fix(qwen4_exp): read host-resident experts without routing them through VRAM

Slice 1 put the *quantised* experts in host memory and the load got
from layer 15 to layer 22 of 48. It should have reached 48. The
telemetry says where the rest went:

  layers_from_cache=15 layers_quantised=8 read_gib=21.2 written_gib=9.9

and the failure moved from "OOM quantising" to "OOM loading
layers.22.mlp.experts/down_proj" — running out while reading the
*source*, not while placing the result.

Which is what was still happening: `experts_vb.get` put each layer's
5.03 GB fused pair on the load device, the quantiser copied it to the
host, and it was dropped — 48 times. Persistent device state is only
about 6 GB, so 31 GB nominally free ran out on fragmentation rather
than occupancy: two multi-gigabyte blocks and 1536 small f32
temporaries per layer, churning a pool nothing trims mid-load. The
image path documents the same failure and answers it by trimming after
every job.

Removing the transit is better than managing it. A second VarBuilder
over the same checkpoint, bound to host memory, reads the fused
tensors straight to where they are going. Built once per load — it maps
the same files, and mapping is not what costs. It also takes 241 GB of
PCIe traffic out of a load, which is not nothing either.

Device-resident models are untouched: `experts_vb` is `None` for them
and the read goes through the same VarBuilder as everything else.

Refs #307, #318.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015v1FQqShiRp9rrJEEcgNER

crates/cortex-core/src/harness.rs

@@ -172,6 +172,39 @@ pub struct ModelSpec {
     /// a belief. A request's own `chat_template_kwargs` still wins.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub preserve_thinking: Option<bool>,
+
+    /// Where the routed experts of a sparse MoE live (#318).
+    ///
+    /// `None` — the default, and every model on the fleet today — keeps
+    /// them on the device with everything else.
+    ///
+    /// `Some(ExpertResidency::Host)` quantises them into host memory
+    /// instead and moves the activations across for the expert matmul.
+    /// That is the only way `qwen4_exp` loads at all: its routed
+    /// experts are 73.0 GB at q4k against 63.7 GB of VRAM across both
+    /// of beast's cards, so no quantisation or tensor-parallel split
+    /// makes them fit. It is slow — a token's experts are ~1.35 GB of
+    /// host reads, so decode is bounded near 37 tok/s by DDR5
+    /// bandwidth before compute — and it is the difference between a
+    /// model that serves and one that does not.
+    ///
+    /// Explicitly opted into rather than inferred from a failed device
+    /// load, because #318's whole point is that placement is measured
+    /// per machine rather than decided once: an automatic fallback
+    /// would quietly pick for every host and hide the measurement it
+    /// exists to inform.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub expert_residency: Option<ExpertResidency>,
+}
+
+/// Where a sparse MoE's routed experts are held (#318).
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum ExpertResidency {
+    /// On the device, with the rest of the model. The default.
+    Device,
+    /// In host memory, with activations crossing for each expert.
+    Host,
 }
 
 /// Per-model token budget advertised by the catalogue or neuron.

crates/cortex-gateway/src/router.rs

@@ -545,6 +545,11 @@ async fn profile_to_spec(
         // this host was configured to hold resident.
         sampling: profile.sampling.clone(),
         preserve_thinking: profile.preserve_thinking,
+        // Expert residency (#318) is a property of the machine, not of
+        // the catalogue: whether a host must hold the experts in RAM
+        // depends on its VRAM, and cortex routes to several. The neuron
+        // decides from its own config.
+        expert_residency: None,
     }
 }
 

crates/helexa-bench/src/client.rs

@@ -218,6 +218,10 @@ impl TargetClient {
             // ever used to load a model the host has no config for.
             sampling: None,
             preserve_thinking: None,
+            // Same limitation, same reason: `GET /models` does not
+            // report where the experts were placed (#318), so a
+            // reconstructed spec cannot carry it either.
+            expert_residency: None,
         })
     }
 

crates/neuron/src/harness/arch/qwen3_5/moe.rs

@@ -30,7 +30,7 @@
 
 use anyhow::{Context, Result};
 use candle_core::quantized::{GgmlDType, QMatMul, QTensor};
-use candle_core::{DType, IndexOp, Module, Tensor};
+use candle_core::{DType, Device, IndexOp, Module, Tensor};
 use candle_nn::Linear;
 use candle_nn::var_builder::ShardedVarBuilder;
 
@@ -88,6 +88,20 @@ pub(crate) struct QuantizedExpert {
     down: QMatMul,
 }
 
+impl QuantizedExpert {
+    /// Where this expert's weights are, when that is knowable.
+    ///
+    /// `None` for a `QMatMul` that dequantised on construction — it
+    /// holds a plain tensor whose device is the load device by
+    /// construction, so the caller's fallback is right.
+    fn device(&self) -> Option<Device> {
+        match &self.gate {
+            QMatMul::QTensor(t) => Some(t.device()),
+            _ => None,
+        }
+    }
+}
+
 /// The quantisation type actually usable for rows of `row` elements.
 ///
 /// ggml quantises along the last dimension, and a row must be a whole
@@ -170,11 +184,31 @@ impl Experts {
     /// The two halves may not end up the same dtype: see
     /// [`quant_for_row`]. `qwen4_exp`'s `down_proj` reduces over 640,
     /// which no k-quant can block.
+    /// `onto` is where the quantised experts come to rest, which need
+    /// not be where the fused source tensor was read (#318). Passing
+    /// `Device::Cpu` for a CUDA load leaves the experts in host memory
+    /// and is what makes a model whose experts exceed VRAM loadable at
+    /// all. The source is dropped either way, so the peak is one
+    /// layer's fused pair regardless.
+    /// Quantise where the source already is. Only the tests want this
+    /// now — the loaders name a destination explicitly, because for
+    /// `qwen4_exp` the answer is not the source's device.
+    #[cfg(test)]
     pub(crate) fn quantize_banked(
         gate_up: &Tensor,
         down: &Tensor,
         intermediate: usize,
         dtype: GgmlDType,
+    ) -> candle_core::Result<Self> {
+        Self::quantize_banked_onto(gate_up, down, intermediate, dtype, gate_up.device())
+    }
+
+    pub(crate) fn quantize_banked_onto(
+        gate_up: &Tensor,
+        down: &Tensor,
+        intermediate: usize,
+        dtype: GgmlDType,
+        onto: &Device,
     ) -> candle_core::Result<Self> {
         let n = gate_up.dims()[0];
         // Decided once, from the shapes, rather than per expert —
@@ -209,8 +243,14 @@ impl Experts {
             // the f32 the quantiser wants comes from.
             let q = |t: Tensor, dtype: GgmlDType| -> candle_core::Result<QMatMul> {
                 let dims = t.dims().to_vec();
+                // This copy to host was always here — it is how the
+                // source comes to own its storage. Building the owned
+                // tensor on `onto` rather than on `t.device()` is the
+                // whole of host residency: the bytes are already in
+                // RAM at this point, and the only question is whether
+                // they go back.
                 let vals = t.to_dtype(DType::F32)?.flatten_all()?.to_vec1::<f32>()?;
-                let owned = Tensor::from_vec(vals, dims, t.device())?;
+                let owned = Tensor::from_vec(vals, dims, onto)?;
                 QMatMul::from_qtensor(QTensor::quantize(&owned, dtype)?)
             };
             out.push(QuantizedExpert {
@@ -222,6 +262,57 @@ impl Experts {
         Ok(Experts::Quantized(out))
     }
 
+    /// The quantised tensors, named for the on-disk cache (#322).
+    ///
+    /// `None` for any layout that is not quantised — there is nothing
+    /// to cache about weights that were never transformed.
+    pub(crate) fn cache_entries(&self) -> Option<Vec<(String, std::sync::Arc<QTensor>)>> {
+        let Experts::Quantized(v) = self else {
+            return None;
+        };
+        let of = |m: &QMatMul| match m {
+            QMatMul::QTensor(t) => Some(t.clone()),
+            // A dequantised-on-load QMatMul has no quantised tensor to
+            // write. `CANDLE_DEQUANTIZE_ALL` produces these; caching
+            // half a layer would be worse than caching none of it.
+            _ => None,
+        };
+        let mut out = Vec::with_capacity(v.len() * 3);
+        for (i, q) in v.iter().enumerate() {
+            out.push((format!("e{i}.gate"), of(&q.gate)?));
+            out.push((format!("e{i}.up"), of(&q.up)?));
+            out.push((format!("e{i}.down"), of(&q.down)?));
+        }
+        Some(out)
+    }
+
+    /// Rebuild from what [`Self::cache_entries`] wrote.
+    ///
+    /// Errors rather than filling gaps: a layer missing an expert is a
+    /// truncated artifact, and serving it would mean one expert of 512
+    /// quietly doing nothing.
+    pub(crate) fn from_cache_entries(
+        mut entries: std::collections::HashMap<String, QTensor>,
+        num_experts: usize,
+    ) -> candle_core::Result<Self> {
+        let mut v = Vec::with_capacity(num_experts);
+        for i in 0..num_experts {
+            let mut take = |part: &str| -> candle_core::Result<QMatMul> {
+                let name = format!("e{i}.{part}");
+                match entries.remove(&name) {
+                    Some(t) => QMatMul::from_qtensor(t),
+                    None => candle_core::bail!("cached expert layer is missing '{name}'"),
+                }
+            };
+            v.push(QuantizedExpert {
+                gate: take("gate")?,
+                up: take("up")?,
+                down: take("down")?,
+            });
+        }
+        Ok(Experts::Quantized(v))
+    }
+
     /// Expert `e` applied to the rows routed to it.
     pub(crate) fn forward_one(&self, e: usize, xs: &Tensor) -> candle_core::Result<Tensor> {
         match self {
@@ -243,13 +334,25 @@ impl Experts {
                 // The quantised kernels accumulate in f32 whatever the
                 // activation dtype, so the cast is explicit here rather
                 // than implied — and the result comes back in the
-                // caller's dtype so the routing arithmetic upstream
-                // does not have to know which layout it got.
+                // caller's dtype, and on the caller's device, so the
+                // routing arithmetic upstream does not have to know
+                // which layout or residency it got.
                 let dtype = xs.dtype();
-                let xs32 = xs.to_dtype(DType::F32)?;
+                let home = q.device().unwrap_or_else(|| xs.device().clone());
+                // Host-resident experts (#318): the activations go to
+                // the weights rather than the weights to the
+                // activations, because the weights are ~2.4 GB a layer
+                // and the activations are the rows routed to one
+                // expert. `to_device` is a clone when the devices
+                // already match, so the device-resident path pays
+                // nothing for this.
+                let xs32 = xs.to_dtype(DType::F32)?.to_device(&home)?;
                 let lhs = candle_nn::ops::silu(&q.gate.forward(&xs32)?)?;
                 let rhs = q.up.forward(&xs32)?;
-                q.down.forward(&(lhs * rhs)?)?.to_dtype(dtype)
+                q.down
+                    .forward(&(lhs * rhs)?)?
+                    .to_device(xs.device())?
+                    .to_dtype(dtype)
             }
         }
     }
@@ -707,6 +810,129 @@ mod tests {
     /// a tiny fixture silently cannot be quantised at all, which is the
     /// first thing that bites when this is tried on a toy model.
     ///
+    /// Host-resident experts compute the same function as device-
+    /// resident ones (#318).
+    ///
+    /// This is the contract that makes the whole placement decision
+    /// safe to take: *where* the weights live may change what a load
+    /// costs, and must not change what it computes. The activations
+    /// cross to the weights and the result crosses back, and nothing
+    /// else about the expert is different.
+    ///
+    /// On a CPU-only test host both "devices" are the same one, so this
+    /// pins the plumbing — the dtype round trip, the `to_device` on the
+    /// way out, the caller getting its own device and dtype back —
+    /// rather than the transfer itself. The transfer is exercised on
+    /// hardware, where the two devices genuinely differ; what a unit
+    /// test can hold is that residency is not silently part of the
+    /// arithmetic.
+    #[test]
+    fn host_resident_experts_compute_the_same_function() {
+        let (n_experts, hidden, inter) = (2usize, 2560usize, 640usize);
+        let gate_up = randn(&[n_experts, inter * 2, hidden]);
+        let down = randn(&[n_experts, hidden, inter]);
+
+        let on_device =
+            Experts::quantize_banked_onto(&gate_up, &down, inter, GgmlDType::Q4K, &Device::Cpu)
+                .unwrap();
+        let on_host =
+            Experts::quantize_banked_onto(&gate_up, &down, inter, GgmlDType::Q4K, &Device::Cpu)
+                .unwrap();
+
+        let xs = randn(&[3, hidden]);
+        for e in 0..n_experts {
+            let a: Vec<f32> = on_device
+                .forward_one(e, &xs)
+                .unwrap()
+                .flatten_all()
+                .unwrap()
+                .to_vec1()
+                .unwrap();
+            let b = on_host.forward_one(e, &xs).unwrap();
+            // The caller's device and dtype come back unchanged,
+            // whatever the weights did in between.
+            assert_eq!(b.dtype(), xs.dtype());
+            assert!(b.device().same_device(xs.device()));
+            let b: Vec<f32> = b.flatten_all().unwrap().to_vec1().unwrap();
+            assert_eq!(a, b, "expert {e} computed differently off-device");
+        }
+    }
+
+    /// Experts restored from the on-disk cache compute the same
+    /// function as the ones that filled it (#322).
+    ///
+    /// The cache's whole risk is that it returns something plausible:
+    /// the model loads, it serves, and its weights are quietly not the
+    /// ones it was asked for. So this asserts the restored experts are
+    /// *bit-identical* in output, not close — there is no arithmetic
+    /// between writing and reading that could legitimately differ.
+    ///
+    /// At the checkpoint's real widths, because they are what decide
+    /// the types involved: 2560 blocks under Q4K, 640 does not and
+    /// falls back to Q5_0, so this exercises a layer whose halves are
+    /// *different* quantisation types — exactly the case a container
+    /// storing one type per file could not represent, and the reason
+    /// the artifact is GGUF.
+    #[test]
+    fn cached_experts_compute_the_same_function() {
+        let dev = Device::Cpu;
+        let (n_experts, hidden, inter) = (2usize, 2560usize, 640usize);
+        let gate_up = randn(&[n_experts, inter * 2, hidden]);
+        let down = randn(&[n_experts, hidden, inter]);
+        let fresh = Experts::quantize_banked(&gate_up, &down, inter, GgmlDType::Q4K).unwrap();
+
+        let entries = fresh
+            .cache_entries()
+            .expect("quantised experts must be cacheable");
+        // The two halves really did take different types; if they had
+        // not, this test would not be covering what it claims to.
+        let dtypes: std::collections::HashSet<GgmlDType> =
+            entries.iter().map(|(_, t)| t.dtype()).collect();
+        assert!(
+            dtypes.len() > 1,
+            "expected a mix of quant types across gate/up and down, got {dtypes:?}"
+        );
+
+        let dir = tempfile::tempdir().unwrap();
+        let cache =
+            crate::harness::isq_cache::IsqCache::new(dir.path(), "test/model", Some("rev0"), "q4k")
+                .unwrap();
+        let refs: Vec<(&str, &QTensor)> = entries
+            .iter()
+            .map(|(n, t)| (n.as_str(), t.as_ref()))
+            .collect();
+        cache
+            .try_store("layer-0", &refs)
+            .expect("the write must succeed, and say why if it does not");
+
+        let restored = Experts::from_cache_entries(
+            cache
+                .load("layer-0", &dev)
+                .expect("the artifact just written"),
+            n_experts,
+        )
+        .unwrap();
+
+        let xs = randn(&[3, hidden]);
+        for e in 0..n_experts {
+            let a: Vec<f32> = fresh
+                .forward_one(e, &xs)
+                .unwrap()
+                .flatten_all()
+                .unwrap()
+                .to_vec1()
+                .unwrap();
+            let b: Vec<f32> = restored
+                .forward_one(e, &xs)
+                .unwrap()
+                .flatten_all()
+                .unwrap()
+                .to_vec1()
+                .unwrap();
+            assert_eq!(a, b, "expert {e} changed value on the cache round trip");
+        }
+    }
+
     /// `qwen4_exp`'s real expert widths, which no k-quant can block.
     ///
     /// `moe_intermediate_size` is 640, so `down_proj` is `[2560, 640]`

crates/neuron/src/harness/arch/qwen4_exp/decoder.rs

@@ -41,8 +41,6 @@ use crate::harness::arch::qwen3_5::linear_attn::GatedDeltaNet;
 use crate::harness::arch::qwen3_5::moe::Qwen3_5MoeBlock;
 use crate::harness::arch::qwen3_5::rope::RotaryEmbedding;
 
-use candle_core::quantized::GgmlDType;
-
 use super::config::TextConfig;
 use super::full_attn::Attention;
 use super::hyper::HyperConnection;
@@ -74,8 +72,8 @@ impl DecoderLayer {
         cfg: &TextConfig,
         rotary: Arc<RotaryEmbedding>,
         layer_idx: usize,
-        quant: Option<GgmlDType>,
         vb: &ShardedVarBuilder,
+        opts: &super::LoadOptions<'_>,
     ) -> Result<Self> {
         let layer_type = cfg
             .layer_types
@@ -92,8 +90,8 @@ impl DecoderLayer {
             rotary,
             layer_type,
             cfg.ple_layers().contains(&layer_idx),
-            quant,
             vb,
+            opts,
         )
     }
 
@@ -108,8 +106,8 @@ impl DecoderLayer {
         rotary: Arc<RotaryEmbedding>,
         layer_type: &str,
         with_ple: bool,
-        quant: Option<GgmlDType>,
         vb: &ShardedVarBuilder,
+        opts: &super::LoadOptions<'_>,
     ) -> Result<Self> {
         let mixer = match layer_type {
             "full_attention" => {
@@ -155,7 +153,7 @@ impl DecoderLayer {
             attn_hc: hc("attn_hyper_connection")?,
             mlp_hc: hc("mlp_hyper_connection")?,
             mixer,
-            mlp: moe::load(cfg, quant, &vb.pp("mlp"))?,
+            mlp: moe::load(cfg, &vb.pp("mlp"), opts)?,
             ple,
         })
     }

crates/neuron/src/harness/arch/qwen4_exp/mod.rs

@@ -44,3 +44,47 @@ pub mod moe;
 pub mod mtp;
 pub mod ple;
 pub mod qsa;
+
+/// What a load needs to know beyond the config and the weights.
+///
+/// Threading these one at a time got the loader signatures to eight
+/// arguments, at which point clippy is making a readability point
+/// rather than a stylistic one: `load(cfg, dtype, quant, device, vb,
+/// paths, cache, onto)` has two devices and two ways to name the
+/// checkpoint in it, and nothing at the call site says which is which.
+///
+/// They also belong together. Every one of them is a decision about
+/// *this load* rather than about the architecture — what precision, on
+/// what hardware, reading from where, reusing what.
+pub struct LoadOptions<'a> {
+    /// In-situ quantisation type (#315). `None` keeps the checkpoint's
+    /// precision, which for this architecture fits nowhere.
+    pub quant: Option<candle_core::quantized::GgmlDType>,
+    /// The checkpoint's own files, for mmapping the PLE n-gram table
+    /// where it already lies (#310). Empty falls back to loading it
+    /// through the VarBuilder, which puts 102.4 GB wherever that
+    /// points.
+    pub safetensors_paths: &'a [std::path::PathBuf],
+    /// Reuse of a previous load's quantisation (#322).
+    pub isq_cache: Option<&'a crate::harness::isq_cache::IsqCache>,
+    /// Where the routed experts come to rest (#318) — the load device
+    /// for an ordinary model, host memory for one whose experts exceed
+    /// VRAM.
+    pub experts_onto: &'a candle_core::Device,
+    /// A VarBuilder rooted at the checkpoint but bound to
+    /// [`Self::experts_onto`], used to read the fused expert tensors
+    /// when that is not the load device.
+    ///
+    /// Without it, host-resident experts still arrive *through* VRAM:
+    /// `get` puts the layer's 5.03 GB fused pair on the load device,
+    /// the quantiser copies it to the host, and the source is dropped
+    /// — 48 times, 241 GB of PCIe traffic, against a pool that has to
+    /// absorb two multi-gigabyte blocks and 1536 small ones per layer.
+    /// Measured: the load reached layer 22 of 48 and ran out with 31 GB
+    /// nominally free, which is fragmentation rather than occupancy —
+    /// the same failure the image path trims after every job to avoid.
+    ///
+    /// Reading them straight into host memory removes the transit
+    /// rather than managing it.
+    pub experts_vb: Option<&'a candle_nn::var_builder::ShardedVarBuilder<'a>>,
+}

crates/neuron/src/harness/arch/qwen4_exp/model.rs

@@ -34,7 +34,6 @@
 //! See `doc/qwen4_exp-port-spec.md`.
 
 use anyhow::{Context, Result, ensure};
-use candle_core::quantized::GgmlDType;
 use candle_core::{DType, Device, IndexOp, Module, Tensor};
 use candle_nn::var_builder::ShardedVarBuilder;
 use candle_nn::{Embedding, Linear};
@@ -137,10 +136,9 @@ impl Qwen4ExpForCausalLM {
     pub fn load(
         cfg: &Config,
         dtype: DType,
-        quant: Option<GgmlDType>,
         device: &Device,
         vb: &ShardedVarBuilder,
-        safetensors_paths: &[std::path::PathBuf],
+        opts: &super::LoadOptions<'_>,
     ) -> Result<Self> {
         let text = &cfg.text_config;
         // `RotaryEmbedding` reads a qwen3_5 config; the rope block is the
@@ -169,7 +167,7 @@ impl Qwen4ExpForCausalLM {
             Some(layer) => Some(load_ngram(
                 text,
                 &layers_vb.pp(*layer).pp("ple"),
-                safetensors_paths,
+                opts.safetensors_paths,
                 device,
             )?),
             None => None,
@@ -178,7 +176,7 @@ impl Qwen4ExpForCausalLM {
         let mut layers = Vec::with_capacity(text.num_hidden_layers);
         for i in 0..text.num_hidden_layers {
             layers.push(
-                DecoderLayer::load(text, rotary.clone(), i, quant, &layers_vb.pp(i))
+                DecoderLayer::load(text, rotary.clone(), i, &layers_vb.pp(i), opts)
                     .with_context(|| format!("load decoder layer {i}"))?,
             );
         }
@@ -804,10 +802,15 @@ mod tests {
         let model = Qwen4ExpForCausalLM::load(
             &cfg,
             DType::F32,
-            None,
             &Device::Cpu,
             &vb,
-            std::slice::from_ref(&path),
+            &super::super::LoadOptions {
+                quant: None,
+                safetensors_paths: std::slice::from_ref(&path),
+                isq_cache: None,
+                experts_onto: &Device::Cpu,
+                experts_vb: None,
+            },
         )
         .unwrap();
         (dir, model)

crates/neuron/src/harness/arch/qwen4_exp/moe.rs

@@ -40,7 +40,6 @@
 
 use anyhow::{Context, Result};
 use candle_core::Tensor;
-use candle_core::quantized::GgmlDType;
 use candle_nn::Linear;
 use candle_nn::var_builder::ShardedVarBuilder;
 
@@ -58,9 +57,10 @@ use super::config::TextConfig;
 /// load of `Qwen3.8-Flash-Next` always passes one.
 pub fn load(
     cfg: &TextConfig,
-    quant: Option<GgmlDType>,
     vb: &ShardedVarBuilder,
+    opts: &super::LoadOptions<'_>,
 ) -> Result<Qwen3_5MoeBlock> {
+    let (quant, isq_cache, experts_onto) = (opts.quant, opts.isq_cache, opts.experts_onto);
     let (h, inter) = (cfg.hidden_size, cfg.moe_intermediate_size);
     anyhow::ensure!(
         cfg.num_experts > 0 && cfg.num_experts_per_tok > 0 && inter > 0,
@@ -77,25 +77,78 @@ pub fn load(
         None,
     );
 
-    let experts_vb = vb.pp("experts");
-    let gate_up = experts_vb
-        .get((cfg.num_experts, inter * 2, h), "gate_up_proj")
-        .with_context(|| format!("load '{}/gate_up_proj'", experts_vb.prefix()))?;
-    let down = experts_vb
-        .get((cfg.num_experts, h, inter), "down_proj")
-        .with_context(|| format!("load '{}/down_proj'", experts_vb.prefix()))?;
-    check_fused_experts(&gate_up, &down, inter)?;
-    // The fused pair is the peak: 5.03 GB per layer at BF16. Quantising
-    // here and letting it drop at the end of this function is what keeps
-    // the whole 241.6 GB from ever existing.
-    let experts = match quant {
-        Some(dtype) => Experts::quantize_banked(&gate_up, &down, inter, dtype)
-            .with_context(|| format!("quantize experts to {dtype:?}"))?,
-        None => Experts::Banked {
-            gate_up,
-            down,
-            intermediate: inter,
-        },
+    // Read the fused tensors wherever the experts are going to live.
+    // When that is the host, this is a VarBuilder bound to it, walked
+    // to the same prefix — so the 5.03 GB pair never touches the
+    // device on its way to being quantised into RAM.
+    let experts_vb = match opts.experts_vb {
+        Some(host_root) => {
+            let mut at = host_root.clone();
+            for segment in vb.prefix().split('.').filter(|s| !s.is_empty()) {
+                at = at.pp(segment);
+            }
+            at.pp("experts")
+        }
+        None => vb.pp("experts"),
+    };
+    // The cache key is the layer's own tensor prefix, so it cannot
+    // collide across layers and needs no separate index to be passed
+    // down alongside it.
+    let cache_key = experts_vb.prefix().replace(['.', '/'], "-");
+
+    // A hit skips reading the fused tensors at all, which is the point:
+    // the cost being avoided is the 5.03 GB read *and* the ~32 s of
+    // quantisation, not just the latter.
+    let cached =
+        match (quant, isq_cache) {
+            (Some(_), Some(cache)) => cache.load(&cache_key, vb.device()).and_then(|entries| {
+                match Experts::from_cache_entries(entries, cfg.num_experts) {
+                    Ok(e) => Some(e),
+                    Err(e) => {
+                        tracing::warn!(key = %cache_key, error = %e,
+                            "isq cache: layer artifact rejected, re-quantising");
+                        None
+                    }
+                }
+            }),
+            _ => None,
+        };
+
+    let experts = match cached {
+        Some(experts) => experts,
+        None => {
+            let gate_up = experts_vb
+                .get((cfg.num_experts, inter * 2, h), "gate_up_proj")
+                .with_context(|| format!("load '{}/gate_up_proj'", experts_vb.prefix()))?;
+            let down = experts_vb
+                .get((cfg.num_experts, h, inter), "down_proj")
+                .with_context(|| format!("load '{}/down_proj'", experts_vb.prefix()))?;
+            check_fused_experts(&gate_up, &down, inter)?;
+            // The fused pair is the peak: 5.03 GB per layer at BF16.
+            // Quantising here and letting it drop at the end of this
+            // function is what keeps the whole 241.6 GB from ever
+            // existing.
+            match quant {
+                Some(dtype) => {
+                    let experts =
+                        Experts::quantize_banked_onto(&gate_up, &down, inter, dtype, experts_onto)
+                            .with_context(|| format!("quantize experts to {dtype:?}"))?;
+                    if let (Some(cache), Some(entries)) = (isq_cache, experts.cache_entries()) {
+                        let refs: Vec<(&str, &candle_core::quantized::QTensor)> = entries
+                            .iter()
+                            .map(|(n, t)| (n.as_str(), t.as_ref()))
+                            .collect();
+                        cache.store(&cache_key, &refs);
+                    }
+                    experts
+                }
+                None => Experts::Banked {
+                    gate_up,
+                    down,
+                    intermediate: inter,
+                },
+            }
+        }
     };
 
     let (shared_expert, shared_expert_gate) = if cfg.shared_expert_intermediate_size > 0 {

crates/neuron/src/harness/arch/qwen4_exp/mtp.rs

@@ -113,8 +113,18 @@ impl MtpHead {
                 rotary,
                 "full_attention",
                 false,
-                quant,
                 &mtp.pp("layers").pp(0),
+                // The MTP head is not on the serving path (#313), so it
+                // has nothing to gain from the cache and nothing to
+                // stale it — and its experts go wherever the rest of it
+                // does.
+                &super::LoadOptions {
+                    quant,
+                    safetensors_paths: &[],
+                    isq_cache: None,
+                    experts_onto: vb.device(),
+                    experts_vb: None,
+                },
             )
             .context("load mtp.layers.0")?,
             mixer: HyperConnection::load(

crates/neuron/src/harness/candle.rs

@@ -2738,6 +2738,12 @@ impl CandleHarness {
         // Resolved out here: `spec` is borrowed and does not outlive the
         // blocking task.
         let isq = super::quant::parse_quant_string(spec.quant.as_deref())?;
+        let isq_cache = self.isq_cache_for(
+            &spec.model_id,
+            &source_id.scheme,
+            spec.quant.as_deref(),
+            &safetensors_paths,
+        );
 
         let arch = tokio::task::spawn_blocking(move || -> Result<ModelArch> {
             let cfg_text = std::fs::read_to_string(&config_path).context("read config.json")?;
@@ -2860,10 +2866,18 @@ impl CandleHarness {
                     let model = super::arch::qwen4_exp::model::Qwen4ExpForCausalLM::load(
                         &cfg,
                         dtype,
-                        isq,
                         &device_for_load,
                         &sharded_vb,
-                        &safetensors_paths,
+                        &super::arch::qwen4_exp::LoadOptions {
+                            quant: isq,
+                            safetensors_paths: &safetensors_paths,
+                            isq_cache: isq_cache.as_ref(),
+                            // The CPU path already has everything in
+                            // host memory, so residency is not a
+                            // choice here.
+                            experts_onto: &device_for_load,
+                            experts_vb: None,
+                        },
                     )
                     .context("build qwen4_exp model")?;
                     Ok(ModelArch::Qwen4Exp(Box::new(model)))
@@ -2900,6 +2914,46 @@ impl CandleHarness {
     /// them match the requested quant — that case is a typo, not an ISQ
     /// request, and turning it into a silent 360 GB dense load would be
     /// a worse answer than the error.
+    /// The quantisation cache for this load, or `None` when the quant
+    /// is unset or the checkpoint's revision cannot be identified
+    /// (#322).
+    ///
+    /// It lives **inside the source's own cache root**, beside the
+    /// weights it is derived from, rather than behind a config key
+    /// naming a path. That is the fleet's existing design: an operator
+    /// who wants this on faster storage symlinks it there, exactly as
+    /// the hot model weights on beast are symlinked from the spinning
+    /// `/archive3` cache onto NVMe. Adding a second, differently-shaped
+    /// way to say the same thing would leave two places to get it
+    /// wrong.
+    ///
+    /// A revision we cannot read means we cannot prove two checkpoints
+    /// are the same bytes, and a cache key that might collide is worse
+    /// than no cache — so an unrecognised layout disables it rather
+    /// than guessing.
+    fn isq_cache_for(
+        &self,
+        model_id: &str,
+        scheme: &str,
+        quant: Option<&str>,
+        safetensors_paths: &[PathBuf],
+    ) -> Option<super::isq_cache::IsqCache> {
+        let quant = quant.filter(|q| !q.is_empty())?;
+        let root = self.hf_cache_for(scheme).ok()?.path().join("neuron-isq");
+        let revision = super::isq_cache::revision_from_paths(safetensors_paths);
+        let cache = super::isq_cache::IsqCache::new(&root, model_id, revision, quant);
+        match &cache {
+            Some(c) => tracing::info!(model = model_id, quant, dir = %c.dir().display(),
+                "isq cache: enabled for this load (#322)"),
+            None => tracing::info!(
+                model = model_id,
+                quant,
+                "isq cache: disabled — the checkpoint revision could not be identified"
+            ),
+        }
+        cache
+    }
+
     async fn quant_selects_a_gguf(
         &self,
         spec: &ModelSpec,
@@ -3958,8 +4012,35 @@ impl Harness for CandleHarness {
                 // too, not just the CPU one — parsed here because
                 // `spec` does not cross the channel.
                 let isq = super::quant::parse_quant_string(spec.quant.as_deref())?;
+                // Built here rather than in the worker: the worker has
+                // no view of the config, and an `IsqCache` is a path,
+                // so it crosses the channel freely.
+                let isq_cache = self.isq_cache_for(
+                    &spec.model_id,
+                    &source_id.scheme,
+                    spec.quant.as_deref(),
+                    &safetensors_paths,
+                );
+                let experts_on_host = matches!(
+                    spec.expert_residency,
+                    Some(cortex_core::harness::ExpertResidency::Host)
+                );
+                if experts_on_host {
+                    tracing::info!(
+                        model = %spec.model_id,
+                        "loading routed experts into host memory (#318); the device \
+                         holds the dense path and the activations cross per expert"
+                    );
+                }
                 let load = w
-                    .load_dense(config_path, safetensors_paths, spec.model_id.clone(), isq)
+                    .load_dense(
+                        config_path,
+                        safetensors_paths,
+                        spec.model_id.clone(),
+                        isq,
+                        isq_cache,
+                        experts_on_host,
+                    )
                     .await
                     .map_err(|e| anyhow::anyhow!("worker load_dense: {e:#}"))?;
                 // Prefix snapshots (#11): the built model answers for

crates/neuron/src/harness/device_worker/dispatch.rs

@@ -147,6 +147,8 @@ pub(crate) fn run(device_index: u32, rx: Receiver<Job>, poisoned: Arc<AtomicBool
                 safetensors_paths,
                 model_id,
                 quant,
+                isq_cache,
+                experts_on_host,
                 reply,
             } => {
                 let result = load_dense_inner(
@@ -155,6 +157,8 @@ pub(crate) fn run(device_index: u32, rx: Receiver<Job>, poisoned: Arc<AtomicBool
                     &safetensors_paths,
                     &model_id,
                     quant,
+                    isq_cache.as_ref(),
+                    experts_on_host,
                 )
                 .map(|arch| {
                     // Ask the model, once, while it is still in
@@ -934,12 +938,32 @@ fn load_gguf_inner(
 }
 
 /// Load a dense safetensors model on the worker thread.
+/// Log the cache summary when the load leaves scope, successful or
+/// not. A load that OOMs half way is precisely when the question "how
+/// much of that was re-quantised?" is worth answering.
+fn scopeguard_summary<'a>(
+    cache: Option<&'a crate::harness::isq_cache::IsqCache>,
+    model_id: &'a str,
+) -> impl Drop + 'a {
+    struct S<'a>(Option<&'a crate::harness::isq_cache::IsqCache>, &'a str);
+    impl Drop for S<'_> {
+        fn drop(&mut self) {
+            if let Some(c) = self.0 {
+                c.log_summary(self.1);
+            }
+        }
+    }
+    S(cache, model_id)
+}
+
 fn load_dense_inner(
     device: &candle_core::Device,
     config_path: &std::path::Path,
     safetensors_paths: &[std::path::PathBuf],
     model_id: &str,
     quant: Option<candle_core::quantized::GgmlDType>,
+    isq_cache: Option<&crate::harness::isq_cache::IsqCache>,
+    experts_on_host: bool,
 ) -> anyhow::Result<ModelArch> {
     use anyhow::Context;
     use candle_core::DType;
@@ -948,6 +972,10 @@ fn load_dense_inner(
     use candle_transformers::models::qwen3 as qwen3_dense;
     use candle_transformers::models::qwen3_moe as qwen3_moe_dense;
 
+    // Report what the cache did, however the load ends: a load that
+    // fails part-way is exactly when someone wants to know how much of
+    // it was cheap.
+    let _summary = scopeguard_summary(isq_cache, model_id);
     let cfg_text = std::fs::read_to_string(config_path).context("read config.json")?;
     crate::harness::candle::check_dense_config_supported(&cfg_text, model_id)?;
     // Peek at model_type to choose the family before the typed
@@ -1053,13 +1081,42 @@ fn load_dense_inner(
                 )
                 .context("build ShardedVarBuilder for qwen4_exp")?
             };
+            // A second view of the same checkpoint, bound to host
+            // memory, so host-resident experts are read straight there
+            // rather than through 5.03 GB of VRAM per layer (#318).
+            // Built once for the model: it maps the same files, and
+            // mapping is not what costs.
+            let host_vb = if experts_on_host {
+                Some(unsafe {
+                    candle_nn::var_builder::ShardedSafeTensors::var_builder(
+                        safetensors_paths,
+                        dtype,
+                        &candle_core::Device::Cpu,
+                    )
+                    .context("build host ShardedVarBuilder for qwen4_exp experts")?
+                })
+            } else {
+                None
+            };
             let model = crate::harness::arch::qwen4_exp::model::Qwen4ExpForCausalLM::load(
                 &cfg,
                 dtype,
-                quant,
                 device,
                 &sharded_vb,
-                safetensors_paths,
+                &crate::harness::arch::qwen4_exp::LoadOptions {
+                    quant,
+                    safetensors_paths,
+                    isq_cache,
+                    experts_vb: host_vb.as_ref(),
+                    // Host residency (#318) is what makes a model whose
+                    // experts exceed VRAM loadable; everything else in
+                    // the model still lands on the device.
+                    experts_onto: if experts_on_host {
+                        &candle_core::Device::Cpu
+                    } else {
+                        device
+                    },
+                },
             )
             .context("build qwen4_exp model")?;
             Ok(ModelArch::Qwen4Exp(Box::new(model)))
@@ -1771,11 +1828,18 @@ mod tests {
         for model_type in crate::harness::candle::DENSE_SUPPORTED_MODEL_TYPES {
             let path = dir.path().join(format!("{model_type}.json"));
             std::fs::write(&path, format!(r#"{{"model_type": "{model_type}"}}"#)).unwrap();
-            let msg =
-                match load_dense_inner(&candle_core::Device::Cpu, &path, &[], model_type, None) {
-                    Ok(_) => panic!("a bare config must not build a model"),
-                    Err(e) => format!("{e:#}"),
-                };
+            let msg = match load_dense_inner(
+                &candle_core::Device::Cpu,
+                &path,
+                &[],
+                model_type,
+                None,
+                None,
+                false,
+            ) {
+                Ok(_) => panic!("a bare config must not build a model"),
+                Err(e) => format!("{e:#}"),
+            };
             assert!(
                 !msg.contains("unrouted"),
                 "`{model_type}` is in DENSE_SUPPORTED_MODEL_TYPES but load_dense_inner \

crates/neuron/src/harness/device_worker/jobs.rs

@@ -132,6 +132,11 @@ pub enum Job {
         /// already parsed. `None` loads at the checkpoint dtype.
         /// Archs that cannot quantise in situ ignore it.
         quant: Option<candle_core::quantized::GgmlDType>,
+        /// Where to cache the quantised weights (#322). Built on the
+        /// async side, which is the only side that can see the config.
+        isq_cache: Option<crate::harness::isq_cache::IsqCache>,
+        /// Keep the routed experts in host memory (#318).
+        experts_on_host: bool,
         reply: oneshot::Sender<Result<DenseLoad>>,
     },
     /// Remove the model from the slab and drop it. The `Drop` runs on

crates/neuron/src/harness/device_worker/mod.rs

@@ -222,6 +222,8 @@ impl DeviceWorkerHandle {
         safetensors_paths: Vec<std::path::PathBuf>,
         model_id: String,
         quant: Option<candle_core::quantized::GgmlDType>,
+        isq_cache: Option<crate::harness::isq_cache::IsqCache>,
+        experts_on_host: bool,
     ) -> Result<DenseLoad, WorkerError> {
         if self.poisoned.load(Ordering::Acquire) {
             return Err(WorkerError::Poisoned {
@@ -235,6 +237,8 @@ impl DeviceWorkerHandle {
                 safetensors_paths,
                 model_id,
                 quant,
+                isq_cache,
+                experts_on_host,
                 reply: reply_tx,
             })
             .map_err(|_| WorkerError::Gone {
@@ -1246,6 +1250,8 @@ mod tests {
                 vec![fixture.join("model.safetensors")],
                 "qwen3_next-tiny".into(),
                 None,
+                None,
+                false,
             )
             .await
             .expect("load tiny fixture")

crates/neuron/src/harness/engine.rs

@@ -1262,6 +1262,8 @@ mod tests {
                 vec![fixture.join("model.safetensors")],
                 "qwen3_next-tiny".into(),
                 None,
+                None,
+                false,
             )
             .await
             .expect("load fixture")

crates/neuron/src/harness/isq_cache.rs

@@ -0,0 +1,451 @@
+//! On-disk cache for in-situ quantisation (#322).
+//!
+//! Quantising a model's weights at load is a pure function of the
+//! checkpoint bytes, the requested type, and the code that does it.
+//! None of those change between loads, and yet neuron recomputed the
+//! answer every time and threw it away on unload. For `qwen4_exp` that
+//! is **~32 s per decoder layer, about 26 minutes for a full load**,
+//! measured on beast — paid again on every restart and every eviction
+//! that is later reversed.
+//!
+//! So the quantised tensors are written out once and read back after.
+//!
+//! ## The key must include our own version
+//!
+//! `(checkpoint revision, quant name)` is not enough. `bf1e80ed`
+//! changed what `quant = "q4k"` *means* for a 640-wide row — k-quants
+//! cannot block it, so it now falls back to a 32-block type — and an
+//! artifact written before that commit and read after it would be
+//! silently wrong in a way no checksum of the inputs could catch.
+//! [`QUANTISER_VERSION`] is part of the path, and bumping it orphans
+//! every stale artifact rather than misreading one.
+//!
+//! ## What is cached, and what is not
+//!
+//! One GGUF per layer, holding that layer's quantised experts. GGUF
+//! because it stores a **type per tensor**, which is exactly what the
+//! per-row fallback produces — a layer whose `down_proj` is `Q5_0`
+//! while its `gate`/`up` are `Q4K` is representable without inventing
+//! a container for it — and because neuron already reads the format.
+//!
+//! Per layer rather than per model because it makes the cache
+//! **incrementally useful**: a load that runs out of VRAM at layer 16
+//! has still written sixteen layers, and the next attempt starts
+//! there. On hardware where the model does not yet fit at all (#318),
+//! that is the difference between the cache being useless and being
+//! the thing that makes iteration bearable.
+//!
+//! ## Where it lives
+//!
+//! Inside the source's own cache root — `<hf cache>/neuron-isq/…` —
+//! beside the weights it is derived from, and not behind a config key
+//! naming a path.
+//!
+//! That is deliberate and it is the fleet's existing design. beast
+//! already keeps its hot model weights on NVMe by symlinking them out
+//! of the spinning `/archive3` cache into `/archive1`, with no config
+//! mapping paths to mounts. An operator who wants these artifacts on
+//! faster storage does the same thing to one directory. A second,
+//! differently-shaped way to express the same intent would be a second
+//! place to get it wrong, and the first one already works.
+//!
+//! Size is the thing to know before symlinking: an artifact is roughly
+//! the served size of the model's quantised weights — ~73 GB for
+//! `qwen4_exp` at q4k.
+//!
+//! ## It is an optimisation, and never load-bearing
+//!
+//! Every failure here is swallowed: a miss, a corrupt file, a full
+//! disk, a directory nobody can write. The load proceeds by
+//! quantising, exactly as it did before this module existed. A cache
+//! that can fail a load is worse than no cache.
+
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
+
+use candle_core::Device;
+use candle_core::quantized::{QTensor, gguf_file};
+
+/// Bump when the *meaning* of a quant name changes: a different
+/// fallback table, a different rounding, a different layout. Artifacts
+/// written by an older quantiser are then ignored rather than
+/// misread, and get overwritten in place as layers are re-quantised.
+///
+/// 1: the initial format. Per-layer GGUF, tensors named
+///    `e{index}.{gate,up,down}`, llama.cpp's `tensor_type_fallback`
+///    table for rows that do not divide the requested block size.
+pub const QUANTISER_VERSION: u32 = 1;
+
+/// Refuse a write that would take more than this share of the free
+/// space.
+///
+/// The rule is about *the write*, not the filesystem. An earlier
+/// version required 2% of the volume to remain free, which read
+/// sensibly and was wrong in the case that matters least and happens
+/// most: on a filesystem already below that line, it refused a
+/// six-megabyte write that was in no way responsible. CI found it
+/// immediately.
+///
+/// Bounding each artifact by half the remaining space says the thing
+/// actually meant — no single artifact eats the disk — at every size,
+/// and needs to know nothing about how big the volume is. An
+/// artifact for one 180 B model is ~73 GB, so on a volume with 100 GB
+/// left it is refused and on beast's 2 TB of free `/archive3` it is
+/// not. A few megabytes is never refused, because a few megabytes is
+/// never the problem.
+const MAX_SHARE_OF_FREE: f64 = 0.5;
+
+/// A cache scoped to one (model, revision, quant, quantiser version).
+pub struct IsqCache {
+    dir: PathBuf,
+    hits: AtomicUsize,
+    misses: AtomicUsize,
+    read: AtomicU64,
+    written: AtomicU64,
+}
+
+impl IsqCache {
+    /// `revision` should identify the checkpoint bytes — the snapshot
+    /// directory name, for an hf-hub cache. `None` disables the cache
+    /// rather than guessing, because a wrong revision is the one
+    /// failure mode that produces a confidently wrong model.
+    pub fn new(base: &Path, model_id: &str, revision: Option<&str>, quant: &str) -> Option<Self> {
+        let revision = revision?;
+        let safe = |s: &str| {
+            s.chars()
+                .map(|c| {
+                    if c.is_ascii_alphanumeric() || c == '-' || c == '.' {
+                        c
+                    } else {
+                        '-'
+                    }
+                })
+                .collect::<String>()
+        };
+        let dir = base
+            .join(safe(model_id))
+            .join(safe(revision))
+            .join(format!("{}-v{QUANTISER_VERSION}", safe(quant)));
+        Some(Self {
+            dir,
+            hits: AtomicUsize::new(0),
+            misses: AtomicUsize::new(0),
+            read: AtomicU64::new(0),
+            written: AtomicU64::new(0),
+        })
+    }
+
+    /// Read a layer's quantised tensors onto `device`.
+    ///
+    /// `None` for a miss and for anything that goes wrong reading —
+    /// a truncated artifact from an interrupted write reads as absent,
+    /// and is rewritten by the load that follows.
+    pub fn load(&self, key: &str, device: &Device) -> Option<HashMap<String, QTensor>> {
+        let path = self.dir.join(format!("{key}.gguf"));
+        let Ok(mut file) = std::fs::File::open(&path) else {
+            self.misses.fetch_add(1, Ordering::Relaxed);
+            return None;
+        };
+        let content = match gguf_file::Content::read(&mut file) {
+            Ok(c) => c,
+            Err(e) => {
+                self.misses.fetch_add(1, Ordering::Relaxed);
+                tracing::warn!(path = %path.display(), error = %e,
+                    "isq cache: unreadable artifact, re-quantising this layer");
+                return None;
+            }
+        };
+        let names: Vec<String> = content.tensor_infos.keys().cloned().collect();
+        let mut out = HashMap::with_capacity(names.len());
+        for name in names {
+            match content.tensor(&mut file, &name, device) {
+                Ok(t) => {
+                    out.insert(name, t);
+                }
+                Err(e) => {
+                    self.misses.fetch_add(1, Ordering::Relaxed);
+                    tracing::warn!(path = %path.display(), tensor = %name, error = %e,
+                        "isq cache: artifact tensor failed to read, re-quantising this layer");
+                    return None;
+                }
+            }
+        }
+        self.hits.fetch_add(1, Ordering::Relaxed);
+        self.read.fetch_add(
+            out.values()
+                .map(|t| t.storage_size_in_bytes() as u64)
+                .sum::<u64>(),
+            Ordering::Relaxed,
+        );
+        Some(out)
+    }
+
+    /// Write a layer's quantised tensors.
+    ///
+    /// Through a temporary file and a rename, so an interrupted write
+    /// cannot leave a half-artifact that a later load would read as
+    /// real. Errors are logged and swallowed.
+    /// Write a layer, refusing one that would take an unreasonable
+    /// share of the disk. The production entry point: every failure is
+    /// logged and swallowed.
+    pub fn store(&self, key: &str, tensors: &[(&str, &QTensor)]) {
+        let bytes: u64 = tensors
+            .iter()
+            .map(|(_, t)| t.storage_size_in_bytes() as u64)
+            .sum();
+        if let Some(free) = free_bytes(&self.dir)
+            && too_large_for(bytes, free)
+        {
+            tracing::warn!(
+                dir = %self.dir.display(), key,
+                gib = bytes as f64 / (1024.0 * 1024.0 * 1024.0),
+                free_gib = free as f64 / (1024.0 * 1024.0 * 1024.0),
+                "isq cache: refusing a write that would take over {:.0}% of the free space",
+                MAX_SHARE_OF_FREE * 100.0,
+            );
+            return;
+        }
+        match self.try_store(key, tensors) {
+            Ok(()) => {
+                self.written.fetch_add(bytes, Ordering::Relaxed);
+            }
+            Err(e) => tracing::warn!(dir = %self.dir.display(), key, error = %e,
+                "isq cache: could not write this layer; the load continues uncached"),
+        }
+    }
+
+    /// Say, once per load, what the cache actually did.
+    ///
+    /// Without this the journal shows only that the cache was enabled,
+    /// which is the shape of problem #321 is about: every light green
+    /// and no way to tell from the outside whether the thing worked.
+    /// A load that served nothing from cache and a load that served
+    /// everything looked identical, and the difference between them is
+    /// eight minutes.
+    pub fn log_summary(&self, model_id: &str) {
+        let gib = |b: u64| b as f64 / (1024.0 * 1024.0 * 1024.0);
+        tracing::info!(
+            model = model_id,
+            dir = %self.dir.display(),
+            layers_from_cache = self.hits.load(Ordering::Relaxed),
+            layers_quantised = self.misses.load(Ordering::Relaxed),
+            read_gib = gib(self.read.load(Ordering::Relaxed)),
+            written_gib = gib(self.written.load(Ordering::Relaxed)),
+            "isq cache: load complete (#322)"
+        );
+    }
+
+    /// Write a layer. Nothing else — no policy, and in particular no
+    /// question asked of the filesystem about its size.
+    ///
+    /// That separation is the point. The disk check lives in
+    /// [`Self::store`], so the round-trip tests exercise the write and
+    /// the read without their result depending on how full the machine
+    /// running them happens to be. A unit test that consults the free
+    /// space of whatever host executes it is not testing this code.
+    pub(crate) fn try_store(&self, key: &str, tensors: &[(&str, &QTensor)]) -> anyhow::Result<()> {
+        std::fs::create_dir_all(&self.dir)?;
+        let final_path = self.dir.join(format!("{key}.gguf"));
+        let tmp = self.dir.join(format!(".{key}.{}.tmp", std::process::id()));
+        let write = (|| -> anyhow::Result<()> {
+            let mut f = std::fs::File::create(&tmp)?;
+            gguf_file::write(&mut f, &[], tensors)?;
+            Ok(())
+        })();
+        if let Err(e) = write {
+            // A partial artifact is dead weight that nothing will ever
+            // clean up — and on the failure this is most likely to see,
+            // a full disk, it is dead weight made of the last free
+            // bytes.
+            let _ = std::fs::remove_file(&tmp);
+            return Err(e);
+        }
+        std::fs::rename(&tmp, &final_path)?;
+        Ok(())
+    }
+
+    pub fn dir(&self) -> &Path {
+        &self.dir
+    }
+}
+
+/// Whether a write of `bytes` takes an unreasonable share of `free`.
+///
+/// Split out from the write so the rule can be tested without needing
+/// a filesystem in a particular state — which is precisely what made
+/// the previous rule's defect invisible until CI ran it.
+fn too_large_for(bytes: u64, free: u64) -> bool {
+    bytes as f64 > free as f64 * MAX_SHARE_OF_FREE
+}
+
+/// Free bytes on the filesystem holding `path`, or `None` if it cannot
+/// be determined — in which case the caller writes anyway rather than
+/// refusing on a guess. `-P` is what guarantees one line per
+/// filesystem, so a long device name cannot wrap the fields apart.
+fn free_bytes(path: &Path) -> Option<u64> {
+    let out = std::process::Command::new("df")
+        .arg("-kP")
+        .arg(path)
+        .output()
+        .ok()?;
+    let text = String::from_utf8(out.stdout).ok()?;
+    let kb = text.lines().nth(1)?.split_whitespace().nth(3)?;
+    kb.parse::<u64>().ok().map(|kb| kb * 1024)
+}
+
+/// The revision an hf-hub-cached checkpoint was resolved at, taken
+/// from the snapshot directory its files live in.
+///
+/// `.../models--org--name/snapshots/<revision>/model-00001-of-N.safetensors`
+/// — the parent directory *is* the revision, which is what makes it a
+/// safe cache key: two checkpoints with different bytes cannot share
+/// one. Returns `None` for any layout that is not that, so an
+/// unrecognised path disables the cache instead of colliding.
+pub fn revision_from_paths(paths: &[PathBuf]) -> Option<&str> {
+    let first = paths.first()?;
+    let dir = first.parent()?;
+    let name = dir.file_name()?.to_str()?;
+    let looks_like_snapshot = dir.parent()?.file_name()?.to_str()? == "snapshots";
+    (looks_like_snapshot && !name.is_empty()).then_some(name)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use candle_core::Tensor;
+    use candle_core::quantized::GgmlDType;
+
+    /// A cached layer must dequantise to the same numbers the fresh
+    /// quantisation produced.
+    ///
+    /// This is the whole contract. A cache that returns *something*
+    /// plausible is worse than no cache, because the model loads, it
+    /// serves, and its weights are quietly not the ones it was asked
+    /// for. Round-tripping the dequantised values rather than the
+    /// bytes is deliberate: it is what the model actually consumes.
+    #[test]
+    fn a_cached_tensor_dequantises_to_what_was_stored() {
+        let dev = Device::Cpu;
+        let dir = tempfile::tempdir().unwrap();
+        let cache = IsqCache::new(dir.path(), "org/model", Some("rev0"), "q4k").unwrap();
+
+        // 256-wide so the k-quant can block it at all.
+        let src = Tensor::rand(-1f32, 1f32, (8, 256), &dev).unwrap();
+        let q = QTensor::quantize(&src, GgmlDType::Q4K).unwrap();
+        let before: Vec<f32> = q
+            .dequantize(&dev)
+            .unwrap()
+            .flatten_all()
+            .unwrap()
+            .to_vec1()
+            .unwrap();
+
+        cache
+            .try_store("layer-0", &[("e0.gate", &q)])
+            .expect("the write must succeed, and say why if it does not");
+        let got = cache
+            .load("layer-0", &dev)
+            .expect("the artifact just written");
+        let after: Vec<f32> = got["e0.gate"]
+            .dequantize(&dev)
+            .unwrap()
+            .flatten_all()
+            .unwrap()
+            .to_vec1()
+            .unwrap();
+
+        assert_eq!(
+            before, after,
+            "a cached tensor changed value on the round trip"
+        );
+        assert_eq!(got["e0.gate"].dtype(), GgmlDType::Q4K, "and kept its type");
+    }
+
+    /// The key separates everything that changes the answer.
+    ///
+    /// The quantiser version is the one that is easy to leave out and
+    /// impossible to detect: `bf1e80ed` changed what `q4k` means for a
+    /// 640-wide row, and an artifact written before it is wrong after
+    /// it in a way no checksum of the *inputs* would reveal.
+    #[test]
+    fn the_key_separates_model_revision_quant_and_quantiser() {
+        let root = std::path::Path::new("/cache");
+        let base = IsqCache::new(root, "org/model", Some("rev0"), "q4k").unwrap();
+        let others = [
+            IsqCache::new(root, "org/other", Some("rev0"), "q4k").unwrap(),
+            IsqCache::new(root, "org/model", Some("rev1"), "q4k").unwrap(),
+            IsqCache::new(root, "org/model", Some("rev0"), "q6k").unwrap(),
+        ];
+        for other in &others {
+            assert_ne!(base.dir(), other.dir());
+        }
+        assert!(
+            base.dir()
+                .to_str()
+                .unwrap()
+                .contains(&format!("v{QUANTISER_VERSION}")),
+            "the quantiser version must be in the path: {}",
+            base.dir().display()
+        );
+    }
+
+    /// No revision, no cache. A key that might collide across
+    /// checkpoints is worse than recomputing.
+    #[test]
+    fn an_unidentifiable_revision_disables_the_cache() {
+        assert!(IsqCache::new(std::path::Path::new("/cache"), "org/model", None, "q4k").is_none());
+
+        // An hf-hub snapshot path yields its revision; anything else
+        // does not, rather than yielding a directory name that happens
+        // to be there.
+        let hf = PathBuf::from(
+            "/cache/models--org--model/snapshots/abc123/model-00001-of-00002.safetensors",
+        );
+        assert_eq!(revision_from_paths(&[hf]), Some("abc123"));
+        assert_eq!(
+            revision_from_paths(&[PathBuf::from("/tmp/model.safetensors")]),
+            None
+        );
+        assert_eq!(revision_from_paths(&[]), None);
+    }
+
+    /// The space guard bounds the *write*, not the filesystem.
+    ///
+    /// The rule it replaced required a fixed share of the volume to
+    /// remain free, which refused a six-megabyte write on a disk that
+    /// was already full — punishing the write least responsible for
+    /// the state. These cases are the ones that matter: a 73 GB
+    /// artifact must not land on a volume with 100 GB left, and a few
+    /// megabytes must always be allowed, however full things are.
+    #[test]
+    fn the_space_guard_bounds_the_write_not_the_volume() {
+        let gib = 1024 * 1024 * 1024;
+
+        // A 73 GB artifact against beast's ~2 TB of free /archive3.
+        assert!(!too_large_for(73 * gib, 2000 * gib));
+        // The same artifact where it would eat most of what is left.
+        assert!(too_large_for(73 * gib, 100 * gib));
+        // Six megabytes on a nearly-full disk: never the problem.
+        assert!(!too_large_for(6 * 1024 * 1024, 200 * 1024 * 1024));
+        // Exactly half is allowed; past half is not.
+        assert!(!too_large_for(50, 100));
+        assert!(too_large_for(51, 100));
+        // And nothing is writable onto nothing.
+        assert!(too_large_for(1, 0));
+    }
+
+    /// A miss and a corrupt artifact are the same thing to the caller:
+    /// re-quantise. Neither may propagate as an error.
+    #[test]
+    fn a_missing_or_corrupt_artifact_reads_as_absent() {
+        let dev = Device::Cpu;
+        let dir = tempfile::tempdir().unwrap();
+        let cache = IsqCache::new(dir.path(), "org/model", Some("rev0"), "q4k").unwrap();
+        assert!(cache.load("layer-0", &dev).is_none(), "a miss");
+
+        std::fs::create_dir_all(cache.dir()).unwrap();
+        std::fs::write(cache.dir().join("layer-0.gguf"), b"not a gguf file at all").unwrap();
+        assert!(cache.load("layer-0", &dev).is_none(), "a corrupt artifact");
+    }
+}

crates/neuron/src/harness/mod.rs

@@ -9,6 +9,7 @@ pub mod context_limit;
 pub mod device_worker;
 pub mod engine;
 pub mod image;
+pub mod isq_cache;
 pub mod prefix_cache;
 pub mod preflight;
 pub mod preprocess;

crates/neuron/src/harness/preflight.rs

@@ -403,6 +403,7 @@ mod tests {
             devices: None,
             sampling: None,
             preserve_thinking: None,
+            expert_residency: None,
         }
     }
 

crates/neuron/tests/activation.rs

@@ -31,6 +31,7 @@ async fn test_load_default_models_skips_unknown_harness() {
             devices: None,
             sampling: None,
             preserve_thinking: None,
+            expert_residency: None,
         },
         ModelSpec {
             model_id: "model-b".into(),
@@ -40,6 +41,7 @@ async fn test_load_default_models_skips_unknown_harness() {
             devices: None,
             sampling: None,
             preserve_thinking: None,
+            expert_residency: None,
         },
     ];
 
@@ -101,6 +103,7 @@ async fn test_load_default_models_skipped_on_driver_mismatch() {
         devices: None,
         sampling: None,
         preserve_thinking: None,
+        expert_residency: None,
     }];
     let activation = ActivationTracker::new(&specs);
     let reason = "host NVIDIA driver/library mismatch (userspace NVML 580.159 vs loaded \
@@ -204,6 +207,7 @@ fn qwen_spec() -> ModelSpec {
         devices: None,
         sampling: None,
         preserve_thinking: None,
+        expert_residency: None,
     }
 }
 

crates/neuron/tests/candle_lifecycle.rs

@@ -55,6 +55,7 @@ async fn test_candle_qwen3_load_unload_lifecycle() {
         devices: Some(vec![0]),
         sampling: None,
         preserve_thinking: None,
+        expert_residency: None,
     };
 
     registry

crates/neuron/tests/fixtures/numerical/README.md

@@ -12,7 +12,8 @@ HuggingFace reference" claim to checked-in numbers.
 | `qwen3_5-0.8b-text` | Qwen/Qwen3.5-0.8B | text (>64-token prompt → chunked GDN prefill) | f32 | `text_logits_match_reference` |
 | `qwen3_5-0.8b-vision` | Qwen/Qwen3.5-0.8B | 448×448 synthetic image + prompt | f32 | `vision_tower_and_logits_match_reference` |
 | `qwen3_6-27b-text` | Qwen/Qwen3.6-27B | text | bf16 | manual (see below) |
-| `qwen4_exp-tiny` | random weights, real architecture | text, 12 tokens | f32 | `qwen4_exp_logits_match_reference` |
+| `qwen4_exp-tiny` | random weights, real architecture | text, 12 tokens, no eos | f32 | `qwen4_exp_logits_match_reference` |
+| `qwen4_exp-tiny-long` | random weights, real architecture | text, 80 tokens, eos at 37 | f32 | `qwen4_exp_long_prompt_logits_match_reference` |
 
 ## `qwen4_exp-tiny` is different, and runs in CI
 
@@ -56,6 +57,28 @@ python -m venv --system-site-packages .venv
     --out crates/neuron/tests/fixtures/numerical/qwen4_exp-tiny
 ```
 
+### Why there are two cases
+
+They are not the same test at two sizes. Each reaches a path the other
+structurally cannot, and a mutation demonstrates it:
+
+- **80 tokens** puts the GatedDeltaNet layers on the *chunked* prefill
+  algorithm (#23), which engages at 64. That is a different
+  factorisation of the same recurrence, not the same code with more
+  input — which is also why the long case's bound is 1e-4 against the
+  short case's 1e-5 (observed 1.3e-5 vs 2e-6). It also gives QSA up to
+  40 blocks against a budget of 4, so selection discards nearly
+  everything instead of one block at the boundary.
+- **The eos at position 37** is a segment boundary the n-gram window
+  must refuse to read across. Making `shift_right_ignore_eos` ignore
+  segments entirely leaves the short fixture at 0.000002 — it contains
+  no eos, so there is nothing to get wrong — and moves the long one by
+  **0.67**.
+
+The short case deliberately has no eos: a fixture that happens to
+contain one is testing segment handling by accident, and would make the
+two cases redundant.
+
 ### The tie-break
 
 Where the reference and this port genuinely differ: at an exact tie in

crates/neuron/tests/fixtures/numerical/qwen4_exp-tiny-long/config.json

@@ -0,0 +1,50 @@
+{
+  "model_type": "qwen4_exp",
+  "tie_word_embeddings": false,
+  "text_config": {
+    "vocab_size": 32,
+    "hidden_size": 8,
+    "num_hidden_layers": 4,
+    "num_attention_heads": 2,
+    "num_key_value_heads": 1,
+    "head_dim": 4,
+    "max_position_embeddings": 128,
+    "rms_norm_eps": 1e-06,
+    "hidden_act": "silu",
+    "eos_token_id": 3,
+    "full_attention_interval": 4,
+    "rope_parameters": {
+      "rope_theta": 10000.0,
+      "partial_rotary_factor": 0.5,
+      "rope_type": "default"
+    },
+    "hc_count": 2,
+    "hc_lowrank": 4,
+    "ple_layer_ids": [
+      2
+    ],
+    "ple_embed_dim": 8,
+    "ple_conv_kernel_size": 2,
+    "ngram_size": 3,
+    "heads_per_ngram": 2,
+    "ngram_vocab_size_base": 10,
+    "make_ngram_vocab_size_divisible_by": 4,
+    "split_ngram_parts": 2,
+    "indexer_n_heads": 2,
+    "indexer_kv_heads": 1,
+    "indexer_head_dim": 4,
+    "indexer_budget": 8,
+    "indexer_compress_ratio": 2,
+    "linear_num_key_heads": 1,
+    "linear_key_head_dim": 4,
+    "linear_num_value_heads": 2,
+    "linear_value_head_dim": 4,
+    "linear_conv_kernel_dim": 2,
+    "mamba_ssm_dtype": "float32",
+    "output_gate_type": "sigmoid",
+    "num_experts": 2,
+    "num_experts_per_tok": 1,
+    "moe_intermediate_size": 4,
+    "shared_expert_intermediate_size": 4
+  }
+}

crates/neuron/tests/fixtures/numerical/qwen4_exp-tiny/config.json

@@ -8,7 +8,7 @@
     "num_attention_heads": 2,
     "num_key_value_heads": 1,
     "head_dim": 4,
-    "max_position_embeddings": 64,
+    "max_position_embeddings": 128,
     "rms_norm_eps": 1e-06,
     "hidden_act": "silu",
     "eos_token_id": 3,

crates/neuron/tests/numerical_reference.rs

@@ -360,11 +360,49 @@ fn vision_tower_and_logits_match_reference() {
 /// against our own reading of it.
 #[test]
 fn qwen4_exp_logits_match_reference() {
+    // The recurrent GDN step and a 12-token prompt: agreement is
+    // essentially exact (observed 2e-6), so the bound sits just off the
+    // floor. At 1e-3 this passed with the fused expert's gate and up
+    // halves swapped (2.4e-4) — a bound wide enough to feel safe is
+    // wide enough to hide what it was written for.
+    qwen4_exp_case("qwen4_exp-tiny", 1e-5);
+}
+
+/// The same comparison over a prompt long enough, and shaped
+/// deliberately enough, to reach two paths the short case cannot.
+///
+/// **80 tokens** puts the GatedDeltaNet layers on the *chunked* prefill
+/// algorithm (#23), which engages at 64 and is a different
+/// implementation from the recurrent step the 12-token case exercises —
+/// the same reason the qwen3_5 fixture above is over 64 tokens. It also
+/// gives QSA up to 40 blocks against a budget of 4, so selection is
+/// discarding almost everything rather than one block at the boundary.
+///
+/// **An eos at position 37** puts a segment boundary inside the n-gram
+/// window. `_shift_right_ignore_eos` refuses to read an n-gram across
+/// one, and that logic — a cummax over eos positions, per row — is
+/// intricate enough to be worth comparing rather than reasoning about.
+/// The short case has no eos at all, on purpose: a fixture that happens
+/// to contain one tests segment handling by accident.
+#[test]
+fn qwen4_exp_long_prompt_logits_match_reference() {
+    // Looser than the short case, and for a stated reason rather than
+    // to go green: at 80 tokens the GDN layers take the chunked delta
+    // rule, which is a different *factorisation* of the same
+    // recurrence, so its f32 rounding differs from the recurrent step
+    // by construction. Observed 1.3e-5 against the 2e-6 of the short
+    // case. The bound is still four orders of magnitude below what the
+    // mutations produce — gate/up swapped moves this fixture by 2.7 —
+    // so it discriminates exactly as well.
+    qwen4_exp_case("qwen4_exp-tiny-long", 1e-4);
+}
+
+fn qwen4_exp_case(name: &str, max_abs_bound: f32) {
     use candle_nn::var_builder::ShardedSafeTensors;
     use neuron::harness::arch::qwen4_exp::config::Config as Qwen4ExpConfig;
     use neuron::harness::arch::qwen4_exp::model::Qwen4ExpForCausalLM;
 
-    let fixture = fixture_root().join("qwen4_exp-tiny");
+    let fixture = fixture_root().join(name);
     let weights = fixture.join("model.safetensors");
     let device = Device::Cpu;
 
@@ -377,10 +415,15 @@ fn qwen4_exp_logits_match_reference() {
     let mut model = Qwen4ExpForCausalLM::load(
         &cfg,
         DType::F32,
-        None,
         &device,
         &vb,
-        std::slice::from_ref(&weights),
+        &neuron::harness::arch::qwen4_exp::LoadOptions {
+            quant: None,
+            safetensors_paths: std::slice::from_ref(&weights),
+            isq_cache: None,
+            experts_onto: &device,
+            experts_vb: None,
+        },
     )
     .expect("load the fixture through the production loader");
 
@@ -400,7 +443,7 @@ fn qwen4_exp_logits_match_reference() {
 
     let c = compare(&ours, &reference);
     eprintln!(
-        "qwen4_exp: max_abs={:.6} cosine={:.6} argmax ours={} ref={}",
+        "{name}: max_abs={:.6} cosine={:.6} argmax ours={} ref={}",
         c.max_abs, c.cosine, c.argmax_ours, c.argmax_ref
     );
     // On failure, break the error down by position before asserting.
@@ -409,7 +452,7 @@ fn qwen4_exp_logits_match_reference() {
     // means a discrete choice went differently (an expert, a QSA
     // block), spread means the arithmetic drifted. Finding that out
     // cost an hour the first time.
-    if c.max_abs >= 1e-5 {
+    if c.max_abs >= max_abs_bound {
         let v = cfg.text_config.vocab_size;
         for t in 0..(ours.len() / v) {
             let pc = compare(&ours[t * v..(t + 1) * v], &reference[t * v..(t + 1) * v]);
@@ -420,13 +463,10 @@ fn qwen4_exp_logits_match_reference() {
         }
     }
     assert_eq!(c.argmax_ours, c.argmax_ref, "argmax token diverged");
-    // f32 against f32 is *exact* here — observed max_abs 0.000000 — so
-    // the bound is set just off the floor rather than at a comfortable
-    // distance. That is deliberate: at 1e-3 this test still passed with
-    // the fused expert's gate and up halves read the wrong way round
-    // (max_abs 2.4e-4), which is the single error the loader's own
-    // comment warns about. A tolerance wide enough to be safe is wide
-    // enough to hide the defect it was written for.
     assert!(c.cosine > 0.999_999, "cosine {:.6} too low", c.cosine);
-    assert!(c.max_abs < 1e-5, "max abs diff {:.6} too high", c.max_abs);
+    assert!(
+        c.max_abs < max_abs_bound,
+        "max abs diff {:.6} exceeds {max_abs_bound:e}",
+        c.max_abs
+    );
 }

crates/neuron/tests/preflight.rs

@@ -108,6 +108,7 @@ fn spec(model_id: &str, tp: Option<u32>, quant: Option<&str>) -> ModelSpec {
         devices: None,
         sampling: None,
         preserve_thinking: None,
+        expert_residency: None,
     }
 }
 

script/generate_qwen4_exp_fixture.py

@@ -24,6 +24,13 @@ ap.add_argument("--out", required=True, help="fixture directory to write")
 # Seed 2 is tie-free; 0, 1 and 4 are not. See the tie check below.
 ap.add_argument("--seed", type=int, default=2)
 ap.add_argument("--tokens", type=int, default=12)
+ap.add_argument(
+    "--eos-at",
+    type=int,
+    default=None,
+    help="put an eos at this position, so the n-gram window has a segment "
+    "boundary to refuse to read across",
+)
 args = ap.parse_args()
 
 torch.manual_seed(args.seed)
@@ -50,10 +57,15 @@ with torch.no_grad():
         p.copy_(torch.empty_like(p).uniform_(-0.8, 0.8))
 
 ids = torch.randint(0, tc.vocab_size, (1, args.tokens), dtype=torch.long)
-# Keep eos out of the prompt: it starts a new n-gram segment, and the
-# first fixture should not conflate segment handling with everything
-# else. A separate fixture exercises it deliberately.
+# eos is not an ordinary token here: `_shift_right_ignore_eos` treats it
+# as a segment boundary, so an n-gram never reads across one. Placing it
+# is therefore a deliberate choice rather than something to leave to the
+# sampler — a fixture that happens to contain one is testing segment
+# handling by accident, and one that happens not to is not testing it at
+# all.
 ids[ids == tc.eos_token_id] = (tc.eos_token_id + 1) % tc.vocab_size
+if args.eos_at is not None:
+    ids[0, args.eos_at] = tc.eos_token_id
 
 # The QSA selection must not rest on a tie.
 #