fix(context): ignore prefill samples too small to measure throughput

A restart collapsed the advertised context to a sixth of what the host
can serve, and stayed there until real traffic happened along. beast,
minutes after a deploy:

    prefill_tok_per_sec=145.16  throughput_ceiling=17419  -> context=17408
    limit: {"context": 17408, "input": 9216, ...}

against a `vram_ceiling` of 182,560 and a hard ceiling of 131,072.

The cause is not a missing guard — the guard exists and is displaced.
`derive_limit` falls back to `bootstrap_prefill_tok_per_sec` (800,
≈96k context) only when the EMA has *no* sample, and the deploy's own
smoke probe supplies one: 19 tokens in ~130 ms, which is almost entirely
fixed per-request cost and reads as 145 tok/s on a host that measures
~1,400. One probe replaced a conservative estimate with a wrong one.

`fold_rate` already intended to prevent this — "No-op for degenerate
inputs so a probe request or a clock blip can't poison the average" —
but only guarded zero tokens and zero elapsed, not the small-prompt case
that actually occurs.

Prefill samples below 512 tokens are now ignored. That excludes the
smoke probe and the short auxiliary calls agent harnesses make (observed
at 19, 143 and 469 tokens) while admitting ordinary prompts: at
~1,400 tok/s a 512-token prefill runs ~370 ms, so fixed overhead is
about a tenth of it rather than all of it.

Fixed here rather than by warming the host on deploy, which would have
made one caller's experience better once and left the next restart to
rediscover it.

Prefill only. `record_decode` has the same shape of flaw — a two-token
completion is equally overhead-dominated — but it feeds the #137 metrics
rather than the advertised limit, so it is left for a change that can be
attributed on its own.

Four tests: the 19-token probe leaves the EMA unset so the bootstrap
value stands, the 143/469-token auxiliary calls likewise, a
representative prefill still records and still tracks a shift, and the
threshold itself counts.

Docs updated: the operating page no longer tells operators to warm a
restarted host by hand.

asset/neuron/beast.toml

@@ -48,6 +48,40 @@ name = "candle"
 # refused.
 max_in_flight = 2
 
+# Snapshot budget for the cross-request prefix cache (#292).
+#
+# 3 GiB, not the 1 GiB default. At the measured 32 KiB/token/card, a
+# 1 GiB budget cannot hold a snapshot beyond ~32k tokens of K/V alone —
+# less once the GatedDeltaNet conv/recurrent state is deep-copied on top,
+# which on this arch is 48 of 64 layers in f32. Agentic sessions here run
+# well past that: `cached_tokens` was observed pinned at 2,016 for ten
+# consecutive turns while the prompt grew to 52,869, so every turn
+# re-prefilled ~50k tokens — 37 s each, against ~20 s of actual decode.
+# The cache was not missing; the snapshot would not fit.
+#
+# The snapshot budget is reserved BEFORE the KV budget, not from what is
+# left over: measured on this host, kv_budget_mb = 5737 - budget_mb,
+# exactly 1:1. So this number is a direct trade against how long a
+# single session can get, and both sides have a floor:
+#
+#   budget_mb   kv_budget   tokens/slot (2 slots)   holds a 52.9k snapshot
+#   1024        4713        75,400                  no  ← the #292 symptom
+#   3072        2665        42,600                  yes, but sessions
+#                                                   cannot reach 52.9k
+#   2048        3689        59,000                  yes
+#
+# 2 GiB is the value that satisfies both: a 52,869-token snapshot is
+# ~1,652 MB of K/V plus the GatedDeltaNet state (~1.73 GB total), and
+# per-slot KV still covers sessions comfortably past the deepest
+# observed. 3 GiB was tried first and starved KV — a session could no
+# longer reach the size whose snapshot the cache was being enlarged to
+# hold, which is self-defeating.
+#
+# max_entries stays at the default 8: the byte budget binds first here,
+# so the count cap is not what is forcing snapshots to be small.
+[harness.candle.prefix_cache]
+budget_mb = 2048
+
 [[default_models]]
 # Qwen3.8-27B is the resident flagship (#251). Architecturally identical
 # to the 3.6-27B it replaced — same model_type, layer counts and vision

crates/cortex-core/src/catalogue.rs

@@ -67,6 +67,13 @@ pub struct ModelProfile {
     /// model, with nothing forcing them to agree.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub sampling: Option<SamplingOverride>,
+    /// The model's own `preserve_thinking` template control, forwarded
+    /// on cold load so a model this gateway loads behaves the same as
+    /// one the host was configured to hold resident. Two config files
+    /// describing one model is how #252 happened; this rides along for
+    /// the same reason `sampling` does.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub preserve_thinking: Option<bool>,
 
     // ── Enrichment (issue #62) ────────────────────────────────
     /// Per-model token budget. When present, advertised in `/v1/models`
@@ -251,6 +258,7 @@ mod tests {
             residency_priority: None,
             source: None,
             sampling: None,
+            preserve_thinking: None,
             limit: None,
             cost: None,
             capabilities: vec![],

crates/cortex-core/src/harness.rs

@@ -55,13 +55,78 @@ pub struct SamplingOverride {
     pub top_p: Option<f64>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub top_k: Option<usize>,
+    /// Multiplier applied to recently-generated tokens before sampling.
+    /// `1.0` disables it; above that, recently-emitted tokens get less
+    /// likely.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub repeat_penalty: Option<f32>,
+    /// How many recently-generated tokens the penalty considers.
+    ///
+    /// The reason this is operator-settable at all: the built-in default
+    /// is 64, inherited from candle's chat example, and a reasoning
+    /// model's think block runs to tens of thousands of tokens. Anything
+    /// restated more than 64 tokens later is invisible to the penalty,
+    /// which is how a 28,672-token think block can contain the same
+    /// derivation five times over.
+    ///
+    /// Raising it is a real trade, not a free win: the penalty is
+    /// token-level, and source code legitimately repeats tokens
+    /// constantly (`const`, `function`, brace runs). A wide window at a
+    /// meaningful penalty distorts exactly the output this fleet is for.
+    /// Sequence-level repetition wants a sequence-level instrument
+    /// (frequency/presence penalties, or a DRY-style sampler); this knob
+    /// exists so the trade can be *measured* per model rather than
+    /// guessed once and frozen into a constant.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub repeat_last_n: Option<usize>,
+    /// Sequence-wide one-time penalty for already-seen tokens.
+    ///
+    /// This, not `repeat_last_n`, is the remedy Qwen prescribe for the
+    /// endless repetition their models fall into during long reasoning
+    /// (0..2, 1.5 when severe). It is scored over everything generated
+    /// so far, so a passage restated 5,000 tokens later is still
+    /// penalised — the case a trailing window cannot see.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub presence_penalty: Option<f32>,
+    /// Sequence-wide penalty proportional to a token's existing count.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub frequency_penalty: Option<f32>,
+    /// Pin the sampler's RNG for every request to this model.
+    ///
+    /// Exists because parameter search needs a control: comparing two
+    /// `presence_penalty` values across unseeded runs cannot separate
+    /// the parameter's effect from sampling variance, and the clients
+    /// that would otherwise pin it cannot — pi's request builder sends
+    /// no `seed` at all, so a caller-side seed is not available on the
+    /// path that matters.
+    ///
+    /// Note what this does and does not give. It removes sampling noise
+    /// as a confounder; it does not make two runs comparable
+    /// token-by-token, because changing a penalty changes the logits
+    /// and the sequences diverge at the first step where a token choice
+    /// flips.
+    ///
+    /// Leaving this set is a product decision, not just a testing one:
+    /// every identical prompt then yields an identical answer for every
+    /// caller. That is defensible — predictable and cacheable — but it
+    /// removes the variation a user may expect from retrying. A request
+    /// that names its own seed still wins.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub seed: Option<u64>,
 }
 
 impl SamplingOverride {
     /// True when the operator set nothing — an empty `[sampling]` table
     /// must behave exactly as no table at all.
     pub fn is_empty(&self) -> bool {
-        self.temperature.is_none() && self.top_p.is_none() && self.top_k.is_none()
+        self.temperature.is_none()
+            && self.top_p.is_none()
+            && self.top_k.is_none()
+            && self.repeat_penalty.is_none()
+            && self.repeat_last_n.is_none()
+            && self.presence_penalty.is_none()
+            && self.frequency_penalty.is_none()
+            && self.seed.is_none()
     }
 }
 
@@ -86,6 +151,27 @@ pub struct ModelSpec {
     /// still round-trips the spec.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub sampling: Option<SamplingOverride>,
+    /// Whether prior assistant turns keep their reasoning when the chat
+    /// template re-renders the conversation.
+    ///
+    /// This is the model's own `preserve_thinking` template control, not
+    /// an invention of ours. `Qwen/Qwen3.8-27B` defaults it to *true* —
+    /// the whole transcript's think blocks are replayed — and `false`
+    /// keeps reasoning only for turns after the last user query, i.e.
+    /// the turn in progress.
+    ///
+    /// `None` leaves the kwarg unset so the template's own default
+    /// applies, which is the only safe default: the model's authors
+    /// chose it, and overriding that fleet-wide on a hunch is how you
+    /// ship a regression you cannot see.
+    ///
+    /// Exposed as operator config so the premise can be A/B'd on a real
+    /// workload — full replay is the larger prompt and pushes against
+    /// the prefix-cache budget and the throughput-derived context
+    /// ceiling, but whether it *helps* the model is a measurement, not
+    /// a belief. A request's own `chat_template_kwargs` still wins.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub preserve_thinking: Option<bool>,
 }
 
 /// Per-model token budget advertised by the catalogue or neuron.
@@ -247,6 +333,16 @@ pub struct ModelInfo {
     /// marker tokens (`<think>` / `<\/think>` or similar).
     #[serde(default)]
     pub reasoning: bool,
+    /// The operator's `preserve_thinking` for this loaded model, or
+    /// `None` when unset and the template's own default applies.
+    ///
+    /// Advertised so a run can be *stamped* with the value it ran
+    /// under. This is an A/B knob whose whole purpose is comparison
+    /// between runs, and a comparison whose arms cannot be told apart
+    /// afterwards is not a comparison. Recovering it from deploy
+    /// history is guesswork — a config sync can land mid-session.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub preserve_thinking: Option<bool>,
 
     /// Whether this node can actually serve a request for this model
     /// right now (#245).

crates/cortex-core/src/openai.rs

@@ -43,6 +43,20 @@ pub struct ChatCompletionRequest {
     /// why it is a knob rather than a constant (#272).
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub repeat_last_n: Option<usize>,
+    /// OpenAI core, `-2.0..=2.0`. Subtracted once from any token that
+    /// has already appeared, regardless of how often.
+    ///
+    /// Unlike `repetition_penalty` this is scored over the **whole**
+    /// generated sequence, not a trailing window — which is what a
+    /// reasoning model needs, and what Qwen recommend (0..2, 1.5 for
+    /// severe cases) for the endless-repetition failure their own
+    /// models exhibit during long thinking.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub presence_penalty: Option<f32>,
+    /// OpenAI core, `-2.0..=2.0`. Subtracted in proportion to how many
+    /// times a token has already appeared.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub frequency_penalty: Option<f32>,
     /// The deprecated spelling of the output cap. Still what most
     /// clients send; see [`ChatCompletionRequest::effective_max_tokens`].
     #[serde(skip_serializing_if = "Option::is_none")]

crates/cortex-core/src/responses.rs

@@ -70,6 +70,11 @@ pub struct ResponsesRequest {
     /// Window `repetition_penalty` considers (#272).
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub repeat_last_n: Option<usize>,
+    /// OpenAI core; forwarded to the chat path unchanged.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub presence_penalty: Option<f32>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub frequency_penalty: Option<f32>,
     /// Chained-conversation identifier. We don't store responses
     /// server-side yet; if this is `Some`, the handler returns 400.
     #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -271,6 +276,15 @@ pub enum ResponsesOutputItem {
         #[serde(default = "default_item_status")]
         status: String,
     },
+    /// A think block, as its own output item ahead of the message —
+    /// the same shape the streaming projector emits, so a client sees
+    /// one contract whether or not it streams (#300).
+    Reasoning {
+        id: String,
+        summary: Vec<ResponsesSummaryPart>,
+        #[serde(default = "default_item_status")]
+        status: String,
+    },
     /// Reserved for the day tool-call extraction lands. The wire
     /// shape mirrors `ResponsesInputItem::FunctionCall`.
     FunctionCall {
@@ -283,6 +297,24 @@ pub enum ResponsesOutputItem {
     },
 }
 
+/// One `summary_text` part of a reasoning output item.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ResponsesSummaryPart {
+    /// Always `"summary_text"`.
+    #[serde(rename = "type")]
+    pub part_type: String,
+    pub text: String,
+}
+
+impl ResponsesSummaryPart {
+    pub fn summary_text(text: impl Into<String>) -> Self {
+        Self {
+            part_type: "summary_text".into(),
+            text: text.into(),
+        }
+    }
+}
+
 fn default_item_status() -> String {
     "completed".into()
 }

crates/cortex-core/src/translate.rs

@@ -130,6 +130,8 @@ pub fn anthropic_to_openai(req: MessagesRequest) -> ChatCompletionRequest {
         seed: None,
         repetition_penalty: None,
         repeat_last_n: None,
+        presence_penalty: None,
+        frequency_penalty: None,
         max_tokens: Some(req.max_tokens),
         // Anthropic has one spelling; it maps onto the legacy field.
         max_completion_tokens: None,
@@ -413,17 +415,26 @@ fn anthropic_content_to_text(content: AnthropicContent) -> String {
 pub fn openai_to_anthropic(resp: ChatCompletionResponse) -> MessagesResponse {
     let choice = resp.choices.into_iter().next();
 
-    let (content_text, stop_reason) = match choice {
+    let (content_text, reasoning_text, stop_reason) = match choice {
         Some(c) => {
             let text = match c.message.content {
                 MessageContent::Text(t) => t,
                 MessageContent::Parts(parts) => serde_json::to_string(&parts).unwrap_or_default(),
                 MessageContent::Null => String::new(),
             };
+            // The model's think block, which the harness returns on
+            // `message.reasoning_content` (#300).
+            let reasoning = c
+                .message
+                .extra
+                .get("reasoning_content")
+                .and_then(|v| v.as_str())
+                .map(str::to_string)
+                .filter(|t| !t.is_empty());
             let stop = c.finish_reason.map(|r| map_stop_reason(&r));
-            (text, stop)
+            (text, reasoning, stop)
         }
-        None => (String::new(), None),
+        None => (String::new(), None, None),
     };
 
     let usage = resp.usage.unwrap_or(Usage {
@@ -439,10 +450,32 @@ pub fn openai_to_anthropic(resp: ChatCompletionResponse) -> MessagesResponse {
         id: resp.id,
         response_type: "message".into(),
         role: "assistant".into(),
-        content: vec![ContentBlock {
-            block_type: "text".into(),
-            data: json!({ "text": content_text }),
-        }],
+        // A `thinking` block ahead of the text, which is Anthropic's
+        // own ordering. This path accepted `thinking.budget_tokens` on
+        // the way in (see `anthropic_to_openai`) and returned nothing on
+        // the way out: the caller spent the tokens, could not see them,
+        // and could not replay the model's reasoning next turn (#300).
+        //
+        // Emitted without a `signature`. Anthropic signs its thinking
+        // blocks and a client that round-trips one may expect the field;
+        // we cannot mint a valid signature and inventing one would be
+        // worse than omitting it. If a real client rejects an unsigned
+        // block, the fix is to gate this behind a flag — not to go back
+        // to silently deleting the model's work.
+        content: {
+            let mut blocks = Vec::with_capacity(2);
+            if let Some(thinking) = reasoning_text {
+                blocks.push(ContentBlock {
+                    block_type: "thinking".into(),
+                    data: json!({ "thinking": thinking }),
+                });
+            }
+            blocks.push(ContentBlock {
+                block_type: "text".into(),
+                data: json!({ "text": content_text }),
+            });
+            blocks
+        },
         model: resp.model,
         stop_reason,
         usage: AnthropicUsage {
@@ -499,6 +532,7 @@ pub struct AnthropicStreamTranslator {
 enum OpenBlock {
     Text,
     ToolUse,
+    Thinking,
 }
 
 impl AnthropicStreamTranslator {
@@ -538,6 +572,32 @@ impl AnthropicStreamTranslator {
         }
 
         for choice in &chunk.choices {
+            // The model's think block. Dropped entirely until #300: this
+            // translator read `content` and `tool_calls` and nothing
+            // else, so an Anthropic client asked for extended thinking
+            // (`thinking.budget_tokens`, which `anthropic_to_openai`
+            // accepts), was billed for the tokens, and received a stream
+            // with no thinking in it — every turn, silently. Fixing only
+            // the non-streaming translation would have missed this,
+            // because the clients that ask for thinking stream.
+            if let Some(text) = choice
+                .delta
+                .get("reasoning_content")
+                .and_then(Value::as_str)
+                && !text.is_empty()
+            {
+                self.ensure_thinking_block(&mut out);
+                let index = self.open_block.as_ref().map(|(i, _)| *i).unwrap_or(0);
+                out.push((
+                    "content_block_delta".to_string(),
+                    json!({
+                        "type": "content_block_delta",
+                        "index": index,
+                        "delta": { "type": "thinking_delta", "thinking": text }
+                    }),
+                ));
+            }
+
             if let Some(text) = choice.delta.get("content").and_then(Value::as_str)
                 && !text.is_empty()
             {
@@ -651,6 +711,32 @@ impl AnthropicStreamTranslator {
         out
     }
 
+    /// Open a `thinking` block, closing whatever preceded it.
+    ///
+    /// Emitted without a `signature`. Anthropic signs its thinking
+    /// blocks; we cannot mint a valid one, and inventing a value would
+    /// be worse than omitting it. If a client rejects an unsigned block
+    /// the fix is a flag — not a return to deleting the model's work.
+    fn ensure_thinking_block(&mut self, out: &mut Vec<(String, Value)>) {
+        match &self.open_block {
+            Some((_, OpenBlock::Thinking)) => {}
+            _ => {
+                self.close_open_block(out);
+                let index = self.next_index;
+                self.next_index += 1;
+                self.open_block = Some((index, OpenBlock::Thinking));
+                out.push((
+                    "content_block_start".to_string(),
+                    json!({
+                        "type": "content_block_start",
+                        "index": index,
+                        "content_block": { "type": "thinking", "thinking": "" }
+                    }),
+                ));
+            }
+        }
+    }
+
     fn ensure_text_block(&mut self, out: &mut Vec<(String, Value)>) {
         match &self.open_block {
             Some((_, OpenBlock::Text)) => {}
@@ -1060,4 +1146,115 @@ mod request_tests {
         let tools = openai.extra.get("tools").and_then(Value::as_array).unwrap();
         assert_eq!(tools[0]["function"]["name"], "x");
     }
+
+    /// The Anthropic path must return the model's thinking (#300).
+    ///
+    /// It accepted `thinking.budget_tokens` on the way in and returned
+    /// a bare text block on the way out: the caller spent the tokens,
+    /// never saw them, and could not replay the model's reasoning on
+    /// the next turn. Taking a parameter and silently discarding what
+    /// it buys is the defect, not the missing field.
+    #[test]
+    fn anthropic_response_carries_a_thinking_block() {
+        let mut extra = serde_json::Map::new();
+        extra.insert("reasoning_content".into(), json!("weighing the options"));
+        let resp = ChatCompletionResponse {
+            id: "id".into(),
+            object: "chat.completion".into(),
+            created: 0,
+            model: "m".into(),
+            choices: vec![crate::openai::ChatCompletionChoice {
+                index: 0,
+                message: ChatMessage {
+                    role: "assistant".into(),
+                    content: MessageContent::Text("the answer".into()),
+                    extra: Value::Object(extra),
+                },
+                finish_reason: Some("stop".into()),
+                extra: Value::Null,
+            }],
+            usage: None,
+            extra: Value::Null,
+        };
+        let a = openai_to_anthropic(resp);
+        assert_eq!(a.content.len(), 2, "expected thinking + text");
+        assert_eq!(a.content[0].block_type, "thinking", "thinking comes first");
+        assert_eq!(a.content[0].data["thinking"], "weighing the options");
+        assert_eq!(a.content[1].block_type, "text");
+        assert_eq!(a.content[1].data["text"], "the answer");
+    }
+
+    /// No thinking means the unchanged single text block.
+    #[test]
+    fn anthropic_response_without_thinking_is_unchanged() {
+        let resp = ChatCompletionResponse {
+            id: "id".into(),
+            object: "chat.completion".into(),
+            created: 0,
+            model: "m".into(),
+            choices: vec![crate::openai::ChatCompletionChoice {
+                index: 0,
+                message: ChatMessage {
+                    role: "assistant".into(),
+                    content: MessageContent::Text("plain".into()),
+                    extra: Value::Object(Default::default()),
+                },
+                finish_reason: Some("stop".into()),
+                extra: Value::Null,
+            }],
+            usage: None,
+            extra: Value::Null,
+        };
+        let a = openai_to_anthropic(resp);
+        assert_eq!(a.content.len(), 1);
+        assert_eq!(a.content[0].block_type, "text");
+    }
+
+    /// Streaming Anthropic must carry thinking too (#300).
+    ///
+    /// Fixing only the non-streaming translation would have missed this
+    /// entirely — and missed it for the clients that matter, since the
+    /// ones asking for extended thinking stream.
+    #[test]
+    fn anthropic_stream_emits_thinking_deltas() {
+        use crate::openai::{ChatCompletionChunk, ChunkChoice};
+        let mut t = AnthropicStreamTranslator::new();
+        let chunk = |delta: Value| ChatCompletionChunk {
+            id: "c".into(),
+            object: "chat.completion.chunk".into(),
+            created: 0,
+            model: "m".into(),
+            choices: vec![ChunkChoice {
+                index: 0,
+                delta,
+                finish_reason: None,
+                extra: Value::Null,
+            }],
+            usage: None,
+            extra: Value::Null,
+        };
+
+        let ev = t.on_chunk(&chunk(json!({ "reasoning_content": "weighing it" })));
+        let start = ev
+            .iter()
+            .find(|(n, _)| n == "content_block_start")
+            .expect("a block must open");
+        assert_eq!(start.1["content_block"]["type"], "thinking");
+        let delta = ev
+            .iter()
+            .find(|(n, _)| n == "content_block_delta")
+            .expect("a delta must follow");
+        assert_eq!(delta.1["delta"]["type"], "thinking_delta");
+        assert_eq!(delta.1["delta"]["thinking"], "weighing it");
+
+        // Visible text opens a separate block, after thinking closes.
+        let ev = t.on_chunk(&chunk(json!({ "content": "the answer" })));
+        let names: Vec<&str> = ev.iter().map(|(n, _)| n.as_str()).collect();
+        assert!(
+            names.contains(&"content_block_stop"),
+            "the thinking block must close first: {names:?}"
+        );
+        let start = ev.iter().find(|(n, _)| n == "content_block_start").unwrap();
+        assert_eq!(start.1["content_block"]["type"], "text");
+    }
 }

crates/cortex-gateway/src/router.rs

@@ -539,6 +539,7 @@ async fn profile_to_spec(
         // (#283), so a cold-loaded model samples the same way as one
         // this host was configured to hold resident.
         sampling: profile.sampling.clone(),
+        preserve_thinking: profile.preserve_thinking,
     }
 }
 
@@ -647,6 +648,7 @@ mod tests {
             residency_priority: None,
             source: source.map(String::from),
             sampling: None,
+            preserve_thinking: None,
             limit: None,
             cost: None,
             capabilities: vec![],

crates/helexa-bench/src/client.rs

@@ -217,6 +217,7 @@ impl TargetClient {
             // entry supplies the override; it would matter if this were
             // ever used to load a model the host has no config for.
             sampling: None,
+            preserve_thinking: None,
         })
     }
 
@@ -281,6 +282,7 @@ impl TargetClient {
                         cost: None,
                         tool_call: false,
                         reasoning: false,
+                        preserve_thinking: None,
                         // A generic OpenAI endpoint says nothing about
                         // serviceability, and absent means "no opinion".
                         servable: None,

crates/neuron/src/api.rs

@@ -486,9 +486,9 @@ async fn responses(
                     input_tokens: u.prompt_tokens,
                     output_tokens: u.completion_tokens,
                     total_tokens: u.prompt_tokens + u.completion_tokens,
-                    // Carry the reasoning sub-count through from the chat
-                    // usage — the non-streaming path now splits off the
-                    // `<think>` span and counts it (see `split_off_reasoning`).
+                    // Carry the reasoning sub-count through from the
+                    // chat usage: the collector counts the `<think>`
+                    // span and returns its text separately.
                     output_tokens_details: u.completion_tokens_details.as_ref().map(|d| {
                         OutputTokensDetails {
                             reasoning_tokens: d.reasoning_tokens,
@@ -502,8 +502,18 @@ async fn responses(
                     model_id,
                     message_item_id: mint_message_item_id(),
                 };
-                let _ = chat_resp; // make the borrow-checker happy if `text` consumed it
-                let resp = openai_responses::build_response(&meta, text, finish, usage);
+                // The model's think block, which the chat collector now
+                // returns on `message.reasoning_content` (#300). Without
+                // this the caller is billed for `reasoning_tokens` it
+                // never sees, and cannot replay the model's own
+                // reasoning on the next turn.
+                let reasoning = chat_resp
+                    .choices
+                    .first()
+                    .and_then(|c| c.message.extra.get("reasoning_content"))
+                    .and_then(|v| v.as_str())
+                    .map(str::to_string);
+                let resp = openai_responses::build_response(&meta, text, finish, usage, reasoning);
                 Json(resp).into_response()
             }
             Err(e) => inference_error_response(e),

crates/neuron/src/harness/candle.rs

@@ -429,6 +429,17 @@ impl LoadedHandle {
         }
     }
 
+    /// The operator's `preserve_thinking` for this model, advertised so
+    /// an A/B run can be stamped with the value it ran under.
+    pub fn preserve_thinking(&self) -> Option<bool> {
+        match self {
+            LoadedHandle::Single(m) => m.preserve_thinking,
+            #[cfg(feature = "cuda")]
+            LoadedHandle::Tp(m) => m.preserve_thinking,
+            LoadedHandle::Image(_) => None,
+        }
+    }
+
     /// Effort rungs the model's template accepts (#290).
     pub fn reasoning_efforts(&self) -> super::reasoning_effort::SupportedEfforts {
         match self {
@@ -653,6 +664,8 @@ fn requested_sampling(request: &ChatCompletionRequest) -> super::sampling::Reque
         seed: request.seed,
         repeat_penalty: request.repetition_penalty,
         repeat_last_n: request.repeat_last_n,
+        presence_penalty: request.presence_penalty,
+        frequency_penalty: request.frequency_penalty,
     }
 }
 
@@ -726,6 +739,9 @@ pub struct LoadedModel {
     /// no effort control. Drives both what `/models` advertises and
     /// which value we forward on a request.
     pub reasoning_efforts: super::reasoning_effort::SupportedEfforts,
+    /// Operator's `preserve_thinking` for this model, or `None` to let
+    /// the template decide (#H7 A/B). See `ModelSpec::preserve_thinking`.
+    pub preserve_thinking: Option<bool>,
     /// Open/close token IDs for the model's tool-call marker
     /// pair (`<tool_call>` / `</tool_call>` on Qwen3-Coder / Hermes
     /// / DeepSeek / gpt-oss). `None` for models that don't emit
@@ -884,6 +900,9 @@ pub struct TpLoadedModel {
     /// no effort control. Drives both what `/models` advertises and
     /// which value we forward on a request.
     pub reasoning_efforts: super::reasoning_effort::SupportedEfforts,
+    /// Operator's `preserve_thinking` for this model, or `None` to let
+    /// the template decide (#H7 A/B). See `ModelSpec::preserve_thinking`.
+    pub preserve_thinking: Option<bool>,
     /// Same shape as [`LoadedModel::tool_call_tokens`].
     pub tool_call_tokens: Option<ToolCallTokenPair>,
     /// Same shape as [`LoadedModel::chat_template`].
@@ -2012,6 +2031,45 @@ async fn acquire_pool_lock<'a>(
     }
 }
 
+/// Subtract OpenAI-semantics presence/frequency penalties from `logits`.
+///
+/// `logit -= frequency * count + presence * (count > 0)`, scored over
+/// the whole generated sequence. Both default to `0.0`, which is
+/// exactly a no-op — an operator or caller who sets neither sees the
+/// previous behaviour, and the tensor is returned untouched rather than
+/// round-tripped through the CPU.
+///
+/// Deliberately additive on logits, not multiplicative like
+/// `repeat_penalty`: that is what the OpenAI parameters mean, and a
+/// client that sends `presence_penalty: 1.5` expecting OpenAI behaviour
+/// must get it.
+fn apply_sequence_penalties(
+    logits: &Tensor,
+    history: &[u32],
+    presence: f32,
+    frequency: f32,
+) -> Result<Tensor> {
+    if (presence == 0.0 && frequency == 0.0) || history.is_empty() {
+        return Ok(logits.clone());
+    }
+    let device = logits.device().clone();
+    let mut values = logits.to_dtype(candle_core::DType::F32)?.to_vec1::<f32>()?;
+    let mut counts: std::collections::HashMap<u32, f32> = std::collections::HashMap::new();
+    for t in history {
+        *counts.entry(*t).or_insert(0.0) += 1.0;
+    }
+    for (token, count) in counts {
+        let Some(slot) = values.get_mut(token as usize) else {
+            // A history token outside the vocabulary would be a bug
+            // elsewhere; skip it rather than panicking mid-generation.
+            continue;
+        };
+        *slot -= frequency * count + presence;
+    }
+    let len = values.len();
+    Ok(Tensor::from_vec(values, len, &device)?)
+}
+
 /// Apply the repetition penalty (if any) to the prediction logits and
 /// then sample. Centralises the prefill / generation-loop call sites
 /// so they share identical sampling behaviour.
@@ -2040,6 +2098,19 @@ pub(crate) fn sample_with_penalty(
             &history[start..],
         )?
     };
+    // Sequence-wide penalties, over everything generated so far rather
+    // than a trailing window. This is what a reasoning model needs: at
+    // `repeat_last_n = 64` inside a 28,672-token think block, a passage
+    // restated 5,000 tokens later is invisible to the windowed penalty,
+    // and Qwen's own guidance for the endless-repetition failure their
+    // models show during long thinking is presence_penalty (0..2, 1.5
+    // when severe) — not a wider window.
+    let penalised = apply_sequence_penalties(
+        &penalised,
+        history,
+        sampling.presence_penalty,
+        sampling.frequency_penalty,
+    )?;
     let sampled = logits_processor.sample(&penalised)?;
     // The reasoning budget governs which token is *emitted*, not which
     // is sampled: over budget inside a think block, the model gets its
@@ -2944,6 +3015,7 @@ impl CandleHarness {
             loaded.chat_template.as_deref(),
             &request,
             &loaded.reasoning_efforts,
+            loaded.preserve_thinking,
         )?;
         let encoding = loaded
             .tokenizer
@@ -3517,6 +3589,7 @@ impl Harness for CandleHarness {
                 cost: None,
                 tool_call: h.has_tool_call(),
                 reasoning: h.has_reasoning(),
+                preserve_thinking: h.preserve_thinking(),
                 servable: h.servability(&self.context_limit_cfg),
                 // The rungs this model's own template accepts (#290),
                 // discovered at load rather than invented. Advertising a
@@ -3556,6 +3629,7 @@ impl Harness for CandleHarness {
                     capabilities: snap.capabilities.clone(),
                     limit: None,
                     cost: None,
+                    preserve_thinking: None,
                     tool_call: false, // snapshot doesn't carry these; safe default
                     reasoning: false,
                     // Mid-recovery: the handle is gone, so there is
@@ -3846,10 +3920,19 @@ impl Harness for CandleHarness {
             .transpose()
             .unwrap_or_default()
             .unwrap_or_default();
+        // Stamp the A/B arm in the journal at load, so a session
+        // reviewed later can be attributed to the value it ran under
+        // without reconstructing deploy history.
+        tracing::info!(
+            model = %spec.model_id,
+            preserve_thinking = ?spec.preserve_thinking,
+            "reasoning replay: preserve_thinking for this load (None = the template's own default)"
+        );
         let loaded = Arc::new(LoadedModel {
             model_id: spec.model_id.clone(),
             generation_defaults,
             reasoning_efforts,
+            preserve_thinking: spec.preserve_thinking,
             arch: arch_local,
             tokenizer,
             device,
@@ -4499,10 +4582,19 @@ impl CandleHarness {
             .transpose()
             .unwrap_or_default()
             .unwrap_or_default();
+        // Stamp the A/B arm in the journal at load, so a session
+        // reviewed later can be attributed to the value it ran under
+        // without reconstructing deploy history.
+        tracing::info!(
+            model = %spec.model_id,
+            preserve_thinking = ?spec.preserve_thinking,
+            "reasoning replay: preserve_thinking for this load (None = the template's own default)"
+        );
         let tp_loaded = StdArc::new(TpLoadedModel {
             model_id: spec.model_id.clone(),
             generation_defaults,
             reasoning_efforts,
+            preserve_thinking: spec.preserve_thinking,
             tokenizer,
             devices: devices.clone(),
             pool: StdArc::new(TMutex::new(pool)),
@@ -4639,8 +4731,12 @@ impl CandleHarness {
             });
         }
 
-        let prompt =
-            build_prompt_for_request(tp.chat_template.as_deref(), &request, &tp.reasoning_efforts)?;
+        let prompt = build_prompt_for_request(
+            tp.chat_template.as_deref(),
+            &request,
+            &tp.reasoning_efforts,
+            tp.preserve_thinking,
+        )?;
         let encoding = tp
             .tokenizer
             .encode(prompt.as_str(), true)
@@ -5904,10 +6000,44 @@ fn apply_effort_kwarg(
     serde_json::Value::Object(obj)
 }
 
+/// Apply the operator's `preserve_thinking`, if they set one.
+///
+/// Precedence is request > operator > the model's own template default.
+/// A caller that sends `chat_template_kwargs.preserve_thinking` has
+/// said something specific and keeps it; an operator who sets nothing
+/// changes nothing, so the template's `|default` still governs.
+///
+/// Deliberately *not* defaulted by us. On `Qwen/Qwen3.8-27B` the
+/// template preserves every turn's reasoning unless told otherwise —
+/// its authors' choice — and `false` narrows replay to the turn in
+/// progress. Both are defensible and the difference is measurable, so
+/// this exists to run that measurement rather than to encode a guess.
+fn apply_preserve_thinking_kwarg(
+    kwargs: serde_json::Value,
+    configured: Option<bool>,
+) -> serde_json::Value {
+    let Some(configured) = configured else {
+        return kwargs;
+    };
+    if kwargs.get("preserve_thinking").is_some() {
+        return kwargs;
+    }
+    let mut obj = match kwargs {
+        serde_json::Value::Object(o) => o,
+        _ => serde_json::Map::new(),
+    };
+    obj.insert(
+        "preserve_thinking".into(),
+        serde_json::Value::Bool(configured),
+    );
+    serde_json::Value::Object(obj)
+}
+
 fn build_prompt_for_request(
     chat_template: Option<&str>,
     request: &ChatCompletionRequest,
     efforts: &super::reasoning_effort::SupportedEfforts,
+    preserve_thinking: Option<bool>,
 ) -> Result<String, InferenceError> {
     if !super::chat_template::chat_templates_enabled() {
         return Ok(format_qwen3_prompt(&request.messages));
@@ -5932,6 +6062,9 @@ fn build_prompt_for_request(
     // request was instructed to think maximally and then truncated at
     // whatever budget the caller had actually asked for.
     let kwargs = apply_effort_kwarg(kwargs, request, efforts);
+    // Whether prior turns keep their think blocks. Unset unless the
+    // operator chose, so the template's own default stands.
+    let kwargs = apply_preserve_thinking_kwarg(kwargs, preserve_thinking);
     let tools = request
         .extra
         .get("tools")
@@ -7321,6 +7454,8 @@ mod tests {
             seed: None,
             repetition_penalty: None,
             repeat_last_n: None,
+            presence_penalty: None,
+            frequency_penalty: None,
             max_tokens: None,
             max_completion_tokens: None,
             stream: None,
@@ -7384,6 +7519,49 @@ mod tests {
         assert_eq!(effort_in(&out), None);
     }
 
+    /// Unset by default: the model's authors chose the template's
+    /// default and we do not override it fleet-wide on a hunch.
+    #[test]
+    fn no_operator_setting_leaves_preserve_thinking_alone() {
+        let out = apply_preserve_thinking_kwarg(serde_json::Value::Null, None);
+        assert!(out.get("preserve_thinking").is_none());
+        assert!(out.is_null(), "an untouched Null must stay Null");
+    }
+
+    /// The A/B knob: an operator can narrow replay to the turn in
+    /// progress, or pin full replay, without a rebuild.
+    #[test]
+    fn an_operator_setting_reaches_the_template() {
+        for configured in [true, false] {
+            let out = apply_preserve_thinking_kwarg(serde_json::Value::Null, Some(configured));
+            assert_eq!(out["preserve_thinking"], serde_json::json!(configured));
+        }
+    }
+
+    /// A caller that named it has said something specific; the
+    /// operator's default must not overwrite it.
+    #[test]
+    fn a_requested_preserve_thinking_outranks_the_operator() {
+        let out = apply_preserve_thinking_kwarg(
+            serde_json::json!({ "preserve_thinking": true }),
+            Some(false),
+        );
+        assert_eq!(out["preserve_thinking"], serde_json::json!(true));
+    }
+
+    /// It must not clobber the effort kwarg sharing the same object —
+    /// both are applied to one `chat_template_kwargs`.
+    #[test]
+    fn it_preserves_kwargs_already_present() {
+        let out = apply_preserve_thinking_kwarg(
+            serde_json::json!({ "reasoning_effort": "medium", "custom": 1 }),
+            Some(false),
+        );
+        assert_eq!(out["reasoning_effort"], "medium");
+        assert_eq!(out["custom"], 1);
+        assert_eq!(out["preserve_thinking"], serde_json::json!(false));
+    }
+
     /// A model whose template has no effort control must be left alone.
     #[test]
     fn a_model_without_rungs_gets_no_effort_kwarg() {
@@ -7507,9 +7685,84 @@ mod tests {
             seed: 0,
             repeat_penalty,
             repeat_last_n,
+            presence_penalty: 0.0,
+            frequency_penalty: 0.0,
         }
     }
 
+    /// Presence is a *one-time* subtraction for any token already seen,
+    /// however often. Frequency scales with the count. These are the
+    /// OpenAI semantics, and a client sending `presence_penalty: 1.5`
+    /// expecting them must get them.
+    #[test]
+    fn sequence_penalties_follow_openai_semantics() {
+        use candle_core::{Device, Tensor};
+        let logits = Tensor::from_vec(vec![0.0f32; 6], 6, &Device::Cpu).unwrap();
+        // token 1 appears three times, token 2 once.
+        let history = [1u32, 1, 1, 2];
+
+        let p = apply_sequence_penalties(&logits, &history, 1.0, 0.0)
+            .unwrap()
+            .to_vec1::<f32>()
+            .unwrap();
+        assert_eq!(p[1], -1.0, "presence subtracts once regardless of count");
+        assert_eq!(p[2], -1.0);
+        assert_eq!(p[0], 0.0, "unseen tokens untouched");
+
+        let f = apply_sequence_penalties(&logits, &history, 0.0, 1.0)
+            .unwrap()
+            .to_vec1::<f32>()
+            .unwrap();
+        assert_eq!(f[1], -3.0, "frequency scales with the count");
+        assert_eq!(f[2], -1.0);
+        assert_eq!(f[0], 0.0);
+
+        // Both together compose additively.
+        let b = apply_sequence_penalties(&logits, &history, 0.5, 1.0)
+            .unwrap()
+            .to_vec1::<f32>()
+            .unwrap();
+        assert_eq!(b[1], -3.5);
+    }
+
+    /// The whole point: a passage restated far outside `repeat_last_n`
+    /// is invisible to the windowed penalty but not to this one. At the
+    /// built-in 64-token window inside a 28,672-token think block, that
+    /// is every repetition worth catching (#299).
+    #[test]
+    fn a_repeat_far_outside_the_window_is_still_penalised() {
+        use candle_core::{Device, Tensor};
+        let logits = Tensor::from_vec(vec![0.0f32; 4], 4, &Device::Cpu).unwrap();
+        // token 3 was emitted 5,000 tokens ago and never since.
+        let mut history = vec![3u32];
+        history.extend(std::iter::repeat_n(0u32, 5_000));
+
+        let windowed = &history[history.len().saturating_sub(64)..];
+        assert!(
+            !windowed.contains(&3),
+            "fixture: the token must be outside the 64-token window"
+        );
+
+        let out = apply_sequence_penalties(&logits, &history, 1.5, 0.0)
+            .unwrap()
+            .to_vec1::<f32>()
+            .unwrap();
+        assert_eq!(out[3], -1.5, "sequence-wide penalty still sees it");
+    }
+
+    /// Zero on both is exactly a no-op — an operator who sets neither
+    /// must get byte-identical behaviour to before this existed.
+    #[test]
+    fn zero_sequence_penalties_change_nothing() {
+        use candle_core::{Device, Tensor};
+        let logits = Tensor::from_vec(vec![1.5f32, -2.0, 0.25], 3, &Device::Cpu).unwrap();
+        let out = apply_sequence_penalties(&logits, &[0u32, 1, 1], 0.0, 0.0)
+            .unwrap()
+            .to_vec1::<f32>()
+            .unwrap();
+        assert_eq!(out, vec![1.5, -2.0, 0.25]);
+    }
+
     /// Two inlets converge on one field: an explicit token count, or
     /// Anthropic's normalised `budget_tokens`, which translation folds
     /// into the same shape.
@@ -7533,6 +7786,8 @@ mod tests {
             seed: None,
             repetition_penalty: None,
             repeat_last_n: None,
+            presence_penalty: None,
+            frequency_penalty: None,
             max_tokens: None,
             max_completion_tokens: None,
             stream: None,
@@ -8542,8 +8797,8 @@ mod tests {
             "tools": [{"type": "function", "function": {"name": "x"}}]
         }))
         .unwrap();
-        let err =
-            build_prompt_for_request(Some(bad), &with_tools, &Default::default()).unwrap_err();
+        let err = build_prompt_for_request(Some(bad), &with_tools, &Default::default(), None)
+            .unwrap_err();
         assert!(matches!(err, InferenceError::TemplateRenderFailed { .. }));
 
         // No tools → falling back is harmless, so it stays Ok.
@@ -8552,6 +8807,6 @@ mod tests {
             "messages": [{"role": "user", "content": "hi"}]
         }))
         .unwrap();
-        assert!(build_prompt_for_request(Some(bad), &no_tools, &Default::default()).is_ok());
+        assert!(build_prompt_for_request(Some(bad), &no_tools, &Default::default(), None).is_ok());
     }
 }

crates/neuron/src/harness/chat_template.rs

@@ -438,6 +438,139 @@ mod tests {
         assert!(out.contains("<|vision_start|>") && out.contains("<|vision_end|>"));
     }
 
+    /// A file read back through a tool result must reach the model
+    /// byte-for-byte.
+    ///
+    /// The reported symptom: the model writes a file correctly, reads it
+    /// back, and sees errors that are not in the file — consistently,
+    /// after every write. Anything that mangles `<`, `&`, `"` or
+    /// newlines on this path would produce exactly that, and would make
+    /// the model burn its budget reconciling a file against a corrupted
+    /// view of itself.
+    #[test]
+    fn a_file_read_back_through_a_tool_result_is_byte_exact() {
+        let template = include_str!("../../tests/data/qwen3.8-27b.chat_template.jinja");
+        // The characters most likely to be mangled, in the shapes a real
+        // web project contains them.
+        let file = "<!DOCTYPE html>\n<html lang=\"en\">\n<head>\n  <meta charset=\"utf-8\">\n</head>\n<body>\n  <script>\n    if (a && b || c < d) { g.fillStyle = '#b8905a'; }\n    const s = `rgba(${r},${g},0.5)`;\n\n    // a comment & an ampersand\n  </script>\n</body>\n</html>\n";
+
+        let mut tool_extras = serde_json::Map::new();
+        tool_extras.insert("tool_call_id".into(), json!("call_0"));
+        let mut assistant_extras = serde_json::Map::new();
+        assistant_extras.insert(
+            "tool_calls".into(),
+            json!([{"id":"call_0","type":"function",
+                    "function":{"name":"read","arguments":"{\"path\":\"/tmp/index.html\"}"}}]),
+        );
+
+        let messages = vec![
+            user_msg("read the file back"),
+            ChatMessage {
+                role: "assistant".into(),
+                content: MessageContent::Null,
+                extra: Value::Object(assistant_extras),
+            },
+            ChatMessage {
+                role: "tool".into(),
+                content: MessageContent::Text(file.to_string()),
+                extra: Value::Object(tool_extras),
+            },
+            user_msg("is it correct?"),
+        ];
+        let out =
+            render_chat_template(template, &messages, &Value::Null, &Value::Null).expect("render");
+
+        for probe in [
+            "<!DOCTYPE html>",
+            "<html lang=\"en\">",
+            "if (a && b || c < d)",
+            "g.fillStyle = '#b8905a';",
+            "`rgba(${r},${g},0.5)`",
+            "// a comment & an ampersand",
+        ] {
+            assert!(
+                out.contains(probe),
+                "the model would not see {probe:?} verbatim — rendered prompt mangles it"
+            );
+        }
+        // And nothing HTML-escaped anywhere.
+        for bad in ["&lt;", "&gt;", "&amp;", "&quot;", "&#x27;", "&#39;"] {
+            assert!(
+                !out.contains(bad),
+                "found {bad:?} in the rendered prompt: content is being HTML-escaped"
+            );
+        }
+    }
+
+    /// Blank lines inside a replayed tool-call argument must survive
+    /// rendering intact.
+    ///
+    /// A model that writes a source file and then re-reads it from its
+    /// own replayed tool call is comparing what it remembers writing
+    /// against what the prompt now shows. Any whitespace the pipeline
+    /// eats makes those disagree, and the model has no way to tell a
+    /// rendering artifact from a mistake of its own — which is a
+    /// plausible route into the announce-then-stop stall (#296).
+    ///
+    /// The JSON string→object normalisation above is the step that
+    /// could plausibly lose them, since it round-trips the arguments
+    /// through a parser.
+    #[test]
+    fn a_replayed_tool_call_keeps_the_blank_lines_in_its_arguments() {
+        let template = include_str!("../../tests/data/qwen3.8-27b.chat_template.jinja");
+        // Blank lines between sections, exactly as a written source file
+        // carries them.
+        let file_body = "'use strict';\n\n// ---- constants ----\nconst A = 1;\n\n\nfunction f() {\n  return A;\n}\n";
+        let args = json!({ "path": "/tmp/game.js", "content": file_body });
+        let mut assistant_extras = serde_json::Map::new();
+        assistant_extras.insert(
+            "tool_calls".into(),
+            json!([{
+                "id": "call_0", "type": "function",
+                "function": {
+                    "name": "write",
+                    // The OpenAI wire form: arguments as a JSON *string*.
+                    "arguments": serde_json::to_string(&args).unwrap()
+                }
+            }]),
+        );
+        let mut tool_extras = serde_json::Map::new();
+        tool_extras.insert("tool_call_id".into(), json!("call_0"));
+
+        let messages = vec![
+            user_msg("write the file"),
+            ChatMessage {
+                role: "assistant".into(),
+                content: MessageContent::Null,
+                extra: Value::Object(assistant_extras),
+            },
+            ChatMessage {
+                role: "tool".into(),
+                content: MessageContent::Text("Successfully wrote 84 bytes".into()),
+                extra: Value::Object(tool_extras),
+            },
+            user_msg("now check it parses"),
+        ];
+        let out =
+            render_chat_template(template, &messages, &Value::Null, &Value::Null).expect("render");
+
+        assert!(
+            out.contains(file_body),
+            "the file body must appear verbatim in the rendered prompt"
+        );
+        let blanks_in = file_body.lines().filter(|l| l.is_empty()).count();
+        assert_eq!(blanks_in, 3, "fixture sanity");
+        // And specifically the double newline the model would notice.
+        assert!(
+            out.contains("'use strict';\n\n// ---- constants ----"),
+            "blank line after the prologue was eaten"
+        );
+        assert!(
+            out.contains("const A = 1;\n\n\nfunction f()"),
+            "consecutive blank lines were collapsed"
+        );
+    }
+
     /// The second turn of an OpenAI-native agentic loop: the client
     /// replays its own assistant turn, which carries `tool_calls` and
     /// `"content": null`. That must render as the tool-call block —

crates/neuron/src/harness/context_limit.rs

@@ -36,6 +36,28 @@ use crate::config::ContextLimitConfig;
 /// few turns.
 const RATE_EMA_ALPHA: f64 = 0.3;
 
+/// Smallest prompt whose prefill time says anything about throughput.
+///
+/// A prefill has a fixed per-request cost — tokenize, launch, sync —
+/// that dominates a short prompt completely. The deploy's own smoke
+/// probe prefills 19 tokens in ~130 ms and reads as **145 tok/s** on a
+/// host that measures ~1,400. Folding that in does not make the
+/// estimate slightly worse; it replaces it.
+///
+/// That mattered because the fallback it displaces is the whole
+/// cold-start guard: with no sample at all, `derive_limit` uses
+/// `bootstrap_prefill_tok_per_sec` (800, ≈96k context at the default
+/// 120 s target). One 19-token probe replaced it with 145 tok/s, and
+/// the host advertised **17,408** — a sixth of what it can serve —
+/// until real traffic happened to warm it (#295).
+///
+/// 512 excludes the smoke probe and the short auxiliary calls agent
+/// harnesses make (observed at 19, 143 and 469 tokens) while admitting
+/// ordinary prompts. At ~1,400 tok/s a 512-token prefill runs ~370 ms,
+/// so fixed overhead is roughly a tenth of it — informative rather than
+/// dominant.
+const MIN_PREFILL_SAMPLE_TOKENS: usize = 512;
+
 /// Fold one throughput sample (`tokens` in `elapsed`) into the EMA held in
 /// `bits`. No-op for degenerate inputs so a probe request or a clock blip
 /// can't poison the average.
@@ -99,7 +121,15 @@ impl ThroughputEma {
 
     /// Fold one prefill measurement (`prompt_tokens` processed in
     /// `elapsed`) into the prefill EMA.
+    ///
+    /// Samples below [`MIN_PREFILL_SAMPLE_TOKENS`] are ignored: they
+    /// measure fixed per-request cost rather than throughput, and the
+    /// value they displace — the bootstrap estimate — is better than
+    /// they are.
     pub fn record(&self, prompt_tokens: usize, elapsed: Duration) {
+        if prompt_tokens < MIN_PREFILL_SAMPLE_TOKENS {
+            return;
+        }
         fold_rate(&self.prefill_bits, prompt_tokens, elapsed);
     }
 
@@ -356,6 +386,60 @@ pub fn derive_limit(
 
 #[cfg(test)]
 mod tests {
+
+    /// The deploy's smoke probe must not set the advertised context.
+    ///
+    /// 19 tokens in 130 ms reads as ~145 tok/s on a host that measures
+    /// ~1,400. Before this guard that one sample displaced the
+    /// bootstrap estimate and the host advertised 17,408 instead of
+    /// ~96,000 — until real traffic happened along (#295).
+    #[test]
+    fn a_smoke_probe_does_not_poison_the_prefill_rate() {
+        let ema = ThroughputEma::new();
+        ema.record(19, Duration::from_millis(130));
+        assert_eq!(
+            ema.get(),
+            None,
+            "a 19-token prefill must leave the EMA unset so the bootstrap value stands"
+        );
+    }
+
+    /// The short auxiliary calls agent harnesses make are the same
+    /// shape as the probe and equally uninformative — observed at 143
+    /// and 469 tokens against prompts three orders of magnitude larger.
+    #[test]
+    fn short_auxiliary_prompts_are_ignored_too() {
+        let ema = ThroughputEma::new();
+        for tokens in [143usize, 469] {
+            ema.record(tokens, Duration::from_millis(200));
+        }
+        assert_eq!(ema.get(), None);
+    }
+
+    /// A real prompt still records, and still tracks a shift.
+    #[test]
+    fn representative_prefills_still_measure() {
+        let ema = ThroughputEma::new();
+        ema.record(1400, Duration::from_secs(1));
+        let first = ema.get().expect("a 1400-token sample must record");
+        assert!((first - 1400.0).abs() < 1.0, "got {first}");
+
+        // A slower host is tracked, not ignored.
+        for _ in 0..20 {
+            ema.record(700, Duration::from_secs(1));
+        }
+        let after = ema.get().unwrap();
+        assert!(after < first && after > 600.0, "got {after}");
+    }
+
+    /// Exactly at the threshold counts — the boundary is inclusive so
+    /// the constant means what it says.
+    #[test]
+    fn the_threshold_itself_is_a_valid_sample() {
+        let ema = ThroughputEma::new();
+        ema.record(MIN_PREFILL_SAMPLE_TOKENS, Duration::from_secs(1));
+        assert!(ema.get().is_some());
+    }
     use super::*;
 
     /// The ceiling a client is told to plan against must not be the

crates/neuron/src/harness/preflight.rs

@@ -402,6 +402,7 @@ mod tests {
             tensor_parallel: tp,
             devices: None,
             sampling: None,
+            preserve_thinking: None,
         }
     }
 

crates/neuron/src/harness/sampling.rs

@@ -75,6 +75,15 @@ pub struct ModelGenerationDefaults {
     pub temperature: Option<f64>,
     pub top_p: Option<f64>,
     pub top_k: Option<usize>,
+    /// Not published in `generation_config.json` — this struct is the
+    /// resolved per-model default layer, and the operator override
+    /// (#283) lands here so the repetition knobs sit in the same
+    /// precedence chain as the rest.
+    pub repeat_penalty: Option<f32>,
+    pub repeat_last_n: Option<usize>,
+    pub presence_penalty: Option<f32>,
+    pub frequency_penalty: Option<f32>,
+    pub seed: Option<u64>,
 }
 
 impl ModelGenerationDefaults {
@@ -155,6 +164,21 @@ impl ModelGenerationDefaults {
         if let Some(k) = op.top_k {
             self.top_k = Some(k);
         }
+        if let Some(r) = op.repeat_penalty {
+            self.repeat_penalty = Some(r);
+        }
+        if let Some(n) = op.repeat_last_n {
+            self.repeat_last_n = Some(n);
+        }
+        if let Some(v) = op.presence_penalty {
+            self.presence_penalty = Some(v);
+        }
+        if let Some(v) = op.frequency_penalty {
+            self.frequency_penalty = Some(v);
+        }
+        if let Some(v) = op.seed {
+            self.seed = Some(v);
+        }
         tracing::info!(
             model = %model_id,
             model_temperature = ?before.temperature,
@@ -163,6 +187,11 @@ impl ModelGenerationDefaults {
             temperature = ?self.temperature,
             top_p = ?self.top_p,
             top_k = ?self.top_k,
+            repeat_penalty = ?self.repeat_penalty,
+            repeat_last_n = ?self.repeat_last_n,
+            presence_penalty = ?self.presence_penalty,
+            frequency_penalty = ?self.frequency_penalty,
+            seed = ?self.seed,
             "sampling: operator override applied over the model's published defaults (#283)"
         );
         self
@@ -179,6 +208,8 @@ pub struct RequestedSampling {
     pub seed: Option<u64>,
     pub repeat_penalty: Option<f32>,
     pub repeat_last_n: Option<usize>,
+    pub presence_penalty: Option<f32>,
+    pub frequency_penalty: Option<f32>,
 }
 
 /// Fully-resolved sampling for one request.
@@ -190,6 +221,9 @@ pub struct SamplingParams {
     pub seed: u64,
     pub repeat_penalty: f32,
     pub repeat_last_n: usize,
+    /// Sequence-wide penalties, OpenAI semantics. `0.0` disables.
+    pub presence_penalty: f32,
+    pub frequency_penalty: f32,
 }
 
 impl SamplingParams {
@@ -215,9 +249,23 @@ impl SamplingParams {
             // whenever the caller stayed quiet.
             top_p: requested.top_p.or(model.top_p),
             top_k: requested.top_k.or(model.top_k),
-            seed: requested.seed.unwrap_or(random_seed),
-            repeat_penalty: requested.repeat_penalty.unwrap_or(DEFAULT_REPEAT_PENALTY),
-            repeat_last_n: requested.repeat_last_n.unwrap_or(DEFAULT_REPEAT_LAST_N),
+            seed: requested.seed.or(model.seed).unwrap_or(random_seed),
+            repeat_penalty: requested
+                .repeat_penalty
+                .or(model.repeat_penalty)
+                .unwrap_or(DEFAULT_REPEAT_PENALTY),
+            repeat_last_n: requested
+                .repeat_last_n
+                .or(model.repeat_last_n)
+                .unwrap_or(DEFAULT_REPEAT_LAST_N),
+            presence_penalty: requested
+                .presence_penalty
+                .or(model.presence_penalty)
+                .unwrap_or(0.0),
+            frequency_penalty: requested
+                .frequency_penalty
+                .or(model.frequency_penalty)
+                .unwrap_or(0.0),
         }
     }
 
@@ -252,6 +300,7 @@ mod tests {
             temperature: Some(1.0),
             top_p: Some(0.95),
             top_k: Some(20),
+            ..Default::default()
         }
     }
 
@@ -298,6 +347,7 @@ mod tests {
             temperature,
             top_p,
             top_k,
+            ..Default::default()
         }
     }
 
@@ -394,6 +444,103 @@ mod tests {
     /// A model with no `generation_config.json` must behave exactly as
     /// it did before this landed, so nothing regresses on the models
     /// that do not ship one.
+    /// An operator can pin the seed, because the client that needs it
+    /// pinned cannot send one: pi's request builder emits no `seed`, so
+    /// without this a parameter search runs unseeded and sampling
+    /// variance is indistinguishable from the effect being measured.
+    #[test]
+    fn an_operator_can_pin_the_seed() {
+        let op = cortex_core::harness::SamplingOverride {
+            seed: Some(424_242),
+            ..Default::default()
+        };
+        let defaults = ModelGenerationDefaults::default().with_operator_override("m", Some(&op));
+        // The random seed argument must be ignored once an operator pinned one.
+        let p = SamplingParams::resolve(&RequestedSampling::default(), &defaults, 999);
+        assert_eq!(p.seed, 424_242);
+    }
+
+    /// Precedence holds: a caller that names its own seed keeps it.
+    #[test]
+    fn a_requested_seed_outranks_the_operator() {
+        let op = cortex_core::harness::SamplingOverride {
+            seed: Some(424_242),
+            ..Default::default()
+        };
+        let defaults = ModelGenerationDefaults::default().with_operator_override("m", Some(&op));
+        let req = RequestedSampling {
+            seed: Some(7),
+            ..Default::default()
+        };
+        assert_eq!(SamplingParams::resolve(&req, &defaults, 999).seed, 7);
+    }
+
+    /// With neither, the per-request random seed still applies — an
+    /// unset override must not accidentally freeze the fleet's RNG.
+    #[test]
+    fn no_pinned_seed_leaves_generation_random() {
+        let d = ModelGenerationDefaults::default().with_operator_override("m", None);
+        assert_eq!(
+            SamplingParams::resolve(&RequestedSampling::default(), &d, 999).seed,
+            999
+        );
+    }
+
+    /// The gap this closes: `repeat_last_n` defaults to 64 — candle's
+    /// chat example value — while a reasoning model's think block runs
+    /// to tens of thousands of tokens. Anything restated more than 64
+    /// tokens later is invisible to the penalty, and before this there
+    /// was no operator route to the knob at all: `SamplingOverride`
+    /// carried only temperature/top_p/top_k, and OpenAI-shaped clients
+    /// do not send repetition parameters. The 64 was effectively
+    /// hardcoded in production.
+    #[test]
+    fn an_operator_can_widen_the_repetition_window() {
+        let op = cortex_core::harness::SamplingOverride {
+            repeat_penalty: Some(1.05),
+            repeat_last_n: Some(2048),
+            presence_penalty: None,
+            frequency_penalty: None,
+            ..Default::default()
+        };
+        let defaults = ModelGenerationDefaults::default().with_operator_override("m", Some(&op));
+        let p = SamplingParams::resolve(&RequestedSampling::default(), &defaults, 7);
+        assert_eq!(p.repeat_last_n, 2048);
+        assert!((p.repeat_penalty - 1.05).abs() < f32::EPSILON);
+    }
+
+    /// Precedence is request > operator > built-in, same as every other
+    /// sampling field — a caller that named a window keeps it.
+    #[test]
+    fn a_requested_repetition_window_outranks_the_operator() {
+        let op = cortex_core::harness::SamplingOverride {
+            repeat_last_n: Some(2048),
+            presence_penalty: None,
+            frequency_penalty: None,
+            ..Default::default()
+        };
+        let defaults = ModelGenerationDefaults::default().with_operator_override("m", Some(&op));
+        let req = RequestedSampling {
+            repeat_last_n: Some(128),
+            presence_penalty: None,
+            frequency_penalty: None,
+            ..Default::default()
+        };
+        assert_eq!(
+            SamplingParams::resolve(&req, &defaults, 7).repeat_last_n,
+            128
+        );
+    }
+
+    /// An operator who sets nothing must not change the constants.
+    #[test]
+    fn no_operator_setting_leaves_the_repetition_defaults_alone() {
+        let d = ModelGenerationDefaults::default().with_operator_override("m", None);
+        let p = SamplingParams::resolve(&RequestedSampling::default(), &d, 7);
+        assert_eq!(p.repeat_last_n, DEFAULT_REPEAT_LAST_N);
+        assert!((p.repeat_penalty - DEFAULT_REPEAT_PENALTY).abs() < f32::EPSILON);
+    }
+
     #[test]
     fn without_model_defaults_the_old_behaviour_is_preserved() {
         let p = SamplingParams::resolve(
@@ -495,6 +642,8 @@ mod tests {
             &RequestedSampling {
                 repeat_penalty: Some(1.05),
                 repeat_last_n: Some(512),
+                presence_penalty: None,
+                frequency_penalty: None,
                 ..Default::default()
             },
             &ModelGenerationDefaults::default(),

crates/neuron/src/wire/openai_chat.rs

@@ -468,11 +468,21 @@ fn usage_chunk(
 /// where the same wire object could be assembled differently — the
 /// shape of defect #252 and #272 both were.
 ///
-/// Semantics deliberately preserved from the path this replaces:
-/// reasoning text is **dropped** from `content` (it is counted in
-/// `usage.completion_tokens_details.reasoning_tokens` but never
-/// rendered), and `finish_reason` becomes `"tool_calls"` whenever any
-/// tool call was parsed.
+/// Reasoning is kept out of `content` — it is not the assistant's
+/// reply — but it is **returned** on `message.reasoning_content`, the
+/// same field the streaming path emits as
+/// `choice.delta.reasoning_content`.
+///
+/// It used to be discarded here (#300). That was inherited from the
+/// path this replaced and was wrong on both counts: the tokens are
+/// metered in `usage.completion_tokens_details.reasoning_tokens`, so a
+/// caller paid for text it could never see; and a client cannot replay
+/// what it never received, so on the next turn the model saw its own
+/// prior turn as a bare assertion with the reasoning that produced it
+/// deleted.
+///
+/// `finish_reason` becomes `"tool_calls"` whenever any tool call was
+/// parsed.
 ///
 /// Two fields that the old path hardcoded to `None` now populate,
 /// because the events carry them and the struct already had the slots:
@@ -488,6 +498,7 @@ pub async fn collect_chat_completion(
     model_id: String,
 ) -> Result<ChatCompletionResponse, String> {
     let mut content = String::new();
+    let mut reasoning = String::new();
     let mut tool_calls: Vec<serde_json::Value> = Vec::new();
     let mut finish: Option<(FinishReason, u32, u32, u32, u32, Option<FinishTiming>)> = None;
 
@@ -495,8 +506,8 @@ pub async fn collect_chat_completion(
         match event {
             InferenceEvent::Start => {}
             InferenceEvent::TextDelta(t) => content.push_str(&t),
-            // Counted, never rendered — see the doc comment.
-            InferenceEvent::ReasoningDelta(_) => {}
+            // Kept out of `content`, returned on `reasoning_content`.
+            InferenceEvent::ReasoningDelta(t) => reasoning.push_str(&t),
             InferenceEvent::ToolCall {
                 index,
                 id: call_id,
@@ -545,11 +556,16 @@ pub async fn collect_chat_completion(
     } else {
         "tool_calls".to_string()
     };
-    let message_extra = if tool_calls.is_empty() {
-        serde_json::Value::Object(Default::default())
-    } else {
-        json!({ "tool_calls": tool_calls })
-    };
+    let mut message_extra = serde_json::Map::new();
+    if !tool_calls.is_empty() {
+        message_extra.insert("tool_calls".into(), json!(tool_calls));
+    }
+    // Absent rather than empty when the model did not think: a
+    // non-reasoning model must keep the unchanged message shape.
+    if !reasoning.is_empty() {
+        message_extra.insert("reasoning_content".into(), json!(reasoning));
+    }
+    let message_extra = serde_json::Value::Object(message_extra);
 
     Ok(ChatCompletionResponse {
         id,
@@ -588,6 +604,86 @@ pub async fn collect_chat_completion(
 
 #[cfg(test)]
 mod tests {
+
+    /// The model's think block must come back on the non-streaming
+    /// path too (#300).
+    ///
+    /// It used to be dropped here while the streaming path emitted it
+    /// as `choice.delta.reasoning_content`. The tokens were metered
+    /// either way, so a non-streaming caller paid for text it could
+    /// never see — and, worse, could not replay the model's reasoning
+    /// on the next turn, leaving the model to face its own prior turn
+    /// as a bare assertion.
+    #[tokio::test]
+    async fn non_streaming_returns_the_models_reasoning() {
+        let (tx, rx) = mpsc::channel::<InferenceEvent>(8);
+        tx.send(InferenceEvent::Start).await.unwrap();
+        tx.send(InferenceEvent::ReasoningDelta(
+            "weighing the options".into(),
+        ))
+        .await
+        .unwrap();
+        tx.send(InferenceEvent::TextDelta("the answer".into()))
+            .await
+            .unwrap();
+        tx.send(InferenceEvent::Finish {
+            reason: FinishReason::Stop,
+            prompt_tokens: 5,
+            completion_tokens: 7,
+            reasoning_tokens: 3,
+            cached_tokens: 0,
+            timing: None,
+        })
+        .await
+        .unwrap();
+        drop(tx);
+
+        let r = collect_chat_completion(rx, "id".into(), 0, "m".into())
+            .await
+            .expect("collect");
+        let msg = &r.choices[0].message;
+        assert_eq!(
+            msg.extra["reasoning_content"], "weighing the options",
+            "reasoning must be returned, not discarded"
+        );
+        // And it must NOT be folded into the visible reply.
+        assert!(
+            matches!(&msg.content, MessageContent::Text(t) if t == "the answer"),
+            "reasoning leaked into content"
+        );
+    }
+
+    /// A non-reasoning model must keep the unchanged message shape —
+    /// absent, not an empty string.
+    #[tokio::test]
+    async fn no_reasoning_means_no_reasoning_field() {
+        let (tx, rx) = mpsc::channel::<InferenceEvent>(8);
+        tx.send(InferenceEvent::Start).await.unwrap();
+        tx.send(InferenceEvent::TextDelta("plain".into()))
+            .await
+            .unwrap();
+        tx.send(InferenceEvent::Finish {
+            reason: FinishReason::Stop,
+            prompt_tokens: 1,
+            completion_tokens: 1,
+            reasoning_tokens: 0,
+            cached_tokens: 0,
+            timing: None,
+        })
+        .await
+        .unwrap();
+        drop(tx);
+        let r = collect_chat_completion(rx, "id".into(), 0, "m".into())
+            .await
+            .expect("collect");
+        assert!(
+            r.choices[0]
+                .message
+                .extra
+                .get("reasoning_content")
+                .is_none()
+        );
+    }
     use super::*;
 
     // ── Non-streaming collector (#285) ────────────────────────────

crates/neuron/src/wire/openai_responses.rs

@@ -176,6 +176,29 @@ pub fn request_to_chat(req: ResponsesRequest) -> Result<ChatCompletionRequest, T
                     {
                         attach_reasoning(&mut msg, text);
                     }
+                    // A `function_call` item belongs to the assistant
+                    // turn that precedes it, not to a turn of its own.
+                    //
+                    // Responses splits one turn across several items —
+                    // `reasoning`, then `message`, then `function_call`
+                    // — while chat carries all three on a single
+                    // assistant message. Pushing each item separately
+                    // produced *consecutive assistant turns*: one that
+                    // deliberates and then ends without acting, and one
+                    // that acts with an empty `<think></think>`.
+                    //
+                    // Neither shape exists in training, and the model
+                    // imitates what it is shown: after four such
+                    // examples in its own replayed context it produced
+                    // a think block and a preamble, then ended the turn
+                    // without the tool call it had just announced
+                    // (#296). Observed on a real session; the model was
+                    // copying our transcript, not misbehaving.
+                    if let Some(prev) = messages.last_mut()
+                        && merge_tool_call_into(prev, &mut msg)
+                    {
+                        continue;
+                    }
                     messages.push(msg);
                 }
             }
@@ -239,6 +262,8 @@ pub fn request_to_chat(req: ResponsesRequest) -> Result<ChatCompletionRequest, T
         seed: req.seed,
         repetition_penalty: req.repetition_penalty,
         repeat_last_n: req.repeat_last_n,
+        presence_penalty: req.presence_penalty,
+        frequency_penalty: req.frequency_penalty,
         max_tokens: req.max_output_tokens,
         // Responses spells the cap `max_output_tokens`, already
         // resolved onto the legacy field above.
@@ -376,6 +401,57 @@ fn apply_reasoning_effort(extra: &mut serde_json::Map<String, Value>) {
     }
 }
 
+/// Fold a `function_call`-derived assistant message into the assistant
+/// turn it belongs to.
+///
+/// Returns `true` when the call was merged and the caller should drop
+/// its message. Only ever merges a *pure* tool-call message (empty
+/// content, no reasoning of its own) into a preceding assistant turn,
+/// so a genuine two-turn sequence is never collapsed and a reasoning
+/// item that arrived with the call keeps its own turn.
+///
+/// Consecutive calls accumulate, which is what parallel tool calling
+/// looks like on the wire: one assistant turn, several `tool_calls`.
+fn merge_tool_call_into(prev: &mut ChatMessage, msg: &mut ChatMessage) -> bool {
+    if prev.role != "assistant" || msg.role != "assistant" {
+        return false;
+    }
+    // Only a bare tool-call carrier folds in.
+    if !matches!(&msg.content, MessageContent::Text(t) if t.is_empty()) {
+        return false;
+    }
+    if msg.extra.get("reasoning_content").is_some() {
+        return false;
+    }
+    let Some(incoming) = msg
+        .extra
+        .get_mut("tool_calls")
+        .and_then(|v| v.as_array_mut())
+        .map(std::mem::take)
+    else {
+        return false;
+    };
+    // Everything else the item carried must survive the fold, or this
+    // trades one silent drop for another.
+    if msg
+        .extra
+        .as_object()
+        .is_some_and(|o| o.keys().any(|k| k != "tool_calls"))
+    {
+        return false;
+    }
+    let Some(obj) = prev.extra.as_object_mut() else {
+        return false;
+    };
+    match obj.get_mut("tool_calls").and_then(|v| v.as_array_mut()) {
+        Some(existing) => existing.extend(incoming),
+        None => {
+            obj.insert("tool_calls".into(), Value::Array(incoming));
+        }
+    }
+    true
+}
+
 /// The text of a replayed reasoning item, under either spelling.
 ///
 /// `summary` is what neuron emits (`summary_text` parts) and `content`
@@ -1173,28 +1249,49 @@ fn finish_to_status(reason: FinishReason) -> &'static str {
 /// Collect a chat-completions response into a non-streaming
 /// [`ResponsesResponse`]. Used by the `/v1/responses` handler when
 /// the request doesn't set `stream: true`.
+///
+/// `reasoning` is the model's think block, when it produced one. It
+/// becomes its own output item ahead of the message — the same shape
+/// and ordering the streaming projector emits. Omitting it (as this did
+/// until #300) meant a non-streaming caller was billed for
+/// `reasoning_tokens` it could never see, and could not replay the
+/// model's own thinking on the next turn.
 pub fn build_response(
     meta: &ResponseMeta,
     full_text: String,
     reason: FinishReason,
     usage: Option<ResponsesUsage>,
+    reasoning: Option<String>,
 ) -> ResponsesResponse {
     let status = finish_to_status(reason).to_string();
+    let mut output: Vec<ResponsesOutputItem> = Vec::with_capacity(2);
+    if let Some(text) = reasoning.filter(|t| !t.is_empty()) {
+        output.push(ResponsesOutputItem::Reasoning {
+            id: reasoning_item_id(meta),
+            summary: vec![cortex_core::responses::ResponsesSummaryPart::summary_text(
+                text,
+            )],
+            status: "completed".into(),
+        });
+    }
     ResponsesResponse {
         id: meta.response_id.clone(),
         object: "response".into(),
         created_at: meta.created_at,
         status: status.clone(),
         model: meta.model_id.clone(),
-        output: vec![ResponsesOutputItem::Message {
-            id: meta.message_item_id.clone(),
-            role: "assistant".into(),
-            content: vec![ResponsesOutputContent::OutputText {
-                text: full_text,
-                annotations: vec![],
-            }],
-            status,
-        }],
+        output: {
+            output.push(ResponsesOutputItem::Message {
+                id: meta.message_item_id.clone(),
+                role: "assistant".into(),
+                content: vec![ResponsesOutputContent::OutputText {
+                    text: full_text,
+                    annotations: vec![],
+                }],
+                status,
+            });
+            output
+        },
         // Same contract as the streaming shell: a truncated response
         // says so, or the caller cannot tell it from a fault.
         incomplete_details: matches!(reason, FinishReason::Length)
@@ -1208,6 +1305,173 @@ mod tests {
     use super::*;
     use cortex_core::openai::MessageContent;
 
+    /// The non-streaming Responses body must carry a reasoning item
+    /// (#300), matching the item the streaming projector emits.
+    ///
+    /// Observed before the fix: a response reporting
+    /// `output_tokens_details.reasoning_tokens: 4000` with an empty
+    /// `output` — 4,000 tokens generated, metered, and invisible.
+    #[test]
+    fn non_streaming_response_carries_a_reasoning_item() {
+        let r = build_response(
+            &meta(),
+            "the answer".into(),
+            FinishReason::Stop,
+            None,
+            Some("weighing the options".into()),
+        );
+        let v = serde_json::to_value(&r).unwrap();
+        let items = v["output"].as_array().expect("output array");
+        assert_eq!(items.len(), 2, "expected a reasoning item and a message");
+        assert_eq!(items[0]["type"], "reasoning", "reasoning comes first");
+        assert_eq!(items[0]["summary"][0]["type"], "summary_text");
+        assert_eq!(items[0]["summary"][0]["text"], "weighing the options");
+        assert_eq!(items[1]["type"], "message");
+        assert_eq!(items[1]["content"][0]["text"], "the answer");
+    }
+
+    /// A model that did not think produces no reasoning item — the
+    /// unchanged single-item shape.
+    #[test]
+    fn no_reasoning_means_no_reasoning_item() {
+        let r = build_response(&meta(), "plain".into(), FinishReason::Stop, None, None);
+        let v = serde_json::to_value(&r).unwrap();
+        let items = v["output"].as_array().unwrap();
+        assert_eq!(items.len(), 1);
+        assert_eq!(items[0]["type"], "message");
+    }
+
+    /// A file read back must survive the *whole* chain: Responses
+    /// `function_call_output` -> chat message -> rendered prompt.
+    ///
+    /// The reported symptom is a model that writes a file correctly,
+    /// reads it back, and consistently sees errors that are not there.
+    /// This walks the real path rather than a hand-built chat message,
+    /// because the translation is the part in between.
+    #[test]
+    fn a_file_read_back_survives_the_whole_translation() {
+        let file = "<!DOCTYPE html>\n<html lang=\"en\">\n\n  <script>\n    if (a && b || c < d) { s = '#b8905a'; }\n  </script>\n</html>\n";
+        let req: ResponsesRequest = serde_json::from_value(serde_json::json!({
+            "model": "m",
+            "input": [
+                {"type": "message", "role": "user",
+                 "content": [{"type": "input_text", "text": "read it back"}]},
+                {"type": "function_call", "call_id": "c1", "name": "read",
+                 "arguments": "{\"path\":\"/tmp/index.html\"}"},
+                // pi sends the tool result as a plain string (its
+                // `convertToolResultOutput` only produces an array when
+                // images are involved).
+                {"type": "function_call_output", "call_id": "c1", "output": file},
+                {"type": "message", "role": "user",
+                 "content": [{"type": "input_text", "text": "any errors?"}]}
+            ]
+        }))
+        .unwrap();
+        let chat = request_to_chat(req).expect("translate");
+
+        // The tool message must carry the file verbatim.
+        let tool = chat
+            .messages
+            .iter()
+            .find(|m| m.role == "tool")
+            .expect("a tool message");
+        let got = match &tool.content {
+            MessageContent::Text(t) => t.clone(),
+            other => panic!("tool content is not text: {other:?}"),
+        };
+        assert_eq!(got, file, "translation altered the file content");
+
+        // And it must survive the render too.
+        let template = include_str!("../../tests/data/qwen3.8-27b.chat_template.jinja");
+        let out = crate::harness::chat_template::render_chat_template(
+            template,
+            &chat.messages,
+            &Value::Null,
+            &Value::Null,
+        )
+        .expect("render");
+        for probe in ["<!DOCTYPE html>", "if (a && b || c < d)", "s = '#b8905a';"] {
+            assert!(out.contains(probe), "rendered prompt lost {probe:?}");
+        }
+        for bad in ["&lt;", "&amp;", "&quot;"] {
+            assert!(!out.contains(bad), "content HTML-escaped: {bad:?}");
+        }
+    }
+
+    /// One agentic step must render as ONE assistant turn.
+    ///
+    /// Responses splits a turn into `reasoning` + `message` +
+    /// `function_call` items. Emitting a message per item produced
+    /// *consecutive assistant turns* — one that deliberates and then
+    /// ends without acting, one that acts with an empty think block.
+    /// Shown four of those in its own replayed context, the model
+    /// copied the pattern: it produced a think block and a preamble,
+    /// then ended the turn without the tool call it had just announced
+    /// (#296).
+    #[test]
+    fn one_agentic_step_becomes_one_assistant_turn() {
+        let req: ResponsesRequest = serde_json::from_value(serde_json::json!({
+            "model": "m",
+            "input": [
+                {"type": "message", "role": "user",
+                 "content": [{"type": "input_text", "text": "build it"}]},
+                {"type": "reasoning",
+                 "content": [{"type": "reasoning_text", "text": "planning the file"}]},
+                {"type": "message", "role": "assistant",
+                 "content": [{"type": "output_text", "text": "Writing index.html:"}]},
+                {"type": "function_call", "call_id": "c1",
+                 "name": "write", "arguments": "{\"path\":\"a.html\"}"},
+                {"type": "function_call_output", "call_id": "c1", "output": "wrote 10 bytes"},
+                {"type": "message", "role": "user",
+                 "content": [{"type": "input_text", "text": "now check it"}]}
+            ]
+        }))
+        .unwrap();
+        let chat = request_to_chat(req).expect("translate");
+        let roles: Vec<&str> = chat.messages.iter().map(|m| m.role.as_str()).collect();
+        assert_eq!(
+            roles,
+            vec!["user", "assistant", "tool", "user"],
+            "no two assistant turns may be adjacent: got {roles:?}"
+        );
+
+        let a = &chat.messages[1];
+        assert!(
+            a.extra.get("reasoning_content").is_some(),
+            "reasoning must ride the turn it describes"
+        );
+        assert!(
+            matches!(&a.content, MessageContent::Text(t) if t.contains("Writing index.html")),
+            "the assistant's own text must survive the fold"
+        );
+        let calls = a.extra["tool_calls"].as_array().expect("tool_calls");
+        assert_eq!(calls.len(), 1);
+        assert_eq!(calls[0]["function"]["name"], "write");
+    }
+
+    /// Parallel tool calls are several calls on one turn, not several
+    /// turns.
+    #[test]
+    fn consecutive_function_calls_accumulate_on_one_turn() {
+        let req: ResponsesRequest = serde_json::from_value(serde_json::json!({
+            "model": "m",
+            "input": [
+                {"type": "message", "role": "user",
+                 "content": [{"type": "input_text", "text": "go"}]},
+                {"type": "message", "role": "assistant",
+                 "content": [{"type": "output_text", "text": "doing both:"}]},
+                {"type": "function_call", "call_id": "c1", "name": "a", "arguments": "{}"},
+                {"type": "function_call", "call_id": "c2", "name": "b", "arguments": "{}"}
+            ]
+        }))
+        .unwrap();
+        let chat = request_to_chat(req).expect("translate");
+        let roles: Vec<&str> = chat.messages.iter().map(|m| m.role.as_str()).collect();
+        assert_eq!(roles, vec!["user", "assistant"]);
+        let calls = chat.messages[1].extra["tool_calls"].as_array().unwrap();
+        assert_eq!(calls.len(), 2, "both calls belong to the same turn");
+    }
+
     /// Wrap typed items as `input` elements. Most translator tests
     /// exercise the typed path; the bare easy-message and unknown-item
     /// paths have dedicated tests below.
@@ -1245,6 +1509,8 @@ mod tests {
             seed: None,
             repetition_penalty: None,
             repeat_last_n: None,
+            presence_penalty: None,
+            frequency_penalty: None,
             previous_response_id: None,
             extra: Value::Object(Default::default()),
         };
@@ -1271,6 +1537,8 @@ mod tests {
             seed: None,
             repetition_penalty: None,
             repeat_last_n: None,
+            presence_penalty: None,
+            frequency_penalty: None,
             previous_response_id: None,
             extra: Value::Object(Default::default()),
         };
@@ -1298,6 +1566,8 @@ mod tests {
             seed: None,
             repetition_penalty: None,
             repeat_last_n: None,
+            presence_penalty: None,
+            frequency_penalty: None,
             previous_response_id: Some("resp_prev".into()),
             extra: Value::Object(Default::default()),
         };
@@ -1334,6 +1604,8 @@ mod tests {
             seed: None,
             repetition_penalty: None,
             repeat_last_n: None,
+            presence_penalty: None,
+            frequency_penalty: None,
             previous_response_id: None,
             extra: Value::Object(Default::default()),
         };
@@ -1368,6 +1640,8 @@ mod tests {
             seed: None,
             repetition_penalty: None,
             repeat_last_n: None,
+            presence_penalty: None,
+            frequency_penalty: None,
             previous_response_id: None,
             extra: Value::Object(Default::default()),
         };
@@ -1417,6 +1691,8 @@ mod tests {
             seed: None,
             repetition_penalty: None,
             repeat_last_n: None,
+            presence_penalty: None,
+            frequency_penalty: None,
             previous_response_id: None,
             extra: Value::Object(Default::default()),
         };
@@ -1458,6 +1734,8 @@ mod tests {
             seed: None,
             repetition_penalty: None,
             repeat_last_n: None,
+            presence_penalty: None,
+            frequency_penalty: None,
             previous_response_id: None,
             extra: Value::Object(Default::default()),
         };
@@ -1494,6 +1772,8 @@ mod tests {
             seed: None,
             repetition_penalty: None,
             repeat_last_n: None,
+            presence_penalty: None,
+            frequency_penalty: None,
             previous_response_id: None,
             extra: Value::Object(Default::default()),
         };
@@ -2504,6 +2784,7 @@ mod tests {
                 output_tokens_details: None,
                 input_tokens_details: None,
             }),
+            None,
         );
         assert_eq!(r.status, "completed");
         match &r.output[0] {
@@ -2529,7 +2810,7 @@ mod tests {
 
     #[test]
     fn build_response_length_yields_incomplete_status() {
-        let r = build_response(&meta(), "trunc".into(), FinishReason::Length, None);
+        let r = build_response(&meta(), "trunc".into(), FinishReason::Length, None, None);
         assert_eq!(r.status, "incomplete");
     }
 }

crates/neuron/tests/activation.rs

@@ -30,6 +30,7 @@ async fn test_load_default_models_skips_unknown_harness() {
             tensor_parallel: None,
             devices: None,
             sampling: None,
+            preserve_thinking: None,
         },
         ModelSpec {
             model_id: "model-b".into(),
@@ -38,6 +39,7 @@ async fn test_load_default_models_skips_unknown_harness() {
             tensor_parallel: None,
             devices: None,
             sampling: None,
+            preserve_thinking: None,
         },
     ];
 
@@ -98,6 +100,7 @@ async fn test_load_default_models_skipped_on_driver_mismatch() {
         tensor_parallel: Some(2),
         devices: None,
         sampling: None,
+        preserve_thinking: None,
     }];
     let activation = ActivationTracker::new(&specs);
     let reason = "host NVIDIA driver/library mismatch (userspace NVML 580.159 vs loaded \
@@ -200,6 +203,7 @@ fn qwen_spec() -> ModelSpec {
         tensor_parallel: None,
         devices: None,
         sampling: None,
+        preserve_thinking: None,
     }
 }
 

crates/neuron/tests/candle_lifecycle.rs

@@ -54,6 +54,7 @@ async fn test_candle_qwen3_load_unload_lifecycle() {
         tensor_parallel: None,
         devices: Some(vec![0]),
         sampling: None,
+        preserve_thinking: None,
     };
 
     registry

crates/neuron/tests/preflight.rs

@@ -107,6 +107,7 @@ fn spec(model_id: &str, tp: Option<u32>, quant: Option<&str>) -> ModelSpec {
         tensor_parallel: tp,
         devices: None,
         sampling: None,
+        preserve_thinking: None,
     }
 }
 

helexa.ai/content/docs/2-operating/03-neurons.md

@@ -58,6 +58,32 @@ Two things that bite:
 - **Cold loads are disk-bound.** A 50 GB model off spinning rust is a
   very different experience from the same model on NVMe.
 
+### Where weights come from
+
+By default a bare `org/name` id is fetched from HuggingFace. Operators
+running a mirror, or serving models from more than one place, can
+declare sources explicitly:
+
+```toml
+[harness.candle]
+default_source = "huggingface"
+
+[harness.candle.sources.huggingface]
+endpoint = "https://huggingface.co"
+auth_env = "HF_TOKEN"          # env var name, never the token itself
+cache_dir = "/archive/llm"     # optional; per-source cache
+```
+
+`auth_env` names an environment variable rather than holding a token, so
+credentials stay out of the config file — worth keeping to even when the
+file is not obviously shared, because config is exactly what gets copied
+between hosts.
+
+Give a source its own `cache_dir` when two sources can serve the same
+`org/name`. The cache tree is keyed on the model id alone, so sharing a
+directory between sources that disagree about what `org/name` means will
+collide on disk.
+
 ## Default models
 
 Models listed as defaults load when the daemon activates, so the host
@@ -89,12 +115,19 @@ Each loaded model has bounded admission control:
 
 ```toml
 [harness.candle.admission]
-max_in_flight = 8        # concurrent requests actually running
+max_in_flight = 2        # concurrent requests actually running
 max_queue_depth = 8      # waiting before rejection
-max_wait_secs = 30       # how long a queued request waits
+max_wait_secs = 30       # how long a queued request waits for a slot
+kv_max_wait_secs = 300   # how long it waits for enough KV budget
 max_per_principal = 2    # fair share, so one caller cannot monopolise
 ```
 
+`kv_max_wait_secs` is deliberately much longer than `max_wait_secs`.
+The first waits for a slot to turn over; the second waits for a whole
+long-context sequence to finish, which on a 27B can be minutes. A
+request that needs KV budget nobody has yet is not a request to reject
+in thirty seconds.
+
 Over-capacity requests are rejected promptly with `429`/`503` and a
 `Retry-After` rather than being allowed to queue invisibly. A fast
 rejection a client can retry is far better than a request that hangs
@@ -102,8 +135,19 @@ until something times out.
 
 `max_in_flight` above 1 enables batched decode on architectures that
 support it, so concurrent requests share a decode step instead of
-serialising. Raise it on big cards, leave it at 1 on small ones — the
-KV cache for several concurrent long contexts is itself substantial.
+serialising.
+
+Higher is not better, because **the model's KV budget is divided by it
+at load**. A host advertising a 131k context with eight slots needs
+eight times the KV of one slot, which it does not have — the practical
+result is that each slot gets a fraction of the advertised window and a
+single long session runs out of room while seven idle slots hold
+reservations. Size it against the concurrency you actually observe: on
+one production host, seven days of request spans showed 89.6% of busy
+time was a single request in flight and 93.6% was two or fewer, so two
+slots cover the real load and give each a usable context. Anything above
+the limit queues on admission with a `Retry-After` rather than being
+refused.
 
 `max_per_principal` is what stops one busy client consuming the whole
 model.
@@ -150,7 +194,8 @@ reservation is doing its job, not that the model is overloaded.
 neuron refuses work it cannot finish rather than crashing partway:
 
 - a **static floor** of free VRAM below which prefill is refused
-  (`NEURON_MIN_FREE_VRAM_MB`, default 1500)
+  (`NEURON_MIN_FREE_VRAM_MB`, or `min_free_floor_mb` under
+  `[harness.candle.context_limit]`; default 1500)
 - a **length-aware check** that estimates the KV cache the request will
   actually need and refuses if it will not fit
 
@@ -159,15 +204,206 @@ can pass the static floor and still be impossible, because KV grows with
 prompt *and* generation length. See [context
 limits](/docs/operating/context-limits) for how the numbers relate.
 
+`activation_headroom_mb` reserves VRAM per card for prefill activations
+on top of the weights and the KV cache, before the context ceiling is
+computed. Raise it if prefill OOMs on a host whose steady-state
+occupancy looks fine — activations are transient and do not show up in
+the resident figure.
+
 A host that cannot currently serve a model reports that on `/health`,
 and cortex stops routing to it — an unservable replica is not a routing
 target.
 
+## Image models
+
+```toml
+[harness.candle.image]
+te_device = "cpu"     # where the text encoder runs
+te_resident = true    # keep it loaded between generations
+max_dim = 2048        # resolution ceiling per side
+```
+
+The text encoder is the placement decision. On CPU the ~8 GB encoder
+never touches the GPU — prompt encoding is one short forward and the
+features are tiny — which is what lets a 24 GB card serve a BF16 DiT at
+all. Move it to `cuda` only on a card with room to spare.
+
+`te_resident` trades memory for latency: resident costs ~16 GB of system
+RAM on CPU (or ~8 GB of VRAM on CUDA) and saves a few seconds per
+request. CPU defaults to resident, CUDA to rebuilding per request, which
+are the right defaults for where each is scarce.
+
+`max_dim` rejects oversized requests **before admission**, so an
+impossible resolution costs nothing. That matters more than it sounds:
+an OOM on the device poisons the worker context, so refusing early is
+the difference between one rejected request and a model that needs
+restarting.
+
 ## Prompt caps
 
 `NEURON_MAX_PROMPT_TOKENS` sets a hard ceiling; larger prompts are
 rejected before any GPU work, so a runaway client costs nothing.
 
+## Sampling
+
+Every model publishes its own sampling defaults in
+`generation_config.json`, and a caller can override them per request.
+Between those two sits an operator override, for the cases where the
+model's published defaults are wrong for how you serve it:
+
+```toml
+[[default_models]]
+model_id = "Qwen/Qwen3.8-27B"
+
+[default_models.sampling]
+temperature = 0.6
+presence_penalty = 1.0
+```
+
+Precedence is **request > operator > model > built-in**. Anything you
+leave unset changes nothing, so an empty table behaves exactly as no
+table at all.
+
+Set the same values in the gateway's `models.toml` under
+`[models.sampling]`. A model that is cold-loaded on demand gets its
+sampling from the catalogue, not from this file, so the two must agree
+or the same model samples differently depending on how it arrived —
+`script/check-config-consistency.py` fails the deploy if they diverge.
+
+### Which knob for which symptom
+
+| symptom | knob |
+|---|---|
+| Structurally malformed output — truncated JSON, unbalanced braces | `temperature` |
+| The think block restates the same passage over and over | `presence_penalty` |
+| Common tokens over-represented across a long answer | `frequency_penalty` |
+| Short immediate loops — "no, no, no…" | `repeat_penalty` |
+| You need two runs to be comparable | `seed` |
+
+**`temperature`** is the blunt one. Qwen3.8-27B publishes `1.0`, which
+measured a 20% structural-defect rate on long agentic turns against 0/60
+at `0.6`; the fleet therefore overrides it. Note that `0.6` is also what
+Qwen recommend for their thinking variants, so this is not fighting the
+model.
+
+**`presence_penalty`** and **`frequency_penalty`** are scored over the
+*whole* generated sequence. They are the ones that matter for reasoning
+models, and Qwen name `presence_penalty` (0–2, 1.5 when severe) as the
+remedy for the endless repetition their models fall into during long
+thinking. Both default to `0.0`, which is off.
+
+**`repeat_penalty`** (default `1.1`) and **`repeat_last_n`** (default
+`64`) are the older, *windowed* form: the penalty only sees the last
+`repeat_last_n` tokens. That is fine for chat and close to useless for a
+think block — a 20,000-token deliberation restating a passage 5,000
+tokens later is entirely outside a 64-token window. Measured across
+recorded sessions, **86% of think blocks over 60k characters contained
+verbatim repeated sentences**, one of them repeating 76% of its own
+content.
+
+Widening the window is the obvious move and the wrong one: the penalty
+is token-level, and source code legitimately repeats tokens constantly
+(`const`, `function`, brace runs), so a wide window at a meaningful
+penalty degrades exactly the output you care about. Reach for
+`presence_penalty` first — what recurs is passages, not tokens.
+
+**`seed`** pins the sampler's RNG for every request to this model. It
+exists because parameter search needs a control and the clients that
+would otherwise pin it often cannot. Leaving it set is a product
+decision, not just a testing one: every identical prompt then returns an
+identical answer to every caller, which is predictable and cacheable but
+removes the variation a user may expect from retrying.
+
+## Reasoning
+
+```toml
+[default_models.sampling]
+# nothing here — reasoning settings live in their own places, below
+```
+
+**`preserve_thinking`** controls whether prior turns keep their think
+blocks when the conversation is re-rendered. It is the model's own chat
+template control, not ours:
+
+```toml
+[[default_models]]
+preserve_thinking = false   # keep reasoning only for the turn in progress
+```
+
+Unset is the default and means the template decides. On Qwen3.8-27B that
+is `true` — every prior turn's reasoning is replayed. Full replay is
+what makes prompt growth track *total* completion rather than visible
+output: one measured session produced 28,672 reasoning tokens on turn 1
+and carried all of them into turn 2's 31,269-token prompt, at ~22 s of
+prefill on every turn thereafter.
+
+Setting it to `false` is a legitimate experiment, not a recommendation.
+Whether narrowing replay *helps* the model is unmeasured. Set it in both
+`models.toml` and this file, like sampling, and the value is reported on
+`/v1/models` and logged at load so a session can be attributed to the
+arm it ran under.
+
+**`reasoning_budget.answer_reserve_tokens`** (default `4096`) holds back
+part of the caller's output budget for the answer:
+
+```toml
+[harness.candle.reasoning_budget]
+answer_reserve_tokens = 4096
+```
+
+A reasoning model with no stopping criterion fills whatever budget it is
+given. Measured on Qwen3.8-27B: given 16,384 tokens it used 16,384;
+given 32,768 it used 32,768 — and both times returned no answer at all.
+The reserve bounds the think block at `max_tokens - answer_reserve` so
+the rest survives for the reply.
+
+Deliberately *not* derived from reasoning effort. Effort is a prompt
+instruction the model is trained to follow; making it also set a
+guillotine is how a model came to be told to reason at `xhigh` and then
+cut off at `medium`'s budget. Set `0` to disable, restoring unbounded
+reasoning.
+
+The trade to know: when the reserve fires, the model is interrupted
+mid-deliberation and its answer phase tends to state an *intention*
+rather than a conclusion. That is better than returning nothing, but it
+is not free.
+
+## The prefix cache
+
+Snapshots of previously-served prefixes, so a follow-up turn re-uses the
+KV cache instead of re-prefilling the whole conversation:
+
+```toml
+[harness.candle.prefix_cache]
+enabled = true
+budget_mb = 2048    # snapshot bytes per loaded model
+max_entries = 8     # live snapshots per model, regardless of budget
+```
+
+**Reach for `budget_mb` when `cached_tokens` stops growing mid-session.**
+That is the symptom: the prompt keeps growing, the cached prefix does
+not, and every turn re-prefills the difference. Observed on a 1 GiB
+default — `cached_tokens` pinned at 2,016 for ten consecutive turns
+while the prompt reached 52,869, so ~50k tokens were re-prefilled every turn
+at roughly 37 s each, against ~20 s of actual decode.
+
+The number is not free, and the direction of the trade is the opposite
+of what it looks like. **The snapshot budget is reserved before the KV
+budget, not from what is left over.** On a 2×32 GB host serving a 27B at
+`max_in_flight = 2`, measured 1:1:
+
+| `budget_mb` | KV budget | tokens per slot | holds a 53k snapshot |
+|---|---|---|---|
+| 1024 | 4713 MB | 75,400 | no |
+| 2048 | 3689 MB | 59,000 | yes |
+| 3072 | 2665 MB | 42,600 | yes, but a session cannot reach 53k |
+
+So raising it too far starves the KV cache and shortens how long a
+single session can run — the model can no longer reach the context whose
+snapshot the larger cache was meant to hold. Size it against the deepest
+session you actually serve, and check `kv_budget_mb` in the journal
+after a reload.
+
 ## Checking on it
 
 ```sh

helexa.ai/content/docs/2-operating/05-context-limits.md

@@ -32,6 +32,43 @@ daemon now does automatically.
 | `limit.input` | same | compaction trigger — opencode compacts to keep the prompt at/under this |
 | `limit.output` | same | generation reserve opencode leaves below the wall |
 
+### Tuning the derived limit
+
+Since #67 the neuron computes `limit` itself (see below), and three
+`[harness.candle.context_limit]` settings shape that computation:
+
+```toml
+[harness.candle.context_limit]
+target_prefill_latency_secs = 120     # longest acceptable prefill per turn
+bootstrap_prefill_tok_per_sec = 800   # cold-start estimate until measured
+output_reserve_tokens = 8192          # what a request gets when it names no cap
+max_output_tokens = 32768             # the largest output a request may name
+```
+
+**`target_prefill_latency_secs`** is the one that usually binds. The
+throughput ceiling is `target_prefill_latency_secs × measured
+prefill tok/s`, so on a host measuring ~1,400 tok/s a 120-second target
+advertises about 168k — above most hard ceilings, in which case it stops
+mattering. On a slower host, or one whose recent traffic has been small
+prompts, it is what caps the window.
+
+**`bootstrap_prefill_tok_per_sec`** applies until the model has measured
+its own rate, and stops the advertised window collapsing on a
+freshly-restarted host. Prefill samples below 512 tokens are ignored
+when computing that rate: a short prompt is dominated by fixed
+per-request cost and says nothing about throughput — the deploy's own
+19-token smoke probe reads as ~145 tok/s on a host that measures ~1,400.
+So a host that has served only small requests keeps the bootstrap
+estimate rather than adopting a misleading one, and no warm-up is
+needed after a restart.
+
+**`output_reserve_tokens`** is what a request that names no cap
+receives. **`max_output_tokens`** is the ceiling advertised to clients
+as `max_output_tokens`, and it is the *ceiling*, not the reserve —
+publishing the reserve tells every client that reads the advertisement
+to cap itself there, which on a reasoning model can be less than a
+single think block.
+
 ## How they must relate
 
 For a single model on a single neuron, all of these must hold:

script/check-config-consistency.py

@@ -15,6 +15,9 @@ Checked, per model id present in both files:
 
   * ``quant``      — the #252 field
   * ``sampling``   — the #283 field, compared field-by-field
+  * ``preserve_thinking`` — the H7 A/B gate: a model that replays
+    prior reasoning when resident and not when cold-loaded would
+    make the A/B unreadable, which is #252's shape again
 
 A model in only one file is not an error: plenty of catalogue entries are
 never resident anywhere, and a host may hold a model the catalogue does
@@ -46,7 +49,7 @@ REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
 # Fields that must agree wherever both files mention the same model.
 # Extend this list whenever a serving-critical field lands in both.
-SCALAR_FIELDS = ["quant"]
+SCALAR_FIELDS = ["quant", "preserve_thinking"]
 TABLE_FIELDS = ["sampling"]
 
 

script/cuda-check.sh

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+# Type-check neuron's `cuda` feature locally, in CI's own image.
+#
+# Why this exists: `#[cfg(feature = "cuda")]` code is invisible to a
+# default-feature build, so `cargo build`, `cargo clippy --all-targets`
+# and the whole test suite can pass on code that does not compile for
+# the target the fleet actually serves on. The only gate that catches it
+# is CI's `cuda-check` job — a ~20 minute round trip, and the failure
+# arrives after the push rather than before it.
+#
+# This runs the same command, with the same env, in the same image.
+#
+# Do NOT try to install the CUDA toolkit on a Fedora 44 host instead:
+# CUDA 13.0's nvcc rejects host compilers newer than gcc 15 and F44
+# ships gcc 16 with no compat package. The image is Fedora 43 for
+# exactly that reason, and additionally patches a glibc 2.41+ /
+# CUDA-header `noexcept` mismatch that nvcc hard-errors on. See
+# gongfoo's images/runner-cuda-13.0/Containerfile.
+#
+# No GPU is required — this is a borrow-check, not a run.
+#
+# First run pulls the image (~GBs) and compiles the candle/flash-attn
+# kernels, so it costs about what CI costs. Afterwards the cargo target
+# volume makes an incremental check a fraction of that, which is the
+# whole point.
+set -euo pipefail
+
+IMAGE="${CUDA_CHECK_IMAGE:-git.lair.cafe/gongfoo/runner-cuda-13.0:latest}"
+ROOT="$(git rev-parse --show-toplevel)"
+
+# A dedicated target volume. Sharing the host's `target/` would make the
+# default-feature and cuda-feature builds evict each other's artifacts on
+# every alternation, which would cost more than it saves.
+TARGET_VOL="${CUDA_CHECK_TARGET_VOL:-helexa-cuda-check-target}"
+# Mounted at the registry rather than over CARGO_HOME: the toolchain
+# binaries live in /usr/local/cargo/bin and must stay visible.
+REGISTRY_VOL="${CUDA_CHECK_REGISTRY_VOL:-helexa-cuda-check-registry}"
+
+if [ "${1:-}" = "--pull" ]; then
+    shift
+    podman pull "${IMAGE}"
+fi
+
+# Default to the exact command ci.yml's cuda-check job runs. Extra
+# arguments replace it, for narrowing to one crate while iterating.
+if [ "$#" -gt 0 ]; then
+    CARGO_CMD=("$@")
+else
+    CARGO_CMD=(cargo check -p neuron --features cuda,flash-attn --all-targets)
+fi
+
+# Allocate a TTY only when there is one to allocate. `-it` against a
+# pipe or a CI log wedges the container before cargo ever starts, which
+# looks exactly like a very slow compile.
+TTY_ARGS=()
+if [ -t 0 ] && [ -t 1 ]; then
+    TTY_ARGS=(-it)
+fi
+
+exec podman run --rm "${TTY_ARGS[@]}" \
+    -v "${ROOT}:/work:Z" \
+    -v "${TARGET_VOL}:/target" \
+    -v "${REGISTRY_VOL}:/usr/local/cargo/registry" \
+    -w /work \
+    -e CARGO_TARGET_DIR=/target \
+    -e CARGO_TERM_COLOR=always \
+    `# candle-kernels' build script falls back to nvidia-smi for` \
+    `# compute-cap detection when this is unset, and there is no GPU` \
+    `# here. Any valid cap borrow-checks; the real per-flavour caps` \
+    `# live in build-prerelease.yml's matrix.` \
+    -e CUDA_COMPUTE_CAP=86 \
+    `# The image ships sccache, but the local run has no shared cache` \
+    `# to talk to; an unset wrapper avoids a hard cargo failure.` \
+    -e RUSTC_WRAPPER= \
+    "${IMAGE}" \
+    bash -lc '
+        set -euo pipefail
+        # CUDA_HOME/PATH/LD_LIBRARY_PATH come from the image. LIBRARY_PATH
+        # does not, and cudarc needs it at link time — ci.yml exports all
+        # three explicitly, so keep this in step with that job.
+        export LIBRARY_PATH="${CUDA_HOME}/targets/x86_64-linux/lib:${CUDA_HOME}/lib64:${LIBRARY_PATH:-}"
+        exec "$@"
+    ' _ "${CARGO_CMD[@]}"