Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
33a62d6aa5
|
||
|
|
d4f7491008
|
||
|
|
cc5b99f91a
|
||
|
|
687753767f
|
||
|
|
2e48533b4d
|
||
|
|
b142bea782
|
||
|
|
943570895d
|
||
|
|
f48035f2ce
|
||
|
|
eecb039875
|
||
|
|
5502cb2893
|
||
|
|
2f358264c6
|
||
|
|
f5c9e17ab6
|
||
|
|
8dd0bb5332
|
||
|
|
b8098c4096
|
||
|
|
7a75703d23
|
||
|
|
0705768f24
|
@@ -0,0 +1,58 @@
|
||||
# Internal-only vhost for the investor portal (helexa-angels).
|
||||
#
|
||||
# Useful where the public name does not resolve or connect correctly from
|
||||
# inside the network that hosts it — common without NAT reflection or
|
||||
# split-horizon DNS. Optional: skip this file if your public name works
|
||||
# from everywhere.
|
||||
#
|
||||
# Cert: internal CA, renewed by step@angels.timer.
|
||||
#
|
||||
# Enable with a relative symlink into sites-enabled:
|
||||
# ln -sf ../sites-available/angels.internal.conf /etc/nginx/sites-enabled/
|
||||
|
||||
upstream angels_up {
|
||||
server gallumbits.kosherinata.internal:8092;
|
||||
keepalive 16;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 127.0.0.1:14443 ssl proxy_protocol;
|
||||
http2 on;
|
||||
server_name angels.internal;
|
||||
|
||||
ssl_certificate /etc/nginx/tls/cert/angels.internal.pem;
|
||||
ssl_certificate_key /etc/nginx/tls/key/angels.internal.pem;
|
||||
ssl_protocols TLSv1.3;
|
||||
ssl_trusted_certificate /etc/pki/ca-trust/source/anchors/root-internal.pem;
|
||||
|
||||
# The material is confidential whichever name reaches it, so the same
|
||||
# headers as the public vhost. `always` so they survive error responses.
|
||||
add_header X-Robots-Tag "noindex, nofollow, noarchive, nosnippet" always;
|
||||
add_header Referrer-Policy "no-referrer" always;
|
||||
add_header X-Content-Type-Options "nosniff" always;
|
||||
add_header X-Frame-Options "DENY" always;
|
||||
add_header Content-Security-Policy
|
||||
"default-src 'none'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; form-action 'self'; base-uri 'none'; frame-ancestors 'none'" always;
|
||||
|
||||
# Same per-IP limits as the public path: an invite code is equally
|
||||
# guessable from here, and argon2 costs the same CPU whoever spends it.
|
||||
location /i/ {
|
||||
limit_req zone=angels_invite burst=5 nodelay;
|
||||
proxy_pass http://angels_up;
|
||||
include /etc/nginx/conf.d/angels-proxy-params.conf;
|
||||
}
|
||||
location /signin {
|
||||
limit_req zone=angels_auth burst=5 nodelay;
|
||||
proxy_pass http://angels_up;
|
||||
include /etc/nginx/conf.d/angels-proxy-params.conf;
|
||||
}
|
||||
location /register {
|
||||
limit_req zone=angels_auth burst=5 nodelay;
|
||||
proxy_pass http://angels_up;
|
||||
include /etc/nginx/conf.d/angels-proxy-params.conf;
|
||||
}
|
||||
location / {
|
||||
proxy_pass http://angels_up;
|
||||
include /etc/nginx/conf.d/angels-proxy-params.conf;
|
||||
}
|
||||
}
|
||||
@@ -259,6 +259,21 @@ pub struct ReasoningBudgetRung {
|
||||
/// Qwen3.8 that silently means `xhigh`.
|
||||
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||
pub default: bool,
|
||||
/// Why this rung is not being offered right now, or `None` when it
|
||||
/// is. Absent from the wire in the normal case.
|
||||
///
|
||||
/// Availability is a *capacity* statement, not a permission: the
|
||||
/// request path does not reject an unavailable rung. A caller that
|
||||
/// insists still gets it, which keeps this from becoming a new
|
||||
/// refusal path for every existing API client. It exists so a UI can
|
||||
/// avoid offering an option that will serve the user badly, and so
|
||||
/// an integrator reading `/v1/models` can see the same thing.
|
||||
///
|
||||
/// Today the only reason is concurrency: on a node running many
|
||||
/// slots, the longest rung is reported unavailable. See
|
||||
/// `reasoning_budget::availability` for the argument and the knob.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub unavailable_reason: Option<String>,
|
||||
}
|
||||
|
||||
/// Operator-set pricing, **USD per 1,000,000 tokens, as JSON numbers**
|
||||
|
||||
@@ -67,6 +67,18 @@ pub struct ModelEntry {
|
||||
/// older persisted/serialised entries deserialisable.
|
||||
#[serde(default)]
|
||||
pub capabilities: Vec<String>,
|
||||
/// The reasoning ladder this *model* offers, with any rung the node
|
||||
/// is currently withholding marked unavailable.
|
||||
///
|
||||
/// Per model, not per node. It was held on `NodeState` on the
|
||||
/// reasoning that "every model on a neuron reports the same rungs",
|
||||
/// which was true while a rung was only a name from the model's
|
||||
/// template. It stopped being true once availability began depending
|
||||
/// on the model's own `max_in_flight`: two models on one host can
|
||||
/// have different admission settings, and the node-level copy would
|
||||
/// attribute one model's withheld rung to the other.
|
||||
#[serde(default)]
|
||||
pub reasoning_budget: Vec<crate::harness::ReasoningBudgetRung>,
|
||||
/// Runtime-detected capability flags from the neuron's `/models`
|
||||
/// response (`ModelInfo`). `false` when the neuron predates these
|
||||
/// fields or hasn't reported them yet.
|
||||
|
||||
@@ -847,8 +847,16 @@ async fn list_models(State(fleet): State<Arc<CortexState>>) -> Json<Value> {
|
||||
// First non-empty wins: a catalogue-seeded entry
|
||||
// starts with none, and neurons serving the same
|
||||
// model are configured alike.
|
||||
// The model's own ladder wins; the node-level copy is
|
||||
// only a fallback for a neuron too old to report one
|
||||
// per model. Preferring the node copy would discard
|
||||
// the per-model availability computed at the source.
|
||||
if e.reasoning_budget.is_empty() {
|
||||
e.reasoning_budget = node.reasoning_budget.clone();
|
||||
e.reasoning_budget = if entry.reasoning_budget.is_empty() {
|
||||
node.reasoning_budget.clone()
|
||||
} else {
|
||||
entry.reasoning_budget.clone()
|
||||
};
|
||||
}
|
||||
})
|
||||
.or_insert_with(|| CortexModelEntry {
|
||||
@@ -872,7 +880,11 @@ async fn list_models(State(fleet): State<Arc<CortexState>>) -> Json<Value> {
|
||||
context_window: None,
|
||||
context_length: None,
|
||||
// What this neuron says each effort level buys (#223).
|
||||
reasoning_budget: node.reasoning_budget.clone(),
|
||||
reasoning_budget: if entry.reasoning_budget.is_empty() {
|
||||
node.reasoning_budget.clone()
|
||||
} else {
|
||||
entry.reasoning_budget.clone()
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -130,12 +130,13 @@ async fn poll_neuron(fleet: &CortexState, name: &str, endpoint: &str) {
|
||||
Ok(resp) if resp.status().is_success() => {
|
||||
match resp.json::<Vec<ModelInfo>>().await {
|
||||
Ok(models) => {
|
||||
// The reasoning-effort ladder is host configuration:
|
||||
// every model on a neuron reports the same rungs, so
|
||||
// it is held once per node rather than copied onto
|
||||
// each model entry where the two could drift (#223).
|
||||
// Taken from the first model that offers one — a
|
||||
// non-reasoning model advertises none.
|
||||
// Node-level ladder, kept only as a fallback for a
|
||||
// model whose own entry carries none. It is NOT the
|
||||
// source of truth any more: availability depends on
|
||||
// the model's `max_in_flight`, so two models on one
|
||||
// host can legitimately offer different rungs and a
|
||||
// single node-level copy would attribute one's
|
||||
// withheld rung to the other. Per-model is set below.
|
||||
node.reasoning_budget = models
|
||||
.iter()
|
||||
.map(|m| m.reasoning_budget.clone())
|
||||
@@ -158,6 +159,7 @@ async fn poll_neuron(fleet: &CortexState, name: &str, endpoint: &str) {
|
||||
// authoritative source the gateway advertises.
|
||||
e.limit = upstream.limit.clone();
|
||||
e.servable = upstream.servable.clone();
|
||||
e.reasoning_budget = upstream.reasoning_budget.clone();
|
||||
})
|
||||
.or_insert_with(|| ModelEntry {
|
||||
id: upstream.id.clone(),
|
||||
@@ -169,6 +171,7 @@ async fn poll_neuron(fleet: &CortexState, name: &str, endpoint: &str) {
|
||||
reasoning: upstream.reasoning,
|
||||
limit: upstream.limit.clone(),
|
||||
servable: upstream.servable.clone(),
|
||||
reasoning_budget: upstream.reasoning_budget.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -482,6 +482,11 @@ async fn cold_load(
|
||||
// would make the cold-load path immediately
|
||||
// un-route what it had just placed.
|
||||
servable: None,
|
||||
// Same reasoning: the ladder and its availability are
|
||||
// the neuron's to report, and the next poll (~10s)
|
||||
// fills it. Guessing here risks advertising a rung
|
||||
// this host is withholding.
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -13,27 +13,56 @@ use cortex_gateway::state::CortexState;
|
||||
use serde_json::json;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU32, Ordering};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
/// Write a `models.toml` with one alias to a unique temp path. Returns
|
||||
/// the path; the file persists for the test process and gets reaped by
|
||||
/// the OS at exit. Using $XDG_RUNTIME_DIR fallback for the temp dir
|
||||
/// keeps the file off shared /tmp on CI without pulling in tempfile.
|
||||
/// Write a `models.toml` with one alias into a directory of its own
|
||||
/// under the system temp dir, and prove the bytes landed before
|
||||
/// returning. The file is reaped by the OS at exit.
|
||||
///
|
||||
/// Each call gets its own directory rather than a distinct filename in
|
||||
/// a shared one. Two things make that worth the extra syscall:
|
||||
///
|
||||
/// - The tests in this file run concurrently in one binary and write
|
||||
/// *different* alias tables. A shared name is one collision away from
|
||||
/// a test loading another's catalogue.
|
||||
/// - The failure is silent. `ModelCatalogue::load` maps a missing,
|
||||
/// unreadable, unparseable **or empty** file onto an empty catalogue,
|
||||
/// and `resolve_alias` then returns the alias unchanged — so the
|
||||
/// mismatch surfaces at whatever assertion happens to touch it, tens
|
||||
/// of lines from the cause. Observed on CI 2026-08-30 as
|
||||
/// `left: "helexa/small", right: "test-model"` while the same commit
|
||||
/// passed on its branch and 60/60 locally.
|
||||
///
|
||||
/// The read-back is for the same reason: an empty catalogue is
|
||||
/// indistinguishable from a catalogue that never loaded, so a test may
|
||||
/// not simply assume its fixture is on disk.
|
||||
fn write_models_toml(alias: &str, target: &str) -> PathBuf {
|
||||
static SEQ: AtomicU32 = AtomicU32::new(0);
|
||||
let contents = format!(
|
||||
r#"
|
||||
[aliases]
|
||||
"{alias}" = "{target}"
|
||||
"#
|
||||
);
|
||||
let mut path = std::env::temp_dir();
|
||||
let mut dir = std::env::temp_dir();
|
||||
let pid = std::process::id();
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos();
|
||||
path.push(format!("cortex-test-models-{pid}-{now}.toml"));
|
||||
std::fs::write(&path, contents).expect("write temp models.toml");
|
||||
let seq = SEQ.fetch_add(1, Ordering::Relaxed);
|
||||
dir.push(format!("cortex-test-models-{pid}-{now}-{seq}"));
|
||||
std::fs::create_dir_all(&dir).expect("create temp catalogue dir");
|
||||
let path = dir.join("models.toml");
|
||||
std::fs::write(&path, &contents).expect("write temp models.toml");
|
||||
let seen = std::fs::read_to_string(&path).expect("read back temp models.toml");
|
||||
assert_eq!(
|
||||
seen,
|
||||
contents,
|
||||
"temp catalogue did not round-trip at {}",
|
||||
path.display()
|
||||
);
|
||||
path
|
||||
}
|
||||
|
||||
@@ -81,6 +110,7 @@ async fn test_alias_resolves_in_chat_completions() {
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -168,6 +198,7 @@ async fn test_aliases_surface_in_v1_models() {
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -256,6 +287,7 @@ async fn test_alias_falls_through_for_unmapped_model() {
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -125,6 +125,7 @@ async fn spawn_gateway(neuron_url: &str, entitlements: EntitlementsConfig) -> St
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -100,6 +100,7 @@ async fn spawn_gateway(neuron_url: &str, key: ApiKeyConfig) -> (Arc<CortexState>
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -519,6 +519,7 @@ pub async fn spawn_gateway_with_state(mock_url: &str) -> (Arc<CortexState>, Stri
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -97,6 +97,7 @@ fn loaded(id: &str, age_secs: i64) -> ModelEntry {
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -168,6 +169,7 @@ async fn test_evict_lru_model() {
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
node.models.insert(
|
||||
@@ -182,6 +184,7 @@ async fn test_evict_lru_model() {
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -250,6 +253,7 @@ async fn test_eviction_increments_lifecycle_cycles() {
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -183,6 +183,7 @@ async fn spawn_keyed_gateway(neuron_url: &str, hard_cap: u64) -> String {
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -38,6 +38,7 @@ async fn seed_loaded(fleet: &CortexState, node: &str, in_flight: usize, queue_de
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
n.model_load.insert(
|
||||
|
||||
@@ -73,6 +73,7 @@ async fn spawn_metered_gateway(neuron_url: &str) -> (Arc<CortexState>, String) {
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -179,6 +180,7 @@ async fn anonymous_request_records_no_spend() {
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -82,6 +82,7 @@ capabilities = ["text"]
|
||||
// No cap: effort is expressed to the model through its
|
||||
// template, not enforced by truncating its thinking.
|
||||
tokens: None,
|
||||
unavailable_reason: None,
|
||||
})
|
||||
.collect();
|
||||
node.models.insert(
|
||||
@@ -105,6 +106,7 @@ capabilities = ["text"]
|
||||
output_ceiling: 32768,
|
||||
}),
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
// A model with no derivable limit: the flat #78 fields must be
|
||||
@@ -121,6 +123,7 @@ capabilities = ["text"]
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -330,6 +330,7 @@ async fn test_poller_removes_stale_models() {
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
node.models.insert(
|
||||
@@ -344,6 +345,7 @@ async fn test_poller_removes_stale_models() {
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -99,6 +99,7 @@ async fn spawn_gateway(neuron: &str, context: usize) -> String {
|
||||
output_ceiling: 0,
|
||||
}),
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -198,6 +198,7 @@ async fn test_recovering_model_returns_503_and_stays_listed() {
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ fn entry(id: &str, servable: Option<ModelServability>) -> ModelEntry {
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable,
|
||||
reasoning_budget: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -90,7 +90,6 @@ fn router_entry(cortex: &str, e: &CortexModelEntry) -> CortexModelEntry {
|
||||
max_output_tokens: None,
|
||||
context_window: None,
|
||||
context_length: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
..e.clone()
|
||||
}
|
||||
}
|
||||
@@ -108,6 +107,7 @@ fn merge_into(acc: &mut CortexModelEntry, cortex: &str, e: &CortexModelEntry) {
|
||||
}
|
||||
acc.tool_call |= e.tool_call;
|
||||
acc.reasoning |= e.reasoning;
|
||||
acc.reasoning_budget = merge_reasoning_budget(&acc.reasoning_budget, &e.reasoning_budget);
|
||||
acc.limit = tightest_limit(acc.limit.take(), e.limit.clone());
|
||||
acc.cost = cheapest_cost(acc.cost.take(), e.cost.clone());
|
||||
}
|
||||
@@ -272,3 +272,114 @@ mod tests {
|
||||
assert_eq!(out[0].max_output_tokens, Some(4096));
|
||||
}
|
||||
}
|
||||
|
||||
/// Fold two operators' reasoning ladders into the federated view.
|
||||
///
|
||||
/// A rung is offered if **any** operator can currently serve it, because
|
||||
/// the router can route there. Marking it unavailable because one busy
|
||||
/// operator is withholding it would hide capacity the federation has.
|
||||
///
|
||||
/// Conversely a rung only carries an unavailable reason when every
|
||||
/// operator offering it is withholding it — and the first such reason is
|
||||
/// kept, since they are diagnostics rather than a set to be merged.
|
||||
///
|
||||
/// Order comes from whichever ladder is longer, so the rungs still read
|
||||
/// in the order a client would climb them.
|
||||
fn merge_reasoning_budget(
|
||||
acc: &[cortex_core::harness::ReasoningBudgetRung],
|
||||
other: &[cortex_core::harness::ReasoningBudgetRung],
|
||||
) -> Vec<cortex_core::harness::ReasoningBudgetRung> {
|
||||
if acc.is_empty() {
|
||||
return other.to_vec();
|
||||
}
|
||||
if other.is_empty() {
|
||||
return acc.to_vec();
|
||||
}
|
||||
let (long, short) = if acc.len() >= other.len() {
|
||||
(acc, other)
|
||||
} else {
|
||||
(other, acc)
|
||||
};
|
||||
long.iter()
|
||||
.map(|rung| {
|
||||
let counterpart = short.iter().find(|r| r.effort == rung.effort);
|
||||
let Some(counterpart) = counterpart else {
|
||||
// Only one operator offers this rung at all; take it as-is.
|
||||
return rung.clone();
|
||||
};
|
||||
let available_somewhere =
|
||||
rung.unavailable_reason.is_none() || counterpart.unavailable_reason.is_none();
|
||||
cortex_core::harness::ReasoningBudgetRung {
|
||||
effort: rung.effort.clone(),
|
||||
tokens: rung.tokens.or(counterpart.tokens),
|
||||
default: rung.default || counterpart.default,
|
||||
unavailable_reason: if available_somewhere {
|
||||
None
|
||||
} else {
|
||||
rung.unavailable_reason
|
||||
.clone()
|
||||
.or_else(|| counterpart.unavailable_reason.clone())
|
||||
},
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod reasoning_budget_merge_tests {
|
||||
use super::*;
|
||||
use cortex_core::harness::ReasoningBudgetRung;
|
||||
|
||||
fn rung(effort: &str, unavailable: Option<&str>) -> ReasoningBudgetRung {
|
||||
ReasoningBudgetRung {
|
||||
effort: effort.into(),
|
||||
tokens: None,
|
||||
default: false,
|
||||
unavailable_reason: unavailable.map(str::to_string),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_rung_one_operator_can_serve_is_offered() {
|
||||
// The federation's whole point: a busy operator withholding xhigh
|
||||
// must not hide an idle operator that can serve it.
|
||||
let busy = vec![rung("low", None), rung("xhigh", Some("max_in_flight=8"))];
|
||||
let idle = vec![rung("low", None), rung("xhigh", None)];
|
||||
let merged = merge_reasoning_budget(&busy, &idle);
|
||||
let x = merged.iter().find(|r| r.effort == "xhigh").expect("xhigh");
|
||||
assert_eq!(x.unavailable_reason, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_rung_every_operator_withholds_stays_unavailable() {
|
||||
let a = vec![rung("xhigh", Some("max_in_flight=8"))];
|
||||
let b = vec![rung("xhigh", Some("max_in_flight=4"))];
|
||||
let merged = merge_reasoning_budget(&a, &b);
|
||||
assert!(merged[0].unavailable_reason.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_ladder_does_not_erase_a_populated_one() {
|
||||
// A non-reasoning model, or a cortex too old to report rungs,
|
||||
// must not blank out an operator that does.
|
||||
let populated = vec![rung("low", None), rung("medium", None)];
|
||||
assert_eq!(merge_reasoning_budget(&populated, &[]).len(), 2);
|
||||
assert_eq!(merge_reasoning_budget(&[], &populated).len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_rung_only_one_operator_offers_survives_the_merge() {
|
||||
let few = vec![rung("low", None)];
|
||||
let many = vec![rung("low", None), rung("xhigh", None)];
|
||||
let merged = merge_reasoning_budget(&few, &many);
|
||||
assert_eq!(merged.len(), 2, "the longer ladder sets the shape");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_default_flag_survives_from_either_side() {
|
||||
let mut d = rung("xhigh", None);
|
||||
d.default = true;
|
||||
let merged = merge_reasoning_budget(&[rung("xhigh", None)], &[d]);
|
||||
assert!(merged[0].default);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,6 +53,18 @@ fn unique_email() -> String {
|
||||
format!("u-{}@test.local", uuid::Uuid::new_v4())
|
||||
}
|
||||
|
||||
/// A fixture value that is unique per run.
|
||||
///
|
||||
/// Rows keyed on a hash of these (`email_tokens.token_hash`,
|
||||
/// `api_keys.key_hash`) are uniquely indexed, so a hard-coded literal
|
||||
/// passes against a virgin database and then fails with `23505` on every
|
||||
/// run afterwards — the tests only looked deterministic because CI always
|
||||
/// gets a fresh one. Anything inserted under a unique constraint gets a
|
||||
/// fresh value here.
|
||||
fn unique_secret(prefix: &str) -> String {
|
||||
format!("{prefix}-{}", uuid::Uuid::new_v4())
|
||||
}
|
||||
|
||||
async fn post(url: String, body: Value, bearer: Option<&str>) -> reqwest::Response {
|
||||
let c = reqwest::Client::new();
|
||||
let mut req = c.post(url).json(&body);
|
||||
@@ -85,13 +97,13 @@ async fn verify_endpoint_consumes_token_once() {
|
||||
.await
|
||||
.unwrap()
|
||||
.get("id");
|
||||
let raw = "verify-raw-token-xyz";
|
||||
let raw = unique_secret("verify-raw-token");
|
||||
pool.execute(
|
||||
sqlx::query(
|
||||
"INSERT INTO email_tokens (token_hash, user_id, kind, expires_at) \
|
||||
VALUES ($1, $2, 'verify', now() + interval '1 hour')",
|
||||
)
|
||||
.bind(sha256(raw))
|
||||
.bind(sha256(&raw))
|
||||
.bind(user_id),
|
||||
)
|
||||
.await
|
||||
@@ -273,13 +285,13 @@ async fn fingerprint_abuse_silently_deactivates_all_no_clue() {
|
||||
.await
|
||||
.unwrap()
|
||||
.get("id");
|
||||
let raw = "sk-helexa-deactivated-probe";
|
||||
let raw = unique_secret("sk-helexa-deactivated-probe");
|
||||
pool.execute(
|
||||
sqlx::query(
|
||||
"INSERT INTO api_keys (account_id, key_hash, key_prefix) VALUES ($1, $2, 'sk-helexa-')",
|
||||
)
|
||||
.bind(acct)
|
||||
.bind(sha256(raw)),
|
||||
.bind(sha256(&raw)),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
@@ -94,7 +94,7 @@ pub struct CandleHarnessConfig {
|
||||
/// configurable because the right values depend on the model and the
|
||||
/// hardware.
|
||||
#[serde(default)]
|
||||
pub reasoning_budget: crate::harness::reasoning_budget::ReasoningBudgetLadder,
|
||||
pub reasoning_budget: crate::harness::reasoning_budget::ReasoningBudgetConfig,
|
||||
|
||||
/// Admission control (#53): bounds the per-model wait queue so a busy
|
||||
/// model returns a fast, retryable `429`/`503` instead of stalling new
|
||||
|
||||
@@ -104,7 +104,7 @@ pub struct CandleHarness {
|
||||
context_limit_cfg: crate::config::ContextLimitConfig,
|
||||
/// Effort-level → reasoning-token rungs (#223), resolved per request
|
||||
/// so a caller can bound how long the model thinks.
|
||||
reasoning_budget_cfg: super::reasoning_budget::ReasoningBudgetLadder,
|
||||
reasoning_budget_cfg: super::reasoning_budget::ReasoningBudgetConfig,
|
||||
/// Admission-control settings (#53), used to build each loaded model's
|
||||
/// [`super::admission::AdmissionController`] at load time.
|
||||
admission_cfg: crate::config::AdmissionConfig,
|
||||
@@ -513,11 +513,21 @@ impl LoadedHandle {
|
||||
LoadedHandle::Image(_) => return None,
|
||||
};
|
||||
let rate = rate.unwrap_or(cfg.bootstrap_prefill_tok_per_sec);
|
||||
// The KV budget admission will actually grant. Advertising past
|
||||
// it hands a client a context it can only discover is a lie by
|
||||
// hitting `413 prompt_too_long_for_vram` mid-session.
|
||||
let kv_budget_mb = match self {
|
||||
LoadedHandle::Single(m) => m.admission.kv_budget_mb(),
|
||||
#[cfg(feature = "cuda")]
|
||||
LoadedHandle::Tp(m) => m.admission.kv_budget_mb(),
|
||||
LoadedHandle::Image(_) => unreachable!("derived_limit returned above"),
|
||||
};
|
||||
let limit = super::context_limit::derive_limit(
|
||||
&profile,
|
||||
free_mb,
|
||||
rate,
|
||||
max_prompt_tokens_clamp(),
|
||||
kv_budget_mb,
|
||||
cfg,
|
||||
);
|
||||
// Refresh the request-path enforcement cap (#67 phase 5b): the
|
||||
@@ -604,12 +614,10 @@ impl LoadedImageModel {
|
||||
/// the four request-handling paths.
|
||||
/// The reasoning-token budget this request asked for, if any (#223).
|
||||
///
|
||||
/// Three inlets converge on `extra.reasoning`: Anthropic's
|
||||
/// `thinking.budget_tokens` (normalised in translation), the explicit
|
||||
/// `reasoning.max_tokens` extension, and OpenAI's `reasoning.effort`
|
||||
/// ladder resolved against the operator's configured rungs. An explicit
|
||||
/// token count wins over a ladder rung — it is the more specific thing
|
||||
/// the caller reached for.
|
||||
/// Two inlets converge on `extra.reasoning`: Anthropic's
|
||||
/// `thinking.budget_tokens` (normalised in translation) and the explicit
|
||||
/// `reasoning.max_tokens` extension. `reasoning.effort` is deliberately
|
||||
/// **not** one of them — see below.
|
||||
///
|
||||
/// An explicit `reasoning.max_tokens` wins. Failing that, the budget is
|
||||
/// the caller's own output budget less `answer_reserve_tokens`, so an
|
||||
@@ -617,7 +625,9 @@ impl LoadedImageModel {
|
||||
/// gets an answer rather than a think block that eats the lot.
|
||||
///
|
||||
/// Effort levels used to be mapped onto a token ladder here and enforced
|
||||
/// by force-injecting `</think>` at the limit. That made effort mean two
|
||||
/// by force-injecting `</think>` at the limit. That ladder was removed
|
||||
/// outright on 2026-08-30 — it had been unreachable since #290 and read
|
||||
/// as though effort still set a budget. That made effort mean two
|
||||
/// incompatible things at once: a prompt instruction to the model, and a
|
||||
/// guillotine applied to it. On `Qwen/Qwen3.8-27B` the model was told to
|
||||
/// reason at `xhigh` (the template default, because the caller's effort
|
||||
@@ -637,7 +647,7 @@ impl LoadedImageModel {
|
||||
/// the output cap bite than to emit a stub).
|
||||
fn requested_reasoning_budget(
|
||||
request: &ChatCompletionRequest,
|
||||
ladder: &super::reasoning_budget::ReasoningBudgetLadder,
|
||||
cfg: &super::reasoning_budget::ReasoningBudgetConfig,
|
||||
max_new: usize,
|
||||
) -> Option<usize> {
|
||||
if let Some(explicit) = request
|
||||
@@ -649,7 +659,7 @@ fn requested_reasoning_budget(
|
||||
{
|
||||
return Some(explicit as usize);
|
||||
}
|
||||
let reserve = ladder.answer_reserve_tokens;
|
||||
let reserve = cfg.answer_reserve_tokens;
|
||||
if reserve == 0 || reserve >= max_new {
|
||||
return None;
|
||||
}
|
||||
@@ -3637,20 +3647,33 @@ impl Harness for CandleHarness {
|
||||
// strongest setting. (Whether a given client reads this
|
||||
// is another matter — pi-ai does not; see
|
||||
// `reasoning_effort`'s module doc.)
|
||||
reasoning_budget: h
|
||||
.reasoning_efforts()
|
||||
.levels
|
||||
.iter()
|
||||
.map(|effort| cortex_core::harness::ReasoningBudgetRung {
|
||||
default: h.reasoning_efforts().default.as_deref() == Some(effort.as_str()),
|
||||
effort: effort.clone(),
|
||||
// No cap: effort reaches the model through its
|
||||
// template, and how many tokens it then spends
|
||||
// is the model's business. Enforcing effort by
|
||||
// truncation is the defect #290 fixed.
|
||||
tokens: None,
|
||||
})
|
||||
.collect(),
|
||||
reasoning_budget: {
|
||||
let efforts = h.reasoning_efforts();
|
||||
let slots = h.capacity().0;
|
||||
// Withdraw the deepest rung on a busy node, but never
|
||||
// a model's only rung — the rule steers a choice, and
|
||||
// with one option there is no choice to steer.
|
||||
let withhold = (efforts.levels.len() > 1
|
||||
&& !super::reasoning_budget::long_rung_available(slots))
|
||||
.then(|| super::reasoning_budget::longest_rung(&efforts.levels))
|
||||
.flatten()
|
||||
.cloned();
|
||||
efforts
|
||||
.levels
|
||||
.iter()
|
||||
.map(|effort| cortex_core::harness::ReasoningBudgetRung {
|
||||
default: efforts.default.as_deref() == Some(effort.as_str()),
|
||||
effort: effort.clone(),
|
||||
// No cap: effort reaches the model through its
|
||||
// template, and how many tokens it then spends
|
||||
// is the model's business. Enforcing effort by
|
||||
// truncation is the defect #290 fixed.
|
||||
tokens: None,
|
||||
unavailable_reason: (withhold.as_deref() == Some(effort.as_str()))
|
||||
.then(|| format!("max_in_flight={slots}")),
|
||||
})
|
||||
.collect()
|
||||
},
|
||||
});
|
||||
}
|
||||
// Models mid-recovery whose registry slot is absent (the
|
||||
@@ -7670,29 +7693,36 @@ mod tests {
|
||||
/// does not vary with effort.
|
||||
#[test]
|
||||
fn effort_no_longer_selects_a_rungs_token_budget() {
|
||||
let ladder = super::super::reasoning_budget::ReasoningBudgetLadder::default();
|
||||
let medium = requested_reasoning_budget(&req_with_effort(Some("medium")), &ladder, 32_768);
|
||||
let xhigh = requested_reasoning_budget(&req_with_effort(Some("xhigh")), &ladder, 32_768);
|
||||
let cfg = super::super::reasoning_budget::ReasoningBudgetConfig::default();
|
||||
let medium = requested_reasoning_budget(&req_with_effort(Some("medium")), &cfg, 32_768);
|
||||
let xhigh = requested_reasoning_budget(&req_with_effort(Some("xhigh")), &cfg, 32_768);
|
||||
assert_eq!(medium, xhigh, "the bound must not depend on effort");
|
||||
// 12,288 was `medium`'s rung in the cfg this replaced. The
|
||||
// literal is the point: if a future change reintroduces a
|
||||
// per-effort budget, this is the number it would land on.
|
||||
assert_ne!(
|
||||
medium,
|
||||
Some(ladder.medium),
|
||||
"medium must not become a {}-token guillotine",
|
||||
ladder.medium
|
||||
Some(12_288),
|
||||
"medium must not become a 12288-token guillotine"
|
||||
);
|
||||
assert_eq!(
|
||||
medium,
|
||||
Some(32_768 - cfg.answer_reserve_tokens),
|
||||
"the bound is the caller's budget less the answer reserve"
|
||||
);
|
||||
}
|
||||
|
||||
/// The regression this reserve exists to prevent (#223, reopened by
|
||||
/// #290): an OpenAI-shaped client cannot send `reasoning.max_tokens`,
|
||||
/// so removing the effort→ladder mapping left it with no bound at
|
||||
/// so removing the effort→cfg mapping left it with no bound at
|
||||
/// all. Qwen3.8-27B then spent the entire 32,768-token budget
|
||||
/// thinking and returned no answer — twice, on real sessions.
|
||||
#[test]
|
||||
fn a_caller_that_names_no_budget_still_keeps_room_to_answer() {
|
||||
let ladder = super::super::reasoning_budget::ReasoningBudgetLadder::default();
|
||||
let budget = requested_reasoning_budget(&req_with_effort(Some("xhigh")), &ladder, 32_768)
|
||||
let cfg = super::super::reasoning_budget::ReasoningBudgetConfig::default();
|
||||
let budget = requested_reasoning_budget(&req_with_effort(Some("xhigh")), &cfg, 32_768)
|
||||
.expect("an unbounded think block is what broke");
|
||||
assert_eq!(budget, 32_768 - ladder.answer_reserve_tokens);
|
||||
assert_eq!(budget, 32_768 - cfg.answer_reserve_tokens);
|
||||
assert!(
|
||||
budget < 32_768,
|
||||
"some of the caller's own budget must survive for the answer"
|
||||
@@ -7703,12 +7733,12 @@ mod tests {
|
||||
/// for, so it wins over the reserve in both directions.
|
||||
#[test]
|
||||
fn an_explicit_max_tokens_still_wins() {
|
||||
let ladder = super::super::reasoning_budget::ReasoningBudgetLadder::default();
|
||||
let cfg = super::super::reasoning_budget::ReasoningBudgetConfig::default();
|
||||
let mut req = req_with_effort(None);
|
||||
if let serde_json::Value::Object(o) = &mut req.extra {
|
||||
o.insert("reasoning".into(), serde_json::json!({ "max_tokens": 500 }));
|
||||
}
|
||||
assert_eq!(requested_reasoning_budget(&req, &ladder, 32_768), Some(500));
|
||||
assert_eq!(requested_reasoning_budget(&req, &cfg, 32_768), Some(500));
|
||||
|
||||
// Above the reserve-derived value, too — not clamped to it.
|
||||
if let serde_json::Value::Object(o) = &mut req.extra {
|
||||
@@ -7717,30 +7747,23 @@ mod tests {
|
||||
serde_json::json!({ "max_tokens": 31_000 }),
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
requested_reasoning_budget(&req, &ladder, 32_768),
|
||||
Some(31_000)
|
||||
);
|
||||
assert_eq!(requested_reasoning_budget(&req, &cfg, 32_768), Some(31_000));
|
||||
}
|
||||
|
||||
/// Reserving from a budget too small to think in would emit a stub.
|
||||
/// Better to think freely and let the output cap bite.
|
||||
#[test]
|
||||
fn a_budget_smaller_than_the_reserve_stays_unbounded() {
|
||||
let ladder = super::super::reasoning_budget::ReasoningBudgetLadder::default();
|
||||
for max_new in [0, 1, ladder.answer_reserve_tokens] {
|
||||
let cfg = super::super::reasoning_budget::ReasoningBudgetConfig::default();
|
||||
for max_new in [0, 1, cfg.answer_reserve_tokens] {
|
||||
assert_eq!(
|
||||
requested_reasoning_budget(&req_with_effort(None), &ladder, max_new),
|
||||
requested_reasoning_budget(&req_with_effort(None), &cfg, max_new),
|
||||
None,
|
||||
"max_new={max_new} must not produce a bound"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
requested_reasoning_budget(
|
||||
&req_with_effort(None),
|
||||
&ladder,
|
||||
ladder.answer_reserve_tokens + 1
|
||||
),
|
||||
requested_reasoning_budget(&req_with_effort(None), &cfg, cfg.answer_reserve_tokens + 1),
|
||||
Some(1),
|
||||
"one token above the reserve is the first bounded case"
|
||||
);
|
||||
@@ -7750,12 +7773,11 @@ mod tests {
|
||||
/// thing creating a bound.
|
||||
#[test]
|
||||
fn a_zero_reserve_restores_unbounded_reasoning() {
|
||||
let ladder = super::super::reasoning_budget::ReasoningBudgetLadder {
|
||||
let cfg = super::super::reasoning_budget::ReasoningBudgetConfig {
|
||||
answer_reserve_tokens: 0,
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
requested_reasoning_budget(&req_with_effort(Some("xhigh")), &ladder, 32_768),
|
||||
requested_reasoning_budget(&req_with_effort(Some("xhigh")), &cfg, 32_768),
|
||||
None
|
||||
);
|
||||
}
|
||||
@@ -7867,7 +7889,7 @@ mod tests {
|
||||
#[test]
|
||||
fn the_reasoning_budget_resolves_from_every_inlet() {
|
||||
use cortex_core::openai::{ChatCompletionRequest, MessageContent};
|
||||
let ladder = crate::harness::reasoning_budget::ReasoningBudgetLadder::default();
|
||||
let cfg = crate::harness::reasoning_budget::ReasoningBudgetConfig::default();
|
||||
let req = |extra: serde_json::Value| ChatCompletionRequest {
|
||||
model: "m".into(),
|
||||
messages: vec![],
|
||||
@@ -7889,14 +7911,14 @@ mod tests {
|
||||
// Every case below shares one output budget, so the only thing
|
||||
// that varies is the caller's reasoning control.
|
||||
const OUT: usize = 32_768;
|
||||
let reserved = OUT - ladder.answer_reserve_tokens;
|
||||
let reserved = OUT - cfg.answer_reserve_tokens;
|
||||
|
||||
// Explicit count — also the shape Anthropic's `budget_tokens`
|
||||
// is normalised into during translation.
|
||||
assert_eq!(
|
||||
requested_reasoning_budget(
|
||||
&req(serde_json::json!({"reasoning": {"max_tokens": 5000}})),
|
||||
&ladder,
|
||||
&cfg,
|
||||
OUT
|
||||
),
|
||||
Some(5000)
|
||||
@@ -7909,7 +7931,7 @@ mod tests {
|
||||
assert_eq!(
|
||||
requested_reasoning_budget(
|
||||
&req(serde_json::json!({"reasoning": {"effort": "medium"}})),
|
||||
&ladder,
|
||||
&cfg,
|
||||
OUT
|
||||
),
|
||||
Some(reserved)
|
||||
@@ -7918,7 +7940,7 @@ mod tests {
|
||||
assert_eq!(
|
||||
requested_reasoning_budget(
|
||||
&req(serde_json::json!({"reasoning": {"effort": "low", "max_tokens": 999}})),
|
||||
&ladder,
|
||||
&cfg,
|
||||
OUT
|
||||
),
|
||||
Some(999)
|
||||
@@ -7927,7 +7949,7 @@ mod tests {
|
||||
// is the case pi sends, and the one that returned no answer at
|
||||
// all while it was unbounded.
|
||||
assert_eq!(
|
||||
requested_reasoning_budget(&req(serde_json::json!({})), &ladder, OUT),
|
||||
requested_reasoning_budget(&req(serde_json::json!({})), &cfg, OUT),
|
||||
Some(reserved)
|
||||
);
|
||||
// `none` disables thinking via the template, so the governor
|
||||
@@ -7935,7 +7957,7 @@ mod tests {
|
||||
assert_eq!(
|
||||
requested_reasoning_budget(
|
||||
&req(serde_json::json!({"reasoning": {"effort": "none"}})),
|
||||
&ladder,
|
||||
&cfg,
|
||||
OUT
|
||||
),
|
||||
Some(reserved)
|
||||
@@ -8728,7 +8750,8 @@ mod tests {
|
||||
let cfg = crate::config::ContextLimitConfig::default();
|
||||
let profile = backstop_profile();
|
||||
|
||||
let limit = super::super::context_limit::derive_limit(&profile, 40_000, 800.0, None, &cfg);
|
||||
let limit =
|
||||
super::super::context_limit::derive_limit(&profile, 40_000, 800.0, None, 0, &cfg);
|
||||
let cap = limit.input.expect("input budget derived");
|
||||
assert_eq!(cap, 87_040);
|
||||
|
||||
|
||||
@@ -283,7 +283,8 @@ const DEFAULT_OUTPUT_CEILING: usize = 32_768;
|
||||
/// output_ceiling = min(max_output_tokens, max_position_embeddings)
|
||||
/// vram_ceiling = (free_tightest − activation_headroom − min_free_floor) / kv_bytes_per_token_per_card
|
||||
/// throughput_ceiling = target_prefill_latency_secs × prefill_tok_per_sec
|
||||
/// context = min(max_position_embeddings, vram_ceiling, throughput_ceiling) [clamped by `hard_ceiling` if set]
|
||||
/// kv_ceiling = kv_budget_mb × 1 MiB / kv_bytes_per_token_per_card
|
||||
/// context = min(max_position_embeddings, vram_ceiling, throughput_ceiling, kv_ceiling) [clamped by `hard_ceiling` if set]
|
||||
/// input = context − output
|
||||
/// ```
|
||||
///
|
||||
@@ -294,6 +295,23 @@ const DEFAULT_OUTPUT_CEILING: usize = 32_768;
|
||||
/// `hard_ceiling` is an optional clamp-only backstop
|
||||
/// (`NEURON_MAX_PROMPT_TOKENS` or a catalogue override); `None` = no clamp.
|
||||
///
|
||||
/// `kv_budget_mb` is what admission will actually hand out
|
||||
/// ([`kv_reservation_mb`](super::candle::kv_reservation_mb) prices every
|
||||
/// request against it). Without this term the advertisement and the
|
||||
/// admission gate answer "how long a prompt fits" independently, and
|
||||
/// they disagree: measured on beast 2026-08-30, a 3689 MiB budget holds
|
||||
/// 118,048 tokens at 32 KiB/token/card while the advertised context was
|
||||
/// the model's full 131,072 — so a client that believed the
|
||||
/// advertisement filled to it and got a hard `413
|
||||
/// prompt_too_long_for_vram` with no compaction path. `0` means the
|
||||
/// budget is not published yet (pre-load, or CPU) → no clamp, matching
|
||||
/// the `free_tightest_mb == 0` sentinel above.
|
||||
///
|
||||
/// Note this bounds the *whole* sequence. A caller that intends to
|
||||
/// request `output_ceiling` output must plan against `context − its own
|
||||
/// declared output`, not against `input`: `input` is `context − output`
|
||||
/// for the default reserve, which is the only output size it can know.
|
||||
///
|
||||
/// `reasoning`: `input = context − output` keeps a generation reserve
|
||||
/// below the wall; `output` (the reserve) is a *sub-budget* of context,
|
||||
/// matching opencode's compaction model.
|
||||
@@ -302,6 +320,7 @@ pub fn derive_limit(
|
||||
free_tightest_mb: u64,
|
||||
prefill_tok_per_sec: f64,
|
||||
hard_ceiling: Option<usize>,
|
||||
kv_budget_mb: u64,
|
||||
cfg: &ContextLimitConfig,
|
||||
) -> ModelLimit {
|
||||
let output = cfg.output_reserve_tokens;
|
||||
@@ -335,10 +354,20 @@ pub fn derive_limit(
|
||||
};
|
||||
let throughput_ceiling = (cfg.target_prefill_latency_secs * tok_per_sec).max(0.0) as usize;
|
||||
|
||||
// Admission ceiling — what the KV gate will actually grant. Zero is
|
||||
// the "not published yet" sentinel (pre-load, CPU) and imposes no
|
||||
// ceiling, like `free_tightest_mb` above.
|
||||
let kv_ceiling = if kv_budget_mb == 0 || profile.kv_bytes_per_token_per_card == 0 {
|
||||
usize::MAX
|
||||
} else {
|
||||
(kv_budget_mb.saturating_mul(1024 * 1024) / profile.kv_bytes_per_token_per_card) as usize
|
||||
};
|
||||
|
||||
let mut context = profile
|
||||
.max_position_embeddings
|
||||
.min(vram_ceiling)
|
||||
.min(throughput_ceiling);
|
||||
.min(throughput_ceiling)
|
||||
.min(kv_ceiling);
|
||||
if let Some(clamp) = hard_ceiling {
|
||||
context = context.min(clamp);
|
||||
}
|
||||
@@ -355,6 +384,8 @@ pub fn derive_limit(
|
||||
prefill_tok_per_sec = tok_per_sec,
|
||||
vram_ceiling,
|
||||
throughput_ceiling,
|
||||
kv_budget_mb,
|
||||
kv_ceiling,
|
||||
?hard_ceiling,
|
||||
context,
|
||||
"derive_limit"
|
||||
@@ -453,7 +484,7 @@ mod tests {
|
||||
kv_bytes_per_token_per_card: 65_536,
|
||||
world_size: 2,
|
||||
};
|
||||
let limit = derive_limit(&profile, 0, 0.0, None, &cfg);
|
||||
let limit = derive_limit(&profile, 0, 0.0, None, 0, &cfg);
|
||||
assert_eq!(limit.output, cfg.output_reserve_tokens, "reserve unchanged");
|
||||
assert_eq!(limit.output_ceiling, DEFAULT_OUTPUT_CEILING);
|
||||
assert!(limit.output_ceiling > limit.output);
|
||||
@@ -469,10 +500,54 @@ mod tests {
|
||||
kv_bytes_per_token_per_card: 65_536,
|
||||
world_size: 1,
|
||||
};
|
||||
let limit = derive_limit(&profile, 0, 0.0, None, &cfg);
|
||||
let limit = derive_limit(&profile, 0, 0.0, None, 0, &cfg);
|
||||
assert_eq!(limit.output_ceiling, 16_384);
|
||||
}
|
||||
|
||||
/// The advertisement may not promise a context the KV gate will
|
||||
/// refuse. Numbers are the live beast measurement (2026-08-30): a
|
||||
/// 3689 MiB budget at 32 KiB/token/card holds 118,048 tokens, while
|
||||
/// the model's own position limit is 131,072. Before this clamp the
|
||||
/// node advertised 131,072, pi filled a session to 90,432 prompt
|
||||
/// tokens believing it, and the turn died on a hard 413.
|
||||
#[test]
|
||||
fn the_context_never_exceeds_what_admission_will_grant() {
|
||||
let cfg = ContextLimitConfig::default();
|
||||
let profile = ContextProfile {
|
||||
max_position_embeddings: 131_072,
|
||||
kv_bytes_per_token_per_card: 32_768,
|
||||
world_size: 2,
|
||||
};
|
||||
// Roomy VRAM and a fast card, so only the KV budget can bind.
|
||||
let limit = derive_limit(&profile, 60_000, 100_000.0, None, 3689, &cfg);
|
||||
let context = limit.context;
|
||||
assert!(
|
||||
context <= 118_048,
|
||||
"advertised {context} tokens against a budget holding 118,048"
|
||||
);
|
||||
assert_eq!(context, 117_760, "118,048 rounded down to the 1024 grid");
|
||||
|
||||
// A caller planning against the advertised ceiling must fit too:
|
||||
// it subtracts its own declared output from `context`.
|
||||
let ceiling = limit.output_ceiling;
|
||||
assert!(context.saturating_sub(ceiling) <= 118_048 - ceiling);
|
||||
}
|
||||
|
||||
/// Zero is "not published yet" (pre-load, or a CPU load), not "no
|
||||
/// room" — the same sentinel `free_tightest_mb` uses. Collapsing the
|
||||
/// limit to zero there would unadvertise every model on startup.
|
||||
#[test]
|
||||
fn an_unpublished_kv_budget_imposes_no_ceiling() {
|
||||
let cfg = ContextLimitConfig::default();
|
||||
let with_budget = derive_limit(&beast_profile(), 9254, 8500.0, None, 3689, &cfg);
|
||||
let without = derive_limit(&beast_profile(), 9254, 8500.0, None, 0, &cfg);
|
||||
assert!(
|
||||
without.context >= with_budget.context,
|
||||
"the sentinel must not bind tighter than a real budget"
|
||||
);
|
||||
assert!(without.context > 0);
|
||||
}
|
||||
|
||||
/// The operator's number wins, which is what makes the default a
|
||||
/// starting point rather than a guess baked into the binary.
|
||||
#[test]
|
||||
@@ -487,7 +562,7 @@ mod tests {
|
||||
world_size: 2,
|
||||
};
|
||||
assert_eq!(
|
||||
derive_limit(&profile, 0, 0.0, None, &cfg).output_ceiling,
|
||||
derive_limit(&profile, 0, 0.0, None, 0, &cfg).output_ceiling,
|
||||
65_536
|
||||
);
|
||||
}
|
||||
@@ -504,10 +579,10 @@ mod tests {
|
||||
kv_bytes_per_token_per_card: 65_536,
|
||||
world_size: 2,
|
||||
};
|
||||
let roomy = derive_limit(&profile, 60_000, 5_000.0, None, &cfg);
|
||||
let roomy = derive_limit(&profile, 60_000, 5_000.0, None, 0, &cfg);
|
||||
// Tight enough that the VRAM term binds — the same shape as the
|
||||
// fleet advertising 109568 one hour and 131072 the next.
|
||||
let tight = derive_limit(&profile, 6_000, 5_000.0, None, &cfg);
|
||||
let tight = derive_limit(&profile, 6_000, 5_000.0, None, 0, &cfg);
|
||||
assert!(
|
||||
tight.context < roomy.context,
|
||||
"the live window should shrink under VRAM pressure"
|
||||
@@ -543,7 +618,7 @@ mod tests {
|
||||
// VRAM ceiling on beast pre-#11. VRAM (~9.2 GB free) allows far
|
||||
// more, max_position_embeddings is 262144, so throughput wins.
|
||||
let cfg = ContextLimitConfig::default();
|
||||
let limit = derive_limit(&beast_profile(), 9254, 850.0, None, &cfg);
|
||||
let limit = derive_limit(&beast_profile(), 9254, 850.0, None, 0, &cfg);
|
||||
// 120 × 850 = 102000 → rounded down to 1024 → 101376.
|
||||
assert_eq!(limit.context, 101376);
|
||||
assert_eq!(limit.output, 8192);
|
||||
@@ -556,8 +631,8 @@ mod tests {
|
||||
// Prefix caching (#11) speeds effective prefill → ceiling rises,
|
||||
// eventually pinned by VRAM / max_position_embeddings.
|
||||
let cfg = ContextLimitConfig::default();
|
||||
let slow = derive_limit(&beast_profile(), 9254, 850.0, None, &cfg);
|
||||
let fast = derive_limit(&beast_profile(), 9254, 8500.0, None, &cfg);
|
||||
let slow = derive_limit(&beast_profile(), 9254, 850.0, None, 0, &cfg);
|
||||
let fast = derive_limit(&beast_profile(), 9254, 8500.0, None, 0, &cfg);
|
||||
assert!(fast.context > slow.context);
|
||||
}
|
||||
|
||||
@@ -565,8 +640,8 @@ mod tests {
|
||||
fn tighter_vram_lowers_the_limit() {
|
||||
// Same model, less free VRAM → VRAM ceiling binds below throughput.
|
||||
let cfg = ContextLimitConfig::default();
|
||||
let roomy = derive_limit(&beast_profile(), 9254, 8500.0, None, &cfg);
|
||||
let tight = derive_limit(&beast_profile(), 2600, 8500.0, None, &cfg);
|
||||
let roomy = derive_limit(&beast_profile(), 9254, 8500.0, None, 0, &cfg);
|
||||
let tight = derive_limit(&beast_profile(), 2600, 8500.0, None, 0, &cfg);
|
||||
assert!(tight.context < roomy.context);
|
||||
}
|
||||
|
||||
@@ -574,10 +649,10 @@ mod tests {
|
||||
fn hard_ceiling_clamps_only_downward() {
|
||||
let cfg = ContextLimitConfig::default();
|
||||
// A backstop below the derived value clamps it.
|
||||
let clamped = derive_limit(&beast_profile(), 9254, 8500.0, Some(49152), &cfg);
|
||||
let clamped = derive_limit(&beast_profile(), 9254, 8500.0, Some(49152), 0, &cfg);
|
||||
assert_eq!(clamped.context, 49152);
|
||||
// A backstop above the derived value is a no-op.
|
||||
let unclamped = derive_limit(&beast_profile(), 9254, 850.0, Some(200000), &cfg);
|
||||
let unclamped = derive_limit(&beast_profile(), 9254, 850.0, Some(200000), 0, &cfg);
|
||||
assert_eq!(unclamped.context, 101376);
|
||||
}
|
||||
|
||||
@@ -629,7 +704,7 @@ mod tests {
|
||||
world_size: 1,
|
||||
};
|
||||
let cfg = ContextLimitConfig::default();
|
||||
let limit = derive_limit(&profile, 8000, 8500.0, None, &cfg);
|
||||
let limit = derive_limit(&profile, 8000, 8500.0, None, 0, &cfg);
|
||||
// max_position_embeddings (32768) binds below throughput (~1.02M).
|
||||
assert_eq!(limit.context, 32768);
|
||||
}
|
||||
|
||||
@@ -13,40 +13,39 @@
|
||||
//!
|
||||
//! Two halves live here:
|
||||
//!
|
||||
//! - [`ReasoningBudget`] — how many reasoning tokens this request may
|
||||
//! spend, resolved from whichever control the caller's wire format
|
||||
//! gave them.
|
||||
//! - [`ReasoningBudgetConfig`] — the operator's single knob,
|
||||
//! `answer_reserve_tokens`. The budget itself is derived per request
|
||||
//! in `candle::requested_reasoning_budget` as
|
||||
//! `declared_max_tokens − answer_reserve_tokens`, or taken verbatim
|
||||
//! from `reasoning.max_tokens` when the caller sets one.
|
||||
//! - [`ReasoningGovernor`] — the per-request state that enforces it, by
|
||||
//! substituting the model's own `</think>` token when the budget runs
|
||||
//! out so generation transitions to answering instead of being cut off.
|
||||
//!
|
||||
//! **The budget does not depend on the effort level, and must not.**
|
||||
//! Effort reaches the model as a prompt instruction through its own chat
|
||||
//! template (`candle::apply_effort_kwarg`, #290); deriving a token cap
|
||||
//! from it as well is what told Qwen3.8-27B to reason at `xhigh` and
|
||||
//! then guillotined it at `medium`'s allowance. This module carried a
|
||||
//! per-rung table for that purpose until 2026-08-30; it had been dead
|
||||
//! since #290 and is gone. If you are looking for where `low` differs
|
||||
//! from `xhigh`, it is the rendered prompt, not a number here.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Per-model mapping from an OpenAI effort level to a token budget.
|
||||
/// How much of a caller's output budget is held back for the answer.
|
||||
///
|
||||
/// OpenAI-shaped clients cannot send a token count — pi-ai's vocabulary
|
||||
/// is the ladder `off | minimal | low | medium | high | xhigh | max` and
|
||||
/// nothing else — so the numbers have to live on the server. Operator
|
||||
/// configurable, because the right values depend on the model and the
|
||||
/// hardware; a derived default would be a guess baked into the binary.
|
||||
/// One field. It was a per-effort ladder until 2026-08-30 — see the
|
||||
/// module doc for why that was the wrong shape and why nothing read it.
|
||||
/// The name stays `[harness.candle.reasoning_budget]` on the wire so
|
||||
/// operator configs keep working; unknown keys are ignored, so a stale
|
||||
/// `medium = 12288` is harmless.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ReasoningBudgetLadder {
|
||||
pub minimal: usize,
|
||||
pub low: usize,
|
||||
pub medium: usize,
|
||||
pub high: usize,
|
||||
/// `xhigh` and `max` are only offered by a client when the model
|
||||
/// declares them; absent here means "not offered", and a request
|
||||
/// naming one falls back to `high` rather than being rejected.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub xhigh: Option<usize>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max: Option<usize>,
|
||||
pub struct ReasoningBudgetConfig {
|
||||
/// Tokens of the caller's **own** output budget held back for the
|
||||
/// answer, when the caller names no explicit `reasoning.max_tokens`.
|
||||
///
|
||||
/// This is what an OpenAI-shaped client gets instead of the rungs
|
||||
/// above, and it is deliberately not derived from effort. Effort is
|
||||
/// Deliberately not derived from effort. Effort is
|
||||
/// a prompt instruction the model is trained to follow (#290);
|
||||
/// making it *also* set a guillotine is what told Qwen3.8-27B to
|
||||
/// reason at `xhigh` and then cut it off at `medium`'s 12,288.
|
||||
@@ -69,7 +68,7 @@ fn default_answer_reserve() -> usize {
|
||||
4_096
|
||||
}
|
||||
|
||||
impl Default for ReasoningBudgetLadder {
|
||||
impl Default for ReasoningBudgetConfig {
|
||||
/// Rungs sized against observed behaviour rather than round numbers:
|
||||
/// a long agentic design turn on this class of model runs to ~27k
|
||||
/// tokens, so `high` sits above that (bounded but rarely binding),
|
||||
@@ -77,57 +76,11 @@ impl Default for ReasoningBudgetLadder {
|
||||
/// and `low` at the scale of a focused single-file change.
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
minimal: 1_024,
|
||||
low: 4_096,
|
||||
medium: 12_288,
|
||||
high: 32_768,
|
||||
xhigh: None,
|
||||
max: None,
|
||||
answer_reserve_tokens: default_answer_reserve(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ReasoningBudgetLadder {
|
||||
/// Resolve an effort level to a token budget.
|
||||
///
|
||||
/// `none`/`off` are handled before this — they disable thinking
|
||||
/// outright via the chat template — so they have no budget here.
|
||||
/// An unrecognised level resolves to `None` (unbounded) rather than
|
||||
/// to a guess: silently substituting a budget nobody asked for is
|
||||
/// the failure mode this whole area has been full of.
|
||||
pub fn for_effort(&self, effort: &str) -> Option<usize> {
|
||||
match effort.trim().to_ascii_lowercase().as_str() {
|
||||
"minimal" => Some(self.minimal),
|
||||
"low" => Some(self.low),
|
||||
"medium" => Some(self.medium),
|
||||
"high" => Some(self.high),
|
||||
"xhigh" => Some(self.xhigh.unwrap_or(self.high)),
|
||||
"max" => Some(self.max.or(self.xhigh).unwrap_or(self.high)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The mapping as a client should see it on `/v1/models` — the rungs
|
||||
/// this deployment actually offers, so a caller picking `low` knows
|
||||
/// what it bought instead of guessing (#274).
|
||||
pub fn advertised(&self) -> Vec<(String, usize)> {
|
||||
let mut out = vec![
|
||||
("minimal".to_string(), self.minimal),
|
||||
("low".to_string(), self.low),
|
||||
("medium".to_string(), self.medium),
|
||||
("high".to_string(), self.high),
|
||||
];
|
||||
if let Some(v) = self.xhigh {
|
||||
out.push(("xhigh".to_string(), v));
|
||||
}
|
||||
if let Some(v) = self.max {
|
||||
out.push(("max".to_string(), v));
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
/// Enforces a resolved budget over one request's token stream.
|
||||
///
|
||||
/// Fed every sampled token; returns the token that should actually be
|
||||
@@ -270,36 +223,102 @@ mod tests {
|
||||
}
|
||||
assert!(!g.forced(), "content after the block must not be governed");
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the longest reasoning rung should be offered on a node
|
||||
/// running `max_in_flight` concurrent slots.
|
||||
///
|
||||
/// Operator-reported (2026-08-29): `xhigh` "hits issues" when this host
|
||||
/// runs eight slots. The mechanism is not yet isolated, so this is a
|
||||
/// precaution rather than a diagnosis, and it is written down as one —
|
||||
/// naming it a fix would stop anyone looking.
|
||||
///
|
||||
/// The plausible account: `xhigh` instructs the model to deliberate
|
||||
/// exhaustively, so a turn can spend tens of thousands of tokens
|
||||
/// reasoning. KV is a shared pool (#291), and a slot holds its
|
||||
/// reservation for the request's whole life, so several concurrent
|
||||
/// `xhigh` turns can hold most of the pool for minutes while shorter
|
||||
/// requests queue on bytes. That is consistent with "issues" but is not
|
||||
/// evidence of them; #305's turn is a candidate to instrument.
|
||||
///
|
||||
/// **2026-08-30: that account is now contradicted, and the constant
|
||||
/// below did not hold.** A controlled A/B/C on this host reproduced the
|
||||
/// defect at `max_in_flight = 2` with `in_flight == 1` — the very
|
||||
/// configuration this function calls safe — while the `medium` arm at
|
||||
/// `max_in_flight = 8` was clean. Concurrency does not predict it.
|
||||
/// Neither does the GatedDeltaNet bf16 state round-trip (#284):
|
||||
/// `NEURON_GDN_STATE_F32=1` changed neither the defect rate nor its
|
||||
/// class on a run matched to within 0.2% on the opening generation.
|
||||
/// What both `xhigh` arms produced was a read against the wrong
|
||||
/// receiver with the correct one in scope on an adjacent line
|
||||
/// (`tw.damage` for `def.damage`; `1 - slowPct` for `slowPct / 100`) —
|
||||
/// the same shape as #305's `<parameter=` for `<function=`. The
|
||||
/// mechanism remains unisolated; this is still a precaution, and the
|
||||
/// evidence now says it is very likely guarding the wrong variable.
|
||||
/// Do not read the number below as a diagnosis.
|
||||
///
|
||||
/// Availability is advertised, never enforced. The request path does not
|
||||
/// reject an unavailable rung — a caller that insists still gets it. The
|
||||
/// point is to keep a UI from offering an option that will serve someone
|
||||
/// badly, not to add a refusal path to every existing API client.
|
||||
pub fn long_rung_available(max_in_flight: usize) -> bool {
|
||||
max_in_flight <= LONG_RUNG_MAX_IN_FLIGHT
|
||||
}
|
||||
|
||||
/// The slot count above which the longest rung stops being offered.
|
||||
///
|
||||
/// Two, because that is the concurrency at which a single deep session
|
||||
/// still has the pool substantially to itself. A named constant rather
|
||||
/// than a literal at the call site so the number is findable when the
|
||||
/// mechanism above is finally isolated and it turns out to be wrong.
|
||||
pub const LONG_RUNG_MAX_IN_FLIGHT: usize = 2;
|
||||
|
||||
/// The rung a deployment considers "longest", if it offers one.
|
||||
///
|
||||
/// Taken as the last entry rather than by matching on `"xhigh"`: the
|
||||
/// rungs come from the model's own template (#290), so the name of the
|
||||
/// deepest one is the model's to choose and a future model may spell it
|
||||
/// differently.
|
||||
pub fn longest_rung(levels: &[String]) -> Option<&String> {
|
||||
levels.last()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod availability_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn the_ladder_resolves_the_levels_clients_can_send() {
|
||||
let l = ReasoningBudgetLadder::default();
|
||||
assert_eq!(l.for_effort("minimal"), Some(1_024));
|
||||
assert_eq!(l.for_effort("LOW"), Some(4_096));
|
||||
assert_eq!(l.for_effort(" medium "), Some(12_288));
|
||||
assert_eq!(l.for_effort("high"), Some(32_768));
|
||||
// Not offered by default → falls back to high rather than
|
||||
// rejecting a level the client was told it could send.
|
||||
assert_eq!(l.for_effort("xhigh"), Some(32_768));
|
||||
assert_eq!(l.for_effort("max"), Some(32_768));
|
||||
// Unknown → unbounded, never a substituted guess.
|
||||
assert_eq!(l.for_effort("enthusiastic"), None);
|
||||
assert_eq!(l.for_effort("none"), None);
|
||||
fn the_long_rung_is_offered_only_at_low_concurrency() {
|
||||
assert!(long_rung_available(1));
|
||||
assert!(long_rung_available(2));
|
||||
assert!(!long_rung_available(3));
|
||||
assert!(!long_rung_available(8));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_advertised_ladder_lists_only_what_is_offered() {
|
||||
let l = ReasoningBudgetLadder::default();
|
||||
let names: Vec<_> = l.advertised().into_iter().map(|(k, _)| k).collect();
|
||||
assert_eq!(names, ["minimal", "low", "medium", "high"]);
|
||||
let l2 = ReasoningBudgetLadder {
|
||||
xhigh: Some(49_152),
|
||||
..ReasoningBudgetLadder::default()
|
||||
};
|
||||
assert!(
|
||||
l2.advertised()
|
||||
.iter()
|
||||
.any(|(k, v)| k == "xhigh" && *v == 49_152)
|
||||
);
|
||||
fn the_longest_rung_is_the_last_one_the_template_offers() {
|
||||
// Not a match on "xhigh": the ladder is the model's, and the
|
||||
// deepest rung is whatever it puts last.
|
||||
let levels = vec!["low".to_string(), "medium".to_string(), "xhigh".to_string()];
|
||||
assert_eq!(longest_rung(&levels).map(String::as_str), Some("xhigh"));
|
||||
|
||||
let other = vec!["brief".to_string(), "exhaustive".to_string()];
|
||||
assert_eq!(longest_rung(&other).map(String::as_str), Some("exhaustive"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_model_with_no_rungs_has_no_longest_one() {
|
||||
assert_eq!(longest_rung(&[]), None);
|
||||
}
|
||||
|
||||
/// A single-rung model must not have its only option withdrawn: the
|
||||
/// rule exists to steer a choice, and there is no choice to steer.
|
||||
#[test]
|
||||
fn a_single_rung_is_never_withdrawn() {
|
||||
let levels = vec!["only".to_string()];
|
||||
assert_eq!(longest_rung(&levels).map(String::as_str), Some("only"));
|
||||
// The caller is responsible for not withdrawing a lone rung;
|
||||
// pinned here so the intent survives a refactor of that caller.
|
||||
assert_eq!(levels.len(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -219,6 +219,9 @@ pub fn request_to_chat(req: ResponsesRequest) -> Result<ChatCompletionRequest, T
|
||||
}
|
||||
}
|
||||
|
||||
// Only the newest turn's thinking is context; the rest is noise.
|
||||
prune_stale_reasoning(&mut messages);
|
||||
|
||||
// Carry the caller's extension fields across the hop (#277).
|
||||
//
|
||||
// This used to start from an empty map and insert exactly one key,
|
||||
@@ -452,6 +455,54 @@ fn merge_tool_call_into(prev: &mut ChatMessage, msg: &mut ChatMessage) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
/// Keep replayed reasoning on the most recent assistant turn only.
|
||||
///
|
||||
/// #277 restored the round-trip, and then over-corrected: a client that
|
||||
/// replays every turn verbatim — which is what an agentic harness does —
|
||||
/// had the model's entire thinking history rendered back into the
|
||||
/// prompt, because Qwen3.8's template defaults `preserve_thinking` to
|
||||
/// true. Measured on a live session: **62% of the assistant content
|
||||
/// being replayed was prior think blocks** (95,311 characters of
|
||||
/// thinking against 467 characters of actual output), and the model's
|
||||
/// per-turn reasoning collapsed to tens of tokens while it made
|
||||
/// increasingly shallow passes at the task.
|
||||
///
|
||||
/// Reasoning models are trained on histories that carry *final content*,
|
||||
/// not the deliberation behind it — Qwen's own multi-turn convention
|
||||
/// strips prior think blocks. But the newest block is different: when a
|
||||
/// turn was cut off mid-thought and the caller says "continue", that
|
||||
/// thinking is the work being resumed, which is what #277 exists to
|
||||
/// preserve.
|
||||
///
|
||||
/// So: newest assistant turn keeps its reasoning, everything older
|
||||
/// loses it. Note the template's own `preserve_thinking: false` is
|
||||
/// exactly backwards for this — it keeps thinking on turns *after* the
|
||||
/// last user message (the agentic case, where it is noise) and drops it
|
||||
/// on turns *before* (the continuation case, where it is the point).
|
||||
fn prune_stale_reasoning(messages: &mut [ChatMessage]) {
|
||||
let newest = messages
|
||||
.iter()
|
||||
.rposition(|m| m.role == "assistant" && m.extra.get("reasoning_content").is_some());
|
||||
let Some(newest) = newest else { return };
|
||||
let mut dropped = 0usize;
|
||||
for (i, m) in messages.iter_mut().enumerate() {
|
||||
if i == newest || m.role != "assistant" {
|
||||
continue;
|
||||
}
|
||||
if let Some(obj) = m.extra.as_object_mut()
|
||||
&& obj.remove("reasoning_content").is_some()
|
||||
{
|
||||
dropped += 1;
|
||||
}
|
||||
}
|
||||
if dropped > 0 {
|
||||
tracing::debug!(
|
||||
dropped,
|
||||
"responses: replaying reasoning for the newest assistant turn only"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The text of a replayed reasoning item, under either spelling.
|
||||
///
|
||||
/// `summary` is what neuron emits (`summary_text` parts) and `content`
|
||||
@@ -1883,6 +1934,69 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// An agentic harness replays every turn verbatim. Rendering all of
|
||||
/// that thinking back into the prompt made the context mostly
|
||||
/// self-quotation — 62% of replayed assistant content on a live
|
||||
/// session — and the model started making shallow passes. Only the
|
||||
/// newest turn's reasoning is context; the rest is noise the model
|
||||
/// was never trained to read.
|
||||
#[test]
|
||||
fn only_the_newest_turns_reasoning_is_replayed() {
|
||||
let raw = r#"{
|
||||
"model": "Qwen/Qwen3.8-27B",
|
||||
"input": [
|
||||
{"type": "message", "role": "user", "content": "build it"},
|
||||
{"type": "reasoning", "summary": [{"type": "summary_text", "text": "old plan"}]},
|
||||
{"type": "message", "role": "assistant", "content": "wrote index.html"},
|
||||
{"type": "message", "role": "user", "content": "keep going"},
|
||||
{"type": "reasoning", "summary": [{"type": "summary_text", "text": "newer plan"}]},
|
||||
{"type": "message", "role": "assistant", "content": "wrote style.css"}
|
||||
]
|
||||
}"#;
|
||||
let req: ResponsesRequest = serde_json::from_str(raw).expect("parse");
|
||||
let chat = request_to_chat(req).expect("translate");
|
||||
let carried: Vec<_> = chat
|
||||
.messages
|
||||
.iter()
|
||||
.filter_map(|m| m.extra.get("reasoning_content").and_then(|v| v.as_str()))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
carried,
|
||||
["newer plan"],
|
||||
"older turns must keep their content but lose their thinking"
|
||||
);
|
||||
// The turns themselves survive — only the thinking is pruned.
|
||||
assert_eq!(
|
||||
chat.messages
|
||||
.iter()
|
||||
.filter(|m| m.role == "assistant")
|
||||
.count(),
|
||||
2
|
||||
);
|
||||
}
|
||||
|
||||
/// The continuation case still works: a turn cut off mid-thought is
|
||||
/// the newest assistant turn, so its reasoning is what survives.
|
||||
#[test]
|
||||
fn a_truncated_turn_keeps_its_reasoning_through_a_continue() {
|
||||
let raw = r#"{
|
||||
"model": "Qwen/Qwen3.8-27B",
|
||||
"input": [
|
||||
{"type": "message", "role": "user", "content": "build it"},
|
||||
{"type": "reasoning", "summary": [{"type": "summary_text", "text": "24k of design"}]},
|
||||
{"type": "message", "role": "user", "content": "continue"}
|
||||
]
|
||||
}"#;
|
||||
let req: ResponsesRequest = serde_json::from_str(raw).expect("parse");
|
||||
let chat = request_to_chat(req).expect("translate");
|
||||
let carried: Vec<_> = chat
|
||||
.messages
|
||||
.iter()
|
||||
.filter_map(|m| m.extra.get("reasoning_content").and_then(|v| v.as_str()))
|
||||
.collect();
|
||||
assert_eq!(carried, ["24k of design"]);
|
||||
}
|
||||
|
||||
/// The control a Responses client actually reaches for (#223): pi-ai
|
||||
/// emits `reasoning: {"effort": "none"}` for thinking-level *off*.
|
||||
/// Before this it went nowhere, and a harness that asked for no
|
||||
|
||||
@@ -58,7 +58,9 @@ if (!fs.existsSync(SSR_ENTRY)) {
|
||||
fail(`${path.relative(ROOT, SSR_ENTRY)} is missing — run the SSR build first`);
|
||||
}
|
||||
|
||||
const { renderDoc, slugs } = await import(url.pathToFileURL(SSR_ENTRY).href);
|
||||
const { renderDoc, slugs, indexes } = await import(
|
||||
url.pathToFileURL(SSR_ENTRY).href
|
||||
);
|
||||
const template = fs.readFileSync(path.join(DIST, "index.html"), "utf8");
|
||||
|
||||
// The shell must contain an empty #root for the markup to go into. If
|
||||
@@ -117,6 +119,40 @@ for (const slug of slugs()) {
|
||||
written.push({ slug, canonical });
|
||||
}
|
||||
|
||||
// Index pages for every directory the loop above created. Without these
|
||||
// nginx answers 403 for /docs and each section: the directory exists, so
|
||||
// `try_files $uri/` matches, and with no index inside and autoindex off
|
||||
// there is nothing to serve and no fallthrough to the SPA shell.
|
||||
for (const idx of indexes()) {
|
||||
const slugPath = idx.path ? `/docs/${idx.path}` : "/docs";
|
||||
const html = withHead(
|
||||
template.replace(ROOT_DIV, `<div id="root">${idx.html}</div>`),
|
||||
idx.title,
|
||||
undefined,
|
||||
`${SITE}${slugPath}`,
|
||||
);
|
||||
const dir = idx.path
|
||||
? path.join(DIST, "docs", ...idx.path.split("/"))
|
||||
: path.join(DIST, "docs");
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, "index.html"), html);
|
||||
}
|
||||
|
||||
// Nothing the prerenderer creates may be left without an index — that is
|
||||
// the 403 above, and it is silent at build time.
|
||||
const missing = [];
|
||||
(function walk(dir) {
|
||||
if (!fs.existsSync(dir)) return;
|
||||
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
if (!entries.some((e) => e.isFile() && e.name === "index.html")) {
|
||||
missing.push(path.relative(DIST, dir));
|
||||
}
|
||||
for (const e of entries) if (e.isDirectory()) walk(path.join(dir, e.name));
|
||||
})(path.join(DIST, "docs"));
|
||||
if (missing.length) {
|
||||
fail(`directories with no index.html would 403: ${missing.join(", ")}`);
|
||||
}
|
||||
|
||||
// A sitemap so the pages are discoverable without relying on a crawler
|
||||
// walking the sidebar — only once they are meant to be found.
|
||||
if (INDEXABLE) {
|
||||
|
||||
+20
-5
@@ -2157,14 +2157,29 @@ html[dir="rtl"] .hx-landing-composer textarea {
|
||||
.hx-thinking-toggle:hover {
|
||||
color: var(--color-text);
|
||||
}
|
||||
.hx-thinking-toggle input {
|
||||
margin: 0;
|
||||
.hx-thinking-select {
|
||||
background: transparent;
|
||||
color: inherit;
|
||||
border: 1px solid var(--color-border-subtle);
|
||||
border-radius: 0.35rem;
|
||||
padding: 0.1rem 0.25rem;
|
||||
font-size: 0.78rem;
|
||||
cursor: pointer;
|
||||
}
|
||||
/* Narrow screens: the composer is already tight, and the label is the
|
||||
* expendable half — the checkbox still reads as a control. */
|
||||
.hx-thinking-select:focus-visible {
|
||||
outline: 2px solid var(--color-accent);
|
||||
outline-offset: 1px;
|
||||
}
|
||||
/* Options inherit the page background in some engines and end up
|
||||
* unreadable on a dark theme; set both explicitly. */
|
||||
.hx-thinking-select option {
|
||||
background: var(--color-bg-elevated);
|
||||
color: var(--color-text);
|
||||
}
|
||||
/* Narrow screens: the composer is already tight, and the word is the
|
||||
* expendable half — the select still reads as a control. */
|
||||
@media (max-width: 30rem) {
|
||||
.hx-thinking-toggle span {
|
||||
.hx-thinking-label {
|
||||
display: none;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "በማስኬድ ላይ…",
|
||||
"showThinking": "ማመዛዘን",
|
||||
"showThinkingHint": "ሞዴሉ ከመመለሱ በፊት እንዲያስብ ጠይቅ፣ እና ማመዛዘኑን አሳይ"
|
||||
"showThinkingHint": "ሞዴሉ ከመመለሱ በፊት እንዲያስብ ጠይቅ፣ እና ማመዛዘኑን አሳይ",
|
||||
"effortOff": "ጠፍቷል",
|
||||
"effort_low": "ዝቅተኛ",
|
||||
"effort_medium": "መካከለኛ",
|
||||
"effort_xhigh": "በጣም ከፍተኛ"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "قيد التشغيل…",
|
||||
"showThinking": "التفكير",
|
||||
"showThinkingHint": "اطلب من النموذج التفكير قبل الإجابة، واعرض تفكيره"
|
||||
"showThinkingHint": "اطلب من النموذج التفكير قبل الإجابة، واعرض تفكيره",
|
||||
"effortOff": "مغلق",
|
||||
"effort_low": "منخفض",
|
||||
"effort_medium": "متوسط",
|
||||
"effort_xhigh": "مرتفع جدًا"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "изпълнява се…",
|
||||
"showThinking": "Разсъждение",
|
||||
"showThinkingHint": "Моделът да мисли преди да отговори и да показва разсъжденията си"
|
||||
"showThinkingHint": "Моделът да мисли преди да отговори и да показва разсъжденията си",
|
||||
"effortOff": "Изкл.",
|
||||
"effort_low": "Ниско",
|
||||
"effort_medium": "Средно",
|
||||
"effort_xhigh": "Много високо"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "en execució…",
|
||||
"showThinking": "Raonament",
|
||||
"showThinkingHint": "Demana al model que pensi abans de respondre i mostra'n el raonament"
|
||||
"showThinkingHint": "Demana al model que pensi abans de respondre i mostra'n el raonament",
|
||||
"effortOff": "Desactivat",
|
||||
"effort_low": "Baix",
|
||||
"effort_medium": "Mitjà",
|
||||
"effort_xhigh": "Molt alt"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "probíhá…",
|
||||
"showThinking": "Uvažování",
|
||||
"showThinkingHint": "Ať model před odpovědí přemýšlí a ukáže své uvažování"
|
||||
"showThinkingHint": "Ať model před odpovědí přemýšlí a ukáže své uvažování",
|
||||
"effortOff": "Vypnuto",
|
||||
"effort_low": "Nízké",
|
||||
"effort_medium": "Střední",
|
||||
"effort_xhigh": "Velmi vysoké"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "kører…",
|
||||
"showThinking": "Ræsonnement",
|
||||
"showThinkingHint": "Bed modellen tænke før den svarer, og vis dens ræsonnement"
|
||||
"showThinkingHint": "Bed modellen tænke før den svarer, og vis dens ræsonnement",
|
||||
"effortOff": "Fra",
|
||||
"effort_low": "Lav",
|
||||
"effort_medium": "Mellem",
|
||||
"effort_xhigh": "Meget høj"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "läuft…",
|
||||
"showThinking": "Denkprozess",
|
||||
"showThinkingHint": "Das Modell vor der Antwort nachdenken lassen und den Denkprozess anzeigen"
|
||||
"showThinkingHint": "Das Modell vor der Antwort nachdenken lassen und den Denkprozess anzeigen",
|
||||
"effortOff": "Aus",
|
||||
"effort_low": "Niedrig",
|
||||
"effort_medium": "Mittel",
|
||||
"effort_xhigh": "Sehr hoch"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "εκτελείται…",
|
||||
"showThinking": "Συλλογισμός",
|
||||
"showThinkingHint": "Ζήτα από το μοντέλο να σκεφτεί πριν απαντήσει και δείξε τον συλλογισμό του"
|
||||
"showThinkingHint": "Ζήτα από το μοντέλο να σκεφτεί πριν απαντήσει και δείξε τον συλλογισμό του",
|
||||
"effortOff": "Ανενεργό",
|
||||
"effort_low": "Χαμηλό",
|
||||
"effort_medium": "Μέτριο",
|
||||
"effort_xhigh": "Πολύ υψηλό"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "running…",
|
||||
"showThinking": "Reasoning",
|
||||
"showThinkingHint": "Ask the model to think before answering, and show its reasoning"
|
||||
"showThinkingHint": "Ask the model to think before answering, and show its reasoning",
|
||||
"effortOff": "Off",
|
||||
"effort_low": "Low",
|
||||
"effort_medium": "Medium",
|
||||
"effort_xhigh": "Extra high"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "ejecutando…",
|
||||
"showThinking": "Razonamiento",
|
||||
"showThinkingHint": "Pide al modelo que piense antes de responder y muestra su razonamiento"
|
||||
"showThinkingHint": "Pide al modelo que piense antes de responder y muestra su razonamiento",
|
||||
"effortOff": "Desactivado",
|
||||
"effort_low": "Bajo",
|
||||
"effort_medium": "Medio",
|
||||
"effort_xhigh": "Muy alto"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "töötab…",
|
||||
"showThinking": "Arutlus",
|
||||
"showThinkingHint": "Palu mudelil enne vastamist mõelda ja näita tema arutlust"
|
||||
"showThinkingHint": "Palu mudelil enne vastamist mõelda ja näita tema arutlust",
|
||||
"effortOff": "Väljas",
|
||||
"effort_low": "Madal",
|
||||
"effort_medium": "Keskmine",
|
||||
"effort_xhigh": "Väga kõrge"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "در حال اجرا…",
|
||||
"showThinking": "استدلال",
|
||||
"showThinkingHint": "از مدل بخواهید پیش از پاسخ فکر کند و استدلالش را نشان دهد"
|
||||
"showThinkingHint": "از مدل بخواهید پیش از پاسخ فکر کند و استدلالش را نشان دهد",
|
||||
"effortOff": "خاموش",
|
||||
"effort_low": "کم",
|
||||
"effort_medium": "متوسط",
|
||||
"effort_xhigh": "خیلی زیاد"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "suoritetaan…",
|
||||
"showThinking": "Päättely",
|
||||
"showThinkingHint": "Pyydä mallia ajattelemaan ennen vastaamista ja näytä sen päättely"
|
||||
"showThinkingHint": "Pyydä mallia ajattelemaan ennen vastaamista ja näytä sen päättely",
|
||||
"effortOff": "Pois",
|
||||
"effort_low": "Matala",
|
||||
"effort_medium": "Keskitaso",
|
||||
"effort_xhigh": "Erittäin korkea"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "en cours…",
|
||||
"showThinking": "Raisonnement",
|
||||
"showThinkingHint": "Demander au modèle de réfléchir avant de répondre et afficher son raisonnement"
|
||||
"showThinkingHint": "Demander au modèle de réfléchir avant de répondre et afficher son raisonnement",
|
||||
"effortOff": "Désactivé",
|
||||
"effort_low": "Faible",
|
||||
"effort_medium": "Moyen",
|
||||
"effort_xhigh": "Très élevé"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "ar siúl…",
|
||||
"showThinking": "Réasúnaíocht",
|
||||
"showThinkingHint": "Iarr ar an tsamhail machnamh a dhéanamh sula bhfreagraíonn sí, agus taispeáin é"
|
||||
"showThinkingHint": "Iarr ar an tsamhail machnamh a dhéanamh sula bhfreagraíonn sí, agus taispeáin é",
|
||||
"effortOff": "As",
|
||||
"effort_low": "Íseal",
|
||||
"effort_medium": "Meánach",
|
||||
"effort_xhigh": "An-ard"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "yana gudana…",
|
||||
"showThinking": "Tunani",
|
||||
"showThinkingHint": "Ka nemi samfurin ya yi tunani kafin amsa, kuma a nuna tunaninsa"
|
||||
"showThinkingHint": "Ka nemi samfurin ya yi tunani kafin amsa, kuma a nuna tunaninsa",
|
||||
"effortOff": "A kashe",
|
||||
"effort_low": "Ƙasa",
|
||||
"effort_medium": "Matsakaici",
|
||||
"effort_xhigh": "Mai tsayi sosai"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "פועל…",
|
||||
"showThinking": "חשיבה",
|
||||
"showThinkingHint": "בקש מהמודל לחשוב לפני שהוא עונה, והצג את החשיבה שלו"
|
||||
"showThinkingHint": "בקש מהמודל לחשוב לפני שהוא עונה, והצג את החשיבה שלו",
|
||||
"effortOff": "כבוי",
|
||||
"effort_low": "נמוך",
|
||||
"effort_medium": "בינוני",
|
||||
"effort_xhigh": "גבוה מאוד"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "izvodi se…",
|
||||
"showThinking": "Zaključivanje",
|
||||
"showThinkingHint": "Zatraži da model razmisli prije odgovora i prikaži njegovo zaključivanje"
|
||||
"showThinkingHint": "Zatraži da model razmisli prije odgovora i prikaži njegovo zaključivanje",
|
||||
"effortOff": "Isključeno",
|
||||
"effort_low": "Nisko",
|
||||
"effort_medium": "Srednje",
|
||||
"effort_xhigh": "Vrlo visoko"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "fut…",
|
||||
"showThinking": "Gondolatmenet",
|
||||
"showThinkingHint": "Kérd meg a modellt, hogy gondolkodjon válasz előtt, és mutasd a gondolatmenetét"
|
||||
"showThinkingHint": "Kérd meg a modellt, hogy gondolkodjon válasz előtt, és mutasd a gondolatmenetét",
|
||||
"effortOff": "Ki",
|
||||
"effort_low": "Alacsony",
|
||||
"effort_medium": "Közepes",
|
||||
"effort_xhigh": "Nagyon magas"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "in esecuzione…",
|
||||
"showThinking": "Ragionamento",
|
||||
"showThinkingHint": "Chiedi al modello di ragionare prima di rispondere e mostra il ragionamento"
|
||||
"showThinkingHint": "Chiedi al modello di ragionare prima di rispondere e mostra il ragionamento",
|
||||
"effortOff": "Disattivato",
|
||||
"effort_low": "Basso",
|
||||
"effort_medium": "Medio",
|
||||
"effort_xhigh": "Molto alto"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "მიმდინარეობს…",
|
||||
"showThinking": "მსჯელობა",
|
||||
"showThinkingHint": "სთხოვე მოდელს იფიქროს პასუხამდე და აჩვენე მისი მსჯელობა"
|
||||
"showThinkingHint": "სთხოვე მოდელს იფიქროს პასუხამდე და აჩვენე მისი მსჯელობა",
|
||||
"effortOff": "გამორთული",
|
||||
"effort_low": "დაბალი",
|
||||
"effort_medium": "საშუალო",
|
||||
"effort_xhigh": "ძალიან მაღალი"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "орындалуда…",
|
||||
"showThinking": "Пайымдау",
|
||||
"showThinkingHint": "Жауап бермес бұрын модель ойлансын және пайымдауын көрсетсін"
|
||||
"showThinkingHint": "Жауап бермес бұрын модель ойлансын және пайымдауын көрсетсін",
|
||||
"effortOff": "Өшірулі",
|
||||
"effort_low": "Төмен",
|
||||
"effort_medium": "Орташа",
|
||||
"effort_xhigh": "Өте жоғары"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "vykdoma…",
|
||||
"showThinking": "Samprotavimas",
|
||||
"showThinkingHint": "Paprašyk modelio pamąstyti prieš atsakant ir rodyk jo samprotavimą"
|
||||
"showThinkingHint": "Paprašyk modelio pamąstyti prieš atsakant ir rodyk jo samprotavimą",
|
||||
"effortOff": "Išjungta",
|
||||
"effort_low": "Žemas",
|
||||
"effort_medium": "Vidutinis",
|
||||
"effort_xhigh": "Labai aukštas"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "notiek…",
|
||||
"showThinking": "Spriešana",
|
||||
"showThinkingHint": "Lūdz modelim padomāt pirms atbildes un rādi tā spriešanu"
|
||||
"showThinkingHint": "Lūdz modelim padomāt pirms atbildes un rādi tā spriešanu",
|
||||
"effortOff": "Izslēgts",
|
||||
"effort_low": "Zems",
|
||||
"effort_medium": "Vidējs",
|
||||
"effort_xhigh": "Ļoti augsts"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "كيخدم…",
|
||||
"showThinking": "التفكير",
|
||||
"showThinkingHint": "طلب من الموديل يفكر قبل ما يجاوب، وبيّن التفكير ديالو"
|
||||
"showThinkingHint": "طلب من الموديل يفكر قبل ما يجاوب، وبيّن التفكير ديالو",
|
||||
"effortOff": "مطفي",
|
||||
"effort_low": "ضعيف",
|
||||
"effort_medium": "متوسط",
|
||||
"effort_xhigh": "عالي بزاف"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "għaddej…",
|
||||
"showThinking": "Raġunament",
|
||||
"showThinkingHint": "Itlob lill-mudell jaħseb qabel iwieġeb, u uri r-raġunament tiegħu"
|
||||
"showThinkingHint": "Itlob lill-mudell jaħseb qabel iwieġeb, u uri r-raġunament tiegħu",
|
||||
"effortOff": "Mitfi",
|
||||
"effort_low": "Baxx",
|
||||
"effort_medium": "Medju",
|
||||
"effort_xhigh": "Għoli ħafna"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "bezig…",
|
||||
"showThinking": "Redenering",
|
||||
"showThinkingHint": "Laat het model nadenken vóór het antwoordt en toon de redenering"
|
||||
"showThinkingHint": "Laat het model nadenken vóór het antwoordt en toon de redenering",
|
||||
"effortOff": "Uit",
|
||||
"effort_low": "Laag",
|
||||
"effort_medium": "Gemiddeld",
|
||||
"effort_xhigh": "Zeer hoog"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "kjører…",
|
||||
"showThinking": "Resonnement",
|
||||
"showThinkingHint": "Be modellen tenke før den svarer, og vis resonnementet"
|
||||
"showThinkingHint": "Be modellen tenke før den svarer, og vis resonnementet",
|
||||
"effortOff": "Av",
|
||||
"effort_low": "Lav",
|
||||
"effort_medium": "Middels",
|
||||
"effort_xhigh": "Svært høy"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "w toku…",
|
||||
"showThinking": "Rozumowanie",
|
||||
"showThinkingHint": "Poproś model, by pomyślał przed odpowiedzią, i pokaż jego rozumowanie"
|
||||
"showThinkingHint": "Poproś model, by pomyślał przed odpowiedzią, i pokaż jego rozumowanie",
|
||||
"effortOff": "Wyłączone",
|
||||
"effort_low": "Niskie",
|
||||
"effort_medium": "Średnie",
|
||||
"effort_xhigh": "Bardzo wysokie"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "em execução…",
|
||||
"showThinking": "Raciocínio",
|
||||
"showThinkingHint": "Pedir ao modelo que pense antes de responder e mostrar o raciocínio"
|
||||
"showThinkingHint": "Pedir ao modelo que pense antes de responder e mostrar o raciocínio",
|
||||
"effortOff": "Desativado",
|
||||
"effort_low": "Baixo",
|
||||
"effort_medium": "Médio",
|
||||
"effort_xhigh": "Muito alto"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "în curs…",
|
||||
"showThinking": "Raționament",
|
||||
"showThinkingHint": "Cere modelului să gândească înainte de a răspunde și arată raționamentul"
|
||||
"showThinkingHint": "Cere modelului să gândească înainte de a răspunde și arată raționamentul",
|
||||
"effortOff": "Oprit",
|
||||
"effort_low": "Scăzut",
|
||||
"effort_medium": "Mediu",
|
||||
"effort_xhigh": "Foarte ridicat"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "выполняется…",
|
||||
"showThinking": "Рассуждение",
|
||||
"showThinkingHint": "Просить модель подумать перед ответом и показывать её рассуждение"
|
||||
"showThinkingHint": "Просить модель подумать перед ответом и показывать её рассуждение",
|
||||
"effortOff": "Выкл.",
|
||||
"effort_low": "Низкое",
|
||||
"effort_medium": "Среднее",
|
||||
"effort_xhigh": "Очень высокое"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "prebieha…",
|
||||
"showThinking": "Uvažovanie",
|
||||
"showThinkingHint": "Nech model pred odpoveďou premýšľa a ukáže svoje uvažovanie"
|
||||
"showThinkingHint": "Nech model pred odpoveďou premýšľa a ukáže svoje uvažovanie",
|
||||
"effortOff": "Vypnuté",
|
||||
"effort_low": "Nízke",
|
||||
"effort_medium": "Stredné",
|
||||
"effort_xhigh": "Veľmi vysoké"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "poteka…",
|
||||
"showThinking": "Sklepanje",
|
||||
"showThinkingHint": "Naj model pred odgovorom premisli in pokaže svoje sklepanje"
|
||||
"showThinkingHint": "Naj model pred odgovorom premisli in pokaže svoje sklepanje",
|
||||
"effortOff": "Izklopljeno",
|
||||
"effort_low": "Nizko",
|
||||
"effort_medium": "Srednje",
|
||||
"effort_xhigh": "Zelo visoko"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "у току…",
|
||||
"showThinking": "Резоновање",
|
||||
"showThinkingHint": "Затражи да модел размисли пре одговора и прикажи његово резоновање"
|
||||
"showThinkingHint": "Затражи да модел размисли пре одговора и прикажи његово резоновање",
|
||||
"effortOff": "Искључено",
|
||||
"effort_low": "Ниско",
|
||||
"effort_medium": "Средње",
|
||||
"effort_xhigh": "Веома високо"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "körs…",
|
||||
"showThinking": "Resonemang",
|
||||
"showThinkingHint": "Be modellen tänka innan den svarar och visa resonemanget"
|
||||
"showThinkingHint": "Be modellen tänka innan den svarar och visa resonemanget",
|
||||
"effortOff": "Av",
|
||||
"effort_low": "Låg",
|
||||
"effort_medium": "Medel",
|
||||
"effort_xhigh": "Mycket hög"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "inaendelea…",
|
||||
"showThinking": "Ufikirio",
|
||||
"showThinkingHint": "Mwombe modeli afikirie kabla ya kujibu, na uonyeshe ufikirio wake"
|
||||
"showThinkingHint": "Mwombe modeli afikirie kabla ya kujibu, na uonyeshe ufikirio wake",
|
||||
"effortOff": "Imezimwa",
|
||||
"effort_low": "Chini",
|
||||
"effort_medium": "Wastani",
|
||||
"effort_xhigh": "Juu sana"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "çalışıyor…",
|
||||
"showThinking": "Akıl yürütme",
|
||||
"showThinkingHint": "Modelden yanıtlamadan önce düşünmesini iste ve akıl yürütmesini göster"
|
||||
"showThinkingHint": "Modelden yanıtlamadan önce düşünmesini iste ve akıl yürütmesini göster",
|
||||
"effortOff": "Kapalı",
|
||||
"effort_low": "Düşük",
|
||||
"effort_medium": "Orta",
|
||||
"effort_xhigh": "Çok yüksek"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "виконується…",
|
||||
"showThinking": "Міркування",
|
||||
"showThinkingHint": "Просити модель подумати перед відповіддю та показувати її міркування"
|
||||
"showThinkingHint": "Просити модель подумати перед відповіддю та показувати її міркування",
|
||||
"effortOff": "Вимк.",
|
||||
"effort_low": "Низьке",
|
||||
"effort_medium": "Середнє",
|
||||
"effort_xhigh": "Дуже високе"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "bajarilmoqda…",
|
||||
"showThinking": "Mulohaza",
|
||||
"showThinkingHint": "Modeldan javobdan oldin o‘ylashini so‘rang va mulohazasini ko‘rsating"
|
||||
"showThinkingHint": "Modeldan javobdan oldin o‘ylashini so‘rang va mulohazasini ko‘rsating",
|
||||
"effortOff": "O‘chiq",
|
||||
"effort_low": "Past",
|
||||
"effort_medium": "O‘rta",
|
||||
"effort_xhigh": "Juda yuqori"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "ń ṣiṣẹ́…",
|
||||
"showThinking": "Ìrònú",
|
||||
"showThinkingHint": "Béèrè pé kí àwòṣe náà ronú kí ó tó dáhùn, kí a sì fi ìrònú rẹ̀ hàn"
|
||||
"showThinkingHint": "Béèrè pé kí àwòṣe náà ronú kí ó tó dáhùn, kí a sì fi ìrònú rẹ̀ hàn",
|
||||
"effortOff": "Pa",
|
||||
"effort_low": "Kékeré",
|
||||
"effort_medium": "Aárin",
|
||||
"effort_xhigh": "Gíga gan"
|
||||
}
|
||||
|
||||
@@ -34,5 +34,9 @@
|
||||
"blockTool": "{{name}}",
|
||||
"blockRunning": "iyasebenza…",
|
||||
"showThinking": "Ukucabanga",
|
||||
"showThinkingHint": "Cela imodeli icabange ngaphambi kokuphendula, futhi ubonise ukucabanga kwayo"
|
||||
"showThinkingHint": "Cela imodeli icabange ngaphambi kokuphendula, futhi ubonise ukucabanga kwayo",
|
||||
"effortOff": "Valiwe",
|
||||
"effort_low": "Phansi",
|
||||
"effort_medium": "Maphakathi",
|
||||
"effort_xhigh": "Kuphakeme kakhulu"
|
||||
}
|
||||
|
||||
@@ -50,16 +50,20 @@ export interface StreamOptions {
|
||||
/** OpenAI tools array; omitted = no tools offered. */
|
||||
tools?: readonly unknown[];
|
||||
/**
|
||||
* Ask the model to reason, and to send that reasoning back.
|
||||
* How hard to think, or `null`/absent for not at all.
|
||||
*
|
||||
* neuron couples generation to surfacing: unless a caller says it will
|
||||
* display reasoning, `enable_thinking` defaults to false and the model
|
||||
* produces none at all. That default is right — an undisplayed think
|
||||
* block still eats the output budget — but it means the reasoning UI
|
||||
* shows nothing until the client opts in. Verified against the live
|
||||
* router: without this, `reasoning_tokens=0` on every request.
|
||||
* Two things travel together and both are needed. The
|
||||
* `x-include-thinking` header says the caller will display reasoning;
|
||||
* without it neuron couples generation to surfacing and the model
|
||||
* produces none at all (`reasoning_tokens=0` on every request, verified
|
||||
* against the live router). The `reasoning.effort` field then selects a
|
||||
* rung of the model's own ladder.
|
||||
*
|
||||
* Rung names are the model's, discovered from `/v1/models` rather than
|
||||
* hardcoded — `Qwen/Qwen3.8-27B` accepts exactly `low`, `medium` and
|
||||
* `xhigh` and its template raises on anything else.
|
||||
*/
|
||||
includeThinking?: boolean;
|
||||
reasoningEffort?: string | null;
|
||||
signal: AbortSignal;
|
||||
}
|
||||
|
||||
@@ -99,7 +103,7 @@ export async function streamChatCompletion(
|
||||
// server-side default and the behaviour every other OpenAI client
|
||||
// gets — worth preserving exactly, since this toggle exists partly
|
||||
// to demonstrate both paths.
|
||||
if (opts.includeThinking) headers["x-include-thinking"] = "true";
|
||||
if (opts.reasoningEffort) headers["x-include-thinking"] = "true";
|
||||
resp = await fetch(`${base}/v1/chat/completions`, {
|
||||
method: "POST",
|
||||
headers,
|
||||
@@ -107,6 +111,11 @@ export async function streamChatCompletion(
|
||||
model: opts.model,
|
||||
messages: opts.messages,
|
||||
...(opts.tools?.length ? { tools: opts.tools } : {}),
|
||||
// The shape neuron reads (`request.extra.reasoning.effort`).
|
||||
// Omitted entirely when off, so the request is byte-identical to
|
||||
// what a vanilla OpenAI client sends — which is the other half of
|
||||
// what this toggle demonstrates.
|
||||
...(opts.reasoningEffort ? { reasoning: { effort: opts.reasoningEffort } } : {}),
|
||||
stream: true,
|
||||
}),
|
||||
signal: ctl.signal,
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
fetchReasoningLadder,
|
||||
reasoningOptions,
|
||||
resolveSelection,
|
||||
type ReasoningRung,
|
||||
} from "./reasoningLadder";
|
||||
|
||||
const LADDER: ReasoningRung[] = [
|
||||
{ effort: "low" },
|
||||
{ effort: "medium" },
|
||||
{ effort: "xhigh", default: true },
|
||||
];
|
||||
|
||||
const BUSY: ReasoningRung[] = [
|
||||
{ effort: "low" },
|
||||
{ effort: "medium" },
|
||||
{ effort: "xhigh", default: true, unavailable_reason: "max_in_flight=8" },
|
||||
];
|
||||
|
||||
describe("reasoningOptions", () => {
|
||||
it("offers every rung on a quiet node", () => {
|
||||
const o = reasoningOptions(LADDER);
|
||||
expect(o.available.map((r) => r.effort)).toEqual(["low", "medium", "xhigh"]);
|
||||
expect(o.withheld).toEqual([]);
|
||||
});
|
||||
|
||||
it("separates a rung the deployment is withholding", () => {
|
||||
const o = reasoningOptions(BUSY);
|
||||
expect(o.available.map((r) => r.effort)).toEqual(["low", "medium"]);
|
||||
expect(o.withheld.map((r) => r.effort)).toEqual(["xhigh"]);
|
||||
});
|
||||
|
||||
it("yields nothing for a model with no ladder", () => {
|
||||
// A non-reasoning model, or a cortex too old to publish one. The
|
||||
// caller should render no control rather than an empty one.
|
||||
expect(reasoningOptions(undefined).available).toEqual([]);
|
||||
expect(reasoningOptions([]).available).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveSelection", () => {
|
||||
it("keeps a remembered rung that is still offered", () => {
|
||||
expect(resolveSelection("medium", reasoningOptions(LADDER))).toBe("medium");
|
||||
});
|
||||
|
||||
it("falls back to off when the remembered rung is withheld", () => {
|
||||
// Deliberately not "promote to the next best rung": that chooses on
|
||||
// the user's behalf without telling them, and the model's own
|
||||
// default is `xhigh` — the very rung most likely to be withheld.
|
||||
expect(resolveSelection("xhigh", reasoningOptions(BUSY))).toBeNull();
|
||||
});
|
||||
|
||||
it("stays off when nothing was remembered", () => {
|
||||
expect(resolveSelection(null, reasoningOptions(LADDER))).toBeNull();
|
||||
});
|
||||
|
||||
it("stays off when the model has no ladder at all", () => {
|
||||
expect(resolveSelection("medium", reasoningOptions([]))).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("fetchReasoningLadder", () => {
|
||||
afterEach(() => vi.unstubAllGlobals());
|
||||
|
||||
function respond(body: unknown, ok = true) {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn().mockResolvedValue({ ok, json: () => Promise.resolve(body) }),
|
||||
);
|
||||
}
|
||||
|
||||
it("returns the ladder for the requested model", async () => {
|
||||
respond({
|
||||
data: [
|
||||
{ id: "other/model", reasoning_budget: [{ effort: "wrong" }] },
|
||||
{ id: "Qwen/Qwen3.8-27B", reasoning_budget: BUSY },
|
||||
],
|
||||
});
|
||||
const rungs = await fetchReasoningLadder("https://x.invalid", "Qwen/Qwen3.8-27B");
|
||||
expect(rungs.map((r) => r.effort)).toEqual(["low", "medium", "xhigh"]);
|
||||
expect(rungs[2].unavailable_reason).toBe("max_in_flight=8");
|
||||
});
|
||||
|
||||
it("returns nothing for a model that is not listed", async () => {
|
||||
respond({ data: [{ id: "other/model", reasoning_budget: LADDER }] });
|
||||
expect(await fetchReasoningLadder("https://x.invalid", "absent")).toEqual([]);
|
||||
});
|
||||
|
||||
it("returns nothing for a model with no ladder published", async () => {
|
||||
respond({ data: [{ id: "m" }] });
|
||||
expect(await fetchReasoningLadder("https://x.invalid", "m")).toEqual([]);
|
||||
});
|
||||
|
||||
describe("degrades quietly rather than surfacing an error", () => {
|
||||
// The reasoning control is an enhancement. A chat that works without
|
||||
// it beats an error banner about a capability nobody asked for.
|
||||
it("on a non-2xx response", async () => {
|
||||
respond({}, false);
|
||||
expect(await fetchReasoningLadder("https://x.invalid", "m")).toEqual([]);
|
||||
});
|
||||
|
||||
it("on a network failure", async () => {
|
||||
vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("offline")));
|
||||
expect(await fetchReasoningLadder("https://x.invalid", "m")).toEqual([]);
|
||||
});
|
||||
|
||||
it("on a body that is not the expected shape", async () => {
|
||||
respond({ unexpected: true });
|
||||
expect(await fetchReasoningLadder("https://x.invalid", "m")).toEqual([]);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,95 @@
|
||||
/**
|
||||
* The reasoning rungs a model currently offers, read from `/v1/models`.
|
||||
*
|
||||
* Deliberately not a hardcoded `["low","medium","xhigh"]`. The rungs come
|
||||
* from the model's own chat template (#290) — `Qwen/Qwen3.8-27B` accepts
|
||||
* exactly those three and raises on anything else, but a different model
|
||||
* spells its ladder differently, and a list frozen into the SPA rots the
|
||||
* first time the catalogue changes. The server already publishes the
|
||||
* truth; this reads it.
|
||||
*
|
||||
* Availability is also the server's to decide. A node running many
|
||||
* concurrent slots withholds its deepest rung, and it reports that rather
|
||||
* than the SPA inferring it from a `max_in_flight` it would have to be
|
||||
* told separately.
|
||||
*/
|
||||
|
||||
/** One rung as `/v1/models` publishes it. */
|
||||
export interface ReasoningRung {
|
||||
effort: string;
|
||||
default?: boolean;
|
||||
/** Present when the deployment is not currently offering this rung. */
|
||||
unavailable_reason?: string;
|
||||
}
|
||||
|
||||
/** What the picker should show for a model, in ladder order. */
|
||||
export interface ReasoningOptions {
|
||||
/** Rungs that can be selected right now. */
|
||||
available: ReasoningRung[];
|
||||
/** Rungs the deployment is withholding, with its stated reason. */
|
||||
withheld: ReasoningRung[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Split a model's advertised ladder into offerable and withheld.
|
||||
*
|
||||
* A model with no ladder — a non-reasoning model, or a cortex too old to
|
||||
* publish one — yields nothing, and the caller should offer no reasoning
|
||||
* control at all rather than a control that cannot work.
|
||||
*/
|
||||
export function reasoningOptions(rungs: readonly ReasoningRung[] | undefined): ReasoningOptions {
|
||||
const all = rungs ?? [];
|
||||
return {
|
||||
available: all.filter((r) => !r.unavailable_reason),
|
||||
withheld: all.filter((r) => !!r.unavailable_reason),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The rung to preselect: the caller's remembered choice when it is still
|
||||
* offered, otherwise nothing.
|
||||
*
|
||||
* Falling back to the model's own `default` would be wrong here. That
|
||||
* default is what the model applies when a caller names no effort at all,
|
||||
* and on Qwen3.8 it is `xhigh` — the rung most likely to be withheld. A
|
||||
* picker that silently promoted a remembered `xhigh` to whatever remains
|
||||
* would be choosing on the user's behalf without saying so; better to
|
||||
* show the reasoning control as off and let them pick again.
|
||||
*/
|
||||
export function resolveSelection(
|
||||
remembered: string | null,
|
||||
options: ReasoningOptions,
|
||||
): string | null {
|
||||
if (!remembered) return null;
|
||||
return options.available.some((r) => r.effort === remembered) ? remembered : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the ladder for one model.
|
||||
*
|
||||
* Failure is not an error worth surfacing: the reasoning control is an
|
||||
* enhancement, and a chat that works without it is better than an error
|
||||
* banner about a capability nobody asked for yet. Returns an empty
|
||||
* ladder, which renders as no control.
|
||||
*/
|
||||
export async function fetchReasoningLadder(
|
||||
baseUrl: string,
|
||||
model: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<ReasoningRung[]> {
|
||||
try {
|
||||
const resp = await fetch(`${baseUrl.replace(/\/$/, "")}/v1/models`, { signal });
|
||||
if (!resp.ok) return [];
|
||||
const body: unknown = await resp.json();
|
||||
const data = (body as { data?: unknown })?.data;
|
||||
if (!Array.isArray(data)) return [];
|
||||
const entry = data.find(
|
||||
(m): m is { id: string; reasoning_budget?: ReasoningRung[] } =>
|
||||
typeof (m as { id?: unknown })?.id === "string" && (m as { id: string }).id === model,
|
||||
);
|
||||
const rungs = entry?.reasoning_budget;
|
||||
return Array.isArray(rungs) ? rungs.filter((r) => typeof r?.effort === "string") : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -59,8 +59,8 @@ export function useChat(opts: {
|
||||
* `true` unconditionally. Also selects the tool-aware system prompt.
|
||||
*/
|
||||
toolsEnabled: boolean;
|
||||
/** Ask the model to reason, and show that reasoning. */
|
||||
includeThinking?: boolean;
|
||||
/** Rung of the model's reasoning ladder, or null for off. */
|
||||
reasoningEffort?: string | null;
|
||||
}): UseChat {
|
||||
const [streaming, setStreaming] = useState(false);
|
||||
const [activity, setActivity] = useState<ToolActivity | null>(null);
|
||||
@@ -179,7 +179,7 @@ export function useChat(opts: {
|
||||
apiKey: opts.apiKey,
|
||||
model: opts.model,
|
||||
messages: reqMessages,
|
||||
includeThinking: opts.includeThinking,
|
||||
reasoningEffort: opts.reasoningEffort,
|
||||
tools: offerTools ? [WEB_SEARCH_TOOL, READ_PAGE_TOOL] : undefined,
|
||||
signal: controller.signal,
|
||||
},
|
||||
|
||||
@@ -15,6 +15,12 @@ import {
|
||||
} from "react-icons/lu";
|
||||
import Markdown from "../components/Markdown";
|
||||
import { TurnBlocks } from "../components/TurnBlocks";
|
||||
import {
|
||||
fetchReasoningLadder,
|
||||
reasoningOptions,
|
||||
resolveSelection,
|
||||
type ReasoningRung,
|
||||
} from "../lib/reasoningLadder";
|
||||
import { CHAT_API_KEY, db } from "../data/db";
|
||||
import {
|
||||
archiveProject,
|
||||
@@ -33,7 +39,8 @@ import { ensureChatKey } from "../lib/ensureChatKey";
|
||||
import { accountApi } from "../api/account";
|
||||
|
||||
/** Persisted across reloads so a demo keeps whatever was last shown. */
|
||||
const THINKING_PREF_KEY = "helexa.chat.showThinking";
|
||||
const THINKING_PREF_KEY = "helexa.chat.reasoningEffort";
|
||||
const ROUTER_BASE = import.meta.env.VITE_ROUTER_BASE_URL || "";
|
||||
|
||||
const ANON_MODEL = import.meta.env.VITE_ANON_MODEL || "helexa/small";
|
||||
const AUTH_MODEL = import.meta.env.VITE_DEFAULT_MODEL || "helexa/balanced";
|
||||
@@ -161,35 +168,54 @@ export default function Chat() {
|
||||
[],
|
||||
);
|
||||
|
||||
// Reasoning is opt-in per #304's showcase argument: neuron only
|
||||
// Reasoning effort is opt-in and server-advertised. neuron only
|
||||
// *generates* reasoning for callers that say they will display it, so
|
||||
// this toggle switches a real server-side behaviour rather than just a
|
||||
// rendering preference. Both paths are worth demonstrating — an
|
||||
// integrator evaluating helexa wants to see the reasoning path and the
|
||||
// clean-output path that every vanilla OpenAI client gets.
|
||||
const [showThinking, setShowThinking] = useState<boolean>(() => {
|
||||
// this picks a real server-side behaviour rather than a rendering
|
||||
// preference — and both paths are worth demonstrating to an integrator:
|
||||
// the reasoning path, and the clean-output path every vanilla OpenAI
|
||||
// client gets.
|
||||
//
|
||||
// The rungs are the model's own (#290) and their availability is the
|
||||
// deployment's; both are read from `/v1/models` rather than hardcoded,
|
||||
// so a busy node withholding its deepest rung simply stops offering it.
|
||||
const [ladder, setLadder] = useState<ReasoningRung[]>([]);
|
||||
const [effort, setEffort] = useState<string | null>(() => {
|
||||
try {
|
||||
return window.localStorage.getItem(THINKING_PREF_KEY) === "1";
|
||||
return window.localStorage.getItem(THINKING_PREF_KEY) || null;
|
||||
} catch {
|
||||
// Private windows and blocked site data throw on access, not on
|
||||
// read — default off rather than taking the page down with it.
|
||||
return false;
|
||||
return null;
|
||||
}
|
||||
});
|
||||
const options = useMemo(() => reasoningOptions(ladder), [ladder]);
|
||||
// Derived, not synchronised. A remembered rung the deployment has since
|
||||
// withdrawn resolves to off at render, so the picker can never show a
|
||||
// selection it cannot honour — and there is no effect writing state
|
||||
// back, which is both a cascading-render hazard and the lint rule that
|
||||
// once let a defect reach main.
|
||||
const effectiveEffort = useMemo(() => resolveSelection(effort, options), [effort, options]);
|
||||
|
||||
useEffect(() => {
|
||||
const ctl = new AbortController();
|
||||
void fetchReasoningLadder(ROUTER_BASE, model, ctl.signal).then(setLadder);
|
||||
return () => ctl.abort();
|
||||
}, [model]);
|
||||
|
||||
useEffect(() => {
|
||||
try {
|
||||
window.localStorage.setItem(THINKING_PREF_KEY, showThinking ? "1" : "0");
|
||||
window.localStorage.setItem(THINKING_PREF_KEY, effort ?? "");
|
||||
} catch {
|
||||
/* preference is a convenience; losing it is not worth an error */
|
||||
}
|
||||
}, [showThinking]);
|
||||
}, [effort]);
|
||||
|
||||
const { streaming, activity, error, send, stop } = useChat({
|
||||
model,
|
||||
apiKey: authed ? (chatApiKey ?? undefined) : undefined,
|
||||
locale: i18n.language,
|
||||
toolsEnabled: authed || anonWebSearch,
|
||||
includeThinking: showThinking,
|
||||
reasoningEffort: effectiveEffort,
|
||||
});
|
||||
const [draft, setDraft] = useState("");
|
||||
const threadRef = useRef<HTMLDivElement>(null);
|
||||
@@ -524,14 +550,25 @@ export default function Chat() {
|
||||
}
|
||||
}}
|
||||
/>
|
||||
<label className="hx-thinking-toggle" title={t("chat:showThinkingHint")}>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={showThinking}
|
||||
onChange={(e) => setShowThinking(e.target.checked)}
|
||||
/>
|
||||
<span>{t("chat:showThinking")}</span>
|
||||
</label>
|
||||
{/* Rendered only when the deployment offers rungs — a control
|
||||
* that cannot work is worse than no control. */}
|
||||
{options.available.length > 0 && (
|
||||
<label className="hx-thinking-toggle" title={t("chat:showThinkingHint")}>
|
||||
<span className="hx-thinking-label">{t("chat:showThinking")}</span>
|
||||
<select
|
||||
className="hx-thinking-select"
|
||||
value={effectiveEffort ?? ""}
|
||||
onChange={(e) => setEffort(e.target.value || null)}
|
||||
>
|
||||
<option value="">{t("chat:effortOff")}</option>
|
||||
{options.available.map((r) => (
|
||||
<option key={r.effort} value={r.effort}>
|
||||
{t(`chat:effort_${r.effort}`, { defaultValue: r.effort })}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
)}
|
||||
{streaming ? (
|
||||
<button
|
||||
type="button"
|
||||
|
||||
@@ -111,6 +111,15 @@ export default function Docs() {
|
||||
return first ? <Navigate to={`/docs/${first}`} replace /> : null;
|
||||
}
|
||||
|
||||
// A section on its own — `/docs/operating` — is a directory rather than
|
||||
// a page. It is reachable: the prerenderer writes an index there, and
|
||||
// trimming a URL back to its parent is a thing people do. Send it to
|
||||
// the section's first page rather than claiming it does not exist.
|
||||
const section = tree.find((s) => s.id === slug);
|
||||
if (section?.pages.length) {
|
||||
return <Navigate to={`/docs/${section.pages[0].slug}`} replace />;
|
||||
}
|
||||
|
||||
if (!doc) {
|
||||
return (
|
||||
<main className="container py-5">
|
||||
|
||||
@@ -116,3 +116,59 @@ function flatten(node: React.ReactNode): string {
|
||||
export function slugs(): string[] {
|
||||
return allDocSlugs();
|
||||
}
|
||||
|
||||
/**
|
||||
* Index pages for the directories that pages live under.
|
||||
*
|
||||
* Prerendering creates real directories on disk, and a real directory
|
||||
* shadows the client-side route of the same name: nginx's
|
||||
* `try_files $uri $uri/ /index.html` matches `$uri/`, looks for an index
|
||||
* inside it, finds none, and answers **403** — it never falls through to
|
||||
* the SPA shell. So `/docs` and each section 403'd while the pages
|
||||
* beneath them served fine.
|
||||
*
|
||||
* Every directory the prerenderer creates therefore needs an index. They
|
||||
* are useful in their own right — a crawler or a reader without
|
||||
* JavaScript gets a table of contents rather than a redirect — and the
|
||||
* SPA replaces them on mount as it does every other prerendered page.
|
||||
*/
|
||||
export function indexes(): { path: string; title: string; html: string }[] {
|
||||
const tree = docTree();
|
||||
const pages = tree.flatMap((s) => s.pages);
|
||||
|
||||
// Every ancestor directory of every page, derived from the slugs rather
|
||||
// than from the section list — a page nested deeper than one level
|
||||
// creates intermediate directories too, and missing one of those is the
|
||||
// same 403 with a less obvious cause.
|
||||
const dirs = new Set<string>([""]);
|
||||
for (const p of pages) {
|
||||
const parts = p.slug.split("/");
|
||||
for (let i = 1; i < parts.length; i++) {
|
||||
dirs.add(parts.slice(0, i).join("/"));
|
||||
}
|
||||
}
|
||||
|
||||
const labelFor = (dir: string): string => {
|
||||
const section = tree.find((s) => s.id === dir);
|
||||
if (section) return section.label;
|
||||
const last = dir.split("/").pop() ?? dir;
|
||||
return last.charAt(0).toUpperCase() + last.slice(1).replace(/[-_]/g, " ");
|
||||
};
|
||||
|
||||
return [...dirs].sort().map((dir) => {
|
||||
const prefix = dir ? `${dir}/` : "";
|
||||
const children = pages.filter((p) => p.slug.startsWith(prefix));
|
||||
const items = children
|
||||
.map(
|
||||
(p) =>
|
||||
`<li><a href="/docs/${p.slug}">${escapeHtml(p.sidebarLabel)}</a></li>`,
|
||||
)
|
||||
.join("");
|
||||
const title = dir ? labelFor(dir) : "Documentation";
|
||||
return {
|
||||
path: dir,
|
||||
title,
|
||||
html: `${navHtml("")}<article><h1>${escapeHtml(title)}</h1><ul>${items}</ul></article>`,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
@@ -453,6 +453,84 @@ for web_host in "${web_hosts[@]}"; do
|
||||
fi
|
||||
done
|
||||
|
||||
# ── angels.internal: internal name for the investor portal ─────────────
|
||||
# For networks where the public name does not work from inside — served by
|
||||
# the edge nearest helexa-angels, with an internal-CA cert renewed by
|
||||
# step@angels.timer. Needs a matching DNS record on the local resolver.
|
||||
angels_int_host="oolon.kosherinata.internal"
|
||||
angels_int_domain="angels.internal"
|
||||
angels_int_cert="/etc/nginx/tls/cert/${angels_int_domain}.pem"
|
||||
angels_int_key="/etc/nginx/tls/key/${angels_int_domain}.pem"
|
||||
|
||||
echo "==> ${angels_int_host}: internal vhost (${angels_int_domain})"
|
||||
ssh "${angels_int_host}" "
|
||||
set -eu
|
||||
sudo install -d -o root -g root -m 0755 /etc/nginx/tls/cert
|
||||
sudo install -d -o root -g root -m 0700 /etc/nginx/tls/key
|
||||
"
|
||||
if ! ssh "${angels_int_host}" "sudo test -f '${angels_int_cert}'"; then
|
||||
prov_pw_local="${HOME}/.step/secrets/provisioner"
|
||||
prov_pw_remote="/root/.helexa-provisioner-pw"
|
||||
if [[ -f "${prov_pw_local}" ]]; then
|
||||
echo " issuing ${angels_int_domain} cert (JWK 'lair' provisioner)…"
|
||||
if rsync --archive --chown root:root --chmod 0600 --rsync-path 'sudo rsync' \
|
||||
"${prov_pw_local}" "${angels_int_host}:${prov_pw_remote}"; then
|
||||
ssh "${angels_int_host}" "
|
||||
trap 'sudo rm -f ${prov_pw_remote}' EXIT
|
||||
sudo step ca certificate ${angels_int_domain} ${angels_int_cert} ${angels_int_key} \
|
||||
--ca-url https://ca.internal \
|
||||
--root /etc/pki/ca-trust/source/anchors/root-internal.pem \
|
||||
--provisioner lair \
|
||||
--provisioner-password-file ${prov_pw_remote} \
|
||||
--force
|
||||
" || echo " WARNING: cert issuance failed — review on ${angels_int_host}"
|
||||
ssh "${angels_int_host}" "sudo rm -f ${prov_pw_remote}"
|
||||
else
|
||||
echo " failed to transfer provisioner secret to ${angels_int_host}"
|
||||
fi
|
||||
else
|
||||
echo " NOTE: no provisioner secret at ${prov_pw_local}; issue ${angels_int_domain} cert manually."
|
||||
fi
|
||||
fi
|
||||
# Only wire the vhost once its cert exists. Without this guard a host
|
||||
# with no provisioner secret takes the "issue it manually" branch above
|
||||
# and then installs a vhost naming a certificate that is not there:
|
||||
# `ln -sf` runs before `nginx -t`, so the broken config is already in
|
||||
# sites-enabled when the test fails, and the next reload or reboot —
|
||||
# including the renewal timer enabled two lines down — fails on it. The
|
||||
# helexa.internal block above guards exactly this way; this one did not.
|
||||
if ! ssh "${angels_int_host}" "sudo test -f '${angels_int_cert}'"; then
|
||||
echo " ${angels_int_domain} cert still absent — vhost not installed"
|
||||
elif
|
||||
# The rate-limit drop-in first: this vhost references the
|
||||
# `angels_invite` and `angels_auth` zones, which are http-context
|
||||
# names defined there. It was only shipped to this host later, by the
|
||||
# public-portal loop below, so on a host that had not run that yet
|
||||
# `nginx -t` failed on an undefined zone.
|
||||
rsync --archive --compress --chown root:root --chmod 0644 \
|
||||
--rsync-path 'sudo rsync' \
|
||||
"${repo_path}/asset/nginx/helexa-ratelimit.conf" \
|
||||
"${angels_int_host}:/etc/nginx/conf.d/helexa-ratelimit.conf" \
|
||||
|| echo " failed to install helexa-ratelimit.conf"
|
||||
rsync --archive --compress --chown root:root --chmod 0644 \
|
||||
--rsync-path 'sudo rsync' \
|
||||
"${repo_path}/asset/nginx/${angels_int_domain}.conf" \
|
||||
"${angels_int_host}:/etc/nginx/sites-available/${angels_int_domain}.conf"
|
||||
then
|
||||
ssh "${angels_int_host}" "
|
||||
set -eu
|
||||
sudo ln -sf ../sites-available/${angels_int_domain}.conf \
|
||||
/etc/nginx/sites-enabled/${angels_int_domain}.conf
|
||||
sudo nginx -t
|
||||
sudo systemctl enable --now step@angels.timer
|
||||
" && echo " vhost installed + renewal timer enabled"
|
||||
if ssh "${angels_int_host}" "systemctl is-active --quiet nginx"; then
|
||||
ssh "${angels_int_host}" "sudo systemctl reload nginx"
|
||||
fi
|
||||
else
|
||||
echo " failed to install ${angels_int_domain} vhost"
|
||||
fi
|
||||
|
||||
# ── angels.helexa.ai: the confidential investor portal ─────────────────
|
||||
# Unlike the other vhosts this one serves NOTHING from disk: every byte
|
||||
# comes from helexa-angels on gallumbits, which assembles each page
|
||||
|
||||
Reference in New Issue
Block a user