Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0705768f24
|
@@ -105,11 +105,6 @@ jobs:
|
||||
echo "commit_timestamp=${COMMIT_TIMESTAMP}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- id: changes
|
||||
env:
|
||||
# The push's own range. Empty on workflow_dispatch and all
|
||||
# zeroes for a new branch; `decide` treats both as unusable and
|
||||
# falls back to the full gate.
|
||||
PUSH_BEFORE: ${{ github.event.before }}
|
||||
run: |
|
||||
set -ux
|
||||
# Default: build everything. Detection only ever narrows
|
||||
@@ -194,30 +189,9 @@ jobs:
|
||||
BUILD_TOOLS=$(decide "$TOOLS_BASE" "$TOOLS_RE")
|
||||
BUILD_ANGELS=$(decide "$ANGELS_BASE" "$ANGELS_RE")
|
||||
if [ "$BUILD_CORTEX" = "true" ] || [ "$BUILD_NEURON" = "true" ] || [ "$BUILD_BENCH" = "true" ] || [ "$BUILD_UPSTREAM" = "true" ] || [ "$BUILD_ROUTER" = "true" ] || [ "$BUILD_TOOLS" = "true" ] || [ "$BUILD_ANGELS" = "true" ]; then
|
||||
# Anything we are about to build gets linted and tested.
|
||||
# This is the invariant that matters: no artefact reaches
|
||||
# the fleet unchecked, however the branch below behaves.
|
||||
CHECK_RUST=true
|
||||
else
|
||||
# Nothing is being built, so nothing will be published.
|
||||
# The only question left is whether THIS PUSH touched
|
||||
# Rust — and the honest base for that is the push's own
|
||||
# range, not a package's publication lag.
|
||||
#
|
||||
# This used to ask "has anything Rust changed since
|
||||
# cortex was last published". Cortex is only rebuilt when
|
||||
# cortex paths change, so after a run of neuron-only work
|
||||
# its base sits days behind and the diff from it sweeps in
|
||||
# every unrelated Rust commit since. A one-character
|
||||
# whitespace fix to a doc then re-ran fmt, clippy and the
|
||||
# full test suite over a delta that had already been
|
||||
# linted on the branch that introduced it (2026-08-18,
|
||||
# base fec5b33).
|
||||
#
|
||||
# `decide` returns true for an unusable base, so a new
|
||||
# branch (all-zero before-sha) or a force-push still gets
|
||||
# the full gate.
|
||||
CHECK_RUST=$(decide "$PUSH_BEFORE" "$RUST_RE")
|
||||
CHECK_RUST=$(decide "$CORTEX_BASE" "$RUST_RE")
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
@@ -262,7 +236,7 @@ jobs:
|
||||
# fast instead of triggering a slower uncached rebuild; only a real
|
||||
# sccache fault drops the cache. See script/ci-cargo-escalate.sh.
|
||||
- name: Clippy (sccache escalation)
|
||||
run: script/ci-cargo-escalate.sh cargo clippy --workspace --all-targets -- -D warnings
|
||||
run: script/ci-cargo-escalate.sh cargo clippy --workspace -- -D warnings
|
||||
|
||||
test:
|
||||
name: Test
|
||||
|
||||
+1
-173
@@ -39,201 +39,31 @@ env:
|
||||
CARGO_TARGET_DIR: target-${{ github.job }}
|
||||
|
||||
jobs:
|
||||
# Two independent questions: does this change need the Rust gate, and
|
||||
# does it need the web gate.
|
||||
#
|
||||
# The SPA under helexa.ai/ is built and shipped by its own deploy job and
|
||||
# is not read by any crate, build script or spec file — so a change
|
||||
# confined to it cannot alter a Rust result. Waiting up to 70 minutes for
|
||||
# a CUDA type-check to prove that is pure latency, and the wait is what
|
||||
# tempts people to merge front-end work unchecked. Documentation is inert
|
||||
# the same way, and more obviously: no crate `include_str!`s a markdown
|
||||
# file, no spec packages `doc/`, no build script reads one.
|
||||
#
|
||||
# Conversely a Rust-only change cannot break the SPA, so it need not
|
||||
# spend three minutes on `npm ci` + lint + typecheck + build.
|
||||
#
|
||||
# The classification stays deliberately one-directional: it only ever
|
||||
# narrows the gate when *every* changed path is known-inert, and every
|
||||
# other outcome — a tag, an unknown base, an empty diff, a mixed
|
||||
# changeset — falls through to the full gate. A new top-level directory
|
||||
# is therefore covered on the day it appears, without anyone
|
||||
# remembering to add it here. Adding an inert pattern is the only way to
|
||||
# widen what gets skipped, which is the property worth keeping.
|
||||
changes:
|
||||
name: Classify changes
|
||||
timeout-minutes: 10
|
||||
runs-on: fedora-43
|
||||
outputs:
|
||||
rust: ${{ steps.classify.outputs.rust }}
|
||||
web: ${{ steps.classify.outputs.web }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: classify
|
||||
run: |
|
||||
set -u
|
||||
full() {
|
||||
echo "→ full gate: $1"
|
||||
echo "rust=true" >> "$GITHUB_OUTPUT"
|
||||
echo "web=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
}
|
||||
|
||||
# Tags build the SRPMs from the whole tree, and the srpm jobs
|
||||
# gate on fmt/clippy/test/cuda-check. Never narrow there.
|
||||
case "${GITHUB_REF}" in
|
||||
refs/tags/*) full "tag push" ;;
|
||||
esac
|
||||
|
||||
# merge-base against main, not the push's before-sha: a new
|
||||
# branch has no before-sha, and a force-push's is unreachable.
|
||||
base=""
|
||||
if git fetch --no-tags origin main >/dev/null 2>&1; then
|
||||
base="$(git merge-base FETCH_HEAD HEAD 2>/dev/null || true)"
|
||||
fi
|
||||
[ -n "${base}" ] || full "no merge-base with main"
|
||||
|
||||
files="$(git diff --name-only "${base}" HEAD)"
|
||||
[ -n "${files}" ] || full "empty diff against ${base}"
|
||||
|
||||
echo "changed files:"
|
||||
printf '%s\n' "${files}" | sed 's/^/ /'
|
||||
|
||||
# Inert for Rust: the SPA, documentation, and fleet config.
|
||||
# Markdown is matched only at the repo root and under doc/ — a
|
||||
# .md inside crates/ is left to fall through, since
|
||||
# `include_str!` could make one load-bearing without this file
|
||||
# knowing.
|
||||
#
|
||||
# Fleet config (models.toml, asset/neuron/<host>.toml,
|
||||
# asset/helexa-bench/<host>.toml) is read by a *running*
|
||||
# daemon, never by a build script or an `include_str!`, so it
|
||||
# cannot change a Rust result. Deploying one is a config rsync
|
||||
# plus a service restart — neuron re-reads it and reloads the
|
||||
# model — so spending ~25 minutes on fmt, clippy, test and a
|
||||
# CUDA type-check to prove a one-line TOML edit compiles is
|
||||
# pure latency (#291).
|
||||
#
|
||||
# The `config` job below stays ungated and still validates
|
||||
# these files, so narrowing here removes the meritless work
|
||||
# without removing the check that matters. build-prerelease
|
||||
# already reaches the same conclusion on main: none of its
|
||||
# per-package path regexes match these files, so a config-only
|
||||
# push builds no binaries.
|
||||
rust_relevant="$(printf '%s\n' "${files}" \
|
||||
| grep -v '^helexa\.ai/' \
|
||||
| grep -v '^doc/' \
|
||||
| grep -vE '^[^/]+\.md$' \
|
||||
| grep -vE '^asset/neuron/[^/]+\.toml$' \
|
||||
| grep -vE '^asset/helexa-bench/[^/]+\.toml$' \
|
||||
| grep -vE '^models\.toml$' \
|
||||
|| true)"
|
||||
if [ -n "${rust_relevant}" ]; then
|
||||
echo "rust=true" >> "$GITHUB_OUTPUT"
|
||||
echo "→ Rust gate: required"
|
||||
else
|
||||
echo "rust=false" >> "$GITHUB_OUTPUT"
|
||||
echo "→ Rust gate: skipped (front-end and/or docs only)"
|
||||
fi
|
||||
|
||||
# The web gate is needed only when the SPA itself changed.
|
||||
if printf '%s\n' "${files}" | grep -q '^helexa\.ai/'; then
|
||||
echo "web=true" >> "$GITHUB_OUTPUT"
|
||||
echo "→ web gate: required"
|
||||
else
|
||||
echo "web=false" >> "$GITHUB_OUTPUT"
|
||||
echo "→ web gate: skipped (nothing under helexa.ai/)"
|
||||
fi
|
||||
|
||||
fmt:
|
||||
name: Format
|
||||
timeout-minutes: 15
|
||||
runs-on: rust
|
||||
needs: changes
|
||||
if: needs.changes.outputs.rust == 'true'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: cargo fmt --check --all
|
||||
|
||||
# models.toml and asset/neuron/<host>.toml both describe how a model is
|
||||
# served, and until #283 nothing forced them to agree. #252 is the bill
|
||||
# for that: models.toml omitted `quant`, so a cortex cold-load served
|
||||
# bf16 and could not prefill, while the same model was fine when the
|
||||
# host loaded it from its own config. The sampling override (#283) is a
|
||||
# second field with the same hazard, so the gate lands with it.
|
||||
#
|
||||
# Deliberately ungated by the `changes` classifier: it is a
|
||||
# sub-second Python run with no build, and the whole point is that it
|
||||
# cannot be skipped on the push that introduces the drift. A config
|
||||
# change is neither `rust` nor `web`, so gating it on either would let
|
||||
# a pure-TOML push through unchecked — precisely the push that breaks
|
||||
# this.
|
||||
config:
|
||||
name: Config consistency
|
||||
timeout-minutes: 5
|
||||
runs-on: fedora-43
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: python3 script/check-config-consistency.py
|
||||
|
||||
web:
|
||||
name: Web (lint + typecheck + i18n + build)
|
||||
timeout-minutes: 20
|
||||
runs-on: fedora-43
|
||||
needs: changes
|
||||
if: needs.changes.outputs.web == 'true'
|
||||
defaults:
|
||||
run:
|
||||
working-directory: helexa.ai
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "20"
|
||||
- run: npm ci
|
||||
# eslint and tsc previously ran nowhere. The deploy job builds the
|
||||
# SPA, but only on main and only after a merge — so a defect landed
|
||||
# first and broke the deploy afterwards. A setState-in-effect error
|
||||
# reached main exactly that way.
|
||||
- run: npm run lint
|
||||
- run: npm run typecheck
|
||||
# 42 locales with strict key parity is a promise that breaks
|
||||
# quietly. Adding an English string without translating it must fail
|
||||
# here, not in review, and certainly not in front of somebody
|
||||
# evaluating the project in their own language.
|
||||
- run: npm run i18n:check
|
||||
- run: npm run i18n:meta
|
||||
- run: npm run i18n:lang-labels
|
||||
- run: npm run build
|
||||
|
||||
clippy:
|
||||
name: Clippy
|
||||
timeout-minutes: 25
|
||||
runs-on: rust
|
||||
needs: changes
|
||||
if: needs.changes.outputs.rust == 'true'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# Failure-aware sccache escalation lives in the shared script (kept
|
||||
# in sync with build-prerelease.yml): a signal death (rustc SIGSEGV
|
||||
# / OOM-kill) keeps the cache and fails fast instead of an uncached
|
||||
# rebuild; only a real sccache fault drops the cache.
|
||||
# --all-targets so `#[cfg(test)]` modules, tests/, benches/ and
|
||||
# examples/ are linted too (#286). Without it clippy sees only lib
|
||||
# and bin targets, so `-D warnings` never gated a single line of
|
||||
# test code — which is how two tests lost their `#[test]` attribute
|
||||
# and stopped running unnoticed.
|
||||
- name: Clippy (sccache escalation)
|
||||
run: script/ci-cargo-escalate.sh cargo clippy --workspace --all-targets -- -D warnings
|
||||
run: script/ci-cargo-escalate.sh cargo clippy --workspace -- -D warnings
|
||||
|
||||
test:
|
||||
name: Test
|
||||
timeout-minutes: 25
|
||||
runs-on: rust
|
||||
needs: changes
|
||||
if: needs.changes.outputs.rust == 'true'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# See script/ci-cargo-escalate.sh for the escalation rationale.
|
||||
@@ -256,8 +86,6 @@ jobs:
|
||||
# sccache + the cargo target cache absorb it afterwards.
|
||||
timeout-minutes: 70
|
||||
runs-on: cuda-13.0
|
||||
needs: changes
|
||||
if: needs.changes.outputs.rust == 'true'
|
||||
# The workflow-level env sets `RUSTC_WRAPPER: sccache`
|
||||
# unconditionally, which hard-fails cargo if the CUDA image
|
||||
# doesn't ship sccache. Clear it at job level; the "Enable
|
||||
|
||||
+32
-292
@@ -37,31 +37,7 @@ env:
|
||||
${{ secrets.RSYNC_SSH_KEY }}
|
||||
|
||||
jobs:
|
||||
# Gate the whole rollout on the two config files agreeing about every
|
||||
# model they both describe. models.toml (cortex, cold loads) and
|
||||
# asset/neuron/<host>.toml (resident models) each say how a model is
|
||||
# served — quant, and since #283 the sampling override. When they
|
||||
# disagree the model is served one way cold and another way resident,
|
||||
# which is #252.
|
||||
#
|
||||
# Deliberately a *gate*, not a step inside deploy-cortex: an
|
||||
# inconsistent pair must stop the neuron rollout too, and stopping it
|
||||
# before anything restarts is much cheaper than after (a beast restart
|
||||
# costs ~5-6 min of 27B cold-load).
|
||||
config-check:
|
||||
name: Config consistency
|
||||
runs-on: fedora-43
|
||||
if: >-
|
||||
${{
|
||||
github.event_name == 'workflow_dispatch'
|
||||
|| github.event.workflow_run.conclusion == 'success'
|
||||
}}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: python3 script/check-config-consistency.py
|
||||
|
||||
deploy-cortex:
|
||||
needs: [config-check]
|
||||
runs-on: fedora-43
|
||||
# Two trigger paths: manual dispatch always runs; workflow_run
|
||||
# only runs if the upstream `build-prerelease` actually succeeded.
|
||||
@@ -71,7 +47,6 @@ jobs:
|
||||
|| github.event.workflow_run.conclusion == 'success'
|
||||
}}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: SSH init
|
||||
run: |
|
||||
mkdir -p ~/.ssh
|
||||
@@ -80,43 +55,6 @@ jobs:
|
||||
ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
|
||||
gitea_ci@hanzalova.internal 'hostname -f'
|
||||
|
||||
# The journal-capture step below needs to know whether this run
|
||||
# actually restarted the service. systemd mints a fresh
|
||||
# InvocationID on every start, so comparing before/after answers
|
||||
# that without depending on the deploy step's control flow.
|
||||
- name: Record cortex.service invocation
|
||||
id: pre
|
||||
run: |
|
||||
echo "inv=$(ssh gitea_ci@hanzalova.internal \
|
||||
'systemctl show -P InvocationID cortex.service' || true)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# The catalogue is tracked (#283) precisely so it can be rolled out
|
||||
# here rather than hand-copied. Sync it *before* the package step,
|
||||
# so the restart that step performs is the one that picks it up —
|
||||
# and report whether it changed, because a catalogue edit with no
|
||||
# new RPM must still restart cortex or the change never takes
|
||||
# effect.
|
||||
#
|
||||
# Compared by digest rather than rsync's own change detection so the
|
||||
# answer is available to the shell that decides on the restart.
|
||||
- name: Sync gateway catalogue (models.toml)
|
||||
id: catalogue
|
||||
run: |
|
||||
local_sum=$(sha256sum models.toml | cut -d' ' -f1)
|
||||
remote_sum=$(ssh gitea_ci@hanzalova.internal \
|
||||
'sha256sum /etc/cortex/models.toml 2>/dev/null | cut -d" " -f1' || true)
|
||||
if [ "${local_sum}" = "${remote_sum}" ]; then
|
||||
echo "changed=0" >> "$GITHUB_OUTPUT"
|
||||
echo "models.toml unchanged (${local_sum})"
|
||||
else
|
||||
rsync --archive --compress \
|
||||
--chown root:root --chmod 0644 \
|
||||
--rsync-path 'sudo rsync' \
|
||||
models.toml gitea_ci@hanzalova.internal:/etc/cortex/models.toml
|
||||
echo "changed=1" >> "$GITHUB_OUTPUT"
|
||||
echo "models.toml updated: ${remote_sum:-<absent>} -> ${local_sum}"
|
||||
fi
|
||||
|
||||
# Gating compares `rpm -q` against the packages.json manifest the
|
||||
# publish job maintains — NOT unprivileged `dnf check-update`,
|
||||
# which proved unreliable as the gitea_ci user (hung on metadata
|
||||
@@ -127,11 +65,6 @@ jobs:
|
||||
ssh gitea_ci@hanzalova.internal 'bash -s' <<'DEPLOY'
|
||||
set -eu
|
||||
pkg=cortex
|
||||
# Default to 0 if the sync step somehow produced no output:
|
||||
# `set -u` plus `[ "" -eq 0 ]` would abort the deploy on an
|
||||
# arithmetic error rather than on anything meaningful.
|
||||
config_changed="${{ steps.catalogue.outputs.changed }}"
|
||||
config_changed="${config_changed:-0}"
|
||||
# `rpm -q` prints "package X is not installed" on STDOUT and exits
|
||||
# 1, so `|| echo not-installed` never fires — it captures the
|
||||
# message text instead. Test the exit code.
|
||||
@@ -147,31 +80,18 @@ jobs:
|
||||
p = max(cands, key=lambda p: p.get("buildTime", 0))
|
||||
print(p["version"] + "-" + p["release"])
|
||||
' "${pkg}" 2>/dev/null || true)
|
||||
# Skip only when BOTH the package and the catalogue are
|
||||
# unchanged — a models.toml edit (a new sampling override, a
|
||||
# corrected quant) must restart cortex even with no new RPM,
|
||||
# exactly as the neuron drop-in does below.
|
||||
if [ -n "${latest}" ] && [ "${latest}" = "${installed}" ] \
|
||||
&& [ "${config_changed}" -eq 0 ]; then
|
||||
echo "${pkg}-${installed} already current; models.toml unchanged — leaving service untouched"
|
||||
if [ -n "${latest}" ] && [ "${latest}" = "${installed}" ]; then
|
||||
echo "${pkg}-${installed} already current — leaving service untouched"
|
||||
exit 0
|
||||
fi
|
||||
pkg_changed=1
|
||||
if [ -n "${latest}" ] && [ "${latest}" = "${installed}" ]; then
|
||||
pkg_changed=0
|
||||
fi
|
||||
echo "installed=${installed} published=${latest:-unknown} pkg_changed=${pkg_changed} config_changed=${config_changed} — deploying"
|
||||
echo "installed=${installed} published=${latest:-unknown} — deploying"
|
||||
if systemctl is-active --quiet cortex.service; then
|
||||
sudo /usr/bin/systemctl stop cortex.service
|
||||
fi
|
||||
# A catalogue-only change still restarts, but must not spend a
|
||||
# dnf metadata refresh proving the package is already current.
|
||||
if [ "${pkg_changed}" -eq 1 ]; then
|
||||
if rpm -q "${pkg}" >/dev/null 2>&1; then
|
||||
sudo /usr/bin/dnf upgrade --refresh --allowerasing -y cortex
|
||||
else
|
||||
sudo /usr/bin/dnf install --refresh --allowerasing -y cortex
|
||||
fi
|
||||
if rpm -q "${pkg}" >/dev/null 2>&1; then
|
||||
sudo /usr/bin/dnf upgrade --refresh --allowerasing -y cortex
|
||||
else
|
||||
sudo /usr/bin/dnf install --refresh --allowerasing -y cortex
|
||||
fi
|
||||
sudo /usr/bin/systemctl daemon-reload
|
||||
# enable --now: start the service AND enable it for boot so the
|
||||
@@ -180,39 +100,17 @@ jobs:
|
||||
DEPLOY
|
||||
|
||||
# Wait for the service to either come up or wedge, then capture
|
||||
# the journal for the invocation this run started. Runs even on
|
||||
# prior failure so a failed start step still leaves a usable
|
||||
# record in the deploy log.
|
||||
# the latest-invocation journal. Runs even on prior failure so a
|
||||
# failed start step still leaves a usable record in the deploy log.
|
||||
- name: Capture cortex.service startup journal
|
||||
if: always()
|
||||
env:
|
||||
PRE_INV: ${{ steps.pre.outputs.inv }}
|
||||
run: |
|
||||
sleep 10
|
||||
ssh gitea_ci@hanzalova.internal bash -s "${PRE_INV}" <<'CAPTURE'
|
||||
set -eu
|
||||
pre=${1:-}
|
||||
unit=cortex.service
|
||||
now=$(systemctl show -P InvocationID "${unit}")
|
||||
if [ -n "${now}" ] && [ "${now}" = "${pre}" ]; then
|
||||
echo "${unit} not restarted this run (invocation ${now}) — nothing to capture"
|
||||
systemctl status --no-pager --lines=0 "${unit}" || true
|
||||
exit 0
|
||||
fi
|
||||
# -I selects the latest invocation, which on a long-lived service
|
||||
# is unbounded — days of logs, hundreds of MB at debug level. Cap
|
||||
# the line count and drop the debug/trace firehose; startup lines
|
||||
# (and panics on stderr) are info and above. An empty invocation
|
||||
# id means the unit never came up, so drop -I and fall back to the
|
||||
# unit's recent tail, which is where the failure will be.
|
||||
timeout 60 journalctl --unit "${unit}" ${now:+-I} \
|
||||
--priority=info --lines=500 --no-pager \
|
||||
|| { echo "(no info-level entries for this invocation, or capture timed out)"
|
||||
systemctl status --no-pager --lines=0 "${unit}" || true; }
|
||||
CAPTURE
|
||||
ssh gitea_ci@hanzalova.internal \
|
||||
'journalctl --unit cortex.service -I --no-pager'
|
||||
|
||||
deploy-neurons:
|
||||
needs: [config-check, deploy-cortex]
|
||||
needs: [deploy-cortex]
|
||||
runs-on: fedora-43
|
||||
strategy:
|
||||
# One neuron failing must not cancel the others. Cortex is up
|
||||
@@ -247,7 +145,6 @@ jobs:
|
||||
load_timeout: 300
|
||||
max_prompt_tokens: 16384
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: SSH init
|
||||
run: |
|
||||
mkdir -p ~/.ssh
|
||||
@@ -256,49 +153,6 @@ jobs:
|
||||
ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
|
||||
gitea_ci@${{ matrix.host }} 'hostname -f'
|
||||
|
||||
# See deploy-cortex for why the capture step needs this.
|
||||
- name: Record neuron.service invocation
|
||||
id: pre
|
||||
run: |
|
||||
echo "inv=$(ssh gitea_ci@${{ matrix.host }} \
|
||||
'systemctl show -P InvocationID neuron.service' || true)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# asset/neuron/<short>.toml is this host's neuron.toml. It carries
|
||||
# the resident-model specs — including the #283 sampling override,
|
||||
# which is inert until the daemon reloads the model, so a change
|
||||
# here has to restart the neuron the same way the context-cap
|
||||
# drop-in does.
|
||||
#
|
||||
# Synced before the package step so the restart that step performs
|
||||
# is the one that picks it up. Digest-compared rather than trusting
|
||||
# rsync's own change detection, because the restart decision needs
|
||||
# the answer and a needless neuron restart costs ~5-6 min of 27B
|
||||
# cold-load (#1).
|
||||
- name: Sync neuron config (${{ matrix.host }})
|
||||
id: neuronconf
|
||||
run: |
|
||||
short="${{ matrix.host }}"; short="${short%%.*}"
|
||||
src="asset/neuron/${short}.toml"
|
||||
if [ ! -f "${src}" ]; then
|
||||
echo "changed=0" >> "$GITHUB_OUTPUT"
|
||||
echo "no ${src} in the repo — leaving this host's config alone"
|
||||
exit 0
|
||||
fi
|
||||
local_sum=$(sha256sum "${src}" | cut -d' ' -f1)
|
||||
remote_sum=$(ssh gitea_ci@${{ matrix.host }} \
|
||||
'sha256sum /etc/neuron/neuron.toml 2>/dev/null | cut -d" " -f1' || true)
|
||||
if [ "${local_sum}" = "${remote_sum}" ]; then
|
||||
echo "changed=0" >> "$GITHUB_OUTPUT"
|
||||
echo "${src} unchanged (${local_sum})"
|
||||
else
|
||||
rsync --archive --compress \
|
||||
--chown root:root --chmod 0644 \
|
||||
--rsync-path 'sudo rsync' \
|
||||
"${src}" gitea_ci@${{ matrix.host }}:/etc/neuron/neuron.toml
|
||||
echo "changed=1" >> "$GITHUB_OUTPUT"
|
||||
echo "${src} updated: ${remote_sum:-<absent>} -> ${local_sum}"
|
||||
fi
|
||||
|
||||
# See deploy-cortex for why gating uses the publish manifest and
|
||||
# not unprivileged `dnf check-update`.
|
||||
- name: Deploy helexa-neuron-${{ matrix.flavour }} (skips when already current)
|
||||
@@ -307,9 +161,6 @@ jobs:
|
||||
set -eu
|
||||
pkg=helexa-neuron-${{ matrix.flavour }}
|
||||
max_prompt_tokens="${{ matrix.max_prompt_tokens }}"
|
||||
# A neuron.toml change counts as a config change for the
|
||||
# restart decision below, exactly like the drop-in.
|
||||
neuron_toml_changed="${{ steps.neuronconf.outputs.changed }}"
|
||||
|
||||
# ── Desired per-model systemd drop-in ─────────────────────────
|
||||
# model.conf carries NEURON_MAX_PROMPT_TOKENS so the context cap
|
||||
@@ -318,7 +169,7 @@ jobs:
|
||||
# deploy-managed value wins over any manual local override of the
|
||||
# same variable. See doc/context-limits.md.
|
||||
conf=/etc/systemd/system/neuron.service.d/model.conf
|
||||
config_changed=${neuron_toml_changed:-0}
|
||||
config_changed=0
|
||||
if [ -n "${max_prompt_tokens}" ]; then
|
||||
desired=$(printf '%s\n%s\n%s\n%s' \
|
||||
"# Managed by .gitea/workflows/deploy.yml - do not edit by hand." \
|
||||
@@ -352,17 +203,13 @@ jobs:
|
||||
# Skip only when BOTH the package and the drop-in are unchanged —
|
||||
# a context-cap change must restart the neuron even with no new RPM.
|
||||
if [ "${pkg_changed}" -eq 0 ] && [ "${config_changed}" -eq 0 ]; then
|
||||
echo "${pkg}-${installed} current; NEURON_MAX_PROMPT_TOKENS=${max_prompt_tokens:-<unset>} and neuron.toml unchanged — leaving service untouched"
|
||||
echo "${pkg}-${installed} current; NEURON_MAX_PROMPT_TOKENS=${max_prompt_tokens:-<unset>} unchanged — leaving service untouched"
|
||||
exit 0
|
||||
fi
|
||||
echo "installed=${installed} published=${latest:-unknown} pkg_changed=${pkg_changed} config_changed=${config_changed} — deploying"
|
||||
|
||||
# Write the drop-in (staged in gitea_ci's dir, installed root-owned).
|
||||
# Guarded on max_prompt_tokens as well as config_changed: since
|
||||
# a neuron.toml change also sets config_changed, an entry with
|
||||
# no max_prompt_tokens would otherwise reach this with `desired`
|
||||
# unset.
|
||||
if [ "${config_changed}" -eq 1 ] && [ -n "${max_prompt_tokens}" ]; then
|
||||
if [ "${config_changed}" -eq 1 ]; then
|
||||
printf '%s\n' "${desired}" > /var/lib/gitea_ci/model.conf
|
||||
sudo /usr/bin/install -o root -g root -m 0644 -D /var/lib/gitea_ci/model.conf "${conf}"
|
||||
rm -f /var/lib/gitea_ci/model.conf
|
||||
@@ -494,32 +341,14 @@ jobs:
|
||||
fi'
|
||||
|
||||
# Wait for the service to either come up or wedge, then capture
|
||||
# the journal for the invocation this run started. Runs even on
|
||||
# prior failure so a failed start step still leaves a usable
|
||||
# record in the deploy log. See deploy-cortex for the bounding
|
||||
# rationale — a neuron that has not restarted in a week is the
|
||||
# most exposed of the fleet to an unbounded -I.
|
||||
# the latest-invocation journal. Runs even on prior failure so a
|
||||
# failed start step still leaves a usable record in the deploy log.
|
||||
- name: Capture neuron.service startup journal
|
||||
if: always()
|
||||
env:
|
||||
PRE_INV: ${{ steps.pre.outputs.inv }}
|
||||
run: |
|
||||
sleep 10
|
||||
ssh gitea_ci@${{ matrix.host }} bash -s "${PRE_INV}" <<'CAPTURE'
|
||||
set -eu
|
||||
pre=${1:-}
|
||||
unit=neuron.service
|
||||
now=$(systemctl show -P InvocationID "${unit}")
|
||||
if [ -n "${now}" ] && [ "${now}" = "${pre}" ]; then
|
||||
echo "${unit} not restarted this run (invocation ${now}) — nothing to capture"
|
||||
systemctl status --no-pager --lines=0 "${unit}" || true
|
||||
exit 0
|
||||
fi
|
||||
timeout 60 journalctl --unit "${unit}" ${now:+-I} \
|
||||
--priority=info --lines=500 --no-pager \
|
||||
|| { echo "(no info-level entries for this invocation, or capture timed out)"
|
||||
systemctl status --no-pager --lines=0 "${unit}" || true; }
|
||||
CAPTURE
|
||||
ssh gitea_ci@${{ matrix.host }} \
|
||||
'journalctl --unit neuron.service -I --no-pager'
|
||||
|
||||
# helexa-bench is a separate package on a separate host (bob), and it
|
||||
# only consumes the fleet's HTTP APIs — it has no deploy-ordering
|
||||
@@ -547,17 +376,6 @@ jobs:
|
||||
ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
|
||||
gitea_ci@gallumbits.kosherinata.internal 'hostname -f'
|
||||
|
||||
# See deploy-cortex for why the capture step needs this. One
|
||||
# `unit=invocation` token per service; ssh flattens the list into
|
||||
# separate argv entries on the far side.
|
||||
- name: Record service invocations
|
||||
id: pre
|
||||
run: |
|
||||
echo "inv=$(ssh gitea_ci@gallumbits.kosherinata.internal '
|
||||
for u in helexa-router helexa-upstream helexa-tools helexa-angels; do
|
||||
printf "%s=%s " "${u}" "$(systemctl show -P InvocationID ${u}.service)"
|
||||
done' || true)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Deploy helexa-router + helexa-upstream + helexa-tools + helexa-angels (skips when already current)
|
||||
run: |
|
||||
ssh gitea_ci@gallumbits.kosherinata.internal 'bash -s' <<'DEPLOY'
|
||||
@@ -619,36 +437,14 @@ jobs:
|
||||
[ "$changed" = "1" ] && sudo /usr/bin/firewall-cmd --reload || true
|
||||
'
|
||||
|
||||
# Per-unit: capture only the invocation this run started, bounded.
|
||||
# See deploy-cortex for the rationale.
|
||||
- name: Capture service startup journals
|
||||
if: always()
|
||||
env:
|
||||
PRE_INV: ${{ steps.pre.outputs.inv }}
|
||||
run: |
|
||||
sleep 10
|
||||
ssh gitea_ci@gallumbits.kosherinata.internal bash -s "${PRE_INV}" <<'CAPTURE'
|
||||
set -eu
|
||||
for spec in "$@"; do
|
||||
u=${spec%%=*}
|
||||
pre=${spec#*=}
|
||||
unit=${u}.service
|
||||
echo "── ${unit} ──"
|
||||
now=$(systemctl show -P InvocationID "${unit}")
|
||||
if [ -n "${now}" ] && [ "${now}" = "${pre}" ]; then
|
||||
echo "${unit} not restarted this run (invocation ${now}) — nothing to capture"
|
||||
systemctl status --no-pager --lines=0 "${unit}" || true
|
||||
continue
|
||||
fi
|
||||
timeout 60 journalctl --unit "${unit}" ${now:+-I} \
|
||||
--priority=info --lines=500 --no-pager \
|
||||
|| { echo "(no info-level entries for this invocation, or capture timed out)"
|
||||
systemctl status --no-pager --lines=0 "${unit}" || true; }
|
||||
done
|
||||
CAPTURE
|
||||
ssh gitea_ci@gallumbits.kosherinata.internal \
|
||||
'journalctl --unit helexa-router.service -I --no-pager; journalctl --unit helexa-upstream.service -I --no-pager; journalctl --unit helexa-tools.service -I --no-pager; journalctl --unit helexa-angels.service -I --no-pager'
|
||||
|
||||
deploy-bench:
|
||||
needs: [config-check]
|
||||
runs-on: fedora-43
|
||||
if: >-
|
||||
${{
|
||||
@@ -656,7 +452,6 @@ jobs:
|
||||
|| github.event.workflow_run.conclusion == 'success'
|
||||
}}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: SSH init
|
||||
run: |
|
||||
mkdir -p ~/.ssh
|
||||
@@ -665,35 +460,6 @@ jobs:
|
||||
ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
|
||||
gitea_ci@bob.hanzalova.internal 'hostname -f'
|
||||
|
||||
# See deploy-cortex for why the capture step needs this.
|
||||
- name: Record helexa-bench.service invocation
|
||||
id: pre
|
||||
run: |
|
||||
echo "inv=$(ssh gitea_ci@bob.hanzalova.internal \
|
||||
'systemctl show -P InvocationID helexa-bench.service' || true)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Same contract as the gateway and neuron configs: the repo is the
|
||||
# only writer, so a config edit must reach the host and restart the
|
||||
# service even when no new RPM landed.
|
||||
- name: Sync bench config
|
||||
id: benchconf
|
||||
run: |
|
||||
src="asset/helexa-bench/bob.toml"
|
||||
local_sum=$(sha256sum "${src}" | cut -d' ' -f1)
|
||||
remote_sum=$(ssh gitea_ci@bob.hanzalova.internal \
|
||||
'sha256sum /etc/helexa-bench/helexa-bench.toml 2>/dev/null | cut -d" " -f1' || true)
|
||||
if [ "${local_sum}" = "${remote_sum}" ]; then
|
||||
echo "changed=0" >> "$GITHUB_OUTPUT"
|
||||
echo "${src} unchanged (${local_sum})"
|
||||
else
|
||||
rsync --archive --compress \
|
||||
--chown root:root --chmod 0644 \
|
||||
--rsync-path 'sudo rsync' \
|
||||
"${src}" gitea_ci@bob.hanzalova.internal:/etc/helexa-bench/helexa-bench.toml
|
||||
echo "changed=1" >> "$GITHUB_OUTPUT"
|
||||
echo "${src} updated: ${remote_sum:-<absent>} -> ${local_sum}"
|
||||
fi
|
||||
|
||||
# See deploy-cortex for why gating uses the publish manifest and
|
||||
# not unprivileged `dnf check-update`.
|
||||
- name: Deploy helexa-bench (skips when already current)
|
||||
@@ -701,8 +467,6 @@ jobs:
|
||||
ssh gitea_ci@bob.hanzalova.internal 'bash -s' <<'DEPLOY'
|
||||
set -eu
|
||||
pkg=helexa-bench
|
||||
config_changed="${{ steps.benchconf.outputs.changed }}"
|
||||
config_changed="${config_changed:-0}"
|
||||
# `rpm -q` prints "package X is not installed" on STDOUT and exits
|
||||
# 1, so `|| echo not-installed` never fires — it captures the
|
||||
# message text instead. Test the exit code.
|
||||
@@ -718,25 +482,18 @@ jobs:
|
||||
p = max(cands, key=lambda p: p.get("buildTime", 0))
|
||||
print(p["version"] + "-" + p["release"])
|
||||
' "${pkg}" 2>/dev/null || true)
|
||||
if [ -n "${latest}" ] && [ "${latest}" = "${installed}" ] \
|
||||
&& [ "${config_changed}" -eq 0 ]; then
|
||||
echo "${pkg}-${installed} already current; config unchanged — leaving service untouched"
|
||||
if [ -n "${latest}" ] && [ "${latest}" = "${installed}" ]; then
|
||||
echo "${pkg}-${installed} already current — leaving service untouched"
|
||||
exit 0
|
||||
fi
|
||||
pkg_changed=1
|
||||
if [ -n "${latest}" ] && [ "${latest}" = "${installed}" ]; then
|
||||
pkg_changed=0
|
||||
fi
|
||||
echo "installed=${installed} published=${latest:-unknown} pkg_changed=${pkg_changed} config_changed=${config_changed} — deploying"
|
||||
echo "installed=${installed} published=${latest:-unknown} — deploying"
|
||||
if systemctl is-active --quiet helexa-bench.service; then
|
||||
sudo /usr/bin/systemctl stop helexa-bench.service
|
||||
fi
|
||||
if [ "${pkg_changed}" -eq 1 ]; then
|
||||
if rpm -q "${pkg}" >/dev/null 2>&1; then
|
||||
sudo /usr/bin/dnf upgrade --refresh --allowerasing -y helexa-bench
|
||||
else
|
||||
sudo /usr/bin/dnf install --refresh --allowerasing -y helexa-bench
|
||||
fi
|
||||
if rpm -q "${pkg}" >/dev/null 2>&1; then
|
||||
sudo /usr/bin/dnf upgrade --refresh --allowerasing -y helexa-bench
|
||||
else
|
||||
sudo /usr/bin/dnf install --refresh --allowerasing -y helexa-bench
|
||||
fi
|
||||
sudo /usr/bin/systemctl daemon-reload
|
||||
# enable --now: start the service AND enable it for boot so the
|
||||
@@ -777,31 +534,14 @@ jobs:
|
||||
fi'
|
||||
|
||||
# Wait for the service to either come up or wedge, then capture
|
||||
# the journal for the invocation this run started. Runs even on
|
||||
# prior failure so a failed start step still leaves a usable
|
||||
# record in the deploy log. See deploy-cortex for the bounding
|
||||
# rationale.
|
||||
# the latest-invocation journal. Runs even on prior failure so a
|
||||
# failed start step still leaves a usable record in the deploy log.
|
||||
- name: Capture helexa-bench.service startup journal
|
||||
if: always()
|
||||
env:
|
||||
PRE_INV: ${{ steps.pre.outputs.inv }}
|
||||
run: |
|
||||
sleep 10
|
||||
ssh gitea_ci@bob.hanzalova.internal bash -s "${PRE_INV}" <<'CAPTURE'
|
||||
set -eu
|
||||
pre=${1:-}
|
||||
unit=helexa-bench.service
|
||||
now=$(systemctl show -P InvocationID "${unit}")
|
||||
if [ -n "${now}" ] && [ "${now}" = "${pre}" ]; then
|
||||
echo "${unit} not restarted this run (invocation ${now}) — nothing to capture"
|
||||
systemctl status --no-pager --lines=0 "${unit}" || true
|
||||
exit 0
|
||||
fi
|
||||
timeout 60 journalctl --unit "${unit}" ${now:+-I} \
|
||||
--priority=info --lines=500 --no-pager \
|
||||
|| { echo "(no info-level entries for this invocation, or capture timed out)"
|
||||
systemctl status --no-pager --lines=0 "${unit}" || true; }
|
||||
CAPTURE
|
||||
ssh gitea_ci@bob.hanzalova.internal \
|
||||
'journalctl --unit helexa-bench.service -I --no-pager'
|
||||
|
||||
# Build the bench UI and publish it to the public nginx vhost on the
|
||||
# gateway (https://bench.helexa.ai). The vhost + Let's Encrypt cert are
|
||||
|
||||
@@ -9,6 +9,7 @@ helexa.ai/.env.local
|
||||
.idea/
|
||||
.vscode/
|
||||
cortex.toml
|
||||
models.toml
|
||||
doc/plan/*
|
||||
/target-cuda/
|
||||
.claude/
|
||||
|
||||
@@ -241,8 +241,7 @@ quant = "Q4_K_M"
|
||||
vram_mb = 19000
|
||||
min_devices = 2
|
||||
min_device_vram_mb = 10000
|
||||
pinned_on = ["beast"] # optional: affinity — run only on these
|
||||
residency_priority = 300 # optional: who may displace whom under VRAM pressure
|
||||
pinned_on = ["beast"] # optional: never evict from these neurons
|
||||
```
|
||||
|
||||
### neuron.toml (per-host)
|
||||
|
||||
@@ -167,7 +167,7 @@ doc-comment; touch points (the `Job` enum, the dispatch handlers, the
|
||||
cargo build --release # build all crates
|
||||
cargo run -p cortex-cli -- serve # run the gateway
|
||||
cargo test # run all tests
|
||||
cargo clippy --workspace --all-targets # lint (test code included)
|
||||
cargo clippy --workspace # lint
|
||||
```
|
||||
|
||||
## CI
|
||||
@@ -177,7 +177,7 @@ pass before merging:
|
||||
|
||||
```sh
|
||||
cargo fmt --check --all # formatting
|
||||
cargo clippy --workspace --all-targets -- -D warnings # lint (warnings are errors)
|
||||
cargo clippy --workspace -- -D warnings # lint (warnings are errors)
|
||||
cargo test --workspace # tests
|
||||
```
|
||||
|
||||
@@ -191,8 +191,7 @@ Work each change on its own branch; `main` stays releasable.
|
||||
|
||||
1. Implement on a feature branch (`fix/<issue>-…`, `feat/<issue>-…`).
|
||||
2. Run the CI triad locally (`cargo fmt --check --all`,
|
||||
`cargo clippy --workspace --all-targets -- -D warnings`,
|
||||
`cargo test --workspace`).
|
||||
`cargo clippy --workspace -- -D warnings`, `cargo test --workspace`).
|
||||
Local builds are **CPU-only** — the `#[cfg(feature = "cuda")]` neuron/TP
|
||||
paths do NOT compile locally. The branch CI's **CUDA type-check** job is
|
||||
the only thing that validates them, so for any neuron change the push to
|
||||
@@ -621,8 +620,7 @@ quant = "Q4_K_M"
|
||||
vram_mb = 19000
|
||||
min_devices = 2
|
||||
min_device_vram_mb = 10000
|
||||
pinned_on = ["beast"] # optional: affinity — run only on these
|
||||
residency_priority = 300 # optional: who may displace whom under VRAM pressure
|
||||
pinned_on = ["beast"] # optional: never evict from these neurons
|
||||
|
||||
[[models]]
|
||||
id = "Qwen/Qwen3-VL-8B"
|
||||
|
||||
@@ -166,16 +166,8 @@ name = "benjy"
|
||||
endpoint = "http://benjy.internal:13131"
|
||||
```
|
||||
|
||||
Model placement profiles — VRAM requirements, quant, device minimums,
|
||||
which neurons a model may run on, and what it may displace when one runs
|
||||
out of VRAM — live in `models.toml`. `models.example.toml` is the field
|
||||
reference; [placement & displacement](https://helexa.ai/docs/operating/placement)
|
||||
explains how the two fit together, and is worth reading before you set
|
||||
`residency_priority` on anything.
|
||||
|
||||
Full documentation — using helexa and operating it — is at
|
||||
[helexa.ai/docs](https://helexa.ai/docs); the source lives under
|
||||
`helexa.ai/content/docs/`.
|
||||
Model placement profiles (VRAM requirements, quant, device minimums,
|
||||
pinning) live in `models.toml` — see `models.example.toml`.
|
||||
|
||||
## Run
|
||||
|
||||
|
||||
@@ -1,10 +1,3 @@
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# MANAGED BY CI — .gitea/workflows/deploy.yml is the only writer.
|
||||
#
|
||||
# Edits made directly to the deployed copy on the host are reverted by
|
||||
# the next deploy, and until then the fleet runs something no commit
|
||||
# describes. Change it here, commit, let the pipeline ship it.
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# helexa-bench config for bob.hanzalova.internal.
|
||||
#
|
||||
# Synced to /etc/helexa-bench/helexa-bench.toml by script/infra-setup.sh
|
||||
@@ -22,20 +15,6 @@ iteration_pause_secs = 2
|
||||
request_timeout_secs = 600
|
||||
db_path = "/var/lib/helexa-bench/bench.sqlite"
|
||||
|
||||
# Identity bench presents to neurons (#288). Without it bench is
|
||||
# anonymous, and since #262 anonymous callers are served only from
|
||||
# leftover capacity — capped below max_in_flight and parked at the class
|
||||
# gate. That moved beast's concurrency:8 from 168.9 tok/s / 0.53s
|
||||
# ttft_p95 to 104.6 / 11.77s the day #262 landed and held there, so every
|
||||
# number since measured the yield policy, not serving capacity.
|
||||
#
|
||||
# A dedicated pair, not a borrowed one: bench then holds its own
|
||||
# fair-share allocation (#54) and cannot starve interactive traffic —
|
||||
# the failure #262 existed to fix.
|
||||
[bench.principal]
|
||||
account_id = "helexa-bench"
|
||||
key_id = "fleet-benchmark"
|
||||
|
||||
[scenarios]
|
||||
prompt_sizes = [128, 4096]
|
||||
max_tokens = 256
|
||||
@@ -43,10 +22,6 @@ max_tokens = 256
|
||||
# carries p95-under-concurrency data before the F3 A/B gate (#94) needs
|
||||
# it for comparison. Levels mirror the real a0/hermes/opencode fan-out.
|
||||
concurrency_levels = [2, 4, 8]
|
||||
# Non-streaming bursts (#285/#288). Kept to the top level: it is where
|
||||
# serialization shows most starkly and each level costs a full burst per
|
||||
# sample per build.
|
||||
concurrency_nonstreaming_levels = [8]
|
||||
concurrency_prompt_tokens = 512
|
||||
|
||||
# Capability probes (#91) — the reasoning/planning axis the speed
|
||||
|
||||
+2
-78
@@ -1,10 +1,3 @@
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# MANAGED BY CI — .gitea/workflows/deploy.yml is the only writer.
|
||||
#
|
||||
# Edits made directly to the deployed copy on the host are reverted by
|
||||
# the next deploy, and until then the fleet runs something no commit
|
||||
# describes. Change it here, commit, let the pipeline ship it.
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# neuron.toml for beast.hanzalova.internal
|
||||
#
|
||||
# 2x RTX 5090 (32 GB each) — TP-2 capable. Pre-warms Qwen3.6-27B with
|
||||
@@ -29,80 +22,11 @@ name = "candle"
|
||||
# count. NEURON_BATCHING=0 (systemd drop-in) is the kill switch;
|
||||
# removing this section reverts to batch-1 serialization.
|
||||
[harness.candle.admission]
|
||||
# 2, not 8 (#291). max_in_flight is both the admission bound and the
|
||||
# batch engine's slot count, and the KV budget is divided by it at load.
|
||||
# At 8 that gave each slot 18,852 tokens out of 150,816 — against a
|
||||
# 131,072-token advertised context, which needs 4 GB of KV per card on
|
||||
# its own and 32 GB across eight slots, on a 31 GB card already holding
|
||||
# ~21 GB of weights. The advertised context and the slot count were
|
||||
# incompatible by roughly 7x, and an agentic session ran in a ~19k
|
||||
# window while seven idle slots held reservations.
|
||||
#
|
||||
# Seven days of paired request spans on this host: 89.6% of busy time
|
||||
# was ONE request in flight, 93.6% was two or fewer, and utilisation was
|
||||
# 4.9%. Most of what little concurrency exists is helexa-bench's own
|
||||
# synthetic bursts. Two slots cover 93.6% of observed load and give each
|
||||
# ~75k tokens; the batch engine is still worth 1.79x there (5.24x at
|
||||
# eight is throughput nobody is asking this host for). Anything above
|
||||
# two queues on admission (#53) with Retry-After rather than being
|
||||
# refused.
|
||||
max_in_flight = 2
|
||||
|
||||
# Snapshot budget for the cross-request prefix cache (#292).
|
||||
#
|
||||
# 3 GiB, not the 1 GiB default. At the measured 32 KiB/token/card, a
|
||||
# 1 GiB budget cannot hold a snapshot beyond ~32k tokens of K/V alone —
|
||||
# less once the GatedDeltaNet conv/recurrent state is deep-copied on top,
|
||||
# which on this arch is 48 of 64 layers in f32. Agentic sessions here run
|
||||
# well past that: `cached_tokens` was observed pinned at 2,016 for ten
|
||||
# consecutive turns while the prompt grew to 52,869, so every turn
|
||||
# re-prefilled ~50k tokens — 37 s each, against ~20 s of actual decode.
|
||||
# The cache was not missing; the snapshot would not fit.
|
||||
#
|
||||
# The snapshot budget is reserved BEFORE the KV budget, not from what is
|
||||
# left over: measured on this host, kv_budget_mb = 5737 - budget_mb,
|
||||
# exactly 1:1. So this number is a direct trade against how long a
|
||||
# single session can get, and both sides have a floor:
|
||||
#
|
||||
# budget_mb kv_budget tokens/slot (2 slots) holds a 52.9k snapshot
|
||||
# 1024 4713 75,400 no ← the #292 symptom
|
||||
# 3072 2665 42,600 yes, but sessions
|
||||
# cannot reach 52.9k
|
||||
# 2048 3689 59,000 yes
|
||||
#
|
||||
# 2 GiB is the value that satisfies both: a 52,869-token snapshot is
|
||||
# ~1,652 MB of K/V plus the GatedDeltaNet state (~1.73 GB total), and
|
||||
# per-slot KV still covers sessions comfortably past the deepest
|
||||
# observed. 3 GiB was tried first and starved KV — a session could no
|
||||
# longer reach the size whose snapshot the cache was being enlarged to
|
||||
# hold, which is self-defeating.
|
||||
#
|
||||
# max_entries stays at the default 8: the byte budget binds first here,
|
||||
# so the count cap is not what is forcing snapshots to be small.
|
||||
[harness.candle.prefix_cache]
|
||||
budget_mb = 2048
|
||||
max_in_flight = 8
|
||||
|
||||
[[default_models]]
|
||||
# Qwen3.8-27B is the resident flagship (#251). Architecturally identical
|
||||
# to the 3.6-27B it replaced — same model_type, layer counts and vision
|
||||
# tower — so the load spec is unchanged. The 3.6-27B stays catalogued at
|
||||
# the same residency class, so it can still be cold-loaded on demand and
|
||||
# the two displace each other in either direction.
|
||||
model_id = "Qwen/Qwen3.8-27B"
|
||||
model_id = "Qwen/Qwen3.6-27B"
|
||||
harness = "candle"
|
||||
quant = "q6k"
|
||||
tensor_parallel = 2
|
||||
devices = [0, 1]
|
||||
|
||||
# Sampling override (#283) — MUST match the `[models.sampling]` block on
|
||||
# this model in models.toml. Two files describe one model; only
|
||||
# script/check-config-consistency.py forces them to agree, and #252 is
|
||||
# what happens when nothing does.
|
||||
#
|
||||
# The model's own generation_config.json says temperature = 1.0, which
|
||||
# measured a 20% structural-defect rate on ~2k-token code generation
|
||||
# against 0/60 at <= 0.6 (Fisher exact one-sided p = 0.0031). Precedence
|
||||
# is request > operator > model, so a caller naming a temperature still
|
||||
# gets what it asked for.
|
||||
[default_models.sampling]
|
||||
temperature = 0.6
|
||||
|
||||
@@ -1,10 +1,3 @@
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# MANAGED BY CI — .gitea/workflows/deploy.yml is the only writer.
|
||||
#
|
||||
# Edits made directly to the deployed copy on the host are reverted by
|
||||
# the next deploy, and until then the fleet runs something no commit
|
||||
# describes. Change it here, commit, let the pipeline ship it.
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# neuron.toml for benjy.hanzalova.internal
|
||||
#
|
||||
# 1x RTX 4090 (24 GB) — largest single-GPU host on the fleet. Pre-warms
|
||||
|
||||
@@ -1,10 +1,3 @@
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# MANAGED BY CI — .gitea/workflows/deploy.yml is the only writer.
|
||||
#
|
||||
# Edits made directly to the deployed copy on the host are reverted by
|
||||
# the next deploy, and until then the fleet runs something no commit
|
||||
# describes. Change it here, commit, let the pipeline ship it.
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# neuron.toml for quadbrat.hanzalova.internal
|
||||
#
|
||||
# 1x RTX 3060 (12 GB) — small / quantised tier. Pre-warms Qwen3-1.7B
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
# Internal-only vhost for the investor portal (helexa-angels).
|
||||
#
|
||||
# Useful where the public name does not resolve or connect correctly from
|
||||
# inside the network that hosts it — common without NAT reflection or
|
||||
# split-horizon DNS. Optional: skip this file if your public name works
|
||||
# from everywhere.
|
||||
#
|
||||
# Cert: internal CA, renewed by step@angels.timer.
|
||||
#
|
||||
# Enable with a relative symlink into sites-enabled:
|
||||
# ln -sf ../sites-available/angels.internal.conf /etc/nginx/sites-enabled/
|
||||
|
||||
upstream angels_up {
|
||||
server gallumbits.kosherinata.internal:8092;
|
||||
keepalive 16;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 127.0.0.1:14443 ssl proxy_protocol;
|
||||
http2 on;
|
||||
server_name angels.internal;
|
||||
|
||||
ssl_certificate /etc/nginx/tls/cert/angels.internal.pem;
|
||||
ssl_certificate_key /etc/nginx/tls/key/angels.internal.pem;
|
||||
ssl_protocols TLSv1.3;
|
||||
ssl_trusted_certificate /etc/pki/ca-trust/source/anchors/root-internal.pem;
|
||||
|
||||
# The material is confidential whichever name reaches it, so the same
|
||||
# headers as the public vhost. `always` so they survive error responses.
|
||||
add_header X-Robots-Tag "noindex, nofollow, noarchive, nosnippet" always;
|
||||
add_header Referrer-Policy "no-referrer" always;
|
||||
add_header X-Content-Type-Options "nosniff" always;
|
||||
add_header X-Frame-Options "DENY" always;
|
||||
add_header Content-Security-Policy
|
||||
"default-src 'none'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; form-action 'self'; base-uri 'none'; frame-ancestors 'none'" always;
|
||||
|
||||
# Same per-IP limits as the public path: an invite code is equally
|
||||
# guessable from here, and argon2 costs the same CPU whoever spends it.
|
||||
location /i/ {
|
||||
limit_req zone=angels_invite burst=5 nodelay;
|
||||
proxy_pass http://angels_up;
|
||||
include /etc/nginx/conf.d/angels-proxy-params.conf;
|
||||
}
|
||||
location /signin {
|
||||
limit_req zone=angels_auth burst=5 nodelay;
|
||||
proxy_pass http://angels_up;
|
||||
include /etc/nginx/conf.d/angels-proxy-params.conf;
|
||||
}
|
||||
location /register {
|
||||
limit_req zone=angels_auth burst=5 nodelay;
|
||||
proxy_pass http://angels_up;
|
||||
include /etc/nginx/conf.d/angels-proxy-params.conf;
|
||||
}
|
||||
location / {
|
||||
proxy_pass http://angels_up;
|
||||
include /etc/nginx/conf.d/angels-proxy-params.conf;
|
||||
}
|
||||
}
|
||||
@@ -47,18 +47,6 @@ server {
|
||||
http2 on;
|
||||
server_name helexa.ai;
|
||||
|
||||
# This vhost listens on an internal port behind the edge SNI router,
|
||||
# so nginx's own idea of "where am I" is 127.0.0.1:14443 rather than
|
||||
# the address the client used. Absolute redirects built from that
|
||||
# leak the internal port -- a request for a directory without a
|
||||
# trailing slash answered 301 to https://<host>:14443/<path>/, which
|
||||
# no client outside this machine can reach.
|
||||
#
|
||||
# Relative redirects sidestep the question: the browser resolves the
|
||||
# Location against the URL it actually requested, which is correct
|
||||
# for every deployment shape this vhost has.
|
||||
absolute_redirect off;
|
||||
|
||||
ssl_certificate /etc/letsencrypt/live/helexa.ai/fullchain.pem;
|
||||
ssl_certificate_key /etc/letsencrypt/live/helexa.ai/privkey.pem;
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
|
||||
@@ -25,18 +25,6 @@ server {
|
||||
listen 127.0.0.1:14443 ssl proxy_protocol;
|
||||
http2 on;
|
||||
|
||||
# This vhost listens on an internal port behind the edge SNI router,
|
||||
# so nginx's own idea of "where am I" is 127.0.0.1:14443 rather than
|
||||
# the address the client used. Absolute redirects built from that
|
||||
# leak the internal port -- a request for a directory without a
|
||||
# trailing slash answered 301 to https://<host>:14443/<path>/, which
|
||||
# no client outside this machine can reach.
|
||||
#
|
||||
# Relative redirects sidestep the question: the browser resolves the
|
||||
# Location against the URL it actually requested, which is correct
|
||||
# for every deployment shape this vhost has.
|
||||
absolute_redirect off;
|
||||
|
||||
ssl_certificate /etc/nginx/tls/cert/helexa.internal.pem;
|
||||
ssl_certificate_key /etc/nginx/tls/key/helexa.internal.pem;
|
||||
ssl_trusted_certificate /etc/pki/ca-trust/source/anchors/root-internal.pem;
|
||||
|
||||
+1
-8
@@ -1,10 +1,4 @@
|
||||
import type {
|
||||
Dimensions,
|
||||
MeasurementRegime,
|
||||
ReportRow,
|
||||
RunRow,
|
||||
SeriesPoint,
|
||||
} from "./types";
|
||||
import type { Dimensions, ReportRow, RunRow, SeriesPoint } from "./types";
|
||||
|
||||
// Empty default → `fetch('/api/...')` hits the dev proxy (vite.config.ts)
|
||||
// or the same origin. For a separately-hosted build, set VITE_API_BASE to
|
||||
@@ -21,7 +15,6 @@ async function getJson<T>(path: string): Promise<T> {
|
||||
|
||||
export const getDimensions = () => getJson<Dimensions>("/api/dimensions");
|
||||
export const getSummary = () => getJson<ReportRow[]>("/api/summary");
|
||||
export const getRegimes = () => getJson<MeasurementRegime[]>("/api/regimes");
|
||||
|
||||
// host is resolved server-side (each model maps to one host today), so the
|
||||
// public UI selects by model + scenario alone.
|
||||
|
||||
+86
-288
@@ -11,8 +11,8 @@ import {
|
||||
XAxis,
|
||||
YAxis,
|
||||
} from "recharts";
|
||||
import { getDimensions, getRegimes, getSeries } from "../api";
|
||||
import type { Dimensions, MeasurementRegime, SeriesPoint } from "../types";
|
||||
import { getDimensions, getSeries } from "../api";
|
||||
import type { Dimensions, SeriesPoint } from "../types";
|
||||
import { BASELINE_SOURCE, baselineFor } from "../baseline";
|
||||
|
||||
function Picker({
|
||||
@@ -40,95 +40,11 @@ function Picker({
|
||||
);
|
||||
}
|
||||
|
||||
type SeriesDef = {
|
||||
key: string;
|
||||
name: string;
|
||||
stroke: string;
|
||||
dashed?: boolean;
|
||||
};
|
||||
|
||||
/** One titled chart over the shared build timeline.
|
||||
*
|
||||
* Every panel draws the same x-axis and the same regime divider, so a
|
||||
* reader can line a change up across metrics — which is the whole point
|
||||
* of having more than two of them. */
|
||||
/** A vertical rule at a build where the metric changed meaning. */
|
||||
type Rule = { at: string; label: string; detail?: string };
|
||||
|
||||
function MetricChart({
|
||||
title,
|
||||
hint,
|
||||
data,
|
||||
lines,
|
||||
rules,
|
||||
unit,
|
||||
}: {
|
||||
title: string;
|
||||
hint?: string;
|
||||
data: Record<string, unknown>[];
|
||||
lines: SeriesDef[];
|
||||
rules?: Rule[];
|
||||
unit?: string;
|
||||
}) {
|
||||
const hasAny = data.some((d) => lines.some((l) => d[l.key] != null));
|
||||
if (!hasAny) return null;
|
||||
return (
|
||||
<>
|
||||
<h5 className="mt-4">
|
||||
{title}
|
||||
{unit ? <span className="text-muted fw-normal"> ({unit})</span> : null}
|
||||
</h5>
|
||||
{hint && <p className="text-muted small mb-2">{hint}</p>}
|
||||
<ResponsiveContainer width="100%" height={260}>
|
||||
<LineChart data={data} margin={{ top: 8, right: 24, bottom: 8, left: 0 }}>
|
||||
<CartesianGrid strokeDasharray="3 3" />
|
||||
<XAxis dataKey="label" />
|
||||
<YAxis />
|
||||
<Tooltip />
|
||||
<Legend />
|
||||
{(rules ?? [])
|
||||
// Only draw a rule whose build is actually on this x-axis;
|
||||
// recharts would otherwise place it at the origin and imply
|
||||
// the boundary sits before all the data.
|
||||
.filter((r) => data.some((d) => d.label === r.at))
|
||||
.map((r) => (
|
||||
<ReferenceLine
|
||||
key={`${r.at}-${r.label}`}
|
||||
x={r.at}
|
||||
stroke="#bbb"
|
||||
strokeDasharray="3 3"
|
||||
label={{
|
||||
value: r.label,
|
||||
position: "top",
|
||||
fill: "#999",
|
||||
fontSize: 11,
|
||||
}}
|
||||
/>
|
||||
))}
|
||||
{lines.map((l) => (
|
||||
<Line
|
||||
key={l.key}
|
||||
type="monotone"
|
||||
dataKey={l.key}
|
||||
name={l.name}
|
||||
stroke={l.stroke}
|
||||
strokeDasharray={l.dashed ? "5 5" : undefined}
|
||||
connectNulls
|
||||
dot={false}
|
||||
/>
|
||||
))}
|
||||
</LineChart>
|
||||
</ResponsiveContainer>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
export default function Trends() {
|
||||
const [dims, setDims] = useState<Dimensions | null>(null);
|
||||
const [model, setModel] = useState("");
|
||||
const [scenario, setScenario] = useState("");
|
||||
const [series, setSeries] = useState<SeriesPoint[]>([]);
|
||||
const [regimes, setRegimes] = useState<MeasurementRegime[]>([]);
|
||||
const [err, setErr] = useState<string | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
@@ -139,10 +55,6 @@ export default function Trends() {
|
||||
if (d.scenarios[0]) setScenario(d.scenarios[0]);
|
||||
})
|
||||
.catch((e) => setErr(String(e)));
|
||||
// A missing/older API just means no rules — not a page failure.
|
||||
getRegimes()
|
||||
.then(setRegimes)
|
||||
.catch(() => setRegimes([]));
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
@@ -156,7 +68,10 @@ export default function Trends() {
|
||||
// Prepend the pre-helexa-bench baseline (dashed, separate keys) so it
|
||||
// anchors the timeline without being merged into the live line. Different
|
||||
// measurement regime — see baseline.ts / doc/benchmarks.md.
|
||||
const base = useMemo(() => baselineFor(model, scenario), [model, scenario]);
|
||||
const base = useMemo(
|
||||
() => baselineFor(model, scenario),
|
||||
[model, scenario],
|
||||
);
|
||||
const data = useMemo(
|
||||
() => [
|
||||
...base.map((p) => ({
|
||||
@@ -170,14 +85,6 @@ export default function Trends() {
|
||||
ttft: p.ttft_s_median,
|
||||
decode: p.decode_tps_median,
|
||||
total: p.total_s_median,
|
||||
ttftP95: p.ttft_p95_s_median,
|
||||
queueWait: p.queue_wait_ms_median,
|
||||
rejected: p.rejected_median,
|
||||
prefillTps: p.prefill_tps_median,
|
||||
reasoning: p.reasoning_tokens_median,
|
||||
cached: p.cached_tokens_median,
|
||||
completion: p.completion_tokens_median,
|
||||
tpot: p.tpot_p95_ms_median,
|
||||
})),
|
||||
],
|
||||
[series, base],
|
||||
@@ -188,55 +95,6 @@ export default function Trends() {
|
||||
const firstLive = series[0]?.git_sha;
|
||||
const showDivider = base.length > 0 && series.length > 0;
|
||||
|
||||
// The build where the measuring identity changed, derived from the
|
||||
// data rather than declared — a constant would go stale the moment
|
||||
// the principal is reconfigured (#288).
|
||||
const identityShift = useMemo(() => {
|
||||
for (let i = 1; i < series.length; i++) {
|
||||
const prev = series[i - 1].principal ?? "anonymous";
|
||||
const cur = series[i].principal ?? "anonymous";
|
||||
if (prev !== cur) {
|
||||
return {
|
||||
at: series[i].git_sha,
|
||||
label: `measured as ${cur === "anonymous" ? "anonymous" : "identified"}`,
|
||||
detail:
|
||||
"The identity bench measured under changed here. Anonymous " +
|
||||
"samples are subject to the #262 yield policy and understate " +
|
||||
"capacity, so points either side are not comparable (#288).",
|
||||
} as Rule;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}, [series]);
|
||||
|
||||
/** Declared boundaries touching `metric`, plus the regime-independent
|
||||
* ones that apply to every panel. */
|
||||
const rulesFor = (metric: string): Rule[] => {
|
||||
const out: Rule[] = [];
|
||||
if (showDivider && firstLive) {
|
||||
out.push({ at: firstLive, label: "bench.py → helexa-bench" });
|
||||
}
|
||||
for (const r of regimes) {
|
||||
if (r.affects.includes(metric)) {
|
||||
out.push({ at: r.first_sha, label: r.label, detail: r.detail });
|
||||
}
|
||||
}
|
||||
if (identityShift) out.push(identityShift);
|
||||
return out;
|
||||
};
|
||||
|
||||
// Anonymous and identified samples are not comparable once #262 is in
|
||||
// the build: an anonymous caller is capped below max_in_flight and
|
||||
// parked at the class gate, so it characterises the yield policy
|
||||
// rather than serving capacity. Say so rather than letting someone
|
||||
// read a step change as an engine regression (#288).
|
||||
const identities = useMemo(
|
||||
() => new Set(series.map((p) => p.principal ?? "anonymous")),
|
||||
[series],
|
||||
);
|
||||
const mixedIdentity = identities.size > 1;
|
||||
const anyAnonymous = identities.has("anonymous");
|
||||
|
||||
if (err) return <Alert variant="danger">{err}</Alert>;
|
||||
if (!dims) return <Spinner animation="border" />;
|
||||
|
||||
@@ -264,25 +122,6 @@ export default function Trends() {
|
||||
</p>
|
||||
)}
|
||||
|
||||
{mixedIdentity && (
|
||||
<Alert variant="warning" className="py-2">
|
||||
<strong>Mixed measurement identity.</strong> Some builds were
|
||||
sampled anonymously and some under a principal. Since{" "}
|
||||
<code>#262</code> an anonymous caller is capped below{" "}
|
||||
<code>max_in_flight</code> and yields to identified traffic, so
|
||||
those points measure the admission policy rather than serving
|
||||
capacity. A step change across that boundary is the instrument,
|
||||
not the engine — see <code>#288</code>.
|
||||
</Alert>
|
||||
)}
|
||||
{!mixedIdentity && anyAnonymous && (
|
||||
<Alert variant="secondary" className="py-2 small">
|
||||
Sampled anonymously. Since <code>#262</code> anonymous callers are
|
||||
served from leftover capacity, so these numbers understate what an
|
||||
authenticated caller gets (<code>#288</code>).
|
||||
</Alert>
|
||||
)}
|
||||
|
||||
{data.length === 0 ? (
|
||||
<Alert variant="info">No data for this selection yet.</Alert>
|
||||
) : (
|
||||
@@ -294,128 +133,87 @@ export default function Trends() {
|
||||
see <code>doc/benchmarks.md</code>.
|
||||
</p>
|
||||
)}
|
||||
<h5 className="mt-3">decode tok/s (higher is better)</h5>
|
||||
<ResponsiveContainer width="100%" height={280}>
|
||||
<LineChart data={data} margin={{ top: 8, right: 24, bottom: 8, left: 0 }}>
|
||||
<CartesianGrid strokeDasharray="3 3" />
|
||||
<XAxis dataKey="label" />
|
||||
<YAxis />
|
||||
<Tooltip />
|
||||
<Legend />
|
||||
{showDivider && firstLive && (
|
||||
<ReferenceLine
|
||||
x={firstLive}
|
||||
stroke="#bbb"
|
||||
strokeDasharray="3 3"
|
||||
label={{
|
||||
value: "bench.py → helexa-bench",
|
||||
position: "top",
|
||||
fill: "#999",
|
||||
fontSize: 11,
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
<Line
|
||||
type="monotone"
|
||||
dataKey="decode"
|
||||
name="decode tok/s"
|
||||
stroke="#0d6efd"
|
||||
connectNulls
|
||||
/>
|
||||
{base.length > 0 && (
|
||||
<Line
|
||||
type="monotone"
|
||||
dataKey="baseDecode"
|
||||
name="baseline (bench.py · gateway)"
|
||||
stroke="#888"
|
||||
strokeDasharray="5 5"
|
||||
connectNulls
|
||||
/>
|
||||
)}
|
||||
</LineChart>
|
||||
</ResponsiveContainer>
|
||||
|
||||
<MetricChart
|
||||
title="decode tok/s"
|
||||
unit="higher is better"
|
||||
data={data}
|
||||
rules={rulesFor("decode")}
|
||||
lines={[
|
||||
{ key: "decode", name: "decode tok/s", stroke: "#0d6efd" },
|
||||
...(base.length > 0
|
||||
? [
|
||||
{
|
||||
key: "baseDecode",
|
||||
name: "baseline (bench.py · gateway)",
|
||||
stroke: "#888",
|
||||
dashed: true,
|
||||
},
|
||||
]
|
||||
: []),
|
||||
]}
|
||||
/>
|
||||
|
||||
<MetricChart
|
||||
title="prefill tok/s"
|
||||
unit="higher is better"
|
||||
hint="The other half of serving speed, derived from prefill_tokens / prefill_ms. A prefix-cache hit shortens prefill_ms while the token count stays whole, so a high rate here is itself the cache-hit signal."
|
||||
data={data}
|
||||
rules={rulesFor("prefillTps")}
|
||||
lines={[
|
||||
{ key: "prefillTps", name: "prefill tok/s", stroke: "#20c997" },
|
||||
]}
|
||||
/>
|
||||
|
||||
<MetricChart
|
||||
title="TTFT"
|
||||
unit="seconds, lower is better"
|
||||
hint="Median and p95 together on purpose. Under concurrency the median is dominated by whichever streams were admitted immediately; the p95 is the one that moves when a caller is made to wait. Charting only the median is how a 0.53 s → 11.77 s tail went unnoticed for a week."
|
||||
data={data}
|
||||
rules={rulesFor("ttft")}
|
||||
lines={[
|
||||
{ key: "ttft", name: "TTFT median (s)", stroke: "#dc3545" },
|
||||
{ key: "ttftP95", name: "TTFT p95 (s)", stroke: "#fd7e14" },
|
||||
...(base.length > 0
|
||||
? [
|
||||
{
|
||||
key: "baseTtft",
|
||||
name: "baseline (bench.py · gateway)",
|
||||
stroke: "#888",
|
||||
dashed: true,
|
||||
},
|
||||
]
|
||||
: []),
|
||||
]}
|
||||
/>
|
||||
|
||||
<MetricChart
|
||||
title="inter-token gap p95"
|
||||
unit="ms, lower is better"
|
||||
hint="Stream smoothness. decode tok/s is a mean over the whole window, so a stream that stalls and then catches up is indistinguishable from one that never stalled — this is the number a user feels."
|
||||
data={data}
|
||||
rules={rulesFor("tpot")}
|
||||
lines={[
|
||||
{ key: "tpot", name: "inter-token p95 (ms)", stroke: "#6f42c1" },
|
||||
]}
|
||||
/>
|
||||
|
||||
<MetricChart
|
||||
title="admission"
|
||||
unit="queue wait ms · requests shed"
|
||||
hint="Separates “the server is slow” from “you were queued behind someone”. Queue wait is TTFT minus server-measured prefill; rejected counts honest backpressure rather than silent failures."
|
||||
data={data}
|
||||
rules={rulesFor("queueWait")}
|
||||
lines={[
|
||||
{ key: "queueWait", name: "queue wait (ms)", stroke: "#d63384" },
|
||||
{ key: "rejected", name: "rejected (count)", stroke: "#adb5bd" },
|
||||
]}
|
||||
/>
|
||||
|
||||
<MetricChart
|
||||
title="tokens per sample"
|
||||
unit="counts"
|
||||
hint="Cost, not speed. Reasoning tokens are the dominant driver on a reasoning model and move independently of every rate above — a template or sampling change can double what the model thinks before answering while the speed charts stay flat. Cached tokens are why prefill timing varies between otherwise identical samples."
|
||||
data={data}
|
||||
rules={rulesFor("completion")}
|
||||
lines={[
|
||||
{ key: "completion", name: "completion tokens", stroke: "#0dcaf0" },
|
||||
{ key: "reasoning", name: "reasoning tokens", stroke: "#ffc107" },
|
||||
{ key: "cached", name: "cached prompt tokens", stroke: "#198754" },
|
||||
]}
|
||||
/>
|
||||
|
||||
{(() => {
|
||||
// Explain every rule actually drawn for this selection. The
|
||||
// chart label is a name; without the reason, a reader still
|
||||
// has to go and rediscover what it meant — which is the cost
|
||||
// this whole feature exists to remove.
|
||||
const shown = [
|
||||
...regimes
|
||||
.filter((r) => series.some((p) => p.git_sha === r.first_sha))
|
||||
.map((r) => ({ at: r.first_sha, label: r.label, detail: r.detail })),
|
||||
...(identityShift ? [identityShift] : []),
|
||||
];
|
||||
if (shown.length === 0) return null;
|
||||
return (
|
||||
<div className="mt-4 pt-3 border-top">
|
||||
<p className="text-muted small mb-2">
|
||||
<strong>Dashed rules mark measurement-regime changes</strong> —
|
||||
builds where a number changed meaning rather than value. A step
|
||||
across one is the instrument, not the engine.
|
||||
</p>
|
||||
<dl className="row small text-muted mb-0">
|
||||
{shown.map((r) => (
|
||||
<div key={`${r.at}-${r.label}`} className="mb-2">
|
||||
<dt className="fw-semibold">
|
||||
<code>{r.at}</code> — {r.label}
|
||||
</dt>
|
||||
<dd className="mb-0">{r.detail}</dd>
|
||||
</div>
|
||||
))}
|
||||
</dl>
|
||||
</div>
|
||||
);
|
||||
})()}
|
||||
<h5 className="mt-4">TTFT seconds (lower is better)</h5>
|
||||
<ResponsiveContainer width="100%" height={280}>
|
||||
<LineChart data={data} margin={{ top: 8, right: 24, bottom: 8, left: 0 }}>
|
||||
<CartesianGrid strokeDasharray="3 3" />
|
||||
<XAxis dataKey="label" />
|
||||
<YAxis />
|
||||
<Tooltip />
|
||||
<Legend />
|
||||
{showDivider && firstLive && (
|
||||
<ReferenceLine
|
||||
x={firstLive}
|
||||
stroke="#bbb"
|
||||
strokeDasharray="3 3"
|
||||
label={{
|
||||
value: "bench.py → helexa-bench",
|
||||
position: "top",
|
||||
fill: "#999",
|
||||
fontSize: 11,
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
<Line
|
||||
type="monotone"
|
||||
dataKey="ttft"
|
||||
name="TTFT (s)"
|
||||
stroke="#dc3545"
|
||||
connectNulls
|
||||
/>
|
||||
{base.length > 0 && (
|
||||
<Line
|
||||
type="monotone"
|
||||
dataKey="baseTtft"
|
||||
name="baseline (bench.py · gateway)"
|
||||
stroke="#888"
|
||||
strokeDasharray="5 5"
|
||||
connectNulls
|
||||
/>
|
||||
)}
|
||||
</LineChart>
|
||||
</ResponsiveContainer>
|
||||
</>
|
||||
)}
|
||||
</>
|
||||
|
||||
@@ -41,17 +41,6 @@ export interface SeriesPoint {
|
||||
ttft_s_median: number | null;
|
||||
decode_tps_median: number | null;
|
||||
total_s_median: number | null;
|
||||
ttft_p95_s_median: number | null;
|
||||
queue_wait_ms_median: number | null;
|
||||
rejected_median: number | null;
|
||||
prefill_tps_median: number | null;
|
||||
reasoning_tokens_median: number | null;
|
||||
cached_tokens_median: number | null;
|
||||
completion_tokens_median: number | null;
|
||||
tpot_p95_ms_median: number | null;
|
||||
/** Identity the samples were taken under, or null if anonymous.
|
||||
* Anonymous and identified points are not comparable (#288). */
|
||||
principal: string | null;
|
||||
samples: number;
|
||||
}
|
||||
|
||||
@@ -78,13 +67,3 @@ export interface RunRow {
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
/** A build at which a metric changed meaning rather than value (#288).
|
||||
* Served from the API so the UI and `report` cite one list. */
|
||||
export interface MeasurementRegime {
|
||||
first_sha: string;
|
||||
label: string;
|
||||
detail: string;
|
||||
/** Series keys this affects, as Trends.tsx names them. */
|
||||
affects: string[];
|
||||
}
|
||||
|
||||
@@ -20,11 +20,6 @@ pub struct MessagesRequest {
|
||||
pub temperature: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub top_p: Option<f64>,
|
||||
/// Anthropic's Messages API carries `top_k` natively, so it maps
|
||||
/// straight onto the OpenAI-side field rather than being dropped in
|
||||
/// translation (#272).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub top_k: Option<usize>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stream: Option<bool>,
|
||||
#[serde(flatten)]
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
//! Model catalogue — profiles describing how to serve each model.
|
||||
|
||||
use crate::discovery::DeviceInfo;
|
||||
use crate::harness::{ModelCost, ModelLimit, SamplingOverride};
|
||||
use crate::harness::{ModelCost, ModelLimit};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
@@ -22,30 +22,9 @@ pub struct ModelProfile {
|
||||
/// Minimum VRAM per device in MB.
|
||||
#[serde(default)]
|
||||
pub min_device_vram_mb: Option<u64>,
|
||||
/// Neurons this model is allowed to run on. Empty = anywhere its
|
||||
/// device constraints are satisfied.
|
||||
///
|
||||
/// This is an *affinity* constraint — where the model may be placed.
|
||||
/// It says nothing about whether the model may be evicted once
|
||||
/// resident; that is [`ModelProfile::residency_priority`]. The two
|
||||
/// were a single field once, which made "run only here" and "never
|
||||
/// evict here" impossible to ask for separately.
|
||||
/// Neurons where this model should never be evicted.
|
||||
#[serde(default)]
|
||||
pub pinned_on: Vec<String>,
|
||||
/// Which residency class this model belongs to when a node runs out
|
||||
/// of VRAM. A model may displace residents of its own class or
|
||||
/// below, and none above it — so models that should take turns on a
|
||||
/// node share a number, and a model is protected by being ranked
|
||||
/// *above* whatever must not evict it.
|
||||
///
|
||||
/// Unset means [`DEFAULT_RESIDENCY_PRIORITY`], except for profiles
|
||||
/// carrying `pinned_on`, which default to
|
||||
/// [`PINNED_RESIDENCY_PRIORITY`] — before these were separate
|
||||
/// fields, `pinned_on` implied immunity from eviction, and a
|
||||
/// catalogue written against that meaning must not silently start
|
||||
/// allowing its flagship to be evicted.
|
||||
#[serde(default)]
|
||||
pub residency_priority: Option<u32>,
|
||||
/// Source scheme this profile's weights come from. When set, the
|
||||
/// router prefixes `id` with `scheme:` before forwarding the load
|
||||
/// request to neuron, ensuring the daemon fetches from the right
|
||||
@@ -57,23 +36,6 @@ pub struct ModelProfile {
|
||||
/// on this being explicit per model rather than implicit.
|
||||
#[serde(default)]
|
||||
pub source: Option<String>,
|
||||
/// Operator sampling override forwarded to neuron on cold load
|
||||
/// (#283). The counterpart to the same field in a host's
|
||||
/// `[[default_models]]` — a model that can be both cold-loaded by
|
||||
/// cortex and resident on a host must carry the same value in both,
|
||||
/// which `script/check-config-consistency.py` enforces in CI.
|
||||
///
|
||||
/// This is the #252 shape of hazard: two config files describing one
|
||||
/// model, with nothing forcing them to agree.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub sampling: Option<SamplingOverride>,
|
||||
/// The model's own `preserve_thinking` template control, forwarded
|
||||
/// on cold load so a model this gateway loads behaves the same as
|
||||
/// one the host was configured to hold resident. Two config files
|
||||
/// describing one model is how #252 happened; this rides along for
|
||||
/// the same reason `sampling` does.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub preserve_thinking: Option<bool>,
|
||||
|
||||
// ── Enrichment (issue #62) ────────────────────────────────
|
||||
/// Per-model token budget. When present, advertised in `/v1/models`
|
||||
@@ -95,18 +57,6 @@ fn default_min_devices() -> u32 {
|
||||
1
|
||||
}
|
||||
|
||||
/// Residency priority for a model that declares none. Deliberately not
|
||||
/// zero: an operator needs room to rank something *below* the ordinary
|
||||
/// case (a scratch or experimental model that should yield to anything)
|
||||
/// without editing every other entry.
|
||||
pub const DEFAULT_RESIDENCY_PRIORITY: u32 = 100;
|
||||
|
||||
/// Residency priority assumed for a profile that carries `pinned_on` but
|
||||
/// no explicit priority. High enough that nothing with a default
|
||||
/// priority can evict it, preserving the immunity `pinned_on` used to
|
||||
/// grant on its own.
|
||||
pub const PINNED_RESIDENCY_PRIORITY: u32 = 1000;
|
||||
|
||||
/// The full model catalogue.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct ModelCatalogue {
|
||||
@@ -145,49 +95,11 @@ impl ModelCatalogue {
|
||||
}
|
||||
}
|
||||
|
||||
/// How strongly `model_id` holds its place. Models absent from the
|
||||
/// catalogue rank at the default — a model can be resident on a
|
||||
/// neuron without a profile (loaded directly, or left over from an
|
||||
/// earlier catalogue), and treating those as unevictable would let
|
||||
/// an unlisted model wedge a node permanently.
|
||||
pub fn residency_priority(&self, model_id: &str) -> u32 {
|
||||
self.get(model_id)
|
||||
.map(|p| {
|
||||
p.residency_priority.unwrap_or({
|
||||
if p.pinned_on.is_empty() {
|
||||
DEFAULT_RESIDENCY_PRIORITY
|
||||
} else {
|
||||
PINNED_RESIDENCY_PRIORITY
|
||||
}
|
||||
})
|
||||
})
|
||||
.unwrap_or(DEFAULT_RESIDENCY_PRIORITY)
|
||||
}
|
||||
|
||||
/// May `incoming` take VRAM from `resident` when a node cannot hold
|
||||
/// both?
|
||||
///
|
||||
/// Greater-or-equal, which makes the priority a *class* rather than
|
||||
/// a strict order: a model may displace anything in its own class or
|
||||
/// below, and nothing above it.
|
||||
///
|
||||
/// Equal rank has to permit displacement, because mutual
|
||||
/// displacement is what cold-swap *is*. Two models that share a node
|
||||
/// and take turns on it — an image generator and a mid-tier text
|
||||
/// model, or two generations of the same flagship being compared —
|
||||
/// each need to evict the other on demand. Under a strict
|
||||
/// greater-than the first one to arrive would win permanently and
|
||||
/// the other could never come back, which reads as "the model
|
||||
/// vanished after we generated an image".
|
||||
///
|
||||
/// Protection therefore comes from ranking a model *above* its
|
||||
/// would-be evictor, not from ranking every model differently.
|
||||
///
|
||||
/// This governs only *whether* a displacement is permitted, never
|
||||
/// whether one is needed. A node with room for both evicts nothing,
|
||||
/// however the two rank.
|
||||
pub fn may_displace(&self, incoming_id: &str, resident_id: &str) -> bool {
|
||||
self.residency_priority(incoming_id) >= self.residency_priority(resident_id)
|
||||
/// Check if a model is pinned on a given neuron.
|
||||
pub fn is_pinned(&self, model_id: &str, neuron_name: &str) -> bool {
|
||||
self.models
|
||||
.iter()
|
||||
.any(|p| p.id == model_id && p.pinned_on.contains(&neuron_name.to_string()))
|
||||
}
|
||||
|
||||
/// Find a profile by model id.
|
||||
@@ -255,10 +167,7 @@ mod tests {
|
||||
min_devices: 2,
|
||||
min_device_vram_mb: Some(24_000),
|
||||
pinned_on: vec![],
|
||||
residency_priority: None,
|
||||
source: None,
|
||||
sampling: None,
|
||||
preserve_thinking: None,
|
||||
limit: None,
|
||||
cost: None,
|
||||
capabilities: vec![],
|
||||
@@ -303,162 +212,6 @@ mod tests {
|
||||
assert!(p.is_feasible_on("anywhere", &devices));
|
||||
}
|
||||
|
||||
/// A catalogue shaped like a real fleet. Two residency classes: the
|
||||
/// big-node class holds a flagship and a frontier model that take
|
||||
/// turns on the one machine large enough for either; the everyday
|
||||
/// class holds an image generator and a mid-tier text model that
|
||||
/// take turns on a smaller one. Nothing in the everyday class may
|
||||
/// touch the big-node class.
|
||||
fn tiered_catalogue() -> ModelCatalogue {
|
||||
toml::from_str(
|
||||
r#"
|
||||
[[models]]
|
||||
id = "flagship"
|
||||
harness = "candle"
|
||||
pinned_on = ["big-node"]
|
||||
residency_priority = 300
|
||||
|
||||
[[models]]
|
||||
id = "frontier"
|
||||
harness = "candle"
|
||||
residency_priority = 300
|
||||
|
||||
[[models]]
|
||||
id = "image"
|
||||
harness = "candle"
|
||||
residency_priority = 200
|
||||
|
||||
[[models]]
|
||||
id = "mid"
|
||||
harness = "candle"
|
||||
residency_priority = 200
|
||||
|
||||
[[models]]
|
||||
id = "tiny"
|
||||
harness = "candle"
|
||||
residency_priority = 100
|
||||
"#,
|
||||
)
|
||||
.expect("parse tiered catalogue")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn image_generation_displaces_the_mid_tier() {
|
||||
assert!(tiered_catalogue().may_displace("image", "mid"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_mid_tier_comes_back_after_an_image_takes_its_node() {
|
||||
// The swap-back half. An image request evicts the text model;
|
||||
// the next text request must be able to evict the image model,
|
||||
// or the text tier disappears until someone restarts something.
|
||||
assert!(tiered_catalogue().may_displace("mid", "image"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_generations_of_a_flagship_can_swap_in_both_directions() {
|
||||
// Comparing a new flagship against the incumbent needs traffic
|
||||
// to move each way on demand. A strict order would let whichever
|
||||
// arrived first hold the node permanently.
|
||||
let cat = tiered_catalogue();
|
||||
assert!(cat.may_displace("frontier", "flagship"));
|
||||
assert!(cat.may_displace("flagship", "frontier"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn image_generation_never_displaces_the_flagship() {
|
||||
// The image generator's device constraints alone would let it
|
||||
// land on the flagship's node, so this is the case priority
|
||||
// exists to prevent -- not a hypothetical one.
|
||||
assert!(!tiered_catalogue().may_displace("image", "flagship"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_frontier_tier_displaces_the_flagship() {
|
||||
// The requirement a boolean pin cannot express: the flagship is
|
||||
// protected from one model and not from another.
|
||||
assert!(tiered_catalogue().may_displace("frontier", "flagship"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_lower_class_cannot_displace_a_higher_one() {
|
||||
let cat = tiered_catalogue();
|
||||
assert!(!cat.may_displace("tiny", "mid"));
|
||||
assert!(!cat.may_displace("tiny", "flagship"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_higher_class_can_displace_a_lower_one() {
|
||||
assert!(tiered_catalogue().may_displace("flagship", "tiny"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pinned_on_alone_still_protects_a_catalogue_written_before_priorities() {
|
||||
// `pinned_on` used to mean "never evict here". A catalogue that
|
||||
// predates the split says nothing about priority, and must not
|
||||
// silently start allowing its flagship to be evicted.
|
||||
let cat: ModelCatalogue = toml::from_str(
|
||||
r#"
|
||||
[[models]]
|
||||
id = "flagship"
|
||||
harness = "candle"
|
||||
pinned_on = ["big-node"]
|
||||
|
||||
[[models]]
|
||||
id = "ordinary"
|
||||
harness = "candle"
|
||||
"#,
|
||||
)
|
||||
.expect("parse legacy catalogue");
|
||||
assert!(!cat.may_displace("ordinary", "flagship"));
|
||||
assert!(cat.may_displace("flagship", "ordinary"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unlisted_resident_is_displaceable_by_a_ranked_model() {
|
||||
// A model can be resident without a profile. Treating it as
|
||||
// unevictable would let an unlisted model wedge a node forever.
|
||||
let cat = tiered_catalogue();
|
||||
assert_eq!(
|
||||
cat.residency_priority("never-heard-of-it"),
|
||||
DEFAULT_RESIDENCY_PRIORITY
|
||||
);
|
||||
assert!(cat.may_displace("image", "never-heard-of-it"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn affinity_and_immunity_are_independently_expressible() {
|
||||
// The whole point of the split: confine a model to a node
|
||||
// without protecting it there, and protect one without
|
||||
// confining it anywhere.
|
||||
let cat: ModelCatalogue = toml::from_str(
|
||||
r#"
|
||||
[[models]]
|
||||
id = "confined-but-evictable"
|
||||
harness = "candle"
|
||||
pinned_on = ["big-node"]
|
||||
residency_priority = 50
|
||||
|
||||
[[models]]
|
||||
id = "roaming-but-protected"
|
||||
harness = "candle"
|
||||
residency_priority = 900
|
||||
"#,
|
||||
)
|
||||
.expect("parse catalogue");
|
||||
let devices = [device(0, 32_000)];
|
||||
|
||||
let confined = cat.get("confined-but-evictable").unwrap();
|
||||
assert!(confined.is_feasible_on("big-node", &devices));
|
||||
assert!(!confined.is_feasible_on("other-node", &devices));
|
||||
|
||||
let roaming = cat.get("roaming-but-protected").unwrap();
|
||||
assert!(roaming.is_feasible_on("other-node", &devices));
|
||||
|
||||
assert!(cat.may_displace("roaming-but-protected", "confined-but-evictable"));
|
||||
assert!(!cat.may_displace("confined-but-evictable", "roaming-but-protected"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_alias_returns_target_when_alias_present() {
|
||||
let mut cat = ModelCatalogue::default();
|
||||
|
||||
@@ -109,40 +109,6 @@ pub struct ModelLoad {
|
||||
/// (#54/#137). `#[serde(default)]` for back-compat.
|
||||
#[serde(default)]
|
||||
pub rejected_per_principal: u64,
|
||||
/// Anonymous requests currently holding a seat (#262). Read against
|
||||
/// `in_flight` it says how much of this model's load is unattributable;
|
||||
/// against `anon_max_in_flight`, how close that traffic is to the
|
||||
/// ceiling that keeps it from starving identified callers.
|
||||
#[serde(default)]
|
||||
pub anon_in_flight: usize,
|
||||
/// Seats anonymous traffic may hold at once (#262); `0` = refused
|
||||
/// outright. `#[serde(default)]` (→ 0) for pre-#262 neurons, which
|
||||
/// applied no ceiling at all — cortex skips the gauge rather than
|
||||
/// publish a 0 that reads as the opposite of the truth.
|
||||
#[serde(default)]
|
||||
pub anon_max_in_flight: usize,
|
||||
/// Cumulative anonymous requests refused for want of leftover capacity
|
||||
/// (#262). Rising here with a flat `rejected_timeout` means the
|
||||
/// reservation is working, not that the model is overloaded.
|
||||
#[serde(default)]
|
||||
pub rejected_anon_yield: u64,
|
||||
/// Cumulative requests that waited for KV budget and never got it
|
||||
/// (#257) — the model is saturated by long-context sequences rather
|
||||
/// than by request count. `#[serde(default)]` for back-compat.
|
||||
#[serde(default)]
|
||||
pub rejected_kv_timeout: u64,
|
||||
/// Cumulative requests whose KV reservation exceeded the model's entire
|
||||
/// budget (#257) — prompts too long for this node at any load, not a
|
||||
/// load-shedding signal. `#[serde(default)]` for back-compat.
|
||||
#[serde(default)]
|
||||
pub rejected_kv_unservable: u64,
|
||||
/// This model's total KV budget in MiB and how much of it is currently
|
||||
/// unreserved (#257). `kv_budget_mb == 0` means the gate is disabled
|
||||
/// (CPU load, or an arch with no context profile).
|
||||
#[serde(default)]
|
||||
pub kv_budget_mb: u64,
|
||||
#[serde(default)]
|
||||
pub kv_available_mb: u64,
|
||||
/// Live prefill throughput EMA in tokens/sec (#137) — prompt tokens
|
||||
/// processed per second. `0.0` before the first sample. `#[serde(default)]`
|
||||
/// for back-compat with pre-#137 neurons.
|
||||
@@ -182,15 +148,8 @@ mod health_load_tests {
|
||||
max_in_flight: 8,
|
||||
max_queue_depth: 8,
|
||||
rejected_queue_full: 0,
|
||||
rejected_kv_timeout: 0,
|
||||
rejected_kv_unservable: 0,
|
||||
kv_budget_mb: 0,
|
||||
kv_available_mb: 0,
|
||||
rejected_timeout: 0,
|
||||
rejected_per_principal: 0,
|
||||
anon_in_flight: 2,
|
||||
anon_max_in_flight: 7,
|
||||
rejected_anon_yield: 0,
|
||||
tok_s_prefill: 0.0,
|
||||
tok_s_decode: 0.0,
|
||||
}],
|
||||
@@ -202,8 +161,6 @@ mod health_load_tests {
|
||||
assert_eq!(back.models[0].queue_depth, 3);
|
||||
assert_eq!(back.models[0].max_in_flight, 8);
|
||||
assert_eq!(back.models[0].max_queue_depth, 8);
|
||||
assert_eq!(back.models[0].anon_in_flight, 2);
|
||||
assert_eq!(back.models[0].anon_max_in_flight, 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -215,10 +172,6 @@ mod health_load_tests {
|
||||
assert_eq!(m.in_flight, 2);
|
||||
assert_eq!(m.max_in_flight, 0);
|
||||
assert_eq!(m.max_queue_depth, 0);
|
||||
// Likewise a pre-#262 neuron, which enforced no anonymous ceiling.
|
||||
assert_eq!(m.anon_in_flight, 0);
|
||||
assert_eq!(m.anon_max_in_flight, 0);
|
||||
assert_eq!(m.rejected_anon_yield, 0);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -26,115 +26,7 @@ pub struct HarnessHealth {
|
||||
pub uptime_secs: Option<u64>,
|
||||
}
|
||||
|
||||
/// Operator-set sampling defaults for one model (#283).
|
||||
///
|
||||
/// A model's `generation_config.json` is the model authors' statement of
|
||||
/// intent, and honouring it (#272) is right by default. It is not always
|
||||
/// right for the workload the operator serves: `Qwen/Qwen3.8-27B`
|
||||
/// publishes `temperature = 1.0`, which measured a 20% structural-defect
|
||||
/// rate on ~2k-token code generation against 0/60 at `<= 0.6`
|
||||
/// (#283, p = 0.0031).
|
||||
///
|
||||
/// This is the operator's explicit, visible correction to that default —
|
||||
/// deliberately *not* a heuristic that inspects the model and guesses.
|
||||
/// A temperature guesser is a footgun; a number in a config file the
|
||||
/// operator wrote is not.
|
||||
///
|
||||
/// Precedence is **request > operator > model > built-in fallback**: the
|
||||
/// override replaces what the model published, but a caller that names a
|
||||
/// value still wins, because an explicit API parameter that is silently
|
||||
/// ignored is a contract break.
|
||||
///
|
||||
/// Every field is optional and overlays independently — setting
|
||||
/// `temperature` alone leaves the model's `top_p`/`top_k` in force.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SamplingOverride {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub temperature: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub top_p: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub top_k: Option<usize>,
|
||||
/// Multiplier applied to recently-generated tokens before sampling.
|
||||
/// `1.0` disables it; above that, recently-emitted tokens get less
|
||||
/// likely.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub repeat_penalty: Option<f32>,
|
||||
/// How many recently-generated tokens the penalty considers.
|
||||
///
|
||||
/// The reason this is operator-settable at all: the built-in default
|
||||
/// is 64, inherited from candle's chat example, and a reasoning
|
||||
/// model's think block runs to tens of thousands of tokens. Anything
|
||||
/// restated more than 64 tokens later is invisible to the penalty,
|
||||
/// which is how a 28,672-token think block can contain the same
|
||||
/// derivation five times over.
|
||||
///
|
||||
/// Raising it is a real trade, not a free win: the penalty is
|
||||
/// token-level, and source code legitimately repeats tokens
|
||||
/// constantly (`const`, `function`, brace runs). A wide window at a
|
||||
/// meaningful penalty distorts exactly the output this fleet is for.
|
||||
/// Sequence-level repetition wants a sequence-level instrument
|
||||
/// (frequency/presence penalties, or a DRY-style sampler); this knob
|
||||
/// exists so the trade can be *measured* per model rather than
|
||||
/// guessed once and frozen into a constant.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub repeat_last_n: Option<usize>,
|
||||
/// Sequence-wide one-time penalty for already-seen tokens.
|
||||
///
|
||||
/// This, not `repeat_last_n`, is the remedy Qwen prescribe for the
|
||||
/// endless repetition their models fall into during long reasoning
|
||||
/// (0..2, 1.5 when severe). It is scored over everything generated
|
||||
/// so far, so a passage restated 5,000 tokens later is still
|
||||
/// penalised — the case a trailing window cannot see.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub presence_penalty: Option<f32>,
|
||||
/// Sequence-wide penalty proportional to a token's existing count.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub frequency_penalty: Option<f32>,
|
||||
/// Pin the sampler's RNG for every request to this model.
|
||||
///
|
||||
/// Exists because parameter search needs a control: comparing two
|
||||
/// `presence_penalty` values across unseeded runs cannot separate
|
||||
/// the parameter's effect from sampling variance, and the clients
|
||||
/// that would otherwise pin it cannot — pi's request builder sends
|
||||
/// no `seed` at all, so a caller-side seed is not available on the
|
||||
/// path that matters.
|
||||
///
|
||||
/// Note what this does and does not give. It removes sampling noise
|
||||
/// as a confounder; it does not make two runs comparable
|
||||
/// token-by-token, because changing a penalty changes the logits
|
||||
/// and the sequences diverge at the first step where a token choice
|
||||
/// flips.
|
||||
///
|
||||
/// Leaving this set is a product decision, not just a testing one:
|
||||
/// every identical prompt then yields an identical answer for every
|
||||
/// caller. That is defensible — predictable and cacheable — but it
|
||||
/// removes the variation a user may expect from retrying. A request
|
||||
/// that names its own seed still wins.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub seed: Option<u64>,
|
||||
}
|
||||
|
||||
impl SamplingOverride {
|
||||
/// True when the operator set nothing — an empty `[sampling]` table
|
||||
/// must behave exactly as no table at all.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.temperature.is_none()
|
||||
&& self.top_p.is_none()
|
||||
&& self.top_k.is_none()
|
||||
&& self.repeat_penalty.is_none()
|
||||
&& self.repeat_last_n.is_none()
|
||||
&& self.presence_penalty.is_none()
|
||||
&& self.frequency_penalty.is_none()
|
||||
&& self.seed.is_none()
|
||||
}
|
||||
}
|
||||
|
||||
/// Specification for loading a model through a harness.
|
||||
///
|
||||
/// Doubles as the `[[default_models]]` entry shape in `neuron.toml`, so
|
||||
/// a field added here is available to both the load API and the
|
||||
/// operator's per-host config without a second type to keep in step.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ModelSpec {
|
||||
pub model_id: String,
|
||||
@@ -142,36 +34,6 @@ pub struct ModelSpec {
|
||||
pub quant: Option<String>,
|
||||
pub tensor_parallel: Option<u32>,
|
||||
pub devices: Option<Vec<u32>>,
|
||||
/// Operator sampling override (#283). Travels with the *model*, not
|
||||
/// the request: neuron is the only place sampling is resolved, so
|
||||
/// the override has to reach it whether the load came from cortex's
|
||||
/// catalogue or from this host's own `[[default_models]]`.
|
||||
///
|
||||
/// `#[serde(default)]` so a cortex or neuron predating this field
|
||||
/// still round-trips the spec.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub sampling: Option<SamplingOverride>,
|
||||
/// Whether prior assistant turns keep their reasoning when the chat
|
||||
/// template re-renders the conversation.
|
||||
///
|
||||
/// This is the model's own `preserve_thinking` template control, not
|
||||
/// an invention of ours. `Qwen/Qwen3.8-27B` defaults it to *true* —
|
||||
/// the whole transcript's think blocks are replayed — and `false`
|
||||
/// keeps reasoning only for turns after the last user query, i.e.
|
||||
/// the turn in progress.
|
||||
///
|
||||
/// `None` leaves the kwarg unset so the template's own default
|
||||
/// applies, which is the only safe default: the model's authors
|
||||
/// chose it, and overriding that fleet-wide on a hunch is how you
|
||||
/// ship a regression you cannot see.
|
||||
///
|
||||
/// Exposed as operator config so the premise can be A/B'd on a real
|
||||
/// workload — full replay is the larger prompt and pushes against
|
||||
/// the prefix-cache budget and the throughput-derived context
|
||||
/// ceiling, but whether it *helps* the model is a measurement, not
|
||||
/// a belief. A request's own `chat_template_kwargs` still wins.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub preserve_thinking: Option<bool>,
|
||||
}
|
||||
|
||||
/// Per-model token budget advertised by the catalogue or neuron.
|
||||
@@ -188,77 +50,8 @@ pub struct ModelLimit {
|
||||
/// back to `context − output`.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input: Option<usize>,
|
||||
/// The generation budget a request gets when it names none, and the
|
||||
/// figure `input = context − output` is derived from.
|
||||
///
|
||||
/// A *reserve*, not a ceiling — a request may ask for more and be
|
||||
/// served it. Named `output` for the shape's history; the number a
|
||||
/// client should plan against is [`ModelLimit::output_ceiling`].
|
||||
/// Maximum number of generation tokens.
|
||||
pub output: usize,
|
||||
/// The largest output a request may name and be served (#278).
|
||||
///
|
||||
/// Distinct from `output`, which is the default and the KV-planning
|
||||
/// reserve. Publishing the reserve as the ceiling meant a client
|
||||
/// that trusted the advertisement asked for a fraction of what the
|
||||
/// model would happily generate — on a reasoning model, often less
|
||||
/// than its own think block costs.
|
||||
///
|
||||
/// `#[serde(default)]` so a neuron predating this field still
|
||||
/// deserializes; zero means "not advertised", and consumers fall
|
||||
/// back to `output`.
|
||||
#[serde(default, skip_serializing_if = "is_zero")]
|
||||
pub output_ceiling: usize,
|
||||
}
|
||||
|
||||
fn is_zero(v: &usize) -> bool {
|
||||
*v == 0
|
||||
}
|
||||
|
||||
/// What each reasoning effort level costs, in reasoning tokens (#223).
|
||||
///
|
||||
/// OpenAI-shaped clients can only send `minimal|low|medium|high` — the
|
||||
/// ladder is their entire vocabulary, and they cannot express a token
|
||||
/// count. So the server picks the numbers, and has to say what they are:
|
||||
/// a client choosing `low` with no idea whether that buys 2k tokens or
|
||||
/// 20k is guessing, which is the failure #274 exists to end.
|
||||
///
|
||||
/// Ordered rungs rather than a map, so the ladder reads in the order a
|
||||
/// client would climb it.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ReasoningBudgetRung {
|
||||
/// The effort level as the caller spells it on the wire.
|
||||
///
|
||||
/// Since #290 these are the rungs the **model's own chat template**
|
||||
/// accepts, discovered by rendering it, rather than a ladder the
|
||||
/// operator invented. `Qwen/Qwen3.8-27B` accepts exactly `low`,
|
||||
/// `medium` and `xhigh` and raises on anything else; we previously
|
||||
/// advertised `minimal`/`low`/`medium`/`high`, two of which the
|
||||
/// template rejects and which omitted the model's own default.
|
||||
///
|
||||
/// Advertise only rungs the template accepts: an invented name is a
|
||||
/// name a caller can send and the model will reject. Note that a
|
||||
/// client need not read this at all — pi-ai takes its levels from a
|
||||
/// static `thinkingLevelMap` in the operator's config — so this is
|
||||
/// a statement of truth, not a control surface we can rely on.
|
||||
pub effort: String,
|
||||
/// Backstop reasoning-token cap for this rung, when the deployment
|
||||
/// sets one.
|
||||
///
|
||||
/// `None` is the normal case and means "named rung, no cap": effort
|
||||
/// is expressed to the model through its template, and the number of
|
||||
/// tokens it then spends is the model's business. A cap is a safety
|
||||
/// net against a runaway think block (#223), not the mechanism by
|
||||
/// which effort is selected — enforcing effort by truncation is what
|
||||
/// #290 fixed.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub tokens: Option<usize>,
|
||||
/// Whether the model applies this rung when the caller names none.
|
||||
///
|
||||
/// Taken from the template's own `|default(...)`. Without it a
|
||||
/// caller that says nothing cannot tell what it is getting — on
|
||||
/// Qwen3.8 that silently means `xhigh`.
|
||||
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||
pub default: bool,
|
||||
}
|
||||
|
||||
/// Operator-set pricing, **USD per 1,000,000 tokens, as JSON numbers**
|
||||
@@ -333,52 +126,6 @@ pub struct ModelInfo {
|
||||
/// marker tokens (`<think>` / `<\/think>` or similar).
|
||||
#[serde(default)]
|
||||
pub reasoning: bool,
|
||||
/// The operator's `preserve_thinking` for this loaded model, or
|
||||
/// `None` when unset and the template's own default applies.
|
||||
///
|
||||
/// Advertised so a run can be *stamped* with the value it ran
|
||||
/// under. This is an A/B knob whose whole purpose is comparison
|
||||
/// between runs, and a comparison whose arms cannot be told apart
|
||||
/// afterwards is not a comparison. Recovering it from deploy
|
||||
/// history is guesswork — a config sync can land mid-session.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub preserve_thinking: Option<bool>,
|
||||
|
||||
/// Whether this node can actually serve a request for this model
|
||||
/// right now (#245).
|
||||
///
|
||||
/// `status == "loaded"` only says the weights are resident. A node
|
||||
/// can hold a model and still reject every request — most commonly
|
||||
/// because something outside neuron has eaten the device's free VRAM
|
||||
/// and the prefill floor check will fail before any device work.
|
||||
/// That happened in production: a leaked desktop compositor took a
|
||||
/// whole tier down while the node answered every poll correctly and
|
||||
/// the fleet reported itself fully healthy.
|
||||
///
|
||||
/// `None` means the node did not say — an older neuron, or a state
|
||||
/// it cannot evaluate. Callers must treat that as "assume servable",
|
||||
/// never as "unservable", or a version skew would evict the fleet
|
||||
/// from its own routing table.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub servable: Option<ModelServability>,
|
||||
/// The reasoning-effort ladder this deployment honours (#223).
|
||||
/// Absent when the model does not reason, or the harness has no
|
||||
/// budget to offer.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub reasoning_budget: Vec<ReasoningBudgetRung>,
|
||||
}
|
||||
|
||||
/// Why a loaded model can or cannot be served right now (#245).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ModelServability {
|
||||
pub ok: bool,
|
||||
/// Machine-readable cause when `ok == false` — e.g.
|
||||
/// `"insufficient_vram"`, matching the error the request would get.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub reason: Option<String>,
|
||||
/// Human-facing detail, for an operator reading a health page.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub detail: Option<String>,
|
||||
}
|
||||
|
||||
/// What an inference harness must do, from neuron's perspective.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use crate::discovery::{ActivationStatus, DiscoveryResponse, ModelLoad};
|
||||
use crate::harness::{ModelCost, ModelLimit, ModelServability};
|
||||
use crate::harness::{ModelCost, ModelLimit};
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
@@ -44,12 +44,6 @@ pub struct NodeState {
|
||||
/// just one whose total topology could. Empty until the first
|
||||
/// /health poll.
|
||||
pub device_health: Vec<crate::discovery::DeviceHealth>,
|
||||
/// The reasoning-effort ladder this neuron honours (#223), copied
|
||||
/// from its `/models` reply at poll time. Per-node rather than
|
||||
/// per-model because it is host configuration — every model on a
|
||||
/// neuron reports the same rungs — and duplicating it onto each
|
||||
/// model entry would invite the two to disagree.
|
||||
pub reasoning_budget: Vec<crate::harness::ReasoningBudgetRung>,
|
||||
}
|
||||
|
||||
/// A model registered on a node, with its runtime status.
|
||||
@@ -82,27 +76,6 @@ pub struct ModelEntry {
|
||||
/// limits are no longer consulted.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub limit: Option<ModelLimit>,
|
||||
/// Whether this node could actually serve the model right now
|
||||
/// (#245), from `ModelInfo.servable` at poll time.
|
||||
///
|
||||
/// `None` means the neuron expressed no opinion — an older build, a
|
||||
/// CPU load, an image model, or a cache not yet seeded. Absent is
|
||||
/// **not** unservable: see [`ModelEntry::is_servable`].
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub servable: Option<ModelServability>,
|
||||
}
|
||||
|
||||
impl ModelEntry {
|
||||
/// Whether the router should treat this location as a candidate.
|
||||
///
|
||||
/// Absent evidence means yes. A neuron that predates the field, or
|
||||
/// one that cannot evaluate its own state, must not be read as
|
||||
/// broken — a version skew would otherwise empty the routing table
|
||||
/// and take the fleet down far more comprehensively than the fault
|
||||
/// this guards against.
|
||||
pub fn is_servable(&self) -> bool {
|
||||
self.servable.as_ref().is_none_or(|s| s.ok)
|
||||
}
|
||||
}
|
||||
|
||||
/// Model lifecycle status.
|
||||
@@ -193,35 +166,12 @@ pub struct CortexModelEntry {
|
||||
/// Served max-seq-len in tokens — mirrors `limit.context`.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_model_len: Option<usize>,
|
||||
/// The same window under the two names generic OpenAI clients look
|
||||
/// for (#278) — a client that probes for one of these rather than
|
||||
/// `max_model_len` would otherwise find no window at all and fall
|
||||
/// back to a number its operator had to invent by hand.
|
||||
///
|
||||
/// Publishing them is cheap and harmless; do not assume any
|
||||
/// *specific* client consumes them. An earlier version of this
|
||||
/// comment credited pi-ai's provider discovery, which was wrong:
|
||||
/// pi requires `contextWindow` as a static field on its model type
|
||||
/// and reads none of these keys at runtime (checked against 0.82.1
|
||||
/// and 0.84.3). The misattribution mattered — it made an operator's
|
||||
/// hand-set 80k window look like our advertisement failing to land,
|
||||
/// when in fact nothing was reading it and the number had to be
|
||||
/// maintained by hand regardless.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub context_window: Option<usize>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub context_length: Option<usize>,
|
||||
/// Usable input budget — mirrors `limit.input` when present.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_input_tokens: Option<usize>,
|
||||
/// The largest output a request may name — mirrors
|
||||
/// `limit.output_ceiling`, falling back to `limit.output` (#278).
|
||||
/// Maximum generation tokens — mirrors `limit.output`.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_output_tokens: Option<usize>,
|
||||
/// What each effort level buys, in reasoning tokens (#223) — so a
|
||||
/// client picking `low` knows what it asked for.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub reasoning_budget: Vec<crate::harness::ReasoningBudgetRung>,
|
||||
}
|
||||
|
||||
impl CortexModelEntry {
|
||||
@@ -233,26 +183,8 @@ impl CortexModelEntry {
|
||||
/// dropped the limit.
|
||||
pub fn sync_flat_limit(&mut self) {
|
||||
self.max_model_len = self.limit.as_ref().map(|l| l.context);
|
||||
self.context_window = self.limit.as_ref().map(|l| l.context);
|
||||
self.context_length = self.limit.as_ref().map(|l| l.context);
|
||||
self.max_input_tokens = self.limit.as_ref().and_then(|l| l.input);
|
||||
// The ceiling, not the reserve (#278): a client that reads this
|
||||
// field turns it into the cap it sends on each request, so
|
||||
// advertising the reserve handed a reasoning model less budget
|
||||
// than its own think block needs. Falls back to the reserve for
|
||||
// a neuron too old to publish a ceiling.
|
||||
//
|
||||
// Observed with dsh. pi is *not* an example: its `maxTokens` is
|
||||
// a required static field in models.json and it reads nothing
|
||||
// from here — an operator has to copy this number across by
|
||||
// hand, and gets no error if they copy it wrong.
|
||||
self.max_output_tokens = self.limit.as_ref().map(|l| {
|
||||
if l.output_ceiling > 0 {
|
||||
l.output_ceiling
|
||||
} else {
|
||||
l.output
|
||||
}
|
||||
});
|
||||
self.max_output_tokens = self.limit.as_ref().map(|l| l.output);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -18,55 +18,8 @@ pub struct ChatCompletionRequest {
|
||||
pub temperature: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub top_p: Option<f64>,
|
||||
/// Truncate sampling to the `k` most likely tokens (#272).
|
||||
///
|
||||
/// Not in the OpenAI schema but universal among serving stacks
|
||||
/// (vLLM, SGLang, llama.cpp, Ollama), and required by models that
|
||||
/// publish it — Qwen3 specifies `top_k = 20`. A named field rather
|
||||
/// than `extra`, because a sampling parameter that lands in `extra`
|
||||
/// is accepted and silently discarded.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub top_k: Option<usize>,
|
||||
/// Pin the sampler's RNG for reproducible output (OpenAI core).
|
||||
///
|
||||
/// Previously ignored on every text path, which called
|
||||
/// `unix_subsec_nanos()` regardless — a caller asking for
|
||||
/// determinism got fresh randomness and a `200`.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub seed: Option<u64>,
|
||||
/// Penalty applied to recently-generated tokens; `1.0` disables.
|
||||
/// Defaults to 1.1 when absent (#272).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub repetition_penalty: Option<f32>,
|
||||
/// How many recent tokens `repetition_penalty` considers. Defaults
|
||||
/// to 64 — fine for chat, short for a long reasoning block, which is
|
||||
/// why it is a knob rather than a constant (#272).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub repeat_last_n: Option<usize>,
|
||||
/// OpenAI core, `-2.0..=2.0`. Subtracted once from any token that
|
||||
/// has already appeared, regardless of how often.
|
||||
///
|
||||
/// Unlike `repetition_penalty` this is scored over the **whole**
|
||||
/// generated sequence, not a trailing window — which is what a
|
||||
/// reasoning model needs, and what Qwen recommend (0..2, 1.5 for
|
||||
/// severe cases) for the endless-repetition failure their own
|
||||
/// models exhibit during long thinking.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub presence_penalty: Option<f32>,
|
||||
/// OpenAI core, `-2.0..=2.0`. Subtracted in proportion to how many
|
||||
/// times a token has already appeared.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub frequency_penalty: Option<f32>,
|
||||
/// The deprecated spelling of the output cap. Still what most
|
||||
/// clients send; see [`ChatCompletionRequest::effective_max_tokens`].
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub max_tokens: Option<u64>,
|
||||
/// OpenAI's current spelling of the output cap, which deprecated
|
||||
/// `max_tokens`. Newer SDKs and clients send only this one — it has
|
||||
/// to be a named field rather than falling into `extra`, or the cap
|
||||
/// is silently ignored and generation runs to the server default.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_completion_tokens: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stream: Option<bool>,
|
||||
/// All other fields (tools, response_format, backend extensions, etc.)
|
||||
@@ -74,49 +27,20 @@ pub struct ChatCompletionRequest {
|
||||
pub extra: Value,
|
||||
}
|
||||
|
||||
impl ChatCompletionRequest {
|
||||
/// The output cap the caller asked for, under either spelling.
|
||||
///
|
||||
/// `max_completion_tokens` wins when both are present — it is the
|
||||
/// current OpenAI field, and cortex's metering already reserves
|
||||
/// against it (`metering::requested_max_output`), so preferring it
|
||||
/// here keeps what we bill and what we generate in agreement.
|
||||
pub fn effective_max_tokens(&self) -> Option<u64> {
|
||||
self.max_completion_tokens.or(self.max_tokens)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ChatMessage {
|
||||
pub role: String,
|
||||
/// Absent on an assistant turn that carries only `tool_calls` —
|
||||
/// see [`MessageContent::Null`]. `#[serde(default)]` so a message
|
||||
/// with no `content` key at all deserializes rather than being
|
||||
/// rejected.
|
||||
#[serde(default)]
|
||||
pub content: MessageContent,
|
||||
#[serde(flatten)]
|
||||
pub extra: Value,
|
||||
}
|
||||
|
||||
/// Content can be a simple string, an array of content parts (for
|
||||
/// vision), or absent.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
/// Content can be a simple string or an array of content parts (for vision).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum MessageContent {
|
||||
Text(String),
|
||||
Parts(Vec<Value>),
|
||||
/// `"content": null`, or no `content` key at all.
|
||||
///
|
||||
/// This is the OpenAI-canonical shape for an assistant turn whose
|
||||
/// only payload is `tool_calls`, and agentic clients replay the
|
||||
/// assistant turn verbatim on the follow-up request that carries
|
||||
/// the tool result — so any client doing OpenAI-native tool
|
||||
/// calling sends it on its second turn. HF chat templates model it
|
||||
/// the same way (`content is none` renders as empty), so it maps
|
||||
/// straight through to the prompt.
|
||||
#[default]
|
||||
Null,
|
||||
}
|
||||
|
||||
// ── Chat completion response (non-streaming) ─────────────────────────
|
||||
@@ -188,9 +112,8 @@ pub struct Usage {
|
||||
/// `completion_tokens`, never added into `total_tokens`).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub completion_tokens_details: Option<CompletionTokensDetails>,
|
||||
/// OpenAI-standard breakdown of `prompt_tokens`, carrying
|
||||
/// `cached_tokens` (#269). Omitted when nothing was reused, so
|
||||
/// "absent" stays distinguishable from "measured as zero".
|
||||
/// OpenAI-standard breakdown of `prompt_tokens`. Populated once
|
||||
/// prompt caching lands (#11); `None` until then.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub prompt_tokens_details: Option<PromptTokensDetails>,
|
||||
/// helexa extension (non-OpenAI): server-measured prefill/decode
|
||||
@@ -222,9 +145,8 @@ pub struct CompletionTokensDetails {
|
||||
/// Sub-counts of `Usage::prompt_tokens`.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct PromptTokensDetails {
|
||||
/// Prompt tokens served from neuron's prefix KV cache (#11), so a
|
||||
/// caller can see the saving rather than assume none (#269). A
|
||||
/// sub-count of `prompt_tokens`, never added into `total_tokens`.
|
||||
/// Prompt tokens served from cache (cache-read rate). Populated
|
||||
/// once prompt caching lands (#11).
|
||||
pub cached_tokens: u64,
|
||||
}
|
||||
|
||||
@@ -248,133 +170,3 @@ pub struct ModelObject {
|
||||
#[serde(flatten)]
|
||||
pub extra: Value,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The follow-up request an OpenAI-native agentic client sends
|
||||
/// after running a tool: it replays its own assistant turn, which
|
||||
/// OpenAI emits with `"content": null` because the payload was
|
||||
/// only `tool_calls`. Rejecting this shape breaks every such
|
||||
/// client on its second turn.
|
||||
#[test]
|
||||
fn assistant_tool_call_turn_with_null_content_deserializes() {
|
||||
let req: ChatCompletionRequest = serde_json::from_str(
|
||||
r#"{
|
||||
"model": "m",
|
||||
"messages": [
|
||||
{"role": "user", "content": "list the files"},
|
||||
{"role": "assistant", "content": null, "tool_calls": [
|
||||
{"id": "call_0", "type": "function",
|
||||
"function": {"name": "bash", "arguments": "{\"command\":\"ls\"}"}}
|
||||
]},
|
||||
{"role": "tool", "content": "a.txt", "tool_call_id": "call_0"}
|
||||
]
|
||||
}"#,
|
||||
)
|
||||
.expect("null assistant content is a valid OpenAI request");
|
||||
|
||||
assert!(matches!(req.messages[1].content, MessageContent::Null));
|
||||
// The tool calls survive in `extra`, where the chat template
|
||||
// reads them from.
|
||||
assert_eq!(
|
||||
req.messages[1].extra["tool_calls"][0]["function"]["name"],
|
||||
"bash"
|
||||
);
|
||||
assert_eq!(req.messages[2].extra["tool_call_id"], "call_0");
|
||||
}
|
||||
|
||||
/// Some clients omit the key entirely rather than sending null.
|
||||
#[test]
|
||||
fn assistant_turn_with_no_content_key_deserializes() {
|
||||
let msg: ChatMessage = serde_json::from_str(
|
||||
r#"{"role": "assistant", "tool_calls": [
|
||||
{"id": "c", "type": "function",
|
||||
"function": {"name": "bash", "arguments": "{}"}}]}"#,
|
||||
)
|
||||
.expect("absent content is a valid OpenAI message");
|
||||
assert!(matches!(msg.content, MessageContent::Null));
|
||||
}
|
||||
|
||||
/// Absent content round-trips back onto the wire as null, so a
|
||||
/// translated or re-serialized request stays OpenAI-shaped.
|
||||
#[test]
|
||||
fn null_content_round_trips() {
|
||||
let msg = ChatMessage {
|
||||
role: "assistant".into(),
|
||||
content: MessageContent::Null,
|
||||
extra: Value::Null,
|
||||
};
|
||||
let v = serde_json::to_value(&msg).expect("serialize");
|
||||
assert_eq!(v["content"], Value::Null);
|
||||
assert!(v.get("content").is_some(), "content key must be present");
|
||||
}
|
||||
|
||||
/// Newer OpenAI SDKs send only `max_completion_tokens`; the cap
|
||||
/// must survive as a named field rather than falling into `extra`,
|
||||
/// where every generation loop would miss it.
|
||||
#[test]
|
||||
fn max_completion_tokens_is_the_effective_cap() {
|
||||
let req: ChatCompletionRequest = serde_json::from_str(
|
||||
r#"{"model": "m", "max_completion_tokens": 240,
|
||||
"messages": [{"role": "user", "content": "hi"}]}"#,
|
||||
)
|
||||
.expect("deserialize");
|
||||
assert_eq!(req.max_completion_tokens, Some(240));
|
||||
assert_eq!(req.max_tokens, None);
|
||||
assert_eq!(req.effective_max_tokens(), Some(240));
|
||||
assert!(
|
||||
req.extra.get("max_completion_tokens").is_none(),
|
||||
"must be a named field, not swept into extra"
|
||||
);
|
||||
}
|
||||
|
||||
/// The legacy spelling keeps working on its own.
|
||||
#[test]
|
||||
fn legacy_max_tokens_still_caps() {
|
||||
let req: ChatCompletionRequest = serde_json::from_str(
|
||||
r#"{"model": "m", "max_tokens": 128,
|
||||
"messages": [{"role": "user", "content": "hi"}]}"#,
|
||||
)
|
||||
.expect("deserialize");
|
||||
assert_eq!(req.effective_max_tokens(), Some(128));
|
||||
}
|
||||
|
||||
/// Clients that send both for compatibility must not be rejected,
|
||||
/// and the current field wins — matching what cortex meters
|
||||
/// against, so billed and generated caps agree.
|
||||
#[test]
|
||||
fn both_spellings_resolve_to_the_current_field() {
|
||||
let req: ChatCompletionRequest = serde_json::from_str(
|
||||
r#"{"model": "m", "max_tokens": 99, "max_completion_tokens": 256,
|
||||
"messages": [{"role": "user", "content": "hi"}]}"#,
|
||||
)
|
||||
.expect("sending both must not be an error");
|
||||
assert_eq!(req.effective_max_tokens(), Some(256));
|
||||
}
|
||||
|
||||
/// Neither spelling present leaves the cap to the server default.
|
||||
#[test]
|
||||
fn absent_cap_is_none() {
|
||||
let req: ChatCompletionRequest = serde_json::from_str(
|
||||
r#"{"model": "m", "messages": [{"role": "user", "content": "hi"}]}"#,
|
||||
)
|
||||
.expect("deserialize");
|
||||
assert_eq!(req.effective_max_tokens(), None);
|
||||
}
|
||||
|
||||
/// The string and array forms keep working unchanged.
|
||||
#[test]
|
||||
fn text_and_parts_content_still_deserialize() {
|
||||
let text: ChatMessage =
|
||||
serde_json::from_str(r#"{"role": "user", "content": "hi"}"#).expect("text");
|
||||
assert!(matches!(text.content, MessageContent::Text(ref t) if t == "hi"));
|
||||
|
||||
let parts: ChatMessage = serde_json::from_str(
|
||||
r#"{"role": "user", "content": [{"type": "text", "text": "hi"}]}"#,
|
||||
)
|
||||
.expect("parts");
|
||||
assert!(matches!(parts.content, MessageContent::Parts(ref p) if p.len() == 1));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,26 +55,6 @@ pub struct ResponsesRequest {
|
||||
pub temperature: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub top_p: Option<f64>,
|
||||
/// Truncate sampling to the `k` most likely tokens (#272). Not in
|
||||
/// the Responses schema, but a sampling parameter that lands in
|
||||
/// `extra` is accepted and silently discarded, which is the defect
|
||||
/// this closes.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub top_k: Option<usize>,
|
||||
/// Pin the sampler's RNG for reproducible output (#272).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub seed: Option<u64>,
|
||||
/// Penalty on recently-generated tokens; `1.0` disables (#272).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub repetition_penalty: Option<f32>,
|
||||
/// Window `repetition_penalty` considers (#272).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub repeat_last_n: Option<usize>,
|
||||
/// OpenAI core; forwarded to the chat path unchanged.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub presence_penalty: Option<f32>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub frequency_penalty: Option<f32>,
|
||||
/// Chained-conversation identifier. We don't store responses
|
||||
/// server-side yet; if this is `Some`, the handler returns 400.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
@@ -150,22 +130,12 @@ pub enum ResponsesInputItem {
|
||||
/// string or an array of content parts; the translator renders
|
||||
/// either form to text rather than losing the tool result.
|
||||
FunctionCallOutput { call_id: String, output: Value },
|
||||
/// A reasoning item the model emitted on an earlier turn, replayed
|
||||
/// by the client so the model can continue its own train of thought.
|
||||
///
|
||||
/// Both spellings are modelled because both are in the wild and one
|
||||
/// of them is ours: OpenAI's o-series carries the text in `content`
|
||||
/// as `reasoning_text` parts, while neuron emits `summary` with a
|
||||
/// `summary_text` part. Clients round-trip the completed item
|
||||
/// verbatim — pi-ai stores our `response.output_item.done` payload
|
||||
/// and replays it unchanged — so whichever field we emit is the
|
||||
/// field we get back, and reading only one of them silently loses
|
||||
/// the turn's reasoning.
|
||||
/// Reasoning items emitted by o-series models. Accepted but
|
||||
/// not forwarded to the model — neuron's candle path doesn't
|
||||
/// surface reasoning separately yet.
|
||||
Reasoning {
|
||||
#[serde(default)]
|
||||
content: Vec<Value>,
|
||||
#[serde(default)]
|
||||
summary: Vec<Value>,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -220,46 +190,11 @@ pub struct ResponsesResponse {
|
||||
pub status: String,
|
||||
pub model: String,
|
||||
pub output: Vec<ResponsesOutputItem>,
|
||||
/// Why a `status: "incomplete"` response stopped short. Required in
|
||||
/// practice, not optional decoration: a client cannot tell an
|
||||
/// honest truncation from a protocol fault without it, and at least
|
||||
/// one treats the difference as fatal.
|
||||
///
|
||||
/// pi-ai's `mapStopReason` maps
|
||||
/// `incomplete` + `reason == "max_output_tokens"` to
|
||||
/// `stopReason: "length"`, and `incomplete` with no reason to
|
||||
/// `stopReason: "error"` ("Response incomplete without a provider
|
||||
/// reason"). Its agent loop then either continues — failing any
|
||||
/// truncated tool calls safely and letting the model recover on the
|
||||
/// next turn — or halts outright. Omitting this field cost a live
|
||||
/// agentic session its whole run.
|
||||
///
|
||||
/// `None` for `completed` and `in_progress`.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub incomplete_details: Option<IncompleteDetails>,
|
||||
/// Populated on completion; `None` while streaming.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub usage: Option<ResponsesUsage>,
|
||||
}
|
||||
|
||||
/// Why a response is `incomplete`.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct IncompleteDetails {
|
||||
/// OpenAI spells the output-cap case `"max_output_tokens"`; that is
|
||||
/// the only reason we produce today. `content_filter` is the other
|
||||
/// value the upstream API defines.
|
||||
pub reason: String,
|
||||
}
|
||||
|
||||
impl IncompleteDetails {
|
||||
/// The response ran into the caller's output budget.
|
||||
pub fn max_output_tokens() -> Self {
|
||||
Self {
|
||||
reason: "max_output_tokens".into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum ResponsesOutputItem {
|
||||
@@ -276,15 +211,6 @@ pub enum ResponsesOutputItem {
|
||||
#[serde(default = "default_item_status")]
|
||||
status: String,
|
||||
},
|
||||
/// A think block, as its own output item ahead of the message —
|
||||
/// the same shape the streaming projector emits, so a client sees
|
||||
/// one contract whether or not it streams (#300).
|
||||
Reasoning {
|
||||
id: String,
|
||||
summary: Vec<ResponsesSummaryPart>,
|
||||
#[serde(default = "default_item_status")]
|
||||
status: String,
|
||||
},
|
||||
/// Reserved for the day tool-call extraction lands. The wire
|
||||
/// shape mirrors `ResponsesInputItem::FunctionCall`.
|
||||
FunctionCall {
|
||||
@@ -297,24 +223,6 @@ pub enum ResponsesOutputItem {
|
||||
},
|
||||
}
|
||||
|
||||
/// One `summary_text` part of a reasoning output item.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ResponsesSummaryPart {
|
||||
/// Always `"summary_text"`.
|
||||
#[serde(rename = "type")]
|
||||
pub part_type: String,
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
impl ResponsesSummaryPart {
|
||||
pub fn summary_text(text: impl Into<String>) -> Self {
|
||||
Self {
|
||||
part_type: "summary_text".into(),
|
||||
text: text.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn default_item_status() -> String {
|
||||
"completed".into()
|
||||
}
|
||||
@@ -341,8 +249,8 @@ pub struct ResponsesUsage {
|
||||
/// sub-count of `output_tokens`, never added into `total_tokens`).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_tokens_details: Option<OutputTokensDetails>,
|
||||
/// OpenAI-standard breakdown of `input_tokens`, carrying
|
||||
/// `cached_tokens` (#269). Omitted when nothing was reused.
|
||||
/// OpenAI-standard breakdown of `input_tokens`. Populated once
|
||||
/// prompt caching lands (#11); `None` until then.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_tokens_details: Option<InputTokensDetails>,
|
||||
}
|
||||
@@ -357,8 +265,8 @@ pub struct OutputTokensDetails {
|
||||
/// Sub-counts of `ResponsesUsage::input_tokens`.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct InputTokensDetails {
|
||||
/// Input tokens served from neuron's prefix KV cache (#11), so a
|
||||
/// caller can see the saving rather than assume none (#269).
|
||||
/// Input tokens served from cache (cache-read rate). Populated
|
||||
/// once prompt caching lands (#11).
|
||||
pub cached_tokens: u64,
|
||||
}
|
||||
|
||||
@@ -381,37 +289,9 @@ pub mod events {
|
||||
pub const OUTPUT_TEXT_DONE: &str = "response.output_text.done";
|
||||
pub const CONTENT_PART_DONE: &str = "response.content_part.done";
|
||||
pub const OUTPUT_ITEM_DONE: &str = "response.output_item.done";
|
||||
/// Reasoning-item event family. A reasoning model's think block is
|
||||
/// its own output item, emitted ahead of the message item, with a
|
||||
/// summary part carrying the text. Streaming these is what keeps the
|
||||
/// connection observably alive while a model thinks — clients time
|
||||
/// out on silence, and SSE comment keep-alives don't count because
|
||||
/// idle timers reset on parsed events, not comments.
|
||||
pub const REASONING_SUMMARY_PART_ADDED: &str = "response.reasoning_summary_part.added";
|
||||
pub const REASONING_SUMMARY_TEXT_DELTA: &str = "response.reasoning_summary_text.delta";
|
||||
pub const REASONING_SUMMARY_TEXT_DONE: &str = "response.reasoning_summary_text.done";
|
||||
pub const REASONING_SUMMARY_PART_DONE: &str = "response.reasoning_summary_part.done";
|
||||
pub const FUNCTION_CALL_ARGUMENTS_DELTA: &str = "response.function_call_arguments.delta";
|
||||
pub const FUNCTION_CALL_ARGUMENTS_DONE: &str = "response.function_call_arguments.done";
|
||||
pub const COMPLETED: &str = "response.completed";
|
||||
/// Terminal frame for a response that stopped short. Carries the
|
||||
/// same `response` payload as [`COMPLETED`], with
|
||||
/// `status: "incomplete"` and `incomplete_details.reason`.
|
||||
///
|
||||
/// A client keyed on the event name — rather than on
|
||||
/// `response.status` — sees no terminal event at all if a truncated
|
||||
/// response is announced as `response.completed`, and waits out its
|
||||
/// idle timeout.
|
||||
pub const INCOMPLETE: &str = "response.incomplete";
|
||||
/// Terminal frame for a response that did not finish because the
|
||||
/// server failed — a poisoned device, an OOM, a producer that died
|
||||
/// mid-stream. Carries `status: "failed"` and an `error` object.
|
||||
///
|
||||
/// The alternative is worse than it looks: defaulting a missing
|
||||
/// finish to a clean stop reports a crash as a complete answer, and
|
||||
/// the caller believes a truncated or empty reply was the model's
|
||||
/// considered response.
|
||||
pub const FAILED: &str = "response.failed";
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -616,7 +496,6 @@ mod tests {
|
||||
}],
|
||||
status: "completed".into(),
|
||||
}],
|
||||
incomplete_details: None,
|
||||
usage: Some(ResponsesUsage {
|
||||
input_tokens: 5,
|
||||
output_tokens: 3,
|
||||
@@ -629,36 +508,5 @@ mod tests {
|
||||
let parsed: ResponsesResponse = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(parsed.id, "resp_1");
|
||||
assert_eq!(parsed.output.len(), 1);
|
||||
assert!(
|
||||
!json.contains("incomplete_details"),
|
||||
"a completed response must not carry an incomplete reason"
|
||||
);
|
||||
}
|
||||
|
||||
/// The field pi-ai's `mapStopReason` keys on. Absent, it maps
|
||||
/// `incomplete` to `stopReason: "error"` and its agent loop halts;
|
||||
/// present with `max_output_tokens`, it maps to `"length"` and the
|
||||
/// loop continues.
|
||||
#[test]
|
||||
fn an_incomplete_response_serialises_its_reason() {
|
||||
let r = ResponsesResponse {
|
||||
id: "resp_2".into(),
|
||||
object: "response".into(),
|
||||
created_at: 1700,
|
||||
status: "incomplete".into(),
|
||||
model: "m".into(),
|
||||
output: vec![],
|
||||
incomplete_details: Some(IncompleteDetails::max_output_tokens()),
|
||||
usage: None,
|
||||
};
|
||||
let v: serde_json::Value = serde_json::to_value(&r).unwrap();
|
||||
assert_eq!(v["status"], "incomplete");
|
||||
assert_eq!(v["incomplete_details"]["reason"], "max_output_tokens");
|
||||
|
||||
let parsed: ResponsesResponse = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(
|
||||
parsed.incomplete_details,
|
||||
Some(IncompleteDetails::max_output_tokens())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -94,28 +94,6 @@ pub fn anthropic_to_openai(req: MessagesRequest) -> ChatCompletionRequest {
|
||||
if let Some(tc) = tool_choice {
|
||||
obj.insert("tool_choice".into(), tc);
|
||||
}
|
||||
// Anthropic's extended-thinking budget is the one quantitative
|
||||
// reasoning control any of our wire formats offers (#223).
|
||||
// Normalised onto `reasoning.max_tokens`, the shape the harness
|
||||
// resolves, so all three inlets converge on one field instead of
|
||||
// each generation loop learning three spellings.
|
||||
let budget = obj
|
||||
.get("thinking")
|
||||
.filter(|t| t.get("type").and_then(Value::as_str) != Some("disabled"))
|
||||
.and_then(|t| t.get("budget_tokens"))
|
||||
.and_then(Value::as_u64)
|
||||
.filter(|n| *n > 0);
|
||||
if let Some(budget) = budget {
|
||||
let reasoning = obj
|
||||
.entry("reasoning")
|
||||
.or_insert_with(|| Value::Object(Default::default()));
|
||||
if !reasoning.is_object() {
|
||||
*reasoning = Value::Object(Default::default());
|
||||
}
|
||||
if let Some(r) = reasoning.as_object_mut() {
|
||||
r.entry("max_tokens").or_insert_with(|| Value::from(budget));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ChatCompletionRequest {
|
||||
@@ -123,18 +101,7 @@ pub fn anthropic_to_openai(req: MessagesRequest) -> ChatCompletionRequest {
|
||||
messages,
|
||||
temperature: req.temperature,
|
||||
top_p: req.top_p,
|
||||
// Anthropic models top_k too, so it survives the hop (#272).
|
||||
top_k: req.top_k,
|
||||
// Anthropic's Messages API has no seed or penalty knobs; absent
|
||||
// means the model's own generation_config decides.
|
||||
seed: None,
|
||||
repetition_penalty: None,
|
||||
repeat_last_n: None,
|
||||
presence_penalty: None,
|
||||
frequency_penalty: None,
|
||||
max_tokens: Some(req.max_tokens),
|
||||
// Anthropic has one spelling; it maps onto the legacy field.
|
||||
max_completion_tokens: None,
|
||||
stream: req.stream,
|
||||
extra,
|
||||
}
|
||||
@@ -415,26 +382,16 @@ fn anthropic_content_to_text(content: AnthropicContent) -> String {
|
||||
pub fn openai_to_anthropic(resp: ChatCompletionResponse) -> MessagesResponse {
|
||||
let choice = resp.choices.into_iter().next();
|
||||
|
||||
let (content_text, reasoning_text, stop_reason) = match choice {
|
||||
let (content_text, stop_reason) = match choice {
|
||||
Some(c) => {
|
||||
let text = match c.message.content {
|
||||
MessageContent::Text(t) => t,
|
||||
MessageContent::Parts(parts) => serde_json::to_string(&parts).unwrap_or_default(),
|
||||
MessageContent::Null => String::new(),
|
||||
};
|
||||
// The model's think block, which the harness returns on
|
||||
// `message.reasoning_content` (#300).
|
||||
let reasoning = c
|
||||
.message
|
||||
.extra
|
||||
.get("reasoning_content")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(str::to_string)
|
||||
.filter(|t| !t.is_empty());
|
||||
let stop = c.finish_reason.map(|r| map_stop_reason(&r));
|
||||
(text, reasoning, stop)
|
||||
(text, stop)
|
||||
}
|
||||
None => (String::new(), None, None),
|
||||
None => (String::new(), None),
|
||||
};
|
||||
|
||||
let usage = resp.usage.unwrap_or(Usage {
|
||||
@@ -450,32 +407,10 @@ pub fn openai_to_anthropic(resp: ChatCompletionResponse) -> MessagesResponse {
|
||||
id: resp.id,
|
||||
response_type: "message".into(),
|
||||
role: "assistant".into(),
|
||||
// A `thinking` block ahead of the text, which is Anthropic's
|
||||
// own ordering. This path accepted `thinking.budget_tokens` on
|
||||
// the way in (see `anthropic_to_openai`) and returned nothing on
|
||||
// the way out: the caller spent the tokens, could not see them,
|
||||
// and could not replay the model's reasoning next turn (#300).
|
||||
//
|
||||
// Emitted without a `signature`. Anthropic signs its thinking
|
||||
// blocks and a client that round-trips one may expect the field;
|
||||
// we cannot mint a valid signature and inventing one would be
|
||||
// worse than omitting it. If a real client rejects an unsigned
|
||||
// block, the fix is to gate this behind a flag — not to go back
|
||||
// to silently deleting the model's work.
|
||||
content: {
|
||||
let mut blocks = Vec::with_capacity(2);
|
||||
if let Some(thinking) = reasoning_text {
|
||||
blocks.push(ContentBlock {
|
||||
block_type: "thinking".into(),
|
||||
data: json!({ "thinking": thinking }),
|
||||
});
|
||||
}
|
||||
blocks.push(ContentBlock {
|
||||
block_type: "text".into(),
|
||||
data: json!({ "text": content_text }),
|
||||
});
|
||||
blocks
|
||||
},
|
||||
content: vec![ContentBlock {
|
||||
block_type: "text".into(),
|
||||
data: json!({ "text": content_text }),
|
||||
}],
|
||||
model: resp.model,
|
||||
stop_reason,
|
||||
usage: AnthropicUsage {
|
||||
@@ -532,7 +467,6 @@ pub struct AnthropicStreamTranslator {
|
||||
enum OpenBlock {
|
||||
Text,
|
||||
ToolUse,
|
||||
Thinking,
|
||||
}
|
||||
|
||||
impl AnthropicStreamTranslator {
|
||||
@@ -572,32 +506,6 @@ impl AnthropicStreamTranslator {
|
||||
}
|
||||
|
||||
for choice in &chunk.choices {
|
||||
// The model's think block. Dropped entirely until #300: this
|
||||
// translator read `content` and `tool_calls` and nothing
|
||||
// else, so an Anthropic client asked for extended thinking
|
||||
// (`thinking.budget_tokens`, which `anthropic_to_openai`
|
||||
// accepts), was billed for the tokens, and received a stream
|
||||
// with no thinking in it — every turn, silently. Fixing only
|
||||
// the non-streaming translation would have missed this,
|
||||
// because the clients that ask for thinking stream.
|
||||
if let Some(text) = choice
|
||||
.delta
|
||||
.get("reasoning_content")
|
||||
.and_then(Value::as_str)
|
||||
&& !text.is_empty()
|
||||
{
|
||||
self.ensure_thinking_block(&mut out);
|
||||
let index = self.open_block.as_ref().map(|(i, _)| *i).unwrap_or(0);
|
||||
out.push((
|
||||
"content_block_delta".to_string(),
|
||||
json!({
|
||||
"type": "content_block_delta",
|
||||
"index": index,
|
||||
"delta": { "type": "thinking_delta", "thinking": text }
|
||||
}),
|
||||
));
|
||||
}
|
||||
|
||||
if let Some(text) = choice.delta.get("content").and_then(Value::as_str)
|
||||
&& !text.is_empty()
|
||||
{
|
||||
@@ -711,32 +619,6 @@ impl AnthropicStreamTranslator {
|
||||
out
|
||||
}
|
||||
|
||||
/// Open a `thinking` block, closing whatever preceded it.
|
||||
///
|
||||
/// Emitted without a `signature`. Anthropic signs its thinking
|
||||
/// blocks; we cannot mint a valid one, and inventing a value would
|
||||
/// be worse than omitting it. If a client rejects an unsigned block
|
||||
/// the fix is a flag — not a return to deleting the model's work.
|
||||
fn ensure_thinking_block(&mut self, out: &mut Vec<(String, Value)>) {
|
||||
match &self.open_block {
|
||||
Some((_, OpenBlock::Thinking)) => {}
|
||||
_ => {
|
||||
self.close_open_block(out);
|
||||
let index = self.next_index;
|
||||
self.next_index += 1;
|
||||
self.open_block = Some((index, OpenBlock::Thinking));
|
||||
out.push((
|
||||
"content_block_start".to_string(),
|
||||
json!({
|
||||
"type": "content_block_start",
|
||||
"index": index,
|
||||
"content_block": { "type": "thinking", "thinking": "" }
|
||||
}),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_text_block(&mut self, out: &mut Vec<(String, Value)>) {
|
||||
match &self.open_block {
|
||||
Some((_, OpenBlock::Text)) => {}
|
||||
@@ -1146,115 +1028,4 @@ mod request_tests {
|
||||
let tools = openai.extra.get("tools").and_then(Value::as_array).unwrap();
|
||||
assert_eq!(tools[0]["function"]["name"], "x");
|
||||
}
|
||||
|
||||
/// The Anthropic path must return the model's thinking (#300).
|
||||
///
|
||||
/// It accepted `thinking.budget_tokens` on the way in and returned
|
||||
/// a bare text block on the way out: the caller spent the tokens,
|
||||
/// never saw them, and could not replay the model's reasoning on
|
||||
/// the next turn. Taking a parameter and silently discarding what
|
||||
/// it buys is the defect, not the missing field.
|
||||
#[test]
|
||||
fn anthropic_response_carries_a_thinking_block() {
|
||||
let mut extra = serde_json::Map::new();
|
||||
extra.insert("reasoning_content".into(), json!("weighing the options"));
|
||||
let resp = ChatCompletionResponse {
|
||||
id: "id".into(),
|
||||
object: "chat.completion".into(),
|
||||
created: 0,
|
||||
model: "m".into(),
|
||||
choices: vec![crate::openai::ChatCompletionChoice {
|
||||
index: 0,
|
||||
message: ChatMessage {
|
||||
role: "assistant".into(),
|
||||
content: MessageContent::Text("the answer".into()),
|
||||
extra: Value::Object(extra),
|
||||
},
|
||||
finish_reason: Some("stop".into()),
|
||||
extra: Value::Null,
|
||||
}],
|
||||
usage: None,
|
||||
extra: Value::Null,
|
||||
};
|
||||
let a = openai_to_anthropic(resp);
|
||||
assert_eq!(a.content.len(), 2, "expected thinking + text");
|
||||
assert_eq!(a.content[0].block_type, "thinking", "thinking comes first");
|
||||
assert_eq!(a.content[0].data["thinking"], "weighing the options");
|
||||
assert_eq!(a.content[1].block_type, "text");
|
||||
assert_eq!(a.content[1].data["text"], "the answer");
|
||||
}
|
||||
|
||||
/// No thinking means the unchanged single text block.
|
||||
#[test]
|
||||
fn anthropic_response_without_thinking_is_unchanged() {
|
||||
let resp = ChatCompletionResponse {
|
||||
id: "id".into(),
|
||||
object: "chat.completion".into(),
|
||||
created: 0,
|
||||
model: "m".into(),
|
||||
choices: vec![crate::openai::ChatCompletionChoice {
|
||||
index: 0,
|
||||
message: ChatMessage {
|
||||
role: "assistant".into(),
|
||||
content: MessageContent::Text("plain".into()),
|
||||
extra: Value::Object(Default::default()),
|
||||
},
|
||||
finish_reason: Some("stop".into()),
|
||||
extra: Value::Null,
|
||||
}],
|
||||
usage: None,
|
||||
extra: Value::Null,
|
||||
};
|
||||
let a = openai_to_anthropic(resp);
|
||||
assert_eq!(a.content.len(), 1);
|
||||
assert_eq!(a.content[0].block_type, "text");
|
||||
}
|
||||
|
||||
/// Streaming Anthropic must carry thinking too (#300).
|
||||
///
|
||||
/// Fixing only the non-streaming translation would have missed this
|
||||
/// entirely — and missed it for the clients that matter, since the
|
||||
/// ones asking for extended thinking stream.
|
||||
#[test]
|
||||
fn anthropic_stream_emits_thinking_deltas() {
|
||||
use crate::openai::{ChatCompletionChunk, ChunkChoice};
|
||||
let mut t = AnthropicStreamTranslator::new();
|
||||
let chunk = |delta: Value| ChatCompletionChunk {
|
||||
id: "c".into(),
|
||||
object: "chat.completion.chunk".into(),
|
||||
created: 0,
|
||||
model: "m".into(),
|
||||
choices: vec![ChunkChoice {
|
||||
index: 0,
|
||||
delta,
|
||||
finish_reason: None,
|
||||
extra: Value::Null,
|
||||
}],
|
||||
usage: None,
|
||||
extra: Value::Null,
|
||||
};
|
||||
|
||||
let ev = t.on_chunk(&chunk(json!({ "reasoning_content": "weighing it" })));
|
||||
let start = ev
|
||||
.iter()
|
||||
.find(|(n, _)| n == "content_block_start")
|
||||
.expect("a block must open");
|
||||
assert_eq!(start.1["content_block"]["type"], "thinking");
|
||||
let delta = ev
|
||||
.iter()
|
||||
.find(|(n, _)| n == "content_block_delta")
|
||||
.expect("a delta must follow");
|
||||
assert_eq!(delta.1["delta"]["type"], "thinking_delta");
|
||||
assert_eq!(delta.1["delta"]["thinking"], "weighing it");
|
||||
|
||||
// Visible text opens a separate block, after thinking closes.
|
||||
let ev = t.on_chunk(&chunk(json!({ "content": "the answer" })));
|
||||
let names: Vec<&str> = ev.iter().map(|(n, _)| n.as_str()).collect();
|
||||
assert!(
|
||||
names.contains(&"content_block_stop"),
|
||||
"the thinking block must close first: {names:?}"
|
||||
);
|
||||
let start = ev.iter().find(|(n, _)| n == "content_block_start").unwrap();
|
||||
assert_eq!(start.1["content_block"]["type"], "text");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,23 +17,11 @@ pub async fn eviction_loop(fleet: Arc<CortexState>) {
|
||||
}
|
||||
}
|
||||
|
||||
/// Evict the least-recently-used model on a given node that `incoming`
|
||||
/// is permitted to displace.
|
||||
///
|
||||
/// `incoming` is the model the caller is trying to make room for. It is
|
||||
/// load-bearing, not diagnostic: whether a displacement is allowed is a
|
||||
/// question about the *pair*, so an evictor that only knows the victim
|
||||
/// cannot answer it. Passing `None` means "no particular model" and
|
||||
/// permits displacing anything at or below the default priority, which
|
||||
/// is the right reading for maintenance-driven eviction rather than a
|
||||
/// cold-load making room for itself.
|
||||
///
|
||||
/// Returns the model ID that was evicted, or None when nothing on the
|
||||
/// node may be displaced.
|
||||
/// Evict the least-recently-used model on a given node.
|
||||
/// Returns the model ID that was evicted, or None if nothing could be evicted.
|
||||
pub async fn evict_lru_on_node(
|
||||
fleet: &CortexState,
|
||||
node_name: &str,
|
||||
incoming: Option<&str>,
|
||||
) -> anyhow::Result<Option<String>> {
|
||||
let (neuron_endpoint, candidate) = {
|
||||
let nodes = fleet.nodes.read().await;
|
||||
@@ -41,18 +29,13 @@ pub async fn evict_lru_on_node(
|
||||
anyhow::bail!("node '{node_name}' not found");
|
||||
};
|
||||
|
||||
// Oldest first, among the models this incoming model outranks.
|
||||
// Find the loaded model with the oldest last_accessed,
|
||||
// excluding models pinned on this neuron (from catalogue).
|
||||
let candidate = node
|
||||
.models
|
||||
.values()
|
||||
.filter(|m| m.status == ModelStatus::Loaded)
|
||||
.filter(|m| match incoming {
|
||||
Some(inc) => fleet.catalogue.may_displace(inc, &m.id),
|
||||
None => {
|
||||
fleet.catalogue.residency_priority(&m.id)
|
||||
<= cortex_core::catalogue::DEFAULT_RESIDENCY_PRIORITY
|
||||
}
|
||||
})
|
||||
.filter(|m| !fleet.catalogue.is_pinned(&m.id, node_name))
|
||||
.min_by_key(|m| m.last_accessed)
|
||||
.map(|m| m.id.clone());
|
||||
|
||||
@@ -60,11 +43,7 @@ pub async fn evict_lru_on_node(
|
||||
};
|
||||
|
||||
let Some(model_id) = candidate else {
|
||||
tracing::info!(
|
||||
node = node_name,
|
||||
incoming = incoming.unwrap_or("(none)"),
|
||||
"no displaceable models found"
|
||||
);
|
||||
tracing::info!(node = node_name, "no evictable models found");
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
|
||||
@@ -730,11 +730,6 @@ async fn list_models(State(fleet): State<Arc<CortexState>>) -> Json<Value> {
|
||||
let now = Utc::now().timestamp() as u64;
|
||||
let nodes = fleet.nodes.read().await;
|
||||
let catalogue = &fleet.catalogue;
|
||||
// Modalities the neurons derived for models nothing has loaded
|
||||
// (#241). Without this a cold model advertises no capabilities at
|
||||
// all, which made image generation undiscoverable — the image model
|
||||
// is evicted almost all of the time.
|
||||
let discovered = fleet.discovered_capabilities.read().await;
|
||||
|
||||
let mut entries: HashMap<String, CortexModelEntry> = HashMap::new();
|
||||
|
||||
@@ -772,16 +767,9 @@ async fn list_models(State(fleet): State<Arc<CortexState>>) -> Json<Value> {
|
||||
loaded: false,
|
||||
feasible_on,
|
||||
locations: Vec::new(),
|
||||
// Start with what the neurons discovered for this model,
|
||||
// falling back to any catalogue declaration; Pass 2
|
||||
// unions runtime-detected ones from loaded neurons.
|
||||
// Discovery wins over the catalogue deliberately: it is
|
||||
// read from the weights the node would actually serve,
|
||||
// so it cannot drift the way a hand-written field does.
|
||||
capabilities: discovered
|
||||
.get(&profile.id)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| profile.capabilities.clone()),
|
||||
// Start with catalogue-declared capabilities; Pass 2 unions
|
||||
// runtime-detected ones from loaded neurons.
|
||||
capabilities: profile.capabilities.clone(),
|
||||
// `limit` is no longer operator-declared (#67): the neuron
|
||||
// self-derives it from live VRAM + throughput and reports it
|
||||
// per loaded model — Pass 2 fills it from the neuron's
|
||||
@@ -798,10 +786,6 @@ async fn list_models(State(fleet): State<Arc<CortexState>>) -> Json<Value> {
|
||||
max_model_len: None,
|
||||
max_input_tokens: None,
|
||||
max_output_tokens: None,
|
||||
context_window: None,
|
||||
context_length: None,
|
||||
// Catalogue-seeded: no neuron has reported rungs yet.
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -843,13 +827,6 @@ async fn list_models(State(fleet): State<Arc<CortexState>>) -> Json<Value> {
|
||||
// so a client never overflows the most-constrained
|
||||
// deployment that might serve it.
|
||||
e.limit = tightest_limit(e.limit.take(), entry.limit.clone());
|
||||
// The reasoning ladder this neuron honours (#223).
|
||||
// First non-empty wins: a catalogue-seeded entry
|
||||
// starts with none, and neurons serving the same
|
||||
// model are configured alike.
|
||||
if e.reasoning_budget.is_empty() {
|
||||
e.reasoning_budget = node.reasoning_budget.clone();
|
||||
}
|
||||
})
|
||||
.or_insert_with(|| CortexModelEntry {
|
||||
id: model_id.clone(),
|
||||
@@ -869,10 +846,6 @@ async fn list_models(State(fleet): State<Arc<CortexState>>) -> Json<Value> {
|
||||
max_model_len: None,
|
||||
max_input_tokens: None,
|
||||
max_output_tokens: None,
|
||||
context_window: None,
|
||||
context_length: None,
|
||||
// What this neuron says each effort level buys (#223).
|
||||
reasoning_budget: node.reasoning_budget.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -923,10 +896,8 @@ async fn list_models(State(fleet): State<Arc<CortexState>>) -> Json<Value> {
|
||||
feasible_on: Vec::new(),
|
||||
locations: vec![location],
|
||||
// A model that's only mid-prewarm has no loaded
|
||||
// location to read capabilities from yet — but a
|
||||
// neuron may still have derived them from its cache
|
||||
// (#241), so prefer that over admitting nothing.
|
||||
capabilities: discovered.get(model_id).cloned().unwrap_or_default(),
|
||||
// location to read capabilities from yet.
|
||||
capabilities: Vec::new(),
|
||||
limit: None,
|
||||
cost: None,
|
||||
tool_call: false,
|
||||
@@ -934,10 +905,6 @@ async fn list_models(State(fleet): State<Arc<CortexState>>) -> Json<Value> {
|
||||
max_model_len: None,
|
||||
max_input_tokens: None,
|
||||
max_output_tokens: None,
|
||||
context_window: None,
|
||||
context_length: None,
|
||||
// What this neuron says each effort level buys (#223).
|
||||
reasoning_budget: node.reasoning_budget.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -975,10 +942,6 @@ async fn list_models(State(fleet): State<Arc<CortexState>>) -> Json<Value> {
|
||||
max_model_len: None,
|
||||
max_input_tokens: None,
|
||||
max_output_tokens: None,
|
||||
context_window: None,
|
||||
context_length: None,
|
||||
// Catalogue-seeded: no neuron has reported rungs yet.
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -1001,55 +964,18 @@ async fn list_models(State(fleet): State<Arc<CortexState>>) -> Json<Value> {
|
||||
}
|
||||
|
||||
/// `GET /health`
|
||||
///
|
||||
/// Reports serviceability, not just reachability (#245). Counting
|
||||
/// reachable nodes alone once let the fleet report `3/3 healthy, ok`
|
||||
/// through a total outage of a whole tier: the node answered every poll
|
||||
/// correctly while rejecting every request, because something outside
|
||||
/// neuron had eaten its VRAM. Liveness was true and useless.
|
||||
async fn health(State(fleet): State<Arc<CortexState>>) -> Json<Value> {
|
||||
let nodes = fleet.nodes.read().await;
|
||||
let healthy_count = nodes.values().filter(|n| n.healthy).count();
|
||||
let total_count = nodes.len();
|
||||
|
||||
// Models resident on a node that cannot currently serve them. Worth
|
||||
// surfacing even when another node can pick the work up, because it
|
||||
// is always a fault on that host — and when no other node can, it is
|
||||
// the difference between "fine" and "a tier is down".
|
||||
let mut unservable: Vec<Value> = Vec::new();
|
||||
for node in nodes.values() {
|
||||
if !node.healthy {
|
||||
continue;
|
||||
Json(json!({
|
||||
"status": if healthy_count > 0 { "ok" } else { "degraded" },
|
||||
"nodes": {
|
||||
"healthy": healthy_count,
|
||||
"total": total_count,
|
||||
}
|
||||
for entry in node.models.values() {
|
||||
if entry.is_servable() {
|
||||
continue;
|
||||
}
|
||||
unservable.push(json!({
|
||||
"node": node.name,
|
||||
"model": entry.id,
|
||||
"reason": entry.servable.as_ref().and_then(|s| s.reason.clone()),
|
||||
"detail": entry.servable.as_ref().and_then(|s| s.detail.clone()),
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
let status = if healthy_count == 0 {
|
||||
"degraded"
|
||||
} else if !unservable.is_empty() {
|
||||
"impaired"
|
||||
} else {
|
||||
"ok"
|
||||
};
|
||||
|
||||
let mut body = json!({
|
||||
"status": status,
|
||||
"nodes": { "healthy": healthy_count, "total": total_count },
|
||||
});
|
||||
if !unservable.is_empty() {
|
||||
body["unservable"] = Value::Array(unservable);
|
||||
}
|
||||
Json(body)
|
||||
}))
|
||||
}
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────
|
||||
|
||||
@@ -141,15 +141,6 @@ pub fn usage_sink(
|
||||
/// returns a guard the caller settles with actual usage; on refusal returns
|
||||
/// the #63 envelope (`rate_limit_exceeded` + `Retry-After` for a resetting
|
||||
/// window, `insufficient_quota` for a hard balance — never `402`).
|
||||
// The `Err` variant is the #63 envelope itself, which is a data-carrying
|
||||
// struct (status, type, code, message, param, retry hint, diagnostics) and
|
||||
// is passed by value everywhere it is built, matched and rendered. Boxing it
|
||||
// at this one boundary would push `Box` conversions through three handlers
|
||||
// and both HTTP adapters to save an allocation on a path taken once per
|
||||
// rejected request. clippy 1.98 lowered what it considers large enough to
|
||||
// flag; the shape is deliberate. Same call as the existing allows on the
|
||||
// figment-returning config loaders.
|
||||
#[allow(clippy::result_large_err)]
|
||||
pub async fn reserve_or_reject(
|
||||
provider: Arc<dyn EntitlementProvider>,
|
||||
principal: &Principal,
|
||||
@@ -288,7 +279,7 @@ mod tests {
|
||||
// Requested output dominates; prompt estimate is small for a tiny body.
|
||||
let body = br#"{"model":"m","max_tokens":1000}"#;
|
||||
let est = reservation_estimate(body, Some(8192));
|
||||
assert!((1000..1100).contains(&est), "est was {est}");
|
||||
assert!(est >= 1000 && est < 1100, "est was {est}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
//! in Prometheus text format.
|
||||
|
||||
use anyhow::Result;
|
||||
use metrics_exporter_prometheus::{Matcher, PrometheusBuilder};
|
||||
use metrics_exporter_prometheus::PrometheusBuilder;
|
||||
use std::net::SocketAddr;
|
||||
|
||||
/// Install the Prometheus metrics recorder and return a handle.
|
||||
@@ -12,7 +12,8 @@ use std::net::SocketAddr;
|
||||
pub fn install(listen: &str) -> Result<()> {
|
||||
let addr: SocketAddr = listen.parse()?;
|
||||
|
||||
with_buckets(PrometheusBuilder::new().with_http_listener(addr))?
|
||||
PrometheusBuilder::new()
|
||||
.with_http_listener(addr)
|
||||
.install()
|
||||
.map_err(|e| anyhow::anyhow!("failed to install Prometheus exporter: {e}"))?;
|
||||
|
||||
@@ -24,77 +25,13 @@ pub fn install(listen: &str) -> Result<()> {
|
||||
/// Install a recorder for testing (no HTTP listener). Returns a handle
|
||||
/// that can render the current metrics as Prometheus text.
|
||||
pub fn install_test_recorder() -> Result<metrics_exporter_prometheus::PrometheusHandle> {
|
||||
let handle = with_buckets(PrometheusBuilder::new())?
|
||||
let handle = PrometheusBuilder::new()
|
||||
.install_recorder()
|
||||
.map_err(|e| anyhow::anyhow!("failed to install test recorder: {e}"))?;
|
||||
describe_metrics();
|
||||
Ok(handle)
|
||||
}
|
||||
|
||||
/// Give every histogram explicit buckets, so it is exported as a
|
||||
/// Prometheus **histogram** rather than a summary.
|
||||
///
|
||||
/// Without this, `metrics-exporter-prometheus` renders every
|
||||
/// `histogram!` as a summary: `{quantile="0.95"}` series and no
|
||||
/// `_bucket` series at all. Three things break as a result, and the
|
||||
/// third is what made it visible.
|
||||
///
|
||||
/// 1. `histogram_quantile()` has nothing to read, so any dashboard
|
||||
/// panel written the idiomatic way returns *No data* forever. The
|
||||
/// fleet dashboard's TTFT panel did exactly that while cortex was
|
||||
/// serving thousands of requests.
|
||||
/// 2. Summary quantiles are computed per process over a rolling
|
||||
/// window, so they **cannot be aggregated**. Averaging p95 across
|
||||
/// two gateways is not p95 of anything.
|
||||
/// 3. They decay: with no samples in the window every quantile reads
|
||||
/// `0`, which is indistinguishable from "genuinely instant" on a
|
||||
/// graph.
|
||||
///
|
||||
/// Buckets are per-metric because the quantities differ by orders of
|
||||
/// magnitude — a request lasting minutes and a TTFT of milliseconds
|
||||
/// have no useful shared scale. Ranges are chosen from observed fleet
|
||||
/// behaviour rather than round numbers: decode runs for minutes on a
|
||||
/// long turn, prefill is seconds on a long prompt, and decode
|
||||
/// throughput sits in the tens of tokens/sec.
|
||||
fn with_buckets(builder: PrometheusBuilder) -> Result<PrometheusBuilder> {
|
||||
let seconds_short = &[
|
||||
0.05, 0.1, 0.25, 0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 30.0, 60.0, 120.0,
|
||||
];
|
||||
let seconds_long = &[
|
||||
0.1, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0, 1200.0,
|
||||
];
|
||||
let tokens_per_second = &[
|
||||
1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0, 40.0, 60.0, 80.0, 120.0, 200.0,
|
||||
];
|
||||
builder
|
||||
// Whole-request latency: a long agentic turn legitimately runs
|
||||
// for many minutes, so the tail has to reach there.
|
||||
.set_buckets_for_metric(
|
||||
Matcher::Full("cortex_request_duration_seconds".into()),
|
||||
seconds_long,
|
||||
)?
|
||||
// Prefill: sub-second for a short prompt, seconds for a long
|
||||
// one. Anything past a minute is pathological and belongs in
|
||||
// the overflow bucket.
|
||||
.set_buckets_for_metric(
|
||||
Matcher::Full("cortex_time_to_first_token_seconds".into()),
|
||||
seconds_short,
|
||||
)?
|
||||
// Image generation is inherently slower than a text turn and
|
||||
// scales with resolution and step count.
|
||||
.set_buckets_for_metric(
|
||||
Matcher::Full("cortex_images_generation_seconds".into()),
|
||||
seconds_long,
|
||||
)?
|
||||
// Not a duration — decode throughput, tens of tokens/sec on
|
||||
// this fleet.
|
||||
.set_buckets_for_metric(
|
||||
Matcher::Full("cortex_tokens_per_second".into()),
|
||||
tokens_per_second,
|
||||
)
|
||||
.map_err(|e| anyhow::anyhow!("failed to configure histogram buckets: {e}"))
|
||||
}
|
||||
|
||||
fn describe_metrics() {
|
||||
metrics::describe_histogram!(
|
||||
"cortex_request_duration_seconds",
|
||||
@@ -121,10 +58,6 @@ fn describe_metrics() {
|
||||
"cortex_request_errors_total",
|
||||
"Total number of failed proxy requests"
|
||||
);
|
||||
metrics::describe_counter!(
|
||||
"cortex_cached_prompt_tokens_total",
|
||||
"Prompt tokens served from neuron's prefix KV cache; over cortex_prompt_tokens_total this is the cache-hit rate (#269)"
|
||||
);
|
||||
metrics::describe_counter!("cortex_evictions_total", "Total number of model evictions");
|
||||
metrics::describe_counter!(
|
||||
"cortex_cold_starts_total",
|
||||
@@ -178,15 +111,7 @@ fn describe_metrics() {
|
||||
);
|
||||
metrics::describe_counter!(
|
||||
"cortex_model_rejections_total",
|
||||
"Admission rejections per neuron:model by reason: queue_full / wait_timeout / per_principal / anon_yield — the load-shedding signal (#137, #262)"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
"cortex_model_anon_in_flight",
|
||||
"Anonymous (unattributable) requests holding a seat on a neuron:model (#262)"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
"cortex_model_anon_max_in_flight",
|
||||
"Seats anonymous traffic may hold at once, so it cannot starve identified callers (#262)"
|
||||
"Admission rejections per neuron:model by reason: queue_full / wait_timeout / per_principal — the load-shedding signal (#137)"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
"cortex_model_tok_s_decode",
|
||||
@@ -197,43 +122,3 @@ fn describe_metrics() {
|
||||
"Live prefill throughput per neuron:model, tokens/sec EMA (#137)"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod bucket_tests {
|
||||
/// Histograms must export as Prometheus **histograms**, with
|
||||
/// `_bucket` series — not as summaries.
|
||||
///
|
||||
/// This asserts the exported *shape*, not the recording call,
|
||||
/// because the recording was never the problem: cortex measured
|
||||
/// TTFT correctly for months while the fleet dashboard's panel sat
|
||||
/// on "No data", since `histogram_quantile()` reads `_bucket`
|
||||
/// series and a summary has none. A metric that is collected but
|
||||
/// cannot be queried is indistinguishable from one that was never
|
||||
/// collected — from the graph, and from the operator's chair.
|
||||
#[test]
|
||||
fn histograms_export_buckets_not_summary_quantiles() {
|
||||
let handle = match super::install_test_recorder() {
|
||||
Ok(h) => h,
|
||||
// Another test in this binary owns the global recorder;
|
||||
// it installs the same buckets, so skipping is honest.
|
||||
Err(_) => return,
|
||||
};
|
||||
metrics::histogram!("cortex_time_to_first_token_seconds", "node" => "n", "model" => "m")
|
||||
.record(0.42);
|
||||
let rendered = handle.render();
|
||||
|
||||
assert!(
|
||||
rendered.contains("cortex_time_to_first_token_seconds_bucket"),
|
||||
"no _bucket series — histogram_quantile() cannot read this:\n{rendered}"
|
||||
);
|
||||
assert!(
|
||||
rendered.contains("# TYPE cortex_time_to_first_token_seconds histogram"),
|
||||
"exported as the wrong metric type:\n{rendered}"
|
||||
);
|
||||
assert!(
|
||||
!rendered
|
||||
.contains(r#"cortex_time_to_first_token_seconds{node="n",model="m",quantile="#),
|
||||
"still exporting summary quantiles, which cannot be aggregated across gateways"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,12 +18,6 @@ const POLL_INTERVAL: Duration = Duration::from_secs(10);
|
||||
/// 10s poll interval this tolerates ~20s of flapping before evicting.
|
||||
const POLL_FAILURE_THRESHOLD: u32 = 3;
|
||||
|
||||
/// How long before re-asking the fleet about a model whose capabilities
|
||||
/// nobody could derive (#241). Long, because the usual cause — no node
|
||||
/// has the weights cached — is resolved by a deliberate operator action
|
||||
/// (a load, a prefetch) rather than by time passing.
|
||||
const CAPABILITY_RETRY: Duration = Duration::from_secs(600);
|
||||
|
||||
/// Record a failed poll for `node`, marking it unhealthy only once failures
|
||||
/// reach [`POLL_FAILURE_THRESHOLD`]. Below the threshold the node keeps its
|
||||
/// last-known health, riding over transient misses. A successful poll resets
|
||||
@@ -130,17 +124,6 @@ async fn poll_neuron(fleet: &CortexState, name: &str, endpoint: &str) {
|
||||
Ok(resp) if resp.status().is_success() => {
|
||||
match resp.json::<Vec<ModelInfo>>().await {
|
||||
Ok(models) => {
|
||||
// The reasoning-effort ladder is host configuration:
|
||||
// every model on a neuron reports the same rungs, so
|
||||
// it is held once per node rather than copied onto
|
||||
// each model entry where the two could drift (#223).
|
||||
// Taken from the first model that offers one — a
|
||||
// non-reasoning model advertises none.
|
||||
node.reasoning_budget = models
|
||||
.iter()
|
||||
.map(|m| m.reasoning_budget.clone())
|
||||
.find(|rungs| !rungs.is_empty())
|
||||
.unwrap_or_default();
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
for upstream in &models {
|
||||
seen.insert(upstream.id.clone());
|
||||
@@ -157,7 +140,6 @@ async fn poll_neuron(fleet: &CortexState, name: &str, endpoint: &str) {
|
||||
// Neuron's self-derived limit (#67) — the
|
||||
// authoritative source the gateway advertises.
|
||||
e.limit = upstream.limit.clone();
|
||||
e.servable = upstream.servable.clone();
|
||||
})
|
||||
.or_insert_with(|| ModelEntry {
|
||||
id: upstream.id.clone(),
|
||||
@@ -168,7 +150,6 @@ async fn poll_neuron(fleet: &CortexState, name: &str, endpoint: &str) {
|
||||
tool_call: upstream.tool_call,
|
||||
reasoning: upstream.reasoning,
|
||||
limit: upstream.limit.clone(),
|
||||
servable: upstream.servable.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -203,8 +184,6 @@ async fn poll_neuron(fleet: &CortexState, name: &str, endpoint: &str) {
|
||||
// Release the write lock before the next HTTP call.
|
||||
drop(nodes);
|
||||
|
||||
discover_capabilities(fleet, name, endpoint).await;
|
||||
|
||||
// Poll /health for the activation snapshot. We don't want this to
|
||||
// flip the node to unhealthy on its own — a neuron that's serving
|
||||
// /models fine is still operational even if /health is briefly
|
||||
@@ -213,97 +192,6 @@ async fn poll_neuron(fleet: &CortexState, name: &str, endpoint: &str) {
|
||||
poll_health(fleet, name, endpoint).await;
|
||||
}
|
||||
|
||||
/// Ask a neuron what the catalogue models it has never loaded can do
|
||||
/// (#241).
|
||||
///
|
||||
/// A loaded model reports its modalities on every `/models` poll, but a
|
||||
/// cold one reported nothing at all — which is why image generation was
|
||||
/// invisible on `/v1/models`, the image model being evicted almost all
|
||||
/// of the time. The neuron derives the answer from its local model cache
|
||||
/// without loading anything.
|
||||
///
|
||||
/// Only ever asks about ids it has no answer for, so a steady fleet
|
||||
/// costs nothing after the first cycle: what a given model id can do is
|
||||
/// a property of the model, not of the moment. A 404 means this node has
|
||||
/// no local evidence — another node may still have the weights cached,
|
||||
/// so nothing is recorded and the id is retried against the next node.
|
||||
///
|
||||
/// Ids that stay unresolved back off to [`CAPABILITY_RETRY`] rather than
|
||||
/// being re-asked every cycle. A catalogue entry whose weights nobody has
|
||||
/// downloaded can never be answered, and at the poll interval that would
|
||||
/// be a permanent trickle of requests and 404s. The retry still exists
|
||||
/// because the answer *can* change: the first node to pull those weights
|
||||
/// starts being able to reply.
|
||||
async fn discover_capabilities(fleet: &CortexState, name: &str, endpoint: &str) {
|
||||
let unknown: Vec<String> = {
|
||||
let known = fleet.discovered_capabilities.read().await;
|
||||
let attempts = fleet.capability_probe_attempts.read().await;
|
||||
fleet
|
||||
.catalogue
|
||||
.models
|
||||
.iter()
|
||||
.map(|p| p.id.clone())
|
||||
.filter(|id| !known.contains_key(id))
|
||||
.filter(|id| {
|
||||
attempts
|
||||
.get(&(name.to_string(), id.clone()))
|
||||
.is_none_or(|at| at.elapsed() >= CAPABILITY_RETRY)
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
if unknown.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
{
|
||||
let now = std::time::Instant::now();
|
||||
let mut attempts = fleet.capability_probe_attempts.write().await;
|
||||
for id in &unknown {
|
||||
attempts.insert((name.to_string(), id.clone()), now);
|
||||
}
|
||||
}
|
||||
|
||||
for model_id in unknown {
|
||||
let url = format!(
|
||||
"{endpoint}/models/{}/capabilities",
|
||||
urlencoding::encode(&model_id)
|
||||
);
|
||||
let resp = fleet
|
||||
.http_client
|
||||
.get(&url)
|
||||
.timeout(Duration::from_secs(5))
|
||||
.send()
|
||||
.await;
|
||||
let Ok(resp) = resp else { continue };
|
||||
if !resp.status().is_success() {
|
||||
continue;
|
||||
}
|
||||
let Ok(body) = resp.json::<serde_json::Value>().await else {
|
||||
continue;
|
||||
};
|
||||
let Some(caps) = body.get("capabilities").and_then(|c| c.as_array()) else {
|
||||
continue;
|
||||
};
|
||||
let caps: Vec<String> = caps
|
||||
.iter()
|
||||
.filter_map(|c| c.as_str().map(str::to_string))
|
||||
.collect();
|
||||
// An empty list is a real answer ("cached, serves nothing"), but
|
||||
// recording it would pin a model to no capabilities on the word
|
||||
// of one node. Leave it unknown so a node with real weights can
|
||||
// still speak up.
|
||||
if caps.is_empty() {
|
||||
continue;
|
||||
}
|
||||
tracing::debug!(node = name, model = %model_id, ?caps, "discovered capabilities");
|
||||
fleet
|
||||
.discovered_capabilities
|
||||
.write()
|
||||
.await
|
||||
.insert(model_id, caps);
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch `/health` and stash the activation snapshot on NodeState.
|
||||
/// Decoupled from the /models poll so a /health glitch doesn't mark
|
||||
/// the neuron unhealthy or evict the model list.
|
||||
@@ -391,21 +279,6 @@ fn export_health_metrics(node: &str, h: &HealthResponse) {
|
||||
counter!("cortex_model_rejections_total",
|
||||
"node" => node.to_string(), "model" => m.id.clone(), "reason" => "per_principal")
|
||||
.absolute(m.rejected_per_principal);
|
||||
counter!("cortex_model_rejections_total",
|
||||
"node" => node.to_string(), "model" => m.id.clone(), "reason" => "anon_yield")
|
||||
.absolute(m.rejected_anon_yield);
|
||||
// How much of this model's load is unattributable (#262), and the
|
||||
// ceiling that keeps it from starving identified callers. 0 for a
|
||||
// pre-#262 neuron, which had no ceiling — skip rather than publish
|
||||
// a 0 that reads as "anonymous is refused" when it means the
|
||||
// opposite.
|
||||
gauge!("cortex_model_anon_in_flight", "node" => node.to_string(), "model" => m.id.clone())
|
||||
.set(m.anon_in_flight as f64);
|
||||
if m.anon_max_in_flight > 0 {
|
||||
gauge!("cortex_model_anon_max_in_flight",
|
||||
"node" => node.to_string(), "model" => m.id.clone())
|
||||
.set(m.anon_max_in_flight as f64);
|
||||
}
|
||||
}
|
||||
for d in &h.devices {
|
||||
let device = d.index.to_string();
|
||||
|
||||
@@ -71,46 +71,15 @@ pub async fn forward_request(
|
||||
})?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
// Buffer the error body so the journal records WHY, then hand the
|
||||
// same bytes on. Streaming is what a success body needs; an error
|
||||
// body is a few hundred bytes and streaming it only guarantees
|
||||
// that nobody ever sees the reason.
|
||||
//
|
||||
// This was a real dead end: neuron answered 503 `insufficient_vram`
|
||||
// with the deficit in the body, and cortex logged only
|
||||
// `status=503`. Neither journal held the cause, and the guard
|
||||
// responsible could only be found by reading neuron's source
|
||||
// (#257). The body is upstream-generated (our own #63 envelope),
|
||||
// not caller content, and it is truncated in the log.
|
||||
const MAX_ERROR_BODY: usize = 64 * 1024;
|
||||
let status = response.status();
|
||||
let (parts, body) = response.into_parts();
|
||||
return match axum::body::to_bytes(body, MAX_ERROR_BODY).await {
|
||||
Ok(bytes) => {
|
||||
let snippet: String = String::from_utf8_lossy(&bytes).chars().take(600).collect();
|
||||
tracing::warn!(
|
||||
node = %route.node_name,
|
||||
url = %url,
|
||||
status = status.as_u16(),
|
||||
body = %snippet,
|
||||
"proxy: upstream returned non-2xx"
|
||||
);
|
||||
Ok(Response::from_parts(parts, axum::body::Body::from(bytes)))
|
||||
}
|
||||
// Over the cap or a read failure: the body is consumed and
|
||||
// cannot be handed on. Say so rather than passing an empty
|
||||
// body off as the upstream's answer.
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
node = %route.node_name,
|
||||
url = %url,
|
||||
status = status.as_u16(),
|
||||
error = %e,
|
||||
"proxy: upstream returned non-2xx (body unreadable, dropped)"
|
||||
);
|
||||
Ok(Response::from_parts(parts, axum::body::Body::empty()))
|
||||
}
|
||||
};
|
||||
// Streaming body — can't snippet without breaking the stream
|
||||
// pass-through. Log status + URL; the client still gets the
|
||||
// upstream status, just without the leaked body.
|
||||
tracing::warn!(
|
||||
node = %route.node_name,
|
||||
url = %url,
|
||||
status = response.status().as_u16(),
|
||||
"proxy: upstream returned non-2xx"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(response)
|
||||
@@ -171,9 +140,6 @@ impl IntoResponse for ProxyError {
|
||||
// over the decode window (first→last chunk); falls back to the
|
||||
// full request duration for single-chunk (non-streaming) bodies
|
||||
// cortex_prompt_tokens_total / cortex_completion_tokens_total (counters)
|
||||
// cortex_cached_prompt_tokens_total (counter) — prompt tokens served
|
||||
// from neuron's prefix cache; over prompt_tokens_total this is the
|
||||
// fleet's cache-hit rate (#269)
|
||||
|
||||
/// Cap on the retained body tail. The usage object rides on the final
|
||||
/// chunk, so a generous tail is plenty; the cap bounds memory on huge
|
||||
@@ -246,15 +212,6 @@ impl ChunkObserver for CortexMetrics {
|
||||
if let Some(prompt) = prompt {
|
||||
metrics::counter!("cortex_prompt_tokens_total", &self.labels).increment(prompt);
|
||||
}
|
||||
// Prefix-cache reuse (#269), read from the same usage object.
|
||||
// Against `cortex_prompt_tokens_total` this gives fleet-wide
|
||||
// cache-hit rate as a ratio, which was previously only
|
||||
// obtainable by grepping `reused=` out of a neuron journal —
|
||||
// per-host, unaggregated, and gone on log rotation.
|
||||
if let Some(cached) = last_count_for(self.tail.as_str(), "cached_tokens") {
|
||||
metrics::counter!("cortex_cached_prompt_tokens_total", &self.labels)
|
||||
.increment(cached);
|
||||
}
|
||||
if let Some(completion) = completion.filter(|c| *c > 0) {
|
||||
metrics::counter!("cortex_completion_tokens_total", &self.labels)
|
||||
.increment(completion);
|
||||
|
||||
@@ -155,27 +155,6 @@ pub async fn resolve(
|
||||
if let Some(entry) = node.models.get(model_id) {
|
||||
match entry.status {
|
||||
ModelStatus::Loaded | ModelStatus::Reloading => {
|
||||
// Resident is not servable (#245). A node whose
|
||||
// device has been squeezed below the prefill
|
||||
// floor rejects every request before doing any
|
||||
// work — routing there converts a recoverable
|
||||
// placement problem into a 503 for the client.
|
||||
// Skipping it lets the rest of this function do
|
||||
// what it already does well: pick another
|
||||
// replica, or cold-load somewhere that fits.
|
||||
if !entry.is_servable() {
|
||||
tracing::warn!(
|
||||
node = %node.name,
|
||||
model = %model_id,
|
||||
reason = entry
|
||||
.servable
|
||||
.as_ref()
|
||||
.and_then(|s| s.reason.as_deref())
|
||||
.unwrap_or("unknown"),
|
||||
"skipping loaded-but-unservable location"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
// Least-busy score: in-flight + queued from the
|
||||
// neuron's last /health (#53). Unknown load (no poll
|
||||
// yet) scores 0 so the replica stays eligible.
|
||||
@@ -254,8 +233,7 @@ pub async fn resolve(
|
||||
// free enough was ranked below one that can).
|
||||
if !fits_free {
|
||||
for _ in 0..3 {
|
||||
match crate::evictor::evict_lru_on_node(fleet, &node_name, Some(&profile.id)).await
|
||||
{
|
||||
match crate::evictor::evict_lru_on_node(fleet, &node_name).await {
|
||||
Ok(Some(evicted)) => {
|
||||
tracing::info!(
|
||||
model = %profile.id,
|
||||
@@ -299,15 +277,10 @@ async fn node_fits_free(fleet: &Arc<CortexState>, node_name: &str, profile: &Mod
|
||||
/// profile. Preference order (#203):
|
||||
/// 1. A neuron from `profile.pinned_on` that is healthy + feasible.
|
||||
/// 2. One whose live free VRAM already fits the profile.
|
||||
/// 3. One whose *displaceable* models — loaded, and outranked by this
|
||||
/// profile's residency priority — plus free VRAM could fit it
|
||||
/// after a cold-swap eviction.
|
||||
/// 3. One whose *evictable* (loaded, not catalogue-pinned) models
|
||||
/// plus free VRAM could fit it after a cold-swap eviction.
|
||||
/// 4. Any healthy + feasible neuron, stable by name.
|
||||
///
|
||||
/// Free-fit outranks evict-fit, so a model never displaces anything
|
||||
/// while a node with room exists. Priority decides who may be
|
||||
/// displaced, never whether a displacement is needed.
|
||||
///
|
||||
/// Returns `(name, endpoint, fits_free)` — `fits_free = false` tells
|
||||
/// the caller a cold-swap eviction is needed before loading.
|
||||
async fn pick_feasible_neuron(
|
||||
@@ -335,16 +308,14 @@ async fn pick_feasible_neuron(
|
||||
.unwrap_or(0);
|
||||
let need = profile.vram_mb.unwrap_or(0);
|
||||
let fits_free = max_free >= need;
|
||||
// Evictable estimate: free + the VRAM of loaded models this
|
||||
// profile is permitted to displace. The predicate must be the
|
||||
// same one the evictor applies, or a node ranks as able to make
|
||||
// room and then declines to make any.
|
||||
// Evictable estimate: free + the VRAM of loaded models that the
|
||||
// catalogue does not pin to this node.
|
||||
let evictable: u64 = node
|
||||
.models
|
||||
.values()
|
||||
.filter(|m| {
|
||||
matches!(m.status, cortex_core::node::ModelStatus::Loaded)
|
||||
&& fleet.catalogue.may_displace(&profile.id, &m.id)
|
||||
&& !fleet.catalogue.is_pinned(&m.id, &node.name)
|
||||
})
|
||||
// neuron reports vram_used_mb: null today; fall back to the
|
||||
// catalogue's declared footprint so the evictable estimate
|
||||
@@ -477,11 +448,6 @@ async fn cold_load(
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
// A model we just loaded is presumed servable until
|
||||
// the next poll says otherwise. Seeding `false` here
|
||||
// would make the cold-load path immediately
|
||||
// un-route what it had just placed.
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -535,11 +501,6 @@ async fn profile_to_spec(
|
||||
quant: profile.quant.clone(),
|
||||
tensor_parallel,
|
||||
devices: Some(devices),
|
||||
// The catalogue's operator override rides along with the load
|
||||
// (#283), so a cold-loaded model samples the same way as one
|
||||
// this host was configured to hold resident.
|
||||
sampling: profile.sampling.clone(),
|
||||
preserve_thinking: profile.preserve_thinking,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -645,10 +606,7 @@ mod tests {
|
||||
min_devices: 1,
|
||||
min_device_vram_mb: None,
|
||||
pinned_on: vec![],
|
||||
residency_priority: None,
|
||||
source: source.map(String::from),
|
||||
sampling: None,
|
||||
preserve_thinking: None,
|
||||
limit: None,
|
||||
cost: None,
|
||||
capabilities: vec![],
|
||||
|
||||
@@ -25,33 +25,6 @@ pub struct CortexState {
|
||||
/// Per-principal served-token tally (#58), reported to upstream for
|
||||
/// operator reconciliation by the flush task when upstream is enabled.
|
||||
pub served_usage: Arc<crate::served_usage::ServedUsage>,
|
||||
/// Modalities discovered for models that are not currently loaded
|
||||
/// (#241), keyed by model id.
|
||||
///
|
||||
/// A loaded model reports its own capabilities every poll, so this
|
||||
/// only carries the cold ones — which is the case that mattered,
|
||||
/// since the image model spends nearly all its time evicted and was
|
||||
/// therefore advertising nothing. Neurons derive these from their
|
||||
/// local model cache; cortex asks once and keeps the answer, because
|
||||
/// what a given model id can do does not change.
|
||||
pub discovered_capabilities: RwLock<HashMap<String, Vec<String>>>,
|
||||
/// When each `(neuron, model id)` pair was last probed for
|
||||
/// capabilities.
|
||||
///
|
||||
/// A model no node has cached can never be answered, and the poll
|
||||
/// loop runs every few seconds — without this it would ask every
|
||||
/// node about every such model forever, turning a one-off lookup
|
||||
/// into steady background traffic and a stream of 404s in the logs.
|
||||
/// Resolved ids stop being probed entirely.
|
||||
///
|
||||
/// Keyed per node, not per model: nodes are polled in config order,
|
||||
/// so a model-only key let whichever node came first record the
|
||||
/// attempt and then, if it answered 404, suppress the question to
|
||||
/// every other node for the whole back-off window. Weights are not
|
||||
/// distributed evenly across the fleet, so that is the common case
|
||||
/// rather than an edge — it hid a GGUF that two of three nodes could
|
||||
/// describe perfectly well.
|
||||
pub capability_probe_attempts: RwLock<HashMap<(String, String), std::time::Instant>>,
|
||||
}
|
||||
|
||||
impl CortexState {
|
||||
@@ -66,9 +39,6 @@ impl CortexState {
|
||||
endpoint: nc.endpoint.clone(),
|
||||
healthy: false,
|
||||
models: HashMap::new(),
|
||||
// Filled from the neuron's /models reply at the
|
||||
// first poll (#223).
|
||||
reasoning_budget: Vec::new(),
|
||||
lifecycle_cycles: 0,
|
||||
last_poll: None,
|
||||
discovery: None,
|
||||
@@ -101,36 +71,13 @@ impl CortexState {
|
||||
neuron_configs: config.neurons.clone(),
|
||||
eviction: config.eviction.clone(),
|
||||
catalogue,
|
||||
// READ timeout, not a total one. reqwest's `.timeout()` is a
|
||||
// total deadline — "from when the request starts connecting
|
||||
// until the response body has finished" — which for a streamed
|
||||
// completion is a hard cap on how long a model may generate,
|
||||
// not a liveness check. At 300s and ~13 tok/s that capped every
|
||||
// stream at roughly 3.9k tokens: cortex severed the connection
|
||||
// mid-answer, the client reported the stream as terminated, and
|
||||
// neuron carried on generating into a socket nobody was reading.
|
||||
// A long agentic turn could therefore never complete through the
|
||||
// gateway however healthy the fleet was, and the failure looked
|
||||
// like a model fault because the model was blameless.
|
||||
//
|
||||
// `.read_timeout()` resets on every successful read, so it times
|
||||
// out a stalled upstream while letting a producing one run as
|
||||
// long as it needs. The same 300s is kept deliberately: it is
|
||||
// the headroom a cold model load needs before its first byte,
|
||||
// which is what the original total timeout was reaching for.
|
||||
//
|
||||
// This pairs with neuron's 1s SSE keep-alive: a healthy stream
|
||||
// puts bytes on the wire every second, so idle here means idle,
|
||||
// and the deadline finally measures what it was meant to.
|
||||
http_client: reqwest::Client::builder()
|
||||
.read_timeout(std::time::Duration::from_secs(300))
|
||||
.timeout(std::time::Duration::from_secs(300))
|
||||
.build()
|
||||
.expect("failed to build HTTP client"),
|
||||
entitlements,
|
||||
require_auth: config.entitlements.require_auth,
|
||||
served_usage: Arc::new(crate::served_usage::ServedUsage::new()),
|
||||
discovered_capabilities: RwLock::new(HashMap::new()),
|
||||
capability_probe_attempts: RwLock::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -80,7 +80,6 @@ async fn test_alias_resolves_in_chat_completions() {
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -167,7 +166,6 @@ async fn test_aliases_surface_in_v1_models() {
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -255,7 +253,6 @@ async fn test_alias_falls_through_for_unmapped_model() {
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -124,7 +124,6 @@ async fn spawn_gateway(neuron_url: &str, entitlements: EntitlementsConfig) -> St
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
//! Integration tests for budget enforcement (#52) — the A0 seatbelt.
|
||||
//!
|
||||
//! A reservation over the key's hard cap is refused *before* neuron is hit,
|
||||
//! with the #63 code matching the cap-window semantics: `rate_limit_exceeded`
|
||||
//! plus `Retry-After` for a resetting window, `insufficient_quota` for a hard
|
||||
//! balance. Spend never exceeds the cap. No 402, ever.
|
||||
//! with the #63 code matching the cap-window semantics (rate_limit_exceeded
|
||||
//! + Retry-After for a resetting window, insufficient_quota for a hard
|
||||
//! balance). Spend never exceeds the cap. No 402, ever.
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::Path;
|
||||
@@ -99,7 +99,6 @@ async fn spawn_gateway(neuron_url: &str, key: ApiKeyConfig) -> (Arc<CortexState>
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,208 +0,0 @@
|
||||
//! Cold models advertise what they can do (#241).
|
||||
//!
|
||||
//! Before this, `/v1/models` reported `capabilities: []` for anything not
|
||||
//! currently loaded, because the only source of capabilities was a loaded
|
||||
//! neuron reporting its own handle. That made image generation
|
||||
//! undiscoverable in practice: the image model is evicted almost all of
|
||||
//! the time, so a client listing our models saw an entry that looked like
|
||||
//! a text model with nothing to say about itself.
|
||||
//!
|
||||
//! The fix is discovery, not configuration — neurons derive modalities
|
||||
//! from their local model cache and cortex asks. These tests pin the two
|
||||
//! properties that matter: a cold model's capabilities reach `/v1/models`,
|
||||
//! and a node with no local evidence cannot erase them.
|
||||
|
||||
mod common;
|
||||
|
||||
use cortex_core::config::{
|
||||
EvictionSettings, EvictionStrategy, GatewayConfig, GatewaySettings, NeuronEndpoint,
|
||||
};
|
||||
use cortex_gateway::state::CortexState;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
/// A catalogue holding one image model that nothing has loaded.
|
||||
fn write_catalogue() -> std::path::PathBuf {
|
||||
static SEQ: AtomicU64 = AtomicU64::new(0);
|
||||
let toml = r#"
|
||||
[[models]]
|
||||
id = "Tongyi-MAI/Z-Image-Turbo"
|
||||
harness = "candle"
|
||||
min_devices = 1
|
||||
|
||||
[aliases]
|
||||
"helexa/image" = "Tongyi-MAI/Z-Image-Turbo"
|
||||
"#;
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"cortex_test_capability_models.{}.{}.toml",
|
||||
std::process::id(),
|
||||
SEQ.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
std::fs::write(&path, toml).unwrap();
|
||||
path
|
||||
}
|
||||
|
||||
fn fleet_for(endpoint: String) -> Arc<CortexState> {
|
||||
fleet_of(vec![("gpu", endpoint)])
|
||||
}
|
||||
|
||||
fn fleet_of(neurons: Vec<(&str, String)>) -> Arc<CortexState> {
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: neurons
|
||||
.into_iter()
|
||||
.map(|(name, endpoint)| NeuronEndpoint {
|
||||
name: name.into(),
|
||||
endpoint,
|
||||
})
|
||||
.collect(),
|
||||
models_config: write_catalogue().to_string_lossy().into_owned(),
|
||||
..Default::default()
|
||||
};
|
||||
Arc::new(CortexState::from_config(&config))
|
||||
}
|
||||
|
||||
/// Serve `fleet` and return its base URL, so a test can assert the
|
||||
/// actual HTTP surface rather than the state behind it.
|
||||
async fn serve(fleet: Arc<CortexState>) -> String {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let app = cortex_gateway::build_app(fleet);
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
format!("http://{addr}")
|
||||
}
|
||||
|
||||
/// The headline case: nothing loaded anywhere, and `/v1/models` still
|
||||
/// says the image model generates images. This is the acceptance
|
||||
/// criterion from #241 — asserted on the response a client actually
|
||||
/// receives, because that is where the bug was visible.
|
||||
#[tokio::test]
|
||||
async fn cold_image_model_advertises_image_capability() {
|
||||
let neuron = common::spawn_mock_neuron_with_capabilities(
|
||||
serde_json::json!([]),
|
||||
&[("Tongyi-MAI/Z-Image-Turbo", vec!["image"])],
|
||||
)
|
||||
.await;
|
||||
let fleet = fleet_for(neuron);
|
||||
|
||||
cortex_gateway::poller::poll_once(&fleet).await;
|
||||
|
||||
let gateway = serve(fleet).await;
|
||||
let body: serde_json::Value = reqwest::get(format!("{gateway}/v1/models"))
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let entry = body["data"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|m| m["id"] == "Tongyi-MAI/Z-Image-Turbo")
|
||||
.expect("the catalogued image model should be listed");
|
||||
assert_eq!(
|
||||
entry["loaded"], false,
|
||||
"precondition: this asserts the *cold* path"
|
||||
);
|
||||
assert_eq!(
|
||||
entry["capabilities"],
|
||||
serde_json::json!(["image"]),
|
||||
"a cold image model must still advertise that it generates images"
|
||||
);
|
||||
|
||||
// The alias is the id most clients will actually send, so it has to
|
||||
// carry the capability too — advertising it only on the concrete id
|
||||
// would leave the public-facing name looking like a text model.
|
||||
let alias = body["data"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|m| m["id"] == "helexa/image")
|
||||
.expect("the tier alias should be listed");
|
||||
assert_eq!(
|
||||
alias["capabilities"],
|
||||
serde_json::json!(["image"]),
|
||||
"the alias must advertise the same capabilities as its target"
|
||||
);
|
||||
}
|
||||
|
||||
/// A node that has never cached the weights answers 404. That must leave
|
||||
/// the id unknown rather than recording an empty list, or the first node
|
||||
/// polled would permanently pin a model to no capabilities — the exact
|
||||
/// failure this work removes, reintroduced from a different direction.
|
||||
#[tokio::test]
|
||||
async fn node_without_local_evidence_records_nothing() {
|
||||
let neuron = common::spawn_mock_neuron_with_capabilities(serde_json::json!([]), &[]).await;
|
||||
let fleet = fleet_for(neuron);
|
||||
|
||||
cortex_gateway::poller::poll_once(&fleet).await;
|
||||
|
||||
assert!(
|
||||
fleet.discovered_capabilities.read().await.is_empty(),
|
||||
"a 404 means 'no local evidence', which must not be cached as an answer"
|
||||
);
|
||||
}
|
||||
|
||||
/// Weights are not spread evenly across a fleet, so the node polled
|
||||
/// first routinely lacks a model that a later one has. The back-off that
|
||||
/// stops unanswerable ids being re-probed every cycle must not let that
|
||||
/// first node's 404 suppress the question to everyone else.
|
||||
///
|
||||
/// Found live: a GGUF that two of three neurons could describe reported
|
||||
/// no capabilities at all, because the one node without it was polled
|
||||
/// first and claimed the attempt for the whole window.
|
||||
#[tokio::test]
|
||||
async fn a_node_that_cannot_answer_does_not_starve_one_that_can() {
|
||||
let empty_handed =
|
||||
common::spawn_mock_neuron_with_capabilities(serde_json::json!([]), &[]).await;
|
||||
let has_weights = common::spawn_mock_neuron_with_capabilities(
|
||||
serde_json::json!([]),
|
||||
&[("Tongyi-MAI/Z-Image-Turbo", vec!["image"])],
|
||||
)
|
||||
.await;
|
||||
// Config order is poll order, so the node that 404s goes first.
|
||||
let fleet = fleet_of(vec![("first", empty_handed), ("second", has_weights)]);
|
||||
|
||||
cortex_gateway::poller::poll_once(&fleet).await;
|
||||
|
||||
assert_eq!(
|
||||
fleet
|
||||
.discovered_capabilities
|
||||
.read()
|
||||
.await
|
||||
.get("Tongyi-MAI/Z-Image-Turbo")
|
||||
.cloned(),
|
||||
Some(vec!["image".to_string()]),
|
||||
"the second node's answer must survive the first node's 404, \
|
||||
in the same poll cycle"
|
||||
);
|
||||
}
|
||||
|
||||
/// An empty list is likewise not an answer worth keeping: a repo cached
|
||||
/// without weights says nothing about what other nodes can serve.
|
||||
#[tokio::test]
|
||||
async fn empty_capability_list_is_not_recorded() {
|
||||
let neuron = common::spawn_mock_neuron_with_capabilities(
|
||||
serde_json::json!([]),
|
||||
&[("Tongyi-MAI/Z-Image-Turbo", vec![])],
|
||||
)
|
||||
.await;
|
||||
let fleet = fleet_for(neuron);
|
||||
|
||||
cortex_gateway::poller::poll_once(&fleet).await;
|
||||
|
||||
assert!(
|
||||
fleet.discovered_capabilities.read().await.is_empty(),
|
||||
"an empty list must not be cached as this model's capabilities"
|
||||
);
|
||||
}
|
||||
@@ -3,7 +3,7 @@
|
||||
use axum::body::Body;
|
||||
use axum::extract::Path;
|
||||
use axum::http::header;
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum::response::Response;
|
||||
use axum::routing::{get, post};
|
||||
use axum::{Json, Router};
|
||||
use cortex_core::config::{
|
||||
@@ -405,73 +405,6 @@ pub async fn spawn_mock_neuron_with_models_and_health(
|
||||
base_url
|
||||
}
|
||||
|
||||
/// Variant that also serves `GET /models/{id}/capabilities` (#241), the
|
||||
/// route a neuron answers from its local model cache for models nothing
|
||||
/// has loaded.
|
||||
///
|
||||
/// `capabilities` maps model id → modalities. Any id not listed gets a
|
||||
/// 404, which is how a real neuron says "these weights were never cached
|
||||
/// here" — distinct from an empty list, and the distinction is load-
|
||||
/// bearing, so the mock has to be able to express both.
|
||||
#[allow(dead_code)]
|
||||
pub async fn spawn_mock_neuron_with_capabilities(
|
||||
models_response: Value,
|
||||
capabilities: &[(&str, Vec<&str>)],
|
||||
) -> String {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let base_url = format!("http://{addr}");
|
||||
let health_response = default_health_response();
|
||||
let table: std::collections::HashMap<String, Vec<String>> = capabilities
|
||||
.iter()
|
||||
.map(|(id, caps)| {
|
||||
(
|
||||
(*id).to_string(),
|
||||
caps.iter().map(|c| (*c).to_string()).collect(),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
let table = Arc::new(table);
|
||||
|
||||
let app = Router::new()
|
||||
.route(
|
||||
"/models",
|
||||
get(move || {
|
||||
let resp = models_response.clone();
|
||||
async move { Json(resp) }
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/health",
|
||||
get(move || {
|
||||
let resp = health_response.clone();
|
||||
async move { Json(resp) }
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/models/{model_id}/capabilities",
|
||||
get(move |Path(model_id): Path<String>| {
|
||||
let table = table.clone();
|
||||
async move {
|
||||
match table.get(&model_id) {
|
||||
Some(caps) => Json(json!({ "capabilities": caps })).into_response(),
|
||||
None => (
|
||||
axum::http::StatusCode::NOT_FOUND,
|
||||
Json(json!({"error": "no local evidence"})),
|
||||
)
|
||||
.into_response(),
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
|
||||
base_url
|
||||
}
|
||||
|
||||
/// Spawns the cortex gateway with a single neuron pointing at `mock_url`.
|
||||
/// The node is pre-seeded as healthy with one loaded model ("test-model").
|
||||
/// Returns the gateway's base URL.
|
||||
@@ -518,7 +451,6 @@ pub async fn spawn_gateway_with_state(mock_url: &str) -> (Arc<CortexState>, Stri
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -533,31 +465,3 @@ pub async fn spawn_gateway_with_state(mock_url: &str) -> (Arc<CortexState>, Stri
|
||||
|
||||
(fleet, format!("http://{addr}"))
|
||||
}
|
||||
|
||||
/// Build a gateway state over two mock neurons (no poller; we seed state).
|
||||
pub async fn two_neuron_fleet(endpoint_a: &str, endpoint_b: &str) -> Arc<CortexState> {
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![
|
||||
NeuronEndpoint {
|
||||
name: "node-a".into(),
|
||||
endpoint: endpoint_a.to_string(),
|
||||
},
|
||||
NeuronEndpoint {
|
||||
name: "node-b".into(),
|
||||
endpoint: endpoint_b.to_string(),
|
||||
},
|
||||
],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
Arc::new(CortexState::from_config(&config))
|
||||
}
|
||||
|
||||
@@ -56,76 +56,6 @@ async fn spawn_eviction_mock() -> (String, Arc<tokio::sync::Mutex<Vec<String>>>)
|
||||
(base_url, unloaded)
|
||||
}
|
||||
|
||||
/// A fleet whose catalogue ranks models, so the displacement rules are
|
||||
/// exercised rather than defaulted. Writes the catalogue to a temp file
|
||||
/// because `CortexState` loads it from a path.
|
||||
fn make_fleet_with_catalogue(
|
||||
endpoint: &str,
|
||||
catalogue_toml: &str,
|
||||
tag: &str,
|
||||
) -> (Arc<CortexState>, std::path::PathBuf) {
|
||||
let path = std::env::temp_dir().join(format!("cortex-evict-catalogue-{tag}.toml"));
|
||||
std::fs::write(&path, catalogue_toml).expect("write test catalogue");
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![NeuronEndpoint {
|
||||
name: "gpu-node".into(),
|
||||
endpoint: endpoint.to_string(),
|
||||
}],
|
||||
models_config: path.to_string_lossy().into_owned(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
(Arc::new(CortexState::from_config(&config)), path)
|
||||
}
|
||||
|
||||
fn loaded(id: &str, age_secs: i64) -> ModelEntry {
|
||||
ModelEntry {
|
||||
id: id.into(),
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: Some(Utc::now() - chrono::Duration::seconds(age_secs)),
|
||||
vram_estimate_mb: Some(8000),
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The fleet policy under test. Two residency classes: image generation
|
||||
/// and the mid tier share a node and take turns on it; the flagship and
|
||||
/// the frontier model share a bigger one and take turns on that. Nothing
|
||||
/// in the everyday class may touch the big-node class.
|
||||
const TIERED: &str = r#"
|
||||
[[models]]
|
||||
id = "flagship"
|
||||
harness = "candle"
|
||||
residency_priority = 300
|
||||
|
||||
[[models]]
|
||||
id = "frontier"
|
||||
harness = "candle"
|
||||
residency_priority = 300
|
||||
|
||||
[[models]]
|
||||
id = "image"
|
||||
harness = "candle"
|
||||
residency_priority = 200
|
||||
|
||||
[[models]]
|
||||
id = "mid"
|
||||
harness = "candle"
|
||||
residency_priority = 200
|
||||
"#;
|
||||
|
||||
fn make_fleet(endpoint: &str, defrag_after: u32) -> Arc<CortexState> {
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
@@ -167,7 +97,6 @@ async fn test_evict_lru_model() {
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
node.models.insert(
|
||||
@@ -181,12 +110,11 @@ async fn test_evict_lru_model() {
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let evicted = cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node", None)
|
||||
let evicted = cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node")
|
||||
.await
|
||||
.expect("eviction should succeed");
|
||||
|
||||
@@ -219,7 +147,7 @@ async fn test_eviction_nothing_to_evict() {
|
||||
nodes.get_mut("gpu-node").unwrap().healthy = true;
|
||||
}
|
||||
|
||||
let evicted = cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node", None)
|
||||
let evicted = cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node")
|
||||
.await
|
||||
.expect("eviction should succeed");
|
||||
|
||||
@@ -249,12 +177,11 @@ async fn test_eviction_increments_lifecycle_cycles() {
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node", None)
|
||||
cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node")
|
||||
.await
|
||||
.expect("eviction should succeed");
|
||||
|
||||
@@ -301,129 +228,3 @@ async fn test_last_accessed_updated_on_request() {
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
|
||||
/// Image generation must take the mid tier's node when it needs it —
|
||||
/// this is existing fleet behaviour and the change must preserve it.
|
||||
#[tokio::test]
|
||||
async fn image_generation_evicts_the_mid_tier() {
|
||||
let (mock_url, unloaded) = spawn_eviction_mock().await;
|
||||
let (fleet, path) = make_fleet_with_catalogue(&mock_url, TIERED, "image-takes-mid");
|
||||
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("gpu-node").unwrap();
|
||||
node.healthy = true;
|
||||
node.models.insert("mid".into(), loaded("mid", 60));
|
||||
}
|
||||
|
||||
let evicted = cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node", Some("image"))
|
||||
.await
|
||||
.expect("eviction should succeed");
|
||||
|
||||
assert_eq!(evicted, Some("mid".to_string()));
|
||||
assert_eq!(unloaded.lock().await.as_slice(), ["mid"]);
|
||||
std::fs::remove_file(path).ok();
|
||||
}
|
||||
|
||||
/// Image generation must never take the flagship's node. Its device
|
||||
/// constraints alone would let it land there, so nothing but priority
|
||||
/// stops this.
|
||||
#[tokio::test]
|
||||
async fn image_generation_cannot_evict_the_flagship() {
|
||||
let (mock_url, unloaded) = spawn_eviction_mock().await;
|
||||
let (fleet, path) = make_fleet_with_catalogue(&mock_url, TIERED, "image-spares-flagship");
|
||||
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("gpu-node").unwrap();
|
||||
node.healthy = true;
|
||||
node.models
|
||||
.insert("flagship".into(), loaded("flagship", 9999));
|
||||
}
|
||||
|
||||
let evicted = cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node", Some("image"))
|
||||
.await
|
||||
.expect("eviction should succeed");
|
||||
|
||||
assert_eq!(
|
||||
evicted, None,
|
||||
"the flagship outranks image generation, however stale it is"
|
||||
);
|
||||
assert!(unloaded.lock().await.is_empty());
|
||||
std::fs::remove_file(path).ok();
|
||||
}
|
||||
|
||||
/// The frontier tier may cold-swap the flagship off its node.
|
||||
#[tokio::test]
|
||||
async fn the_frontier_tier_evicts_the_flagship() {
|
||||
let (mock_url, unloaded) = spawn_eviction_mock().await;
|
||||
let (fleet, path) = make_fleet_with_catalogue(&mock_url, TIERED, "frontier-takes-flagship");
|
||||
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("gpu-node").unwrap();
|
||||
node.healthy = true;
|
||||
node.models
|
||||
.insert("flagship".into(), loaded("flagship", 10));
|
||||
}
|
||||
|
||||
let evicted = cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node", Some("frontier"))
|
||||
.await
|
||||
.expect("eviction should succeed");
|
||||
|
||||
assert_eq!(evicted, Some("flagship".to_string()));
|
||||
assert_eq!(unloaded.lock().await.as_slice(), ["flagship"]);
|
||||
std::fs::remove_file(path).ok();
|
||||
}
|
||||
|
||||
/// LRU still decides *which* victim, but only among the models the
|
||||
/// incoming one outranks. Here the flagship is by far the stalest, so a
|
||||
/// purely age-ordered evictor would take it.
|
||||
#[tokio::test]
|
||||
async fn lru_picks_the_oldest_displaceable_model_not_the_oldest_model() {
|
||||
let (mock_url, unloaded) = spawn_eviction_mock().await;
|
||||
let (fleet, path) = make_fleet_with_catalogue(&mock_url, TIERED, "lru-within-rank");
|
||||
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("gpu-node").unwrap();
|
||||
node.healthy = true;
|
||||
node.models
|
||||
.insert("flagship".into(), loaded("flagship", 9999));
|
||||
node.models.insert("mid".into(), loaded("mid", 60));
|
||||
}
|
||||
|
||||
let evicted = cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node", Some("image"))
|
||||
.await
|
||||
.expect("eviction should succeed");
|
||||
|
||||
assert_eq!(evicted, Some("mid".to_string()));
|
||||
assert_eq!(unloaded.lock().await.as_slice(), ["mid"]);
|
||||
std::fs::remove_file(path).ok();
|
||||
}
|
||||
|
||||
/// The other half of the cold-swap: after an image generation has taken
|
||||
/// the node, the next text request must be able to take it back. A
|
||||
/// strict priority order would let whichever model arrived first hold
|
||||
/// the node forever, which looks to a user like the text tier vanishing
|
||||
/// once somebody generated an image.
|
||||
#[tokio::test]
|
||||
async fn the_mid_tier_takes_its_node_back_from_image_generation() {
|
||||
let (mock_url, unloaded) = spawn_eviction_mock().await;
|
||||
let (fleet, path) = make_fleet_with_catalogue(&mock_url, TIERED, "mid-swaps-back");
|
||||
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("gpu-node").unwrap();
|
||||
node.healthy = true;
|
||||
node.models.insert("image".into(), loaded("image", 30));
|
||||
}
|
||||
|
||||
let evicted = cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node", Some("mid"))
|
||||
.await
|
||||
.expect("eviction should succeed");
|
||||
|
||||
assert_eq!(evicted, Some("image".to_string()));
|
||||
assert_eq!(unloaded.lock().await.as_slice(), ["image"]);
|
||||
std::fs::remove_file(path).ok();
|
||||
}
|
||||
|
||||
@@ -182,7 +182,6 @@ async fn spawn_keyed_gateway(neuron_url: &str, hard_cap: u64) -> String {
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -37,7 +37,6 @@ async fn seed_loaded(fleet: &CortexState, node: &str, in_flight: usize, queue_de
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
n.model_load.insert(
|
||||
@@ -49,15 +48,8 @@ async fn seed_loaded(fleet: &CortexState, node: &str, in_flight: usize, queue_de
|
||||
max_in_flight: 8,
|
||||
max_queue_depth: 8,
|
||||
rejected_queue_full: 0,
|
||||
rejected_kv_timeout: 0,
|
||||
rejected_kv_unservable: 0,
|
||||
kv_budget_mb: 0,
|
||||
kv_available_mb: 0,
|
||||
rejected_timeout: 0,
|
||||
rejected_per_principal: 0,
|
||||
anon_in_flight: 0,
|
||||
anon_max_in_flight: 7,
|
||||
rejected_anon_yield: 0,
|
||||
tok_s_prefill: 0.0,
|
||||
tok_s_decode: 0.0,
|
||||
},
|
||||
|
||||
@@ -72,7 +72,6 @@ async fn spawn_metered_gateway(neuron_url: &str) -> (Arc<CortexState>, String) {
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -178,7 +177,6 @@ async fn anonymous_request_records_no_spend() {
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -69,21 +69,6 @@ capabilities = ["text"]
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("mock-node").expect("node exists");
|
||||
node.healthy = true;
|
||||
// The rungs the neuron reported at poll time. Since #290 these
|
||||
// come from the model's own chat template rather than operator
|
||||
// configuration, so the fixture uses the shape a real reasoning
|
||||
// model ships: Qwen3.8-27B accepts exactly low/medium/xhigh and
|
||||
// defaults to xhigh.
|
||||
node.reasoning_budget = ["low", "medium", "xhigh"]
|
||||
.into_iter()
|
||||
.map(|effort| cortex_core::harness::ReasoningBudgetRung {
|
||||
default: effort == "xhigh",
|
||||
effort: effort.into(),
|
||||
// No cap: effort is expressed to the model through its
|
||||
// template, not enforced by truncating its thinking.
|
||||
tokens: None,
|
||||
})
|
||||
.collect();
|
||||
node.models.insert(
|
||||
"test-model".into(),
|
||||
ModelEntry {
|
||||
@@ -100,11 +85,7 @@ capabilities = ["text"]
|
||||
context: 49152,
|
||||
input: Some(40960),
|
||||
output: 8192,
|
||||
// The ceiling a request may name, distinct from the
|
||||
// 8192-token reserve above (#278).
|
||||
output_ceiling: 32768,
|
||||
}),
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
// A model with no derivable limit: the flat #78 fields must be
|
||||
@@ -120,7 +101,6 @@ capabilities = ["text"]
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -165,53 +145,7 @@ capabilities = ["text"]
|
||||
// vLLM-convention probes (Hermes Agent) auto-detect the window.
|
||||
assert_eq!(entry["max_model_len"], 49152);
|
||||
assert_eq!(entry["max_input_tokens"], 40960);
|
||||
// The ceiling, not the reserve (#278). A client that reads this
|
||||
// field turns it into the cap it sends on each request; advertising
|
||||
// the 8192 reserve told a reasoning model's harness to stop below
|
||||
// the cost of a single think block. Observed with dsh — pi takes
|
||||
// `maxTokens` from static config and reads nothing here.
|
||||
assert_eq!(entry["max_output_tokens"], 32768);
|
||||
// The rungs the model itself accepts (#290). A caller can only send
|
||||
// a rung name, so a name we invent is one the template will reject.
|
||||
// Publishing the true set is correct regardless of who reads it —
|
||||
// pi-ai, for one, takes its levels from static config instead.
|
||||
let rungs = entry["reasoning_budget"]
|
||||
.as_array()
|
||||
.expect("reasoning_budget advertised for a reasoning model");
|
||||
let names: Vec<&str> = rungs
|
||||
.iter()
|
||||
.map(|r| r["effort"].as_str().expect("effort"))
|
||||
.collect();
|
||||
assert_eq!(names, ["low", "medium", "xhigh"]);
|
||||
// Which rung applies when the caller names none. Without this a
|
||||
// silent caller cannot tell what it is getting — on Qwen3.8 that is
|
||||
// xhigh, the most expensive rung on the ladder.
|
||||
let default_rung: Vec<&str> = rungs
|
||||
.iter()
|
||||
.filter(|r| r["default"].as_bool().unwrap_or(false))
|
||||
.map(|r| r["effort"].as_str().expect("effort"))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
default_rung,
|
||||
["xhigh"],
|
||||
"the model's own default must be visible"
|
||||
);
|
||||
// A named rung carries no token count: effort reaches the model
|
||||
// through its template, and enforcing it by truncation is the defect
|
||||
// #290 fixed.
|
||||
assert!(
|
||||
rungs.iter().all(|r| r.get("tokens").is_none()),
|
||||
"a named rung must not masquerade as a token budget"
|
||||
);
|
||||
|
||||
// The window under the two names a generic client may read, for one
|
||||
// that probes these rather than `max_model_len`. Publish all three;
|
||||
// do not assume a particular client consumes any of them. pi, in
|
||||
// particular, reads none — `contextWindow` is a required static
|
||||
// field on its model type (checked against 0.82.1 and 0.84.3), so
|
||||
// its window is hand-maintained whatever we advertise.
|
||||
assert_eq!(entry["context_window"], 49152);
|
||||
assert_eq!(entry["context_length"], 49152);
|
||||
assert_eq!(entry["max_output_tokens"], 8192);
|
||||
|
||||
// No limit → flat fields omitted entirely, never 0 or a guess.
|
||||
let unknown = body["data"]
|
||||
@@ -221,13 +155,7 @@ capabilities = ["text"]
|
||||
.find(|m| m["id"] == "no-limit-model")
|
||||
.expect("no-limit-model present in /v1/models");
|
||||
assert!(unknown.get("limit").is_none());
|
||||
for key in [
|
||||
"max_model_len",
|
||||
"max_input_tokens",
|
||||
"max_output_tokens",
|
||||
"context_window",
|
||||
"context_length",
|
||||
] {
|
||||
for key in ["max_model_len", "max_input_tokens", "max_output_tokens"] {
|
||||
assert!(
|
||||
unknown.get(key).is_none(),
|
||||
"{key} must be omitted when the window is unknown"
|
||||
|
||||
@@ -329,7 +329,6 @@ async fn test_poller_removes_stale_models() {
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
node.models.insert(
|
||||
@@ -343,7 +342,6 @@ async fn test_poller_removes_stale_models() {
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -96,9 +96,7 @@ async fn spawn_gateway(neuron: &str, context: usize) -> String {
|
||||
context,
|
||||
input: None,
|
||||
output: 16,
|
||||
output_ceiling: 0,
|
||||
}),
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -197,7 +197,6 @@ async fn test_recovering_model_returns_503_and_stays_listed() {
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,189 +0,0 @@
|
||||
//! A node that cannot serve is not a routing target (#245).
|
||||
//!
|
||||
//! The incident: quadbrat held `helexa/small` with 1405 MiB free against
|
||||
//! a 1500 MiB prefill floor, so every request was rejected before any
|
||||
//! device work. The model was `loaded`, the node answered every poll,
|
||||
//! and cortex reported `3/3 healthy` — while the whole small tier
|
||||
//! returned 503 to clients. Liveness was true and useless.
|
||||
//!
|
||||
//! Two properties are pinned here, and the second matters as much as the
|
||||
//! first: an unservable location must be skipped, and a *silent* neuron
|
||||
//! must not be mistaken for an unservable one. Reading absent evidence
|
||||
//! as "broken" would let a version skew empty the routing table and take
|
||||
//! the fleet down far more thoroughly than the fault being guarded.
|
||||
|
||||
mod common;
|
||||
|
||||
use cortex_core::harness::ModelServability;
|
||||
use cortex_core::node::{ModelEntry, ModelStatus};
|
||||
|
||||
fn entry(id: &str, servable: Option<ModelServability>) -> ModelEntry {
|
||||
ModelEntry {
|
||||
id: id.into(),
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: None,
|
||||
vram_estimate_mb: Some(4000),
|
||||
capabilities: vec!["text".into()],
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable,
|
||||
}
|
||||
}
|
||||
|
||||
fn squeezed() -> ModelServability {
|
||||
ModelServability {
|
||||
ok: false,
|
||||
reason: Some("insufficient_vram".into()),
|
||||
detail: Some("1405 MiB free, need at least 1500 MiB to start a prefill".into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// The safety property. A neuron that says nothing — an older build, a
|
||||
/// CPU load, an image model, a cache not yet seeded — stays eligible.
|
||||
#[test]
|
||||
fn silence_means_servable() {
|
||||
assert!(
|
||||
entry("m", None).is_servable(),
|
||||
"absent evidence must not be read as unservable"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_explicit_ok_is_servable() {
|
||||
assert!(
|
||||
entry(
|
||||
"m",
|
||||
Some(ModelServability {
|
||||
ok: true,
|
||||
reason: None,
|
||||
detail: None
|
||||
})
|
||||
)
|
||||
.is_servable()
|
||||
);
|
||||
}
|
||||
|
||||
/// The incident condition itself.
|
||||
#[test]
|
||||
fn a_squeezed_node_is_not_servable() {
|
||||
let e = entry("helexa/small", Some(squeezed()));
|
||||
assert!(!e.is_servable());
|
||||
assert_eq!(
|
||||
e.servable.as_ref().and_then(|s| s.reason.as_deref()),
|
||||
Some("insufficient_vram"),
|
||||
"the reason should match the code the request itself would fail with"
|
||||
);
|
||||
}
|
||||
|
||||
/// `/health` must stop calling the fleet `ok` while a resident model
|
||||
/// cannot be served. Reporting healthy through a tier outage is what
|
||||
/// made the incident invisible for as long as it was.
|
||||
#[tokio::test]
|
||||
async fn health_reports_impaired_when_a_model_cannot_be_served() {
|
||||
let neuron = common::spawn_mock_neuron().await;
|
||||
let (fleet, gateway) = common::spawn_gateway_with_state(&neuron).await;
|
||||
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("mock-node").expect("seeded node");
|
||||
node.healthy = true;
|
||||
node.models
|
||||
.insert("test-model".into(), entry("test-model", Some(squeezed())));
|
||||
}
|
||||
|
||||
let body: serde_json::Value = reqwest::get(format!("{gateway}/health"))
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
body["status"], "impaired",
|
||||
"a reachable node that cannot serve is not 'ok'"
|
||||
);
|
||||
let listed = body["unservable"].as_array().expect("unservable listed");
|
||||
assert_eq!(listed.len(), 1);
|
||||
assert_eq!(listed[0]["node"], "mock-node");
|
||||
assert_eq!(listed[0]["model"], "test-model");
|
||||
assert_eq!(listed[0]["reason"], "insufficient_vram");
|
||||
}
|
||||
|
||||
/// The control: the same fleet with nothing squeezed reports `ok`, so
|
||||
/// the assertion above is detecting the condition rather than always
|
||||
/// firing.
|
||||
#[tokio::test]
|
||||
async fn health_reports_ok_when_everything_can_serve() {
|
||||
let neuron = common::spawn_mock_neuron().await;
|
||||
let (fleet, gateway) = common::spawn_gateway_with_state(&neuron).await;
|
||||
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("mock-node").expect("seeded node");
|
||||
node.healthy = true;
|
||||
node.models
|
||||
.insert("test-model".into(), entry("test-model", None));
|
||||
}
|
||||
|
||||
let body: serde_json::Value = reqwest::get(format!("{gateway}/health"))
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(body["status"], "ok");
|
||||
assert!(body.get("unservable").is_none());
|
||||
}
|
||||
|
||||
// ── Routing ──────────────────────────────────────────────────────────
|
||||
|
||||
/// The behavioural fix. Two replicas, one squeezed: the request must go
|
||||
/// to the one that can actually answer it.
|
||||
///
|
||||
/// This is the shape the incident took, minus the detail that made it
|
||||
/// total — quadbrat was the *only* node holding `helexa/small`, so there
|
||||
/// was no second replica and the request 503'd. With the location
|
||||
/// skipped, that case falls through to the catalogue cold-load path
|
||||
/// instead, which places the model somewhere it fits.
|
||||
#[tokio::test]
|
||||
async fn routing_skips_a_loaded_but_unservable_replica() {
|
||||
let neuron_a = common::spawn_mock_neuron().await;
|
||||
let neuron_b = common::spawn_mock_neuron().await;
|
||||
let fleet = common::two_neuron_fleet(&neuron_a, &neuron_b).await;
|
||||
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
for name in ["node-a", "node-b"] {
|
||||
let n = nodes.get_mut(name).expect("node exists");
|
||||
n.healthy = true;
|
||||
}
|
||||
// node-a is squeezed; node-b is fine. node-a sorts first by
|
||||
// name, so a router that ignored servability would pick it.
|
||||
nodes
|
||||
.get_mut("node-a")
|
||||
.unwrap()
|
||||
.models
|
||||
.insert("test-model".into(), entry("test-model", Some(squeezed())));
|
||||
nodes.get_mut("node-b").unwrap().models.insert(
|
||||
"test-model".into(),
|
||||
entry(
|
||||
"test-model",
|
||||
Some(ModelServability {
|
||||
ok: true,
|
||||
reason: None,
|
||||
detail: None,
|
||||
}),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
let route = cortex_gateway::router::resolve(&fleet, "test-model")
|
||||
.await
|
||||
.expect("one replica can serve");
|
||||
assert_eq!(
|
||||
route.node_name, "node-b",
|
||||
"must route past the squeezed replica, not to it"
|
||||
);
|
||||
}
|
||||
@@ -86,15 +86,10 @@ pub async fn enter(
|
||||
HeaderMap::new(),
|
||||
auth::pending_invite_cookie(&code, state.config.session.secure),
|
||||
);
|
||||
// Land on "create account", not "sign in". Nearly everyone arriving on
|
||||
// an invitation is new here — defaulting to sign-in asks the majority
|
||||
// to notice a tab before they can do the only thing available to them.
|
||||
// The sign-in tab is one click away for the few who already have a
|
||||
// helexa.ai account.
|
||||
Ok((
|
||||
StatusCode::SEE_OTHER,
|
||||
out,
|
||||
[(header::LOCATION, "/register".to_string())],
|
||||
[(header::LOCATION, "/signin".to_string())],
|
||||
)
|
||||
.into_response())
|
||||
}
|
||||
|
||||
@@ -28,15 +28,7 @@ pub async fn register(state: &AppState, email: &str, password: &str) -> Result<(
|
||||
let resp = state
|
||||
.http
|
||||
.post(&url)
|
||||
// Tell upstream which front end this signup came from, so the
|
||||
// confirmation mail sends the investor back to the portal rather
|
||||
// than to helexa.ai, where the material they were invited to read
|
||||
// is nowhere to be seen. Upstream allowlists this.
|
||||
.json(&json!({
|
||||
"email": email,
|
||||
"password": password,
|
||||
"origin": state.config.site.base_url,
|
||||
}))
|
||||
.json(&json!({ "email": email, "password": password }))
|
||||
.send()
|
||||
.await
|
||||
.with_context(|| format!("calling {url}"))?;
|
||||
|
||||
@@ -148,12 +148,8 @@
|
||||
<footer class="bot">
|
||||
<div class="bot-inner">
|
||||
<p style="margin:0 0 .4rem">
|
||||
{% if user_email %}
|
||||
Confidential. Prepared for the named recipient above and not for
|
||||
onward distribution.
|
||||
{% else %}
|
||||
Confidential. Access is limited to invited recipients.
|
||||
{% endif %}
|
||||
Confidential. Prepared for the named recipient above and not for
|
||||
onward distribution.
|
||||
</p>
|
||||
<p style="margin:0">
|
||||
{{ brand_name }} · <a href="/privacy">How we handle your data</a>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}{% if tab == 'register' %}Create your account{% else %}Sign in{% endif %} — {{ site_name }}{% endblock %}
|
||||
{% block title %}Sign in — {{ site_name }}{% endblock %}
|
||||
{% block content %}
|
||||
<div class="card card-narrow">
|
||||
<div class="eyebrow">{{ invite_label or "Investor portal" }}</div>
|
||||
|
||||
@@ -34,7 +34,6 @@ pub fn api_routes(state: ApiState) -> Router {
|
||||
Router::new()
|
||||
.route("/api/health", get(health))
|
||||
.route("/api/dimensions", get(dimensions))
|
||||
.route("/api/regimes", get(regimes))
|
||||
.route("/api/summary", get(summary))
|
||||
.route("/api/scaling", get(scaling))
|
||||
.route("/api/concurrency", get(concurrency))
|
||||
@@ -66,14 +65,6 @@ async fn health(State(s): State<ApiState>) -> Result<Json<serde_json::Value>, Ap
|
||||
Ok(Json(json!({ "status": "ok", "run_count": count })))
|
||||
}
|
||||
|
||||
/// Declared measurement-regime boundaries (#288).
|
||||
///
|
||||
/// Static — no store access — but served from the API rather than
|
||||
/// hardcoded in the frontend so `report` and the UI cite one list.
|
||||
async fn regimes() -> Json<&'static [crate::regime::MeasurementRegime]> {
|
||||
Json(crate::regime::REGIMES)
|
||||
}
|
||||
|
||||
async fn dimensions(State(s): State<ApiState>) -> Result<Json<crate::store::Dimensions>, ApiError> {
|
||||
let store = s.lock().await;
|
||||
store.dimensions().map(Json).map_err(err500)
|
||||
|
||||
@@ -6,7 +6,6 @@ use crate::config::{TargetConfig, TargetKind};
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use cortex_core::build_info::BuildInfo;
|
||||
use cortex_core::discovery::{DiscoveryResponse, HealthResponse};
|
||||
use cortex_core::entitlements::{HEADER_ACCOUNT_ID, HEADER_KEY_ID};
|
||||
use cortex_core::harness::{ModelInfo, ModelSpec};
|
||||
use cortex_core::openai::ModelsResponse;
|
||||
use std::time::Duration;
|
||||
@@ -20,49 +19,10 @@ pub struct TargetClient {
|
||||
|
||||
impl TargetClient {
|
||||
pub fn new(request_timeout: Duration) -> Result<Self> {
|
||||
Self::with_principal(request_timeout, None)
|
||||
}
|
||||
|
||||
/// Build a client that stamps `principal`'s identity headers on every
|
||||
/// request (#288).
|
||||
///
|
||||
/// Set on the client rather than at each call site so no scenario can
|
||||
/// be added later that silently measures as anonymous — which is the
|
||||
/// failure this fixes, and it was invisible for a week.
|
||||
///
|
||||
/// `None` keeps the historical anonymous behaviour, which is correct
|
||||
/// for `openai` targets (the headers mean nothing to a foreign
|
||||
/// engine) and for a fresh install with no principal configured.
|
||||
pub fn with_principal(
|
||||
request_timeout: Duration,
|
||||
principal: Option<&crate::config::PrincipalSettings>,
|
||||
) -> Result<Self> {
|
||||
let mut builder = reqwest::Client::builder().timeout(request_timeout);
|
||||
if let Some(p) = principal {
|
||||
let mut headers = reqwest::header::HeaderMap::new();
|
||||
headers.insert(
|
||||
HEADER_ACCOUNT_ID,
|
||||
reqwest::header::HeaderValue::from_str(&p.account_id)
|
||||
.context("principal.account_id is not a valid header value")?,
|
||||
);
|
||||
headers.insert(
|
||||
HEADER_KEY_ID,
|
||||
reqwest::header::HeaderValue::from_str(&p.key_id)
|
||||
.context("principal.key_id is not a valid header value")?,
|
||||
);
|
||||
tracing::info!(
|
||||
account_id = %p.account_id,
|
||||
key_id = %p.key_id,
|
||||
"bench authenticates: measurements reflect identified-caller capacity (#288)"
|
||||
);
|
||||
builder = builder.default_headers(headers);
|
||||
} else {
|
||||
tracing::warn!(
|
||||
"bench has no [bench.principal]: requests are anonymous, so since #262 \
|
||||
they measure the anonymous-yield policy rather than serving capacity (#288)"
|
||||
);
|
||||
}
|
||||
let http = builder.build().context("building HTTP client")?;
|
||||
let http = reqwest::Client::builder()
|
||||
.timeout(request_timeout)
|
||||
.build()
|
||||
.context("building HTTP client")?;
|
||||
Ok(TargetClient { http })
|
||||
}
|
||||
|
||||
@@ -209,15 +169,6 @@ impl TargetClient {
|
||||
quant: None,
|
||||
tensor_parallel: (info.devices.len() > 1).then_some(info.devices.len() as u32),
|
||||
devices: Some(info.devices.clone()),
|
||||
// `GET /models` does not report the operator sampling
|
||||
// override (#283), so a spec reconstructed here cannot carry
|
||||
// it — same limitation as `quant` directly above. Benign
|
||||
// while this is only used to re-load a model the host
|
||||
// already has a `[[default_models]]` entry for, since that
|
||||
// entry supplies the override; it would matter if this were
|
||||
// ever used to load a model the host has no config for.
|
||||
sampling: None,
|
||||
preserve_thinking: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -282,11 +233,6 @@ impl TargetClient {
|
||||
cost: None,
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
preserve_thinking: None,
|
||||
// A generic OpenAI endpoint says nothing about
|
||||
// serviceability, and absent means "no opinion".
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
@@ -67,44 +67,6 @@ pub struct BenchSettings {
|
||||
/// SQLite system-of-record path.
|
||||
#[serde(default = "default_db_path")]
|
||||
pub db_path: String,
|
||||
/// Identity bench presents to a neuron (#288).
|
||||
///
|
||||
/// Without this bench is *anonymous*, and since #262 anonymous
|
||||
/// callers are served only from capacity left over once identified
|
||||
/// traffic is satisfied — they are capped below `max_in_flight` and
|
||||
/// park at the class gate ahead of every other gate. Measured on
|
||||
/// beast, that moved `concurrency:8` from 168.9 tok/s / 0.53 s
|
||||
/// ttft_p95 to 104.6 / 11.77 s the day #262 landed, and held there.
|
||||
///
|
||||
/// So an unauthenticated bench does not measure serving capacity; it
|
||||
/// measures the yield policy. Real traffic arrives through cortex
|
||||
/// with a principal and is subject to none of it.
|
||||
///
|
||||
/// Deliberately not an admission *exemption*: bench should be
|
||||
/// subject to the same rules as any caller, or it measures something
|
||||
/// no user experiences — the same class of error in the other
|
||||
/// direction. Give it a principal and let it queue like everyone
|
||||
/// else.
|
||||
#[serde(default)]
|
||||
pub principal: Option<PrincipalSettings>,
|
||||
}
|
||||
|
||||
/// The account/key pair bench stamps on inference requests (#288).
|
||||
///
|
||||
/// These are the headers cortex asserts after a bearer resolves (#49).
|
||||
/// Bench talks to neurons **directly** over the WireGuard mesh, so it
|
||||
/// stamps them itself — the same trust model the link already relies on,
|
||||
/// since a neuron accepts them from cortex on exactly that basis.
|
||||
///
|
||||
/// Give bench its *own* pair rather than borrowing a real caller's: it
|
||||
/// then holds its own fair-share allocation (#54) and cannot starve
|
||||
/// interactive traffic, which is the failure #262 was fixing.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct PrincipalSettings {
|
||||
/// Stamped as `x-helexa-account-id`.
|
||||
pub account_id: String,
|
||||
/// Stamped as `x-helexa-key-id`.
|
||||
pub key_id: String,
|
||||
}
|
||||
|
||||
impl Default for BenchSettings {
|
||||
@@ -115,7 +77,6 @@ impl Default for BenchSettings {
|
||||
iteration_pause_secs: default_iter_pause(),
|
||||
request_timeout_secs: default_timeout(),
|
||||
db_path: default_db_path(),
|
||||
principal: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -149,18 +110,6 @@ pub struct ScenarioConfig {
|
||||
/// deliberately, e.g. `concurrency_levels = [2, 4, 8]`.
|
||||
#[serde(default)]
|
||||
pub concurrency_levels: Vec<u32>,
|
||||
/// Concurrency levels to run **non-streaming** (#285). Separate from
|
||||
/// `concurrency_levels` because the two shapes are not
|
||||
/// interchangeable: streaming multiplexes through the batch engine,
|
||||
/// non-streaming did not until #285 and serialized completely — a
|
||||
/// measured 1.00x aggregate throughput at every level against 3.98x
|
||||
/// streamed.
|
||||
///
|
||||
/// Defaults to the top level only. The signal is starkest there and
|
||||
/// each extra level costs a full burst per sample per build; an
|
||||
/// operator who wants the whole curve can list it.
|
||||
#[serde(default = "default_concurrency_nonstreaming_levels")]
|
||||
pub concurrency_nonstreaming_levels: Vec<u32>,
|
||||
/// Square image sizes (px per side) — one `image:<px>` scenario per
|
||||
/// entry (#203), run only against models advertising the `image`
|
||||
/// capability. Defaults to `[1024]`.
|
||||
@@ -199,7 +148,6 @@ impl Default for ScenarioConfig {
|
||||
prompt_sizes: default_prompt_sizes(),
|
||||
max_tokens: default_max_tokens(),
|
||||
concurrency_levels: Vec::new(),
|
||||
concurrency_nonstreaming_levels: Vec::new(),
|
||||
concurrency_prompt_tokens: default_concurrency_prompt_tokens(),
|
||||
image_sizes: default_image_sizes(),
|
||||
capability_probes: Vec::new(),
|
||||
@@ -267,10 +215,6 @@ fn default_timeout() -> u64 {
|
||||
fn default_db_path() -> String {
|
||||
"/var/lib/helexa-bench/bench.sqlite".to_string()
|
||||
}
|
||||
fn default_concurrency_nonstreaming_levels() -> Vec<u32> {
|
||||
vec![8]
|
||||
}
|
||||
|
||||
fn default_api_enabled() -> bool {
|
||||
true
|
||||
}
|
||||
@@ -299,44 +243,6 @@ fn default_capability_max_tokens() -> u64 {
|
||||
// figment's, not ours, so suppress the lint here.
|
||||
#[allow(clippy::result_large_err)]
|
||||
mod tests {
|
||||
|
||||
/// #288: a config with no `[bench.principal]` must still parse — the
|
||||
/// anonymous behaviour is the historical one and correct for
|
||||
/// `openai` targets — but a configured one must round-trip, because
|
||||
/// a silently-dropped principal reverts the measurement to the
|
||||
/// anonymous-yield path without changing a visible number.
|
||||
#[test]
|
||||
fn principal_is_optional_and_round_trips() {
|
||||
let parse = |t: &str| -> BenchConfig {
|
||||
figment::Figment::new()
|
||||
.merge(figment::providers::Toml::string(t))
|
||||
.extract()
|
||||
.expect("parse")
|
||||
};
|
||||
let none = parse("[bench]\nsamples_per_version = 3\n");
|
||||
assert!(none.bench.principal.is_none());
|
||||
|
||||
let some = parse(
|
||||
"[bench]\nsamples_per_version = 3\n\n\
|
||||
[bench.principal]\naccount_id = \"acct\"\nkey_id = \"k1\"\n",
|
||||
);
|
||||
let p = some.bench.principal.expect("principal");
|
||||
assert_eq!((p.account_id.as_str(), p.key_id.as_str()), ("acct", "k1"));
|
||||
}
|
||||
|
||||
/// The non-streaming burst defaults on (#285). If it defaulted off,
|
||||
/// the path that serialized completely would stay unmeasured on every
|
||||
/// fleet that had not opted in — which is how it went unnoticed.
|
||||
#[test]
|
||||
fn nonstreaming_concurrency_defaults_to_the_top_level() {
|
||||
let cfg: BenchConfig = figment::Figment::new()
|
||||
.merge(figment::providers::Toml::string(
|
||||
"[scenarios]\nprompt_sizes = [128]\n",
|
||||
))
|
||||
.extract()
|
||||
.expect("parse");
|
||||
assert_eq!(cfg.scenarios.concurrency_nonstreaming_levels, vec![8]);
|
||||
}
|
||||
use super::*;
|
||||
use figment::Jail;
|
||||
|
||||
|
||||
@@ -7,7 +7,6 @@
|
||||
pub mod api;
|
||||
pub mod client;
|
||||
pub mod config;
|
||||
pub mod regime;
|
||||
pub mod report;
|
||||
pub mod scenario;
|
||||
pub mod store;
|
||||
|
||||
@@ -1,148 +0,0 @@
|
||||
//! Measurement-regime boundaries — builds where a number changed
|
||||
//! meaning rather than value.
|
||||
//!
|
||||
//! A trend line silently spans them, and every reader re-derives the
|
||||
//! same thing: that a cliff was the instrument, not the engine. Three
|
||||
//! have already cost real investigation time:
|
||||
//!
|
||||
//! - beast's `concurrency:8` fell 168.9 → 104.6 tok/s with `ttft_p95`
|
||||
//! rising 0.53 → 11.77 s, and it read as a 38% throughput regression
|
||||
//! until the one commit in the window turned out to be an admission
|
||||
//! *policy* change that bench, being anonymous, was subject to (#288).
|
||||
//! - quadbrat's `capability:debug-reason` TTFT fell 88.8 → 0.1 s, which
|
||||
//! read as a latency win and was a redefinition of when the clock
|
||||
//! stops.
|
||||
//! - the same redefinition put decode rates at ~250 tok/s on a 3060
|
||||
//! beforehand, by dividing reasoning-inclusive token counts by a
|
||||
//! visible-content-only window.
|
||||
//!
|
||||
//! So the boundaries are declared, not remembered. Declared here rather
|
||||
//! than in the frontend because `report` renders the same series into
|
||||
//! `benchmarks.md` and needs the same caveats.
|
||||
//!
|
||||
//! ## What belongs here
|
||||
//!
|
||||
//! Only changes to **what a metric means**. A change that made the
|
||||
//! server genuinely faster, slower, or differently behaved does *not*
|
||||
//! belong — that is the signal these lines exist to keep readable. #280
|
||||
//! (fetch the model's own chat template) moved quadbrat's
|
||||
//! `completion_tokens` from 4096 to 1053 by ending a truncation loop;
|
||||
//! that is a real behavioural change and stays unmarked.
|
||||
//!
|
||||
//! The identified/anonymous split is deliberately absent too: it is
|
||||
//! recorded per-row in `runs.principal`, so the UI derives it from the
|
||||
//! data instead of trusting a constant to stay true.
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
/// A build at which one or more metrics changed meaning.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct MeasurementRegime {
|
||||
/// Short SHA of the first build measured under the new semantics.
|
||||
/// Matched against `SeriesPoint::git_sha`, so a boundary whose build
|
||||
/// never appears in a selection simply does not draw.
|
||||
pub first_sha: &'static str,
|
||||
/// Terse label for the chart rule.
|
||||
pub label: &'static str,
|
||||
/// What changed, and why the step is not a performance result.
|
||||
pub detail: &'static str,
|
||||
/// Series keys the change affects, as the API/UI name them. A
|
||||
/// boundary only draws on panels carrying one of these — marking
|
||||
/// every panel would train people to ignore the rules.
|
||||
pub affects: &'static [&'static str],
|
||||
}
|
||||
|
||||
/// The declared boundaries, oldest first.
|
||||
pub const REGIMES: &[MeasurementRegime] = &[
|
||||
MeasurementRegime {
|
||||
first_sha: "92dddc0",
|
||||
label: "#262 anonymous callers yield",
|
||||
detail: "Anonymous traffic became servable only from capacity left over \
|
||||
once identified traffic is satisfied — capped below max_in_flight \
|
||||
and parked at the class gate. An anonymous bench therefore \
|
||||
characterises the yield policy rather than serving capacity. \
|
||||
Steps here are admission policy, not the engine (#288).",
|
||||
affects: &["decode", "ttft", "ttftP95", "queueWait", "rejected"],
|
||||
},
|
||||
MeasurementRegime {
|
||||
first_sha: "0fe7aa3",
|
||||
label: "#117 reasoning counts as liveness",
|
||||
detail: "TTFT previously waited for the first *visible content* chunk, so \
|
||||
on a thinking model it measured the whole reasoning span (88.8 s \
|
||||
on quadbrat). It now stops at the first delta of any kind. The \
|
||||
same change fixed decode rates computed by dividing \
|
||||
reasoning-inclusive token counts by a content-only window, which \
|
||||
had produced ~250 tok/s on a 3060.",
|
||||
affects: &["decode", "ttft"],
|
||||
},
|
||||
];
|
||||
|
||||
/// Boundaries affecting `metric`, in declaration order.
|
||||
pub fn regimes_for(metric: &str) -> Vec<&'static MeasurementRegime> {
|
||||
REGIMES
|
||||
.iter()
|
||||
.filter(|r| r.affects.contains(&metric))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A boundary that names no metric would draw nowhere and mislead by
|
||||
/// omission; one naming an unknown key would draw nowhere and look
|
||||
/// fine. Both are typos this catches.
|
||||
#[test]
|
||||
fn every_regime_affects_a_known_series_key() {
|
||||
// The keys `Trends.tsx` maps `SeriesPoint` onto.
|
||||
const KNOWN: &[&str] = &[
|
||||
"decode",
|
||||
"ttft",
|
||||
"ttftP95",
|
||||
"queueWait",
|
||||
"rejected",
|
||||
"prefillTps",
|
||||
"reasoning",
|
||||
"cached",
|
||||
"completion",
|
||||
"tpot",
|
||||
];
|
||||
for r in REGIMES {
|
||||
assert!(
|
||||
!r.affects.is_empty(),
|
||||
"{} affects nothing, so it would never draw",
|
||||
r.first_sha
|
||||
);
|
||||
for key in r.affects {
|
||||
assert!(
|
||||
KNOWN.contains(key),
|
||||
"{} names unknown series key {key:?}",
|
||||
r.first_sha
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn regimes_for_selects_by_metric() {
|
||||
assert!(regimes_for("ttft").iter().any(|r| r.first_sha == "0fe7aa3"));
|
||||
assert!(
|
||||
regimes_for("completion").is_empty(),
|
||||
"#280 changed behaviour, not meaning — it must not be marked"
|
||||
);
|
||||
}
|
||||
|
||||
/// Short SHAs are matched against `SeriesPoint::git_sha`, which the
|
||||
/// store records at the width `/version` reports.
|
||||
#[test]
|
||||
fn declared_shas_are_short_form() {
|
||||
for r in REGIMES {
|
||||
assert_eq!(
|
||||
r.first_sha.len(),
|
||||
7,
|
||||
"{} is not a 7-char short sha; it would never match a series point",
|
||||
r.first_sha
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,6 @@
|
||||
use crate::config::ScenarioConfig;
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use async_trait::async_trait;
|
||||
use cortex_core::entitlements::HEADER_KEY_ID;
|
||||
use cortex_core::harness::ModelInfo;
|
||||
use cortex_core::openai::ChatCompletionChunk;
|
||||
use eventsource_stream::Eventsource;
|
||||
@@ -47,17 +46,6 @@ pub struct RunCtx<'a> {
|
||||
pub model_id: String,
|
||||
pub max_tokens: u64,
|
||||
pub timeout: Duration,
|
||||
/// Base `x-helexa-key-id` bench presents (#288), when configured.
|
||||
///
|
||||
/// The client already sends this as a default header, so every
|
||||
/// scenario is identified without having to remember. A concurrency
|
||||
/// burst overrides it per stream — see
|
||||
/// [`ConcurrencyScenario`] — because fair-share (#54) bounds one
|
||||
/// principal to `max_per_principal` in-flight-or-queued requests,
|
||||
/// which defaults to 2. Eight streams under one identity would be
|
||||
/// capped at two and the rest rejected, measuring the fair-share cap
|
||||
/// instead of the server.
|
||||
pub principal_key_id: Option<String>,
|
||||
}
|
||||
|
||||
/// Operator-felt metrics for a single measured request.
|
||||
@@ -84,34 +72,6 @@ pub struct ScenarioMetrics {
|
||||
pub decode_ms: Option<u64>,
|
||||
/// Tokens submitted to prefill — the denominator for prefill tok/s.
|
||||
pub prefill_tokens: Option<u64>,
|
||||
/// Tokens spent inside the reasoning span, from
|
||||
/// `usage.completion_tokens_details.reasoning_tokens` (#223). A
|
||||
/// sub-count of `completion_tokens`.
|
||||
///
|
||||
/// Worth a trend of its own: on a reasoning model this is the
|
||||
/// dominant cost driver and it moves independently of speed. A
|
||||
/// template change, a sampling change (#283) or a reasoning-budget
|
||||
/// change can double what the model thinks before answering while
|
||||
/// every tok/s number stays flat — the bill doubles and no chart
|
||||
/// moves.
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
/// p95 inter-token arrival gap in milliseconds — the tail of the
|
||||
/// stream's smoothness, client-observed.
|
||||
///
|
||||
/// `decode_tps` is a mean over the whole decode window, so a stream
|
||||
/// that stalls for a second and then catches up is indistinguishable
|
||||
/// from one that never stalled. This is the number a user actually
|
||||
/// feels, and the one a batching stall or a mid-stream rebatch shows
|
||||
/// up in. `None` for non-streaming, which has no inter-token gaps by
|
||||
/// construction, and for streams too short to have a tail.
|
||||
pub tpot_p95_ms: Option<f64>,
|
||||
/// Prompt tokens served from neuron's prefix KV cache (#269), from
|
||||
/// `usage.prompt_tokens_details.cached_tokens`.
|
||||
///
|
||||
/// The reason prefill timing varies between otherwise identical
|
||||
/// samples. #269 exists because the saving was invisible and every
|
||||
/// client reported a 0% hit rate; bench then threw the number away.
|
||||
pub cached_tokens: Option<u64>,
|
||||
// ── Concurrency / agentic-load fields (#89) ──────────────────────────
|
||||
// Set only by the concurrency scenario, which fans out N simultaneous
|
||||
// streams to characterize the real a0/hermes/opencode workload that
|
||||
@@ -243,9 +203,6 @@ impl Scenario for ImageLatencyScenario {
|
||||
.as_u64()
|
||||
.map(|d| d + timing["decode_ms"].as_u64().unwrap_or(0)),
|
||||
prefill_tokens: None,
|
||||
reasoning_tokens: None,
|
||||
cached_tokens: None,
|
||||
tpot_p95_ms: None,
|
||||
concurrency: None,
|
||||
ttft_p95_s: None,
|
||||
queue_wait_ms_median: None,
|
||||
@@ -276,18 +233,6 @@ pub fn build_scenarios(cfg: &ScenarioConfig) -> Vec<Box<dyn Scenario>> {
|
||||
id: format!("concurrency:{n}"),
|
||||
concurrency: n,
|
||||
approx_prompt_tokens: cfg.concurrency_prompt_tokens,
|
||||
streaming: true,
|
||||
}) as Box<dyn Scenario>);
|
||||
}
|
||||
// Non-streaming bursts (#285). A separate cell id so the two shapes
|
||||
// never average together — they measured 1.00x against 3.98x before
|
||||
// #285, and a combined number would have hidden both.
|
||||
for &n in &cfg.concurrency_nonstreaming_levels {
|
||||
scenarios.push(Box::new(ConcurrencyScenario {
|
||||
id: format!("concurrency:{n}:nostream"),
|
||||
concurrency: n,
|
||||
approx_prompt_tokens: cfg.concurrency_prompt_tokens,
|
||||
streaming: false,
|
||||
}) as Box<dyn Scenario>);
|
||||
}
|
||||
for &side in &cfg.image_sizes {
|
||||
@@ -312,7 +257,7 @@ pub fn build_scenarios(cfg: &ScenarioConfig) -> Vec<Box<dyn Scenario>> {
|
||||
pub async fn cold_probe(ctx: &RunCtx<'_>) -> Result<ScenarioMetrics> {
|
||||
let prompt = build_prompt(128);
|
||||
let payload = chat_payload(ctx, &prompt);
|
||||
tokio::time::timeout(ctx.timeout, stream_and_measure(ctx, &payload, None))
|
||||
tokio::time::timeout(ctx.timeout, stream_and_measure(ctx, &payload))
|
||||
.await
|
||||
.map_err(|_| anyhow!("cold probe timed out after {:?}", ctx.timeout))?
|
||||
}
|
||||
@@ -350,7 +295,7 @@ impl Scenario for ChatLatencyScenario {
|
||||
async fn run(&self, ctx: &RunCtx) -> Result<ScenarioMetrics> {
|
||||
let prompt = build_prompt(self.approx_prompt_tokens);
|
||||
let payload = chat_payload(ctx, &prompt);
|
||||
let fut = stream_and_measure(ctx, &payload, None);
|
||||
let fut = stream_and_measure(ctx, &payload);
|
||||
tokio::time::timeout(ctx.timeout, fut)
|
||||
.await
|
||||
.map_err(|_| anyhow!("request timed out after {:?}", ctx.timeout))?
|
||||
@@ -368,16 +313,6 @@ pub struct ConcurrencyScenario {
|
||||
id: String,
|
||||
concurrency: u32,
|
||||
approx_prompt_tokens: u32,
|
||||
/// Whether the burst streams (#285).
|
||||
///
|
||||
/// Both shapes matter and they are not interchangeable. Streaming
|
||||
/// multiplexes through the batch engine; non-streaming did not until
|
||||
/// #285, and serialized completely — 1.00x aggregate throughput at
|
||||
/// every concurrency level against 3.98x streamed. A bench that only
|
||||
/// streams cannot see that, which is why it went unmeasured: the
|
||||
/// neuron reports `in_flight: 8`, admission accepts, `/health` looks
|
||||
/// healthy, and throughput is single-stream.
|
||||
streaming: bool,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -392,26 +327,13 @@ impl Scenario for ConcurrencyScenario {
|
||||
|
||||
async fn run(&self, ctx: &RunCtx) -> Result<ScenarioMetrics> {
|
||||
let prompt = build_prompt(self.approx_prompt_tokens);
|
||||
let payload = if self.streaming {
|
||||
chat_payload(ctx, &prompt)
|
||||
} else {
|
||||
nonstreaming_payload(ctx, &prompt)
|
||||
};
|
||||
let payload = chat_payload(ctx, &prompt);
|
||||
|
||||
// Fire all requests at once; each is independently timed and capped
|
||||
// by the per-request timeout so one hung request can't stall the
|
||||
// burst.
|
||||
// Fire all streams at once; each is independently timed and capped by
|
||||
// the per-request timeout so one hung stream can't stall the burst.
|
||||
let burst_start = Instant::now();
|
||||
let streaming = self.streaming;
|
||||
let payload = &payload;
|
||||
let futs = (0..self.concurrency).map(move |i| async move {
|
||||
// Each stream presents its own identity — see
|
||||
// `with_stream_identity`.
|
||||
if streaming {
|
||||
tokio::time::timeout(ctx.timeout, stream_and_measure(ctx, payload, Some(i))).await
|
||||
} else {
|
||||
tokio::time::timeout(ctx.timeout, request_and_measure(ctx, payload, Some(i))).await
|
||||
}
|
||||
let futs = (0..self.concurrency).map(|_| async {
|
||||
tokio::time::timeout(ctx.timeout, stream_and_measure(ctx, &payload)).await
|
||||
});
|
||||
let results = futures::future::join_all(futs).await;
|
||||
let burst_window = burst_start.elapsed().as_secs_f64();
|
||||
@@ -429,7 +351,7 @@ impl Scenario for ConcurrencyScenario {
|
||||
}
|
||||
if streams.is_empty() {
|
||||
return Err(anyhow!(
|
||||
"all {} concurrent requests failed ({rejected} shed by admission)",
|
||||
"all {} concurrent streams failed ({rejected} shed by admission)",
|
||||
self.concurrency
|
||||
));
|
||||
}
|
||||
@@ -458,50 +380,9 @@ impl Scenario for ConcurrencyScenario {
|
||||
total_s: burst_window,
|
||||
prompt_tokens: streams.iter().find_map(|m| m.prompt_tokens),
|
||||
completion_tokens: total_tokens,
|
||||
// Median server-side phase timing across the burst. These
|
||||
// were `None`, which left the prefill-tok/s panel blank for
|
||||
// exactly the cells where queue behaviour matters most, and
|
||||
// made `queue_wait_ms` (TTFT minus server prefill, #85)
|
||||
// uncomputable for a burst — the one number that separates
|
||||
// "the server is slow" from "you were queued".
|
||||
//
|
||||
// Median rather than sum: every stream prefills its own copy
|
||||
// of the same prompt, so the representative per-stream cost
|
||||
// is the meaningful figure and it composes with `ttft_s`,
|
||||
// which is also a median.
|
||||
prefill_ms: median(
|
||||
&streams
|
||||
.iter()
|
||||
.filter_map(|m| m.prefill_ms.map(|v| v as f64))
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
.map(|v| v as u64),
|
||||
decode_ms: median(
|
||||
&streams
|
||||
.iter()
|
||||
.filter_map(|m| m.decode_ms.map(|v| v as f64))
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
.map(|v| v as u64),
|
||||
prefill_tokens: median(
|
||||
&streams
|
||||
.iter()
|
||||
.filter_map(|m| m.prefill_tokens.map(|v| v as f64))
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
.map(|v| v as u64),
|
||||
// Summed across the burst, like `completion_tokens`: the cell
|
||||
// describes the whole burst, so a per-stream figure here
|
||||
// would not compose with it.
|
||||
reasoning_tokens: sum_opt(&streams, |m| m.reasoning_tokens),
|
||||
cached_tokens: sum_opt(&streams, |m| m.cached_tokens),
|
||||
// The worst stream in the burst, not a median of medians:
|
||||
// under load the question is how bad it got for somebody,
|
||||
// and averaging tails is how a stall stays invisible.
|
||||
tpot_p95_ms: streams
|
||||
.iter()
|
||||
.filter_map(|m| m.tpot_p95_ms)
|
||||
.max_by(|a, b| a.total_cmp(b)),
|
||||
prefill_ms: None,
|
||||
decode_ms: None,
|
||||
prefill_tokens: None,
|
||||
concurrency: Some(self.concurrency),
|
||||
ttft_p95_s: percentile(&ttfts, 95.0),
|
||||
queue_wait_ms_median: median(&queue_waits),
|
||||
@@ -512,146 +393,6 @@ impl Scenario for ConcurrencyScenario {
|
||||
}
|
||||
}
|
||||
|
||||
/// Sum an optional per-stream count across a burst, `None` when no
|
||||
/// stream reported it — so "nobody measured it" stays distinguishable
|
||||
/// from "measured as zero", the same distinction #269 exists to
|
||||
/// preserve.
|
||||
fn sum_opt(
|
||||
streams: &[ScenarioMetrics],
|
||||
f: impl Fn(&ScenarioMetrics) -> Option<u64>,
|
||||
) -> Option<u64> {
|
||||
let vals: Vec<u64> = streams.iter().filter_map(f).collect();
|
||||
(!vals.is_empty()).then(|| vals.iter().sum())
|
||||
}
|
||||
|
||||
/// Per-stream identity for a concurrency burst (#288/#54).
|
||||
///
|
||||
/// Fair-share bounds *one principal* to `max_per_principal`
|
||||
/// in-flight-or-queued requests, defaulting to 2. A burst of 8 under a
|
||||
/// single identity would therefore have 6 rejected with `PrincipalCap`,
|
||||
/// and the scenario would measure the fair-share cap rather than the
|
||||
/// server.
|
||||
///
|
||||
/// Giving each stream its own key is also the more faithful model: a
|
||||
/// concurrency-8 burst is meant to stand in for eight simultaneous
|
||||
/// *callers*, not one caller issuing eight requests. Fair-share is still
|
||||
/// exercised — each identity is subject to it — just not self-inflicted.
|
||||
fn with_stream_identity(
|
||||
rb: reqwest::RequestBuilder,
|
||||
ctx: &RunCtx<'_>,
|
||||
stream: Option<u32>,
|
||||
) -> reqwest::RequestBuilder {
|
||||
match (ctx.principal_key_id.as_deref(), stream) {
|
||||
(Some(base), Some(i)) => rb.header(HEADER_KEY_ID, format!("{base}-{i}")),
|
||||
// No principal configured, or a single-request scenario: the
|
||||
// client's default headers already carry the base identity.
|
||||
_ => rb,
|
||||
}
|
||||
}
|
||||
|
||||
/// The non-streaming counterpart to [`chat_payload`] (#285).
|
||||
///
|
||||
/// `stream_options` is deliberately absent: usage is part of the body on
|
||||
/// this shape, and sending the streaming-only option to a strict server
|
||||
/// is a needless compatibility risk.
|
||||
fn nonstreaming_payload(ctx: &RunCtx, prompt: &str) -> serde_json::Value {
|
||||
json!({
|
||||
"model": ctx.model_id,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": ctx.max_tokens,
|
||||
"temperature": 0,
|
||||
"stream": false,
|
||||
})
|
||||
}
|
||||
|
||||
/// Time one non-streaming chat completion (#285).
|
||||
///
|
||||
/// There is no first-chunk moment on this shape — the whole body arrives
|
||||
/// at once — so `ttft_s` is the full request wall-clock rather than a
|
||||
/// separate measurement. That is the honest reading: for a non-streaming
|
||||
/// caller, time-to-first-token *is* time-to-everything, and it is exactly
|
||||
/// what makes the serialization this scenario exists to detect so
|
||||
/// expensive.
|
||||
///
|
||||
/// `decode_tps` is per-request; the burst aggregate is computed by the
|
||||
/// caller over the whole window, the same way the streaming path does it.
|
||||
async fn request_and_measure(
|
||||
ctx: &RunCtx<'_>,
|
||||
payload: &serde_json::Value,
|
||||
stream: Option<u32>,
|
||||
) -> Result<ScenarioMetrics> {
|
||||
let start = Instant::now();
|
||||
let resp = with_stream_identity(ctx.client.post(&ctx.chat_url), ctx, stream)
|
||||
.json(payload)
|
||||
.send()
|
||||
.await
|
||||
.context("sending non-streaming chat request")?;
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
return Err(anyhow!("upstream returned {status}: {}", body.trim()));
|
||||
}
|
||||
let v: serde_json::Value = resp
|
||||
.json()
|
||||
.await
|
||||
.context("non-streaming chat response was not JSON")?;
|
||||
let total_s = start.elapsed().as_secs_f64();
|
||||
|
||||
let usage = v.get("usage");
|
||||
let completion_tokens = usage
|
||||
.and_then(|u| u.get("completion_tokens"))
|
||||
.and_then(|t| t.as_u64())
|
||||
.unwrap_or(0);
|
||||
let prompt_tokens = usage
|
||||
.and_then(|u| u.get("prompt_tokens"))
|
||||
.and_then(|t| t.as_u64());
|
||||
let reasoning_tokens = usage
|
||||
.and_then(|u| u.get("completion_tokens_details"))
|
||||
.and_then(|d| d.get("reasoning_tokens"))
|
||||
.and_then(|t| t.as_u64());
|
||||
let cached_tokens = usage
|
||||
.and_then(|u| u.get("prompt_tokens_details"))
|
||||
.and_then(|d| d.get("cached_tokens"))
|
||||
.and_then(|t| t.as_u64());
|
||||
let timing = usage.and_then(|u| u.get("helexa_timing"));
|
||||
let field = |name: &str| timing.and_then(|t| t.get(name)).and_then(|x| x.as_u64());
|
||||
let (prefill_ms, decode_ms, prefill_tokens) = (
|
||||
field("prefill_ms"),
|
||||
field("decode_ms"),
|
||||
field("prefill_tokens"),
|
||||
);
|
||||
|
||||
// Prefer the server's own decode window when it reports one (#85);
|
||||
// it excludes prefill, so it is comparable with the streaming path's
|
||||
// decode-window rate rather than being diluted by it.
|
||||
let decode_tps = match decode_ms {
|
||||
Some(ms) if ms > 200 => Some(completion_tokens as f64 / (ms as f64 / 1000.0)),
|
||||
_ if total_s > 0.2 => Some(completion_tokens as f64 / total_s),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
Ok(ScenarioMetrics {
|
||||
ttft_s: total_s,
|
||||
decode_tps,
|
||||
total_s,
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
prefill_ms,
|
||||
decode_ms,
|
||||
prefill_tokens,
|
||||
reasoning_tokens,
|
||||
cached_tokens,
|
||||
// Non-streaming has no inter-token gaps: the body arrives whole.
|
||||
tpot_p95_ms: None,
|
||||
concurrency: None,
|
||||
ttft_p95_s: None,
|
||||
queue_wait_ms_median: None,
|
||||
rejected: None,
|
||||
artifact: None,
|
||||
image_units: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Quality probe (#91): runs a fixed prompt and stores the full generated
|
||||
/// text as an artifact for later scoring (manual now, LLM-judge later). The
|
||||
/// point is to compare reasoning/planning quality across models — the axis
|
||||
@@ -684,7 +425,7 @@ impl Scenario for CapabilityScenario {
|
||||
"stream": true,
|
||||
"stream_options": {"include_usage": true},
|
||||
});
|
||||
let fut = stream_and_measure_inner(ctx, &payload, true, None);
|
||||
let fut = stream_and_measure_inner(ctx, &payload, true);
|
||||
tokio::time::timeout(ctx.timeout, fut)
|
||||
.await
|
||||
.map_err(|_| anyhow!("capability probe timed out after {:?}", ctx.timeout))?
|
||||
@@ -727,9 +468,8 @@ fn percentile(values: &[f64], p: f64) -> Option<f64> {
|
||||
async fn stream_and_measure(
|
||||
ctx: &RunCtx<'_>,
|
||||
payload: &serde_json::Value,
|
||||
stream: Option<u32>,
|
||||
) -> Result<ScenarioMetrics> {
|
||||
stream_and_measure_inner(ctx, payload, false, stream).await
|
||||
stream_and_measure_inner(ctx, payload, false).await
|
||||
}
|
||||
|
||||
/// As [`stream_and_measure`] but accumulates the full visible text when
|
||||
@@ -739,10 +479,11 @@ async fn stream_and_measure_inner(
|
||||
ctx: &RunCtx<'_>,
|
||||
payload: &serde_json::Value,
|
||||
capture_text: bool,
|
||||
stream_ix: Option<u32>,
|
||||
) -> Result<ScenarioMetrics> {
|
||||
let start = Instant::now();
|
||||
let resp = with_stream_identity(ctx.client.post(&ctx.chat_url), ctx, stream_ix)
|
||||
let resp = ctx
|
||||
.client
|
||||
.post(&ctx.chat_url)
|
||||
.json(payload)
|
||||
.send()
|
||||
.await
|
||||
@@ -762,10 +503,6 @@ async fn stream_and_measure_inner(
|
||||
let mut prefill_ms: Option<u64> = None;
|
||||
let mut decode_ms: Option<u64> = None;
|
||||
let mut prefill_tokens: Option<u64> = None;
|
||||
let mut reasoning_tokens: Option<u64> = None;
|
||||
let mut cached_tokens: Option<u64> = None;
|
||||
// Inter-token arrival gaps, for the p95 tail.
|
||||
let mut gaps_ms: Vec<f64> = Vec::new();
|
||||
let mut captured = String::new();
|
||||
|
||||
while let Some(event) = stream.next().await {
|
||||
@@ -803,11 +540,6 @@ async fn stream_and_measure_inner(
|
||||
if content.is_some() || reasoning.is_some() {
|
||||
if first.is_none() {
|
||||
first = Some(now);
|
||||
} else if let Some(prev) = last {
|
||||
// Gaps only *between* generated deltas: the wait for
|
||||
// the first one is TTFT and belongs to prefill, not
|
||||
// to stream smoothness.
|
||||
gaps_ms.push(now.duration_since(prev).as_secs_f64() * 1000.0);
|
||||
}
|
||||
last = Some(now);
|
||||
chunk_count += 1;
|
||||
@@ -819,14 +551,6 @@ async fn stream_and_measure_inner(
|
||||
if let Some(usage) = chunk.usage {
|
||||
prompt_tokens = Some(usage.prompt_tokens);
|
||||
completion_tokens = Some(usage.completion_tokens);
|
||||
reasoning_tokens = usage
|
||||
.completion_tokens_details
|
||||
.as_ref()
|
||||
.map(|d| d.reasoning_tokens);
|
||||
cached_tokens = usage
|
||||
.prompt_tokens_details
|
||||
.as_ref()
|
||||
.map(|d| d.cached_tokens);
|
||||
if let Some(t) = usage.helexa_timing {
|
||||
prefill_ms = Some(t.prefill_ms);
|
||||
decode_ms = Some(t.decode_ms);
|
||||
@@ -865,9 +589,6 @@ async fn stream_and_measure_inner(
|
||||
prefill_ms,
|
||||
decode_ms,
|
||||
prefill_tokens,
|
||||
reasoning_tokens,
|
||||
cached_tokens,
|
||||
tpot_p95_ms: percentile(&gaps_ms, 95.0),
|
||||
// Concurrency fields unset on the single-request path; the
|
||||
// concurrency scenario builds its own aggregate (#89).
|
||||
concurrency: None,
|
||||
@@ -883,113 +604,6 @@ async fn stream_and_measure_inner(
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A burst must report server-side phase timing, or `queue_wait_ms`
|
||||
/// (TTFT minus server prefill, #85) cannot be computed for the very
|
||||
/// cells it was designed for, and the prefill-tok/s panel stays
|
||||
/// blank under concurrency.
|
||||
#[test]
|
||||
fn a_burst_reports_median_phase_timing_not_none() {
|
||||
let s = |prefill_ms, decode_ms, prefill_tokens| ScenarioMetrics {
|
||||
ttft_s: 0.1,
|
||||
decode_tps: Some(10.0),
|
||||
total_s: 1.0,
|
||||
prompt_tokens: Some(100),
|
||||
completion_tokens: 10,
|
||||
prefill_ms,
|
||||
decode_ms,
|
||||
prefill_tokens,
|
||||
reasoning_tokens: None,
|
||||
cached_tokens: None,
|
||||
tpot_p95_ms: Some(5.0),
|
||||
concurrency: None,
|
||||
ttft_p95_s: None,
|
||||
queue_wait_ms_median: None,
|
||||
rejected: None,
|
||||
artifact: None,
|
||||
image_units: None,
|
||||
};
|
||||
let streams = [
|
||||
s(Some(10), Some(100), Some(512)),
|
||||
s(Some(20), Some(200), Some(512)),
|
||||
s(Some(30), Some(300), Some(512)),
|
||||
];
|
||||
let med = |f: fn(&ScenarioMetrics) -> Option<u64>| {
|
||||
median(
|
||||
&streams
|
||||
.iter()
|
||||
.filter_map(|m| f(m).map(|v| v as f64))
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
.map(|v| v as u64)
|
||||
};
|
||||
assert_eq!(med(|m| m.prefill_ms), Some(20));
|
||||
assert_eq!(med(|m| m.decode_ms), Some(200));
|
||||
// Every stream prefills the same prompt, so the median is the
|
||||
// prompt size rather than a sum across the burst.
|
||||
assert_eq!(med(|m| m.prefill_tokens), Some(512));
|
||||
}
|
||||
|
||||
/// #288/#54: a burst must present one identity *per stream*.
|
||||
///
|
||||
/// Fair-share bounds a single principal to `max_per_principal`
|
||||
/// in-flight-or-queued requests, which defaults to 2. Eight streams
|
||||
/// under one key would have six rejected with `PrincipalCap`, and
|
||||
/// `concurrency:8` would silently measure the fair-share cap instead
|
||||
/// of the server — a worse failure than the anonymity it replaced,
|
||||
/// because it looks like backpressure.
|
||||
#[test]
|
||||
fn a_burst_presents_one_identity_per_stream() {
|
||||
let client = reqwest::Client::new();
|
||||
let ctx = RunCtx {
|
||||
client: &client,
|
||||
chat_url: "http://x/v1/chat/completions".into(),
|
||||
model_id: "m".into(),
|
||||
max_tokens: 16,
|
||||
timeout: Duration::from_secs(1),
|
||||
principal_key_id: Some("fleet-benchmark".into()),
|
||||
};
|
||||
let key_of = |stream: Option<u32>| -> Option<String> {
|
||||
with_stream_identity(client.post(&ctx.chat_url), &ctx, stream)
|
||||
.build()
|
||||
.expect("build")
|
||||
.headers()
|
||||
.get(HEADER_KEY_ID)
|
||||
.map(|v| v.to_str().expect("utf8").to_string())
|
||||
};
|
||||
let keys: Vec<Option<String>> = (0..3).map(|i| key_of(Some(i))).collect();
|
||||
assert_eq!(
|
||||
keys,
|
||||
vec![
|
||||
Some("fleet-benchmark-0".to_string()),
|
||||
Some("fleet-benchmark-1".to_string()),
|
||||
Some("fleet-benchmark-2".to_string()),
|
||||
]
|
||||
);
|
||||
// Distinct, or the cap bites anyway.
|
||||
let uniq: std::collections::HashSet<_> = keys.iter().collect();
|
||||
assert_eq!(uniq.len(), 3);
|
||||
}
|
||||
|
||||
/// A single-request scenario adds no override: the client's default
|
||||
/// headers already carry the base identity, and a per-request copy
|
||||
/// would be a second place for it to drift.
|
||||
#[test]
|
||||
fn a_single_request_scenario_keeps_the_client_identity() {
|
||||
let client = reqwest::Client::new();
|
||||
let ctx = RunCtx {
|
||||
client: &client,
|
||||
chat_url: "http://x/v1/chat/completions".into(),
|
||||
model_id: "m".into(),
|
||||
max_tokens: 16,
|
||||
timeout: Duration::from_secs(1),
|
||||
principal_key_id: Some("fleet-benchmark".into()),
|
||||
};
|
||||
let req = with_stream_identity(client.post(&ctx.chat_url), &ctx, None)
|
||||
.build()
|
||||
.expect("build");
|
||||
assert!(req.headers().get(HEADER_KEY_ID).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prompt_grows_with_token_target() {
|
||||
let small = build_prompt(128);
|
||||
@@ -1032,7 +646,6 @@ mod tests {
|
||||
prompt_sizes: vec![128],
|
||||
max_tokens: 64,
|
||||
concurrency_levels: vec![2, 8],
|
||||
concurrency_nonstreaming_levels: vec![8],
|
||||
concurrency_prompt_tokens: 512,
|
||||
capability_probes: vec![CapabilityProbe {
|
||||
name: "plan".into(),
|
||||
|
||||
@@ -14,22 +14,6 @@ use std::path::Path;
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RunRecord {
|
||||
pub ts: String, // RFC3339
|
||||
/// Identity this sample was taken under (#288), `account/key`, or
|
||||
/// `None` for an anonymous run. Anonymous samples measure the
|
||||
/// admission yield policy rather than serving capacity once #262 is
|
||||
/// in the build, so this is the column that says whether two rows
|
||||
/// are comparable at all.
|
||||
pub principal: Option<String>,
|
||||
/// Tokens spent reasoning (#223) — the dominant cost driver on a
|
||||
/// reasoning model, and one that moves independently of every tok/s
|
||||
/// number on the dashboard.
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
/// Prompt tokens served from the prefix cache (#269) — the reason
|
||||
/// prefill timing varies between otherwise identical samples.
|
||||
pub cached_tokens: Option<u64>,
|
||||
/// p95 inter-token arrival gap (ms) — stream smoothness, which a
|
||||
/// mean tok/s cannot express.
|
||||
pub tpot_p95_ms: Option<f64>,
|
||||
// target
|
||||
pub target_name: String,
|
||||
pub target_kind: String,
|
||||
@@ -220,18 +204,6 @@ impl Store {
|
||||
("artifact", "TEXT"),
|
||||
("quality_score", "REAL"),
|
||||
("scorer", "TEXT"),
|
||||
// #288: which identity the sample was taken under. Rows
|
||||
// predating this are NULL, which is honest — they were
|
||||
// anonymous, and since #262 that measures a different
|
||||
// thing. Without the column the 2026-08-18 discontinuity
|
||||
// is folklore rather than data.
|
||||
("principal", "TEXT"),
|
||||
// #223 / #269: both are on the wire and were being discarded.
|
||||
("reasoning_tokens", "INTEGER"),
|
||||
("cached_tokens", "INTEGER"),
|
||||
// #285: inter-token tail. `decode_tps` is a mean and hides
|
||||
// a stall that later catches up.
|
||||
("tpot_p95_ms", "REAL"),
|
||||
],
|
||||
)?;
|
||||
Ok(())
|
||||
@@ -292,7 +264,6 @@ impl Store {
|
||||
swap_unload_ms, swap_load_ms,
|
||||
artifact, quality_score, scorer,
|
||||
image_units,
|
||||
principal, reasoning_tokens, cached_tokens, tpot_p95_ms,
|
||||
ok, error
|
||||
) VALUES (
|
||||
?1, ?2, ?3, ?4,
|
||||
@@ -309,8 +280,7 @@ impl Store {
|
||||
?42, ?43,
|
||||
?44, ?45, ?46,
|
||||
?47,
|
||||
?48, ?49, ?50, ?51,
|
||||
?52, ?53
|
||||
?48, ?49
|
||||
)",
|
||||
params![
|
||||
r.ts,
|
||||
@@ -360,10 +330,6 @@ impl Store {
|
||||
r.quality_score,
|
||||
r.scorer,
|
||||
r.image_units,
|
||||
r.principal,
|
||||
r.reasoning_tokens,
|
||||
r.cached_tokens,
|
||||
r.tpot_p95_ms,
|
||||
r.ok as i64,
|
||||
r.error,
|
||||
],
|
||||
@@ -737,15 +703,7 @@ impl Store {
|
||||
}
|
||||
};
|
||||
let mut stmt = self.conn.prepare(
|
||||
"SELECT git_sha, build_timestamp, package_version, ttft_s, decode_tps, total_s,
|
||||
ttft_p95_s, queue_wait_ms, rejected,
|
||||
-- Prefill rate derived here rather than stored: the
|
||||
-- inputs are already recorded and a stored rate
|
||||
-- would be a third thing to keep consistent.
|
||||
CASE WHEN prefill_ms > 0 AND prefill_tokens IS NOT NULL
|
||||
THEN (prefill_tokens * 1000.0) / prefill_ms END AS prefill_tps,
|
||||
reasoning_tokens, cached_tokens, completion_tokens,
|
||||
tpot_p95_ms, principal, ts
|
||||
"SELECT git_sha, build_timestamp, package_version, ttft_s, decode_tps, total_s, ts
|
||||
FROM runs
|
||||
WHERE ok=1 AND target_name=?1 AND model_id=?2 AND scenario_id=?3
|
||||
ORDER BY id",
|
||||
@@ -759,16 +717,7 @@ impl Store {
|
||||
ttft_s: r.get(3)?,
|
||||
decode_tps: r.get(4)?,
|
||||
total_s: r.get(5)?,
|
||||
ttft_p95_s: r.get(6)?,
|
||||
queue_wait_ms: r.get(7)?,
|
||||
rejected: r.get(8)?,
|
||||
prefill_tps: r.get(9)?,
|
||||
reasoning_tokens: r.get(10)?,
|
||||
cached_tokens: r.get(11)?,
|
||||
completion_tokens: r.get(12)?,
|
||||
tpot_p95_ms: r.get(13)?,
|
||||
principal: r.get(14)?,
|
||||
ts: r.get(15)?,
|
||||
ts: r.get(6)?,
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<_>>()?;
|
||||
@@ -889,33 +838,6 @@ pub struct SeriesPoint {
|
||||
pub ttft_s_median: Option<f64>,
|
||||
pub decode_tps_median: Option<f64>,
|
||||
pub total_s_median: Option<f64>,
|
||||
// ── Added because the two above cannot show what changed ──────
|
||||
/// Tail TTFT. The median above stayed flat at ~0.3 s across the
|
||||
/// build where p95 went 0.53 s → 11.77 s (#288); charting only the
|
||||
/// median is why that went unnoticed for a week.
|
||||
pub ttft_p95_s_median: Option<f64>,
|
||||
/// Admission queue wait — separates "the server is slow" from
|
||||
/// "you were queued behind someone".
|
||||
pub queue_wait_ms_median: Option<f64>,
|
||||
/// Requests shed by admission during the burst.
|
||||
pub rejected_median: Option<f64>,
|
||||
/// Prefill throughput, derived from `prefill_tokens / prefill_ms`.
|
||||
/// The other half of serving speed; only decode was ever charted.
|
||||
pub prefill_tps_median: Option<f64>,
|
||||
/// Tokens spent reasoning (#223) — the dominant cost driver on a
|
||||
/// reasoning model, and independent of every speed metric here.
|
||||
pub reasoning_tokens_median: Option<f64>,
|
||||
/// Prompt tokens served from the prefix cache (#269).
|
||||
pub cached_tokens_median: Option<f64>,
|
||||
/// Total generated tokens — catches a model that starts truncating
|
||||
/// or rambling, which no rate metric shows.
|
||||
pub completion_tokens_median: Option<f64>,
|
||||
/// p95 inter-token gap (ms) — stream smoothness.
|
||||
pub tpot_p95_ms_median: Option<f64>,
|
||||
/// Identity the samples were taken under (#288), or `None` if
|
||||
/// anonymous. Anonymous and identified points are **not
|
||||
/// comparable** once #262 is in the build.
|
||||
pub principal: Option<String>,
|
||||
pub samples: usize,
|
||||
}
|
||||
|
||||
@@ -926,15 +848,6 @@ struct SeriesRaw {
|
||||
ttft_s: Option<f64>,
|
||||
decode_tps: Option<f64>,
|
||||
total_s: Option<f64>,
|
||||
ttft_p95_s: Option<f64>,
|
||||
queue_wait_ms: Option<f64>,
|
||||
rejected: Option<f64>,
|
||||
prefill_tps: Option<f64>,
|
||||
reasoning_tokens: Option<f64>,
|
||||
cached_tokens: Option<f64>,
|
||||
completion_tokens: Option<f64>,
|
||||
tpot_p95_ms: Option<f64>,
|
||||
principal: Option<String>,
|
||||
ts: String,
|
||||
}
|
||||
|
||||
@@ -967,19 +880,6 @@ fn aggregate_series(raws: Vec<SeriesRaw>) -> Vec<SeriesPoint> {
|
||||
ttft_s_median: median(rows.iter().filter_map(|r| r.ttft_s)),
|
||||
decode_tps_median: median(rows.iter().filter_map(|r| r.decode_tps)),
|
||||
total_s_median: median(rows.iter().filter_map(|r| r.total_s)),
|
||||
ttft_p95_s_median: median(rows.iter().filter_map(|r| r.ttft_p95_s)),
|
||||
queue_wait_ms_median: median(rows.iter().filter_map(|r| r.queue_wait_ms)),
|
||||
rejected_median: median(rows.iter().filter_map(|r| r.rejected)),
|
||||
prefill_tps_median: median(rows.iter().filter_map(|r| r.prefill_tps)),
|
||||
reasoning_tokens_median: median(rows.iter().filter_map(|r| r.reasoning_tokens)),
|
||||
cached_tokens_median: median(rows.iter().filter_map(|r| r.cached_tokens)),
|
||||
completion_tokens_median: median(rows.iter().filter_map(|r| r.completion_tokens)),
|
||||
tpot_p95_ms_median: median(rows.iter().filter_map(|r| r.tpot_p95_ms)),
|
||||
// Any row's principal: a build's samples are all taken
|
||||
// under one identity, and a mixture would mean the
|
||||
// config changed mid-build, which the chart should show
|
||||
// as a discontinuity rather than average away.
|
||||
principal: rows.iter().find_map(|r| r.principal.clone()),
|
||||
samples: rows.len(),
|
||||
};
|
||||
(sort_key, point)
|
||||
@@ -1381,10 +1281,6 @@ mod tests {
|
||||
fn rec(target: &str, sha: &str, model: &str, scenario: &str, ok: bool) -> RunRecord {
|
||||
RunRecord {
|
||||
ts: "2026-06-13T00:00:00Z".into(),
|
||||
principal: None,
|
||||
reasoning_tokens: None,
|
||||
cached_tokens: None,
|
||||
tpot_p95_ms: None,
|
||||
target_name: target.into(),
|
||||
target_kind: "neuron".into(),
|
||||
endpoint: "http://x:13131".into(),
|
||||
|
||||
@@ -78,10 +78,7 @@ pub struct Sweeper {
|
||||
|
||||
impl Sweeper {
|
||||
pub fn new(cfg: BenchConfig) -> Result<Self> {
|
||||
let client = TargetClient::with_principal(
|
||||
cfg.bench.request_timeout(),
|
||||
cfg.bench.principal.as_ref(),
|
||||
)?;
|
||||
let client = TargetClient::new(cfg.bench.request_timeout())?;
|
||||
let store = Store::open(&cfg.bench.db_path)?;
|
||||
Ok(Sweeper { cfg, client, store })
|
||||
}
|
||||
@@ -172,7 +169,6 @@ impl Sweeper {
|
||||
model_id: model.id.clone(),
|
||||
max_tokens: self.cfg.scenarios.max_tokens,
|
||||
timeout: self.cfg.bench.request_timeout(),
|
||||
principal_key_id: self.cfg.bench.principal.as_ref().map(|p| p.key_id.clone()),
|
||||
};
|
||||
let cold = crate::scenario::cold_probe(&ctx).await;
|
||||
let swap = SwapTiming { unload_ms, load_ms };
|
||||
@@ -256,7 +252,6 @@ impl Sweeper {
|
||||
model_id: model.id.clone(),
|
||||
max_tokens: self.cfg.scenarios.max_tokens,
|
||||
timeout: self.cfg.bench.request_timeout(),
|
||||
principal_key_id: self.cfg.bench.principal.as_ref().map(|p| p.key_id.clone()),
|
||||
};
|
||||
|
||||
// One unmeasured warmup when the cell is empty (matches
|
||||
@@ -351,15 +346,6 @@ impl Sweeper {
|
||||
|
||||
RunRecord {
|
||||
ts: chrono::Utc::now().to_rfc3339(),
|
||||
// #288: stamp the identity this sample was taken under, so a
|
||||
// later reader can tell an anonymous row from an identified
|
||||
// one instead of inferring it from the date.
|
||||
principal: self
|
||||
.cfg
|
||||
.bench
|
||||
.principal
|
||||
.as_ref()
|
||||
.map(|p| format!("{}/{}", p.account_id, p.key_id)),
|
||||
target_name: target.name.clone(),
|
||||
target_kind: kind_str(target.kind).to_string(),
|
||||
endpoint: target.endpoint.clone(),
|
||||
@@ -397,9 +383,6 @@ impl Sweeper {
|
||||
prefill_ms: m.and_then(|m| m.prefill_ms),
|
||||
decode_ms: m.and_then(|m| m.decode_ms),
|
||||
prefill_tokens: m.and_then(|m| m.prefill_tokens),
|
||||
reasoning_tokens: m.and_then(|x| x.reasoning_tokens),
|
||||
cached_tokens: m.and_then(|x| x.cached_tokens),
|
||||
tpot_p95_ms: m.and_then(|x| x.tpot_p95_ms),
|
||||
vram_used_mb: health.map(|h| h.vram_used_mb),
|
||||
gpu_util_pct: health.map(|h| h.gpu_util_pct),
|
||||
gpu_temp_c: health.map(|h| h.gpu_temp_c),
|
||||
|
||||
@@ -15,10 +15,6 @@ fn rec(
|
||||
ok: bool,
|
||||
) -> RunRecord {
|
||||
RunRecord {
|
||||
principal: None,
|
||||
reasoning_tokens: None,
|
||||
cached_tokens: None,
|
||||
tpot_p95_ms: None,
|
||||
ts: "2026-06-13T00:00:00Z".into(),
|
||||
target_name: host.into(),
|
||||
target_kind: "neuron".into(),
|
||||
|
||||
@@ -80,7 +80,6 @@ async fn spawn_mock(sha: &str) -> (String, Arc<Mutex<String>>) {
|
||||
fn config_for(endpoint: String, db_path: String) -> BenchConfig {
|
||||
BenchConfig {
|
||||
bench: BenchSettings {
|
||||
principal: None,
|
||||
sweep_interval_secs: 1,
|
||||
samples_per_version: 2,
|
||||
iteration_pause_secs: 0,
|
||||
@@ -92,7 +91,6 @@ fn config_for(endpoint: String, db_path: String) -> BenchConfig {
|
||||
prompt_sizes: vec![128], // single scenario keeps assertions simple
|
||||
max_tokens: 16,
|
||||
concurrency_levels: Vec::new(),
|
||||
concurrency_nonstreaming_levels: vec![],
|
||||
concurrency_prompt_tokens: 512,
|
||||
capability_probes: Vec::new(),
|
||||
},
|
||||
|
||||
@@ -80,19 +80,10 @@ fn router_entry(cortex: &str, e: &CortexModelEntry) -> CortexModelEntry {
|
||||
cost: e.cost.clone(),
|
||||
tool_call: e.tool_call,
|
||||
reasoning: e.reasoning,
|
||||
// Every flat ecosystem field is derived from `limit` by the
|
||||
// final sync pass in aggregate_models, so they are cleared here
|
||||
// rather than copied. Spread from `e` keeps a field added to the
|
||||
// entry later from being silently dropped at this hop.
|
||||
..CortexModelEntry {
|
||||
max_model_len: None,
|
||||
max_input_tokens: None,
|
||||
max_output_tokens: None,
|
||||
context_window: None,
|
||||
context_length: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
..e.clone()
|
||||
}
|
||||
// Derived from `limit` by the final sync pass in aggregate_models.
|
||||
max_model_len: None,
|
||||
max_input_tokens: None,
|
||||
max_output_tokens: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -173,9 +164,6 @@ mod tests {
|
||||
max_model_len: None,
|
||||
max_input_tokens: None,
|
||||
max_output_tokens: None,
|
||||
context_window: None,
|
||||
context_length: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -236,7 +224,6 @@ mod tests {
|
||||
context: 32_768,
|
||||
input: None,
|
||||
output: 4096,
|
||||
output_ceiling: 0,
|
||||
});
|
||||
a.cost = Some(ModelCost {
|
||||
input: 0.50,
|
||||
@@ -249,7 +236,6 @@ mod tests {
|
||||
context: 16_384, // tighter
|
||||
input: None,
|
||||
output: 4096,
|
||||
output_ceiling: 0,
|
||||
});
|
||||
b.cost = Some(ModelCost {
|
||||
input: 0.20, // cheaper
|
||||
|
||||
@@ -42,9 +42,6 @@ fn model_entry(loaded: bool, feasible: bool) -> CortexModelEntry {
|
||||
max_model_len: None,
|
||||
max_input_tokens: None,
|
||||
max_output_tokens: None,
|
||||
context_window: None,
|
||||
context_length: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -58,11 +58,6 @@ mod tests {
|
||||
assert!(cfg.max_redirects > 0);
|
||||
}
|
||||
|
||||
// The closure's error type is `figment::Error`, which is 208 bytes and
|
||||
// not ours to shrink; `Jail::expect_with` dictates the signature. The
|
||||
// usual remedy (box the Err) is not available through a third-party
|
||||
// callback contract, and this is a test closure run once.
|
||||
#[allow(clippy::result_large_err)]
|
||||
#[test]
|
||||
fn loads_toml_with_env_override() {
|
||||
figment::Jail::expect_with(|jail| {
|
||||
|
||||
@@ -58,20 +58,6 @@ pub struct AuthSettings {
|
||||
/// Public base URL of the frontend, used to build verify/reset links.
|
||||
#[serde(default = "default_app_base_url")]
|
||||
pub app_base_url: String,
|
||||
/// Other front ends allowed to originate a signup, so their
|
||||
/// verification mail points back at the property the person actually
|
||||
/// used (e.g. the investor portal). Sending someone who registered on
|
||||
/// one site to a different one is a poor first impression, and for the
|
||||
/// portal it strands them in the chat app instead of the material they
|
||||
/// were invited to read.
|
||||
///
|
||||
/// An **allowlist**, not a free parameter: this value ends up as a URL
|
||||
/// in an email we send. Accepting a caller-supplied origin unchecked
|
||||
/// would make the signup endpoint a phishing-link generator with our
|
||||
/// domain and DKIM signature on it. Anything not listed here falls
|
||||
/// back to `app_base_url`.
|
||||
#[serde(default)]
|
||||
pub additional_app_origins: Vec<String>,
|
||||
}
|
||||
|
||||
impl Default for AuthSettings {
|
||||
@@ -83,7 +69,6 @@ impl Default for AuthSettings {
|
||||
unverified_grace_secs: default_unverified_grace(),
|
||||
unverified_sweep_interval_secs: default_unverified_sweep_interval(),
|
||||
app_base_url: default_app_base_url(),
|
||||
additional_app_origins: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,15 +27,7 @@ use sqlx::Row;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub fn router(state: &AppState) -> Router<AppState> {
|
||||
// Session-only: identity management and the full account view.
|
||||
//
|
||||
// Key management stays here deliberately (#264). If an inference key
|
||||
// could mint keys, one leaked credential would become a foothold that
|
||||
// SURVIVES revoking the original — a spending problem turning into an
|
||||
// eviction problem. `/account` stays too: it carries `angel_access`,
|
||||
// and whatever field is added to it next would otherwise be exposed to
|
||||
// inference credentials by default.
|
||||
let session_only = Router::new()
|
||||
let protected = Router::new()
|
||||
.route("/web/v1/account", get(account))
|
||||
.route("/web/v1/keys", get(list_keys).post(create_key))
|
||||
.route("/web/v1/keys/{id}/archive", post(archive_key))
|
||||
@@ -43,23 +35,11 @@ pub fn router(state: &AppState) -> Router<AppState> {
|
||||
"/web/v1/keys/{id}/limit",
|
||||
axum::routing::patch(update_key_limit),
|
||||
)
|
||||
.layer(axum::middleware::from_fn_with_state(
|
||||
state.clone(),
|
||||
require_session,
|
||||
));
|
||||
|
||||
// Session **or** API key: reading and refilling the allocation.
|
||||
//
|
||||
// A service that watches its own balance should not have to hold its
|
||||
// owner's password — the credential that can mint and revoke keys — to
|
||||
// read a number it is already entitled to spend against.
|
||||
let account_auth = Router::new()
|
||||
.route("/web/v1/allocation", get(allocation))
|
||||
.route("/web/v1/redeem", post(redeem))
|
||||
.route("/web/v1/topup/request", post(request_topup))
|
||||
.layer(axum::middleware::from_fn_with_state(
|
||||
state.clone(),
|
||||
require_account,
|
||||
require_session,
|
||||
));
|
||||
|
||||
Router::new()
|
||||
@@ -69,8 +49,7 @@ pub fn router(state: &AppState) -> Router<AppState> {
|
||||
.route("/web/v1/login", post(login))
|
||||
.route("/web/v1/password-reset/request", post(reset_request))
|
||||
.route("/web/v1/password-reset/confirm", post(reset_confirm))
|
||||
.merge(session_only)
|
||||
.merge(account_auth)
|
||||
.merge(protected)
|
||||
}
|
||||
|
||||
// ── errors ──────────────────────────────────────────────────────────
|
||||
@@ -163,71 +142,6 @@ async fn require_session(State(state): State<AppState>, mut req: Request, next:
|
||||
}
|
||||
}
|
||||
|
||||
/// Authenticated **account**, injected by [`require_account`]. Unlike
|
||||
/// [`AuthUser`] this says nothing about *who* is calling — only which
|
||||
/// account the call bills to — because an API key identifies an account
|
||||
/// without identifying a person.
|
||||
#[derive(Clone, Copy)]
|
||||
struct AuthAccount(Uuid);
|
||||
|
||||
/// Accept **either** a web session or an inference API key, and resolve
|
||||
/// both to an account id (#264).
|
||||
///
|
||||
/// Routes behind this may read and refill an allocation. They may not
|
||||
/// touch keys: a service watching its own balance should not have to
|
||||
/// hold the credential that can mint and revoke credentials. See the
|
||||
/// router for why key management stays session-only.
|
||||
///
|
||||
/// The two token shapes are unambiguous — API keys carry the
|
||||
/// `sk-helexa-` prefix (`crypto::generate_api_key`) and JWTs cannot — so
|
||||
/// this dispatches on the prefix rather than trying to decode a session
|
||||
/// and falling back, which would log a decode failure for every
|
||||
/// perfectly valid key.
|
||||
async fn require_account(State(state): State<AppState>, mut req: Request, next: Next) -> Response {
|
||||
let Some(token) = req
|
||||
.headers()
|
||||
.get(header::AUTHORIZATION)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.strip_prefix("Bearer "))
|
||||
.map(str::trim)
|
||||
.map(str::to_string)
|
||||
else {
|
||||
return WebError::Unauthorized.into_response();
|
||||
};
|
||||
|
||||
let account = if token.starts_with("sk-helexa-") {
|
||||
// `resolve_key` already requires both key and account to be
|
||||
// `active`, so an archived key or a deactivated account
|
||||
// authenticates nothing — no extra check needed here.
|
||||
match crate::ledger::resolve_key(&state.pool, &sha256(&token)).await {
|
||||
Ok(Some(principal)) => principal.account_id,
|
||||
Ok(None) => return WebError::Unauthorized.into_response(),
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e, "key resolve failed on account route");
|
||||
return WebError::Internal.into_response();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let uid = decode::<Claims>(
|
||||
&token,
|
||||
&DecodingKey::from_secret(state.config.auth.jwt_secret.as_bytes()),
|
||||
&Validation::default(),
|
||||
)
|
||||
.ok()
|
||||
.and_then(|d| Uuid::parse_str(&d.claims.sub).ok());
|
||||
let Some(uid) = uid else {
|
||||
return WebError::Unauthorized.into_response();
|
||||
};
|
||||
match account_id_for(&state, uid).await {
|
||||
Ok(a) => a,
|
||||
Err(e) => return e.into_response(),
|
||||
}
|
||||
};
|
||||
|
||||
req.extensions_mut().insert(AuthAccount(account));
|
||||
next.run(req).await
|
||||
}
|
||||
|
||||
/// The caller's single account id.
|
||||
async fn account_id_for(state: &AppState, user_id: Uuid) -> WebResult<Uuid> {
|
||||
let row = sqlx::query("SELECT id FROM accounts WHERE owner_user_id = $1")
|
||||
@@ -260,40 +174,6 @@ struct RegisterReq {
|
||||
password: String,
|
||||
#[serde(default)]
|
||||
fingerprint: Option<String>,
|
||||
/// Front end the signup came from, so the verification mail sends the
|
||||
/// person back where they started. Allowlisted — see
|
||||
/// [`resolve_origin`].
|
||||
#[serde(default)]
|
||||
origin: Option<String>,
|
||||
}
|
||||
|
||||
/// Resolve a caller-supplied origin against the configured allowlist.
|
||||
///
|
||||
/// The result is embedded in an email we send and sign, so an unchecked
|
||||
/// value would turn signup into a phishing-link generator carrying our
|
||||
/// domain's reputation. Unknown or absent → the default front end.
|
||||
fn resolve_origin(state: &AppState, requested: Option<&str>) -> String {
|
||||
resolve_origin_with(&state.config.auth, requested)
|
||||
}
|
||||
|
||||
/// Split out from [`resolve_origin`] so the allowlist can be tested without
|
||||
/// standing up an `AppState` — this is the check that keeps /register from
|
||||
/// becoming a phishing-link generator, so it needs to be exercised.
|
||||
fn resolve_origin_with(auth: &crate::config::AuthSettings, requested: Option<&str>) -> String {
|
||||
let default = auth.app_base_url.clone();
|
||||
let Some(req) = requested else { return default };
|
||||
let req = req.trim_end_matches('/');
|
||||
if req == default.trim_end_matches('/')
|
||||
|| auth
|
||||
.additional_app_origins
|
||||
.iter()
|
||||
.any(|o| o.trim_end_matches('/') == req)
|
||||
{
|
||||
req.to_string()
|
||||
} else {
|
||||
tracing::warn!(origin = %req, "signup origin not allowlisted — using the default");
|
||||
default
|
||||
}
|
||||
}
|
||||
|
||||
/// `POST /web/v1/register` — always returns `202`, regardless of whether the
|
||||
@@ -368,25 +248,13 @@ async fn register_inner(state: &AppState, req: RegisterReq) -> WebResult<()> {
|
||||
.execute(&state.pool)
|
||||
.await?;
|
||||
|
||||
let origin = resolve_origin(state, req.origin.as_deref());
|
||||
let link = format!("{origin}/verify?token={token}");
|
||||
let link = format!("{}/verify?token={token}", state.config.auth.app_base_url);
|
||||
let _ = state
|
||||
.email
|
||||
.send(
|
||||
&req.email,
|
||||
"Confirm your helexa email address",
|
||||
&format!(
|
||||
"Welcome to helexa.\n\n\
|
||||
Confirm this address to finish setting up your account:\n\n\
|
||||
{link}\n\n\
|
||||
The link is valid for 24 hours. Once confirmed, sign in at \
|
||||
{app} and you're ready to go.\n\n\
|
||||
If you didn't create a helexa account, you can ignore this \
|
||||
message — nothing has been set up and the address will be \
|
||||
released.\n\n\
|
||||
helexa\n",
|
||||
app = origin,
|
||||
),
|
||||
"Verify your helexa account",
|
||||
&format!("Welcome to helexa. Verify your email:\n\n{link}\n"),
|
||||
)
|
||||
.await;
|
||||
Ok(())
|
||||
@@ -488,19 +356,8 @@ async fn resend_verification_if_unverified(
|
||||
.email
|
||||
.send(
|
||||
email,
|
||||
"Confirm your helexa email address",
|
||||
&format!(
|
||||
"Here is a fresh confirmation link.\n\n\
|
||||
Confirm this address to finish setting up your account:\n\n\
|
||||
{link}\n\n\
|
||||
The link is valid for 24 hours. Once confirmed, sign in at \
|
||||
{app} and you're ready to go.\n\n\
|
||||
If you didn't create a helexa account, you can ignore this \
|
||||
message — nothing has been set up and the address will be \
|
||||
released.\n\n\
|
||||
helexa\n",
|
||||
app = state.config.auth.app_base_url,
|
||||
),
|
||||
"Verify your helexa account",
|
||||
&format!("Welcome to helexa. Verify your email:\n\n{link}\n"),
|
||||
)
|
||||
.await;
|
||||
Ok(())
|
||||
@@ -633,16 +490,7 @@ async fn reset_request_inner(state: &AppState, email: &str) -> WebResult<()> {
|
||||
.send(
|
||||
email,
|
||||
"Reset your helexa password",
|
||||
&format!(
|
||||
"Someone asked to reset the password for this helexa \
|
||||
account.\n\n\
|
||||
{link}\n\n\
|
||||
The link is valid for 24 hours and can be used once.\n\n\
|
||||
If that wasn't you, ignore this message — your password has \
|
||||
not changed and nobody can use this link without access to \
|
||||
your inbox.\n\n\
|
||||
helexa\n"
|
||||
),
|
||||
&format!("Reset your password:\n\n{link}\n"),
|
||||
)
|
||||
.await;
|
||||
Ok(())
|
||||
@@ -721,46 +569,6 @@ async fn account(
|
||||
.into_response())
|
||||
}
|
||||
|
||||
/// `GET /web/v1/allocation` — the account's token balance, readable with
|
||||
/// **either** a session or an API key (#264).
|
||||
///
|
||||
/// Deliberately narrower than [`account`]: balance and top-up
|
||||
/// availability, nothing else. `/account` also returns `angel_access`,
|
||||
/// and whatever field is added to it next would otherwise be handed to
|
||||
/// every inference key by default. A separate projection means widening
|
||||
/// the account view can never widen what a key can see.
|
||||
async fn allocation(
|
||||
State(state): State<AppState>,
|
||||
Extension(AuthAccount(acct)): Extension<AuthAccount>,
|
||||
) -> WebResult<Response> {
|
||||
let row = sqlx::query(
|
||||
"SELECT allocation_total, allocation_spent, allocation_reserved FROM accounts WHERE id = $1",
|
||||
)
|
||||
.bind(acct)
|
||||
.fetch_one(&state.pool)
|
||||
.await?;
|
||||
// Same display-only eligibility the dashboard uses, so an automated
|
||||
// caller can check before asking and avoid provoking a 409. The grant
|
||||
// path re-checks, so a stale read cannot conjure a top-up.
|
||||
let (topup_available, topup_reason) =
|
||||
match crate::topup::auto_eligibility(&state.pool, acct).await {
|
||||
Ok(Ok(())) => (true, None),
|
||||
Ok(Err(reason)) => (false, Some(reason.to_string())),
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "top-up eligibility check failed");
|
||||
(false, None)
|
||||
}
|
||||
};
|
||||
Ok(Json(json!({
|
||||
"allocation_total": row.get::<i64, _>("allocation_total"),
|
||||
"allocation_spent": row.get::<i64, _>("allocation_spent"),
|
||||
"allocation_reserved": row.get::<i64, _>("allocation_reserved"),
|
||||
"topup_available": topup_available,
|
||||
"topup_reason": topup_reason,
|
||||
}))
|
||||
.into_response())
|
||||
}
|
||||
|
||||
/// Whether this user holds any investor-portal grant.
|
||||
///
|
||||
/// A **boolean, and deliberately nothing more**. helexa.ai is a static
|
||||
@@ -806,9 +614,10 @@ async fn has_angel_access(state: &AppState, user_id: Uuid) -> bool {
|
||||
/// caller is authenticated and asking about their own account.
|
||||
async fn request_topup(
|
||||
State(state): State<AppState>,
|
||||
Extension(AuthAccount(acct)): Extension<AuthAccount>,
|
||||
Extension(user): Extension<AuthUser>,
|
||||
) -> WebResult<Response> {
|
||||
use crate::topup::AutoTopUpError;
|
||||
let acct = account_id_for(&state, user.0).await?;
|
||||
match crate::topup::auto_grant(&state.pool, acct).await {
|
||||
Ok(grant) => {
|
||||
tracing::info!(
|
||||
@@ -984,9 +793,10 @@ struct RedeemReq {
|
||||
/// or already-redeemed code (no oracle).
|
||||
async fn redeem(
|
||||
State(state): State<AppState>,
|
||||
Extension(AuthAccount(acct)): Extension<AuthAccount>,
|
||||
Extension(user): Extension<AuthUser>,
|
||||
Json(req): Json<RedeemReq>,
|
||||
) -> WebResult<Response> {
|
||||
let acct = account_id_for(&state, user.0).await?;
|
||||
match crate::topup::redeem(&state.pool, acct, &req.code).await {
|
||||
Ok(new_total) => Ok(Json(json!({ "allocation_total": new_total })).into_response()),
|
||||
Err(crate::topup::TopUpError::Invalid) => {
|
||||
@@ -998,47 +808,3 @@ async fn redeem(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::AuthSettings;
|
||||
|
||||
#[test]
|
||||
fn signup_origin_must_be_allowlisted() {
|
||||
// The resolved origin is embedded in an email we send and DKIM-sign.
|
||||
// If a caller could supply any value, /register would become a
|
||||
// phishing-link generator carrying our domain's reputation.
|
||||
let cfg = AuthSettings {
|
||||
app_base_url: "https://helexa.ai".into(),
|
||||
additional_app_origins: vec!["https://angels.helexa.ai".into()],
|
||||
..Default::default()
|
||||
};
|
||||
let origin = |req: Option<&str>| resolve_origin_with(&cfg, req);
|
||||
|
||||
assert_eq!(origin(None), "https://helexa.ai");
|
||||
assert_eq!(
|
||||
origin(Some("https://angels.helexa.ai")),
|
||||
"https://angels.helexa.ai"
|
||||
);
|
||||
// A trailing slash is the same origin, not a bypass.
|
||||
assert_eq!(
|
||||
origin(Some("https://angels.helexa.ai/")),
|
||||
"https://angels.helexa.ai"
|
||||
);
|
||||
// Everything else falls back rather than being echoed into a mail.
|
||||
for hostile in [
|
||||
"https://evil.example",
|
||||
"https://angels.helexa.ai.evil.example",
|
||||
"https://helexa.ai.evil.example",
|
||||
"javascript:alert(1)",
|
||||
"//evil.example",
|
||||
] {
|
||||
assert_eq!(
|
||||
origin(Some(hostile)),
|
||||
"https://helexa.ai",
|
||||
"leaked: {hostile}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -424,157 +424,3 @@ async fn topup_concurrent_double_redeem_one_winner() {
|
||||
.count();
|
||||
assert_eq!(wins, 1, "exactly one redemption wins the single-use code");
|
||||
}
|
||||
|
||||
/// An API key may read its own allocation and refill it, but must never
|
||||
/// reach key management (#264).
|
||||
///
|
||||
/// The second half is the assertion that matters. If an inference key
|
||||
/// could mint keys, one leaked credential would become a foothold that
|
||||
/// survives revoking the original — a spending problem turning into an
|
||||
/// eviction problem. The first half is the point of the change: a service
|
||||
/// watching its own balance should not need the account password.
|
||||
#[tokio::test]
|
||||
async fn api_key_reads_allocation_but_cannot_touch_keys() {
|
||||
let Some((base, pool)) = spawn_or_skip("api_key_reads_allocation_but_cannot_touch_keys").await
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let client = reqwest::Client::new();
|
||||
let email = unique_email();
|
||||
post(
|
||||
format!("{base}/web/v1/register"),
|
||||
json!({"email": email, "password": "password123"}),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
pool.execute(
|
||||
sqlx::query("UPDATE users SET email_verified = true WHERE email = $1").bind(&email),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let session = post(
|
||||
format!("{base}/web/v1/login"),
|
||||
json!({"email": email, "password": "password123"}),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.json::<Value>()
|
||||
.await
|
||||
.unwrap()["token"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.to_string();
|
||||
|
||||
let created = post(
|
||||
format!("{base}/web/v1/keys"),
|
||||
json!({"label": "service"}),
|
||||
Some(&session),
|
||||
)
|
||||
.await
|
||||
.json::<Value>()
|
||||
.await
|
||||
.unwrap();
|
||||
let raw_key = created["key"].as_str().unwrap().to_string();
|
||||
let key_id = created["id"].as_str().unwrap().to_string();
|
||||
|
||||
// ── the key CAN read its own allocation ──────────────────────────
|
||||
let r = client
|
||||
.get(format!("{base}/web/v1/allocation"))
|
||||
.bearer_auth(&raw_key)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
r.status(),
|
||||
200,
|
||||
"an API key must be able to read allocation"
|
||||
);
|
||||
let body: Value = r.json().await.unwrap();
|
||||
assert!(body["allocation_total"].is_i64());
|
||||
assert!(body["allocation_spent"].is_i64());
|
||||
assert!(body["allocation_reserved"].is_i64());
|
||||
assert!(body["topup_available"].is_boolean());
|
||||
// Narrower than /account on purpose: nothing about the human.
|
||||
assert!(
|
||||
body.get("angel_access").is_none(),
|
||||
"allocation must not leak account-holder fields to an inference key: {body}"
|
||||
);
|
||||
|
||||
// A session reaches the same route.
|
||||
let r = client
|
||||
.get(format!("{base}/web/v1/allocation"))
|
||||
.bearer_auth(&session)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(r.status(), 200, "a session must still work here");
|
||||
|
||||
// ── the key CANNOT manage keys ───────────────────────────────────
|
||||
let r = post(
|
||||
format!("{base}/web/v1/keys"),
|
||||
json!({"label": "minted-by-a-key"}),
|
||||
Some(&raw_key),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
r.status(),
|
||||
401,
|
||||
"an API key must not be able to mint another key"
|
||||
);
|
||||
|
||||
let r = client
|
||||
.get(format!("{base}/web/v1/keys"))
|
||||
.bearer_auth(&raw_key)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(r.status(), 401, "an API key must not be able to list keys");
|
||||
|
||||
let r = post(
|
||||
format!("{base}/web/v1/keys/{key_id}/archive"),
|
||||
json!({}),
|
||||
Some(&raw_key),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
r.status(),
|
||||
401,
|
||||
"an API key must not be able to archive a key"
|
||||
);
|
||||
|
||||
// /account stays session-only — it carries account-holder fields.
|
||||
let r = client
|
||||
.get(format!("{base}/web/v1/account"))
|
||||
.bearer_auth(&raw_key)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(r.status(), 401, "/account must remain session-only");
|
||||
|
||||
// ── an archived key authenticates nothing ────────────────────────
|
||||
let r = post(
|
||||
format!("{base}/web/v1/keys/{key_id}/archive"),
|
||||
json!({}),
|
||||
Some(&session),
|
||||
)
|
||||
.await;
|
||||
assert!(r.status().is_success(), "archive via session should work");
|
||||
let r = client
|
||||
.get(format!("{base}/web/v1/allocation"))
|
||||
.bearer_auth(&raw_key)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(r.status(), 401, "an archived key must authenticate nothing");
|
||||
|
||||
// Garbage bearers are refused on both shapes.
|
||||
for bogus in ["sk-helexa-nonsense", "not-a-jwt"] {
|
||||
let r = client
|
||||
.get(format!("{base}/web/v1/allocation"))
|
||||
.bearer_auth(bogus)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(r.status(), 401, "bogus bearer {bogus} must be refused");
|
||||
}
|
||||
}
|
||||
|
||||
+17
-249
@@ -21,7 +21,7 @@ use futures::stream::{self, StreamExt};
|
||||
use serde_json::{Value, json};
|
||||
use std::convert::Infallible;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use tokio::sync::RwLock;
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
|
||||
@@ -49,7 +49,6 @@ pub fn neuron_routes() -> Router<Arc<NeuronState>> {
|
||||
.route("/models/load", post(load_model))
|
||||
.route("/models/unload", post(unload_model))
|
||||
.route("/models/{model_id}/endpoint", get(model_endpoint))
|
||||
.route("/models/{model_id}/capabilities", get(model_capabilities))
|
||||
.route("/v1/chat/completions", post(chat_completions))
|
||||
.route("/v1/images/generations", post(images_generations))
|
||||
.route("/v1/responses", post(responses))
|
||||
@@ -97,7 +96,7 @@ async fn list_models(State(state): State<Arc<NeuronState>>) -> impl IntoResponse
|
||||
|
||||
async fn load_model(
|
||||
State(state): State<Arc<NeuronState>>,
|
||||
ApiJson(spec): ApiJson<ModelSpec>,
|
||||
Json(spec): Json<ModelSpec>,
|
||||
) -> impl IntoResponse {
|
||||
// Driver/library mismatch preflight (#19): every CUDA load is
|
||||
// guaranteed to fail until the host reboots. Reject up front with
|
||||
@@ -168,7 +167,7 @@ fn preflight_kind(err: &PreflightError) -> &'static str {
|
||||
|
||||
async fn unload_model(
|
||||
State(state): State<Arc<NeuronState>>,
|
||||
ApiJson(body): ApiJson<Value>,
|
||||
Json(body): Json<Value>,
|
||||
) -> impl IntoResponse {
|
||||
let model_id = match body.get("model_id").and_then(|v| v.as_str()) {
|
||||
Some(id) => id.to_string(),
|
||||
@@ -207,36 +206,6 @@ async fn model_endpoint(
|
||||
}
|
||||
}
|
||||
|
||||
/// `GET /models/{id}/capabilities` — what modalities this node believes
|
||||
/// `id` serves, loaded or not (#241).
|
||||
///
|
||||
/// Answers from the loaded handle when the model is resident, and from
|
||||
/// the cached repo layout otherwise, so a cold model still advertises
|
||||
/// itself. 404 means "this node cannot say" — the repo was never cached
|
||||
/// here, or the id doesn't resolve to a source. That is deliberately not
|
||||
/// an empty list: a caller must be able to tell "no idea" from "serves
|
||||
/// nothing", or it will cache a wrong answer forever.
|
||||
async fn model_capabilities(
|
||||
State(state): State<Arc<NeuronState>>,
|
||||
Path(model_id): Path<String>,
|
||||
) -> impl IntoResponse {
|
||||
let registry = state.registry.read().await;
|
||||
let discovered = match registry.candle() {
|
||||
Some(candle) => candle.discovered_capabilities(&model_id).await,
|
||||
None => None,
|
||||
};
|
||||
match discovered {
|
||||
Some(capabilities) => Json(json!({ "capabilities": capabilities })).into_response(),
|
||||
None => (
|
||||
StatusCode::NOT_FOUND,
|
||||
Json(json!({
|
||||
"error": format!("no local evidence for model '{}'", model_id)
|
||||
})),
|
||||
)
|
||||
.into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Default `chat_template_kwargs.enable_thinking` to `include_thinking`
|
||||
/// when the client didn't set it explicitly, leaving any explicit client
|
||||
/// choice untouched. See the call site in [`chat_completions`] for the
|
||||
@@ -272,35 +241,6 @@ fn default_enable_thinking(req: &mut ChatCompletionRequest, include_thinking: bo
|
||||
/// client-supplied copy and asserts the authoritative value, so over the
|
||||
/// trusted WireGuard link these are safe to key fair-share on. `None` for an
|
||||
/// unauthenticated/direct request — exempt from the per-principal cap.
|
||||
/// SSE keep-alive for the inference streams.
|
||||
///
|
||||
/// A streamed completion emits its `role` delta immediately and then says
|
||||
/// **nothing at all until prefill finishes** — there is no token to send
|
||||
/// yet. On a long prompt that silence is seconds long: a 7.4k-token prompt
|
||||
/// on a TP-2 27B prefills in about six.
|
||||
///
|
||||
/// `KeepAlive::default()` is axum's 15s, which is longer than almost every
|
||||
/// prefill we serve, so in practice it never fired and the gap went out on
|
||||
/// the wire as dead air. Clients read dead air as a dropped stream: the
|
||||
/// DeepSeek Harness gives up after roughly a second and reports
|
||||
/// `terminated`, then retries — and because the abandoned request keeps its
|
||||
/// admission permit and the TP pool lock until it finishes, each retry
|
||||
/// queues behind the last one and waits *longer*, so the session never
|
||||
/// recovers on its own. Every attempt failed while the model was healthy
|
||||
/// and, by then, three sequences deep in work nobody was listening to.
|
||||
///
|
||||
/// One second is chosen to sit under the shortest client idle timeout we
|
||||
/// have met rather than to be tuned to any particular one. The cost is a
|
||||
/// 3-byte comment frame per idle second, which SSE parsers discard by
|
||||
/// spec; the benefit is that a prefill of any length looks alive.
|
||||
///
|
||||
/// This does not make a slow prefill fast — see the `chunked prefill`
|
||||
/// path for that — it stops a slow prefill being indistinguishable from a
|
||||
/// dead connection.
|
||||
fn prefill_keep_alive() -> KeepAlive {
|
||||
KeepAlive::new().interval(Duration::from_secs(1))
|
||||
}
|
||||
|
||||
fn principal_key(headers: &axum::http::HeaderMap) -> Option<String> {
|
||||
let account = headers.get(HEADER_ACCOUNT_ID)?.to_str().ok()?;
|
||||
let key = headers.get(HEADER_KEY_ID)?.to_str().ok()?;
|
||||
@@ -313,7 +253,7 @@ fn principal_key(headers: &axum::http::HeaderMap) -> Option<String> {
|
||||
async fn chat_completions(
|
||||
State(state): State<Arc<NeuronState>>,
|
||||
headers: axum::http::HeaderMap,
|
||||
ApiJson(mut req): ApiJson<ChatCompletionRequest>,
|
||||
Json(mut req): Json<ChatCompletionRequest>,
|
||||
) -> impl IntoResponse {
|
||||
let Some(candle) = state.candle.as_ref().map(Arc::clone) else {
|
||||
return (
|
||||
@@ -323,13 +263,6 @@ async fn chat_completions(
|
||||
.into_response();
|
||||
};
|
||||
|
||||
// Collapse the two spellings of the output cap onto the one field
|
||||
// the generation loops read. Clients that send only
|
||||
// `max_completion_tokens` (OpenAI's current field — the DeepSeek
|
||||
// Harness among them) would otherwise have their cap silently
|
||||
// ignored and generate to the server default.
|
||||
req.max_tokens = req.effective_max_tokens();
|
||||
|
||||
// Reasoning-content opt-in. Off by default → naïve clients
|
||||
// (Zed's commit-message generator, vanilla OpenAI clients)
|
||||
// never see `<think>` blocks. On when the caller sends
|
||||
@@ -375,7 +308,7 @@ async fn chat_completions(
|
||||
let done_stream =
|
||||
stream::once(async { Ok::<_, Infallible>(Event::default().data("[DONE]")) });
|
||||
Sse::new(body_stream.chain(done_stream))
|
||||
.keep_alive(prefill_keep_alive())
|
||||
.keep_alive(KeepAlive::default())
|
||||
.into_response()
|
||||
}
|
||||
Err(e) => inference_error_response(e),
|
||||
@@ -395,7 +328,7 @@ async fn chat_completions(
|
||||
async fn responses(
|
||||
State(state): State<Arc<NeuronState>>,
|
||||
headers: axum::http::HeaderMap,
|
||||
ApiJson(req): ApiJson<ResponsesRequest>,
|
||||
Json(req): Json<ResponsesRequest>,
|
||||
) -> impl IntoResponse {
|
||||
let Some(candle) = state.candle.as_ref().map(Arc::clone) else {
|
||||
return (
|
||||
@@ -448,7 +381,7 @@ async fn responses(
|
||||
Ok::<_, Infallible>(Event::default().event(frame.event_name).data(body))
|
||||
});
|
||||
Sse::new(body_stream)
|
||||
.keep_alive(prefill_keep_alive())
|
||||
.keep_alive(KeepAlive::default())
|
||||
.into_response()
|
||||
}
|
||||
Err(e) => inference_error_response(e),
|
||||
@@ -467,11 +400,11 @@ async fn responses(
|
||||
.first()
|
||||
.map(|c| match &c.message.content {
|
||||
MessageContent::Text(t) => t.clone(),
|
||||
MessageContent::Parts(_) | MessageContent::Null => {
|
||||
MessageContent::Parts(_) => {
|
||||
// Candle output is always text today;
|
||||
// a Parts or absent response would be
|
||||
// surprising. Empty-string fallback is
|
||||
// safer than a panic.
|
||||
// a Parts response would be surprising.
|
||||
// Empty-string fallback is safer than
|
||||
// a panic.
|
||||
String::new()
|
||||
}
|
||||
})
|
||||
@@ -486,9 +419,9 @@ async fn responses(
|
||||
input_tokens: u.prompt_tokens,
|
||||
output_tokens: u.completion_tokens,
|
||||
total_tokens: u.prompt_tokens + u.completion_tokens,
|
||||
// Carry the reasoning sub-count through from the
|
||||
// chat usage: the collector counts the `<think>`
|
||||
// span and returns its text separately.
|
||||
// Carry the reasoning sub-count through from the chat
|
||||
// usage — the non-streaming path now splits off the
|
||||
// `<think>` span and counts it (see `split_off_reasoning`).
|
||||
output_tokens_details: u.completion_tokens_details.as_ref().map(|d| {
|
||||
OutputTokensDetails {
|
||||
reasoning_tokens: d.reasoning_tokens,
|
||||
@@ -502,18 +435,8 @@ async fn responses(
|
||||
model_id,
|
||||
message_item_id: mint_message_item_id(),
|
||||
};
|
||||
// The model's think block, which the chat collector now
|
||||
// returns on `message.reasoning_content` (#300). Without
|
||||
// this the caller is billed for `reasoning_tokens` it
|
||||
// never sees, and cannot replay the model's own
|
||||
// reasoning on the next turn.
|
||||
let reasoning = chat_resp
|
||||
.choices
|
||||
.first()
|
||||
.and_then(|c| c.message.extra.get("reasoning_content"))
|
||||
.and_then(|v| v.as_str())
|
||||
.map(str::to_string);
|
||||
let resp = openai_responses::build_response(&meta, text, finish, usage, reasoning);
|
||||
let _ = chat_resp; // make the borrow-checker happy if `text` consumed it
|
||||
let resp = openai_responses::build_response(&meta, text, finish, usage);
|
||||
Json(resp).into_response()
|
||||
}
|
||||
Err(e) => inference_error_response(e),
|
||||
@@ -529,7 +452,7 @@ async fn responses(
|
||||
async fn images_generations(
|
||||
State(state): State<Arc<NeuronState>>,
|
||||
headers: axum::http::HeaderMap,
|
||||
ApiJson(req): ApiJson<cortex_core::images::ImagesGenerationRequest>,
|
||||
Json(req): Json<cortex_core::images::ImagesGenerationRequest>,
|
||||
) -> impl IntoResponse {
|
||||
use cortex_core::error_envelope::OpenAiError;
|
||||
use cortex_core::images as wire;
|
||||
@@ -690,24 +613,6 @@ fn inference_error_response(err: InferenceError) -> axum::response::Response {
|
||||
.with_retry_after(5)
|
||||
.with_extra("free_mb", json!(free_mb))
|
||||
.with_extra("required_mb", json!(required_mb)),
|
||||
// Not backpressure: an idle node would refuse this prompt too, so
|
||||
// there is nothing to retry (#257). 413 rather than 400 — the
|
||||
// request is well-formed, it is the payload that is too large.
|
||||
InferenceError::PromptTooLongForVram {
|
||||
required_mb,
|
||||
budget_mb,
|
||||
} => OpenAiError::new(
|
||||
413,
|
||||
"invalid_request_error",
|
||||
"prompt_too_long_for_vram",
|
||||
format!(
|
||||
"prompt needs {required_mb} MiB of KV cache but this model's \
|
||||
entire budget is {budget_mb} MiB. Shorten the conversation \
|
||||
or use a model with more headroom."
|
||||
),
|
||||
)
|
||||
.with_extra("required_mb", json!(required_mb))
|
||||
.with_extra("budget_mb", json!(budget_mb)),
|
||||
InferenceError::VisionUnsupported { model_id } => OpenAiError::new(
|
||||
400,
|
||||
"invalid_request_error",
|
||||
@@ -768,37 +673,8 @@ fn inference_error_response(err: InferenceError) -> axum::response::Response {
|
||||
"too many concurrent requests for this key; retry shortly",
|
||||
)
|
||||
.with_retry_after(retry_after_secs),
|
||||
// Draining (#256). 503 + Retry-After, like any transient capacity
|
||||
// condition: the node is coming back, and a well-behaved client
|
||||
// should retry rather than surface a failure. Deliberately not
|
||||
// `rate_limit_exceeded` — nothing about this is the caller's
|
||||
// fault or their rate.
|
||||
InferenceError::ShuttingDown => OpenAiError::new(
|
||||
503,
|
||||
"api_error",
|
||||
"shutting_down",
|
||||
"neuron is shutting down; retry shortly",
|
||||
)
|
||||
.with_retry_after(15),
|
||||
InferenceError::Other(e) => OpenAiError::without_code(500, "api_error", format!("{e:#}")),
|
||||
};
|
||||
// Every failed request says why, once, here (#63). Before this the
|
||||
// envelope went to the client and nothing reached the journal: a 503
|
||||
// `insufficient_vram` was returned in 225 ms with no server-side
|
||||
// trace, and the guard responsible could only be identified by
|
||||
// reading the source and reproducing its arithmetic (#257).
|
||||
//
|
||||
// `extra` carries the numbers that make a rejection actionable —
|
||||
// free_mb/required_mb, prompt_len/max, retry_after — so the log line
|
||||
// is self-contained rather than a pointer to go and look elsewhere.
|
||||
tracing::warn!(
|
||||
status = env.status,
|
||||
error_type = %env.error_type,
|
||||
code = env.code.as_deref().unwrap_or("-"),
|
||||
detail = %env.message,
|
||||
extra = %serde_json::to_string(&env.extra).unwrap_or_default(),
|
||||
"request rejected"
|
||||
);
|
||||
envelope_response(env)
|
||||
}
|
||||
|
||||
@@ -1031,111 +907,3 @@ mod error_envelope_tests {
|
||||
assert!(error["message"].as_str().unwrap().contains("kaboom"));
|
||||
}
|
||||
}
|
||||
|
||||
// ── Request-body extraction (#63) ────────────────────────────────────
|
||||
|
||||
/// `Json<T>` that fails like the rest of our API instead of like axum.
|
||||
///
|
||||
/// axum's own extractor answers a malformed body with a bare string —
|
||||
/// `"Failed to deserialize the JSON body into the target type: …"` —
|
||||
/// at 400/422, *before any handler runs*, so nothing logs it either.
|
||||
/// Both halves of that hurt: no OpenAI client can parse the body (they
|
||||
/// look for `error.code` / `error.type`), and the operator sees nothing
|
||||
/// at all in the journal.
|
||||
///
|
||||
/// That combination cost a real session (#253): a client sent a
|
||||
/// spec-legal shape our types didn't model, got 422 with an unparseable
|
||||
/// body, surfaced nothing to its user, and neuron logged not one line —
|
||||
/// the cause was findable only by reading the wire types.
|
||||
///
|
||||
/// So: same #63 envelope as every other error, plus a `WARN` naming the
|
||||
/// endpoint and the deserialization detail.
|
||||
pub struct ApiJson<T>(pub T);
|
||||
|
||||
impl<T, S> axum::extract::FromRequest<S> for ApiJson<T>
|
||||
where
|
||||
T: serde::de::DeserializeOwned,
|
||||
S: Send + Sync,
|
||||
{
|
||||
type Rejection = axum::response::Response;
|
||||
|
||||
async fn from_request(req: axum::extract::Request, state: &S) -> Result<Self, Self::Rejection> {
|
||||
use axum::extract::rejection::JsonRejection;
|
||||
let (method, uri) = (req.method().clone(), req.uri().clone());
|
||||
match Json::<T>::from_request(req, state).await {
|
||||
Ok(Json(value)) => Ok(ApiJson(value)),
|
||||
Err(rejection) => {
|
||||
// Distinguish "shape we don't accept" (422 — the body
|
||||
// parsed as JSON but not as this request) from "not JSON
|
||||
// at all" (400) and a wrong content-type (415), because
|
||||
// they need different fixes from the caller.
|
||||
let (status, code) = match &rejection {
|
||||
JsonRejection::JsonDataError(_) => (422, "invalid_request_body"),
|
||||
JsonRejection::JsonSyntaxError(_) => (400, "malformed_json"),
|
||||
JsonRejection::MissingJsonContentType(_) => (415, "unsupported_media_type"),
|
||||
_ => (400, "invalid_request_body"),
|
||||
};
|
||||
let detail = rejection.body_text();
|
||||
tracing::warn!(
|
||||
method = %method,
|
||||
uri = %uri,
|
||||
status,
|
||||
code,
|
||||
detail = %detail,
|
||||
"request body rejected before dispatch"
|
||||
);
|
||||
Err(envelope_response(
|
||||
cortex_core::error_envelope::OpenAiError::new(
|
||||
status,
|
||||
"invalid_request_error",
|
||||
code,
|
||||
detail,
|
||||
),
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod keep_alive_tests {
|
||||
use super::*;
|
||||
use futures::StreamExt;
|
||||
|
||||
/// A stream that emits its opening frame and then goes quiet — the
|
||||
/// shape of every streamed completion while it prefills — must keep
|
||||
/// putting bytes on the wire.
|
||||
///
|
||||
/// The deadline here is the point: with axum's 15s default this test
|
||||
/// times out, which is exactly the production failure. A 7.4k-token
|
||||
/// prompt prefills in ~6s, so the default never fired once, the
|
||||
/// connection sat silent for the whole prefill, and clients dropped it
|
||||
/// as dead — the DeepSeek Harness after ~1s, reporting `terminated`.
|
||||
///
|
||||
/// Asserted against observable output rather than the configured
|
||||
/// interval because `KeepAlive` exposes no getter, and because what
|
||||
/// broke was the behaviour, not the number.
|
||||
#[tokio::test]
|
||||
async fn a_silent_stream_still_sends_bytes_while_prefilling() {
|
||||
let silent = stream::pending::<Result<Event, Infallible>>();
|
||||
let response = Sse::new(silent)
|
||||
.keep_alive(prefill_keep_alive())
|
||||
.into_response();
|
||||
|
||||
let mut body = response.into_body().into_data_stream();
|
||||
let frame = tokio::time::timeout(Duration::from_secs(3), body.next())
|
||||
.await
|
||||
.expect("a silent stream must emit a keep-alive well inside any client idle timeout")
|
||||
.expect("stream ended instead of keeping alive")
|
||||
.expect("keep-alive frame");
|
||||
|
||||
// SSE comment frames start with ':' and are discarded by parsers
|
||||
// per spec, so this costs the client nothing but proves liveness.
|
||||
assert_eq!(
|
||||
frame.first(),
|
||||
Some(&b':'),
|
||||
"expected an SSE comment frame, got {:?}",
|
||||
String::from_utf8_lossy(&frame)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -86,16 +86,6 @@ pub struct CandleHarnessConfig {
|
||||
#[serde(default)]
|
||||
pub context_limit: ContextLimitConfig,
|
||||
|
||||
/// What each OpenAI effort level costs, in reasoning tokens (#223).
|
||||
///
|
||||
/// OpenAI-shaped clients cannot send a token count — their whole
|
||||
/// vocabulary is the `minimal|low|medium|high` ladder — so the
|
||||
/// numbers live here and are advertised on `/models`. Operator
|
||||
/// configurable because the right values depend on the model and the
|
||||
/// hardware.
|
||||
#[serde(default)]
|
||||
pub reasoning_budget: crate::harness::reasoning_budget::ReasoningBudgetLadder,
|
||||
|
||||
/// Admission control (#53): bounds the per-model wait queue so a busy
|
||||
/// model returns a fast, retryable `429`/`503` instead of stalling new
|
||||
/// requests until their client times out.
|
||||
@@ -190,44 +180,9 @@ pub struct AdmissionConfig {
|
||||
/// for any single principal (resolved from the `x-helexa-*` headers
|
||||
/// cortex stamps), so one client can't monopolize the queue while others
|
||||
/// wait. Over-cap → `429 rate_limit_exceeded` + `Retry-After`. `0`
|
||||
/// disables the cap.
|
||||
///
|
||||
/// Anonymous requests carry no principal and so cannot be capped by it.
|
||||
/// They are bounded instead by `anon_max_in_flight` / `anon_max_pending`
|
||||
/// below (#262).
|
||||
/// disables the cap; anonymous requests are always exempt.
|
||||
#[serde(default = "default_admission_max_per_principal")]
|
||||
pub max_per_principal: usize,
|
||||
/// Ceiling on **anonymous** requests holding an in-flight seat (#262).
|
||||
///
|
||||
/// Anonymous traffic is served from capacity left over once identified
|
||||
/// traffic is satisfied: it never takes a seat while an identified
|
||||
/// request is waiting, and it may never hold every seat at once — which
|
||||
/// is what this bounds, because a request already running cannot be
|
||||
/// preempted. Reserving one seat is what caps how long a newly arrived
|
||||
/// identified request waits: one request's duration, not all of them.
|
||||
///
|
||||
/// Unset (the default) derives `max_in_flight - 1`, floored at 1 so a
|
||||
/// single-seat model still serves anonymous callers. `0` refuses
|
||||
/// anonymous traffic outright.
|
||||
#[serde(default)]
|
||||
pub anon_max_in_flight: Option<usize>,
|
||||
/// Ceiling on anonymous requests in flight **plus** queued (#262).
|
||||
///
|
||||
/// The queue needs the same reservation as the seats: without it an
|
||||
/// anonymous flood fills the queue and identified callers are refused at
|
||||
/// the door, before priority can help them.
|
||||
///
|
||||
/// Unset derives `max_in_flight + max_queue_depth - 1`, floored at 1.
|
||||
#[serde(default)]
|
||||
pub anon_max_pending: Option<usize>,
|
||||
/// Maximum seconds a request waits for enough **KV budget** to free
|
||||
/// before it is refused (#257). Deliberately far longer than
|
||||
/// `max_wait_secs`: that one waits for a slot to turn over, this one
|
||||
/// waits for a whole long-context sequence to finish, which on a 27B at
|
||||
/// 64k context is minutes rather than seconds. Too short and the queue
|
||||
/// merely converts a fast rejection into a slow one.
|
||||
#[serde(default = "default_admission_kv_max_wait_secs")]
|
||||
pub kv_max_wait_secs: u64,
|
||||
}
|
||||
|
||||
impl Default for AdmissionConfig {
|
||||
@@ -237,9 +192,6 @@ impl Default for AdmissionConfig {
|
||||
max_queue_depth: default_admission_max_queue_depth(),
|
||||
max_wait_secs: default_admission_max_wait_secs(),
|
||||
max_per_principal: default_admission_max_per_principal(),
|
||||
anon_max_in_flight: None,
|
||||
anon_max_pending: None,
|
||||
kv_max_wait_secs: default_admission_kv_max_wait_secs(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -343,24 +295,8 @@ pub struct ContextLimitConfig {
|
||||
/// Generation reserve (tokens) left below the context wall:
|
||||
/// `input = context − output_reserve_tokens`. Defaults to neuron's
|
||||
/// default `max_tokens`.
|
||||
///
|
||||
/// This is the budget a request gets when it names none, and the
|
||||
/// figure the input/output split is derived from. It is **not** the
|
||||
/// most a request may ask for — see `max_output_tokens` (#278).
|
||||
#[serde(default = "default_output_reserve_tokens")]
|
||||
pub output_reserve_tokens: usize,
|
||||
|
||||
/// The largest output a request may name, advertised as
|
||||
/// `max_output_tokens` on `/v1/models` (#278).
|
||||
///
|
||||
/// `None` derives it as `min(max_position_embeddings,
|
||||
/// DEFAULT_OUTPUT_CEILING)`. Separate from the reserve because the
|
||||
/// two answer different questions and were previously the same
|
||||
/// number: publishing the reserve told every client that read the
|
||||
/// advertisement to cap itself at the default, which on a reasoning
|
||||
/// model can be less than one think block.
|
||||
#[serde(default)]
|
||||
pub max_output_tokens: Option<usize>,
|
||||
}
|
||||
|
||||
impl Default for ContextLimitConfig {
|
||||
@@ -372,7 +308,6 @@ impl Default for ContextLimitConfig {
|
||||
activation_headroom_mb: default_activation_headroom_mb(),
|
||||
min_free_floor_mb: default_context_min_free_floor_mb(),
|
||||
output_reserve_tokens: default_output_reserve_tokens(),
|
||||
max_output_tokens: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -393,13 +328,6 @@ fn default_bootstrap_prefill_tok_per_sec() -> f64 {
|
||||
800.0
|
||||
}
|
||||
|
||||
/// Long enough for a multi-minute long-context turn ahead of us to finish
|
||||
/// and return its KV, but inside cortex's 300 s proxy timeout so the
|
||||
/// caller is still listening when we answer.
|
||||
fn default_admission_kv_max_wait_secs() -> u64 {
|
||||
240
|
||||
}
|
||||
|
||||
fn default_activation_headroom_mb() -> u64 {
|
||||
2048
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -380,23 +380,7 @@ impl GatedDeltaNet {
|
||||
self.head_v_dim,
|
||||
)?;
|
||||
// Stash the updated recurrent state for the next call.
|
||||
//
|
||||
// The delta rule ran in f32 and `state_init` upcasts back to f32
|
||||
// on entry, so storing at the model dtype is a round trip that
|
||||
// buys only memory (#284). Decode takes it once per token.
|
||||
//
|
||||
// Storing f32 keeps `new_state` as-is, which on the CUDA decode
|
||||
// path is the same buffer the kernel updated in place — so the
|
||||
// stored state aliases it, where the `to_dtype` branch always
|
||||
// allocated afresh. That is sound: the pre-update value is dead
|
||||
// by this point, and prefix snapshots deep-copy
|
||||
// (`snapshot_state`) precisely so they never share storage with
|
||||
// a buffer a later forward will overwrite.
|
||||
self.state.recurrent_state = Some(if gdn_state_f32() {
|
||||
new_state
|
||||
} else {
|
||||
new_state.to_dtype(dtype)?
|
||||
});
|
||||
self.state.recurrent_state = Some(new_state.to_dtype(dtype)?);
|
||||
|
||||
// core_attn_out: (B, H, L, D_v) → (B, L, H, D_v) → (B*L*H, D_v).
|
||||
let core_attn_out = core_attn_out.transpose(1, 2)?.contiguous()?; // (B, L, H, D_v)
|
||||
@@ -480,33 +464,6 @@ fn chunked_prefill_enabled() -> bool {
|
||||
})
|
||||
}
|
||||
|
||||
/// `NEURON_GDN_STATE_F32=1` keeps the recurrent state in f32 between
|
||||
/// forwards instead of storing it at the model's activation dtype (#284).
|
||||
///
|
||||
/// The delta rule computes in f32 throughout, so storing the result at
|
||||
/// BF16 and upcasting it again next call is a pure round-trip loss.
|
||||
/// Measured on the chunked path: bit-exact carrying f32 at every chunk
|
||||
/// size, against 1.3e-3 relative error after 16 prefill chunks and
|
||||
/// **1.0e-2 after 4096 decode steps** — decode round-trips once per
|
||||
/// token, across 48 of this arch's 64 layers.
|
||||
///
|
||||
/// Off by default on purpose. Every #283 measurement was taken against
|
||||
/// the bf16 baseline, and flipping this silently would move the ground
|
||||
/// underneath the sampling A/B and the decode-rate work (#282, #285)
|
||||
/// at the same time. Turn it on deliberately, measure, then decide the
|
||||
/// default.
|
||||
///
|
||||
/// Costs ~2× the recurrent slab: +37.7 MB per GPU per sequence on the
|
||||
/// deployed 27B under TP=2, so +302 MB/GPU at `max_in_flight = 8`.
|
||||
fn gdn_state_f32() -> bool {
|
||||
static ENABLED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
|
||||
*ENABLED.get_or_init(|| {
|
||||
std::env::var("NEURON_GDN_STATE_F32")
|
||||
.map(|v| v == "1" || v.eq_ignore_ascii_case("true"))
|
||||
.unwrap_or(false)
|
||||
})
|
||||
}
|
||||
|
||||
/// Chunk-parallel gated delta rule — a faithful port of the HF
|
||||
/// reference `torch_chunk_gated_delta_rule` (chunk_size = 64) in
|
||||
/// `transformers/models/qwen3_5/modeling_qwen3_5.py`, minus the steps
|
||||
@@ -1140,94 +1097,6 @@ mod tests {
|
||||
assert!(d_state < 2e-4, "final state diverged: {d_state}");
|
||||
}
|
||||
|
||||
/// #284: the state carried *between* forwards must not lose
|
||||
/// precision. Every other parity test here is single-call and
|
||||
/// all-f32, so none of them crosses the boundary where
|
||||
/// `GatedDeltaNet::forward` stores the state at the model's
|
||||
/// activation dtype — which is why the defect lived.
|
||||
///
|
||||
/// Two properties, one test:
|
||||
///
|
||||
/// 1. carrying f32, an N-chunk run equals a single call **exactly**,
|
||||
/// so the chunk-boundary math itself is sound (this kills the
|
||||
/// "chunked prefill corrupts state" hypothesis, #283 H1);
|
||||
/// 2. carrying the model dtype loses a measurable, chunk-count-
|
||||
/// dependent amount — the defect the `NEURON_GDN_STATE_F32` flag
|
||||
/// exists to switch off.
|
||||
///
|
||||
/// Asserting (2) is deliberate: if a future change makes the bf16
|
||||
/// carry lossless this test fails loudly and the flag can go, rather
|
||||
/// than the flag quietly outliving its reason.
|
||||
#[test]
|
||||
fn state_carried_across_chunks_is_exact_in_f32_and_lossy_in_bf16() {
|
||||
let (b, h, dk, dv) = (1, 2, 32, 32);
|
||||
let dev = Device::Cpu;
|
||||
let l = 1024;
|
||||
// Long-memory decay: g near zero keeps the state alive across
|
||||
// every boundary, which is both the realistic regime for
|
||||
// long-context retrieval and the one that accumulates most.
|
||||
let scale = 1.0 / (dk as f64).sqrt();
|
||||
let q = Tensor::randn(0f32, 1.0, (b, h, l, dk), &dev).unwrap();
|
||||
let q = (l2norm(&q, 1e-6).unwrap() * scale).unwrap();
|
||||
let k = Tensor::randn(0f32, 1.0, (b, h, l, dk), &dev).unwrap();
|
||||
let k = l2norm(&k, 1e-6).unwrap();
|
||||
let v = (Tensor::randn(0f32, 1.0, (b, h, l, dv), &dev).unwrap() * 0.5).unwrap();
|
||||
let g = (Tensor::rand(0f32, 1f32, (b, h, l), &dev).unwrap() * -0.02).unwrap();
|
||||
let beta = Tensor::rand(0.05f32, 0.95f32, (b, h, l), &dev).unwrap();
|
||||
let zeros = || Tensor::zeros((b, h, dk, dv), DType::F32, &dev).unwrap();
|
||||
|
||||
// Ground truth: the whole sequence in one call, f32 throughout.
|
||||
let (out_ref, _) = run_chunk_gated_delta_rule(&q, &k, &v, &g, &beta, zeros()).unwrap();
|
||||
|
||||
// Replay it in `chunk`-sized pieces, carrying the state the way
|
||||
// `forward` does.
|
||||
let replay = |chunk: usize, carry: Option<DType>| {
|
||||
let mut state = zeros();
|
||||
let mut outs = Vec::new();
|
||||
let mut pos = 0;
|
||||
while pos < l {
|
||||
let n = chunk.min(l - pos);
|
||||
let take = |t: &Tensor| t.narrow(2, pos, n).unwrap().contiguous().unwrap();
|
||||
let (o, s) = run_chunk_gated_delta_rule(
|
||||
&take(&q),
|
||||
&take(&k),
|
||||
&take(&v),
|
||||
&take(&g),
|
||||
&take(&beta),
|
||||
state,
|
||||
)
|
||||
.unwrap();
|
||||
state = match carry {
|
||||
Some(dt) => s.to_dtype(dt).unwrap().to_dtype(DType::F32).unwrap(),
|
||||
None => s,
|
||||
};
|
||||
outs.push(o);
|
||||
pos += n;
|
||||
}
|
||||
Tensor::cat(&outs, 2).unwrap()
|
||||
};
|
||||
|
||||
for chunk in [128usize, 256, 512] {
|
||||
let exact = max_abs_diff(&out_ref, &replay(chunk, None));
|
||||
assert_eq!(
|
||||
exact, 0.0,
|
||||
"chunk={chunk}: an f32 state carry must reproduce the \
|
||||
single-call result bit for bit, got {exact:e}"
|
||||
);
|
||||
}
|
||||
|
||||
// And the dtype round trip is what costs us. Compared on the
|
||||
// final chunk, where the accumulated state has done the most work.
|
||||
let tail = |t: &Tensor| t.narrow(2, l - 128, 128).unwrap().contiguous().unwrap();
|
||||
let lossy = max_abs_diff(&tail(&out_ref), &tail(&replay(128, Some(DType::BF16))));
|
||||
assert!(
|
||||
lossy > 1e-5,
|
||||
"storing the state at BF16 between chunks is expected to lose \
|
||||
precision (#284); if this now passes losslessly the flag can \
|
||||
be retired, got {lossy:e}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Adversarially correlated inputs: near-identical keys with
|
||||
/// beta ≈ 1 and negligible decay make the UT-transform matrix T
|
||||
/// maximally coherent — raw powers of T grow combinatorially
|
||||
|
||||
@@ -562,9 +562,9 @@ mod tests {
|
||||
.forward_batch_decode(&input, &positions, mask.as_ref())
|
||||
.expect("batched step");
|
||||
assert_eq!(h.dims()[0], 3);
|
||||
for (row, exp) in expected.iter().enumerate() {
|
||||
for row in 0..3 {
|
||||
let got: Vec<f32> = h.i((row, 0, ..)).unwrap().to_vec1().unwrap();
|
||||
let diff = max_abs_diff(&exp[t], &got);
|
||||
let diff = max_abs_diff(&expected[row][t], &got);
|
||||
assert!(diff < 1e-4, "row {row} step {t} diverged: {diff}");
|
||||
}
|
||||
}
|
||||
@@ -610,9 +610,9 @@ mod tests {
|
||||
let h = model
|
||||
.forward_batch_decode(&input, &[3, 3], None)
|
||||
.expect("step");
|
||||
for (row, exp) in expected.iter().enumerate() {
|
||||
for row in 0..2 {
|
||||
let got: Vec<f32> = h.i((row, 0, ..)).unwrap().to_vec1().unwrap();
|
||||
let diff = max_abs_diff(exp, &got);
|
||||
let diff = max_abs_diff(&expected[row], &got);
|
||||
assert!(diff < 1e-4, "row {row} diverged: {diff}");
|
||||
}
|
||||
}
|
||||
@@ -674,9 +674,9 @@ mod tests {
|
||||
let h = model
|
||||
.forward_batch_decode(&input, &positions, mask.as_ref())
|
||||
.expect("batched step");
|
||||
for (row, exp) in expected.iter().enumerate() {
|
||||
for row in 0..2 {
|
||||
let got: Vec<f32> = h.i((row, 0, ..)).unwrap().to_vec1().unwrap();
|
||||
let diff = max_abs_diff(&exp[t], &got);
|
||||
let diff = max_abs_diff(&expected[row][t], &got);
|
||||
assert!(diff < 1e-4, "batched row {row} step {t}: {diff}");
|
||||
}
|
||||
}
|
||||
|
||||
+1392
-2085
File diff suppressed because it is too large
Load Diff
@@ -1,155 +0,0 @@
|
||||
//! Capability discovery for models that are not loaded (#241).
|
||||
//!
|
||||
//! A loaded model reports its own modalities — the handle knows whether
|
||||
//! it built a vision tower or a diffusion pipeline. A model sitting cold
|
||||
//! in the cache reports nothing, which is how image generation came to be
|
||||
//! invisible: the image model is *normally* cold, so `/v1/models` showed
|
||||
//! an entry with no capabilities and no hint that it produces pictures.
|
||||
//!
|
||||
//! The alternative to discovery is an operator writing `capabilities =
|
||||
//! ["image"]` into a catalogue file. That is a fact the runtime already
|
||||
//! holds, restated by hand in a second place, where it can rot silently —
|
||||
//! nothing fails when the catalogue is wrong, the model just lies about
|
||||
//! itself. So derive it instead.
|
||||
//!
|
||||
//! Everything here reads the local Hugging Face cache and nothing else:
|
||||
//! no network, no weight load, no device. The two signals are the same
|
||||
//! ones the loader itself dispatches on, which is what keeps this honest
|
||||
//! rather than a parallel guess:
|
||||
//!
|
||||
//! - **image** — `model_index.json` at the repo root, the diffusers
|
||||
//! layout marker. [`super::preflight::classify`] already returns
|
||||
//! [`SourceFormat::Diffusers`] for it, and `load_model` routes on
|
||||
//! exactly that to reach the image path.
|
||||
//! - **vision** — a `vision_config` block in `config.json`, the same
|
||||
//! thing `VisionMeta::from_config_path` keys on to decide whether a
|
||||
//! loaded model advertises `"vision"`.
|
||||
//!
|
||||
//! A repo that was never cached yields `None` rather than a guess. That
|
||||
//! is a real limitation — a model no node has downloaded stays
|
||||
//! undiscoverable until one does — but reporting "unknown" is the honest
|
||||
//! answer, and it is strictly better than the empty list it replaces.
|
||||
|
||||
use super::preflight::{self, SourceFormat};
|
||||
use std::path::Path;
|
||||
|
||||
/// Modalities a model would serve, derived from its cached repo.
|
||||
///
|
||||
/// `None` when the repo has no usable snapshot in this cache — the model
|
||||
/// was never downloaded here, so this node cannot say anything about it.
|
||||
pub fn from_cache(cache_path: &Path, repo_path: &str) -> Option<Vec<String>> {
|
||||
let files = preflight::cached_snapshot_files(cache_path, repo_path)?;
|
||||
let refs: Vec<&str> = files.iter().map(String::as_str).collect();
|
||||
Some(classify_files(&refs, || {
|
||||
let dir = preflight::cached_snapshot_dir(cache_path, repo_path)?;
|
||||
std::fs::read_to_string(dir.join("config.json")).ok()
|
||||
}))
|
||||
}
|
||||
|
||||
/// Classify a repo's file listing into modalities.
|
||||
///
|
||||
/// `read_config` is called lazily and only for non-diffusers layouts, so
|
||||
/// a diffusers repo costs no file read at all. Split from [`from_cache`]
|
||||
/// so the tests can drive it from fixtures without a cache on disk.
|
||||
pub fn classify_files(
|
||||
filenames: &[&str],
|
||||
read_config: impl FnOnce() -> Option<String>,
|
||||
) -> Vec<String> {
|
||||
match preflight::classify(filenames) {
|
||||
SourceFormat::Diffusers => vec!["image".to_string()],
|
||||
// A repo with no recognised weights (tokenizer-only, or an empty
|
||||
// entry) is not servable, so claiming "text" would be a lie of
|
||||
// the same kind this module exists to remove.
|
||||
SourceFormat::Empty => Vec::new(),
|
||||
_ => {
|
||||
let mut caps = vec!["text".to_string()];
|
||||
if read_config().is_some_and(|text| declares_vision(&text)) {
|
||||
caps.push("vision".to_string());
|
||||
}
|
||||
caps
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a `config.json` describes a model with a vision tower.
|
||||
fn declares_vision(config_json: &str) -> bool {
|
||||
serde_json::from_str::<serde_json::Value>(config_json)
|
||||
.ok()
|
||||
.is_some_and(|v| v.get("vision_config").is_some())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The Z-Image layout: component subdirectories under a
|
||||
/// `model_index.json`, with safetensors nested inside them. The
|
||||
/// nested shards are why this must be checked before the
|
||||
/// safetensors sniff.
|
||||
#[test]
|
||||
fn diffusers_repo_is_an_image_model() {
|
||||
let files = [
|
||||
"model_index.json",
|
||||
"transformer/config.json",
|
||||
"transformer/diffusion_pytorch_model.safetensors",
|
||||
"vae/config.json",
|
||||
"text_encoder/model.safetensors",
|
||||
];
|
||||
assert_eq!(
|
||||
classify_files(&files, || panic!("config must not be read for diffusers")),
|
||||
vec!["image".to_string()]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dense_repo_without_vision_config_is_text_only() {
|
||||
let files = ["config.json", "tokenizer.json", "model.safetensors"];
|
||||
let caps = classify_files(&files, || Some(r#"{"model_type":"qwen3"}"#.to_string()));
|
||||
assert_eq!(caps, vec!["text".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dense_repo_with_vision_config_adds_vision() {
|
||||
let files = [
|
||||
"config.json",
|
||||
"tokenizer.json",
|
||||
"model.safetensors.index.json",
|
||||
"model-00001-of-00002.safetensors",
|
||||
];
|
||||
let caps = classify_files(&files, || {
|
||||
Some(r#"{"model_type":"qwen3_5","vision_config":{"patch_size":16}}"#.to_string())
|
||||
});
|
||||
assert_eq!(caps, vec!["text".to_string(), "vision".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gguf_repo_is_text() {
|
||||
let files = ["Qwen3-0.6B-Q4_K_M.gguf", "tokenizer.json"];
|
||||
assert_eq!(classify_files(&files, || None), vec!["text".to_string()]);
|
||||
}
|
||||
|
||||
/// A tokenizer-only repo has nothing to serve. Reporting "text"
|
||||
/// would advertise a model that cannot answer.
|
||||
#[test]
|
||||
fn repo_without_weights_claims_nothing() {
|
||||
let files = ["tokenizer.json", "README.md"];
|
||||
assert!(classify_files(&files, || None).is_empty());
|
||||
}
|
||||
|
||||
/// An unparseable or absent config must not promote to vision.
|
||||
#[test]
|
||||
fn unreadable_config_does_not_claim_vision() {
|
||||
let files = ["config.json", "model.safetensors"];
|
||||
assert_eq!(
|
||||
classify_files(&files, || Some("{ not json".to_string())),
|
||||
vec!["text".to_string()]
|
||||
);
|
||||
assert_eq!(classify_files(&files, || None), vec!["text".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uncached_repo_is_unknown_not_empty() {
|
||||
let dir = std::env::temp_dir().join("helexa-capability-probe-missing");
|
||||
assert!(from_cache(&dir, "does-not/exist").is_none());
|
||||
}
|
||||
}
|
||||
@@ -227,30 +227,9 @@ pub fn render_chat_template(
|
||||
let content_value = match &m.content {
|
||||
MessageContent::Text(s) => Value::String(s.clone()),
|
||||
MessageContent::Parts(parts) => Value::Array(parts.clone()),
|
||||
// Rendered as JSON null so the template's own
|
||||
// `content is none` branch handles it — that is how HF
|
||||
// templates model a tool-calls-only assistant turn.
|
||||
MessageContent::Null => Value::Null,
|
||||
};
|
||||
let mut obj = serde_json::Map::new();
|
||||
// `developer` is OpenAI's successor to `system` and is what
|
||||
// a client sends once a model is declared reasoning-capable
|
||||
// (pi-ai: `model.reasoning && supportsDeveloperRole`). Most
|
||||
// chat templates — every one on this fleet — only know the
|
||||
// older spelling and `raise_exception` on anything else, so
|
||||
// an unmodified client got a 422 for a system prompt it sent
|
||||
// correctly.
|
||||
//
|
||||
// Rewritten only when the template does not handle the role
|
||||
// itself: a template that knows `developer` may treat it as
|
||||
// distinct from `system`, and substituting there would be us
|
||||
// deciding something the model's authors already decided.
|
||||
let role = if m.role == "developer" && !template.contains("developer") {
|
||||
"system"
|
||||
} else {
|
||||
m.role.as_str()
|
||||
};
|
||||
obj.insert("role".into(), Value::String(role.to_string()));
|
||||
obj.insert("role".into(), Value::String(m.role.clone()));
|
||||
obj.insert("content".into(), content_value);
|
||||
// Forward extras (e.g. tool_calls on assistant turns,
|
||||
// tool_call_id on tool result turns). HF templates that
|
||||
@@ -438,189 +417,6 @@ mod tests {
|
||||
assert!(out.contains("<|vision_start|>") && out.contains("<|vision_end|>"));
|
||||
}
|
||||
|
||||
/// A file read back through a tool result must reach the model
|
||||
/// byte-for-byte.
|
||||
///
|
||||
/// The reported symptom: the model writes a file correctly, reads it
|
||||
/// back, and sees errors that are not in the file — consistently,
|
||||
/// after every write. Anything that mangles `<`, `&`, `"` or
|
||||
/// newlines on this path would produce exactly that, and would make
|
||||
/// the model burn its budget reconciling a file against a corrupted
|
||||
/// view of itself.
|
||||
#[test]
|
||||
fn a_file_read_back_through_a_tool_result_is_byte_exact() {
|
||||
let template = include_str!("../../tests/data/qwen3.8-27b.chat_template.jinja");
|
||||
// The characters most likely to be mangled, in the shapes a real
|
||||
// web project contains them.
|
||||
let file = "<!DOCTYPE html>\n<html lang=\"en\">\n<head>\n <meta charset=\"utf-8\">\n</head>\n<body>\n <script>\n if (a && b || c < d) { g.fillStyle = '#b8905a'; }\n const s = `rgba(${r},${g},0.5)`;\n\n // a comment & an ampersand\n </script>\n</body>\n</html>\n";
|
||||
|
||||
let mut tool_extras = serde_json::Map::new();
|
||||
tool_extras.insert("tool_call_id".into(), json!("call_0"));
|
||||
let mut assistant_extras = serde_json::Map::new();
|
||||
assistant_extras.insert(
|
||||
"tool_calls".into(),
|
||||
json!([{"id":"call_0","type":"function",
|
||||
"function":{"name":"read","arguments":"{\"path\":\"/tmp/index.html\"}"}}]),
|
||||
);
|
||||
|
||||
let messages = vec![
|
||||
user_msg("read the file back"),
|
||||
ChatMessage {
|
||||
role: "assistant".into(),
|
||||
content: MessageContent::Null,
|
||||
extra: Value::Object(assistant_extras),
|
||||
},
|
||||
ChatMessage {
|
||||
role: "tool".into(),
|
||||
content: MessageContent::Text(file.to_string()),
|
||||
extra: Value::Object(tool_extras),
|
||||
},
|
||||
user_msg("is it correct?"),
|
||||
];
|
||||
let out =
|
||||
render_chat_template(template, &messages, &Value::Null, &Value::Null).expect("render");
|
||||
|
||||
for probe in [
|
||||
"<!DOCTYPE html>",
|
||||
"<html lang=\"en\">",
|
||||
"if (a && b || c < d)",
|
||||
"g.fillStyle = '#b8905a';",
|
||||
"`rgba(${r},${g},0.5)`",
|
||||
"// a comment & an ampersand",
|
||||
] {
|
||||
assert!(
|
||||
out.contains(probe),
|
||||
"the model would not see {probe:?} verbatim — rendered prompt mangles it"
|
||||
);
|
||||
}
|
||||
// And nothing HTML-escaped anywhere.
|
||||
for bad in ["<", ">", "&", """, "'", "'"] {
|
||||
assert!(
|
||||
!out.contains(bad),
|
||||
"found {bad:?} in the rendered prompt: content is being HTML-escaped"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Blank lines inside a replayed tool-call argument must survive
|
||||
/// rendering intact.
|
||||
///
|
||||
/// A model that writes a source file and then re-reads it from its
|
||||
/// own replayed tool call is comparing what it remembers writing
|
||||
/// against what the prompt now shows. Any whitespace the pipeline
|
||||
/// eats makes those disagree, and the model has no way to tell a
|
||||
/// rendering artifact from a mistake of its own — which is a
|
||||
/// plausible route into the announce-then-stop stall (#296).
|
||||
///
|
||||
/// The JSON string→object normalisation above is the step that
|
||||
/// could plausibly lose them, since it round-trips the arguments
|
||||
/// through a parser.
|
||||
#[test]
|
||||
fn a_replayed_tool_call_keeps_the_blank_lines_in_its_arguments() {
|
||||
let template = include_str!("../../tests/data/qwen3.8-27b.chat_template.jinja");
|
||||
// Blank lines between sections, exactly as a written source file
|
||||
// carries them.
|
||||
let file_body = "'use strict';\n\n// ---- constants ----\nconst A = 1;\n\n\nfunction f() {\n return A;\n}\n";
|
||||
let args = json!({ "path": "/tmp/game.js", "content": file_body });
|
||||
let mut assistant_extras = serde_json::Map::new();
|
||||
assistant_extras.insert(
|
||||
"tool_calls".into(),
|
||||
json!([{
|
||||
"id": "call_0", "type": "function",
|
||||
"function": {
|
||||
"name": "write",
|
||||
// The OpenAI wire form: arguments as a JSON *string*.
|
||||
"arguments": serde_json::to_string(&args).unwrap()
|
||||
}
|
||||
}]),
|
||||
);
|
||||
let mut tool_extras = serde_json::Map::new();
|
||||
tool_extras.insert("tool_call_id".into(), json!("call_0"));
|
||||
|
||||
let messages = vec![
|
||||
user_msg("write the file"),
|
||||
ChatMessage {
|
||||
role: "assistant".into(),
|
||||
content: MessageContent::Null,
|
||||
extra: Value::Object(assistant_extras),
|
||||
},
|
||||
ChatMessage {
|
||||
role: "tool".into(),
|
||||
content: MessageContent::Text("Successfully wrote 84 bytes".into()),
|
||||
extra: Value::Object(tool_extras),
|
||||
},
|
||||
user_msg("now check it parses"),
|
||||
];
|
||||
let out =
|
||||
render_chat_template(template, &messages, &Value::Null, &Value::Null).expect("render");
|
||||
|
||||
assert!(
|
||||
out.contains(file_body),
|
||||
"the file body must appear verbatim in the rendered prompt"
|
||||
);
|
||||
let blanks_in = file_body.lines().filter(|l| l.is_empty()).count();
|
||||
assert_eq!(blanks_in, 3, "fixture sanity");
|
||||
// And specifically the double newline the model would notice.
|
||||
assert!(
|
||||
out.contains("'use strict';\n\n// ---- constants ----"),
|
||||
"blank line after the prologue was eaten"
|
||||
);
|
||||
assert!(
|
||||
out.contains("const A = 1;\n\n\nfunction f()"),
|
||||
"consecutive blank lines were collapsed"
|
||||
);
|
||||
}
|
||||
|
||||
/// The second turn of an OpenAI-native agentic loop: the client
|
||||
/// replays its own assistant turn, which carries `tool_calls` and
|
||||
/// `"content": null`. That must render as the tool-call block —
|
||||
/// not as the literal string "none", and not as a render error
|
||||
/// (which would silently drop back to the no-tools fallback
|
||||
/// template and break tool calling for the rest of the session).
|
||||
#[test]
|
||||
fn real_template_renders_null_content_assistant_tool_call_turn() {
|
||||
let template = include_str!("testdata/qwen3_6_chat_template.jinja");
|
||||
let mut assistant_extras = serde_json::Map::new();
|
||||
assistant_extras.insert(
|
||||
"tool_calls".into(),
|
||||
json!([{
|
||||
"id": "call_0", "type": "function",
|
||||
"function": {"name": "bash", "arguments": "{\"command\":\"ls\"}"}
|
||||
}]),
|
||||
);
|
||||
let mut tool_extras = serde_json::Map::new();
|
||||
tool_extras.insert("tool_call_id".into(), json!("call_0"));
|
||||
|
||||
let messages = vec![
|
||||
user_msg("list the files"),
|
||||
ChatMessage {
|
||||
role: "assistant".into(),
|
||||
content: MessageContent::Null,
|
||||
extra: Value::Object(assistant_extras),
|
||||
},
|
||||
ChatMessage {
|
||||
role: "tool".into(),
|
||||
content: MessageContent::Text("a.txt".into()),
|
||||
extra: Value::Object(tool_extras),
|
||||
},
|
||||
];
|
||||
|
||||
let out = render_chat_template(template, &messages, &Value::Null, &Value::Null)
|
||||
.expect("null assistant content must render, not error");
|
||||
assert!(
|
||||
out.contains("<function=bash>"),
|
||||
"tool call must survive the null-content turn; rendered:\n{out}"
|
||||
);
|
||||
assert!(
|
||||
out.contains("<tool_response>\na.txt"),
|
||||
"tool result must render; rendered:\n{out}"
|
||||
);
|
||||
assert!(
|
||||
!out.contains("none"),
|
||||
"null content must render as empty, not the literal \"none\"; rendered:\n{out}"
|
||||
);
|
||||
}
|
||||
|
||||
fn user_msg(text: &str) -> ChatMessage {
|
||||
ChatMessage {
|
||||
role: "user".into(),
|
||||
@@ -648,78 +444,6 @@ mod tests {
|
||||
/// Minimal Qwen3-style template — enough surface to confirm
|
||||
/// our renderer threads role + content correctly without
|
||||
/// loading a real model's tokenizer_config.json.
|
||||
/// A reasoning-capable model makes OpenAI clients send the system
|
||||
/// prompt as `developer` — and every chat template on this fleet
|
||||
/// raises on an unknown role, so the caller got a 422 for a prompt
|
||||
/// it had sent correctly. Observed live: pi with `reasoning: true`,
|
||||
/// `template_render_failed: Unexpected message role`.
|
||||
#[test]
|
||||
fn the_developer_role_reaches_a_template_that_only_knows_system() {
|
||||
const SYSTEM_ONLY: &str = "{%- for message in messages -%}\
|
||||
{%- if message.role == 'system' -%}SYS:{{ message.content }}\
|
||||
{%- elif message.role == 'user' -%}USR:{{ message.content }}\
|
||||
{%- else -%}{{ raise_exception('Unexpected message role.') }}\
|
||||
{%- endif -%}\
|
||||
{%- endfor -%}";
|
||||
let messages = vec![
|
||||
ChatMessage {
|
||||
role: "developer".into(),
|
||||
content: MessageContent::Text("be terse".into()),
|
||||
extra: Value::Object(Default::default()),
|
||||
},
|
||||
user_msg("hi"),
|
||||
];
|
||||
let out = render_chat_template(SYSTEM_ONLY, &messages, &Value::Null, &Value::Null)
|
||||
.expect("a developer role must not blow up a system-only template");
|
||||
assert!(out.contains("SYS:be terse"), "got {out:?}");
|
||||
}
|
||||
|
||||
/// A template that models `developer` itself keeps it: the role may
|
||||
/// mean something distinct there, and substituting would override a
|
||||
/// decision the model's authors already made.
|
||||
#[test]
|
||||
fn a_template_that_knows_developer_keeps_it() {
|
||||
const KNOWS_DEVELOPER: &str = "{%- for message in messages -%}\
|
||||
{%- if message.role == 'developer' -%}DEV:{{ message.content }}\
|
||||
{%- else -%}{{ message.role }}:{{ message.content }}\
|
||||
{%- endif -%}\
|
||||
{%- endfor -%}";
|
||||
let messages = vec![ChatMessage {
|
||||
role: "developer".into(),
|
||||
content: MessageContent::Text("be terse".into()),
|
||||
extra: Value::Object(Default::default()),
|
||||
}];
|
||||
let out = render_chat_template(KNOWS_DEVELOPER, &messages, &Value::Null, &Value::Null)
|
||||
.expect("render");
|
||||
assert!(out.contains("DEV:be terse"), "got {out:?}");
|
||||
}
|
||||
|
||||
/// The fetcher and the reader must agree on the filenames, or the
|
||||
/// reader looks for something nobody downloaded and falls back to a
|
||||
/// built-in prompt format without anything failing (#280 — observed
|
||||
/// live: every model neuron downloaded itself was served with the
|
||||
/// fallback because no sidecar was ever fetched).
|
||||
#[test]
|
||||
fn every_template_source_this_reads_is_one_the_loader_fetches() {
|
||||
for name in crate::harness::candle::CandleHarness::SIDECAR_FILES {
|
||||
assert!(
|
||||
!name.is_empty(),
|
||||
"sidecar names are the contract between fetch and read"
|
||||
);
|
||||
}
|
||||
// The three the reader consults, in its own priority order.
|
||||
for source in [
|
||||
"chat_template.jinja",
|
||||
"chat_template.json",
|
||||
"tokenizer_config.json",
|
||||
] {
|
||||
assert!(
|
||||
crate::harness::candle::CandleHarness::SIDECAR_FILES.contains(&source),
|
||||
"{source} is read by load_chat_template_alongside but never fetched"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const QWEN3_LIKE: &str = "{%- for message in messages -%}\
|
||||
<|im_start|>{{ message.role }}\n{{ message.content }}<|im_end|>\n\
|
||||
{%- endfor -%}\
|
||||
|
||||
@@ -36,28 +36,6 @@ use crate::config::ContextLimitConfig;
|
||||
/// few turns.
|
||||
const RATE_EMA_ALPHA: f64 = 0.3;
|
||||
|
||||
/// Smallest prompt whose prefill time says anything about throughput.
|
||||
///
|
||||
/// A prefill has a fixed per-request cost — tokenize, launch, sync —
|
||||
/// that dominates a short prompt completely. The deploy's own smoke
|
||||
/// probe prefills 19 tokens in ~130 ms and reads as **145 tok/s** on a
|
||||
/// host that measures ~1,400. Folding that in does not make the
|
||||
/// estimate slightly worse; it replaces it.
|
||||
///
|
||||
/// That mattered because the fallback it displaces is the whole
|
||||
/// cold-start guard: with no sample at all, `derive_limit` uses
|
||||
/// `bootstrap_prefill_tok_per_sec` (800, ≈96k context at the default
|
||||
/// 120 s target). One 19-token probe replaced it with 145 tok/s, and
|
||||
/// the host advertised **17,408** — a sixth of what it can serve —
|
||||
/// until real traffic happened to warm it (#295).
|
||||
///
|
||||
/// 512 excludes the smoke probe and the short auxiliary calls agent
|
||||
/// harnesses make (observed at 19, 143 and 469 tokens) while admitting
|
||||
/// ordinary prompts. At ~1,400 tok/s a 512-token prefill runs ~370 ms,
|
||||
/// so fixed overhead is roughly a tenth of it — informative rather than
|
||||
/// dominant.
|
||||
const MIN_PREFILL_SAMPLE_TOKENS: usize = 512;
|
||||
|
||||
/// Fold one throughput sample (`tokens` in `elapsed`) into the EMA held in
|
||||
/// `bits`. No-op for degenerate inputs so a probe request or a clock blip
|
||||
/// can't poison the average.
|
||||
@@ -121,15 +99,7 @@ impl ThroughputEma {
|
||||
|
||||
/// Fold one prefill measurement (`prompt_tokens` processed in
|
||||
/// `elapsed`) into the prefill EMA.
|
||||
///
|
||||
/// Samples below [`MIN_PREFILL_SAMPLE_TOKENS`] are ignored: they
|
||||
/// measure fixed per-request cost rather than throughput, and the
|
||||
/// value they displace — the bootstrap estimate — is better than
|
||||
/// they are.
|
||||
pub fn record(&self, prompt_tokens: usize, elapsed: Duration) {
|
||||
if prompt_tokens < MIN_PREFILL_SAMPLE_TOKENS {
|
||||
return;
|
||||
}
|
||||
fold_rate(&self.prefill_bits, prompt_tokens, elapsed);
|
||||
}
|
||||
|
||||
@@ -264,23 +234,10 @@ fn round_down(tokens: usize, granularity: usize) -> usize {
|
||||
|
||||
const CONTEXT_GRANULARITY: usize = 1024;
|
||||
|
||||
/// Default ceiling on what a request may name as its output budget.
|
||||
///
|
||||
/// Not derived from anything — a deliberate figure, and configurable
|
||||
/// per host (`[context_limit] max_output_tokens`). The evidence behind
|
||||
/// it: a reasoning model asked to build a small application front-loads
|
||||
/// one long think block before it acts, measured at 27,504 tokens for a
|
||||
/// frontier model on a task Qwen3.8-27B was cut off attempting at
|
||||
/// 24,433 (#271, #278). A ceiling below that turns a capable model into
|
||||
/// one that never reaches its first tool call, so the default sits
|
||||
/// above the observed cost of that class of turn.
|
||||
const DEFAULT_OUTPUT_CEILING: usize = 32_768;
|
||||
|
||||
/// Derive `limit{context,input,output}` for a loaded model.
|
||||
///
|
||||
/// ```text
|
||||
/// output = output_reserve_tokens (default + KV planning)
|
||||
/// output_ceiling = min(max_output_tokens, max_position_embeddings)
|
||||
/// output = output_reserve_tokens
|
||||
/// vram_ceiling = (free_tightest − activation_headroom − min_free_floor) / kv_bytes_per_token_per_card
|
||||
/// throughput_ceiling = target_prefill_latency_secs × prefill_tok_per_sec
|
||||
/// context = min(max_position_embeddings, vram_ceiling, throughput_ceiling) [clamped by `hard_ceiling` if set]
|
||||
@@ -361,163 +318,17 @@ pub fn derive_limit(
|
||||
);
|
||||
|
||||
let input = context.saturating_sub(output);
|
||||
// The ceiling a request may name, distinct from the reserve above
|
||||
// (#278). Bounded by the model's own position limit rather than by
|
||||
// the live context, because the advertisement is read once and
|
||||
// cached by clients: a ceiling that shrank with fleet VRAM would be
|
||||
// stale in whichever direction hurt — either rejecting what we said
|
||||
// we would serve, or leaving budget unclaimed. A request that will
|
||||
// not actually fit is refused at request time, with the numbers,
|
||||
// instead of being under-promised here.
|
||||
let output_ceiling = round_down(
|
||||
cfg.max_output_tokens
|
||||
.unwrap_or(DEFAULT_OUTPUT_CEILING)
|
||||
.min(profile.max_position_embeddings),
|
||||
CONTEXT_GRANULARITY,
|
||||
)
|
||||
.max(output);
|
||||
ModelLimit {
|
||||
context,
|
||||
input: Some(input),
|
||||
output,
|
||||
output_ceiling,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
/// The deploy's smoke probe must not set the advertised context.
|
||||
///
|
||||
/// 19 tokens in 130 ms reads as ~145 tok/s on a host that measures
|
||||
/// ~1,400. Before this guard that one sample displaced the
|
||||
/// bootstrap estimate and the host advertised 17,408 instead of
|
||||
/// ~96,000 — until real traffic happened along (#295).
|
||||
#[test]
|
||||
fn a_smoke_probe_does_not_poison_the_prefill_rate() {
|
||||
let ema = ThroughputEma::new();
|
||||
ema.record(19, Duration::from_millis(130));
|
||||
assert_eq!(
|
||||
ema.get(),
|
||||
None,
|
||||
"a 19-token prefill must leave the EMA unset so the bootstrap value stands"
|
||||
);
|
||||
}
|
||||
|
||||
/// The short auxiliary calls agent harnesses make are the same
|
||||
/// shape as the probe and equally uninformative — observed at 143
|
||||
/// and 469 tokens against prompts three orders of magnitude larger.
|
||||
#[test]
|
||||
fn short_auxiliary_prompts_are_ignored_too() {
|
||||
let ema = ThroughputEma::new();
|
||||
for tokens in [143usize, 469] {
|
||||
ema.record(tokens, Duration::from_millis(200));
|
||||
}
|
||||
assert_eq!(ema.get(), None);
|
||||
}
|
||||
|
||||
/// A real prompt still records, and still tracks a shift.
|
||||
#[test]
|
||||
fn representative_prefills_still_measure() {
|
||||
let ema = ThroughputEma::new();
|
||||
ema.record(1400, Duration::from_secs(1));
|
||||
let first = ema.get().expect("a 1400-token sample must record");
|
||||
assert!((first - 1400.0).abs() < 1.0, "got {first}");
|
||||
|
||||
// A slower host is tracked, not ignored.
|
||||
for _ in 0..20 {
|
||||
ema.record(700, Duration::from_secs(1));
|
||||
}
|
||||
let after = ema.get().unwrap();
|
||||
assert!(after < first && after > 600.0, "got {after}");
|
||||
}
|
||||
|
||||
/// Exactly at the threshold counts — the boundary is inclusive so
|
||||
/// the constant means what it says.
|
||||
#[test]
|
||||
fn the_threshold_itself_is_a_valid_sample() {
|
||||
let ema = ThroughputEma::new();
|
||||
ema.record(MIN_PREFILL_SAMPLE_TOKENS, Duration::from_secs(1));
|
||||
assert!(ema.get().is_some());
|
||||
}
|
||||
use super::*;
|
||||
|
||||
/// The ceiling a client is told to plan against must not be the
|
||||
/// reserve. Advertising the reserve is what told a reasoning model's
|
||||
/// harness to cap itself below the cost of one think block (#278).
|
||||
#[test]
|
||||
fn the_advertised_ceiling_is_not_the_reserve() {
|
||||
let cfg = ContextLimitConfig::default();
|
||||
let profile = ContextProfile {
|
||||
max_position_embeddings: 131_072,
|
||||
kv_bytes_per_token_per_card: 65_536,
|
||||
world_size: 2,
|
||||
};
|
||||
let limit = derive_limit(&profile, 0, 0.0, None, &cfg);
|
||||
assert_eq!(limit.output, cfg.output_reserve_tokens, "reserve unchanged");
|
||||
assert_eq!(limit.output_ceiling, DEFAULT_OUTPUT_CEILING);
|
||||
assert!(limit.output_ceiling > limit.output);
|
||||
}
|
||||
|
||||
/// A model whose own position limit is below the default ceiling
|
||||
/// cannot generate past it, so the advertisement must not claim it.
|
||||
#[test]
|
||||
fn the_ceiling_never_exceeds_the_models_own_position_limit() {
|
||||
let cfg = ContextLimitConfig::default();
|
||||
let profile = ContextProfile {
|
||||
max_position_embeddings: 16_384,
|
||||
kv_bytes_per_token_per_card: 65_536,
|
||||
world_size: 1,
|
||||
};
|
||||
let limit = derive_limit(&profile, 0, 0.0, None, &cfg);
|
||||
assert_eq!(limit.output_ceiling, 16_384);
|
||||
}
|
||||
|
||||
/// The operator's number wins, which is what makes the default a
|
||||
/// starting point rather than a guess baked into the binary.
|
||||
#[test]
|
||||
fn a_configured_ceiling_overrides_the_default() {
|
||||
let cfg = ContextLimitConfig {
|
||||
max_output_tokens: Some(65_536),
|
||||
..ContextLimitConfig::default()
|
||||
};
|
||||
let profile = ContextProfile {
|
||||
max_position_embeddings: 131_072,
|
||||
kv_bytes_per_token_per_card: 65_536,
|
||||
world_size: 2,
|
||||
};
|
||||
assert_eq!(
|
||||
derive_limit(&profile, 0, 0.0, None, &cfg).output_ceiling,
|
||||
65_536
|
||||
);
|
||||
}
|
||||
|
||||
/// The ceiling is deliberately *not* a function of live VRAM: it is
|
||||
/// read once and cached by clients, so it has to mean the same thing
|
||||
/// tomorrow. The live context still moves — that is what a request
|
||||
/// is judged against — but the contract does not.
|
||||
#[test]
|
||||
fn the_ceiling_holds_still_while_the_context_moves() {
|
||||
let cfg = ContextLimitConfig::default();
|
||||
let profile = ContextProfile {
|
||||
max_position_embeddings: 131_072,
|
||||
kv_bytes_per_token_per_card: 65_536,
|
||||
world_size: 2,
|
||||
};
|
||||
let roomy = derive_limit(&profile, 60_000, 5_000.0, None, &cfg);
|
||||
// Tight enough that the VRAM term binds — the same shape as the
|
||||
// fleet advertising 109568 one hour and 131072 the next.
|
||||
let tight = derive_limit(&profile, 6_000, 5_000.0, None, &cfg);
|
||||
assert!(
|
||||
tight.context < roomy.context,
|
||||
"the live window should shrink under VRAM pressure"
|
||||
);
|
||||
assert_eq!(
|
||||
tight.output_ceiling, roomy.output_ceiling,
|
||||
"the advertised ceiling must not move with fleet load"
|
||||
);
|
||||
}
|
||||
|
||||
/// beast Qwen3.6-27B: 16 full-attn layers, 4 kv heads, head_dim 256,
|
||||
/// f16 (2 B), TP=2 → 64 KiB/token total, 32 KiB/token/card.
|
||||
fn beast_profile() -> ContextProfile {
|
||||
|
||||
@@ -850,7 +850,7 @@ fn load_gguf_inner(
|
||||
tracing::info!(model = %model_id, path = ?gguf_path, "loading GGUF");
|
||||
let mut file = std::fs::File::open(gguf_path).context("open GGUF file")?;
|
||||
let content =
|
||||
gguf_file::Content::read(&mut file).map_err(|e| anyhow::anyhow!("parse GGUF: {e:#}"))?;
|
||||
gguf_file::Content::read(&mut file).map_err(|e| anyhow::anyhow!("parse GGUF: {e}"))?;
|
||||
|
||||
let architecture = content
|
||||
.metadata
|
||||
@@ -865,7 +865,7 @@ fn load_gguf_inner(
|
||||
match architecture.as_str() {
|
||||
"qwen3" => {
|
||||
let weights = QuantizedQwen3Weights::from_gguf(content, &mut file, device)
|
||||
.map_err(|e| anyhow::anyhow!("from_gguf qwen3: {e:#}"))?;
|
||||
.map_err(|e| anyhow::anyhow!("from_gguf qwen3: {e}"))?;
|
||||
Ok(ModelArch::Qwen3Quantized(weights))
|
||||
}
|
||||
"qwen3moe" => {
|
||||
@@ -874,12 +874,12 @@ fn load_gguf_inner(
|
||||
// accumulation precision and gives the best tokens/sec on
|
||||
// consumer cards.
|
||||
let weights = GGUFQWenMoE::from_gguf(content, &mut file, device, DType::F16)
|
||||
.map_err(|e| anyhow::anyhow!("from_gguf qwen3_moe: {e:#}"))?;
|
||||
.map_err(|e| anyhow::anyhow!("from_gguf qwen3_moe: {e}"))?;
|
||||
Ok(ModelArch::Qwen3MoeQuantized(weights))
|
||||
}
|
||||
"llama" => {
|
||||
let weights = QuantizedLlamaWeights::from_gguf(content, &mut file, device)
|
||||
.map_err(|e| anyhow::anyhow!("from_gguf llama: {e:#}"))?;
|
||||
.map_err(|e| anyhow::anyhow!("from_gguf llama: {e}"))?;
|
||||
Ok(ModelArch::LlamaQuantized(weights))
|
||||
}
|
||||
other => anyhow::bail!(
|
||||
@@ -943,14 +943,14 @@ fn load_dense_inner(
|
||||
let cfg: qwen3_dense::Config =
|
||||
serde_json::from_str(&cfg_text).context("parse Qwen3 config.json")?;
|
||||
let model = qwen3_dense::ModelForCausalLM::new(&cfg, vb)
|
||||
.map_err(|e| anyhow::anyhow!("build Qwen3 dense model: {e:#}"))?;
|
||||
.map_err(|e| anyhow::anyhow!("build Qwen3 dense model: {e}"))?;
|
||||
Ok(ModelArch::Qwen3Dense(model))
|
||||
}
|
||||
"qwen3_moe" => {
|
||||
let cfg: qwen3_moe_dense::Config =
|
||||
serde_json::from_str(&cfg_text).context("parse Qwen3 MoE config.json")?;
|
||||
let model = qwen3_moe_dense::ModelForCausalLM::new(&cfg, vb)
|
||||
.map_err(|e| anyhow::anyhow!("build Qwen3 MoE dense model: {e:#}"))?;
|
||||
.map_err(|e| anyhow::anyhow!("build Qwen3 MoE dense model: {e}"))?;
|
||||
Ok(ModelArch::Qwen3MoeDense(model))
|
||||
}
|
||||
"llama" => {
|
||||
@@ -960,7 +960,7 @@ fn load_dense_inner(
|
||||
let cache = llama_dense::Cache::new(true, dtype, &config, device)
|
||||
.context("build Llama Cache")?;
|
||||
let model = llama_dense::Llama::load(vb, &config)
|
||||
.map_err(|e| anyhow::anyhow!("build Llama dense model: {e:#}"))?;
|
||||
.map_err(|e| anyhow::anyhow!("build Llama dense model: {e}"))?;
|
||||
Ok(ModelArch::LlamaDense(Box::new(
|
||||
crate::harness::candle::LlamaDense::from_parts(
|
||||
model,
|
||||
|
||||
@@ -170,7 +170,7 @@ async fn op_step(
|
||||
(BackendConfig::Single { worker, handle, .. }, ActiveSession::Single { .. }) => worker
|
||||
.forward_logits_batch(*handle, tokens, prefix_lens, padded_len, step)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("batched decode step {step}: {e:#}")),
|
||||
.map_err(|e| anyhow::anyhow!("batched decode step {step}: {e}")),
|
||||
#[cfg(feature = "cuda")]
|
||||
(BackendConfig::Tp { .. }, ActiveSession::Tp { tp, pool }) => pool
|
||||
.generate_step_batch(
|
||||
@@ -182,7 +182,7 @@ async fn op_step(
|
||||
step,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("TP batched decode step {step}: {e:#}")),
|
||||
.map_err(|e| anyhow::anyhow!("TP batched decode step {step}: {e}")),
|
||||
#[cfg(feature = "cuda")]
|
||||
_ => anyhow::bail!("engine backend/session mismatch"),
|
||||
}
|
||||
@@ -204,13 +204,13 @@ async fn op_assemble(
|
||||
worker
|
||||
.assemble_kv_batch(*handle, seqs)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("assemble_kv_batch: {e:#}"))
|
||||
.map_err(|e| anyhow::anyhow!("assemble_kv_batch: {e}"))
|
||||
}
|
||||
#[cfg(feature = "cuda")]
|
||||
(BackendConfig::Tp { .. }, ActiveSession::Tp { tp, pool }) => pool
|
||||
.assemble_kv_batch(&tp.model_id, tp.leader_handle, seqs)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("TP assemble_kv_batch: {e:#}")),
|
||||
.map_err(|e| anyhow::anyhow!("TP assemble_kv_batch: {e}")),
|
||||
#[cfg(feature = "cuda")]
|
||||
_ => anyhow::bail!("engine backend/session mismatch"),
|
||||
}
|
||||
@@ -229,7 +229,7 @@ async fn op_extract(
|
||||
(BackendConfig::Single { worker, handle, .. }, ActiveSession::Single { .. }) => Ok(worker
|
||||
.extract_kv_rows(*handle, rows, padded_len, steps)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("extract_kv_rows: {e:#}"))?
|
||||
.map_err(|e| anyhow::anyhow!("extract_kv_rows: {e}"))?
|
||||
.into_iter()
|
||||
.map(|(id, _bytes)| id.0)
|
||||
.collect()),
|
||||
@@ -247,7 +247,7 @@ async fn op_extract(
|
||||
ids.clone(),
|
||||
)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("TP extract_kv_rows: {e:#}"))?;
|
||||
.map_err(|e| anyhow::anyhow!("TP extract_kv_rows: {e}"))?;
|
||||
Ok(ids)
|
||||
}
|
||||
#[cfg(feature = "cuda")]
|
||||
@@ -262,13 +262,13 @@ async fn op_snapshot_seq(cfg: &EngineConfig, session: &mut ActiveSession) -> Res
|
||||
.snapshot_kv(*handle)
|
||||
.await
|
||||
.map(|(id, _bytes)| id.0)
|
||||
.map_err(|e| anyhow::anyhow!("snapshot after prefill: {e:#}")),
|
||||
.map_err(|e| anyhow::anyhow!("snapshot after prefill: {e}")),
|
||||
#[cfg(feature = "cuda")]
|
||||
(BackendConfig::Tp { .. }, ActiveSession::Tp { tp, pool }) => {
|
||||
let id = tp.next_snapshot_id.fetch_add(1, Ordering::Relaxed);
|
||||
pool.snapshot_kv_cache(&tp.model_id, tp.leader_handle, id)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("TP snapshot after prefill: {e:#}"))?;
|
||||
.map_err(|e| anyhow::anyhow!("TP snapshot after prefill: {e}"))?;
|
||||
Ok(id)
|
||||
}
|
||||
#[cfg(feature = "cuda")]
|
||||
@@ -297,23 +297,14 @@ async fn op_drop_snap(cfg: &EngineConfig, session: &mut ActiveSession, id: u64)
|
||||
|
||||
/// Prefill one joining request at B=1 through the backend's existing
|
||||
/// chunked-prefill + prefix-cache paths. Returns the prefill logits.
|
||||
/// Prefill, returning the logits **and** how many leading tokens the
|
||||
/// prefix cache let us skip (#269).
|
||||
///
|
||||
/// The reuse count is computed here and was previously only logged, so
|
||||
/// no caller could report it — every client saw a 0% cache hit rate on
|
||||
/// a cache that was working. Returning it is what lets `usage` carry
|
||||
/// `cached_tokens`.
|
||||
async fn op_prefill(
|
||||
cfg: &EngineConfig,
|
||||
session: &mut ActiveSession,
|
||||
prompt_tokens: &[u32],
|
||||
) -> Result<(Vec<f32>, usize)> {
|
||||
) -> Result<Vec<f32>> {
|
||||
let prompt_len = prompt_tokens.len();
|
||||
let prefill_start = std::time::Instant::now();
|
||||
// Each arm yields its logits plus how many leading tokens the
|
||||
// prefix cache served, so neither can be forgotten.
|
||||
let (logits, cached_tokens) = match (&cfg.backend, session) {
|
||||
let logits = match (&cfg.backend, session) {
|
||||
(
|
||||
BackendConfig::Single {
|
||||
worker,
|
||||
@@ -349,7 +340,7 @@ async fn op_prefill(
|
||||
None => chunked_prefill_via_worker(worker, *handle, prompt_tokens, reused).await?,
|
||||
};
|
||||
prefill_rate.record(prompt_len, prefill_start.elapsed());
|
||||
(logits, reused)
|
||||
logits
|
||||
}
|
||||
#[cfg(feature = "cuda")]
|
||||
(BackendConfig::Tp { .. }, ActiveSession::Tp { tp, pool }) => {
|
||||
@@ -393,12 +384,12 @@ async fn op_prefill(
|
||||
}
|
||||
};
|
||||
tp.prefill_rate.record(prompt_len, prefill_start.elapsed());
|
||||
(logits, reused)
|
||||
logits
|
||||
}
|
||||
#[cfg(feature = "cuda")]
|
||||
_ => anyhow::bail!("engine backend/session mismatch"),
|
||||
};
|
||||
Ok((logits, cached_tokens))
|
||||
Ok(logits)
|
||||
}
|
||||
|
||||
/// One queued request. Admission has already been passed — the permit
|
||||
@@ -406,14 +397,9 @@ async fn op_prefill(
|
||||
pub struct EngineRequest {
|
||||
pub prompt_tokens: Vec<u32>,
|
||||
pub max_new: usize,
|
||||
/// Sampling resolved once from request → model `generation_config`
|
||||
/// → fallback (#272), rather than loose scalars each caller could
|
||||
/// default differently.
|
||||
pub sampling: crate::harness::sampling::SamplingParams,
|
||||
/// Reasoning-token budget for this request (#223). `None` is
|
||||
/// unbounded — the default, and the behaviour before it existed.
|
||||
/// Per-request because slots in one batch can differ.
|
||||
pub reasoning_budget: Option<usize>,
|
||||
pub temperature: f64,
|
||||
pub top_p: Option<f64>,
|
||||
pub seed: u64,
|
||||
pub eos_id: Option<u32>,
|
||||
pub tool_schemas: ToolSchemas,
|
||||
pub tx: mpsc::Sender<InferenceEvent>,
|
||||
@@ -453,8 +439,6 @@ enum RouterMsg {
|
||||
reason: FinishReason,
|
||||
prompt_tokens: u32,
|
||||
completion_tokens: u32,
|
||||
/// Leading prompt tokens the prefix cache let us skip (#269).
|
||||
cached_tokens: u32,
|
||||
timing: FinishTiming,
|
||||
},
|
||||
}
|
||||
@@ -471,12 +455,6 @@ struct Slot {
|
||||
max_new: usize,
|
||||
eos_id: Option<u32>,
|
||||
lp: LogitsProcessor,
|
||||
/// The slot's resolved sampling (#272). Held per-slot because the
|
||||
/// repetition penalty and its window are per-request knobs, and
|
||||
/// slots in one batch can carry different ones.
|
||||
sampling: crate::harness::sampling::SamplingParams,
|
||||
/// Bounds this slot's think block (#223); `None` = unbounded.
|
||||
governor: Option<crate::harness::reasoning_budget::ReasoningGovernor>,
|
||||
router: mpsc::Sender<RouterMsg>,
|
||||
/// Set by the router when the consumer hangs up; the engine stops
|
||||
/// feeding the slot and compacts it out.
|
||||
@@ -484,9 +462,6 @@ struct Slot {
|
||||
finished: Option<FinishReason>,
|
||||
prefill_ms: u32,
|
||||
prefill_tokens: u32,
|
||||
/// Prompt tokens served from the prefix cache (#269), reported to
|
||||
/// the caller as `cached_tokens` so a working cache is visible.
|
||||
cached_tokens: u32,
|
||||
decode_start: std::time::Instant,
|
||||
_admit: AdmissionPermit,
|
||||
}
|
||||
@@ -584,13 +559,7 @@ async fn run_engine(cfg: EngineConfig, mut rx: mpsc::Receiver<EngineRequest>) {
|
||||
break;
|
||||
}
|
||||
};
|
||||
let nt = match sample_with_penalty(
|
||||
&logits,
|
||||
&slot.generated,
|
||||
&mut slot.lp,
|
||||
&slot.sampling,
|
||||
slot.governor.as_mut(),
|
||||
) {
|
||||
let nt = match sample_with_penalty(&logits, &slot.generated, &mut slot.lp) {
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
let health = logits_health_slice(&logits_vec);
|
||||
@@ -658,7 +627,6 @@ async fn finish_slot(slot: &mut Slot, reason: FinishReason, rate: &PrefillRateEm
|
||||
reason,
|
||||
prompt_tokens: slot.prompt_len as u32,
|
||||
completion_tokens: slot.generated.len() as u32,
|
||||
cached_tokens: slot.cached_tokens,
|
||||
timing: FinishTiming {
|
||||
prefill_ms: slot.prefill_ms,
|
||||
decode_ms: slot.decode_start.elapsed().as_millis() as u32,
|
||||
@@ -791,11 +759,14 @@ async fn prefill_join(
|
||||
session: &mut ActiveSession,
|
||||
req: EngineRequest,
|
||||
) -> Result<Option<(Slot, u64)>> {
|
||||
use candle_transformers::generation::Sampling;
|
||||
|
||||
let EngineRequest {
|
||||
prompt_tokens,
|
||||
max_new,
|
||||
sampling,
|
||||
reasoning_budget,
|
||||
temperature,
|
||||
top_p,
|
||||
seed,
|
||||
eos_id,
|
||||
tool_schemas,
|
||||
tx,
|
||||
@@ -803,37 +774,37 @@ async fn prefill_join(
|
||||
span,
|
||||
} = req;
|
||||
|
||||
let mut lp = LogitsProcessor::from_sampling(sampling.seed, sampling.to_sampling());
|
||||
// Built before prefill's first sample so a prompt that opens the
|
||||
// think block is governed from its very first token (#223).
|
||||
let mut governor = crate::harness::reasoning_budget::ReasoningGovernor::new(
|
||||
reasoning_budget,
|
||||
cfg.reasoning_tokens
|
||||
.as_ref()
|
||||
.map(|p| (p.open_id, p.close_id)),
|
||||
prompt_opens_reasoning(&prompt_tokens, cfg.reasoning_tokens.as_ref()),
|
||||
);
|
||||
let mut lp = {
|
||||
let sampling = if temperature <= 0.0 {
|
||||
Sampling::ArgMax
|
||||
} else {
|
||||
match top_p {
|
||||
Some(p) => Sampling::TopP { p, temperature },
|
||||
None => Sampling::All { temperature },
|
||||
}
|
||||
};
|
||||
LogitsProcessor::from_sampling(seed, sampling)
|
||||
};
|
||||
|
||||
let prompt_len = prompt_tokens.len();
|
||||
let prefill_start = std::time::Instant::now();
|
||||
let (logits_vec, cached_tokens) = op_prefill(cfg, session, &prompt_tokens).await?;
|
||||
let logits_vec = op_prefill(cfg, session, &prompt_tokens).await?;
|
||||
let prefill_elapsed = prefill_start.elapsed();
|
||||
|
||||
// First token from the prefill logits.
|
||||
let generated: Vec<u32> = Vec::new();
|
||||
let logits = Tensor::new(logits_vec.as_slice(), &Device::Cpu)?;
|
||||
let first =
|
||||
match sample_with_penalty(&logits, &generated, &mut lp, &sampling, governor.as_mut()) {
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
let health = logits_health_slice(&logits_vec);
|
||||
tracing::warn!(
|
||||
?health,
|
||||
"batch engine: prefill sample failed; logits unhealthy"
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
let first = match sample_with_penalty(&logits, &generated, &mut lp) {
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
let health = logits_health_slice(&logits_vec);
|
||||
tracing::warn!(
|
||||
?health,
|
||||
"batch engine: prefill sample failed; logits unhealthy"
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
|
||||
// Router task for this slot.
|
||||
let hangup = Arc::new(AtomicBool::new(false));
|
||||
@@ -856,14 +827,11 @@ async fn prefill_join(
|
||||
let mut slot = Slot {
|
||||
prefix_len: prompt_len,
|
||||
prompt_len,
|
||||
cached_tokens: cached_tokens as u32,
|
||||
generated,
|
||||
next_token: first,
|
||||
max_new,
|
||||
eos_id,
|
||||
lp,
|
||||
sampling,
|
||||
governor,
|
||||
router: router_tx,
|
||||
hangup,
|
||||
finished: None,
|
||||
@@ -922,9 +890,6 @@ async fn run_router(
|
||||
let mut tool_call_idx: usize = 0;
|
||||
let mut emitted_tool_call = false;
|
||||
let mut consumer_alive = true;
|
||||
// Whether a Finish was seen, so the fall-through below can tell a
|
||||
// clean end from a dropped slot.
|
||||
let mut finished = false;
|
||||
|
||||
while let Some(msg) = rx.recv().await {
|
||||
match msg {
|
||||
@@ -1013,43 +978,17 @@ async fn run_router(
|
||||
mut reason,
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
cached_tokens,
|
||||
timing,
|
||||
} => {
|
||||
if emitted_tool_call && reason == FinishReason::Stop {
|
||||
reason = FinishReason::ToolCalls;
|
||||
}
|
||||
// Close the journal trail for this turn (#268). The
|
||||
// non-streaming path has always logged this; streamed
|
||||
// turns logged nothing at all, so whether one finished,
|
||||
// hit its output cap, or was abandoned mid-flight was
|
||||
// unanswerable afterwards — and an absence of lines was
|
||||
// repeatedly misread as an absence of work.
|
||||
//
|
||||
// `run_router` is instrumented with the request's span,
|
||||
// so req_id and model come along without being passed.
|
||||
tracing::info!(
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
reasoning_tokens = reasoning_token_count,
|
||||
cached_tokens,
|
||||
finish_reason = ?reason,
|
||||
prefill_ms = timing.prefill_ms,
|
||||
decode_ms = timing.decode_ms,
|
||||
// The turn ran to completion with nobody reading it:
|
||||
// it held its admission permit and the pool lock the
|
||||
// whole time. Invisible until now.
|
||||
client_gone = !consumer_alive,
|
||||
"chat_completion (stream): done"
|
||||
);
|
||||
finished = true;
|
||||
let _ = tx
|
||||
.send(InferenceEvent::Finish {
|
||||
reason,
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
reasoning_tokens: reasoning_token_count,
|
||||
cached_tokens,
|
||||
timing: Some(timing),
|
||||
})
|
||||
.await;
|
||||
@@ -1057,17 +996,6 @@ async fn run_router(
|
||||
}
|
||||
}
|
||||
}
|
||||
// Fell out of the loop without a Finish: the engine dropped this
|
||||
// slot's channel (fatal worker error, shutdown). The client's stream
|
||||
// ends either way, but silently — so say so rather than leaving the
|
||||
// turn's last trace to be a prefill line (#268).
|
||||
if !finished {
|
||||
tracing::warn!(
|
||||
reasoning_tokens = reasoning_token_count,
|
||||
client_gone = !consumer_alive,
|
||||
"chat_completion (stream): ended without a finish event — engine dropped the slot"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// `tracing::Instrument` without importing the trait at every use
|
||||
@@ -1085,92 +1013,6 @@ mod tests {
|
||||
use crate::config::AdmissionConfig;
|
||||
use crate::harness::admission::AdmissionController;
|
||||
|
||||
/// A streamed turn must close its own journal trail (#268).
|
||||
///
|
||||
/// Streaming used to end silently: no token counts, no
|
||||
/// `finish_reason`, no duration. That made a finished turn, a turn
|
||||
/// that hit its output cap, and a turn abandoned by a disconnected
|
||||
/// client indistinguishable after the fact — and during the #267
|
||||
/// investigation an absence of log lines was twice mistaken for an
|
||||
/// absence of work on a daemon that was healthy and decoding.
|
||||
///
|
||||
/// Asserts the emitted line rather than the struct, because a
|
||||
/// `Finish` event that reaches the client but leaves no server-side
|
||||
/// record is exactly the state this fixes.
|
||||
#[tokio::test]
|
||||
async fn a_streamed_turn_logs_how_it_ended() {
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tracing_subscriber::prelude::*;
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct Capture(Arc<Mutex<Vec<u8>>>);
|
||||
impl std::io::Write for Capture {
|
||||
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
||||
self.0
|
||||
.lock()
|
||||
.expect("capture poisoned")
|
||||
.extend_from_slice(buf);
|
||||
Ok(buf.len())
|
||||
}
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for Capture {
|
||||
type Writer = Self;
|
||||
fn make_writer(&'a self) -> Self::Writer {
|
||||
self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
let capture = Capture::default();
|
||||
let subscriber = tracing_subscriber::registry()
|
||||
.with(tracing_subscriber::fmt::layer().with_writer(capture.clone()));
|
||||
let _guard = tracing::subscriber::set_default(subscriber);
|
||||
|
||||
let (router_tx, router_rx) = mpsc::channel::<RouterMsg>(8);
|
||||
let (tx, mut rx) = mpsc::channel::<InferenceEvent>(8);
|
||||
let handle = tokio::spawn(run_router(
|
||||
tiny_tokenizer(8),
|
||||
None,
|
||||
None,
|
||||
ToolSchemas::default(),
|
||||
false,
|
||||
tx,
|
||||
Arc::new(AtomicBool::new(false)),
|
||||
router_rx,
|
||||
));
|
||||
|
||||
router_tx
|
||||
.send(RouterMsg::Finish {
|
||||
reason: FinishReason::Length,
|
||||
prompt_tokens: 7413,
|
||||
completion_tokens: 24576,
|
||||
cached_tokens: 2068,
|
||||
timing: FinishTiming {
|
||||
prefill_ms: 6000,
|
||||
decode_ms: 900_000,
|
||||
prefill_tokens: 7413,
|
||||
},
|
||||
})
|
||||
.await
|
||||
.expect("router accepts finish");
|
||||
drop(router_tx);
|
||||
handle.await.expect("router task joins");
|
||||
while rx.recv().await.is_some() {}
|
||||
|
||||
let logged = String::from_utf8(capture.0.lock().expect("capture poisoned").clone())
|
||||
.expect("utf8 log output");
|
||||
assert!(
|
||||
logged.contains("chat_completion (stream): done"),
|
||||
"streamed turn left no completion line:\n{logged}"
|
||||
);
|
||||
// The counts are the point — "it ended" without them cannot
|
||||
// distinguish a complete answer from an exhausted budget.
|
||||
assert!(logged.contains("24576"), "no completion_tokens:\n{logged}");
|
||||
assert!(logged.contains("Length"), "no finish_reason:\n{logged}");
|
||||
}
|
||||
|
||||
/// A WordLevel tokenizer whose vocab covers the whole fixture
|
||||
/// vocab (`w0`..`w511`), so every decoded token maps to a unique
|
||||
/// word and the emitted text uniquely encodes the token sequence.
|
||||
@@ -1204,20 +1046,9 @@ mod tests {
|
||||
.submit(EngineRequest {
|
||||
prompt_tokens: prompt,
|
||||
max_new,
|
||||
// Greedy, deterministic: the fixture asserts exact
|
||||
// token sequences.
|
||||
sampling: crate::harness::sampling::SamplingParams::resolve(
|
||||
&crate::harness::sampling::RequestedSampling {
|
||||
temperature: Some(0.0),
|
||||
seed: Some(0),
|
||||
..Default::default()
|
||||
},
|
||||
&crate::harness::sampling::ModelGenerationDefaults::default(),
|
||||
0,
|
||||
),
|
||||
// Unbounded: this fixture asserts exact token sequences,
|
||||
// so nothing may substitute a close marker (#223).
|
||||
reasoning_budget: None,
|
||||
temperature: 0.0, // greedy — deterministic
|
||||
top_p: None,
|
||||
seed: 0,
|
||||
eos_id: None,
|
||||
tool_schemas: ToolSchemas::new(),
|
||||
tx,
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
pub mod admission;
|
||||
pub mod arch;
|
||||
pub mod candle;
|
||||
pub mod capability;
|
||||
pub mod chat_template;
|
||||
pub mod context_limit;
|
||||
pub mod device_worker;
|
||||
@@ -12,9 +11,6 @@ pub mod image;
|
||||
pub mod prefix_cache;
|
||||
pub mod preflight;
|
||||
pub mod preprocess;
|
||||
pub mod reasoning_budget;
|
||||
pub mod reasoning_effort;
|
||||
pub mod sampling;
|
||||
pub mod speculative;
|
||||
pub mod tp;
|
||||
|
||||
|
||||
@@ -248,27 +248,15 @@ pub async fn preflight(
|
||||
/// in `siblings[].rfilename` — or `None` when the repo has no usable
|
||||
/// snapshot (never cached, or the ref/snapshot is missing).
|
||||
pub fn cached_snapshot_files(cache_path: &std::path::Path, repo_path: &str) -> Option<Vec<String>> {
|
||||
let snapshot = cached_snapshot_dir(cache_path, repo_path)?;
|
||||
let repo_dir = cache_path.join(format!("models--{}", repo_path.replace('/', "--")));
|
||||
let commit = std::fs::read_to_string(repo_dir.join("refs").join("main")).ok()?;
|
||||
let snapshot = repo_dir.join("snapshots").join(commit.trim());
|
||||
let mut files = Vec::new();
|
||||
collect_snapshot_files(&snapshot, &snapshot, &mut files);
|
||||
files.sort();
|
||||
if files.is_empty() { None } else { Some(files) }
|
||||
}
|
||||
|
||||
/// Directory holding a repo's cached snapshot, or `None` when the repo
|
||||
/// was never cached. Split out of [`cached_snapshot_files`] so callers
|
||||
/// that need to read a specific file (rather than list them all) can
|
||||
/// resolve the same path without duplicating the layout knowledge.
|
||||
pub fn cached_snapshot_dir(
|
||||
cache_path: &std::path::Path,
|
||||
repo_path: &str,
|
||||
) -> Option<std::path::PathBuf> {
|
||||
let repo_dir = cache_path.join(format!("models--{}", repo_path.replace('/', "--")));
|
||||
let commit = std::fs::read_to_string(repo_dir.join("refs").join("main")).ok()?;
|
||||
let snapshot = repo_dir.join("snapshots").join(commit.trim());
|
||||
snapshot.is_dir().then_some(snapshot)
|
||||
}
|
||||
|
||||
/// Recursive walk of a snapshot directory. Snapshot entries are
|
||||
/// symlinks into `blobs/`; `is_dir`/`exists` follow them, so a symlink
|
||||
/// whose blob was pruned is skipped rather than reported as present.
|
||||
@@ -401,8 +389,6 @@ mod tests {
|
||||
quant: quant.map(String::from),
|
||||
tensor_parallel: tp,
|
||||
devices: None,
|
||||
sampling: None,
|
||||
preserve_thinking: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,305 +0,0 @@
|
||||
//! Bounding how long a model may think before it must answer (#223).
|
||||
//!
|
||||
//! A reasoning model with no stopping criterion will fill whatever
|
||||
//! output budget it is given. Measured on Qwen3.8-27B: given 16,384
|
||||
//! tokens it used 16,384; given 32,768 it used 32,768; both times it was
|
||||
//! still opening new topics when the cap bit, and both times the caller
|
||||
//! received no answer at all.
|
||||
//!
|
||||
//! The fix is not to stop the model thinking — the run that did produce
|
||||
//! files designed for 27,157 tokens first, and that design was good. It
|
||||
//! is to bound the thinking so the rest of the output budget survives
|
||||
//! for the answer.
|
||||
//!
|
||||
//! Two halves live here:
|
||||
//!
|
||||
//! - [`ReasoningBudget`] — how many reasoning tokens this request may
|
||||
//! spend, resolved from whichever control the caller's wire format
|
||||
//! gave them.
|
||||
//! - [`ReasoningGovernor`] — the per-request state that enforces it, by
|
||||
//! substituting the model's own `</think>` token when the budget runs
|
||||
//! out so generation transitions to answering instead of being cut off.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Per-model mapping from an OpenAI effort level to a token budget.
|
||||
///
|
||||
/// OpenAI-shaped clients cannot send a token count — pi-ai's vocabulary
|
||||
/// is the ladder `off | minimal | low | medium | high | xhigh | max` and
|
||||
/// nothing else — so the numbers have to live on the server. Operator
|
||||
/// configurable, because the right values depend on the model and the
|
||||
/// hardware; a derived default would be a guess baked into the binary.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ReasoningBudgetLadder {
|
||||
pub minimal: usize,
|
||||
pub low: usize,
|
||||
pub medium: usize,
|
||||
pub high: usize,
|
||||
/// `xhigh` and `max` are only offered by a client when the model
|
||||
/// declares them; absent here means "not offered", and a request
|
||||
/// naming one falls back to `high` rather than being rejected.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub xhigh: Option<usize>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max: Option<usize>,
|
||||
/// Tokens of the caller's **own** output budget held back for the
|
||||
/// answer, when the caller names no explicit `reasoning.max_tokens`.
|
||||
///
|
||||
/// This is what an OpenAI-shaped client gets instead of the rungs
|
||||
/// above, and it is deliberately not derived from effort. Effort is
|
||||
/// a prompt instruction the model is trained to follow (#290);
|
||||
/// making it *also* set a guillotine is what told Qwen3.8-27B to
|
||||
/// reason at `xhigh` and then cut it off at `medium`'s 12,288.
|
||||
///
|
||||
/// A reserve has no such double meaning. The model thinks as long
|
||||
/// as it likes within the budget the caller declared, minus enough
|
||||
/// to answer — so the caller always gets an answer, and the size of
|
||||
/// the think block stays the model's decision rather than ours.
|
||||
///
|
||||
/// Set to `0` to disable, restoring fully unbounded reasoning.
|
||||
#[serde(default = "default_answer_reserve")]
|
||||
pub answer_reserve_tokens: usize,
|
||||
}
|
||||
|
||||
/// Enough for a substantive answer with tool calls, not so much that it
|
||||
/// meaningfully shortens a think block: at the 32,768 budget pi sends,
|
||||
/// this leaves ~28.7 k for reasoning — above the ~27.2 k a full agentic
|
||||
/// design turn was measured to use.
|
||||
fn default_answer_reserve() -> usize {
|
||||
4_096
|
||||
}
|
||||
|
||||
impl Default for ReasoningBudgetLadder {
|
||||
/// Rungs sized against observed behaviour rather than round numbers:
|
||||
/// a long agentic design turn on this class of model runs to ~27k
|
||||
/// tokens, so `high` sits above that (bounded but rarely binding),
|
||||
/// `medium` at roughly half (enough to plan a whole implementation),
|
||||
/// and `low` at the scale of a focused single-file change.
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
minimal: 1_024,
|
||||
low: 4_096,
|
||||
medium: 12_288,
|
||||
high: 32_768,
|
||||
xhigh: None,
|
||||
max: None,
|
||||
answer_reserve_tokens: default_answer_reserve(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ReasoningBudgetLadder {
|
||||
/// Resolve an effort level to a token budget.
|
||||
///
|
||||
/// `none`/`off` are handled before this — they disable thinking
|
||||
/// outright via the chat template — so they have no budget here.
|
||||
/// An unrecognised level resolves to `None` (unbounded) rather than
|
||||
/// to a guess: silently substituting a budget nobody asked for is
|
||||
/// the failure mode this whole area has been full of.
|
||||
pub fn for_effort(&self, effort: &str) -> Option<usize> {
|
||||
match effort.trim().to_ascii_lowercase().as_str() {
|
||||
"minimal" => Some(self.minimal),
|
||||
"low" => Some(self.low),
|
||||
"medium" => Some(self.medium),
|
||||
"high" => Some(self.high),
|
||||
"xhigh" => Some(self.xhigh.unwrap_or(self.high)),
|
||||
"max" => Some(self.max.or(self.xhigh).unwrap_or(self.high)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The mapping as a client should see it on `/v1/models` — the rungs
|
||||
/// this deployment actually offers, so a caller picking `low` knows
|
||||
/// what it bought instead of guessing (#274).
|
||||
pub fn advertised(&self) -> Vec<(String, usize)> {
|
||||
let mut out = vec![
|
||||
("minimal".to_string(), self.minimal),
|
||||
("low".to_string(), self.low),
|
||||
("medium".to_string(), self.medium),
|
||||
("high".to_string(), self.high),
|
||||
];
|
||||
if let Some(v) = self.xhigh {
|
||||
out.push(("xhigh".to_string(), v));
|
||||
}
|
||||
if let Some(v) = self.max {
|
||||
out.push(("max".to_string(), v));
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
/// Enforces a resolved budget over one request's token stream.
|
||||
///
|
||||
/// Fed every sampled token; returns the token that should actually be
|
||||
/// emitted. While the model is inside a think block and over budget, it
|
||||
/// returns the close marker instead, exactly once — after which the
|
||||
/// model is answering and the governor is inert.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ReasoningGovernor {
|
||||
limit: usize,
|
||||
open_id: u32,
|
||||
close_id: u32,
|
||||
inside: bool,
|
||||
used: usize,
|
||||
forced: bool,
|
||||
}
|
||||
|
||||
impl ReasoningGovernor {
|
||||
/// `None` when the request named no budget, or the model has no
|
||||
/// reasoning markers to enforce one with — in both cases generation
|
||||
/// proceeds exactly as it did before this existed.
|
||||
pub fn new(
|
||||
limit: Option<usize>,
|
||||
markers: Option<(u32, u32)>,
|
||||
prompt_opened_reasoning: bool,
|
||||
) -> Option<Self> {
|
||||
let limit = limit?;
|
||||
let (open_id, close_id) = markers?;
|
||||
Some(Self {
|
||||
limit,
|
||||
open_id,
|
||||
close_id,
|
||||
inside: prompt_opened_reasoning,
|
||||
used: 0,
|
||||
forced: false,
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether the forced close has already been spent.
|
||||
pub fn forced(&self) -> bool {
|
||||
self.forced
|
||||
}
|
||||
|
||||
/// Reasoning tokens counted so far.
|
||||
pub fn used(&self) -> usize {
|
||||
self.used
|
||||
}
|
||||
|
||||
/// Govern one sampled token, returning the token to emit.
|
||||
pub fn govern(&mut self, sampled: u32) -> u32 {
|
||||
if sampled == self.open_id {
|
||||
self.inside = true;
|
||||
return sampled;
|
||||
}
|
||||
if sampled == self.close_id {
|
||||
self.inside = false;
|
||||
return sampled;
|
||||
}
|
||||
if !self.inside {
|
||||
return sampled;
|
||||
}
|
||||
if self.used >= self.limit {
|
||||
// Budget spent: hand the model its own close marker so it
|
||||
// transitions to answering. Cutting the stream here instead
|
||||
// would reproduce the very failure this exists to prevent.
|
||||
self.inside = false;
|
||||
self.forced = true;
|
||||
return self.close_id;
|
||||
}
|
||||
self.used += 1;
|
||||
sampled
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const OPEN: u32 = 100;
|
||||
const CLOSE: u32 = 101;
|
||||
|
||||
fn governor(limit: usize, opened: bool) -> ReasoningGovernor {
|
||||
ReasoningGovernor::new(Some(limit), Some((OPEN, CLOSE)), opened).expect("governor")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_request_with_no_budget_is_ungoverned() {
|
||||
assert!(ReasoningGovernor::new(None, Some((OPEN, CLOSE)), true).is_none());
|
||||
}
|
||||
|
||||
/// A model with no detected markers cannot be governed — better
|
||||
/// unbounded than truncated at an arbitrary token.
|
||||
#[test]
|
||||
fn a_model_without_markers_is_ungoverned() {
|
||||
assert!(ReasoningGovernor::new(Some(10), None, true).is_none());
|
||||
}
|
||||
|
||||
/// The case that motivated this: the chat template left the think
|
||||
/// block open, so the model is reasoning from the first token.
|
||||
#[test]
|
||||
fn a_prompt_opened_block_is_bounded_and_force_closed() {
|
||||
let mut g = governor(3, true);
|
||||
assert_eq!(g.govern(7), 7);
|
||||
assert_eq!(g.govern(8), 8);
|
||||
assert_eq!(g.govern(9), 9);
|
||||
assert_eq!(g.govern(10), CLOSE, "the fourth token must close the block");
|
||||
assert!(g.forced());
|
||||
assert_eq!(g.used(), 3);
|
||||
// Answering now — the governor is inert.
|
||||
assert_eq!(g.govern(11), 11);
|
||||
assert_eq!(g.govern(12), 12);
|
||||
}
|
||||
|
||||
/// Content before the block opens is never counted against the
|
||||
/// reasoning budget.
|
||||
#[test]
|
||||
fn only_tokens_inside_the_block_are_counted() {
|
||||
let mut g = governor(2, false);
|
||||
assert_eq!(g.govern(5), 5);
|
||||
assert_eq!(g.govern(6), 6);
|
||||
assert_eq!(g.used(), 0);
|
||||
assert_eq!(g.govern(OPEN), OPEN);
|
||||
assert_eq!(g.govern(7), 7);
|
||||
assert_eq!(g.govern(8), 8);
|
||||
assert_eq!(g.govern(9), CLOSE);
|
||||
assert_eq!(g.used(), 2);
|
||||
}
|
||||
|
||||
/// A model that finishes thinking on its own is untouched — the
|
||||
/// budget is a ceiling, not a target.
|
||||
#[test]
|
||||
fn a_model_that_stops_on_its_own_is_not_forced() {
|
||||
let mut g = governor(100, true);
|
||||
for t in 20..30 {
|
||||
assert_eq!(g.govern(t), t);
|
||||
}
|
||||
assert_eq!(g.govern(CLOSE), CLOSE);
|
||||
assert!(!g.forced());
|
||||
for t in 40..50 {
|
||||
assert_eq!(g.govern(t), t);
|
||||
}
|
||||
assert!(!g.forced(), "content after the block must not be governed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_ladder_resolves_the_levels_clients_can_send() {
|
||||
let l = ReasoningBudgetLadder::default();
|
||||
assert_eq!(l.for_effort("minimal"), Some(1_024));
|
||||
assert_eq!(l.for_effort("LOW"), Some(4_096));
|
||||
assert_eq!(l.for_effort(" medium "), Some(12_288));
|
||||
assert_eq!(l.for_effort("high"), Some(32_768));
|
||||
// Not offered by default → falls back to high rather than
|
||||
// rejecting a level the client was told it could send.
|
||||
assert_eq!(l.for_effort("xhigh"), Some(32_768));
|
||||
assert_eq!(l.for_effort("max"), Some(32_768));
|
||||
// Unknown → unbounded, never a substituted guess.
|
||||
assert_eq!(l.for_effort("enthusiastic"), None);
|
||||
assert_eq!(l.for_effort("none"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_advertised_ladder_lists_only_what_is_offered() {
|
||||
let l = ReasoningBudgetLadder::default();
|
||||
let names: Vec<_> = l.advertised().into_iter().map(|(k, _)| k).collect();
|
||||
assert_eq!(names, ["minimal", "low", "medium", "high"]);
|
||||
let l2 = ReasoningBudgetLadder {
|
||||
xhigh: Some(49_152),
|
||||
..ReasoningBudgetLadder::default()
|
||||
};
|
||||
assert!(
|
||||
l2.advertised()
|
||||
.iter()
|
||||
.any(|(k, v)| k == "xhigh" && *v == 49_152)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,302 +0,0 @@
|
||||
//! Which reasoning-effort rungs a model actually supports, discovered
|
||||
//! from the model itself (#290).
|
||||
//!
|
||||
//! ## Why this is discovered rather than configured
|
||||
//!
|
||||
//! `Qwen/Qwen3.8-27B` accepts exactly three efforts and hard-fails on
|
||||
//! anything else:
|
||||
//!
|
||||
//! ```jinja
|
||||
//! {%- set resolved_reasoning_effort = reasoning_effort|default('xhigh') %}
|
||||
//! {%- if resolved_reasoning_effort not in ('xhigh', 'medium', 'low') %}
|
||||
//! {{- raise_exception('Unexpected reasoning effort ...') }}
|
||||
//! ```
|
||||
//!
|
||||
//! We previously advertised an invented ladder — `minimal`, `low`,
|
||||
//! `medium`, `high` — of which two values the template rejects outright
|
||||
//! and which omits `xhigh`, the model's own default and strongest rung.
|
||||
//!
|
||||
//! A caution about *why* that mattered, because the obvious story is
|
||||
//! wrong. It is tempting to say clients read our advertisement and were
|
||||
//! misled by it. pi-ai does not: its `getSupportedThinkingLevels` reads
|
||||
//! `model.thinkingLevelMap`, a static field in the operator's
|
||||
//! `models.json`, and treats `xhigh`/`max` as unavailable unless that
|
||||
//! map names them explicitly. So a pi user saw exactly
|
||||
//! `minimal, low, medium, high` — the invented ladder, to the value —
|
||||
//! for a reason that had nothing to do with what we published.
|
||||
//!
|
||||
//! The coincidence is the trap: our wrong advertisement and pi's
|
||||
//! default produced the same symptom, which made the advertisement look
|
||||
//! causal. Publishing rungs the model actually accepts is still correct
|
||||
//! — it is the truth, other clients may read it, and it is what makes
|
||||
//! `apply_effort_kwarg` right — but do not assume a given client
|
||||
//! consumes it. For pi the operator must also set `thinkingLevelMap`,
|
||||
//! by hand, and nothing validates that they match.
|
||||
//!
|
||||
//! A hardcoded per-model table would drift the moment a model ships a
|
||||
//! new template, which is exactly the class of defect #280 fixed by
|
||||
//! fetching the template instead of assuming it. So the rungs come from
|
||||
//! the artifact.
|
||||
//!
|
||||
//! ## How the probe works
|
||||
//!
|
||||
//! `raise_exception` is already wired to a render error, so the template
|
||||
//! answers the question directly: render it once per candidate effort
|
||||
//! and see which ones survive.
|
||||
//!
|
||||
//! Two ambiguities the probe has to resolve, or it would report
|
||||
//! confident nonsense:
|
||||
//!
|
||||
//! 1. **A template that ignores `reasoning_effort` entirely** renders
|
||||
//! successfully for every candidate. Supporting "all six" would be a
|
||||
//! lie. So a candidate only counts when the rendered prompt actually
|
||||
//! *differs* across efforts — if every render is identical the model
|
||||
//! has no effort control and we advertise none.
|
||||
//! 2. **The default** is whichever supported candidate renders
|
||||
//! identically to a render with no effort supplied. Qwen3.8 defaults
|
||||
//! to `xhigh`; without detecting that, a caller who says nothing gets
|
||||
//! the strongest rung while we believe it got nothing.
|
||||
|
||||
use anyhow::Result;
|
||||
use cortex_core::openai::{ChatMessage, MessageContent};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
/// Effort names in ascending order of thinking, as clients spell them.
|
||||
///
|
||||
/// Deliberately a superset of what any one model supports: this is the
|
||||
/// *candidate* set the probe tries, and the conventional vocabulary
|
||||
/// OpenAI-compatible clients use (pi-ai's `EXTENDED_THINKING_LEVELS`
|
||||
/// minus `off`, which is not an effort but a separate switch handled by
|
||||
/// `enable_thinking`).
|
||||
pub const CANDIDATE_EFFORTS: &[&str] = &["minimal", "low", "medium", "high", "xhigh", "max"];
|
||||
|
||||
/// What one model's template accepts.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct SupportedEfforts {
|
||||
/// Accepted efforts, ascending, as the template spells them.
|
||||
pub levels: Vec<String>,
|
||||
/// The effort applied when the caller names none — the template's
|
||||
/// own `|default(...)`. `None` when it cannot be determined.
|
||||
pub default: Option<String>,
|
||||
}
|
||||
|
||||
impl SupportedEfforts {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.levels.is_empty()
|
||||
}
|
||||
|
||||
pub fn supports(&self, effort: &str) -> bool {
|
||||
self.levels.iter().any(|l| l == effort)
|
||||
}
|
||||
|
||||
/// Map a caller's effort onto the nearest rung this model has.
|
||||
///
|
||||
/// Exact match wins. Otherwise walk **down** from the requested
|
||||
/// level first, then up — a caller asking for more thinking than the
|
||||
/// model offers should get the most it has, and a caller asking for
|
||||
/// less should not be silently upgraded into a slower, costlier
|
||||
/// response than they asked for.
|
||||
///
|
||||
/// Descending-first is the opposite of pi-ai's `clampThinkingLevel`,
|
||||
/// which searches upward first. That is deliberate: pi is choosing
|
||||
/// among levels it believes exist, whereas we are the ones who know
|
||||
/// what exists, and over-delivering thinking is the failure that
|
||||
/// costs the caller money and latency.
|
||||
///
|
||||
/// `None` when the model has no rungs at all.
|
||||
pub fn nearest(&self, requested: &str) -> Option<&str> {
|
||||
if self.levels.is_empty() {
|
||||
return None;
|
||||
}
|
||||
if let Some(exact) = self.levels.iter().find(|l| *l == requested) {
|
||||
return Some(exact.as_str());
|
||||
}
|
||||
let idx = CANDIDATE_EFFORTS.iter().position(|c| *c == requested)?;
|
||||
for c in CANDIDATE_EFFORTS[..idx].iter().rev() {
|
||||
if let Some(hit) = self.levels.iter().find(|l| l.as_str() == *c) {
|
||||
return Some(hit.as_str());
|
||||
}
|
||||
}
|
||||
for c in &CANDIDATE_EFFORTS[idx + 1..] {
|
||||
if let Some(hit) = self.levels.iter().find(|l| l.as_str() == *c) {
|
||||
return Some(hit.as_str());
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Render a probe prompt at `effort`, or `None` if the template rejects
|
||||
/// it.
|
||||
fn probe_render(template: &str, effort: Option<&str>) -> Option<String> {
|
||||
let messages = [ChatMessage {
|
||||
role: "user".into(),
|
||||
content: MessageContent::Text("probe".into()),
|
||||
extra: Value::Null,
|
||||
}];
|
||||
let kwargs = match effort {
|
||||
Some(e) => json!({ "reasoning_effort": e }),
|
||||
None => Value::Null,
|
||||
};
|
||||
super::chat_template::render_chat_template(template, &messages, &Value::Null, &kwargs).ok()
|
||||
}
|
||||
|
||||
/// Discover which efforts `template` accepts (#290).
|
||||
///
|
||||
/// Errors are not propagated: a template that cannot render even the
|
||||
/// baseline probe yields no rungs, and the caller behaves as it did
|
||||
/// before this existed.
|
||||
pub fn probe(template: &str, model_id: &str) -> Result<SupportedEfforts> {
|
||||
let Some(baseline) = probe_render(template, None) else {
|
||||
tracing::debug!(
|
||||
model = %model_id,
|
||||
"reasoning effort: template does not render a baseline probe; advertising no rungs"
|
||||
);
|
||||
return Ok(SupportedEfforts::default());
|
||||
};
|
||||
|
||||
let mut rendered: Vec<(&str, String)> = Vec::new();
|
||||
for candidate in CANDIDATE_EFFORTS {
|
||||
if let Some(out) = probe_render(template, Some(candidate)) {
|
||||
rendered.push((candidate, out));
|
||||
}
|
||||
}
|
||||
|
||||
// A template that never reads `reasoning_effort` renders the same
|
||||
// bytes whatever we pass. Advertising every candidate in that case
|
||||
// would be the invented-ladder mistake again, in a new place.
|
||||
let effort_aware = rendered.windows(2).any(|w| w[0].1 != w[1].1)
|
||||
|| rendered.iter().any(|(_, out)| *out != baseline);
|
||||
if !effort_aware {
|
||||
tracing::debug!(
|
||||
model = %model_id,
|
||||
"reasoning effort: template ignores reasoning_effort; advertising no rungs"
|
||||
);
|
||||
return Ok(SupportedEfforts::default());
|
||||
}
|
||||
|
||||
let default = rendered
|
||||
.iter()
|
||||
.find(|(_, out)| *out == baseline)
|
||||
.map(|(name, _)| (*name).to_string());
|
||||
let levels: Vec<String> = rendered
|
||||
.iter()
|
||||
.map(|(name, _)| (*name).to_string())
|
||||
.collect();
|
||||
|
||||
tracing::info!(
|
||||
model = %model_id,
|
||||
levels = %levels.join(","),
|
||||
default = ?default,
|
||||
"reasoning effort: rungs discovered from the model's chat template (#290)"
|
||||
);
|
||||
Ok(SupportedEfforts { levels, default })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The probe against the **real** deployed Qwen3.8-27B template,
|
||||
/// not a hand-written approximation of it. A synthetic template
|
||||
/// proves the probe logic; only the shipped artifact proves the
|
||||
/// answer, and the artifact is what #290 is about.
|
||||
#[test]
|
||||
fn the_real_qwen38_template_yields_low_medium_xhigh_defaulting_to_xhigh() {
|
||||
let path = concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/tests/data/qwen3.8-27b.chat_template.jinja"
|
||||
);
|
||||
let Ok(template) = std::fs::read_to_string(path) else {
|
||||
// The fixture is checked in; if it is ever removed this test
|
||||
// should fail loudly rather than silently pass.
|
||||
panic!("missing fixture {path}");
|
||||
};
|
||||
let s = probe(&template, "Qwen/Qwen3.8-27B").expect("probe");
|
||||
assert_eq!(
|
||||
s.levels,
|
||||
vec!["low", "medium", "xhigh"],
|
||||
"the shipped template accepts exactly these"
|
||||
);
|
||||
assert_eq!(s.default.as_deref(), Some("xhigh"));
|
||||
// The two rungs we used to advertise are rejected outright.
|
||||
assert!(!s.supports("minimal"));
|
||||
assert!(!s.supports("high"));
|
||||
}
|
||||
|
||||
/// The shape Qwen3.8-27B actually ships: three rungs, `xhigh`
|
||||
/// default, `raise_exception` on anything else.
|
||||
const QWEN38: &str = r#"
|
||||
{%- set resolved = reasoning_effort|default('xhigh') %}
|
||||
{%- if resolved not in ('xhigh', 'medium', 'low') %}
|
||||
{{- raise_exception('Unexpected reasoning effort ' ~ resolved) }}
|
||||
{%- endif %}
|
||||
{%- if resolved == 'xhigh' %}EFFORT-XHIGH{% elif resolved == 'low' %}EFFORT-LOW{% else %}EFFORT-MEDIUM{% endif %}
|
||||
{%- for m in messages %}<|im_start|>{{ m.role }}
|
||||
{{ m.content }}<|im_end|>
|
||||
{%- endfor %}
|
||||
"#;
|
||||
|
||||
/// A template with no effort control at all.
|
||||
const PLAIN: &str = r#"
|
||||
{%- for m in messages %}<|im_start|>{{ m.role }}
|
||||
{{ m.content }}<|im_end|>
|
||||
{%- endfor %}
|
||||
"#;
|
||||
|
||||
#[test]
|
||||
fn discovers_exactly_the_rungs_the_template_accepts() {
|
||||
let s = probe(QWEN38, "test").expect("probe");
|
||||
assert_eq!(s.levels, vec!["low", "medium", "xhigh"]);
|
||||
assert!(!s.supports("minimal"), "template raises on minimal");
|
||||
assert!(!s.supports("high"), "template raises on high");
|
||||
}
|
||||
|
||||
/// Without this, a caller who names no effort silently receives the
|
||||
/// strongest rung while we believe it received nothing — which is
|
||||
/// how the xhigh instruction ended up on every request in #290.
|
||||
#[test]
|
||||
fn detects_the_templates_own_default() {
|
||||
let s = probe(QWEN38, "test").expect("probe");
|
||||
assert_eq!(s.default.as_deref(), Some("xhigh"));
|
||||
}
|
||||
|
||||
/// A template that ignores the kwarg must advertise nothing rather
|
||||
/// than appearing to support all six.
|
||||
#[test]
|
||||
fn a_template_without_effort_control_advertises_no_rungs() {
|
||||
let s = probe(PLAIN, "test").expect("probe");
|
||||
assert!(s.is_empty(), "got {:?}", s.levels);
|
||||
assert_eq!(s.nearest("medium"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_exact_match_is_returned_unchanged() {
|
||||
let s = probe(QWEN38, "test").expect("probe");
|
||||
assert_eq!(s.nearest("medium"), Some("medium"));
|
||||
assert_eq!(s.nearest("xhigh"), Some("xhigh"));
|
||||
}
|
||||
|
||||
/// `high` does not exist on this model. Descending first means it
|
||||
/// resolves to `medium`, not `xhigh`: a caller is never given more
|
||||
/// thinking (slower, dearer) than it asked for.
|
||||
#[test]
|
||||
fn an_unsupported_level_resolves_downward_not_upward() {
|
||||
let s = probe(QWEN38, "test").expect("probe");
|
||||
assert_eq!(s.nearest("high"), Some("medium"));
|
||||
}
|
||||
|
||||
/// `minimal` is below every rung the model has, so there is nothing
|
||||
/// to descend to and the nearest is the weakest available.
|
||||
#[test]
|
||||
fn a_level_below_every_rung_resolves_up_to_the_weakest() {
|
||||
let s = probe(QWEN38, "test").expect("probe");
|
||||
assert_eq!(s.nearest("minimal"), Some("low"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unknown_level_name_resolves_to_nothing_rather_than_a_guess() {
|
||||
let s = probe(QWEN38, "test").expect("probe");
|
||||
assert_eq!(s.nearest("turbo"), None);
|
||||
}
|
||||
}
|
||||
@@ -1,687 +0,0 @@
|
||||
//! Sampling parameter resolution (#272, #283).
|
||||
//!
|
||||
//! Four sources, in priority order:
|
||||
//!
|
||||
//! 1. what the **request** asked for,
|
||||
//! 2. what the **operator** set for this model (#283),
|
||||
//! 3. what the **model** published in `generation_config.json`,
|
||||
//! 4. a built-in fallback.
|
||||
//!
|
||||
//! Tier (2) exists because honouring the model's own file is right by
|
||||
//! default and still wrong sometimes: `Qwen/Qwen3.8-27B` publishes
|
||||
//! `temperature = 1.0`, which measured a 20% structural-defect rate on
|
||||
//! ~2k-token code generation against 0/60 at `<= 0.6` (#283,
|
||||
//! p = 0.0031). The operator's correction is explicit and lives in a
|
||||
//! config file — never a heuristic that inspects the model and guesses,
|
||||
//! which would be a footgun.
|
||||
//!
|
||||
//! The operator sits *below* the request on purpose. An explicit API
|
||||
//! parameter that is silently ignored is a contract break; the observed
|
||||
//! failure came from callers that named nothing, so a default-tier
|
||||
//! override is sufficient to fix it.
|
||||
//!
|
||||
//! Mechanically, tier (2) is folded into tier (3) once at load time by
|
||||
//! [`ModelGenerationDefaults::with_operator_override`], so the
|
||||
//! per-request resolve path stays a two-tier lookup and every call site
|
||||
//! gets the override for free — there is no site that can forget it.
|
||||
//!
|
||||
//! Before this, (2) did not exist and (3) was a hardcoded
|
||||
//! `temperature = 0.7` with `top_p = None` — and `None` selects
|
||||
//! `Sampling::All`, meaning no truncation at all. So a caller that
|
||||
//! omitted sampling fields (the common case: the DeepSeek Harness sends
|
||||
//! only `temperature`) got untruncated sampling across the full ~250k
|
||||
//! vocabulary on a model whose authors published `top_k = 20,
|
||||
//! top_p = 0.95, temperature = 1.0`.
|
||||
//!
|
||||
//! Picking "no truncation" for an absent field is not a neutral default,
|
||||
//! it is the widest possible one. The neutral default is what the model
|
||||
//! shipped with.
|
||||
//!
|
||||
//! Resolved **once** per request and threaded whole. The eight
|
||||
//! `LogitsProcessor` construction sites this replaces could each have
|
||||
//! disagreed about a default — the same shape of bug as #252, where two
|
||||
//! config files described one model and drifted.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::Path;
|
||||
|
||||
/// Fallback temperature when neither the request nor the model says.
|
||||
/// Retained from the previous hardcoded value so a model without a
|
||||
/// `generation_config.json` behaves exactly as it did before.
|
||||
const FALLBACK_TEMPERATURE: f64 = 0.7;
|
||||
|
||||
/// Repetition penalty applied to recently-generated tokens before
|
||||
/// sampling. 1.0 disables it; >1.0 makes recently-emitted tokens less
|
||||
/// likely. mistral.rs and llama.cpp default to 1.1, which is enough to
|
||||
/// stop small quantized models from degenerating into "Wait, no, no..."
|
||||
/// loops without distorting normal output.
|
||||
pub const DEFAULT_REPEAT_PENALTY: f32 = 1.1;
|
||||
|
||||
/// Number of recently-generated tokens fed into the repetition penalty.
|
||||
/// Matches the candle quantized-qwen3 example default.
|
||||
///
|
||||
/// Defensible for chat and poorly suited to a 24k-token reasoning block,
|
||||
/// where a derivation repeated 500 tokens later is well outside the
|
||||
/// window — which is why it is now a per-request knob rather than a
|
||||
/// constant.
|
||||
pub const DEFAULT_REPEAT_LAST_N: usize = 64;
|
||||
|
||||
/// What a model published in its `generation_config.json`.
|
||||
///
|
||||
/// Every field optional: the file is a HuggingFace convention, not a
|
||||
/// schema, and models populate different subsets of it.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct ModelGenerationDefaults {
|
||||
pub temperature: Option<f64>,
|
||||
pub top_p: Option<f64>,
|
||||
pub top_k: Option<usize>,
|
||||
/// Not published in `generation_config.json` — this struct is the
|
||||
/// resolved per-model default layer, and the operator override
|
||||
/// (#283) lands here so the repetition knobs sit in the same
|
||||
/// precedence chain as the rest.
|
||||
pub repeat_penalty: Option<f32>,
|
||||
pub repeat_last_n: Option<usize>,
|
||||
pub presence_penalty: Option<f32>,
|
||||
pub frequency_penalty: Option<f32>,
|
||||
pub seed: Option<u64>,
|
||||
}
|
||||
|
||||
impl ModelGenerationDefaults {
|
||||
/// Read `generation_config.json` from a model snapshot directory.
|
||||
///
|
||||
/// Absent or unparseable is not an error — we fall back exactly as
|
||||
/// before — but it is logged, because silently guessing the model's
|
||||
/// sampling is how the untruncated default went unnoticed.
|
||||
pub fn load_from_dir(dir: &Path) -> Self {
|
||||
let path = dir.join("generation_config.json");
|
||||
let text = match std::fs::read_to_string(&path) {
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
tracing::debug!(
|
||||
path = %path.display(),
|
||||
error = %e,
|
||||
"sampling: no generation_config.json; using built-in fallbacks"
|
||||
);
|
||||
return Self::default();
|
||||
}
|
||||
};
|
||||
match serde_json::from_str::<Self>(&text) {
|
||||
Ok(d) => {
|
||||
tracing::info!(
|
||||
path = %path.display(),
|
||||
temperature = ?d.temperature,
|
||||
top_p = ?d.top_p,
|
||||
top_k = ?d.top_k,
|
||||
"sampling: model generation defaults loaded"
|
||||
);
|
||||
d
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
path = %path.display(),
|
||||
error = %e,
|
||||
"sampling: generation_config.json did not parse; using built-in fallbacks"
|
||||
);
|
||||
Self::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Overlay the operator's per-model override (#283) onto what the
|
||||
/// model published, field by field.
|
||||
///
|
||||
/// Applied **once at load**, not per request, so the override cannot
|
||||
/// be forgotten at one of the resolve sites — the same reasoning
|
||||
/// that made #272 resolve sampling once rather than at eight
|
||||
/// `LogitsProcessor` construction sites.
|
||||
///
|
||||
/// Each field overlays independently: setting only `temperature`
|
||||
/// leaves the model's `top_p`/`top_k` in force, because an operator
|
||||
/// correcting one number should not silently blank the other two
|
||||
/// back to "no truncation".
|
||||
///
|
||||
/// Logged at `info` whenever it changes something — an operator
|
||||
/// override that is in force but invisible is how a fleet ends up
|
||||
/// with two configs that disagree and nobody noticing (#252).
|
||||
pub fn with_operator_override(
|
||||
mut self,
|
||||
model_id: &str,
|
||||
operator: Option<&cortex_core::harness::SamplingOverride>,
|
||||
) -> Self {
|
||||
let Some(op) = operator else {
|
||||
return self;
|
||||
};
|
||||
if op.is_empty() {
|
||||
return self;
|
||||
}
|
||||
let before = self.clone();
|
||||
if let Some(t) = op.temperature {
|
||||
self.temperature = Some(t);
|
||||
}
|
||||
if let Some(p) = op.top_p {
|
||||
self.top_p = Some(p);
|
||||
}
|
||||
if let Some(k) = op.top_k {
|
||||
self.top_k = Some(k);
|
||||
}
|
||||
if let Some(r) = op.repeat_penalty {
|
||||
self.repeat_penalty = Some(r);
|
||||
}
|
||||
if let Some(n) = op.repeat_last_n {
|
||||
self.repeat_last_n = Some(n);
|
||||
}
|
||||
if let Some(v) = op.presence_penalty {
|
||||
self.presence_penalty = Some(v);
|
||||
}
|
||||
if let Some(v) = op.frequency_penalty {
|
||||
self.frequency_penalty = Some(v);
|
||||
}
|
||||
if let Some(v) = op.seed {
|
||||
self.seed = Some(v);
|
||||
}
|
||||
tracing::info!(
|
||||
model = %model_id,
|
||||
model_temperature = ?before.temperature,
|
||||
model_top_p = ?before.top_p,
|
||||
model_top_k = ?before.top_k,
|
||||
temperature = ?self.temperature,
|
||||
top_p = ?self.top_p,
|
||||
top_k = ?self.top_k,
|
||||
repeat_penalty = ?self.repeat_penalty,
|
||||
repeat_last_n = ?self.repeat_last_n,
|
||||
presence_penalty = ?self.presence_penalty,
|
||||
frequency_penalty = ?self.frequency_penalty,
|
||||
seed = ?self.seed,
|
||||
"sampling: operator override applied over the model's published defaults (#283)"
|
||||
);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// What the caller asked for. All optional — absent means "no opinion",
|
||||
/// which defers to the model's published value.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct RequestedSampling {
|
||||
pub temperature: Option<f64>,
|
||||
pub top_p: Option<f64>,
|
||||
pub top_k: Option<usize>,
|
||||
pub seed: Option<u64>,
|
||||
pub repeat_penalty: Option<f32>,
|
||||
pub repeat_last_n: Option<usize>,
|
||||
pub presence_penalty: Option<f32>,
|
||||
pub frequency_penalty: Option<f32>,
|
||||
}
|
||||
|
||||
/// Fully-resolved sampling for one request.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct SamplingParams {
|
||||
pub temperature: f64,
|
||||
pub top_p: Option<f64>,
|
||||
pub top_k: Option<usize>,
|
||||
pub seed: u64,
|
||||
pub repeat_penalty: f32,
|
||||
pub repeat_last_n: usize,
|
||||
/// Sequence-wide penalties, OpenAI semantics. `0.0` disables.
|
||||
pub presence_penalty: f32,
|
||||
pub frequency_penalty: f32,
|
||||
}
|
||||
|
||||
impl SamplingParams {
|
||||
/// Resolve request over model defaults over fallback.
|
||||
///
|
||||
/// `random_seed` is the per-request seed used when the caller did
|
||||
/// not pin one; it is passed in rather than generated here so this
|
||||
/// stays a pure function and the "same seed reproduces" property is
|
||||
/// testable without a clock.
|
||||
pub fn resolve(
|
||||
requested: &RequestedSampling,
|
||||
model: &ModelGenerationDefaults,
|
||||
random_seed: u64,
|
||||
) -> Self {
|
||||
Self {
|
||||
temperature: requested
|
||||
.temperature
|
||||
.or(model.temperature)
|
||||
.unwrap_or(FALLBACK_TEMPERATURE),
|
||||
// No `unwrap_or` here on purpose: `None` still means "do not
|
||||
// truncate", but it is now only reachable when neither the
|
||||
// caller nor the model expressed a preference, rather than
|
||||
// whenever the caller stayed quiet.
|
||||
top_p: requested.top_p.or(model.top_p),
|
||||
top_k: requested.top_k.or(model.top_k),
|
||||
seed: requested.seed.or(model.seed).unwrap_or(random_seed),
|
||||
repeat_penalty: requested
|
||||
.repeat_penalty
|
||||
.or(model.repeat_penalty)
|
||||
.unwrap_or(DEFAULT_REPEAT_PENALTY),
|
||||
repeat_last_n: requested
|
||||
.repeat_last_n
|
||||
.or(model.repeat_last_n)
|
||||
.unwrap_or(DEFAULT_REPEAT_LAST_N),
|
||||
presence_penalty: requested
|
||||
.presence_penalty
|
||||
.or(model.presence_penalty)
|
||||
.unwrap_or(0.0),
|
||||
frequency_penalty: requested
|
||||
.frequency_penalty
|
||||
.or(model.frequency_penalty)
|
||||
.unwrap_or(0.0),
|
||||
}
|
||||
}
|
||||
|
||||
/// The candle sampling strategy these parameters select.
|
||||
///
|
||||
/// `temperature <= 0` is greedy regardless of the rest — a caller
|
||||
/// asking for determinism that way should not have it silently
|
||||
/// softened by a `top_k` the model happened to publish.
|
||||
pub fn to_sampling(&self) -> candle_transformers::generation::Sampling {
|
||||
use candle_transformers::generation::Sampling;
|
||||
if self.temperature <= 0.0 {
|
||||
return Sampling::ArgMax;
|
||||
}
|
||||
let temperature = self.temperature;
|
||||
match (self.top_k, self.top_p) {
|
||||
(Some(k), Some(p)) => Sampling::TopKThenTopP { k, p, temperature },
|
||||
(Some(k), None) => Sampling::TopK { k, temperature },
|
||||
(None, Some(p)) => Sampling::TopP { p, temperature },
|
||||
(None, None) => Sampling::All { temperature },
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use candle_transformers::generation::Sampling;
|
||||
use cortex_core::harness::SamplingOverride;
|
||||
|
||||
fn qwen3_defaults() -> ModelGenerationDefaults {
|
||||
ModelGenerationDefaults {
|
||||
temperature: Some(1.0),
|
||||
top_p: Some(0.95),
|
||||
top_k: Some(20),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// The case that motivated #271: a caller that names no sampling
|
||||
/// parameters must get the model's published ones, not untruncated
|
||||
/// sampling at a hardcoded 0.7.
|
||||
#[test]
|
||||
fn a_silent_request_inherits_the_models_published_sampling() {
|
||||
let p = SamplingParams::resolve(&RequestedSampling::default(), &qwen3_defaults(), 42);
|
||||
assert_eq!(p.temperature, 1.0);
|
||||
assert_eq!(p.top_p, Some(0.95));
|
||||
assert_eq!(p.top_k, Some(20));
|
||||
assert_eq!(
|
||||
p.to_sampling(),
|
||||
Sampling::TopKThenTopP {
|
||||
k: 20,
|
||||
p: 0.95,
|
||||
temperature: 1.0
|
||||
},
|
||||
"the combination Qwen3 asks for must be expressible"
|
||||
);
|
||||
}
|
||||
|
||||
/// The caller still wins. A harness that deliberately sets a low
|
||||
/// temperature must not have it overridden by the model's file.
|
||||
#[test]
|
||||
fn the_request_overrides_the_model() {
|
||||
let requested = RequestedSampling {
|
||||
temperature: Some(0.2),
|
||||
top_k: Some(5),
|
||||
..Default::default()
|
||||
};
|
||||
let p = SamplingParams::resolve(&requested, &qwen3_defaults(), 42);
|
||||
assert_eq!(p.temperature, 0.2);
|
||||
assert_eq!(p.top_k, Some(5));
|
||||
// Untouched fields still fall through to the model.
|
||||
assert_eq!(p.top_p, Some(0.95));
|
||||
}
|
||||
|
||||
// ── Operator override (#283) ──────────────────────────────
|
||||
|
||||
fn op(temperature: Option<f64>, top_p: Option<f64>, top_k: Option<usize>) -> SamplingOverride {
|
||||
SamplingOverride {
|
||||
temperature,
|
||||
top_p,
|
||||
top_k,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn with_op(o: SamplingOverride) -> ModelGenerationDefaults {
|
||||
qwen3_defaults().with_operator_override("Qwen/Qwen3.8-27B", Some(&o))
|
||||
}
|
||||
|
||||
/// The #283 case: the model publishes `temperature = 1.0`, which
|
||||
/// measured a 20% structural-defect rate on code generation. The
|
||||
/// operator sets 0.6 and a silent caller must get 0.6.
|
||||
#[test]
|
||||
fn the_operator_override_replaces_what_the_model_published() {
|
||||
let p = SamplingParams::resolve(
|
||||
&RequestedSampling::default(),
|
||||
&with_op(op(Some(0.6), None, None)),
|
||||
42,
|
||||
);
|
||||
assert_eq!(p.temperature, 0.6);
|
||||
}
|
||||
|
||||
/// Precedence is request > operator > model. An explicit API
|
||||
/// parameter that is silently ignored is a contract break, so a
|
||||
/// caller naming a temperature still wins over the operator.
|
||||
#[test]
|
||||
fn an_explicit_request_still_beats_the_operator() {
|
||||
let requested = RequestedSampling {
|
||||
temperature: Some(0.9),
|
||||
..Default::default()
|
||||
};
|
||||
let p = SamplingParams::resolve(&requested, &with_op(op(Some(0.6), None, None)), 42);
|
||||
assert_eq!(p.temperature, 0.9);
|
||||
}
|
||||
|
||||
/// Overriding one field must not blank the others back to "no
|
||||
/// truncation" — that was the #271 failure mode, reintroduced by a
|
||||
/// wholesale replace instead of a field-by-field overlay.
|
||||
#[test]
|
||||
fn overriding_temperature_alone_leaves_top_p_and_top_k_intact() {
|
||||
let d = with_op(op(Some(0.6), None, None));
|
||||
assert_eq!(d.temperature, Some(0.6));
|
||||
assert_eq!(d.top_p, Some(0.95), "the model's top_p must survive");
|
||||
assert_eq!(d.top_k, Some(20), "the model's top_k must survive");
|
||||
}
|
||||
|
||||
/// Each field overlays independently.
|
||||
#[test]
|
||||
fn every_field_can_be_overridden_independently() {
|
||||
let d = with_op(op(None, Some(0.8), Some(40)));
|
||||
assert_eq!(d.temperature, Some(1.0), "untouched, so the model's");
|
||||
assert_eq!(d.top_p, Some(0.8));
|
||||
assert_eq!(d.top_k, Some(40));
|
||||
}
|
||||
|
||||
/// No override, and an override that sets nothing, must both be
|
||||
/// indistinguishable from the pre-#283 behaviour.
|
||||
#[test]
|
||||
fn absent_or_empty_override_changes_nothing() {
|
||||
let base = qwen3_defaults();
|
||||
let none = qwen3_defaults().with_operator_override("m", None);
|
||||
let empty =
|
||||
qwen3_defaults().with_operator_override("m", Some(&SamplingOverride::default()));
|
||||
for d in [none, empty] {
|
||||
assert_eq!(d.temperature, base.temperature);
|
||||
assert_eq!(d.top_p, base.top_p);
|
||||
assert_eq!(d.top_k, base.top_k);
|
||||
}
|
||||
}
|
||||
|
||||
/// An operator override on a model that ships no
|
||||
/// `generation_config.json` must still take effect, rather than
|
||||
/// falling through to the built-in 0.7.
|
||||
#[test]
|
||||
fn the_override_applies_even_without_a_generation_config() {
|
||||
let d = ModelGenerationDefaults::default()
|
||||
.with_operator_override("m", Some(&op(Some(0.6), None, None)));
|
||||
let p = SamplingParams::resolve(&RequestedSampling::default(), &d, 42);
|
||||
assert_eq!(p.temperature, 0.6);
|
||||
assert_ne!(p.temperature, FALLBACK_TEMPERATURE);
|
||||
}
|
||||
|
||||
/// `temperature = 0` from an operator must select greedy decoding,
|
||||
/// not be treated as "unset" by an `unwrap_or`-shaped bug.
|
||||
#[test]
|
||||
fn an_operator_temperature_of_zero_is_greedy_not_unset() {
|
||||
let p = SamplingParams::resolve(
|
||||
&RequestedSampling::default(),
|
||||
&with_op(op(Some(0.0), None, None)),
|
||||
42,
|
||||
);
|
||||
assert_eq!(p.temperature, 0.0);
|
||||
assert_eq!(p.to_sampling(), Sampling::ArgMax);
|
||||
}
|
||||
|
||||
/// A model with no `generation_config.json` must behave exactly as
|
||||
/// it did before this landed, so nothing regresses on the models
|
||||
/// that do not ship one.
|
||||
/// An operator can pin the seed, because the client that needs it
|
||||
/// pinned cannot send one: pi's request builder emits no `seed`, so
|
||||
/// without this a parameter search runs unseeded and sampling
|
||||
/// variance is indistinguishable from the effect being measured.
|
||||
#[test]
|
||||
fn an_operator_can_pin_the_seed() {
|
||||
let op = cortex_core::harness::SamplingOverride {
|
||||
seed: Some(424_242),
|
||||
..Default::default()
|
||||
};
|
||||
let defaults = ModelGenerationDefaults::default().with_operator_override("m", Some(&op));
|
||||
// The random seed argument must be ignored once an operator pinned one.
|
||||
let p = SamplingParams::resolve(&RequestedSampling::default(), &defaults, 999);
|
||||
assert_eq!(p.seed, 424_242);
|
||||
}
|
||||
|
||||
/// Precedence holds: a caller that names its own seed keeps it.
|
||||
#[test]
|
||||
fn a_requested_seed_outranks_the_operator() {
|
||||
let op = cortex_core::harness::SamplingOverride {
|
||||
seed: Some(424_242),
|
||||
..Default::default()
|
||||
};
|
||||
let defaults = ModelGenerationDefaults::default().with_operator_override("m", Some(&op));
|
||||
let req = RequestedSampling {
|
||||
seed: Some(7),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(SamplingParams::resolve(&req, &defaults, 999).seed, 7);
|
||||
}
|
||||
|
||||
/// With neither, the per-request random seed still applies — an
|
||||
/// unset override must not accidentally freeze the fleet's RNG.
|
||||
#[test]
|
||||
fn no_pinned_seed_leaves_generation_random() {
|
||||
let d = ModelGenerationDefaults::default().with_operator_override("m", None);
|
||||
assert_eq!(
|
||||
SamplingParams::resolve(&RequestedSampling::default(), &d, 999).seed,
|
||||
999
|
||||
);
|
||||
}
|
||||
|
||||
/// The gap this closes: `repeat_last_n` defaults to 64 — candle's
|
||||
/// chat example value — while a reasoning model's think block runs
|
||||
/// to tens of thousands of tokens. Anything restated more than 64
|
||||
/// tokens later is invisible to the penalty, and before this there
|
||||
/// was no operator route to the knob at all: `SamplingOverride`
|
||||
/// carried only temperature/top_p/top_k, and OpenAI-shaped clients
|
||||
/// do not send repetition parameters. The 64 was effectively
|
||||
/// hardcoded in production.
|
||||
#[test]
|
||||
fn an_operator_can_widen_the_repetition_window() {
|
||||
let op = cortex_core::harness::SamplingOverride {
|
||||
repeat_penalty: Some(1.05),
|
||||
repeat_last_n: Some(2048),
|
||||
presence_penalty: None,
|
||||
frequency_penalty: None,
|
||||
..Default::default()
|
||||
};
|
||||
let defaults = ModelGenerationDefaults::default().with_operator_override("m", Some(&op));
|
||||
let p = SamplingParams::resolve(&RequestedSampling::default(), &defaults, 7);
|
||||
assert_eq!(p.repeat_last_n, 2048);
|
||||
assert!((p.repeat_penalty - 1.05).abs() < f32::EPSILON);
|
||||
}
|
||||
|
||||
/// Precedence is request > operator > built-in, same as every other
|
||||
/// sampling field — a caller that named a window keeps it.
|
||||
#[test]
|
||||
fn a_requested_repetition_window_outranks_the_operator() {
|
||||
let op = cortex_core::harness::SamplingOverride {
|
||||
repeat_last_n: Some(2048),
|
||||
presence_penalty: None,
|
||||
frequency_penalty: None,
|
||||
..Default::default()
|
||||
};
|
||||
let defaults = ModelGenerationDefaults::default().with_operator_override("m", Some(&op));
|
||||
let req = RequestedSampling {
|
||||
repeat_last_n: Some(128),
|
||||
presence_penalty: None,
|
||||
frequency_penalty: None,
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
SamplingParams::resolve(&req, &defaults, 7).repeat_last_n,
|
||||
128
|
||||
);
|
||||
}
|
||||
|
||||
/// An operator who sets nothing must not change the constants.
|
||||
#[test]
|
||||
fn no_operator_setting_leaves_the_repetition_defaults_alone() {
|
||||
let d = ModelGenerationDefaults::default().with_operator_override("m", None);
|
||||
let p = SamplingParams::resolve(&RequestedSampling::default(), &d, 7);
|
||||
assert_eq!(p.repeat_last_n, DEFAULT_REPEAT_LAST_N);
|
||||
assert!((p.repeat_penalty - DEFAULT_REPEAT_PENALTY).abs() < f32::EPSILON);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn without_model_defaults_the_old_behaviour_is_preserved() {
|
||||
let p = SamplingParams::resolve(
|
||||
&RequestedSampling::default(),
|
||||
&ModelGenerationDefaults::default(),
|
||||
42,
|
||||
);
|
||||
assert_eq!(p.temperature, FALLBACK_TEMPERATURE);
|
||||
assert_eq!(p.top_p, None);
|
||||
assert_eq!(p.top_k, None);
|
||||
assert_eq!(p.to_sampling(), Sampling::All { temperature: 0.7 });
|
||||
}
|
||||
|
||||
/// A pinned seed is honoured; an absent one takes the per-request
|
||||
/// random value. Reproducibility is the whole point — the text paths
|
||||
/// previously called `unix_subsec_nanos()` unconditionally, so
|
||||
/// `seed` was accepted and discarded.
|
||||
#[test]
|
||||
fn seed_is_pinned_when_given_and_random_otherwise() {
|
||||
let pinned = SamplingParams::resolve(
|
||||
&RequestedSampling {
|
||||
seed: Some(7),
|
||||
..Default::default()
|
||||
},
|
||||
&ModelGenerationDefaults::default(),
|
||||
42,
|
||||
);
|
||||
assert_eq!(pinned.seed, 7);
|
||||
|
||||
let unpinned = SamplingParams::resolve(
|
||||
&RequestedSampling::default(),
|
||||
&ModelGenerationDefaults::default(),
|
||||
42,
|
||||
);
|
||||
assert_eq!(unpinned.seed, 42);
|
||||
}
|
||||
|
||||
/// Greedy beats every truncation setting: a caller asking for
|
||||
/// `temperature = 0` wants determinism, not a softened version of it.
|
||||
#[test]
|
||||
fn zero_temperature_is_greedy_even_with_model_top_k() {
|
||||
let requested = RequestedSampling {
|
||||
temperature: Some(0.0),
|
||||
..Default::default()
|
||||
};
|
||||
let p = SamplingParams::resolve(&requested, &qwen3_defaults(), 42);
|
||||
assert_eq!(p.to_sampling(), Sampling::ArgMax);
|
||||
}
|
||||
|
||||
/// Each truncation knob alone selects the matching strategy.
|
||||
#[test]
|
||||
fn single_knobs_select_their_own_strategy() {
|
||||
let only_k = SamplingParams::resolve(
|
||||
&RequestedSampling {
|
||||
top_k: Some(20),
|
||||
..Default::default()
|
||||
},
|
||||
&ModelGenerationDefaults::default(),
|
||||
1,
|
||||
);
|
||||
assert_eq!(
|
||||
only_k.to_sampling(),
|
||||
Sampling::TopK {
|
||||
k: 20,
|
||||
temperature: 0.7
|
||||
}
|
||||
);
|
||||
|
||||
let only_p = SamplingParams::resolve(
|
||||
&RequestedSampling {
|
||||
top_p: Some(0.9),
|
||||
..Default::default()
|
||||
},
|
||||
&ModelGenerationDefaults::default(),
|
||||
1,
|
||||
);
|
||||
assert_eq!(
|
||||
only_p.to_sampling(),
|
||||
Sampling::TopP {
|
||||
p: 0.9,
|
||||
temperature: 0.7
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
/// Repetition penalty and its window are now per-request, defaulting
|
||||
/// to the values that were previously constants.
|
||||
#[test]
|
||||
fn repetition_knobs_default_to_the_former_constants() {
|
||||
let d = SamplingParams::resolve(
|
||||
&RequestedSampling::default(),
|
||||
&ModelGenerationDefaults::default(),
|
||||
1,
|
||||
);
|
||||
assert_eq!(d.repeat_penalty, DEFAULT_REPEAT_PENALTY);
|
||||
assert_eq!(d.repeat_last_n, DEFAULT_REPEAT_LAST_N);
|
||||
|
||||
let tuned = SamplingParams::resolve(
|
||||
&RequestedSampling {
|
||||
repeat_penalty: Some(1.05),
|
||||
repeat_last_n: Some(512),
|
||||
presence_penalty: None,
|
||||
frequency_penalty: None,
|
||||
..Default::default()
|
||||
},
|
||||
&ModelGenerationDefaults::default(),
|
||||
1,
|
||||
);
|
||||
assert_eq!(tuned.repeat_penalty, 1.05);
|
||||
assert_eq!(tuned.repeat_last_n, 512);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_generation_config_is_not_an_error() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let d = ModelGenerationDefaults::load_from_dir(dir.path());
|
||||
assert!(d.temperature.is_none() && d.top_p.is_none() && d.top_k.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_real_generation_config_parses() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
// Verbatim from Qwen/Qwen3.8-27B, extra fields included — the
|
||||
// reader must tolerate the ones it does not model.
|
||||
std::fs::write(
|
||||
dir.path().join("generation_config.json"),
|
||||
r#"{"bos_token_id":248044,"do_sample":true,"eos_token_id":[248046,248044],
|
||||
"pad_token_id":248044,"temperature":1.0,"top_k":20,"top_p":0.95}"#,
|
||||
)
|
||||
.expect("write");
|
||||
let d = ModelGenerationDefaults::load_from_dir(dir.path());
|
||||
assert_eq!(d.temperature, Some(1.0));
|
||||
assert_eq!(d.top_p, Some(0.95));
|
||||
assert_eq!(d.top_k, Some(20));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_generation_config_falls_back_rather_than_failing() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
std::fs::write(dir.path().join("generation_config.json"), "{not json").expect("write");
|
||||
let d = ModelGenerationDefaults::load_from_dir(dir.path());
|
||||
assert!(d.temperature.is_none());
|
||||
}
|
||||
}
|
||||
@@ -30,10 +30,6 @@ pub mod worker;
|
||||
use anyhow::{Context, Result};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Stdio;
|
||||
#[cfg(any(feature = "cuda", test))]
|
||||
use std::sync::Arc;
|
||||
#[cfg(any(feature = "cuda", test))]
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Lines};
|
||||
use tokio::process::{Child, ChildStdin, ChildStdout, Command};
|
||||
|
||||
@@ -323,87 +319,6 @@ pub struct WorkerPool {
|
||||
leader_comm: Option<nccl_state::SendComm>,
|
||||
}
|
||||
|
||||
/// How long the watchdog waits for `ncclCommAbort` to return before
|
||||
/// concluding the process is unrecoverable (#265). Generous relative to a
|
||||
/// healthy abort (milliseconds) and short relative to a human noticing.
|
||||
#[cfg(any(feature = "cuda", test))]
|
||||
fn tp_abort_deadline() -> std::time::Duration {
|
||||
let secs = std::env::var("NEURON_TP_ABORT_DEADLINE_S")
|
||||
.ok()
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.filter(|s| *s > 0)
|
||||
.unwrap_or(60);
|
||||
std::time::Duration::from_secs(secs)
|
||||
}
|
||||
|
||||
/// Wait for the abort thread to signal completion. `true` if it returned
|
||||
/// in time (recovery proceeds normally), `false` if the deadline elapsed
|
||||
/// (the process is unrecoverable).
|
||||
///
|
||||
/// Split out from the deadman thread so the decision is testable without
|
||||
/// a GPU: the branch that ends the process is the one most worth
|
||||
/// exercising, and the one hardest to reach by accident.
|
||||
#[cfg(any(feature = "cuda", test))]
|
||||
fn wait_for_abort(done: &AtomicBool, deadline: std::time::Duration) -> bool {
|
||||
let start = std::time::Instant::now();
|
||||
while start.elapsed() < deadline {
|
||||
if done.load(Ordering::Acquire) {
|
||||
return true;
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(25));
|
||||
}
|
||||
// One last look: the abort may have landed inside the final sleep.
|
||||
done.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
/// Exit non-zero so systemd's `Restart=on-failure` restarts us (#265).
|
||||
///
|
||||
/// Called only when the process cannot recover on its own: the NCCL
|
||||
/// collective is wedged and the abort that would unblock it has itself
|
||||
/// hung. The daemon then serves nothing — on beast 2026-08-17 it sat in
|
||||
/// that state for 70 minutes with GPUs idle, weights resident, 27
|
||||
/// connections queued on the listener, and `systemctl` reporting it
|
||||
/// active. A process that knows it is unrecoverable should not wait for a
|
||||
/// human.
|
||||
///
|
||||
/// **Everything diagnostic is logged first.** A restart destroys the
|
||||
/// state that explains the hang, so trading a diagnosable outage for an
|
||||
/// undiagnosable self-healing one would be a poor bargain.
|
||||
///
|
||||
/// `process::exit` deliberately skips destructors and the graceful-drain
|
||||
/// path. That is normally the wrong instinct, but draining a wedged
|
||||
/// process cannot work — and `TimeoutStopSec=120s` plus
|
||||
/// `TimeoutStopFailureMode=abort` would add two more minutes of outage on
|
||||
/// top of the wedge (#256).
|
||||
#[cfg(feature = "cuda")]
|
||||
fn exit_unrecoverable(model_id: &str, reason: &str) -> ! {
|
||||
// Read straight from the driver rather than any cached value: the
|
||||
// cache is refreshed by the very machinery that is wedged.
|
||||
let vram = std::process::Command::new("nvidia-smi")
|
||||
.args([
|
||||
"--query-gpu=index,memory.used,memory.free,utilization.gpu",
|
||||
"--format=csv,noheader",
|
||||
])
|
||||
.output()
|
||||
.ok()
|
||||
.map(|o| String::from_utf8_lossy(&o.stdout).replace('\n', " | "))
|
||||
.unwrap_or_else(|| "unavailable".into());
|
||||
tracing::error!(
|
||||
model = %model_id,
|
||||
reason = %reason,
|
||||
vram = %vram,
|
||||
pid = std::process::id(),
|
||||
"tp watchdog: UNRECOVERABLE — exiting non-zero so systemd restarts this daemon (#265). \
|
||||
The GPU state above is from immediately before the exit; the wedged collective and its \
|
||||
request id are in the preceding watchdog lines."
|
||||
);
|
||||
// Give the logging layer a moment to flush before the process dies —
|
||||
// an exit that eats its own explanation is the failure mode this
|
||||
// whole change exists to avoid.
|
||||
std::thread::sleep(std::time::Duration::from_millis(500));
|
||||
std::process::exit(70) // EX_SOFTWARE
|
||||
}
|
||||
|
||||
/// Per-step deadline for a TP forward (#17 Stage 2). A healthy decode
|
||||
/// step or chunked prefill completes in well under a second; a wedged
|
||||
/// NCCL collective never returns. Generous default so no legitimate step
|
||||
@@ -434,66 +349,23 @@ impl WorkerPool {
|
||||
"tp watchdog: leader forward exceeded deadline — NCCL collective wedged; \
|
||||
aborting comm to unblock the leader thread for auto-recovery"
|
||||
);
|
||||
let Some(comm) = self.leader_comm.as_ref().map(|c| c.0.clone()) else {
|
||||
tracing::error!(
|
||||
match &self.leader_comm {
|
||||
Some(c) => match c.0.abort() {
|
||||
Ok(()) => tracing::error!(
|
||||
model = %model_id,
|
||||
"tp watchdog: ncclCommAbort succeeded — wedged collective unblocked; \
|
||||
failing the step so the model auto-recovers (unload+reload)"
|
||||
),
|
||||
Err(e) => tracing::error!(
|
||||
model = %model_id, error = ?e,
|
||||
"tp watchdog: ncclCommAbort failed — recovery may stall until a process restart"
|
||||
),
|
||||
},
|
||||
None => tracing::error!(
|
||||
model = %model_id,
|
||||
"tp watchdog: no cached leader comm handle — cannot abort; exiting so \
|
||||
systemd restarts us (#265)"
|
||||
);
|
||||
exit_unrecoverable(model_id, "no leader comm handle to abort");
|
||||
};
|
||||
|
||||
// `ncclCommAbort` can itself block forever — observed on beast
|
||||
// 2026-08-17, where neither the success nor the failure branch below
|
||||
// was ever reached and the daemon sat wedged for 70 minutes with 27
|
||||
// connections queued on its listener (#265). So the abort runs on its
|
||||
// own thread and the deadline below does NOT depend on it returning:
|
||||
// an abort that hangs must not take the supervisor with it.
|
||||
let done = Arc::new(AtomicBool::new(false));
|
||||
let signal = Arc::clone(&done);
|
||||
let owned_model = model_id.to_string();
|
||||
std::thread::Builder::new()
|
||||
.name("tp-watchdog-abort".into())
|
||||
.spawn(move || {
|
||||
match comm.abort() {
|
||||
Ok(()) => tracing::error!(
|
||||
model = %owned_model,
|
||||
"tp watchdog: ncclCommAbort succeeded — wedged collective unblocked; \
|
||||
failing the step so the model auto-recovers (unload+reload)"
|
||||
),
|
||||
Err(e) => tracing::error!(
|
||||
model = %owned_model, error = ?e,
|
||||
"tp watchdog: ncclCommAbort failed — the collective is still wedged"
|
||||
),
|
||||
}
|
||||
// Returning at all is the signal, success or failure: a
|
||||
// returned abort leaves the process responsive, so
|
||||
// auto-recovery can run and we must not exit under it.
|
||||
signal.store(true, Ordering::Release);
|
||||
})
|
||||
.map(|_| ())
|
||||
.unwrap_or_else(|e| {
|
||||
// Cannot even spawn a thread — nothing here can recover.
|
||||
tracing::error!(model = %model_id, error = %e, "tp watchdog: abort thread spawn failed");
|
||||
exit_unrecoverable(model_id, "could not spawn the abort thread");
|
||||
});
|
||||
|
||||
// Deadman. Disarmed only by the abort returning.
|
||||
let deadline = tp_abort_deadline();
|
||||
let owned_model = model_id.to_string();
|
||||
if let Err(e) = std::thread::Builder::new()
|
||||
.name("tp-watchdog-deadman".into())
|
||||
.spawn(move || {
|
||||
if wait_for_abort(&done, deadline) {
|
||||
return; // abort came back; the normal path owns recovery
|
||||
}
|
||||
exit_unrecoverable(
|
||||
&owned_model,
|
||||
"ncclCommAbort did not return within the deadline",
|
||||
);
|
||||
})
|
||||
{
|
||||
tracing::error!(model = %model_id, error = %e, "tp watchdog: deadman spawn failed");
|
||||
"tp watchdog: no cached leader comm handle — cannot abort; recovery will rely \
|
||||
on a process restart"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -625,7 +497,7 @@ impl WorkerPool {
|
||||
.leader_worker
|
||||
.nccl_init(leader_cfg, comm_id.clone())
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("leader NCCL init via device worker: {e:#}"))?;
|
||||
.map_err(|e| anyhow::anyhow!("leader NCCL init via device worker: {e}"))?;
|
||||
match leader_resp {
|
||||
rpc::WorkerResponse::InitOk => {}
|
||||
rpc::WorkerResponse::Error { kind, message } => {
|
||||
@@ -697,7 +569,7 @@ impl WorkerPool {
|
||||
.leader_worker
|
||||
.nccl_sanity()
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("leader NCCL sanity via device worker: {e:#}"))?;
|
||||
.map_err(|e| anyhow::anyhow!("leader NCCL sanity via device worker: {e}"))?;
|
||||
|
||||
let expected = self.world_size;
|
||||
let leader_sum = match leader_resp {
|
||||
@@ -822,7 +694,7 @@ impl WorkerPool {
|
||||
world_size,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("leader TP shard load via device worker: {e:#}"))?;
|
||||
.map_err(|e| anyhow::anyhow!("leader TP shard load via device worker: {e}"))?;
|
||||
|
||||
// 3. Collect worker confirmations. Anything other than
|
||||
// LoadDenseShardOk aborts the whole load — the leader's
|
||||
@@ -1378,7 +1250,7 @@ impl WorkerPool {
|
||||
.leader_worker
|
||||
.query_vram()
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("leader query_vram: {e:#}"))?;
|
||||
.map_err(|e| anyhow::anyhow!("leader query_vram: {e}"))?;
|
||||
let mut frees = vec![leader_free_mb];
|
||||
let worker_errors = drain_workers(&mut self.workers, |r| match r {
|
||||
WorkerResponse::VramInfo { free_mb, .. } => {
|
||||
@@ -1570,73 +1442,3 @@ impl WorkerPool {
|
||||
&self.exe
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod watchdog_tests {
|
||||
use super::*;
|
||||
use std::time::Duration;
|
||||
|
||||
/// An abort that returns disarms the deadman — the normal path then
|
||||
/// owns recovery and the process must NOT exit.
|
||||
#[test]
|
||||
fn abort_returning_disarms_the_deadman() {
|
||||
let done = Arc::new(AtomicBool::new(false));
|
||||
let signal = Arc::clone(&done);
|
||||
std::thread::spawn(move || {
|
||||
std::thread::sleep(Duration::from_millis(50));
|
||||
signal.store(true, Ordering::Release);
|
||||
});
|
||||
assert!(
|
||||
wait_for_abort(&done, Duration::from_secs(5)),
|
||||
"a returning abort must disarm the deadman"
|
||||
);
|
||||
}
|
||||
|
||||
/// The #265 case: `ncclCommAbort` never returns. The deadline must
|
||||
/// elapse and report unrecoverable, which is what triggers the exit.
|
||||
#[test]
|
||||
fn abort_that_never_returns_trips_the_deadline() {
|
||||
let done = Arc::new(AtomicBool::new(false)); // never set
|
||||
let start = std::time::Instant::now();
|
||||
assert!(
|
||||
!wait_for_abort(&done, Duration::from_millis(200)),
|
||||
"an abort that never returns must trip the deadline"
|
||||
);
|
||||
assert!(
|
||||
start.elapsed() >= Duration::from_millis(200),
|
||||
"must actually wait the deadline rather than failing fast"
|
||||
);
|
||||
}
|
||||
|
||||
/// An abort landing in the last instant still counts as returned —
|
||||
/// exiting on a race would restart a process that had just recovered.
|
||||
#[test]
|
||||
fn abort_landing_at_the_deadline_still_counts() {
|
||||
let done = Arc::new(AtomicBool::new(false));
|
||||
let signal = Arc::clone(&done);
|
||||
std::thread::spawn(move || {
|
||||
std::thread::sleep(Duration::from_millis(180));
|
||||
signal.store(true, Ordering::Release);
|
||||
});
|
||||
assert!(
|
||||
wait_for_abort(&done, Duration::from_millis(200)),
|
||||
"an abort landing at the deadline must not be treated as a hang"
|
||||
);
|
||||
}
|
||||
|
||||
/// The deadline is operator-tunable but never zero — a zero deadline
|
||||
/// would exit the moment any watchdog fired.
|
||||
#[test]
|
||||
fn deadline_defaults_sanely_and_rejects_zero() {
|
||||
assert_eq!(tp_abort_deadline(), Duration::from_secs(60));
|
||||
unsafe { std::env::set_var("NEURON_TP_ABORT_DEADLINE_S", "0") };
|
||||
assert_eq!(
|
||||
tp_abort_deadline(),
|
||||
Duration::from_secs(60),
|
||||
"zero must fall back, not exit instantly on every watchdog"
|
||||
);
|
||||
unsafe { std::env::set_var("NEURON_TP_ABORT_DEADLINE_S", "15") };
|
||||
assert_eq!(tp_abort_deadline(), Duration::from_secs(15));
|
||||
unsafe { std::env::remove_var("NEURON_TP_ABORT_DEADLINE_S") };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2445,7 +2445,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "cuda"))]
|
||||
fn vb_over(path: &std::path::Path) -> ShardedVarBuilder<'_> {
|
||||
fn vb_over(path: &std::path::Path) -> ShardedVarBuilder {
|
||||
// SAFETY: mmap of a file the test just wrote; nothing mutates it.
|
||||
unsafe {
|
||||
candle_nn::var_builder::ShardedSafeTensors::var_builder(
|
||||
|
||||
@@ -186,9 +186,6 @@ async fn daemon(args: Args) -> Result<()> {
|
||||
let registry = HarnessRegistry::from_configs(&cfg.harnesses, &bind_url, &cfg.harness);
|
||||
discovery_result.harnesses = registry.names();
|
||||
let candle = registry.candle();
|
||||
// A second handle for the shutdown future: it must close admission
|
||||
// before the drain begins, and `candle` is moved into NeuronState.
|
||||
let shutdown_candle = candle.clone();
|
||||
|
||||
let health_cache = Arc::new(health::HealthCache::new());
|
||||
health_cache
|
||||
@@ -249,7 +246,7 @@ async fn daemon(args: Args) -> Result<()> {
|
||||
}
|
||||
|
||||
axum::serve(listener, app)
|
||||
.with_graceful_shutdown(startup::shutdown_signal(shutdown_candle))
|
||||
.with_graceful_shutdown(startup::shutdown_signal())
|
||||
.await?;
|
||||
|
||||
// Deactivation: serve has returned (graceful shutdown signal
|
||||
|
||||
@@ -9,7 +9,6 @@ use crate::activation::ActivationTracker;
|
||||
use crate::harness::HarnessRegistry;
|
||||
use crate::harness::preflight::PreflightError;
|
||||
use cortex_core::harness::ModelSpec;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::signal;
|
||||
|
||||
@@ -166,18 +165,7 @@ fn preflight_kind(err: &PreflightError) -> &'static str {
|
||||
/// Wired into `axum::serve(...).with_graceful_shutdown(shutdown_signal())`
|
||||
/// so the HTTP listener stops accepting new connections, lets in-flight
|
||||
/// requests drain, and then yields control back to main for cleanup.
|
||||
///
|
||||
/// **Before returning, it closes admission and wakes every waiter (#256).**
|
||||
/// Order matters: axum's graceful shutdown waits for *all* in-flight
|
||||
/// requests, and a request queued behind the TP pool lock is in-flight as
|
||||
/// far as axum is concerned. Those waits are unbounded — held by whichever
|
||||
/// request is mid-forward — so without this the drain cannot finish and
|
||||
/// systemd SIGKILLs at `TimeoutStopSec`. Observed 2026-08-15: SIGTERM at
|
||||
/// 13:03:14, one queued request logging "still waiting on pool lock" every
|
||||
/// two seconds, `Failed with result 'timeout'` at 13:05:24. The same node
|
||||
/// drained in about a second when idle, which is what identified the
|
||||
/// queued waiter as the cause.
|
||||
pub async fn shutdown_signal(candle: Option<Arc<crate::harness::candle::CandleHarness>>) {
|
||||
pub async fn shutdown_signal() {
|
||||
let ctrl_c = async {
|
||||
signal::ctrl_c().await.ok();
|
||||
};
|
||||
@@ -191,13 +179,6 @@ pub async fn shutdown_signal(candle: Option<Arc<crate::harness::candle::CandleHa
|
||||
_ = ctrl_c => tracing::info!("received SIGINT, shutting down"),
|
||||
_ = terminate => tracing::info!("received SIGTERM, shutting down"),
|
||||
}
|
||||
// Refuse new admissions and abandon the unbounded waits, so the drain
|
||||
// that starts the moment this future resolves has a bounded amount of
|
||||
// work left to wait for.
|
||||
crate::harness::candle::begin_shutdown();
|
||||
if let Some(candle) = candle {
|
||||
candle.close_all_admission().await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Unload every model currently registered. Called from `main.rs` after
|
||||
|
||||
@@ -84,13 +84,6 @@ pub enum InferenceEvent {
|
||||
/// `output_tokens_details.reasoning_tokens` (responses).
|
||||
/// Zero for non-reasoning models.
|
||||
reasoning_tokens: u32,
|
||||
/// Leading prompt tokens served from the prefix cache (#269) — a
|
||||
/// sub-count of `prompt_tokens`, never added to `total_tokens`.
|
||||
/// Surfaced as `prompt_tokens_details.cached_tokens` (chat) /
|
||||
/// `input_tokens_details.cached_tokens` (responses). The cache
|
||||
/// has worked since #11; without this the saving was invisible
|
||||
/// and every client reported a 0% hit rate.
|
||||
cached_tokens: u32,
|
||||
/// Server-measured prefill/decode timing for the request, or
|
||||
/// `None` on paths that don't measure it (CPU fallback that
|
||||
/// doesn't instrument, tests). Streaming projectors surface
|
||||
|
||||
@@ -27,8 +27,7 @@
|
||||
//! propagate without us writing any logic.
|
||||
|
||||
use cortex_core::openai::{
|
||||
ChatCompletionChoice, ChatCompletionChunk, ChatCompletionResponse, ChatMessage, ChunkChoice,
|
||||
CompletionTokensDetails, HelexaTiming, MessageContent, PromptTokensDetails, Usage,
|
||||
ChatCompletionChunk, ChunkChoice, CompletionTokensDetails, HelexaTiming, Usage,
|
||||
};
|
||||
use serde_json::json;
|
||||
use tokio::sync::mpsc;
|
||||
@@ -140,35 +139,11 @@ pub fn project_chat_stream_with(
|
||||
}
|
||||
InferenceEvent::ReasoningDelta(text) => {
|
||||
if !config.include_thinking {
|
||||
// Default path. This used to `continue` —
|
||||
// dropping the token and emitting nothing at
|
||||
// all — to keep model-internal scratchpad out
|
||||
// of naïve clients' UIs. That goal was right;
|
||||
// the method silenced the stream.
|
||||
//
|
||||
// A reasoning model can think for minutes on a
|
||||
// hard prompt, and for that whole span not one
|
||||
// SSE event went out. Clients time that out: the
|
||||
// DeepSeek Harness reports `pi-ai stream idle
|
||||
// timeout after 300000ms` and retries, so a task
|
||||
// that reasons for over five minutes could never
|
||||
// complete no matter how healthy the model was.
|
||||
// SSE comment keep-alives do not help — idle
|
||||
// timers reset on events, not comments.
|
||||
//
|
||||
// Emitting `reasoning_content` instead keeps the
|
||||
// original guarantee (it is not `content`, so
|
||||
// unaware clients still show clean output) while
|
||||
// the stream stays visibly alive, and hands
|
||||
// reasoning-aware clients the field they are
|
||||
// already looking for.
|
||||
if text.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let chunk = reasoning_chunk(&id, created, &model_id, &text);
|
||||
if tx.send(chunk).await.is_err() {
|
||||
return;
|
||||
}
|
||||
// Default path — reasoning has no slot in
|
||||
// chat completions, so it's dropped. Naïve
|
||||
// clients (Zed commit-message generator,
|
||||
// any vanilla OpenAI client) get clean
|
||||
// output.
|
||||
continue;
|
||||
}
|
||||
let Some(markers) = config.reasoning_markers.as_ref() else {
|
||||
@@ -220,7 +195,6 @@ pub fn project_chat_stream_with(
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
reasoning_tokens,
|
||||
cached_tokens,
|
||||
timing,
|
||||
} => {
|
||||
// The finish_reason chunk, then an OpenAI-style
|
||||
@@ -235,12 +209,9 @@ pub fn project_chat_stream_with(
|
||||
&id,
|
||||
created,
|
||||
&model_id,
|
||||
UsageCounts {
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
reasoning_tokens,
|
||||
cached_tokens,
|
||||
},
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
reasoning_tokens,
|
||||
timing,
|
||||
),
|
||||
]
|
||||
@@ -275,35 +246,6 @@ fn role_chunk(id: &str, created: u64, model_id: &str) -> ChatCompletionChunk {
|
||||
}
|
||||
}
|
||||
|
||||
/// A reasoning delta, in the field reasoning-aware clients read.
|
||||
///
|
||||
/// `reasoning_content` is not in the OpenAI spec, but it is the
|
||||
/// de-facto slot: DeepSeek's API emits it, and so do vLLM and SGLang
|
||||
/// for reasoning models. Our own consumers already expect it —
|
||||
/// `helexa-acp`'s OpenAI provider reads `choice.delta.reasoning_content`,
|
||||
/// and `helexa-bench` times a stream's first reasoning token from it.
|
||||
///
|
||||
/// Crucially it is NOT `content`, so a client that has never heard of
|
||||
/// reasoning ignores the unknown field and sees clean output — the
|
||||
/// property the old drop-it-entirely default was protecting, kept
|
||||
/// without the silence that came with it.
|
||||
fn reasoning_chunk(id: &str, created: u64, model_id: &str, text: &str) -> ChatCompletionChunk {
|
||||
ChatCompletionChunk {
|
||||
id: id.into(),
|
||||
object: "chat.completion.chunk".into(),
|
||||
created,
|
||||
model: model_id.into(),
|
||||
choices: vec![ChunkChoice {
|
||||
index: 0,
|
||||
delta: json!({ "reasoning_content": text }),
|
||||
finish_reason: None,
|
||||
extra: serde_json::Value::Object(Default::default()),
|
||||
}],
|
||||
usage: None,
|
||||
extra: serde_json::Value::Object(Default::default()),
|
||||
}
|
||||
}
|
||||
|
||||
fn content_chunk(id: &str, created: u64, model_id: &str, text: &str) -> ChatCompletionChunk {
|
||||
ChatCompletionChunk {
|
||||
id: id.into(),
|
||||
@@ -390,32 +332,15 @@ fn final_chunk(
|
||||
/// ignore the empty-choices chunk; clients that do (opencode, and
|
||||
/// cortex's Anthropic translator) get the token counts they need to
|
||||
/// track context.
|
||||
/// The token counts a finished turn reports. Grouped rather than passed
|
||||
/// loose: adding `cached_tokens` (#269) put the list past the point
|
||||
/// where four adjacent bare `u32`s are safe to read at a call site.
|
||||
struct UsageCounts {
|
||||
prompt_tokens: u32,
|
||||
completion_tokens: u32,
|
||||
/// Sub-count of `completion_tokens` — tokens spent reasoning.
|
||||
reasoning_tokens: u32,
|
||||
/// Sub-count of `prompt_tokens` — tokens served from the prefix
|
||||
/// cache. Neither sub-count is added into `total_tokens`.
|
||||
cached_tokens: u32,
|
||||
}
|
||||
|
||||
fn usage_chunk(
|
||||
id: &str,
|
||||
created: u64,
|
||||
model_id: &str,
|
||||
counts: UsageCounts,
|
||||
prompt_tokens: u32,
|
||||
completion_tokens: u32,
|
||||
reasoning_tokens: u32,
|
||||
timing: Option<FinishTiming>,
|
||||
) -> ChatCompletionChunk {
|
||||
let UsageCounts {
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
reasoning_tokens,
|
||||
cached_tokens,
|
||||
} = counts;
|
||||
ChatCompletionChunk {
|
||||
id: id.into(),
|
||||
object: "chat.completion.chunk".into(),
|
||||
@@ -431,12 +356,7 @@ fn usage_chunk(
|
||||
completion_tokens_details: (reasoning_tokens > 0).then_some(CompletionTokensDetails {
|
||||
reasoning_tokens: reasoning_tokens as u64,
|
||||
}),
|
||||
// Prefix-cache reuse (#269). Omitted at zero so a client
|
||||
// sees unchanged JSON when nothing was cached, and so
|
||||
// "absent" stays distinguishable from "measured as none".
|
||||
prompt_tokens_details: (cached_tokens > 0).then_some(PromptTokensDetails {
|
||||
cached_tokens: cached_tokens as u64,
|
||||
}),
|
||||
prompt_tokens_details: None,
|
||||
// helexa extension (#85): server-measured prefill/decode
|
||||
// timing for the bench harness. Omitted on paths that don't
|
||||
// measure it so standard clients see unchanged JSON.
|
||||
@@ -450,379 +370,10 @@ fn usage_chunk(
|
||||
}
|
||||
}
|
||||
|
||||
/// Drain an [`InferenceEvent`] stream into a single non-streaming
|
||||
/// [`ChatCompletionResponse`] (#285).
|
||||
///
|
||||
/// Non-streaming used to be a *separate inference path* that took the
|
||||
/// per-model `inference_lock` for the whole request. That made it
|
||||
/// invisible to the batch engine (#98), so concurrent non-streaming
|
||||
/// callers serialized completely — measured on beast at 1.00x aggregate
|
||||
/// throughput across 1/2/4/8 concurrency, against 3.98x for the same
|
||||
/// work streamed. `max_in_flight` bought those requests admission and
|
||||
/// nothing else.
|
||||
///
|
||||
/// Expressing non-streaming as "run the stream, buffer the result"
|
||||
/// collapses the two paths onto one producer, so batching, prefix-cache
|
||||
/// reuse, tool-call extraction, the reasoning governor and server-side
|
||||
/// timing all apply identically to both. It also removes a second place
|
||||
/// where the same wire object could be assembled differently — the
|
||||
/// shape of defect #252 and #272 both were.
|
||||
///
|
||||
/// Reasoning is kept out of `content` — it is not the assistant's
|
||||
/// reply — but it is **returned** on `message.reasoning_content`, the
|
||||
/// same field the streaming path emits as
|
||||
/// `choice.delta.reasoning_content`.
|
||||
///
|
||||
/// It used to be discarded here (#300). That was inherited from the
|
||||
/// path this replaced and was wrong on both counts: the tokens are
|
||||
/// metered in `usage.completion_tokens_details.reasoning_tokens`, so a
|
||||
/// caller paid for text it could never see; and a client cannot replay
|
||||
/// what it never received, so on the next turn the model saw its own
|
||||
/// prior turn as a bare assertion with the reasoning that produced it
|
||||
/// deleted.
|
||||
///
|
||||
/// `finish_reason` becomes `"tool_calls"` whenever any tool call was
|
||||
/// parsed.
|
||||
///
|
||||
/// Two fields that the old path hardcoded to `None` now populate,
|
||||
/// because the events carry them and the struct already had the slots:
|
||||
/// `prompt_tokens_details.cached_tokens` and `helexa_timing`.
|
||||
///
|
||||
/// A stream that ends without a `Finish` is an error, not an empty
|
||||
/// reply: that is how a dropped engine slot or a fatal worker fault
|
||||
/// presents, and returning a 200 with partial text would hide it.
|
||||
pub async fn collect_chat_completion(
|
||||
mut rx: mpsc::Receiver<InferenceEvent>,
|
||||
id: String,
|
||||
created: u64,
|
||||
model_id: String,
|
||||
) -> Result<ChatCompletionResponse, String> {
|
||||
let mut content = String::new();
|
||||
let mut reasoning = String::new();
|
||||
let mut tool_calls: Vec<serde_json::Value> = Vec::new();
|
||||
let mut finish: Option<(FinishReason, u32, u32, u32, u32, Option<FinishTiming>)> = None;
|
||||
|
||||
while let Some(event) = rx.recv().await {
|
||||
match event {
|
||||
InferenceEvent::Start => {}
|
||||
InferenceEvent::TextDelta(t) => content.push_str(&t),
|
||||
// Kept out of `content`, returned on `reasoning_content`.
|
||||
InferenceEvent::ReasoningDelta(t) => reasoning.push_str(&t),
|
||||
InferenceEvent::ToolCall {
|
||||
index,
|
||||
id: call_id,
|
||||
name,
|
||||
arguments,
|
||||
} => tool_calls.push(json!({
|
||||
"index": index,
|
||||
"id": call_id,
|
||||
"type": "function",
|
||||
"function": { "name": name, "arguments": arguments },
|
||||
})),
|
||||
InferenceEvent::Finish {
|
||||
reason,
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
reasoning_tokens,
|
||||
cached_tokens,
|
||||
timing,
|
||||
} => {
|
||||
finish = Some((
|
||||
reason,
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
reasoning_tokens,
|
||||
cached_tokens,
|
||||
timing,
|
||||
));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let Some((reason, prompt_tokens, completion_tokens, reasoning_tokens, cached_tokens, timing)) =
|
||||
finish
|
||||
else {
|
||||
return Err(
|
||||
"inference stream ended without a finish event (engine slot dropped?)".to_string(),
|
||||
);
|
||||
};
|
||||
|
||||
// A parsed tool call outranks whatever the sampler stopped on, so a
|
||||
// model that emits a call and then hits EOS still reports
|
||||
// `tool_calls` and the client dispatches instead of rendering.
|
||||
let finish_reason = if tool_calls.is_empty() {
|
||||
reason.as_openai_str().to_string()
|
||||
} else {
|
||||
"tool_calls".to_string()
|
||||
};
|
||||
let mut message_extra = serde_json::Map::new();
|
||||
if !tool_calls.is_empty() {
|
||||
message_extra.insert("tool_calls".into(), json!(tool_calls));
|
||||
}
|
||||
// Absent rather than empty when the model did not think: a
|
||||
// non-reasoning model must keep the unchanged message shape.
|
||||
if !reasoning.is_empty() {
|
||||
message_extra.insert("reasoning_content".into(), json!(reasoning));
|
||||
}
|
||||
let message_extra = serde_json::Value::Object(message_extra);
|
||||
|
||||
Ok(ChatCompletionResponse {
|
||||
id,
|
||||
object: "chat.completion".into(),
|
||||
created,
|
||||
model: model_id,
|
||||
choices: vec![ChatCompletionChoice {
|
||||
index: 0,
|
||||
message: ChatMessage {
|
||||
role: "assistant".into(),
|
||||
content: MessageContent::Text(content),
|
||||
extra: message_extra,
|
||||
},
|
||||
finish_reason: Some(finish_reason),
|
||||
extra: serde_json::Value::Object(Default::default()),
|
||||
}],
|
||||
usage: Some(Usage {
|
||||
prompt_tokens: prompt_tokens as u64,
|
||||
completion_tokens: completion_tokens as u64,
|
||||
total_tokens: (prompt_tokens + completion_tokens) as u64,
|
||||
completion_tokens_details: (reasoning_tokens > 0).then_some(CompletionTokensDetails {
|
||||
reasoning_tokens: reasoning_tokens as u64,
|
||||
}),
|
||||
prompt_tokens_details: (cached_tokens > 0).then_some(PromptTokensDetails {
|
||||
cached_tokens: cached_tokens as u64,
|
||||
}),
|
||||
helexa_timing: timing.map(|t| HelexaTiming {
|
||||
prefill_ms: t.prefill_ms as u64,
|
||||
decode_ms: t.decode_ms as u64,
|
||||
prefill_tokens: t.prefill_tokens as u64,
|
||||
}),
|
||||
}),
|
||||
extra: serde_json::Value::Object(Default::default()),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
/// The model's think block must come back on the non-streaming
|
||||
/// path too (#300).
|
||||
///
|
||||
/// It used to be dropped here while the streaming path emitted it
|
||||
/// as `choice.delta.reasoning_content`. The tokens were metered
|
||||
/// either way, so a non-streaming caller paid for text it could
|
||||
/// never see — and, worse, could not replay the model's reasoning
|
||||
/// on the next turn, leaving the model to face its own prior turn
|
||||
/// as a bare assertion.
|
||||
#[tokio::test]
|
||||
async fn non_streaming_returns_the_models_reasoning() {
|
||||
let (tx, rx) = mpsc::channel::<InferenceEvent>(8);
|
||||
tx.send(InferenceEvent::Start).await.unwrap();
|
||||
tx.send(InferenceEvent::ReasoningDelta(
|
||||
"weighing the options".into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
tx.send(InferenceEvent::TextDelta("the answer".into()))
|
||||
.await
|
||||
.unwrap();
|
||||
tx.send(InferenceEvent::Finish {
|
||||
reason: FinishReason::Stop,
|
||||
prompt_tokens: 5,
|
||||
completion_tokens: 7,
|
||||
reasoning_tokens: 3,
|
||||
cached_tokens: 0,
|
||||
timing: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
drop(tx);
|
||||
|
||||
let r = collect_chat_completion(rx, "id".into(), 0, "m".into())
|
||||
.await
|
||||
.expect("collect");
|
||||
let msg = &r.choices[0].message;
|
||||
assert_eq!(
|
||||
msg.extra["reasoning_content"], "weighing the options",
|
||||
"reasoning must be returned, not discarded"
|
||||
);
|
||||
// And it must NOT be folded into the visible reply.
|
||||
assert!(
|
||||
matches!(&msg.content, MessageContent::Text(t) if t == "the answer"),
|
||||
"reasoning leaked into content"
|
||||
);
|
||||
}
|
||||
|
||||
/// A non-reasoning model must keep the unchanged message shape —
|
||||
/// absent, not an empty string.
|
||||
#[tokio::test]
|
||||
async fn no_reasoning_means_no_reasoning_field() {
|
||||
let (tx, rx) = mpsc::channel::<InferenceEvent>(8);
|
||||
tx.send(InferenceEvent::Start).await.unwrap();
|
||||
tx.send(InferenceEvent::TextDelta("plain".into()))
|
||||
.await
|
||||
.unwrap();
|
||||
tx.send(InferenceEvent::Finish {
|
||||
reason: FinishReason::Stop,
|
||||
prompt_tokens: 1,
|
||||
completion_tokens: 1,
|
||||
reasoning_tokens: 0,
|
||||
cached_tokens: 0,
|
||||
timing: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
drop(tx);
|
||||
let r = collect_chat_completion(rx, "id".into(), 0, "m".into())
|
||||
.await
|
||||
.expect("collect");
|
||||
assert!(
|
||||
r.choices[0]
|
||||
.message
|
||||
.extra
|
||||
.get("reasoning_content")
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
use super::*;
|
||||
|
||||
// ── Non-streaming collector (#285) ────────────────────────────
|
||||
|
||||
async fn collect_from(events: Vec<InferenceEvent>) -> Result<ChatCompletionResponse, String> {
|
||||
let (tx, rx) = mpsc::channel(64);
|
||||
for e in events {
|
||||
tx.send(e).await.expect("send");
|
||||
}
|
||||
drop(tx);
|
||||
collect_chat_completion(rx, "chatcmpl-test".into(), 1_700_000_000, "m".into()).await
|
||||
}
|
||||
|
||||
fn finish(reason: FinishReason) -> InferenceEvent {
|
||||
InferenceEvent::Finish {
|
||||
reason,
|
||||
prompt_tokens: 10,
|
||||
completion_tokens: 4,
|
||||
reasoning_tokens: 0,
|
||||
cached_tokens: 0,
|
||||
timing: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collector_concatenates_text_deltas() {
|
||||
let r = collect_from(vec![
|
||||
InferenceEvent::Start,
|
||||
InferenceEvent::TextDelta("hel".into()),
|
||||
InferenceEvent::TextDelta("lo".into()),
|
||||
finish(FinishReason::Stop),
|
||||
])
|
||||
.await
|
||||
.expect("collected");
|
||||
let c = &r.choices[0];
|
||||
assert!(matches!(&c.message.content, MessageContent::Text(t) if t == "hello"));
|
||||
assert_eq!(c.finish_reason.as_deref(), Some("stop"));
|
||||
assert_eq!(r.object, "chat.completion");
|
||||
}
|
||||
|
||||
/// Reasoning is counted but never rendered — the behaviour of the
|
||||
/// path this replaced. A regression here would leak scratchpad into
|
||||
/// `content` for every non-streaming caller.
|
||||
#[tokio::test]
|
||||
async fn collector_drops_reasoning_text_but_keeps_the_count() {
|
||||
let (tx, rx) = mpsc::channel(64);
|
||||
tx.send(InferenceEvent::ReasoningDelta("secret".into()))
|
||||
.await
|
||||
.expect("send");
|
||||
tx.send(InferenceEvent::TextDelta("answer".into()))
|
||||
.await
|
||||
.expect("send");
|
||||
tx.send(InferenceEvent::Finish {
|
||||
reason: FinishReason::Stop,
|
||||
prompt_tokens: 10,
|
||||
completion_tokens: 9,
|
||||
reasoning_tokens: 5,
|
||||
cached_tokens: 0,
|
||||
timing: None,
|
||||
})
|
||||
.await
|
||||
.expect("send");
|
||||
drop(tx);
|
||||
let r = collect_chat_completion(rx, "id".into(), 1, "m".into())
|
||||
.await
|
||||
.expect("collected");
|
||||
let c = &r.choices[0];
|
||||
assert!(matches!(&c.message.content, MessageContent::Text(t) if t == "answer"));
|
||||
let u = r.usage.expect("usage");
|
||||
assert_eq!(
|
||||
u.completion_tokens_details
|
||||
.expect("details")
|
||||
.reasoning_tokens,
|
||||
5
|
||||
);
|
||||
}
|
||||
|
||||
/// A parsed tool call outranks the sampler's stop reason, matching
|
||||
/// the streaming projector.
|
||||
#[tokio::test]
|
||||
async fn collector_promotes_finish_reason_to_tool_calls() {
|
||||
let r = collect_from(vec![
|
||||
InferenceEvent::ToolCall {
|
||||
index: 0,
|
||||
id: "call_1".into(),
|
||||
name: "read_file".into(),
|
||||
arguments: r#"{"path":"a.txt"}"#.into(),
|
||||
},
|
||||
finish(FinishReason::Stop),
|
||||
])
|
||||
.await
|
||||
.expect("collected");
|
||||
let c = &r.choices[0];
|
||||
assert_eq!(c.finish_reason.as_deref(), Some("tool_calls"));
|
||||
let calls = c.message.extra.get("tool_calls").expect("tool_calls");
|
||||
assert_eq!(calls[0]["function"]["name"], "read_file");
|
||||
assert_eq!(calls[0]["type"], "function");
|
||||
assert_eq!(calls[0]["id"], "call_1");
|
||||
}
|
||||
|
||||
/// cached_tokens and helexa_timing were hardcoded to `None` on the
|
||||
/// old non-streaming path; the events carried them all along.
|
||||
#[tokio::test]
|
||||
async fn collector_surfaces_cached_tokens_and_timing() {
|
||||
let (tx, rx) = mpsc::channel(8);
|
||||
tx.send(InferenceEvent::Finish {
|
||||
reason: FinishReason::Stop,
|
||||
prompt_tokens: 100,
|
||||
completion_tokens: 5,
|
||||
reasoning_tokens: 0,
|
||||
cached_tokens: 64,
|
||||
timing: Some(FinishTiming {
|
||||
prefill_ms: 12,
|
||||
decode_ms: 34,
|
||||
prefill_tokens: 100,
|
||||
}),
|
||||
})
|
||||
.await
|
||||
.expect("send");
|
||||
drop(tx);
|
||||
let r = collect_chat_completion(rx, "id".into(), 1, "m".into())
|
||||
.await
|
||||
.expect("collected");
|
||||
let u = r.usage.expect("usage");
|
||||
assert_eq!(u.prompt_tokens_details.expect("details").cached_tokens, 64);
|
||||
let t = u.helexa_timing.expect("timing");
|
||||
assert_eq!((t.prefill_ms, t.decode_ms, t.prefill_tokens), (12, 34, 100));
|
||||
}
|
||||
|
||||
/// A dropped engine slot ends the channel with no Finish. Returning
|
||||
/// a 200 with partial text would hide a fatal worker fault.
|
||||
#[tokio::test]
|
||||
async fn collector_errors_when_the_stream_ends_without_finish() {
|
||||
let err = collect_from(vec![InferenceEvent::TextDelta("partial".into())])
|
||||
.await
|
||||
.expect_err("must not succeed");
|
||||
assert!(err.contains("without a finish"), "unexpected: {err}");
|
||||
}
|
||||
|
||||
/// Drain the projection's output into a Vec for assertion.
|
||||
async fn collect(mut rx: mpsc::Receiver<ChatCompletionChunk>) -> Vec<ChatCompletionChunk> {
|
||||
let mut out = Vec::new();
|
||||
@@ -854,7 +405,6 @@ mod tests {
|
||||
prompt_tokens: 0,
|
||||
completion_tokens: 0,
|
||||
reasoning_tokens: 0,
|
||||
cached_tokens: 0,
|
||||
timing: None,
|
||||
})
|
||||
.await
|
||||
@@ -893,7 +443,6 @@ mod tests {
|
||||
prompt_tokens: 128,
|
||||
completion_tokens: 64,
|
||||
reasoning_tokens: 0,
|
||||
cached_tokens: 0,
|
||||
timing: Some(FinishTiming {
|
||||
prefill_ms: 200,
|
||||
decode_ms: 1500,
|
||||
@@ -918,73 +467,6 @@ mod tests {
|
||||
assert_eq!(timing.prefill_tokens, 128);
|
||||
}
|
||||
|
||||
/// A cache hit must reach the caller (#269).
|
||||
///
|
||||
/// Prefix caching has worked since #11 — neuron logs `reused=2068`
|
||||
/// on prefills — but nothing carried the number onto the wire, so
|
||||
/// every client reported a 0% hit rate on a cache that was saving
|
||||
/// real prefill. dsh showed `Cache hit 0%` against a 42.1K-token
|
||||
/// context, and an agentic harness that believes nothing is cached
|
||||
/// may restructure the very prefix that was being reused.
|
||||
#[tokio::test]
|
||||
async fn cached_tokens_reach_the_usage_chunk() {
|
||||
let (tx, rx) = mpsc::channel::<InferenceEvent>(4);
|
||||
let out_rx = project_chat_stream(rx, "id".into(), 1, "m".into());
|
||||
tx.send(InferenceEvent::Finish {
|
||||
reason: FinishReason::Stop,
|
||||
prompt_tokens: 7413,
|
||||
completion_tokens: 12,
|
||||
reasoning_tokens: 0,
|
||||
cached_tokens: 2068,
|
||||
timing: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
drop(tx);
|
||||
|
||||
let out = collect(out_rx).await;
|
||||
let usage = out
|
||||
.iter()
|
||||
.find_map(|c| c.usage.as_ref())
|
||||
.expect("usage chunk present");
|
||||
let details = usage
|
||||
.prompt_tokens_details
|
||||
.as_ref()
|
||||
.expect("prompt_tokens_details populated when the cache was hit");
|
||||
assert_eq!(details.cached_tokens, 2068);
|
||||
// A sub-count of prompt_tokens, never added into the total —
|
||||
// otherwise a cache hit would inflate what a caller is billed.
|
||||
assert_eq!(usage.prompt_tokens, 7413);
|
||||
assert_eq!(usage.total_tokens, 7425);
|
||||
}
|
||||
|
||||
/// No reuse means no `prompt_tokens_details` at all, so a client
|
||||
/// sees byte-identical JSON to before this landed — and "absent"
|
||||
/// stays distinguishable from "measured as zero".
|
||||
#[tokio::test]
|
||||
async fn no_cache_hit_omits_the_details_object() {
|
||||
let (tx, rx) = mpsc::channel::<InferenceEvent>(4);
|
||||
let out_rx = project_chat_stream(rx, "id".into(), 1, "m".into());
|
||||
tx.send(InferenceEvent::Finish {
|
||||
reason: FinishReason::Stop,
|
||||
prompt_tokens: 10,
|
||||
completion_tokens: 1,
|
||||
reasoning_tokens: 0,
|
||||
cached_tokens: 0,
|
||||
timing: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
drop(tx);
|
||||
|
||||
let usage = collect(out_rx)
|
||||
.await
|
||||
.iter()
|
||||
.find_map(|c| c.usage.clone())
|
||||
.expect("usage chunk present");
|
||||
assert!(usage.prompt_tokens_details.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_text_delta_is_dropped() {
|
||||
let (tx, rx) = mpsc::channel::<InferenceEvent>(4);
|
||||
@@ -1006,7 +488,6 @@ mod tests {
|
||||
prompt_tokens: 0,
|
||||
completion_tokens: 0,
|
||||
reasoning_tokens: 0,
|
||||
cached_tokens: 0,
|
||||
timing: None,
|
||||
})
|
||||
.await
|
||||
@@ -1019,24 +500,10 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
/// Reasoning is streamed as `reasoning_content`, never as `content`.
|
||||
///
|
||||
/// Both halves matter. Keeping it out of `content` is what stops
|
||||
/// scratchpad material appearing in the UI of a client that has
|
||||
/// never heard of reasoning — the guarantee the previous
|
||||
/// drop-everything default provided.
|
||||
///
|
||||
/// But emitting *something* is what keeps the connection alive. A
|
||||
/// model can think for minutes; while it did, this projection sent
|
||||
/// no events at all, and clients closed the stream as dead. The
|
||||
/// DeepSeek Harness reports `pi-ai stream idle timeout after
|
||||
/// 300000ms`, so any task whose reasoning outlasts the client's
|
||||
/// idle timeout could never complete. SSE comment keep-alives do
|
||||
/// not help, because idle timers reset on events, not comments.
|
||||
async fn reasoning_delta_is_streamed_as_reasoning_content() {
|
||||
async fn reasoning_delta_is_dropped_in_chat_projection() {
|
||||
let (tx, rx) = mpsc::channel::<InferenceEvent>(4);
|
||||
let out_rx = project_chat_stream(rx, "id".into(), 1, "m".into());
|
||||
tx.send(InferenceEvent::ReasoningDelta("pondering".into()))
|
||||
tx.send(InferenceEvent::ReasoningDelta("<think>".into()))
|
||||
.await
|
||||
.unwrap();
|
||||
tx.send(InferenceEvent::TextDelta("real".into()))
|
||||
@@ -1044,27 +511,8 @@ mod tests {
|
||||
.unwrap();
|
||||
drop(tx);
|
||||
let out = collect(out_rx).await;
|
||||
assert_eq!(out.len(), 2, "reasoning must produce a chunk, not silence");
|
||||
assert_eq!(out[0].choices[0].delta["reasoning_content"], "pondering");
|
||||
assert!(
|
||||
out[0].choices[0].delta.get("content").is_none(),
|
||||
"reasoning must never arrive as content"
|
||||
);
|
||||
assert_eq!(out[1].choices[0].delta["content"], "real");
|
||||
}
|
||||
|
||||
/// An empty reasoning delta is still nothing to say — it must not
|
||||
/// manufacture a chunk, matching the text path.
|
||||
#[tokio::test]
|
||||
async fn empty_reasoning_delta_is_dropped() {
|
||||
let (tx, rx) = mpsc::channel::<InferenceEvent>(4);
|
||||
let out_rx = project_chat_stream(rx, "id".into(), 1, "m".into());
|
||||
tx.send(InferenceEvent::ReasoningDelta(String::new()))
|
||||
.await
|
||||
.unwrap();
|
||||
drop(tx);
|
||||
let out = collect(out_rx).await;
|
||||
assert!(out.is_empty(), "empty reasoning deltas produce no chunks");
|
||||
assert_eq!(out.len(), 1);
|
||||
assert_eq!(out[0].choices[0].delta["content"], "real");
|
||||
}
|
||||
|
||||
fn pair() -> ReasoningTokenPair {
|
||||
@@ -1103,7 +551,6 @@ mod tests {
|
||||
prompt_tokens: 0,
|
||||
completion_tokens: 0,
|
||||
reasoning_tokens: 0,
|
||||
cached_tokens: 0,
|
||||
timing: None,
|
||||
})
|
||||
.await
|
||||
@@ -1156,7 +603,6 @@ mod tests {
|
||||
prompt_tokens: 0,
|
||||
completion_tokens: 0,
|
||||
reasoning_tokens: 0,
|
||||
cached_tokens: 0,
|
||||
timing: None,
|
||||
})
|
||||
.await
|
||||
@@ -1203,7 +649,6 @@ mod tests {
|
||||
prompt_tokens: 0,
|
||||
completion_tokens: 0,
|
||||
reasoning_tokens: 0,
|
||||
cached_tokens: 0,
|
||||
timing: None,
|
||||
})
|
||||
.await
|
||||
@@ -1222,12 +667,10 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
/// With the flag off, reasoning stays out of `content` whether or
|
||||
/// not markers are configured — the flag is the gate, not the
|
||||
/// marker presence. It now travels as `reasoning_content` instead
|
||||
/// of being discarded, so this asserts the containment guarantee
|
||||
/// only: nothing model-internal reaches a naive client's `content`.
|
||||
async fn include_thinking_off_keeps_reasoning_out_of_content() {
|
||||
async fn include_thinking_off_drops_reasoning_even_with_markers() {
|
||||
// Default behaviour even when markers happen to be
|
||||
// configured. The flag is the gate, not the marker
|
||||
// presence.
|
||||
let (tx, rx) = mpsc::channel::<InferenceEvent>(4);
|
||||
let out_rx = project_chat_stream_with(
|
||||
rx,
|
||||
@@ -1250,7 +693,6 @@ mod tests {
|
||||
prompt_tokens: 0,
|
||||
completion_tokens: 0,
|
||||
reasoning_tokens: 0,
|
||||
cached_tokens: 0,
|
||||
timing: None,
|
||||
})
|
||||
.await
|
||||
@@ -1279,7 +721,6 @@ mod tests {
|
||||
prompt_tokens: 42,
|
||||
completion_tokens: 5,
|
||||
reasoning_tokens: 2,
|
||||
cached_tokens: 0,
|
||||
timing: None,
|
||||
})
|
||||
.await
|
||||
@@ -1314,7 +755,6 @@ mod tests {
|
||||
prompt_tokens: 10,
|
||||
completion_tokens: 7,
|
||||
reasoning_tokens: 0,
|
||||
cached_tokens: 0,
|
||||
timing: None,
|
||||
})
|
||||
.await
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -29,8 +29,6 @@ async fn test_load_default_models_skips_unknown_harness() {
|
||||
quant: None,
|
||||
tensor_parallel: None,
|
||||
devices: None,
|
||||
sampling: None,
|
||||
preserve_thinking: None,
|
||||
},
|
||||
ModelSpec {
|
||||
model_id: "model-b".into(),
|
||||
@@ -38,8 +36,6 @@ async fn test_load_default_models_skips_unknown_harness() {
|
||||
quant: None,
|
||||
tensor_parallel: None,
|
||||
devices: None,
|
||||
sampling: None,
|
||||
preserve_thinking: None,
|
||||
},
|
||||
];
|
||||
|
||||
@@ -99,8 +95,6 @@ async fn test_load_default_models_skipped_on_driver_mismatch() {
|
||||
quant: Some("q6k".into()),
|
||||
tensor_parallel: Some(2),
|
||||
devices: None,
|
||||
sampling: None,
|
||||
preserve_thinking: None,
|
||||
}];
|
||||
let activation = ActivationTracker::new(&specs);
|
||||
let reason = "host NVIDIA driver/library mismatch (userspace NVML 580.159 vs loaded \
|
||||
@@ -202,8 +196,6 @@ fn qwen_spec() -> ModelSpec {
|
||||
quant: None,
|
||||
tensor_parallel: None,
|
||||
devices: None,
|
||||
sampling: None,
|
||||
preserve_thinking: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -709,71 +709,3 @@ async fn test_images_chat_at_image_endpoint_is_not_found_for_text_model() {
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), 404);
|
||||
}
|
||||
|
||||
/// A malformed body must come back as the #63 envelope, not axum's bare
|
||||
/// string (#253). An OpenAI client reads `error.code` / `error.type`; a
|
||||
/// plain string leaves it with nothing to act on, which is exactly how a
|
||||
/// harness ended up quitting silently mid-session.
|
||||
#[tokio::test]
|
||||
async fn malformed_body_returns_a_parseable_error_envelope() {
|
||||
let base = spawn_neuron(fake_discovery()).await;
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
// Valid JSON, wrong shape for the request type → 422.
|
||||
let resp = client
|
||||
.post(format!("{base}/v1/chat/completions"))
|
||||
.json(&json!({"model": "m", "messages": "not-an-array"}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), 422);
|
||||
let body: serde_json::Value = resp.json().await.expect("body must be JSON");
|
||||
assert_eq!(body["error"]["type"], "invalid_request_error");
|
||||
assert_eq!(body["error"]["code"], "invalid_request_body");
|
||||
assert!(
|
||||
body["error"]["message"]
|
||||
.as_str()
|
||||
.is_some_and(|m| !m.is_empty()),
|
||||
"message must say what was wrong: {body}"
|
||||
);
|
||||
|
||||
// Not JSON at all → 400, distinguished from the above so the caller
|
||||
// knows whether to fix its serializer or its schema.
|
||||
let resp = client
|
||||
.post(format!("{base}/v1/chat/completions"))
|
||||
.header("content-type", "application/json")
|
||||
.body("{not json")
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), 400);
|
||||
let body: serde_json::Value = resp.json().await.expect("body must be JSON");
|
||||
assert_eq!(body["error"]["code"], "malformed_json");
|
||||
}
|
||||
|
||||
/// The shape from #253 itself: an assistant turn carrying only
|
||||
/// `tool_calls`. It must be *accepted* now — this guards the envelope
|
||||
/// work from quietly re-breaking what #253 fixed.
|
||||
#[tokio::test]
|
||||
async fn null_assistant_content_is_not_a_body_rejection() {
|
||||
let base = spawn_neuron(fake_discovery()).await;
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{base}/v1/chat/completions"))
|
||||
.json(&json!({
|
||||
"model": "m",
|
||||
"messages": [
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "assistant", "content": null, "tool_calls": [
|
||||
{"id": "c1", "type": "function",
|
||||
"function": {"name": "b", "arguments": "{}"}}]},
|
||||
{"role": "tool", "content": "ok", "tool_call_id": "c1"}
|
||||
]
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
// 503 (no candle harness in this fixture) — the point is that it got
|
||||
// past body extraction rather than being rejected as malformed.
|
||||
assert_ne!(resp.status(), 422, "null content must not be a body error");
|
||||
assert_ne!(resp.status(), 400, "null content must not be a body error");
|
||||
}
|
||||
|
||||
@@ -53,8 +53,6 @@ async fn test_candle_qwen3_load_unload_lifecycle() {
|
||||
quant: Some(quant),
|
||||
tensor_parallel: None,
|
||||
devices: Some(vec![0]),
|
||||
sampling: None,
|
||||
preserve_thinking: None,
|
||||
};
|
||||
|
||||
registry
|
||||
|
||||
@@ -1,170 +0,0 @@
|
||||
{%- set image_count = namespace(value=0) %}
|
||||
{%- set video_count = namespace(value=0) %}
|
||||
{%- macro render_content(content, do_vision_count, is_system_content=false) %}
|
||||
{%- if content is string %}
|
||||
{{- content }}
|
||||
{%- elif content is iterable and content is not mapping %}
|
||||
{%- for item in content %}
|
||||
{%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
|
||||
{%- if is_system_content %}
|
||||
{{- raise_exception('System message cannot contain images.') }}
|
||||
{%- endif %}
|
||||
{%- if do_vision_count %}
|
||||
{%- set image_count.value = image_count.value + 1 %}
|
||||
{%- endif %}
|
||||
{%- if add_vision_id %}
|
||||
{{- 'Picture ' ~ image_count.value ~ ': ' }}
|
||||
{%- endif %}
|
||||
{{- '<|vision_start|><|image_pad|><|vision_end|>' }}
|
||||
{%- elif 'video' in item or item.type == 'video' %}
|
||||
{%- if is_system_content %}
|
||||
{{- raise_exception('System message cannot contain videos.') }}
|
||||
{%- endif %}
|
||||
{%- if do_vision_count %}
|
||||
{%- set video_count.value = video_count.value + 1 %}
|
||||
{%- endif %}
|
||||
{%- if add_vision_id %}
|
||||
{{- 'Video ' ~ video_count.value ~ ': ' }}
|
||||
{%- endif %}
|
||||
{{- '<|vision_start|><|video_pad|><|vision_end|>' }}
|
||||
{%- elif 'text' in item %}
|
||||
{{- item.text }}
|
||||
{%- else %}
|
||||
{{- raise_exception('Unexpected item type in content.') }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- elif content is none or content is undefined %}
|
||||
{{- '' }}
|
||||
{%- else %}
|
||||
{{- raise_exception('Unexpected content type.') }}
|
||||
{%- endif %}
|
||||
{%- endmacro %}
|
||||
{%- if not messages %}
|
||||
{{- raise_exception('No messages provided.') }}
|
||||
{%- endif %}
|
||||
{%- set reasoning_instructions = '' %}
|
||||
{%- if enable_thinking is undefined or enable_thinking is true %}
|
||||
{%- set resolved_reasoning_effort = reasoning_effort|default('xhigh') %}
|
||||
{%- if resolved_reasoning_effort not in ('xhigh', 'medium', 'low') %}
|
||||
{{- raise_exception('Unexpected reasoning effort ' ~ reasoning_effort ~ '. Supported types are xhigh (default), medium, and low.') }}
|
||||
{%- endif %}
|
||||
{%- if resolved_reasoning_effort == 'xhigh' %}
|
||||
{%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %}
|
||||
{%- elif resolved_reasoning_effort == 'low' %}
|
||||
{%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if tools and tools is iterable and tools is not mapping %}
|
||||
{{- '<|im_start|>system\n' }}
|
||||
{%- if reasoning_instructions %}
|
||||
{{- reasoning_instructions + '\n\n' }}
|
||||
{%- endif %}
|
||||
{{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{{- "\n" }}
|
||||
{{- tool | tojson }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>" }}
|
||||
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
|
||||
{%- if messages[0].role == 'system' %}
|
||||
{%- set content = render_content(messages[0].content, false, true)|trim %}
|
||||
{%- if content %}
|
||||
{{- '\n\n' + content }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{%- if messages[0].role == 'system' %}
|
||||
{%- set content = render_content(messages[0].content, false, true)|trim %}
|
||||
{%- if content %}
|
||||
{{- '<|im_start|>system\n' + (reasoning_instructions + '\n\n' if reasoning_instructions else '') + content + '<|im_end|>\n' }}
|
||||
{%- elif reasoning_instructions %}
|
||||
{{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- elif reasoning_instructions %}
|
||||
{{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
|
||||
{%- for message in messages[::-1] %}
|
||||
{%- set index = (messages|length - 1) - loop.index0 %}
|
||||
{%- if ns.multi_step_tool and message.role == "user" %}
|
||||
{%- set content = render_content(message.content, false)|trim %}
|
||||
{%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
|
||||
{%- set ns.multi_step_tool = false %}
|
||||
{%- set ns.last_query_index = index %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if ns.multi_step_tool %}
|
||||
{{- raise_exception('No user query found in messages.') }}
|
||||
{%- endif %}
|
||||
{%- for message in messages %}
|
||||
{%- set content = render_content(message.content, true)|trim %}
|
||||
{%- if message.role == "system" %}
|
||||
{%- if not loop.first %}
|
||||
{{- raise_exception('System message must be at the beginning.') }}
|
||||
{%- endif %}
|
||||
{%- elif message.role == "user" %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
|
||||
{%- elif message.role == "assistant" %}
|
||||
{%- set reasoning_content = '' %}
|
||||
{%- if message.reasoning_content is string %}
|
||||
{%- set reasoning_content = message.reasoning_content %}
|
||||
{%- endif %}
|
||||
{%- set reasoning_content = reasoning_content|trim %}
|
||||
{%- if preserve_thinking is undefined or preserve_thinking is true or loop.index0 > ns.last_query_index %}
|
||||
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + content }}
|
||||
{%- endif %}
|
||||
{%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{%- if loop.first %}
|
||||
{%- if content|trim %}
|
||||
{{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
||||
{%- else %}
|
||||
{{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
||||
{%- endif %}
|
||||
{%- if tool_call.arguments is defined and tool_call.arguments != '' %}
|
||||
{%- for args_name, args_value in tool_call.arguments|items %}
|
||||
{{- '<parameter=' + args_name + '>\n' }}
|
||||
{%- set args_value = args_value | string if args_value is string else args_value | tojson | safe %}
|
||||
{{- args_value }}
|
||||
{{- '\n</parameter>\n' }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '</function>\n</tool_call>' }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if loop.previtem and loop.previtem.role != "tool" %}
|
||||
{{- '<|im_start|>user' }}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_response>\n' }}
|
||||
{{- content }}
|
||||
{{- '\n</tool_response>' }}
|
||||
{%- if not loop.last and loop.nextitem.role != "tool" %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif loop.last %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- raise_exception('Unexpected message role.') }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- if enable_thinking is defined and enable_thinking is false %}
|
||||
{{- '<think>\n\n</think>\n\n' }}
|
||||
{%- else %}
|
||||
{{- '<think>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
@@ -60,9 +60,7 @@ fn read_f32(path: &Path) -> Vec<f32> {
|
||||
let bytes = std::fs::read(path).unwrap_or_else(|e| panic!("read {path:?}: {e}"));
|
||||
assert!(bytes.len().is_multiple_of(4), "truncated f32 file {path:?}");
|
||||
bytes
|
||||
.as_chunks::<4>()
|
||||
.0
|
||||
.iter()
|
||||
.chunks_exact(4)
|
||||
.map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]]))
|
||||
.collect()
|
||||
}
|
||||
|
||||
@@ -106,8 +106,6 @@ fn spec(model_id: &str, tp: Option<u32>, quant: Option<&str>) -> ModelSpec {
|
||||
quant: quant.map(String::from),
|
||||
tensor_parallel: tp,
|
||||
devices: None,
|
||||
sampling: None,
|
||||
preserve_thinking: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -37,9 +37,7 @@ fn fixture_dir() -> PathBuf {
|
||||
fn read_f32(path: &Path) -> Vec<f32> {
|
||||
let bytes = std::fs::read(path).unwrap_or_else(|e| panic!("read {path:?}: {e}"));
|
||||
bytes
|
||||
.as_chunks::<4>()
|
||||
.0
|
||||
.iter()
|
||||
.chunks_exact(4)
|
||||
.map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]))
|
||||
.collect()
|
||||
}
|
||||
|
||||
@@ -2,22 +2,6 @@
|
||||
Description=Neuron — per-node GPU discovery and harness daemon for cortex
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
# Start-limit bound for the self-restart path (#265). When a TP forward
|
||||
# wedges and the NCCL abort that would unblock it also hangs, neuron
|
||||
# exits 70 so Restart=on-failure recovers it in seconds instead of the
|
||||
# 70-minute outage that needed a human on 2026-08-17.
|
||||
#
|
||||
# That only helps if the wedge is transient. A deterministic one — a bad
|
||||
# build, a sick card, a model that wedges on load — would otherwise
|
||||
# restart forever, and a flapping node is WORSE than a down one: cortex
|
||||
# marks it healthy between restarts and keeps routing to it, so every
|
||||
# client sees intermittent failures instead of one node cleanly out.
|
||||
#
|
||||
# Four failures in ten minutes stops the unit and leaves it `failed`,
|
||||
# where `systemctl status` and an alert can see it. Visibly broken beats
|
||||
# invisibly cycling.
|
||||
StartLimitIntervalSec=600
|
||||
StartLimitBurst=4
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
@@ -40,11 +24,6 @@ TimeoutStartSec=1800s
|
||||
# On stop, neuron drains in-flight requests then unloads every model
|
||||
# to release CUDA contexts cleanly. Allow generous time for big-model
|
||||
# unloads; systemd will SIGKILL after this bound.
|
||||
#
|
||||
# Note this bound does NOT apply to the self-restart path (#265): a
|
||||
# wedged process exits directly rather than draining, precisely because
|
||||
# draining cannot work when the collective is stuck — and waiting this
|
||||
# out would add two minutes to an outage that is already total.
|
||||
TimeoutStopSec=120s
|
||||
KillSignal=SIGTERM
|
||||
|
||||
|
||||
@@ -1,8 +1,3 @@
|
||||
---
|
||||
title: Context limits
|
||||
description: How the numeric knobs governing usable context fit together.
|
||||
---
|
||||
|
||||
# Context-window & token-limit settings
|
||||
|
||||
How the numeric knobs that govern usable context fit together, what the
|
||||
@@ -32,43 +27,6 @@ daemon now does automatically.
|
||||
| `limit.input` | same | compaction trigger — opencode compacts to keep the prompt at/under this |
|
||||
| `limit.output` | same | generation reserve opencode leaves below the wall |
|
||||
|
||||
### Tuning the derived limit
|
||||
|
||||
Since #67 the neuron computes `limit` itself (see below), and three
|
||||
`[harness.candle.context_limit]` settings shape that computation:
|
||||
|
||||
```toml
|
||||
[harness.candle.context_limit]
|
||||
target_prefill_latency_secs = 120 # longest acceptable prefill per turn
|
||||
bootstrap_prefill_tok_per_sec = 800 # cold-start estimate until measured
|
||||
output_reserve_tokens = 8192 # what a request gets when it names no cap
|
||||
max_output_tokens = 32768 # the largest output a request may name
|
||||
```
|
||||
|
||||
**`target_prefill_latency_secs`** is the one that usually binds. The
|
||||
throughput ceiling is `target_prefill_latency_secs × measured
|
||||
prefill tok/s`, so on a host measuring ~1,400 tok/s a 120-second target
|
||||
advertises about 168k — above most hard ceilings, in which case it stops
|
||||
mattering. On a slower host, or one whose recent traffic has been small
|
||||
prompts, it is what caps the window.
|
||||
|
||||
**`bootstrap_prefill_tok_per_sec`** applies until the model has measured
|
||||
its own rate, and stops the advertised window collapsing on a
|
||||
freshly-restarted host. Prefill samples below 512 tokens are ignored
|
||||
when computing that rate: a short prompt is dominated by fixed
|
||||
per-request cost and says nothing about throughput — the deploy's own
|
||||
19-token smoke probe reads as ~145 tok/s on a host that measures ~1,400.
|
||||
So a host that has served only small requests keeps the bootstrap
|
||||
estimate rather than adopting a misleading one, and no warm-up is
|
||||
needed after a restart.
|
||||
|
||||
**`output_reserve_tokens`** is what a request that names no cap
|
||||
receives. **`max_output_tokens`** is the ceiling advertised to clients
|
||||
as `max_output_tokens`, and it is the *ceiling*, not the reserve —
|
||||
publishing the reserve tells every client that reads the advertisement
|
||||
to cap itself there, which on a reasoning model can be less than a
|
||||
single think block.
|
||||
|
||||
## How they must relate
|
||||
|
||||
For a single model on a single neuron, all of these must hold:
|
||||
@@ -50,12 +50,6 @@ url = "postgres://helexa:helexa@localhost/helexa_upstream"
|
||||
# email_token_ttl_secs = 86400 # 24 hours
|
||||
# Frontend base URL used to build verify/reset links in emails.
|
||||
app_base_url = "https://helexa.ai"
|
||||
# Other front ends allowed to originate a signup, so their confirmation
|
||||
# mail points back where the person registered rather than at the default
|
||||
# site. An allowlist, deliberately: the value is embedded in mail we send
|
||||
# and sign, so an unchecked caller-supplied origin would make signup a
|
||||
# phishing-link generator.
|
||||
additional_app_origins = []
|
||||
|
||||
# Unverified-signup reaping (see docs): frees an address whose signup
|
||||
# was never completed. Must be >= email_token_ttl_secs.
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user