Compare commits
2 Commits
fix/quant-
...
v0.1.15
| Author | SHA1 | Date | |
|---|---|---|---|
0184ccab28
|
|||
|
471b9b7629
|
File diff suppressed because it is too large
Load Diff
@@ -1,26 +1,12 @@
|
||||
name: CI
|
||||
|
||||
# Pushes to main are deliberately excluded: build-prerelease.yml runs
|
||||
# its own lint/test jobs there (gating publish), and running both
|
||||
# workflows on the same push made them queue against each other on the
|
||||
# same runner labels — ~12 minutes of added latency per deploy. Feature
|
||||
# branches, PRs to main, and release tags keep the full gate here.
|
||||
on:
|
||||
push:
|
||||
branches-ignore: [main]
|
||||
branches: ["**"]
|
||||
tags: ["v*"]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
# Coalesce same-ref pushes; a newer push supersedes the in-flight run.
|
||||
# (The old shared `cortex-runner-pool` group with build-prerelease.yml
|
||||
# is gone — the workflows no longer trigger on the same refs, and
|
||||
# ephemeral one-VM-per-job runners removed the shared-workspace race
|
||||
# that group existed to serialize.)
|
||||
concurrency:
|
||||
group: ci-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
CARGO_INCREMENTAL: "0"
|
||||
RUSTC_WRAPPER: sccache
|
||||
@@ -30,306 +16,40 @@ env:
|
||||
SCCACHE_S3_USE_SSL: "false"
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.SCCACHE_S3_ACCESS_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_S3_SECRET_KEY }}
|
||||
# fmt, clippy, and test all run in parallel on the same `rust` runner
|
||||
# and would otherwise share /root/.cache/act/<hash>/hostexecutor/target/,
|
||||
# racing each other's cargo temp files (.tmpXXXXXX) and failing builds
|
||||
# mid-compile. Give each job its own target directory so the invocations
|
||||
# don't collide. sccache still backs the actual rustc cache, so the
|
||||
# rebuild penalty is small.
|
||||
CARGO_TARGET_DIR: target-${{ github.job }}
|
||||
|
||||
jobs:
|
||||
# Two independent questions: does this change need the Rust gate, and
|
||||
# does it need the web gate.
|
||||
#
|
||||
# The SPA under helexa.ai/ is built and shipped by its own deploy job and
|
||||
# is not read by any crate, build script or spec file — so a change
|
||||
# confined to it cannot alter a Rust result. Waiting up to 70 minutes for
|
||||
# a CUDA type-check to prove that is pure latency, and the wait is what
|
||||
# tempts people to merge front-end work unchecked. Documentation is inert
|
||||
# the same way, and more obviously: no crate `include_str!`s a markdown
|
||||
# file, no spec packages `doc/`, no build script reads one.
|
||||
#
|
||||
# Conversely a Rust-only change cannot break the SPA, so it need not
|
||||
# spend three minutes on `npm ci` + lint + typecheck + build.
|
||||
#
|
||||
# The classification stays deliberately one-directional: it only ever
|
||||
# narrows the gate when *every* changed path is known-inert, and every
|
||||
# other outcome — a tag, an unknown base, an empty diff, a mixed
|
||||
# changeset — falls through to the full gate. A new top-level directory
|
||||
# is therefore covered on the day it appears, without anyone
|
||||
# remembering to add it here. Adding an inert pattern is the only way to
|
||||
# widen what gets skipped, which is the property worth keeping.
|
||||
changes:
|
||||
name: Classify changes
|
||||
timeout-minutes: 10
|
||||
runs-on: fedora-43
|
||||
outputs:
|
||||
rust: ${{ steps.classify.outputs.rust }}
|
||||
web: ${{ steps.classify.outputs.web }}
|
||||
check:
|
||||
name: Format, lint, build, test
|
||||
runs-on: fedora
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: classify
|
||||
run: |
|
||||
set -u
|
||||
full() {
|
||||
echo "→ full gate: $1"
|
||||
echo "rust=true" >> "$GITHUB_OUTPUT"
|
||||
echo "web=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
}
|
||||
|
||||
# Tags build the SRPMs from the whole tree, and the srpm jobs
|
||||
# gate on fmt/clippy/test/cuda-check. Never narrow there.
|
||||
case "${GITHUB_REF}" in
|
||||
refs/tags/*) full "tag push" ;;
|
||||
esac
|
||||
|
||||
# merge-base against main, not the push's before-sha: a new
|
||||
# branch has no before-sha, and a force-push's is unreachable.
|
||||
base=""
|
||||
if git fetch --no-tags origin main >/dev/null 2>&1; then
|
||||
base="$(git merge-base FETCH_HEAD HEAD 2>/dev/null || true)"
|
||||
fi
|
||||
[ -n "${base}" ] || full "no merge-base with main"
|
||||
|
||||
files="$(git diff --name-only "${base}" HEAD)"
|
||||
[ -n "${files}" ] || full "empty diff against ${base}"
|
||||
|
||||
echo "changed files:"
|
||||
printf '%s\n' "${files}" | sed 's/^/ /'
|
||||
|
||||
# Inert for Rust: the SPA, documentation, and fleet config.
|
||||
# Markdown is matched only at the repo root and under doc/ — a
|
||||
# .md inside crates/ is left to fall through, since
|
||||
# `include_str!` could make one load-bearing without this file
|
||||
# knowing.
|
||||
#
|
||||
# Fleet config (models.toml, asset/neuron/<host>.toml,
|
||||
# asset/helexa-bench/<host>.toml) is read by a *running*
|
||||
# daemon, never by a build script or an `include_str!`, so it
|
||||
# cannot change a Rust result. Deploying one is a config rsync
|
||||
# plus a service restart — neuron re-reads it and reloads the
|
||||
# model — so spending ~25 minutes on fmt, clippy, test and a
|
||||
# CUDA type-check to prove a one-line TOML edit compiles is
|
||||
# pure latency (#291).
|
||||
#
|
||||
# The `config` job below stays ungated and still validates
|
||||
# these files, so narrowing here removes the meritless work
|
||||
# without removing the check that matters. build-prerelease
|
||||
# already reaches the same conclusion on main: none of its
|
||||
# per-package path regexes match these files, so a config-only
|
||||
# push builds no binaries.
|
||||
rust_relevant="$(printf '%s\n' "${files}" \
|
||||
| grep -v '^helexa\.ai/' \
|
||||
| grep -v '^doc/' \
|
||||
| grep -vE '^[^/]+\.md$' \
|
||||
| grep -vE '^asset/neuron/[^/]+\.toml$' \
|
||||
| grep -vE '^asset/helexa-bench/[^/]+\.toml$' \
|
||||
| grep -vE '^models\.toml$' \
|
||||
|| true)"
|
||||
if [ -n "${rust_relevant}" ]; then
|
||||
echo "rust=true" >> "$GITHUB_OUTPUT"
|
||||
echo "→ Rust gate: required"
|
||||
else
|
||||
echo "rust=false" >> "$GITHUB_OUTPUT"
|
||||
echo "→ Rust gate: skipped (front-end and/or docs only)"
|
||||
fi
|
||||
|
||||
# The web gate is needed only when the SPA itself changed.
|
||||
if printf '%s\n' "${files}" | grep -q '^helexa\.ai/'; then
|
||||
echo "web=true" >> "$GITHUB_OUTPUT"
|
||||
echo "→ web gate: required"
|
||||
else
|
||||
echo "web=false" >> "$GITHUB_OUTPUT"
|
||||
echo "→ web gate: skipped (nothing under helexa.ai/)"
|
||||
fi
|
||||
|
||||
fmt:
|
||||
name: Format
|
||||
timeout-minutes: 15
|
||||
runs-on: rust
|
||||
needs: changes
|
||||
if: needs.changes.outputs.rust == 'true'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: cargo fmt --check --all
|
||||
|
||||
# models.toml and asset/neuron/<host>.toml both describe how a model is
|
||||
# served, and until #283 nothing forced them to agree. #252 is the bill
|
||||
# for that: models.toml omitted `quant`, so a cortex cold-load served
|
||||
# bf16 and could not prefill, while the same model was fine when the
|
||||
# host loaded it from its own config. The sampling override (#283) is a
|
||||
# second field with the same hazard, so the gate lands with it.
|
||||
#
|
||||
# Deliberately ungated by the `changes` classifier: it is a
|
||||
# sub-second Python run with no build, and the whole point is that it
|
||||
# cannot be skipped on the push that introduces the drift. A config
|
||||
# change is neither `rust` nor `web`, so gating it on either would let
|
||||
# a pure-TOML push through unchecked — precisely the push that breaks
|
||||
# this.
|
||||
config:
|
||||
name: Config consistency
|
||||
timeout-minutes: 5
|
||||
runs-on: fedora-43
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: python3 script/check-config-consistency.py
|
||||
# Two files describe one routing decision and nothing forced them
|
||||
# to agree. On 2026-08-27 the router still aliased
|
||||
# `helexa/balanced` to a model retired from the catalogue, so
|
||||
# every web-chat turn evicted beast's resident flagship to
|
||||
# cold-load it. Same shape as the check above (#252/#287).
|
||||
- run: python3 script/check-alias-targets.py
|
||||
|
||||
web:
|
||||
name: Web (lint + typecheck + i18n + build)
|
||||
timeout-minutes: 20
|
||||
runs-on: fedora-43
|
||||
needs: changes
|
||||
if: needs.changes.outputs.web == 'true'
|
||||
defaults:
|
||||
run:
|
||||
working-directory: helexa.ai
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "20"
|
||||
- run: npm ci
|
||||
# eslint and tsc previously ran nowhere. The deploy job builds the
|
||||
# SPA, but only on main and only after a merge — so a defect landed
|
||||
# first and broke the deploy afterwards. A setState-in-effect error
|
||||
# reached main exactly that way.
|
||||
- run: npm run lint
|
||||
- run: npm run typecheck
|
||||
# Behaviour, not just types. Gating from the first commit was the
|
||||
# deliberate call (#304): with no existing suite there is nothing to
|
||||
# stabilise, so a warn-first period would only create a window in
|
||||
# which a vacuous test lands and gets normalised — and a suite
|
||||
# nobody trusts is worse than none, because it gets cited.
|
||||
- run: npm test
|
||||
# 42 locales with strict key parity is a promise that breaks
|
||||
# quietly. Adding an English string without translating it must fail
|
||||
# here, not in review, and certainly not in front of somebody
|
||||
# evaluating the project in their own language.
|
||||
- run: npm run i18n:check
|
||||
- run: npm run i18n:meta
|
||||
- run: npm run i18n:lang-labels
|
||||
- run: npm run build
|
||||
|
||||
clippy:
|
||||
name: Clippy
|
||||
timeout-minutes: 25
|
||||
runs-on: rust
|
||||
needs: changes
|
||||
if: needs.changes.outputs.rust == 'true'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# Failure-aware sccache escalation lives in the shared script (kept
|
||||
# in sync with build-prerelease.yml): a signal death (rustc SIGSEGV
|
||||
# / OOM-kill) keeps the cache and fails fast instead of an uncached
|
||||
# rebuild; only a real sccache fault drops the cache.
|
||||
# --all-targets so `#[cfg(test)]` modules, tests/, benches/ and
|
||||
# examples/ are linted too (#286). Without it clippy sees only lib
|
||||
# and bin targets, so `-D warnings` never gated a single line of
|
||||
# test code — which is how two tests lost their `#[test]` attribute
|
||||
# and stopped running unnoticed.
|
||||
- name: Clippy (sccache escalation)
|
||||
run: script/ci-cargo-escalate.sh cargo clippy --workspace --all-targets -- -D warnings
|
||||
|
||||
test:
|
||||
name: Test
|
||||
timeout-minutes: 25
|
||||
runs-on: rust
|
||||
needs: changes
|
||||
if: needs.changes.outputs.rust == 'true'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# See script/ci-cargo-escalate.sh for the escalation rationale.
|
||||
- name: Test (sccache escalation)
|
||||
run: script/ci-cargo-escalate.sh cargo test --workspace
|
||||
|
||||
# Type-check the CUDA-only code path. Borrow-check-only — we
|
||||
# never run the tests here (the runner has no GPU). This catches
|
||||
# the category of bug where a refactor compiles fine under the
|
||||
# default feature set (which is what the `clippy` and `test` jobs
|
||||
# exercise) but fails inside a `#[cfg(feature = "cuda")]` block.
|
||||
# `runs-on: cuda-13.0` selects the runner that ships nvcc /
|
||||
# cudarc's build prerequisites. The generic `rust` and `rpm`
|
||||
# runners don't have them (the previous label `rpm` was tried
|
||||
# first and tripped cudarc's `nvcc --version` build script —
|
||||
# see commit history).
|
||||
cuda-check:
|
||||
name: CUDA type-check
|
||||
# flash-attn kernel compilation dominates the first uncached run;
|
||||
# sccache + the cargo target cache absorb it afterwards.
|
||||
timeout-minutes: 70
|
||||
runs-on: cuda-13.0
|
||||
needs: changes
|
||||
if: needs.changes.outputs.rust == 'true'
|
||||
# The workflow-level env sets `RUSTC_WRAPPER: sccache`
|
||||
# unconditionally, which hard-fails cargo if the CUDA image
|
||||
# doesn't ship sccache. Clear it at job level; the "Enable
|
||||
# sccache when available" step opts back in only after probing
|
||||
# for the binary. SCCACHE_*/AWS creds stay set — harmless when
|
||||
# the wrapper is off, required when it's on.
|
||||
env:
|
||||
RUSTC_WRAPPER: ""
|
||||
# candle-kernels' build script falls back to `nvidia-smi` for
|
||||
# compute-cap detection when this is unset — and the GPU-less
|
||||
# builder image doesn't ship nvidia-smi. Any valid cap works for
|
||||
# a borrow-check; the real per-flavour caps live in
|
||||
# build-prerelease.yml's matrix.
|
||||
CUDA_COMPUTE_CAP: "86"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# sccache probing + failure classification lives in the shared
|
||||
# script (see build-prerelease.yml's neuron build for the same
|
||||
# pattern). It probes for sccache and, on a rustc SIGSEGV / OOM,
|
||||
# keeps the cache and fails fast rather than rebuilding uncached.
|
||||
# candle-flash-attn's build script clones CUTLASS from GitHub, and the
|
||||
# runners are one VM per job so that cache starts empty every time.
|
||||
# A burst of concurrent CUDA jobs is enough to start getting refused
|
||||
# ("could not read Username for 'https://github.com'"), which fails a
|
||||
# long job on a transient rate limit. Seed the cache first, with the
|
||||
# retries the build script does not have. Both commands are
|
||||
# best-effort: on failure the build proceeds exactly as it would have
|
||||
# and reports the authoritative error itself.
|
||||
- name: Seed the CUTLASS checkout
|
||||
run: |
|
||||
cargo fetch || true
|
||||
script/seed-cutlass.sh
|
||||
- name: cargo check --features cuda (sccache escalation)
|
||||
run: |
|
||||
# act launches the step shell without /etc/profile, so the
|
||||
# gitea_runner user's inherited PATH lacks /usr/local/cuda-13.0/bin.
|
||||
# cudarc's build.rs shells out to `nvcc --version` (the neuron
|
||||
# crate enables cuda-version-from-build-system) and panics with
|
||||
# ENOENT if nvcc isn't resolvable — keep this export in sync
|
||||
# with build-prerelease.yml.
|
||||
export PATH="/usr/local/cuda-13.0/bin:${PATH}"
|
||||
export LD_LIBRARY_PATH="/usr/local/cuda-13.0/targets/x86_64-linux/lib:/usr/local/cuda-13.0/lib64:${LD_LIBRARY_PATH:-}"
|
||||
export LIBRARY_PATH="/usr/local/cuda-13.0/targets/x86_64-linux/lib:/usr/local/cuda-13.0/lib64:${LIBRARY_PATH:-}"
|
||||
script/ci-cargo-escalate.sh cargo check -p neuron --features cuda,flash-attn --all-targets
|
||||
- name: Ensure sccache with S3 support
|
||||
env:
|
||||
# cudaforge shells out to `git clone` for CUTLASS if the seed
|
||||
# step above did not populate the cache. Give that clone the
|
||||
# same anonymous git environment the seed script uses: CUTLASS
|
||||
# is public, and an inherited credential turns a 200 into a
|
||||
# 401. Scoped to this step — actions/checkout writes global
|
||||
# config and must not see these.
|
||||
GIT_CONFIG_GLOBAL: /dev/null
|
||||
GIT_CONFIG_SYSTEM: /dev/null
|
||||
GIT_CONFIG_COUNT: "0"
|
||||
RUSTC_WRAPPER: ""
|
||||
run: |
|
||||
if sccache --version 2>/dev/null && sccache --show-stats 2>/dev/null; then
|
||||
echo "sccache with S3 support already installed"
|
||||
else
|
||||
cargo install sccache --features s3 --locked
|
||||
fi
|
||||
|
||||
- name: Check formatting
|
||||
run: cargo fmt --check --all
|
||||
|
||||
- name: Clippy
|
||||
run: cargo clippy --workspace -- -D warnings
|
||||
|
||||
- name: Test
|
||||
run: cargo test --workspace
|
||||
|
||||
- name: Show sccache stats
|
||||
run: sccache --show-stats
|
||||
|
||||
srpm-cortex:
|
||||
name: Build cortex SRPM
|
||||
timeout-minutes: 25
|
||||
runs-on: rpm
|
||||
needs: [fmt, clippy, test, cuda-check]
|
||||
runs-on: fedora
|
||||
needs: check
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -388,9 +108,8 @@ jobs:
|
||||
|
||||
srpm-neuron:
|
||||
name: Build neuron SRPM
|
||||
timeout-minutes: 25
|
||||
runs-on: rpm
|
||||
needs: [fmt, clippy, test, cuda-check]
|
||||
runs-on: fedora
|
||||
needs: check
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -449,8 +168,7 @@ jobs:
|
||||
|
||||
copr-cortex:
|
||||
name: Publish cortex to COPR
|
||||
timeout-minutes: 60
|
||||
runs-on: fedora-43
|
||||
runs-on: fedora
|
||||
needs: srpm-cortex
|
||||
steps:
|
||||
- name: Download SRPM
|
||||
@@ -467,8 +185,7 @@ jobs:
|
||||
|
||||
copr-neuron:
|
||||
name: Publish neuron to COPR
|
||||
timeout-minutes: 60
|
||||
runs-on: fedora-43
|
||||
runs-on: fedora
|
||||
needs: srpm-neuron
|
||||
steps:
|
||||
- name: Download SRPM
|
||||
@@ -485,8 +202,7 @@ jobs:
|
||||
|
||||
bump-version:
|
||||
name: Bump version in source
|
||||
timeout-minutes: 15
|
||||
runs-on: rust
|
||||
runs-on: fedora
|
||||
needs: [copr-cortex, copr-neuron]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -529,6 +245,6 @@ jobs:
|
||||
echo "Nothing to commit for ${VERSION}"
|
||||
else
|
||||
git commit -m "chore: bump version to ${VERSION}"
|
||||
git remote set-url origin "https://gitea-actions:${GITEA_TOKEN}@git.lair.cafe/${{ github.repository }}.git"
|
||||
git remote set-url origin "https://gitea-actions:${GITEA_TOKEN}@git.lair.cafe/helexa/cortex.git"
|
||||
git push origin HEAD:main
|
||||
fi
|
||||
|
||||
@@ -1,136 +0,0 @@
|
||||
name: deploy-dev
|
||||
|
||||
# Fast-path iteration deploy for a SINGLE neuron host: build one CUDA
|
||||
# flavour, copy the raw binary to the host, restart neuron.service.
|
||||
# Skips the other two flavours, all RPM packaging, signing, repo
|
||||
# publish, and dnf — push-to-testable drops from ~20 min to roughly
|
||||
# one CUDA build plus a service restart.
|
||||
#
|
||||
# This is a DEV convenience, not a release path:
|
||||
# - the binary lands at /usr/bin/neuron *outside* RPM ownership;
|
||||
# the next regular deploy.yml run reconciles the host back to the
|
||||
# packaged binary (dnf sees the newer RPM and reinstalls). `rpm -V
|
||||
# helexa-neuron-<flavour>` flagging a modified /usr/bin/neuron in
|
||||
# the interim is expected.
|
||||
# - nothing is published; other hosts are untouched.
|
||||
# - requires the `install` sudoers rule from
|
||||
# asset/sudoers.d/neuron-host.conf (re-run script/infra-setup.sh
|
||||
# after updating it).
|
||||
#
|
||||
# Trigger from the Gitea UI: Actions → deploy-dev → Run workflow,
|
||||
# pick the target host. Defaults to the ref you dispatch from, so it
|
||||
# works from feature branches without touching main.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
target:
|
||||
description: "neuron host to deploy to"
|
||||
required: true
|
||||
type: choice
|
||||
options: [beast, benjy, quadbrat]
|
||||
default: beast
|
||||
|
||||
# One dev deploy at a time; a newer dispatch for the same host wins.
|
||||
concurrency:
|
||||
group: deploy-dev-${{ inputs.target }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
CARGO_INCREMENTAL: "0"
|
||||
CARGO_TERM_COLOR: "always"
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build neuron (${{ inputs.target }})
|
||||
runs-on: cuda-13.0
|
||||
outputs:
|
||||
flavour: ${{ steps.map.outputs.flavour }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
# host → flavour → compute cap. Keep in sync with the
|
||||
# build-neuron matrix in build-prerelease.yml and the
|
||||
# deploy-neurons matrix in deploy.yml.
|
||||
- id: map
|
||||
run: |
|
||||
case "${{ inputs.target }}" in
|
||||
beast) flavour=blackwell cap=120 ;;
|
||||
benjy) flavour=ada cap=89 ;;
|
||||
quadbrat) flavour=ampere cap=86 ;;
|
||||
*) echo "unknown target ${{ inputs.target }}"; exit 1 ;;
|
||||
esac
|
||||
echo "flavour=${flavour}" >> "$GITHUB_OUTPUT"
|
||||
echo "cap=${cap}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Build neuron with CUDA
|
||||
run: |
|
||||
set -eux
|
||||
export PATH="/usr/local/cuda-13.0/bin:${PATH}"
|
||||
export LD_LIBRARY_PATH="/usr/local/cuda-13.0/targets/x86_64-linux/lib:/usr/local/cuda-13.0/lib64:${LD_LIBRARY_PATH:-}"
|
||||
export LIBRARY_PATH="/usr/local/cuda-13.0/targets/x86_64-linux/lib:/usr/local/cuda-13.0/lib64:${LIBRARY_PATH:-}"
|
||||
cargo build --release -p neuron --features "cuda cudnn"
|
||||
env:
|
||||
CUDA_COMPUTE_CAP: ${{ steps.map.outputs.cap }}
|
||||
CARGO_BUILD_JOBS: "8"
|
||||
NVCC_THREADS: "4"
|
||||
|
||||
- name: Stage binary
|
||||
run: |
|
||||
mkdir --parents artifacts
|
||||
cp target/release/neuron artifacts/neuron-dev
|
||||
file artifacts/neuron-dev
|
||||
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: neuron-dev-${{ inputs.target }}
|
||||
path: artifacts/neuron-dev
|
||||
retention-days: 1
|
||||
|
||||
deploy:
|
||||
name: Deploy to ${{ inputs.target }}
|
||||
needs: build
|
||||
runs-on: fedora-43
|
||||
env:
|
||||
DEPLOY_KEY: |
|
||||
${{ secrets.RSYNC_SSH_KEY }}
|
||||
TARGET_HOST: ${{ inputs.target }}.hanzalova.internal
|
||||
steps:
|
||||
- name: SSH init
|
||||
run: |
|
||||
mkdir -p ~/.ssh
|
||||
echo "${DEPLOY_KEY}" > ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
|
||||
"gitea_ci@${TARGET_HOST}" 'hostname -f'
|
||||
|
||||
- uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: neuron-dev-${{ inputs.target }}
|
||||
path: artifacts/
|
||||
|
||||
- name: Copy binary to host
|
||||
run: |
|
||||
scp artifacts/neuron-dev "gitea_ci@${TARGET_HOST}:/var/lib/gitea_ci/neuron-dev"
|
||||
|
||||
- name: Install binary and restart neuron.service
|
||||
run: |
|
||||
ssh "gitea_ci@${TARGET_HOST}" '
|
||||
set -eu
|
||||
if systemctl is-active --quiet neuron.service; then
|
||||
sudo /usr/bin/systemctl stop neuron.service
|
||||
fi
|
||||
# Exact command form required by the sudoers rule in
|
||||
# asset/sudoers.d/neuron-host.conf — change both together.
|
||||
sudo /usr/bin/install -o root -g root -m 0755 /var/lib/gitea_ci/neuron-dev /usr/bin/neuron
|
||||
# enable --now so a dev deploy also leaves the unit enabled
|
||||
# for boot, consistent with deploy.yml.
|
||||
sudo /usr/bin/systemctl enable --now neuron.service
|
||||
rm -f /var/lib/gitea_ci/neuron-dev'
|
||||
|
||||
- name: Capture neuron.service startup journal
|
||||
if: always()
|
||||
run: |
|
||||
sleep 10
|
||||
ssh "gitea_ci@${TARGET_HOST}" \
|
||||
'journalctl --unit neuron.service -I --no-pager'
|
||||
File diff suppressed because it is too large
Load Diff
23
.gitignore
vendored
23
.gitignore
vendored
@@ -1,30 +1,7 @@
|
||||
/target
|
||||
/bench/node_modules
|
||||
/bench/dist
|
||||
/helexa.ai/node_modules
|
||||
/helexa.ai/dist
|
||||
helexa.ai/.env.local
|
||||
*.swp
|
||||
*.swo
|
||||
.idea/
|
||||
.vscode/
|
||||
# Secret-bearing service configs. These carry API keys, a JWT secret, DB
|
||||
# passwords and SMTP credentials, so they are hand-synced by
|
||||
# script/infra-setup.sh rather than shipped by CI (#287 Gap 1).
|
||||
#
|
||||
# helexa-router.toml is deliberately NOT here: it holds no secrets, only
|
||||
# a listen address, the product-tier aliases and a mesh endpoint. It was
|
||||
# ignored by association, and that is precisely how its `helexa/balanced`
|
||||
# alias came to point at a retired model for long enough to take beast
|
||||
# down (2026-08-27). A file nothing can diff is a file nothing can catch.
|
||||
# Operator-owned credential layer, merged after the CI-deployed config
|
||||
# by cortex / helexa-upstream / helexa-angels. Must never be committed.
|
||||
secrets.toml
|
||||
**/secrets.toml
|
||||
|
||||
cortex.toml
|
||||
doc/plan/*
|
||||
/target-cuda/
|
||||
.claude/
|
||||
helexa-upstream.toml
|
||||
helexa-angels.toml
|
||||
|
||||
269
AGENTS.md
269
AGENTS.md
@@ -1,269 +0,0 @@
|
||||
# AGENTS.md — helexa/cortex
|
||||
|
||||
## Project Overview
|
||||
|
||||
helexa is a self-hosted LLM serving stack for multi-node GPU inference clusters. It has two components:
|
||||
|
||||
- **cortex** — the per-operator control plane and LLM proxy. A Rust reverse-proxy that sits in front of the fleet and presents a unified OpenAI + Anthropic compatible API surface. It handles model routing, lifecycle management (load/unload/evict), request translation, and metrics collection.
|
||||
- **neuron** — the per-host LLM harness. One instance runs on every GPU host, serving candle-based in-process inference and managing local hardware discovery and model lifecycle.
|
||||
|
||||
## Repository Layout
|
||||
|
||||
```
|
||||
cortex/
|
||||
├── Cargo.toml # workspace root (Rust 2024 edition, GPL-3.0)
|
||||
├── cortex.example.toml # example gateway config
|
||||
├── models.example.toml # example model catalogue
|
||||
├── neuron.example.toml # example neuron config
|
||||
├── README.md # public-facing documentation
|
||||
├── CLAUDE.md # detailed design rationale and implementation history
|
||||
├── AGENTS.md # ← you are here
|
||||
├── cortex.spec # RPM spec for cortex
|
||||
├── helexa-neuron.spec # RPM spec for neuron (renamed to avoid Fedora collision)
|
||||
├── rpm/ # prerelease RPM specs
|
||||
│ ├── cortex-prerelease.spec
|
||||
│ ├── helexa-neuron-prerelease.spec
|
||||
│ └── helexa-bench-prerelease.spec
|
||||
├── data/ # systemd units and example configs for packaging
|
||||
│ ├── cortex.service
|
||||
│ ├── neuron.service
|
||||
│ ├── cortex.example.toml
|
||||
│ ├── neuron.example.toml
|
||||
│ └── models.example.toml
|
||||
└── crates/
|
||||
├── cortex-core/ # shared types, config, envelopes
|
||||
│ └── src/
|
||||
│ ├── lib.rs
|
||||
│ ├── build_info.rs # BuildInfo type for /version endpoint
|
||||
│ ├── config.rs # figment-based config structs
|
||||
│ ├── catalogue.rs # ModelProfile, placement matching
|
||||
│ ├── discovery.rs # DeviceInfo, DiscoveryResponse
|
||||
│ ├── harness.rs # Harness trait, HarnessConfig, HarnessHealth
|
||||
│ ├── node.rs # NodeState, ModelStatus
|
||||
│ ├── openai.rs # OpenAI request/response types
|
||||
│ ├── anthropic.rs # Anthropic request/response types
|
||||
│ ├── translate.rs # OpenAI <-> Anthropic translation
|
||||
│ └── metrics.rs # RequestMetrics, histogram helpers
|
||||
├── cortex-gateway/ # the HTTP proxy server
|
||||
│ └── src/
|
||||
│ ├── lib.rs
|
||||
│ ├── state.rs # CortexState: Arc<RwLock<...>>
|
||||
│ ├── router.rs # model -> node routing logic
|
||||
│ ├── proxy.rs # streaming HTTP proxy to backends
|
||||
│ ├── evictor.rs # LRU/priority eviction logic
|
||||
│ ├── poller.rs # background task polling neuron status
|
||||
│ ├── handlers.rs # axum handlers (chat, completions, models, etc.)
|
||||
│ └── metrics.rs # prometheus exporter endpoint
|
||||
├── cortex-cli/ # CLI entrypoint
|
||||
│ └── src/main.rs # binary: `cortex`
|
||||
├── neuron/ # per-host LLM daemon (replaces cortex-agent)
|
||||
│ ├── Cargo.toml # features: cuda, cudnn, flash-attn, cuda-integration
|
||||
│ ├── build.rs # compiles CUDA kernels, emits build metadata
|
||||
│ └── src/
|
||||
│ ├── main.rs # binary: `neuron`
|
||||
│ ├── discovery.rs # nvidia-smi parsing, device enumeration
|
||||
│ ├── health.rs # runtime GPU polling
|
||||
│ ├── api.rs # HTTP handlers for /discovery, /models, etc.
|
||||
│ ├── version.rs # GET /version endpoint with BuildInfo
|
||||
│ ├── models.rs # local model lifecycle orchestration
|
||||
│ └── harness/ # in-process candle inference
|
||||
│ ├── device_worker/ # per-device CUDA worker threads
|
||||
│ │ ├── mod.rs # canonical narrative for worker architecture
|
||||
│ │ ├── jobs.rs # Job enum, dispatch handlers
|
||||
│ │ └── dispatch.rs # DeviceWorkerState struct
|
||||
│ ├── candle.rs # candle model implementation
|
||||
│ └── tp/ # tensor parallelism
|
||||
│ └── worker.rs # TP worker subprocesses
|
||||
├── helexa-acp/ # Agent Client Protocol bridge (Apache-2.0)
|
||||
│ └── src/main.rs # binary: `helexa-acp`, self-contained (no workspace deps)
|
||||
└── helexa-bench/ # benchmark harness
|
||||
└── src/main.rs # binary: `helexa-bench`, SQLite-backed, version-aware
|
||||
```
|
||||
|
||||
## Key Design Decisions
|
||||
|
||||
### Architecture
|
||||
- **cortex** is the control plane. It exposes the unified API, routes requests, manages model lifecycle across the fleet, and collects metrics.
|
||||
- **neuron** is the node plane. One instance runs on every GPU host. It discovers local hardware, manages in-process candle inference, handles NCCL tensor parallelism, and reports runtime state.
|
||||
- cortex never shells out to `nvidia-smi`, never touches systemd units, and never talks directly to a harness. It talks only to neurons via HTTP API on port 13131.
|
||||
|
||||
### Per-device worker thread (neuron)
|
||||
Every CUDA device gets one dedicated OS thread that owns its `CudaContext` for the daemon's lifetime. All CUDA operations route through this thread via a `std::sync::mpsc` job channel. Tensors never escape the worker thread alive. Inference replies carry `Vec<f32>` CPU-side logits; sampled tokens come back as `u32`. The opaque `ArchHandle(u64)` and `TpHandle(u64)` are indices into the worker's state slab, not pointers.
|
||||
|
||||
CPU loads (`Device::Cpu` fallback) keep the legacy `tokio::task::spawn_blocking + Arc<Mutex<ModelArch>>` path — there's no context to own and the channel hop would only add latency. Four `spawn_blocking` references in `harness/candle.rs` are deliberate CPU fallback.
|
||||
|
||||
### candle-native (not mistral.rs)
|
||||
neuron builds directly on [candle](https://github.com/huggingface/candle). Every model architecture it serves is implemented in this repository, ported against the HuggingFace reference. No external inference server to babysit. The Harness trait remains as an internal seam for adding future engines (vision/audio/diffusion) but its only implementation is in-process candle.
|
||||
|
||||
### Streaming proxy
|
||||
Chat completions are proxied as SSE streams. The gateway must:
|
||||
1. Parse the inbound request to extract the model name
|
||||
2. Route to the correct backend neuron
|
||||
3. Stream the response back, capturing token timing for metrics
|
||||
4. NOT buffer the full response — true streaming passthrough
|
||||
|
||||
### Anthropic translation
|
||||
When a request arrives at `/v1/messages` (Anthropic format), the gateway translates it to OpenAI format before proxying to neuron, then translates the response back. This is stateless envelope transformation. Non-streaming round-trip is implemented; streaming SSE translation deferred.
|
||||
|
||||
### Eviction
|
||||
The evictor runs as a background task. Before loading a model on a node where VRAM is tight:
|
||||
1. Check if the model is already loaded elsewhere → route there instead
|
||||
2. Find the LRU model on the target node (excluding pinned models)
|
||||
3. Call `POST {neuron}/models/unload` on that model
|
||||
4. The incoming request's lazy-load triggers the new model load
|
||||
|
||||
### Metrics
|
||||
Per-request: model, node, prompt_tokens, completion_tokens, total_tokens, tok_per_sec, time_to_first_token_ms, total_latency_ms. Exposed as Prometheus histograms/counters on a separate port (31314).
|
||||
|
||||
## Tech Stack
|
||||
|
||||
- **Rust 2024 edition** — workspace with 6 crates
|
||||
- **Axum 0.8** — HTTP framework
|
||||
- **reqwest** — HTTP client for proxying to backends
|
||||
- **figment** — config loading (TOML + env vars)
|
||||
- **tokio** — async runtime
|
||||
- **metrics + metrics-exporter-prometheus** — observability
|
||||
- **tracing** — structured logging
|
||||
- **candle** — in-process inference engine (neuron only, with CUDA support)
|
||||
- **cudarc** — patched for neuron's needs (see workspace `[patch]`)
|
||||
- **clap** — CLI parsing
|
||||
- **rusqlite** (bundled) — helexa-bench SQLite system-of-record
|
||||
|
||||
## Build Commands
|
||||
|
||||
```sh
|
||||
cargo build --release # build all crates
|
||||
cargo run -p cortex-cli -- serve # run the gateway
|
||||
cargo test # run all tests
|
||||
cargo clippy --workspace # lint
|
||||
```
|
||||
|
||||
### neuron Features
|
||||
- `cuda`: Enables CUDA acceleration in candle and cudarc/nccl bindings. Without it, falls back to CPU.
|
||||
- `cudnn`: Use cuDNN for convolution/attention kernels (requires `cuda`).
|
||||
- `flash-attn`: FlashAttention kernels (requires `cuda`).
|
||||
- `cuda-integration`: Reserved for GPU-only integration tests (requires multiple CUDA devices + libnccl).
|
||||
|
||||
### Build Scripts
|
||||
- `neuron/build.rs`: Compiles CUDA kernels (`src/cuda/*.cu`) using `cudaforge::KernelBuilder` when `cuda` feature is enabled. Handles compute capability checks (sm_<80 disables bf16 intrinsics). Also captures build metadata: git SHA, dirty flag, timestamp, rustc version, profile, features, candle-core version.
|
||||
|
||||
## CI
|
||||
|
||||
Gitea Actions runs on every push to any branch. All three checks must pass before merging:
|
||||
|
||||
```sh
|
||||
cargo fmt --check --all # formatting
|
||||
cargo clippy --workspace -- -D warnings # lint (warnings are errors)
|
||||
cargo test --workspace # tests
|
||||
```
|
||||
|
||||
Run these locally before pushing. `cargo fmt --all` fixes formatting automatically. Clippy warnings must be resolved, not suppressed with `#[allow(...)]` unless there is a clear rationale.
|
||||
|
||||
Tagged releases (`v*`) build SRPMs for `cortex`, `helexa-neuron`, and `helexa-bench` and publish to COPR (`helexa/helexa`). Build metadata SHA injection: CI sets `HELEXA_BUILD_SHA=$(git rev-parse HEAD)`.
|
||||
|
||||
## Environment
|
||||
|
||||
- Targets Fedora 43 (systemd, SELinux enforcing)
|
||||
- Nodes communicate over a private network (e.g. WireGuard mesh)
|
||||
- cortex listens on port 31313 (API) and 31314 (metrics)
|
||||
- neuron listens on port 13131 on each GPU host
|
||||
- TLS terminated at gateway or via nginx; internal traffic is plaintext over WireGuard
|
||||
|
||||
## Conventions
|
||||
|
||||
- Error handling: `anyhow` for binaries, `thiserror` for library crates
|
||||
- No `unwrap()` in library code; `expect()` only with clear rationale
|
||||
- All public types derive `Debug, Clone, Serialize, Deserialize` where sensible
|
||||
- Config structs use `figment` with TOML as primary source, env vars as override
|
||||
- Prefer `Arc<RwLock<...>>` for shared fleet state; minimize lock duration
|
||||
- SSE streaming uses `tokio_stream` + `eventsource-stream` for parsing
|
||||
- Log at `info` for request routing, `debug` for proxy details, `warn` for eviction and node health, `error` for proxy failures
|
||||
|
||||
## Testing
|
||||
|
||||
### Gateway tests
|
||||
Use mock neurons spawned via axum in `crates/cortex-gateway/tests/common/mod.rs`. Helpers: `spawn_mock_backend()`, `spawn_gateway()`.
|
||||
|
||||
### neuron integration tests
|
||||
- Numerical reference tests (`numerical_reference.rs`) require `NEURON_REF_MODEL_PATH` env var pointing to a HF snapshot directory. Fixtures are f32-based for precision validation against HuggingFace transformers.
|
||||
- CUDA integration tests (`tp_worker_lifecycle_cuda.rs`) gated behind `cuda-integration` feature; requires 2+ CUDA devices (e.g., 2x RTX 5090).
|
||||
|
||||
### Metrics testing
|
||||
Use `install_test_recorder()` in test code to capture metrics without the HTTP listener.
|
||||
|
||||
## helexa-bench
|
||||
|
||||
A continuous, version-aware benchmark harness. Hits each neuron directly on `:13131`, exercises each warm model with a Scenario suite (chat-latency family), and records results into SQLite stamped with the neuron's full `BuildInfo`. The loop is version-aware: skips any (target, build SHA, model, scenario) cell already at `samples_per_version`.
|
||||
|
||||
Packaged as `helexa-bench` RPM (prebuilt-binary spec). One systemd unit, typically on the metrics host.
|
||||
|
||||
## helexa-acp
|
||||
|
||||
Agent Client Protocol bridge — connects ACP editors (Zed, etc.) to any OpenAI-compatible endpoint, cortex by default. Intentionally self-contained: no workspace crate dependencies. Uses `agent-client-protocol` with `unstable_session_model` feature for Zed model picker support. Licensed Apache-2.0 (workspace is GPL-3.0).
|
||||
|
||||
## RPM Packaging
|
||||
|
||||
- `cortex.spec` — installs the `cortex` binary
|
||||
- `helexa-neuron.spec` — installs the `neuron` binary under package name `helexa-neuron` (renamed to avoid Fedora's NEURON neural-simulation package collision)
|
||||
- Systemd units in `data/cortex.service`, `data/neuron.service`
|
||||
- Example configs: `cortex.example.toml`, `neuron.example.toml`, `models.example.toml`
|
||||
|
||||
Install:
|
||||
```sh
|
||||
dnf copr enable helexa/helexa
|
||||
dnf install cortex # gateway host
|
||||
dnf install helexa-neuron # GPU nodes
|
||||
```
|
||||
|
||||
## Configuration Files
|
||||
|
||||
### cortex.toml (gateway)
|
||||
```toml
|
||||
[gateway]
|
||||
listen = "0.0.0.0:31313"
|
||||
metrics_listen = "0.0.0.0:31314"
|
||||
|
||||
[eviction]
|
||||
strategy = "lru" # lru | priority
|
||||
defrag_after_cycles = 50
|
||||
|
||||
[[neurons]]
|
||||
name = "beast"
|
||||
endpoint = "http://beast.internal:13131"
|
||||
```
|
||||
|
||||
### models.toml (catalogue)
|
||||
```toml
|
||||
[[models]]
|
||||
id = "Qwen/Qwen3-Coder-30B-A3B-Instruct"
|
||||
harness = "candle"
|
||||
quant = "Q4_K_M"
|
||||
vram_mb = 19000
|
||||
min_devices = 2
|
||||
min_device_vram_mb = 10000
|
||||
pinned_on = ["beast"] # optional: affinity — run only on these
|
||||
residency_priority = 300 # optional: who may displace whom under VRAM pressure
|
||||
```
|
||||
|
||||
### neuron.toml (per-host)
|
||||
Configured via figment + env override. See `neuron.example.toml` for reference.
|
||||
|
||||
## neuron API Endpoints
|
||||
|
||||
```
|
||||
GET /discovery → hardware discovery (hostname, OS, CUDA, devices, harnesses)
|
||||
GET /health → runtime GPU stats (VRAM, utilization, temperature)
|
||||
GET /models → loaded/unloaded models with VRAM usage
|
||||
POST /models/load → load a model with spec (quant, TP, devices)
|
||||
POST /models/unload → unload a model, freeing device memory
|
||||
GET /models/{id}/endpoint → inference URL for a model
|
||||
GET /version → build metadata (SHA, features, candle version, etc.)
|
||||
```
|
||||
|
||||
## Sources of Truth
|
||||
|
||||
When prose documentation conflicts with code, trust:
|
||||
1. Executable configuration (`*.toml`, `Cargo.toml` features)
|
||||
2. Type definitions in `cortex-core/`
|
||||
3. Test files in `crates/*/tests/` and `*/src/**/*_test.rs`
|
||||
4. `CLAUDE.md` for historical design rationale
|
||||
586
CLAUDE.md
586
CLAUDE.md
@@ -1,26 +1,16 @@
|
||||
# CLAUDE.md — helexa
|
||||
# CLAUDE.md — cortex
|
||||
|
||||
## Project overview
|
||||
|
||||
helexa is a self-hosted LLM serving stack for multi-node GPU inference
|
||||
clusters. It has two components:
|
||||
|
||||
- **cortex** — the per-operator control plane and LLM proxy. A Rust
|
||||
reverse-proxy that sits in front of the fleet and presents a unified
|
||||
OpenAI + Anthropic compatible API surface. It handles model routing,
|
||||
lifecycle management (load/unload/evict), request translation, and
|
||||
metrics collection.
|
||||
- **neuron** — the per-host LLM harness. One instance runs on every GPU
|
||||
host, serving candle-based in-process inference and managing local
|
||||
hardware discovery and model lifecycle.
|
||||
|
||||
(Historical note: cortex originally proxied to mistral.rs nodes; neuron
|
||||
replaced that — see the 2026-05-18 candle-native addendum below.)
|
||||
cortex is a Rust reverse-proxy that sits in front of multiple
|
||||
mistral.rs inference nodes and presents a unified OpenAI + Anthropic
|
||||
compatible API surface. It handles model routing, lifecycle management
|
||||
(load/unload/evict), request translation, and metrics collection.
|
||||
|
||||
## Repository layout
|
||||
|
||||
```
|
||||
helexa/
|
||||
cortex/
|
||||
├── Cargo.toml # workspace root
|
||||
├── cortex.toml # example gateway config
|
||||
├── README.md
|
||||
@@ -94,63 +84,6 @@ Per-request: model, node, prompt_tokens, completion_tokens, total_tokens,
|
||||
tok_per_sec, time_to_first_token_ms, total_latency_ms.
|
||||
Exposed as Prometheus histograms/counters on a separate port.
|
||||
|
||||
### Per-device worker thread (neuron)
|
||||
The neuron daemon dedicates one OS thread per CUDA device it loads
|
||||
onto. That thread binds the device's `CudaContext` once at startup and
|
||||
owns it for the daemon's lifetime; every model load, forward step,
|
||||
KV-cache reset, VRAM query, NCCL init/sanity, NCCL all_reduce, and
|
||||
model drop on that device routes through this thread via a
|
||||
`std::sync::mpsc` job channel. Replies cross back via
|
||||
`tokio::sync::oneshot`.
|
||||
|
||||
Three properties this gives us, in order of weight:
|
||||
|
||||
1. **Context locality.** cudarc binds the CUDA context per OS thread
|
||||
via `cuCtxSetCurrent`. Before this refactor, ad-hoc
|
||||
`tokio::task::spawn_blocking` calls bound the context onto a
|
||||
different thread per request — and `device_vram_mb()` from an
|
||||
async task bound it onto whichever tokio worker happened to be
|
||||
running. Pinning the context to one named thread ends that.
|
||||
2. **Drop safety.** Every `CudaSlice` in a `Tensor`, every
|
||||
`cudarc::nccl::Comm`, and the `CudaContext` itself call `cuMemFree` /
|
||||
`ncclCommDestroy` / `cuCtxDestroy` during `Drop` — and require the
|
||||
right context current. With the worker owning the model slab,
|
||||
`Drop` always runs on the right thread. The cudarc Drop constraint
|
||||
is structurally enforced.
|
||||
3. **Poisoning blast radius.** When a CUDA driver error makes the
|
||||
context unrecoverable, the poison flag lives on the
|
||||
`DeviceWorkerHandle` itself. Subsequent `submit()` calls fast-reject
|
||||
at the channel boundary with a clear "device worker is poisoned"
|
||||
error before any further CUDA work is attempted. The thread doesn't
|
||||
exit (dropping the slab would re-touch the broken context) — it
|
||||
enters a drain-only mode and replies error to everything until the
|
||||
daemon restarts.
|
||||
|
||||
Tensors never escape the worker thread alive. Inference replies carry
|
||||
`Vec<f32>` CPU-side logits; the async caller wraps them in a CPU
|
||||
candle tensor and runs `apply_repeat_penalty` + `LogitsProcessor::sample`
|
||||
without ever rebinding the device context. Sampled tokens come back as
|
||||
`u32`; VRAM queries as `(u64, u64)`. The opaque `ArchHandle(u64)` and
|
||||
`TpHandle(u64)` are the only "references" callers hold to loaded
|
||||
models — they're indices into the worker's state slab, not pointers.
|
||||
|
||||
The TP worker subprocesses in `harness/tp/worker.rs` are the same
|
||||
pattern out-of-process — a dedicated context-owning process per
|
||||
non-zero NCCL rank. The in-process worker in `harness/device_worker/`
|
||||
brings the discipline to rank 0.
|
||||
|
||||
CPU loads (`Device::Cpu` fallback when CUDA is unavailable) keep the
|
||||
legacy `tokio::task::spawn_blocking + Arc<Mutex<ModelArch>>` path —
|
||||
there's no context to own and the channel hop would only add latency.
|
||||
Four `spawn_blocking` references in `harness/candle.rs` are deliberate
|
||||
CPU fallback.
|
||||
|
||||
Canonical narrative lives in
|
||||
`crates/neuron/src/harness/device_worker/mod.rs`'s module
|
||||
doc-comment; touch points (the `Job` enum, the dispatch handlers, the
|
||||
`DeviceWorkerState` struct) are in the sibling `jobs.rs` and
|
||||
`dispatch.rs`.
|
||||
|
||||
## Tech stack
|
||||
|
||||
- **Rust 2024 edition** — workspace with 4 crates
|
||||
@@ -167,7 +100,7 @@ doc-comment; touch points (the `Job` enum, the dispatch handlers, the
|
||||
cargo build --release # build all crates
|
||||
cargo run -p cortex-cli -- serve # run the gateway
|
||||
cargo test # run all tests
|
||||
cargo clippy --workspace --all-targets # lint (test code included)
|
||||
cargo clippy --workspace # lint
|
||||
```
|
||||
|
||||
## CI
|
||||
@@ -177,7 +110,7 @@ pass before merging:
|
||||
|
||||
```sh
|
||||
cargo fmt --check --all # formatting
|
||||
cargo clippy --workspace --all-targets -- -D warnings # lint (warnings are errors)
|
||||
cargo clippy --workspace -- -D warnings # lint (warnings are errors)
|
||||
cargo test --workspace # tests
|
||||
```
|
||||
|
||||
@@ -185,96 +118,6 @@ Run these locally before pushing. `cargo fmt --all` fixes formatting
|
||||
automatically. Clippy warnings must be resolved, not suppressed with
|
||||
`#[allow(...)]` unless there is a clear rationale.
|
||||
|
||||
## Development workflow
|
||||
|
||||
Work each change on its own branch; `main` stays releasable.
|
||||
|
||||
1. Implement on a feature branch (`fix/<issue>-…`, `feat/<issue>-…`).
|
||||
2. Run the CI triad locally (`cargo fmt --check --all`,
|
||||
`cargo clippy --workspace --all-targets -- -D warnings`,
|
||||
`cargo test --workspace`).
|
||||
Local builds are **CPU-only** — the `#[cfg(feature = "cuda")]` neuron/TP
|
||||
paths do NOT compile locally. The branch CI's **CUDA type-check** job is
|
||||
the only thing that validates them, so for any neuron change the push to
|
||||
Gitea is the real gate, not a rubber stamp.
|
||||
3. Push the branch on local-green (no need to ask first), and background-watch
|
||||
its CI run via the gitea-mcp `actions_run_read` tools. Start the next piece
|
||||
of work meanwhile.
|
||||
4. Merge to `main` when the four **validation** jobs are green — Format,
|
||||
Clippy, Test, CUDA type-check. The SRPM / COPR / version-bump jobs are the
|
||||
deploy pipeline (they run on `main`), not validation — don't wait on them.
|
||||
5. Merging/pushing to `main` triggers the auto-deploy pipeline.
|
||||
|
||||
Docs-only changes (no `#[cfg(feature = "cuda")]` impact) can go straight to
|
||||
`main` — there's nothing for the CUDA type-check to prove.
|
||||
|
||||
## Epic execution playbook
|
||||
|
||||
Distilled from the #197 text-to-image epic (2026-07-29/30): seven
|
||||
children — an engine integration, two upstream candle fixes, a new API
|
||||
surface across neuron + cortex + router, metering, fleet rollout, and
|
||||
quantization — planned, implemented, live-validated, and closed in one
|
||||
multi-hour session with no operator input beyond one up-front decision
|
||||
round. This is the template for large, unblocked, well-planned epics;
|
||||
future sessions should reproduce it deliberately.
|
||||
|
||||
1. **De-risk before planning.** Run the throwaway spike on real
|
||||
hardware first. Hand-driving the candle example on beast produced
|
||||
the VRAM/latency numbers and the three findings that became the
|
||||
port spec — the epic was written from evidence, and every design
|
||||
claim in it had already been observed. An epic whose engine risk is
|
||||
retired before filing is an epic that can be finished.
|
||||
2. **Front-load operator decisions, then don't come back.** Identify
|
||||
the genuinely operator-owned decision surfaces (for #197: metering
|
||||
unit, v1 placement, public exposure, optional-scope selection) and
|
||||
settle them in ONE question round before implementation starts.
|
||||
Record the answers as issue comments so they survive context loss.
|
||||
Everything else is implementation detail the session owns.
|
||||
3. **Decisions are goals, not designs.** When live evidence
|
||||
contradicts the planned design, change the design autonomously,
|
||||
keep the decided goal fixed, and record the divergence + rationale
|
||||
on the issue. #197 examples: the text encoder moved to host CPU
|
||||
when the 24 GB tier couldn't hold it beside the DiT; tiled VAE
|
||||
decode was pulled forward a stage when 1024² OOMed on the actual
|
||||
deployment card; allocator-pool trims were added when mixed-size
|
||||
request sequences fragmented into OOMs. The operator decided
|
||||
*benjy cold-swap*, not *how the bytes fit*.
|
||||
4. **Branch-per-child, stack when dependent, never wait idle.** Push
|
||||
on local green, let branch CI (especially the CUDA type-check)
|
||||
validate in the background, and start the next child meanwhile.
|
||||
Merge in dependency order as runs green. CI wall-clock is the
|
||||
session's biggest fixed cost — always have a second workstream
|
||||
(the next child, fleet prep, issue bookkeeping) to overlap it.
|
||||
5. **Validate live at every stage, with guaranteed restore.**
|
||||
Production hosts are the test bench, used through guarded windows:
|
||||
check `in_flight == 0`, evict, test, restore, verify — as one
|
||||
scripted sequence, so a dropped ssh session can't leave the fleet
|
||||
degraded. Every stage's acceptance is an observable behaviour on
|
||||
real hardware; unit tests gate merges, live probes gate closure.
|
||||
6. **The definition of done is external.** #197 closed only when an
|
||||
unmodified OpenAI client got a metered image through the public
|
||||
chain, `script/validate-image.sh` passed against the *deployed RPM
|
||||
build* (not a dev binary), and the cold-swap ran both directions.
|
||||
`/version` probes confirm what is actually live; "merged" is not
|
||||
"done".
|
||||
7. **Write everything down as you go.** Issues get closing comments
|
||||
with real numbers; surprises become documented gotchas the same
|
||||
hour they bite; session memory is checkpointed at each phase
|
||||
boundary so a context break loses nothing. The issue trail should
|
||||
let a cold reader reconstruct both what happened and why.
|
||||
8. **Fix the environment, not just the code.** One-shotting means
|
||||
owning the whole path: rebasing the cudarc fork to unblock a candle
|
||||
bump, forking candle for upstream defects (and filing the upstream
|
||||
PR so the pin is temporary), re-dispatching a flaky CI run, syncing
|
||||
27 GB of weights across the mesh, force-updating a host. Blockers
|
||||
are work items, not stop signs — but production-touching steps get
|
||||
the same guarded-window discipline as tests, and destructive or
|
||||
scope-changing moves still go back to the operator.
|
||||
|
||||
SSH note: the gitea remote host offers multiple agent keys and cuts the
|
||||
connection before reaching the right one. This repo pins the working key via
|
||||
`git config core.sshCommand "ssh -i ~/.ssh/id_grenade -o IdentitiesOnly=yes"`.
|
||||
|
||||
## Environment
|
||||
|
||||
- Targets Fedora 43 (systemd, SELinux enforcing)
|
||||
@@ -621,8 +464,7 @@ quant = "Q4_K_M"
|
||||
vram_mb = 19000
|
||||
min_devices = 2
|
||||
min_device_vram_mb = 10000
|
||||
pinned_on = ["beast"] # optional: affinity — run only on these
|
||||
residency_priority = 300 # optional: who may displace whom under VRAM pressure
|
||||
pinned_on = ["beast"] # optional: never evict from these neurons
|
||||
|
||||
[[models]]
|
||||
id = "Qwen/Qwen3-VL-8B"
|
||||
@@ -649,7 +491,7 @@ and the hardcoded `vram_mb` per node.
|
||||
## Revised repository layout
|
||||
|
||||
```
|
||||
helexa/
|
||||
cortex/
|
||||
├── Cargo.toml
|
||||
├── cortex.toml # gateway config (neurons only)
|
||||
├── models.toml # model catalogue
|
||||
@@ -774,372 +616,58 @@ dnf install cortex # gateway host
|
||||
dnf install helexa-neuron # GPU nodes
|
||||
```
|
||||
|
||||
## 2026-05-18 addendum: candle-native pivot
|
||||
### Phase 11: llama.cpp harness stub
|
||||
|
||||
Phases 11 (llama.cpp harness) and 12 (mistral.rs COPR) below are
|
||||
**superseded**. The project no longer treats mistral.rs or llama.cpp as
|
||||
dependencies — both are conceptually out of scope. neuron becomes a
|
||||
candle-native inference daemon, with `Harness` retained as an
|
||||
internal seam for adding future engines (vision/audio/diffusion) but
|
||||
its only implementation being in-process candle.
|
||||
**Goal:** Prove the harness abstraction works with a second engine.
|
||||
|
||||
The full staged plan for this pivot lives at
|
||||
`~/.claude/plans/create-a-more-aggressive-calm-naur.md`. Summary:
|
||||
**Steps:**
|
||||
1. `crates/neuron/src/harness/llamacpp.rs` — implement the `Harness`
|
||||
trait for llama.cpp's `llama-server`.
|
||||
- `start()` — launch `llama-server` with the correct model path,
|
||||
`--port`, `--n-gpu-layers`, `--tensor-split` args. Track the
|
||||
child process.
|
||||
- `stop()` — send SIGTERM to the child process.
|
||||
- `list_models()` — llama-server serves one model per process, so
|
||||
return a single-element list.
|
||||
- `load_model()` — start a new llama-server process for this model.
|
||||
- `unload_model()` — stop the process.
|
||||
- `inference_endpoint()` — return `http://localhost:{assigned_port}`.
|
||||
2. Port allocation: neuron assigns ports from a range (e.g. 8100-8199)
|
||||
to llama-server instances.
|
||||
3. Register in `HarnessRegistry` when configured:
|
||||
```toml
|
||||
[[harnesses]]
|
||||
name = "llamacpp"
|
||||
binary = "/usr/local/bin/llama-server"
|
||||
port_range = [8100, 8199]
|
||||
```
|
||||
4. Tests: mock llama-server (simple HTTP server returning canned
|
||||
responses), test load/unload/endpoint lifecycle.
|
||||
|
||||
- **Stage 1 (this commit):** delete `mistralrs.rs` and `llamacpp.rs`,
|
||||
scaffold inert `CandleHarness`, drop `endpoint`/`systemd_unit` from
|
||||
`HarnessConfig`, default no-op `start`/`stop` on the `Harness` trait.
|
||||
- **Stages 2–4:** wire up candle model load/unload (quantized Qwen3
|
||||
first), add OpenAI-compatible inference endpoint in neuron, then SSE
|
||||
streaming.
|
||||
- **Stages 5–6:** load-on-activation (default models in config) and
|
||||
unload-on-deactivation (graceful shutdown).
|
||||
- **Stages 7–8:** multi-GPU tensor parallelism and broader model/quant
|
||||
coverage.
|
||||
**Done when:** A model with `harness = "llamacpp"` in `models.toml` can
|
||||
be loaded and served through cortex. Tests pass with mock llama-server.
|
||||
|
||||
Sections of this document that describe mistral.rs HTTP behaviour
|
||||
("mistral.rs API gotchas") are retained as historical context for
|
||||
Phases 1–10 — they document what was true while the project depended
|
||||
on mistral.rs. They do not describe current behaviour.
|
||||
### Phase 12 (lower priority): mistral.rs COPR packaging
|
||||
|
||||
---
|
||||
**Goal:** Fedora RPMs for mistral.rs built against specific CUDA versions.
|
||||
|
||||
### Phase 11 (superseded): llama.cpp harness stub
|
||||
**Steps:**
|
||||
1. `mistralrs-cuda.spec` — RPM spec that clones a pinned mistral.rs git
|
||||
tag, builds with `--features cuda`, links against the system CUDA
|
||||
toolkit. Produces `mistralrs-cuda13-server` (CUDA 13.x / sm_120) and
|
||||
`mistralrs-cuda12-server` (CUDA 12.x / sm_89). Install binary to
|
||||
`/usr/local/bin/mistralrs`.
|
||||
2. COPR build config: enable the NVIDIA CUDA repo as a build dependency.
|
||||
Pin the CUDA toolkit version in `BuildRequires`.
|
||||
3. Gitea Actions or manual workflow: bump the mistral.rs tag in the spec,
|
||||
trigger COPR rebuild.
|
||||
4. neuron's mistralrs harness config references which binary/package
|
||||
provides the mistral.rs binary. neuron could warn at startup if the
|
||||
installed mistral.rs CUDA version doesn't match the discovered driver.
|
||||
|
||||
~~Originally planned as a second engine to prove the harness
|
||||
abstraction.~~ Replaced by the candle harness work in the 2026-05-18
|
||||
addendum above. llama.cpp's any-model/any-hardware breadth is no
|
||||
longer in scope for helexa.
|
||||
**Done when:** `dnf install mistralrs-cuda13-server` on beast provides a
|
||||
working `mistralrs` binary built for Blackwell GPUs. `dnf install
|
||||
mistralrs-cuda12-server` on benjy provides one built for Ada GPUs.
|
||||
|
||||
### Phase 12 (superseded): mistral.rs COPR packaging
|
||||
|
||||
~~Originally planned to ship CUDA-versioned mistral.rs RPMs.~~ Replaced
|
||||
by the candle harness work in the 2026-05-18 addendum above. With
|
||||
mistral.rs out of the dependency tree, there is nothing to package.
|
||||
|
||||
## 2026-05-27 addendum: per-device worker thread
|
||||
|
||||
Replaced the ad-hoc `tokio::task::spawn_blocking` pattern that drove
|
||||
every leader-side CUDA op with one dedicated OS thread per CUDA device,
|
||||
permanently bound to that device's `CudaContext`. All leader-side
|
||||
inference work (GGUF + dense + TP shard load, forward, kv-cache clear,
|
||||
NCCL init/sanity, NCCL all_reduce, VRAM query, model drop) routes
|
||||
through the worker via a `std::sync::mpsc` channel; tensors never
|
||||
escape the worker thread alive. See "Per-device worker thread (neuron)"
|
||||
above and `crates/neuron/src/harness/device_worker/mod.rs` for the
|
||||
canonical narrative.
|
||||
|
||||
Motivated by the 2026-05-26 silent-hang on beast: a CUDA OOM cascade
|
||||
poisoned the device context on whichever spawn_blocking thread caught
|
||||
it, and subsequent requests stalled invisibly on the pool lock. After
|
||||
the refactor, the same failure mode shows up in journalctl as
|
||||
`prefill sample failed; logits unhealthy nan: 248320/248320` followed
|
||||
by `failed, model marked poisoned`. The thread stays alive and rejects
|
||||
subsequent requests at the channel boundary.
|
||||
|
||||
Landed in four PRs:
|
||||
|
||||
- **Phase 1** (`081b532`) — device_worker module + 8 VRAM-query sites
|
||||
route through the worker. CPU build only; smoke on beast confirmed
|
||||
a persistent `cuda-dev-0` thread.
|
||||
- **Phase 2** (`b179204`) — single-GPU forward + clear_kv + drop via
|
||||
the worker. `LoadedModel.arch_handle: Option<ArchHandle>` replaces
|
||||
`Arc<Mutex<ModelArch>>` for CUDA loads. CPU keeps the legacy path.
|
||||
- **Phase 3** (`76ab24d`) — TP forward + NCCL init/sanity + leader
|
||||
KV-clear routed through the worker. `WorkerPool.leader_nccl` moves
|
||||
into the worker's state. `TpLoadedModel.leader_handle: TpHandle`
|
||||
replaces `Arc<Mutex<TpLeaderModel>>`. CUDA-only TP smoke deferred to
|
||||
next deploy.
|
||||
- **Phase 4** (`b4f3576`) — GGUF + dense + TP shard loads move onto
|
||||
the worker. The `Job::TransferIn` / `Job::CloneLeaderComm` bridges
|
||||
from Phases 2/3 deleted; `SendComm` newtype no longer needed in the
|
||||
load path. `grep -rn spawn_blocking crates/neuron/src/harness/`
|
||||
returns only deliberate CPU-fallback hits after this PR.
|
||||
|
||||
## 2026-06-13 addendum: build metadata + helexa-bench
|
||||
|
||||
Two coupled additions so fleet performance can be tracked automatically
|
||||
across neuron updates instead of by hand-running `script/bench.py` and
|
||||
editing `doc/benchmarks.md`.
|
||||
|
||||
**neuron build metadata + `GET /version`.** neuron's `build.rs` now also
|
||||
captures build identity (`HELEXA_GIT_SHA` — preferring a CI/RPM-injected
|
||||
`HELEXA_BUILD_SHA`, falling back to git, else `unknown` — plus dirty
|
||||
flag, build timestamp, rustc version, profile, enabled cargo features,
|
||||
and a best-effort `candle-core` version from `Cargo.lock`). These are
|
||||
exposed as `cortex_core::build_info::BuildInfo` (new module) from a new
|
||||
`GET /version` endpoint (`neuron/src/version.rs`, wired in `api.rs`) and
|
||||
in clap's `--version` long form. The SHA is injected in CI
|
||||
(`build-prerelease.yml` build-neuron step: `export HELEXA_BUILD_SHA=$(git
|
||||
rev-parse HEAD)`) and via `--define helexa_commit` in the source-build
|
||||
spec, so tarball-built RPMs report the real SHA. `/version` is now the
|
||||
canonical "which build is live" probe (supersedes the per-host RPM-sha
|
||||
check in the fleet-validation flow).
|
||||
|
||||
**`crates/helexa-bench`** — a new binary: a continuous, version-aware
|
||||
benchmark harness (one systemd unit, typically on the metrics host). It
|
||||
hits each neuron **directly** on `:13131`, exercises each **warm**
|
||||
(`status == "loaded"`) model with an extensible `Scenario` suite (phase
|
||||
1: the chat-latency family ported verbatim from `bench.py` — synthetic
|
||||
128/4096-tok prompts, `/no_think`, streamed TTFT + decode-window
|
||||
tok/s), and records each run into a SQLite system-of-record stamped with
|
||||
the neuron's full `BuildInfo`. The loop is **version-aware**: it skips
|
||||
any (target, build SHA, model, scenario) cell already at
|
||||
`samples_per_version`, so a steady fleet costs only cheap `/version` +
|
||||
`/models` polls until a new SHA ships. `helexa-bench report` regenerates
|
||||
the `benchmarks.md`-style table from the DB. `kind = "openai"` targets
|
||||
(mistral.rs/llama.cpp comparison) are scaffolded but not yet wired.
|
||||
Packaged as the `helexa-bench` RPM (prebuilt-binary spec, outbound-only
|
||||
so no firewalld service) via the same `build-prerelease.yml` pipeline.
|
||||
|
||||
## 2026-07-09 addendum: concurrency & admission model
|
||||
|
||||
Historical note: earlier docs described inference as "batch-1,
|
||||
serialized per model, no admission control" and the README ruled out
|
||||
continuous batching "permanently." **Both are obsolete.** Each loaded
|
||||
model on a neuron now serves multiple concurrent requests behind bounded
|
||||
admission control, and cortex routes load-aware — not as a pure
|
||||
passthrough.
|
||||
|
||||
**Admission control (#53, #54).** Every loaded model owns an
|
||||
`AdmissionController` (`crates/neuron/src/harness/admission.rs`). It
|
||||
bounds `max_in_flight` running requests plus a `max_queue_depth` waiting
|
||||
queue with a `max_wait_secs` deadline; over-capacity requests fast-reject
|
||||
with `429`/`503` + `Retry-After` rather than hanging. A per-principal
|
||||
`max_per_principal` cap (keyed on the account/key headers cortex stamps)
|
||||
gives fair-share so one client cannot monopolise a model. Defaults live
|
||||
in `config.rs` (`max_in_flight=1`, `max_queue_depth=8`, `max_wait_secs=30`,
|
||||
`max_per_principal=2`); production overrides per host — e.g. beast runs
|
||||
`max_in_flight=8`. A request holds its `AdmissionPermit` for its whole
|
||||
lifetime; the forward itself is still serialized behind the single-GPU
|
||||
`inference_lock` / TP pool mutex.
|
||||
|
||||
**Batched decode (#98).** When `max_in_flight > 1` on a snapshot-capable
|
||||
arch (`qwen3_5` / `qwen3_next`), the lockstep batch engine
|
||||
(`crates/neuron/src/harness/engine.rs`) multiplexes the concurrent
|
||||
sequences into one decode step (ragged prompts, mid-stream joins), so
|
||||
those models are genuinely not batch-1. Runtime kill switch
|
||||
`NEURON_BATCHING=0`. Archs without snapshot support, or
|
||||
`max_in_flight=1`, keep the serialized single-sequence path.
|
||||
|
||||
**Live load surface.** neuron `GET /health` publishes per-model
|
||||
`ModelLoad { in_flight, queue_depth }` and per-device
|
||||
`DeviceHealth { vram_used_mb, vram_free_mb, utilization_pct, temp_c }`.
|
||||
cortex's poller reads these (~10s) into `NodeState.model_load` and picks
|
||||
the least-busy replica in `router.rs` — this is why cortex is load-aware,
|
||||
not first-loaded-node. The concurrency capacity of a neuron:model is
|
||||
therefore a real number today: `max_in_flight` (+ `max_queue_depth`
|
||||
burst absorption), with `in_flight`/`queue_depth` the live utilisation.
|
||||
|
||||
**Observability gap.** That live load — and the device health — is
|
||||
currently consumed only by routing; it is **not** exported to cortex's
|
||||
Prometheus surface, and neuron has no `/metrics` endpoint. Nor is live
|
||||
tok/s aggregated (per-request `FinishTiming` goes to the caller in
|
||||
`usage.helexa_timing` only) or `max_in_flight` advertised. Closing this
|
||||
(export the polled load, advertise the ceiling, roll up tok/s, count
|
||||
rejections) is tracked separately — it is a plumbing gap, not a
|
||||
measurement one, since the signals already exist in memory.
|
||||
|
||||
## Application-owned system prompts (#179)
|
||||
|
||||
System prompts belong to the **calling application**, never to the
|
||||
operator or the serving chain. cortex and helexa-router proxy inference
|
||||
bodies **without adding to them**: no injected prompt, no house style, no
|
||||
default when the caller sends none, no rewriting or reordering of what
|
||||
was sent.
|
||||
|
||||
This is a contract with API consumers, documented in the README
|
||||
("Tailoring model behaviour"), not an incidental behaviour — treat any
|
||||
change that puts content into a proxied request as a breaking change.
|
||||
When adding a surface or a translation path, the system slot must map
|
||||
straight through:
|
||||
|
||||
- `/v1/chat/completions` — `messages[role == "system"]`, all of them, in
|
||||
order (the model sees the last one last, so last-wins is what users
|
||||
observe).
|
||||
- `/v1/responses` — `instructions`, plus `input` items with
|
||||
`role: system`.
|
||||
- `/v1/messages` — top-level `system`, both the string and the
|
||||
content-block-array forms, translated into a system message.
|
||||
|
||||
Pinned by `crates/cortex-gateway/tests/system_prompt.rs` (asserts what
|
||||
cortex *forwarded upstream*, including a control proving nothing is
|
||||
injected when the caller sends no prompt) and the system-prompt cases in
|
||||
`crates/neuron/src/harness/chat_template.rs`.
|
||||
|
||||
## 2026-07-30 addendum: text-to-image serving (epic #197)
|
||||
|
||||
Image generation is a first-class modality. **Tongyi-MAI/Z-Image-Turbo**
|
||||
(6B single-stream DiT + Qwen3-4B text encoder + VAE, Apache 2.0,
|
||||
flow-matching, 9-step turbo) serves candle-natively through the same
|
||||
device-worker discipline as the text archs. Epic #197, PRs #205–#213.
|
||||
|
||||
**Engine (`crates/neuron/src/harness/image.rs`).** `ZImagePipeline`
|
||||
lives in the device worker's image slab (`ImageHandle`; `LoadImage` /
|
||||
`DropImage` / `GenerateImage` jobs — one job per generation, since a
|
||||
denoise loop saturates the device). CPU-side RGB replies preserve the
|
||||
tensors-never-escape invariant; PNG encoding happens async-side.
|
||||
Component placement is the VRAM story:
|
||||
|
||||
- The **text encoder runs on the host CPU** by default (f32, resident
|
||||
in system RAM; `[harness.candle.image] te_device`) — prompt encoding
|
||||
is one short forward and the features are tiny, and a 24 GB card
|
||||
cannot hold an 8 GB GPU TE beside the resident DiT.
|
||||
- **Tiled VAE decode** (64-latent tiles, 16-latent seam overlap,
|
||||
weighted blending) bounds decoder transients at any resolution;
|
||||
full-frame decode OOMs 24 GB at 1024² and 32 GB at ≥1536².
|
||||
- **`quant = "q8_0"`** ISQ-quantizes the DiT in situ from the dense
|
||||
safetensors (no GGUF artifact): ~6.4 GB DiT, f32 activations,
|
||||
fixed-seed output visually identical to BF16. Opens the 12 GB tier.
|
||||
- The cudarc **allocator pool is trimmed after every image job** —
|
||||
mixed-resolution sequences otherwise fragment it until a generation
|
||||
that would fit a clean slate OOMs.
|
||||
|
||||
**candle is pinned to a fork** (`[patch.crates-io]` →
|
||||
`grenade/candle`, branch `z-image-mask-elision`) carrying: all-ones
|
||||
attention-mask elision so flash-attn actually engages (upstream PR
|
||||
huggingface/candle#3798 — drop the pin when it lands), the
|
||||
`QuantizedZImageTransformer2DModel`, and an f32→bf16 cast around
|
||||
flash-attn for the quantized path. The cudarc fork is rebased onto
|
||||
0.19.8 so one cudarc serves both neuron's NCCL layer and candle 0.11.
|
||||
|
||||
**Surface.** `POST /v1/images/generations` on neuron, cortex, and the
|
||||
public router (the router has an explicit route allowlist — new
|
||||
endpoints must be added there; the edge nginx forwards `/v1/*` by
|
||||
prefix). OpenAI envelope: `n=1`, `b64_json`, PNG; extensions `seed`,
|
||||
`negative_prompt` (CFG doubles per-step cost), `guidance_scale`,
|
||||
`num_steps`. Errors conform to #63 (`wrong_modality` 422 for
|
||||
cross-modality requests, `invalid_image_params` 400 pre-admission).
|
||||
Wire types in `cortex-core/src/images.rs`. Admission reuses #53/#54
|
||||
with `max_in_flight = 1`.
|
||||
|
||||
**Metering (#202).** Unit is **megapixel-steps**
|
||||
(`w × h × steps / 1e6`, CFG ×2, 1-unit floor), reported as
|
||||
`usage.helexa_image_units` + `usage.helexa_timing`
|
||||
(encode/denoise/decode ms). Budget enforcement bridges into the token
|
||||
ledger at `TOKENS_PER_IMAGE_UNIT = 1000` — fail-closed pre-dispatch,
|
||||
settled from actual units. A native image unit in the clearing-house
|
||||
contract is the open follow-up on #202.
|
||||
|
||||
**Placement is automatic cold-swap.** The router ranks catalogue
|
||||
cold-load targets pinned → free-fit → evictable-fit → most-free using
|
||||
live per-device VRAM (now stored on `NodeState.device_health`), with a
|
||||
catalogue-`vram_mb` fallback for the evictable estimate (neuron
|
||||
reports `vram_used_mb: null`), and evicts unpinned LRU models on the
|
||||
chosen node before loading. Catalogue pinning steers: beast's 27B is
|
||||
pinned, so image loads land on benjy and swap the 8B out; the next
|
||||
text request swaps it back. Measured: image cold-swap 10.9 s,
|
||||
text restore 2.6 s.
|
||||
|
||||
**Serving tiers** (bench + live numbers, 2026-07-29/30):
|
||||
|
||||
| Host | Precision | Resident | Ceiling | 1024² |
|
||||
|---|---|---|---|---|
|
||||
| beast (5090, flash-attn flavour) | BF16 / q8 | 14.6 / 9.4 GB | 2048² (37.7 u) | q8 denoise 2.8 s |
|
||||
| benjy (4090) — v1 placement | BF16 | 14.6 GB | 1024² | 11.4 s wall |
|
||||
| quadbrat (3060) | q8_0 | 8.7 GB | 768² | OOM (naive f32 attn) |
|
||||
|
||||
The 3060 ceiling lifts to 1024² if #95 extends flash-attn to the
|
||||
ampere flavour.
|
||||
|
||||
**Ops sharp edges.**
|
||||
|
||||
- neuron binaries are **per-sm** (`CUDA_COMPUTE_CAP`; CI builds
|
||||
ampere/ada/blackwell flavours): a blackwell build throws
|
||||
`CUDA_ERROR_INVALID_PTX` on ada. Local hand-builds must also match
|
||||
the driver's CUDA version (13.0), not the `/usr/local/cuda` symlink
|
||||
(13.2 on beast), or loads die with
|
||||
`CUDA_ERROR_UNSUPPORTED_PTX_VERSION`.
|
||||
- The production neuron runs as the `neuron` system user with its
|
||||
**own HF cache** (`/var/lib/neuron/.cache`) — first image load on a
|
||||
fresh host is a one-time ~29 GB fetch (beast and benjy are warm).
|
||||
- Fleet validation: `script/validate-image.sh <host>` (evicts and
|
||||
restores a co-resident text model around the probe window).
|
||||
helexa-bench has an `image:<px>` scenario (default `[1024]`),
|
||||
gated on the `image` capability; text scenarios gate image models
|
||||
out.
|
||||
- Deferred by operator decision: Z-Image base variant, operator
|
||||
fine-tunes, Z-Image-Edit (`images/edits`). Qwen-Image-2512 is the
|
||||
parked "frontier image" tier (own epic if taken).
|
||||
|
||||
## 2026-09-02 addendum: qwen4_exp (Qwen3.8-Flash-Next) text path
|
||||
|
||||
`Qwen/Qwen3.8-Flash-Next` is `model_type: qwen4_exp` — a new
|
||||
architecture generation, not a Qwen3 variant. Epic #307; the port spec
|
||||
is `doc/qwen4_exp-port-spec.md` and remains the reference for every
|
||||
dimension. The text path is implemented in `crates/neuron/src/harness/
|
||||
arch/qwen4_exp/` and routed in `harness/candle.rs`; it loads a
|
||||
checkpoint and returns logits.
|
||||
|
||||
**What is genuinely new**, and where each lives:
|
||||
|
||||
- `hyper.rs` — **four residual streams**, not one. The inter-layer
|
||||
residual is `hidden_size * hc_count` (10240) wide, initialised as four
|
||||
copies of the token embedding. There is **no `input_layernorm`, no
|
||||
`post_attention_layernorm` and no final `model.norm`** in the
|
||||
checkpoint: the `hc_norm` inside each hyper-connection does all three
|
||||
jobs. Do not go looking for the missing tensor.
|
||||
- `ple.rs` — a hashed n-gram embedding on **zero-indexed layer 1**
|
||||
(`ple_layer_ids: [2]` is one-indexed upstream) over a 320,001,536-row
|
||||
table, 28.44% of the model's parameters. Three pieces: addressing
|
||||
(`NGramHasher`), the gather (`NGramTable` trait — residency is #310),
|
||||
and consumption (`PleBlock`).
|
||||
- `qsa.rs` — sparse attention. Blocks of 4 past positions are scored by
|
||||
a 4-head side channel and only the best 512 attended, which is what
|
||||
makes a 262k window affordable. Keeps a **second KV cache** (12
|
||||
layers × 1 head × 128 dims, ~3 KiB/token) that every `kv_budget_mb`
|
||||
arithmetic must account for.
|
||||
- `decoder.rs`, `model.rs` — the composition.
|
||||
|
||||
**Reused from `arch/qwen3_5/` unchanged**: `rope.rs` (identical
|
||||
interleaved M-RoPE config), the GatedDeltaNet for 36 of 48 layers, and
|
||||
the MoE routing. Those reuses needed three small seams in `qwen3_5`,
|
||||
all behaviour-preserving: `apply_partial_rotary` / `cos_sin_at` on the
|
||||
rotary, `GatedDeltaNetParams` on the linear attention, and `from_parts`
|
||||
constructors on the MoE and MLP.
|
||||
|
||||
### Sharp edges
|
||||
|
||||
- **Read `model.safetensors.index.json` before writing a loader.** It is
|
||||
170 KB, needs no auth, and `curl`-able from HF at the pinned revision.
|
||||
It caught a real bug: the QSA indexer is at
|
||||
`...self_attn.indexer.index_qk_proj.weight`, a **submodule of the
|
||||
attention block**, where the spec's prose implied a sibling. That
|
||||
failure would otherwise have surfaced after a 360 GB download.
|
||||
- **The three PLE buffers are I64 in a bf16 checkpoint.** Read through
|
||||
the model's VarBuilder they round multipliers of ~1e13 and prime vocab
|
||||
sizes of ~2e7 into nonsense; the table is then sized wrong and the
|
||||
addressing still returns rows. Force the dtype.
|
||||
- **A QSA mask must never reach the flash-attn path.**
|
||||
`attention_context` used to take `Option<&Tensor>` and read
|
||||
`is_some()` as "causal", never looking at the tensor — so an arbitrary
|
||||
mask was silently replaced by a dense causal one. It is now
|
||||
`AttnMask::{None, Causal, Additive}`, and `Additive` forces the eager
|
||||
path. Whether QSA can use flash-attn at all (it needs a *gather*, not
|
||||
a mask) is open and now on the critical path.
|
||||
- **PLE's short conv is dilated** by `ngram_size`, so it reaches 9
|
||||
positions back and its cached context is 9 wide, not `kernel_size`. It
|
||||
cannot reuse `run_causal_conv1d`, which assumes dilation 1.
|
||||
- **`output_gate_type: sigmoid` and `mamba_ssm_dtype: float32`** are the
|
||||
checkpoint's explicit choices, against our SiLU fallback and #284's
|
||||
bf16 round-trip default. A port must not answer an upstream choice
|
||||
with our own.
|
||||
|
||||
### What the text path does not yet do
|
||||
|
||||
- **No batched decode.** `supports_kv_snapshot` is false, so it serves
|
||||
one request at a time whatever `max_in_flight` says. Extending
|
||||
`snapshot.rs` to carry PLE's two state slots and the indexer cache is
|
||||
its own piece of work.
|
||||
- **No tensor parallelism** — absent from `TP_SUPPORTED_MODEL_TYPES`, so
|
||||
a `tensor_parallel` request is refused rather than silently served
|
||||
from one device.
|
||||
- **The n-gram table's residency is undecided** (#310). The shipped
|
||||
`ShardedNGramTable` holds the shards wherever the VarBuilder put them,
|
||||
which for 27 GB is the problem that issue exists to solve; it is the
|
||||
control, not the answer.
|
||||
- **Gate 6 (full-model logits parity against a reference trace) is
|
||||
open.** Gates 4 and 5 — QSA below budget is exactly dense, and above
|
||||
budget matches a naive oracle — are closed.
|
||||
- **MTP (#313), vision (#314), quantisation and rollout (#315)** are
|
||||
untouched.
|
||||
This is a separate repo/spec — not part of the cortex workspace — but
|
||||
tightly coupled operationally. Track it as a sibling project.
|
||||
|
||||
3815
Cargo.lock
generated
3815
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
32
Cargo.toml
32
Cargo.toml
@@ -5,20 +5,13 @@ members = [
|
||||
"crates/cortex-gateway",
|
||||
"crates/cortex-cli",
|
||||
"crates/neuron",
|
||||
"crates/helexa-acp",
|
||||
"crates/helexa-angels",
|
||||
"crates/helexa-bench",
|
||||
"crates/helexa-router",
|
||||
"crates/helexa-stream",
|
||||
"crates/helexa-tools",
|
||||
"crates/helexa-upstream",
|
||||
]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.1.16"
|
||||
version = "0.1.12"
|
||||
edition = "2024"
|
||||
license = "GPL-3.0-or-later"
|
||||
repository = "https://git.lair.cafe/helexa/helexa"
|
||||
repository = "https://git.lair.cafe/helexa/cortex"
|
||||
|
||||
[workspace.dependencies]
|
||||
# async runtime
|
||||
@@ -34,7 +27,7 @@ serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
toml = "0.8"
|
||||
|
||||
# http client (for proxying to neuron backends)
|
||||
# http client (for proxying to mistralrs backends)
|
||||
reqwest = { version = "0.12", features = ["json", "stream"] }
|
||||
|
||||
# observability
|
||||
@@ -67,22 +60,3 @@ eventsource-stream = "0.2"
|
||||
# workspace crates
|
||||
cortex-core = { path = "crates/cortex-core" }
|
||||
cortex-gateway = { path = "crates/cortex-gateway" }
|
||||
|
||||
# Patched cudarc (affects neuron's 0.19.x only; candle's 0.17.x is
|
||||
# untouched since the fork is 0.19.8, rebased for the candle 0.11 bump, and now also serves candle's ^0.19.8 req — one unified cudarc). Adds
|
||||
# Comm::abort / get_async_error / raw comm() — needed for #17 Stage 2 TP
|
||||
# hang-recovery (abort a wedged collective from another thread, then
|
||||
# rebuild the comm). Pinned to a fork revision pending upstream review
|
||||
# (grenade/cudarc @ nccl-comm-abort).
|
||||
[patch.crates-io]
|
||||
cudarc = { git = "https://github.com/grenade/cudarc", rev = "6904cf2e651a0384eac318f003ce6417f47f5d94" }
|
||||
# Patched candle (#199): z_image all-ones attention-mask elision so the
|
||||
# flash-attn path actually engages — unlocks >=1536^2 denoising on the
|
||||
# blackwell (flash-attn) builds. Upstream PR:
|
||||
# https://github.com/huggingface/candle/pull/3798 — drop this pin (and
|
||||
# the fork) when it lands in a release. All four crates pin to the same
|
||||
# rev so candle types unify across the graph.
|
||||
candle-core = { git = "https://github.com/grenade/candle", rev = "f8fa85fd2753b334cb4f42e7357f8ba9d9bb2940" }
|
||||
candle-nn = { git = "https://github.com/grenade/candle", rev = "f8fa85fd2753b334cb4f42e7357f8ba9d9bb2940" }
|
||||
candle-transformers = { git = "https://github.com/grenade/candle", rev = "f8fa85fd2753b334cb4f42e7357f8ba9d9bb2940" }
|
||||
candle-flash-attn = { git = "https://github.com/grenade/candle", rev = "f8fa85fd2753b334cb4f42e7357f8ba9d9bb2940" }
|
||||
|
||||
306
README.md
306
README.md
@@ -1,68 +1,24 @@
|
||||
# helexa
|
||||
# cortex
|
||||
|
||||
**Near-frontier AI for mortals.**
|
||||
A Rust reverse-proxy and fleet management layer for multi-node
|
||||
[mistral.rs](https://github.com/EricLBuehler/mistral.rs) inference clusters.
|
||||
|
||||
helexa is a self-hosted LLM serving stack, written in Rust, for people
|
||||
who run open-weight models on their own consumer GPUs. It has two
|
||||
components:
|
||||
## Problem
|
||||
|
||||
- **cortex** — the per-operator control plane and LLM proxy. It sits in
|
||||
front of your GPU fleet and presents a unified OpenAI + Anthropic
|
||||
compatible API surface, handling model routing, lifecycle management
|
||||
(load / unload / evict), request translation, and metrics.
|
||||
- **neuron** — the per-host LLM harness. One instance runs on every GPU
|
||||
host, serving candle-based in-process inference and managing local
|
||||
hardware discovery and model lifecycle.
|
||||
Running local LLMs across multiple GPU nodes (different VRAM tiers, different
|
||||
model affinities) requires a unified API surface that:
|
||||
|
||||
## Why
|
||||
|
||||
Two principles constrain everything in this repository:
|
||||
|
||||
1. **Frontier or close to it.** helexa serves the open-weight models
|
||||
that get nearest to frontier capability — not every architecture
|
||||
ever published.
|
||||
2. **Consumer hardware.** Everything must run on the cards mortals can
|
||||
actually buy: a 3060 here, a 4090 there, a 5090 if you got lucky.
|
||||
Mixed VRAM tiers across mismatched boxes are the expected topology,
|
||||
not a degraded case.
|
||||
|
||||
GPU acquisition is harder than it was a year ago, and the gap between
|
||||
what cloud providers charge and what your own silicon costs keeps
|
||||
widening. The intersection of those two principles — near-frontier
|
||||
models, squeezed onto hardware you own — is helexa's entire niche.
|
||||
|
||||
The secondary objective is **predictable consumption**. If you own the
|
||||
hardware, your tooling shouldn't break because a cloud provider changed
|
||||
billing, deprecated a model, or reshaped an API. cortex's OpenAI and
|
||||
Anthropic surfaces are a stability contract: point your editor, agent,
|
||||
or CLI at it once, and it keeps working.
|
||||
|
||||
## What helexa is not
|
||||
|
||||
This is an intentionally different path from vLLM, SGLang, and peers —
|
||||
not a smaller version of them. Out of scope, permanently:
|
||||
|
||||
- Any-model breadth. Architectures are ported because they're at or
|
||||
near the frontier, not to complete a compatibility matrix.
|
||||
- Datacenter-class scheduling. No sophisticated continuous-batching /
|
||||
paged-attention machinery — the workload is a handful of operators
|
||||
and their agents, not 200 QPS.
|
||||
- Wrapping external inference engines. neuron builds directly on
|
||||
[candle](https://github.com/huggingface/candle); every model
|
||||
architecture it serves is implemented in this repository, ported
|
||||
against the HuggingFace reference.
|
||||
|
||||
One thing that is *not* a principle: CUDA exclusivity. All high-end
|
||||
consumer hardware is in scope. helexa is CUDA-only today because
|
||||
that's the hardware on the bench — nothing ships untested — and ROCm
|
||||
or other consumer accelerators join as soon as there's real hardware
|
||||
to build against.
|
||||
|
||||
In scope, and where the engineering effort goes: aggressive
|
||||
quantization (GGUF Q4_K_M / Q6_K / Q8_0), NCCL tensor parallelism
|
||||
across heterogeneous consumer GPUs, careful CUDA failure handling, and
|
||||
single-request latency — the performance that one operator at a
|
||||
keyboard actually feels.
|
||||
- Presents a **single `/v1/models` catalogue** merging every model across every
|
||||
node.
|
||||
- **Routes requests** to the correct node based on where a model is loaded (or
|
||||
*can* be loaded).
|
||||
- Manages **model lifecycle** — unload cold models, reload on demand, pin
|
||||
critical ones — using the mistral.rs
|
||||
`/v1/models/{unload,reload,status}` HTTP API (PR #1828+).
|
||||
- Translates between **OpenAI and Anthropic** request/response envelopes so
|
||||
every client in the homelab speaks whichever dialect it prefers.
|
||||
- Captures **per-request metrics** (tokens, tok/s, TTFT, latency) and exposes
|
||||
them as Prometheus counters/histograms.
|
||||
|
||||
## Architecture
|
||||
|
||||
@@ -72,83 +28,65 @@ keyboard actually feels.
|
||||
└──────┬───────┘ └─────┬────┘ └──────┬─────┘ └──────┬─────┘
|
||||
│ │ │ │
|
||||
└────────────────┴──────┬───────┴───────────────┘
|
||||
│ OpenAI + Anthropic APIs
|
||||
│
|
||||
┌──────────▼──────────┐
|
||||
│ cortex │
|
||||
│ (cortex-gateway) │
|
||||
│ cortex │
|
||||
│ (cortex-gateway) │
|
||||
│ │
|
||||
│ Router · Metrics │
|
||||
│ Evictor · Translate│
|
||||
└──┬──────┬────────┬──┘
|
||||
│ │ │
|
||||
┌──────────▼┐ ┌──▼─────┐ ┌▼──────────┐
|
||||
│ neuron │ │ neuron │ │ neuron │
|
||||
│ :13131 │ │ :13131 │ │ :13131 │
|
||||
│ candle │ │ candle │ │ candle │
|
||||
│ gpu-large │ │gpu-med │ │ gpu-small │
|
||||
│ mistralrs │ │mistral │ │ mistralrs │
|
||||
│ serve │ │rs serve│ │ serve │
|
||||
│ :8080 │ │ :8080 │ │ :8080 │
|
||||
└───────────┘ └────────┘ └───────────┘
|
||||
private network (.internal)
|
||||
```
|
||||
|
||||
cortex discovers each neuron's hardware (devices, VRAM, compute
|
||||
capability) at runtime and matches it against a model catalogue
|
||||
(`models.toml`) to decide placement: which models fit where, what to
|
||||
evict when VRAM is tight, where to route a request right now. Adding a
|
||||
GPU host to the fleet is one `[[neurons]]` entry — no device specs in
|
||||
config.
|
||||
|
||||
### Crates
|
||||
|
||||
| Crate | Purpose |
|
||||
|---|---|
|
||||
| `cortex-core` | Shared types: config, node/model state, metrics, OpenAI/Anthropic envelopes, harness trait, discovery types |
|
||||
| `cortex-gateway` | Axum HTTP server: proxy, router, evictor, poller, metrics exporter |
|
||||
| `neuron` | Per-host daemon: GPU discovery, in-process candle inference, NCCL tensor parallelism, model lifecycle API |
|
||||
| `cortex-core` | Shared types: config, node/model state, metrics, OpenAI/Anthropic request/response envelopes |
|
||||
| `cortex-gateway` | Axum HTTP server: proxy, router, evictor, metrics exporter |
|
||||
| `cortex-agent` | Per-node sidecar: polls local mistralrs, reports to gateway, handles restart/defrag |
|
||||
| `cortex-cli` | CLI entrypoint (`cortex serve`, `cortex status`, etc.) |
|
||||
| `helexa-acp` | Agent Client Protocol bridge — connects ACP editors (Zed, etc.) to any OpenAI-compatible endpoint, cortex by default |
|
||||
|
||||
## The engine
|
||||
## Node setup
|
||||
|
||||
neuron runs inference in-process on candle — there is no external
|
||||
inference server to babysit. The parts that earn their keep:
|
||||
Each GPU node runs `mistralrs serve` with a multi-model config. Models are
|
||||
declared but start **unloaded** — mistral.rs lazy-loads on first request and
|
||||
the gateway can explicitly unload/reload via the HTTP API.
|
||||
|
||||
- **Per-device worker threads.** Every CUDA device gets one dedicated
|
||||
OS thread that owns its CUDA context for the daemon's lifetime. All
|
||||
loads, forward passes, KV-cache resets, NCCL collectives, VRAM
|
||||
queries, and unloads route through it; tensors never escape it
|
||||
alive. Context binding is pinned to a known thread, the CUDA `Drop`
|
||||
contract is structurally safe, and a driver error poisons one worker
|
||||
— visibly — instead of hanging the whole process.
|
||||
- **Tensor parallelism on consumer cards.** Megatron-style row/column
|
||||
parallel layers with NCCL all-reduce, spanning the mismatched GPUs
|
||||
you actually have. A step watchdog aborts wedged collectives instead
|
||||
of letting a request hang forever.
|
||||
- **Text-to-image.** Z-Image-Turbo (6B S3-DiT, Apache 2.0) served
|
||||
candle-native through the same device-worker discipline: OpenAI
|
||||
`/v1/images/generations` end to end, ~11 s for a 1024x1024 image on
|
||||
an RTX 4090, metered in megapixel-steps.
|
||||
- **Current model focus: the Qwen3 family** — dense and GGUF-quantized,
|
||||
including the hybrid linear-attention (Gated DeltaNet) generation.
|
||||
Vision support is in progress. Each architecture is ported against
|
||||
its HuggingFace reference implementation.
|
||||
Example node systemd unit:
|
||||
|
||||
See `CLAUDE.md` for design rationale and
|
||||
`crates/neuron/src/harness/device_worker/` for the worker narrative.
|
||||
```ini
|
||||
# /etc/systemd/system/mistralrs.service
|
||||
[Unit]
|
||||
Description=mistral.rs inference server
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
## Install
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/local/bin/mistralrs serve \
|
||||
--from-config /etc/mistralrs/config.toml \
|
||||
--port 8080
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
Environment=CUDA_VISIBLE_DEVICES=0,1
|
||||
|
||||
Pre-built RPMs for Fedora:
|
||||
|
||||
```sh
|
||||
dnf copr enable helexa/helexa
|
||||
dnf install cortex # on the gateway host
|
||||
dnf install helexa-neuron # on each GPU host
|
||||
systemctl enable --now cortex # or neuron, respectively
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
## Configure
|
||||
## Gateway config
|
||||
|
||||
```toml
|
||||
# /etc/cortex/cortex.toml
|
||||
# cortex.toml
|
||||
[gateway]
|
||||
listen = "0.0.0.0:31313"
|
||||
metrics_listen = "0.0.0.0:31314"
|
||||
@@ -157,117 +95,35 @@ metrics_listen = "0.0.0.0:31314"
|
||||
strategy = "lru" # lru | priority
|
||||
defrag_after_cycles = 50
|
||||
|
||||
[[neurons]]
|
||||
name = "beast"
|
||||
endpoint = "http://beast.internal:13131"
|
||||
[[nodes]]
|
||||
name = "gpu-large"
|
||||
endpoint = "http://gpu-large.internal:8080"
|
||||
vram_mb = 49_152 # e.g. 2x RTX 4090
|
||||
pinned = ["your-org/large-model"]
|
||||
|
||||
[[neurons]]
|
||||
name = "benjy"
|
||||
endpoint = "http://benjy.internal:13131"
|
||||
[[nodes]]
|
||||
name = "gpu-medium"
|
||||
endpoint = "http://gpu-medium.internal:8080"
|
||||
vram_mb = 24_576 # e.g. RTX 4090
|
||||
pinned = ["your-org/medium-model"]
|
||||
|
||||
[[nodes]]
|
||||
name = "gpu-small"
|
||||
endpoint = "http://gpu-small.internal:8080"
|
||||
vram_mb = 12_288 # e.g. RTX 3060
|
||||
pinned = ["your-org/embedding-model"]
|
||||
```
|
||||
|
||||
Model placement profiles — VRAM requirements, quant, device minimums,
|
||||
which neurons a model may run on, and what it may displace when one runs
|
||||
out of VRAM — live in `models.toml`. `models.example.toml` is the field
|
||||
reference; [placement & displacement](https://helexa.ai/docs/operating/placement)
|
||||
explains how the two fit together, and is worth reading before you set
|
||||
`residency_priority` on anything.
|
||||
|
||||
Full documentation — using helexa and operating it — is at
|
||||
[helexa.ai/docs](https://helexa.ai/docs); the source lives under
|
||||
`helexa.ai/content/docs/`.
|
||||
|
||||
## Run
|
||||
|
||||
```sh
|
||||
# start the gateway
|
||||
cortex serve --config /etc/cortex/cortex.toml
|
||||
|
||||
# check fleet status
|
||||
cortex status
|
||||
|
||||
# one catalogue across every node
|
||||
curl http://localhost:31313/v1/models
|
||||
```
|
||||
|
||||
## Tailoring model behaviour
|
||||
|
||||
System prompts are **application-owned**. Send yours through the standard
|
||||
field for whichever API you speak, and the serving chain — edge → router →
|
||||
cortex → neuron — passes it to the model **verbatim**.
|
||||
|
||||
```sh
|
||||
# OpenAI chat completions
|
||||
curl http://localhost:31313/v1/chat/completions \
|
||||
-H 'content-type: application/json' \
|
||||
-d '{"model": "helexa/balanced",
|
||||
"messages": [{"role": "system", "content": "Reply only in French."},
|
||||
{"role": "user", "content": "Good morning"}]}'
|
||||
|
||||
# OpenAI responses — the `instructions` field is the system slot
|
||||
curl http://localhost:31313/v1/responses \
|
||||
-H 'content-type: application/json' \
|
||||
-d '{"model": "helexa/balanced",
|
||||
"instructions": "Reply only in French.",
|
||||
"input": "Good morning"}'
|
||||
|
||||
# Anthropic messages — top-level `system`, string or content-block array
|
||||
curl http://localhost:31313/v1/messages \
|
||||
-H 'content-type: application/json' \
|
||||
-d '{"model": "helexa/balanced", "max_tokens": 256,
|
||||
"system": "Reply only in French.",
|
||||
"messages": [{"role": "user", "content": "Good morning"}]}'
|
||||
```
|
||||
|
||||
### The passthrough guarantee
|
||||
|
||||
helexa will never, to any request you proxy through it:
|
||||
|
||||
- inject a system prompt, house style or preamble of its own,
|
||||
- rewrite, truncate or reorder the one you sent,
|
||||
- supply a default when you send none.
|
||||
|
||||
If you send no system prompt, the model receives none. Behaviour you did
|
||||
not ask for is a bug — please report it.
|
||||
|
||||
Streaming and non-streaming behave identically, and the guarantee holds on
|
||||
every surface above. Two details worth knowing:
|
||||
|
||||
- **Several system messages** are all forwarded, unmerged and in order.
|
||||
The model sees the last one last, so in practice the last instruction
|
||||
wins. Send one message if you want certainty.
|
||||
- **`/no_think`** in a prompt is a Qwen-family convention the *model*
|
||||
interprets to skip its reasoning block. It is the model's feature, not
|
||||
ours — helexa neither adds nor strips it. Note it currently suppresses
|
||||
reasoning on `/v1/chat/completions` but **not** on `/v1/responses`,
|
||||
where a reasoning model may think regardless (#223); with a small
|
||||
`max_output_tokens` the whole budget can go on reasoning and the reply
|
||||
comes back empty with `status: "incomplete"`. Give Responses requests
|
||||
room (a few hundred tokens) when the model reasons.
|
||||
|
||||
### Why it works this way
|
||||
|
||||
The ecosystem serves an unenumerable diversity of workloads through
|
||||
OpenAI- and Anthropic-compatible APIs. An operator cannot curate prompts
|
||||
for use cases they will never see, and a proxy that quietly edits your
|
||||
payload makes model behaviour impossible to reason about. So the split is:
|
||||
**applications own the prompt, operators own the fleet.**
|
||||
|
||||
If you specifically want centrally-managed prompts across your own
|
||||
workloads, run your own helexa mesh — it is open source, and that is a
|
||||
different deployment from the shared helexa.ai ecosystem.
|
||||
|
||||
Operators: this is a contract, not a default you may flip. cortex and
|
||||
helexa-router proxy inference bodies without adding to them; nothing in
|
||||
the chain is a place to put prompt content.
|
||||
|
||||
## Build from source
|
||||
## Building
|
||||
|
||||
```sh
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
CI runs on every push; keep it green locally:
|
||||
## CI
|
||||
|
||||
Every push triggers format, lint, and test checks. Ensure these pass
|
||||
locally before pushing:
|
||||
|
||||
```sh
|
||||
cargo fmt --check --all # must be clean
|
||||
@@ -275,18 +131,20 @@ cargo clippy --workspace -- -D warnings # warnings are errors
|
||||
cargo test --workspace # all tests must pass
|
||||
```
|
||||
|
||||
Tagged releases (`v*`) build SRPMs for `cortex` and `helexa-neuron`
|
||||
and publish to COPR.
|
||||
Tagged releases (`v*`) additionally build an SRPM and publish to COPR.
|
||||
|
||||
## Status
|
||||
## Running
|
||||
|
||||
Pre-1.0 and moving fast. The gateway path (routing, eviction,
|
||||
translation, metrics) is stable and tested; the candle-native engine
|
||||
is under active development — expect the supported-model list to track
|
||||
the open-weight frontier, deliberately narrowly.
|
||||
```sh
|
||||
# start the gateway
|
||||
cortex serve --config cortex.toml
|
||||
|
||||
Development happens at <https://git.lair.cafe/helexa/helexa>;
|
||||
<https://github.com/helexa-ai/helexa> is a read-only mirror.
|
||||
# check fleet status
|
||||
cortex status
|
||||
|
||||
# list all models across nodes
|
||||
curl http://localhost:31313/v1/models
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
|
||||
@@ -1,92 +0,0 @@
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# MANAGED BY CI — .gitea/workflows/deploy.yml is the only writer.
|
||||
#
|
||||
# Edits made directly to the deployed copy on the host are reverted by
|
||||
# the next deploy, and until then the fleet runs something no commit
|
||||
# describes. Change it here, commit, let the pipeline ship it.
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# helexa-bench config for bob.hanzalova.internal.
|
||||
#
|
||||
# Synced to /etc/helexa-bench/helexa-bench.toml by script/infra-setup.sh
|
||||
# (the helexa-bench RPM ships helexa-bench.example.toml as a
|
||||
# %config(noreplace) default; this per-host file overrides it).
|
||||
#
|
||||
# bob is a client host (it also runs Agent Zero); helexa-bench here hits
|
||||
# every neuron on the fleet directly and records build-stamped results
|
||||
# into the local SQLite store.
|
||||
|
||||
[bench]
|
||||
sweep_interval_secs = 1800
|
||||
samples_per_version = 5
|
||||
iteration_pause_secs = 2
|
||||
request_timeout_secs = 600
|
||||
db_path = "/var/lib/helexa-bench/bench.sqlite"
|
||||
|
||||
# Identity bench presents to neurons (#288). Without it bench is
|
||||
# anonymous, and since #262 anonymous callers are served only from
|
||||
# leftover capacity — capped below max_in_flight and parked at the class
|
||||
# gate. That moved beast's concurrency:8 from 168.9 tok/s / 0.53s
|
||||
# ttft_p95 to 104.6 / 11.77s the day #262 landed and held there, so every
|
||||
# number since measured the yield policy, not serving capacity.
|
||||
#
|
||||
# A dedicated pair, not a borrowed one: bench then holds its own
|
||||
# fair-share allocation (#54) and cannot starve interactive traffic —
|
||||
# the failure #262 existed to fix.
|
||||
[bench.principal]
|
||||
account_id = "helexa-bench"
|
||||
key_id = "fleet-benchmark"
|
||||
|
||||
[scenarios]
|
||||
prompt_sizes = [128, 4096]
|
||||
max_tokens = 256
|
||||
# Concurrency / agentic-load scenarios (#89), enabled so the 27B baseline
|
||||
# carries p95-under-concurrency data before the F3 A/B gate (#94) needs
|
||||
# it for comparison. Levels mirror the real a0/hermes/opencode fan-out.
|
||||
concurrency_levels = [2, 4, 8]
|
||||
# Non-streaming bursts (#285/#288). Kept to the top level: it is where
|
||||
# serialization shows most starkly and each level costs a full burst per
|
||||
# sample per build.
|
||||
concurrency_nonstreaming_levels = [8]
|
||||
concurrency_prompt_tokens = 512
|
||||
|
||||
# Capability probes (#91) — the reasoning/planning axis the speed
|
||||
# scenarios miss; scored manually via `helexa-bench score` (O7). Enabled
|
||||
# for the same reason: the F3 gate compares 80B-A3B variants against the
|
||||
# 27B on planning quality, so the 27B needs scored artifacts first.
|
||||
[[scenarios.capability_probes]]
|
||||
name = "rust-plan"
|
||||
max_tokens = 4096
|
||||
prompt = """
|
||||
Write an implementation plan for adding rate limiting to an Axum service.
|
||||
Honor existing conventions, call out trade-offs, and sequence the work.
|
||||
"""
|
||||
|
||||
[[scenarios.capability_probes]]
|
||||
name = "debug-reason"
|
||||
max_tokens = 4096
|
||||
prompt = """
|
||||
A Rust axum server streams SSE responses through a reverse proxy. Clients
|
||||
report that streams stall for exactly 60 seconds and then resume, but only
|
||||
when response chunks are small and infrequent. Curling the backend directly
|
||||
never stalls. List the most likely causes in order of probability, explain
|
||||
the mechanism behind each, and describe the smallest experiment that would
|
||||
confirm or eliminate each cause.
|
||||
"""
|
||||
|
||||
# Read-only JSON API consumed by the bench UI (hosted separately) and for
|
||||
# programmatic access. Served alongside the sweep loop.
|
||||
[api]
|
||||
enabled = true
|
||||
listen = "0.0.0.0:13132"
|
||||
|
||||
[[targets]]
|
||||
name = "beast"
|
||||
endpoint = "http://beast.hanzalova.internal:13131"
|
||||
|
||||
[[targets]]
|
||||
name = "benjy"
|
||||
endpoint = "http://benjy.hanzalova.internal:13131"
|
||||
|
||||
[[targets]]
|
||||
name = "quadbrat"
|
||||
endpoint = "http://quadbrat.hanzalova.internal:13131"
|
||||
@@ -1,81 +0,0 @@
|
||||
# Fleet monitoring — Prometheus scrape + Grafana dashboard
|
||||
|
||||
Visibility into the cortex `#137` capacity metrics (per-neuron:model load,
|
||||
saturation, tok/s, load-shedding, and per-device GPU health) beyond what the
|
||||
bench UI shows.
|
||||
|
||||
## Topology
|
||||
|
||||
- **cortex** runs on `hanzalova.internal` (10.6.0.46) and is the **only**
|
||||
Prometheus target — it exposes every `cortex_*` metric on `:31314`, already
|
||||
labelled by `{node,model}` / `{node,device}` from its neuron poller. neuron
|
||||
has no `/metrics` endpoint.
|
||||
- **Prometheus + Grafana** run on `golgafrinchans.kosherinata.internal` as
|
||||
podman quadlets. Host ports (from the registered band — 9090 is Cockpit,
|
||||
3000 was avoided): **Prometheus `:26559`** (container config bind-mounted at
|
||||
`/etc/prometheus/prometheus.yml`, `--web.enable-lifecycle` on), **Grafana
|
||||
`:28767`**. Its mesh IP toward hanzalova is `10.3.101.4`.
|
||||
|
||||
## Apply (three steps, in order)
|
||||
|
||||
### 1. Open the metrics port on the cortex host
|
||||
|
||||
cortex binds `0.0.0.0:31314` but firewalld has no rule for it, so a
|
||||
cross-host scrape times out. Open it to the monitoring host **only**:
|
||||
|
||||
```sh
|
||||
# on hanzalova.internal, as root
|
||||
./firewalld-cortex-metrics.sh
|
||||
```
|
||||
|
||||
### 2. Add the scrape job on the monitoring host
|
||||
|
||||
Append `prometheus-cortex.scrape.yml` into the `scrape_configs:` list of
|
||||
`/etc/prometheus/prometheus.yml` on `golgafrinchans.kosherinata.internal`,
|
||||
then hot-reload (no restart, lifecycle API is enabled):
|
||||
|
||||
```sh
|
||||
curl -X POST http://localhost:26559/-/reload
|
||||
# verify the target is UP:
|
||||
curl -s 'http://localhost:26559/api/v1/targets' | jq '.data.activeTargets[]|select(.labels.job=="cortex")|{health,lastError}'
|
||||
```
|
||||
|
||||
### 3. Import the dashboard
|
||||
|
||||
`grafana-helexa-fleet.json` is a raw dashboard model with a `datasource`
|
||||
template variable, so it binds to whichever Prometheus data source you pick
|
||||
at import. Import via the API (creds in `/etc/grafana/grafana.env`):
|
||||
|
||||
```sh
|
||||
# on golgafrinchans; GF_SECURITY_ADMIN_USER/PASSWORD live in the root-only
|
||||
# env file, Grafana is published on :28767
|
||||
set -a; source <(sudo cat /etc/grafana/grafana.env); set +a
|
||||
jq -n --slurpfile d grafana-helexa-fleet.json \
|
||||
'{dashboard: $d[0], overwrite: true, folderUid: null}' \
|
||||
| curl -s -u "$GF_SECURITY_ADMIN_USER:$GF_SECURITY_ADMIN_PASSWORD" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d @- http://localhost:28767/api/dashboards/db | jq '{status,uid,version}'
|
||||
```
|
||||
|
||||
The dashboard lands at uid `helexa-fleet`. Re-importing with `overwrite:true`
|
||||
updates it in place.
|
||||
|
||||
## What the dashboard shows
|
||||
|
||||
| Row | Panels |
|
||||
|---|---|
|
||||
| Capacity & saturation | saturation % (in_flight ÷ max_in_flight), in-flight vs ceiling, queue depth |
|
||||
| Throughput | decode tok/s (revenue capacity), prefill tok/s |
|
||||
| Backpressure & traffic | rejection rate by reason, request & error rate, TTFT p95/p50 |
|
||||
| GPU health | VRAM used per device, GPU utilization %, temperature |
|
||||
|
||||
Templating: pick the Prometheus data source, then filter by `neuron` (node)
|
||||
and `model`. Values are live at the cortex poll cadence (~10 s) scraped every
|
||||
15 s.
|
||||
|
||||
## Note on the empirical knee
|
||||
|
||||
The live gauges above answer "how loaded is it right now". The
|
||||
**sustainable-concurrency knee** (max N before latency/shedding breaks) is a
|
||||
load-test result, not a live gauge — see `helexa-bench report --concurrency`
|
||||
and `GET /api/concurrency` (#137 T3).
|
||||
@@ -1,28 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Open cortex's Prometheus metrics port (:31314) on the cortex gateway host
|
||||
# (hanzalova.internal, 10.6.0.46) to the monitoring host ONLY
|
||||
# (golgafrinchans.kosherinata.internal, which reaches hanzalova as 10.3.101.4).
|
||||
#
|
||||
# cortex binds 0.0.0.0:31314 but firewalld has no rule for it, so a
|
||||
# cross-host scrape times out. This adds a scoped rich rule — the metrics
|
||||
# stay closed to everything except the Prometheus host.
|
||||
#
|
||||
# Run as root on hanzalova.internal. Idempotent.
|
||||
set -euo pipefail
|
||||
|
||||
MONITOR_IP="10.3.101.4" # golgafrinchans.kosherinata.internal (mesh source)
|
||||
PORT="31314"
|
||||
|
||||
rule="rule family=\"ipv4\" source address=\"${MONITOR_IP}/32\" port port=\"${PORT}\" protocol=\"tcp\" accept"
|
||||
|
||||
if firewall-cmd --permanent --query-rich-rule="${rule}" >/dev/null 2>&1; then
|
||||
echo "rich rule already present; nothing to do"
|
||||
else
|
||||
firewall-cmd --permanent --add-rich-rule="${rule}"
|
||||
firewall-cmd --reload
|
||||
echo "opened tcp/${PORT} to ${MONITOR_IP} and reloaded firewalld"
|
||||
fi
|
||||
|
||||
# Verify from this host; the real check is a scrape from the monitoring host:
|
||||
# curl -s -o /dev/null -w '%{http_code}\n' http://hanzalova.internal:31314/metrics
|
||||
firewall-cmd --list-rich-rules | grep "${PORT}" || true
|
||||
@@ -1,841 +0,0 @@
|
||||
{
|
||||
"uid": "helexa-fleet",
|
||||
"title": "helexa fleet \u2014 capacity & throughput",
|
||||
"tags": [
|
||||
"helexa",
|
||||
"cortex",
|
||||
"capacity"
|
||||
],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "15s",
|
||||
"time": {
|
||||
"from": "now-3h",
|
||||
"to": "now"
|
||||
},
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "datasource",
|
||||
"label": "Prometheus",
|
||||
"type": "datasource",
|
||||
"query": "prometheus",
|
||||
"current": {},
|
||||
"hide": 0
|
||||
},
|
||||
{
|
||||
"name": "node",
|
||||
"label": "neuron",
|
||||
"type": "query",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"query": "label_values(cortex_model_max_in_flight, node)",
|
||||
"refresh": 2,
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"allValue": ".*",
|
||||
"current": {
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "model",
|
||||
"label": "model",
|
||||
"type": "query",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"query": "label_values(cortex_model_max_in_flight{node=~\"$node\"}, model)",
|
||||
"refresh": 2,
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"allValue": ".*",
|
||||
"current": {
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "row",
|
||||
"title": "Capacity & saturation",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "timeseries",
|
||||
"title": "Saturation % (in_flight \u00f7 max_in_flight)",
|
||||
"description": "How full each neuron:model is against its admission ceiling. 100% = every slot busy; sustained >100% headroom pressure means it's about to queue/shed.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 1
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"custom": {
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 70
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 90
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "100 * cortex_model_in_flight{node=~\"$node\", model=~\"$model\"} / clamp_min(cortex_model_max_in_flight{node=~\"$node\", model=~\"$model\"}, 1)",
|
||||
"legendFormat": "{{node}} \u00b7 {{model}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "timeseries",
|
||||
"title": "In-flight vs ceiling",
|
||||
"description": "Live concurrent requests (in_flight) against the configured max_in_flight ceiling.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 1
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "none",
|
||||
"min": 0,
|
||||
"custom": {
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never"
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": ".*ceiling.*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "custom.lineStyle",
|
||||
"value": {
|
||||
"dash": [
|
||||
8,
|
||||
6
|
||||
],
|
||||
"fill": "dash"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "custom.fillOpacity",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "cortex_model_in_flight{node=~\"$node\", model=~\"$model\"}",
|
||||
"legendFormat": "{{node}} \u00b7 {{model}}"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "cortex_model_max_in_flight{node=~\"$node\", model=~\"$model\"}",
|
||||
"legendFormat": "{{node}} \u00b7 {{model}} ceiling"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "timeseries",
|
||||
"title": "Queue depth",
|
||||
"description": "Requests waiting in admission beyond the in-flight slots \u2014 the backpressure building before rejection.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 1
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "none",
|
||||
"min": 0,
|
||||
"custom": {
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "cortex_model_queue_depth{node=~\"$node\", model=~\"$model\"}",
|
||||
"legendFormat": "{{node}} \u00b7 {{model}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "row",
|
||||
"title": "Throughput",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 9
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"type": "timeseries",
|
||||
"title": "Decode tok/s (generation throughput)",
|
||||
"description": "Live decode tokens/sec EMA per neuron:model \u2014 the revenue-capacity number. Honest under batch concurrency (folded in at every serving finish site).",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 10
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "none",
|
||||
"min": 0,
|
||||
"custom": {
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never",
|
||||
"axisLabel": "tok/s"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"mean",
|
||||
"max"
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "cortex_model_tok_s_decode{node=~\"$node\", model=~\"$model\"}",
|
||||
"legendFormat": "{{node}} \u00b7 {{model}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"type": "timeseries",
|
||||
"title": "Prefill tok/s (prompt-processing throughput)",
|
||||
"description": "Live prefill tokens/sec EMA per neuron:model.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 10
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "none",
|
||||
"min": 0,
|
||||
"custom": {
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never",
|
||||
"axisLabel": "tok/s"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"mean",
|
||||
"max"
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "cortex_model_tok_s_prefill{node=~\"$node\", model=~\"$model\"}",
|
||||
"legendFormat": "{{node}} \u00b7 {{model}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 20,
|
||||
"type": "row",
|
||||
"title": "Backpressure & traffic",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 18
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 21,
|
||||
"type": "timeseries",
|
||||
"title": "Rejection rate by reason (load shedding)",
|
||||
"description": "rate() of admission rejections. Any sustained non-zero here means the neuron:model is turning work away \u2014 queue_full/wait_timeout = at capacity, per_principal = a client hit its fair-share cap.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 19
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps",
|
||||
"min": 0,
|
||||
"custom": {
|
||||
"fillOpacity": 15,
|
||||
"showPoints": "never",
|
||||
"stacking": {
|
||||
"mode": "normal"
|
||||
}
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"sum"
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "sum by (node, model, reason) (rate(cortex_model_rejections_total{node=~\"$node\", model=~\"$model\"}[5m]))",
|
||||
"legendFormat": "{{node}} \u00b7 {{model}} \u00b7 {{reason}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 22,
|
||||
"type": "timeseries",
|
||||
"title": "Request & error rate",
|
||||
"description": "Proxied request rate and error rate per neuron:model.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 19
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps",
|
||||
"min": 0,
|
||||
"custom": {
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never"
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": ".*error.*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "red"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "sum by (node, model) (rate(cortex_requests_total{node=~\"$node\", model=~\"$model\"}[5m]))",
|
||||
"legendFormat": "{{node}} \u00b7 {{model}}"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "sum by (node, model) (rate(cortex_request_errors_total{node=~\"$node\", model=~\"$model\"}[5m]))",
|
||||
"legendFormat": "{{node}} \u00b7 {{model}} error"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 23,
|
||||
"type": "timeseries",
|
||||
"title": "TTFT p95 / p50",
|
||||
"description": "Time-to-first-token latency percentiles from the request histogram.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 19
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"min": 0,
|
||||
"custom": {
|
||||
"fillOpacity": 0,
|
||||
"showPoints": "never"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "histogram_quantile(0.95, sum by (le, node, model) (rate(cortex_time_to_first_token_seconds_bucket{node=~\"$node\", model=~\"$model\"}[5m])))",
|
||||
"legendFormat": "p95 {{node}} \u00b7 {{model}}"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "histogram_quantile(0.50, sum by (le, node, model) (rate(cortex_time_to_first_token_seconds_bucket{node=~\"$node\", model=~\"$model\"}[5m])))",
|
||||
"legendFormat": "p50 {{node}} \u00b7 {{model}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 30,
|
||||
"type": "row",
|
||||
"title": "GPU health",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 27
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 31,
|
||||
"type": "timeseries",
|
||||
"title": "VRAM used per device",
|
||||
"description": "Per-device VRAM used (MB). Pair with free to see headroom for loading bigger models.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 28
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "decmbytes",
|
||||
"min": 0,
|
||||
"custom": {
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "cortex_device_vram_used_mb{node=~\"$node\"}",
|
||||
"legendFormat": "{{node}} \u00b7 gpu{{device}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 32,
|
||||
"type": "timeseries",
|
||||
"title": "GPU utilization %",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 28
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"custom": {
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"mean"
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "cortex_device_utilization_pct{node=~\"$node\"}",
|
||||
"legendFormat": "{{node}} \u00b7 gpu{{device}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 33,
|
||||
"type": "timeseries",
|
||||
"title": "GPU temperature \u00b0C",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 28
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "celsius",
|
||||
"min": 0,
|
||||
"custom": {
|
||||
"fillOpacity": 0,
|
||||
"showPoints": "never"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 75
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 87
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "cortex_device_temp_c{node=~\"$node\"}",
|
||||
"legendFormat": "{{node}} \u00b7 gpu{{device}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 34,
|
||||
"type": "row",
|
||||
"title": "Text-to-image (#203)",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 36
|
||||
},
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 35,
|
||||
"type": "timeseries",
|
||||
"title": "Images generated / hour",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 37
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (model) (increase(cortex_images_generated_total[1h]))",
|
||||
"legendFormat": "{{model}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 36,
|
||||
"type": "timeseries",
|
||||
"title": "Generation latency (p50/p95)",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 37
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(cortex_images_generation_seconds_bucket[10m])))",
|
||||
"legendFormat": "p95",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, sum by (le) (rate(cortex_images_generation_seconds_bucket[10m])))",
|
||||
"legendFormat": "p50",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 37,
|
||||
"type": "timeseries",
|
||||
"title": "Image spend (Mp-steps / hour by account)",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 37
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (account) (increase(cortex_spend_image_milliunits_total[1h])) / 1000",
|
||||
"legendFormat": "{{account}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {},
|
||||
"overrides": []
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
# helexa cortex gateway scrape job — append into the `scrape_configs:` list
|
||||
# of /etc/prometheus/prometheus.yml on golgafrinchans.kosherinata.internal,
|
||||
# then reload (Prometheus is published on host port 26559, NOT 9090 — 9090 is
|
||||
# Cockpit on that box): curl -X POST http://localhost:26559/-/reload
|
||||
#
|
||||
# cortex runs on hanzalova.internal (10.6.0.46) and exposes ALL fleet metrics
|
||||
# on :31314 — it is the only Prometheus target in the stack (neuron has no
|
||||
# /metrics; cortex's poller already relabels everything by {node,model} /
|
||||
# {node,device}). Reaching it cross-DC (kosherinata -> hanzalova) requires the
|
||||
# firewalld rule in asset/monitoring/firewalld-cortex-metrics.sh, which opens
|
||||
# :31314 to this host's mesh IP (10.3.101.4) only.
|
||||
#
|
||||
# Metrics exposed here include (see #137):
|
||||
# cortex_model_in_flight / _queue_depth / _max_in_flight / _max_queue_depth
|
||||
# cortex_model_tok_s_decode / _tok_s_prefill
|
||||
# cortex_model_rejections_total{reason=queue_full|wait_timeout|per_principal}
|
||||
# cortex_device_vram_used_mb / _vram_free_mb / _utilization_pct / _temp_c
|
||||
# cortex_requests_total / _request_errors_total / _cold_starts_total
|
||||
# cortex_request_duration_seconds / _time_to_first_token_seconds (histograms)
|
||||
# cortex_prompt_tokens_total / _completion_tokens_total / cortex_spend_*
|
||||
- job_name: cortex
|
||||
# cortex refreshes the load/device gauges from each neuron's /health on a
|
||||
# ~10s poll; 15s scrape (the global default) tracks it without aliasing.
|
||||
static_configs:
|
||||
- targets: ['hanzalova.internal:31314']
|
||||
labels:
|
||||
fleet: hanzalova
|
||||
@@ -1,121 +0,0 @@
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# MANAGED BY CI — .gitea/workflows/deploy.yml is the only writer.
|
||||
#
|
||||
# Edits made directly to the deployed copy on the host are reverted by
|
||||
# the next deploy, and until then the fleet runs something no commit
|
||||
# describes. Change it here, commit, let the pipeline ship it.
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# neuron.toml for beast.hanzalova.internal
|
||||
#
|
||||
# 2x RTX 5090 (32 GB each) — TP-2 capable. Pre-warms Qwen3.8-27B with
|
||||
# q6k ISQ across both GPUs at activation, matching the validate-neuron
|
||||
# invocation: `validate-neuron.sh beast.hanzalova.internal
|
||||
# Qwen/Qwen3.8-27B q6k 2`.
|
||||
#
|
||||
# Synced to /etc/neuron/neuron.toml by script/infra-setup.sh. Edits
|
||||
# take effect after the next deploy workflow run restarts the service
|
||||
# (default_models is read at activation).
|
||||
|
||||
port = 13131
|
||||
|
||||
[[harnesses]]
|
||||
name = "candle"
|
||||
|
||||
[harness.candle]
|
||||
|
||||
# Batched decode engine (#98): up to 8 concurrent text streams
|
||||
# multiplex through one lockstep (B,1) forward per decode step.
|
||||
# max_in_flight is both the admission bound and the engine's slot
|
||||
# count. NEURON_BATCHING=0 (systemd drop-in) is the kill switch;
|
||||
# removing this section reverts to batch-1 serialization.
|
||||
[harness.candle.admission]
|
||||
# 8. max_in_flight bounds how many requests run concurrently. It does
|
||||
# NOT partition the KV budget: since #257 (`eca42510`, 2026-08-15) KV is
|
||||
# a shared pool and each request reserves what its own prompt needs via
|
||||
# `enter_with_kv(principal, kv_reservation_mb(prompt_len, ...))`. A
|
||||
# request too big for what is free waits for bytes to come back rather
|
||||
# than being handed a fixed 1/N slice.
|
||||
#
|
||||
# This value was briefly set to 2 (#291) on the stated grounds that "the
|
||||
# KV budget is divided by it at load, so 8 gives each slot 18,852
|
||||
# tokens". That was true before #257 and false after it, and #291 was
|
||||
# filed and actioned after that landed. Nothing in the tree divides the
|
||||
# budget by the slot count — `kv_budget` is a semaphore of MiB permits
|
||||
# and the engine's `max_slots` is only the batch width.
|
||||
#
|
||||
# The concurrency measurement in #291 stands as data (89.6% of busy time
|
||||
# at one request in flight, 4.9% utilisation over seven days) but the
|
||||
# conclusion drawn from it does not, and the traffic it described has
|
||||
# changed: while it was taken, the router's `helexa/balanced` alias
|
||||
# pointed web chat at a second 27B, so per-model concurrency was split
|
||||
# across two models that were evicting each other. With both tiers now
|
||||
# resolving to this model, chat concurrency lands here.
|
||||
#
|
||||
# What actually bounds a busy moment is the pool: 3,689 MiB at
|
||||
# 32 KiB/token/card is ~118k tokens of KV to share. Eight ordinary chat
|
||||
# turns fit easily; a few very long agentic sessions will queue on bytes,
|
||||
# which is the correct behaviour and is what #257 built.
|
||||
max_in_flight = 8
|
||||
|
||||
# Snapshot budget for the cross-request prefix cache (#292).
|
||||
#
|
||||
# 3 GiB, not the 1 GiB default. At the measured 32 KiB/token/card, a
|
||||
# 1 GiB budget cannot hold a snapshot beyond ~32k tokens of K/V alone —
|
||||
# less once the GatedDeltaNet conv/recurrent state is deep-copied on top,
|
||||
# which on this arch is 48 of 64 layers in f32. Agentic sessions here run
|
||||
# well past that: `cached_tokens` was observed pinned at 2,016 for ten
|
||||
# consecutive turns while the prompt grew to 52,869, so every turn
|
||||
# re-prefilled ~50k tokens — 37 s each, against ~20 s of actual decode.
|
||||
# The cache was not missing; the snapshot would not fit.
|
||||
#
|
||||
# The snapshot budget is reserved BEFORE the KV budget, not from what is
|
||||
# left over: measured on this host, kv_budget_mb = 5737 - budget_mb,
|
||||
# exactly 1:1. So this number is a direct trade against how long a
|
||||
# single session can get, and both sides have a floor:
|
||||
#
|
||||
# budget_mb kv_budget tokens/slot (2 slots) holds a 52.9k snapshot
|
||||
# 1024 4713 75,400 no ← the #292 symptom
|
||||
# 3072 2665 42,600 yes, but sessions
|
||||
# cannot reach 52.9k
|
||||
# 2048 3689 59,000 yes
|
||||
#
|
||||
# 2 GiB is the value that satisfies both: a 52,869-token snapshot is
|
||||
# ~1,652 MB of K/V plus the GatedDeltaNet state (~1.73 GB total), and
|
||||
# per-slot KV still covers sessions comfortably past the deepest
|
||||
# observed. 3 GiB was tried first and starved KV — a session could no
|
||||
# longer reach the size whose snapshot the cache was being enlarged to
|
||||
# hold, which is self-defeating.
|
||||
#
|
||||
# max_entries stays at the default 8: the byte budget binds first here,
|
||||
# so the count cap is not what is forcing snapshots to be small.
|
||||
[harness.candle.prefix_cache]
|
||||
budget_mb = 2048
|
||||
|
||||
[[default_models]]
|
||||
# Qwen3.8-27B is the resident flagship (#251). It replaced the
|
||||
# 3.6-27B — architecturally identical, same model_type, layer counts and
|
||||
# vision tower — so the load spec is unchanged.
|
||||
#
|
||||
# The 3.6-27B is no longer catalogued (retired 2026-08-27). While both
|
||||
# were listed at the same residency class they could displace each other
|
||||
# in either direction, which is what let a stale `helexa/balanced` alias
|
||||
# evict this model on every authenticated web-chat turn. See models.toml
|
||||
# for the incident that ended the arrangement.
|
||||
model_id = "Qwen/Qwen3.8-27B"
|
||||
harness = "candle"
|
||||
quant = "q6k"
|
||||
tensor_parallel = 2
|
||||
devices = [0, 1]
|
||||
|
||||
# Sampling override (#283) — MUST match the `[models.sampling]` block on
|
||||
# this model in models.toml. Two files describe one model; only
|
||||
# script/check-config-consistency.py forces them to agree, and #252 is
|
||||
# what happens when nothing does.
|
||||
#
|
||||
# The model's own generation_config.json says temperature = 1.0, which
|
||||
# measured a 20% structural-defect rate on ~2k-token code generation
|
||||
# against 0/60 at <= 0.6 (Fisher exact one-sided p = 0.0031). Precedence
|
||||
# is request > operator > model, so a caller naming a temperature still
|
||||
# gets what it asked for.
|
||||
[default_models.sampling]
|
||||
temperature = 0.6
|
||||
@@ -1,48 +0,0 @@
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# MANAGED BY CI — .gitea/workflows/deploy.yml is the only writer.
|
||||
#
|
||||
# Edits made directly to the deployed copy on the host are reverted by
|
||||
# the next deploy, and until then the fleet runs something no commit
|
||||
# describes. Change it here, commit, let the pipeline ship it.
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# neuron.toml for benjy.hanzalova.internal
|
||||
#
|
||||
# 1x RTX 4090 (24 GB) — largest single-GPU host on the fleet. Pre-warms
|
||||
# Qwen3-8B (bf16, ~18 GB), leaving ~6 GB for KV cache + activations on
|
||||
# moderate-length contexts.
|
||||
#
|
||||
# Synced to /etc/neuron/neuron.toml by script/infra-setup.sh.
|
||||
|
||||
port = 13131
|
||||
|
||||
[[harnesses]]
|
||||
name = "candle"
|
||||
|
||||
[harness.candle]
|
||||
|
||||
[[default_models]]
|
||||
model_id = "Qwen/Qwen3-8B"
|
||||
harness = "candle"
|
||||
devices = [0]
|
||||
|
||||
# Admission (#53): Qwen3-8B is the classic qwen3 arch, which has no KV
|
||||
# snapshot support — forwards serialize on the inference lock, so the
|
||||
# batch engine (and any real max_in_flight > 1) doesn't apply here; the
|
||||
# 24 GB card is not the limiter. Capacity for bursts comes from the
|
||||
# QUEUE: ~5-8 s per chat reply means a full queue drains in under a
|
||||
# minute, and waiters beyond max_wait get an honest 429 instead of an
|
||||
# invisible lock wait. Revisit with prom traffic data; the real unlock
|
||||
# is porting KV snapshots to the qwen3 dense arch.
|
||||
[harness.candle.admission]
|
||||
max_in_flight = 1
|
||||
max_queue_depth = 24
|
||||
max_wait_secs = 60
|
||||
|
||||
# Text-to-image (#197/#203). Z-Image-Turbo cold-swaps against Qwen3-8B
|
||||
# on the 4090 (operator decision 2026-07-29): cortex evicts the 8B on
|
||||
# an images request and the 8B cold-loads back on demand. The Qwen3-4B
|
||||
# text encoder runs on the host CPU (the default) — GPU TE + resident
|
||||
# DiT OOMs 24 GB — and stays resident in system RAM.
|
||||
[harness.candle.image]
|
||||
te_device = "cpu"
|
||||
max_dim = 2048
|
||||
@@ -1,35 +0,0 @@
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# MANAGED BY CI — .gitea/workflows/deploy.yml is the only writer.
|
||||
#
|
||||
# Edits made directly to the deployed copy on the host are reverted by
|
||||
# the next deploy, and until then the fleet runs something no commit
|
||||
# describes. Change it here, commit, let the pipeline ship it.
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# neuron.toml for quadbrat.hanzalova.internal
|
||||
#
|
||||
# 1x RTX 3060 (12 GB) — small / quantised tier. Pre-warms Qwen3-1.7B
|
||||
# (bf16, ~4 GB), leaving ~7 GB for KV cache so long contexts on a small
|
||||
# model still have plenty of room.
|
||||
#
|
||||
# Synced to /etc/neuron/neuron.toml by script/infra-setup.sh.
|
||||
|
||||
port = 13131
|
||||
|
||||
[[harnesses]]
|
||||
name = "candle"
|
||||
|
||||
[harness.candle]
|
||||
|
||||
[[default_models]]
|
||||
model_id = "Qwen/Qwen3-1.7B"
|
||||
harness = "candle"
|
||||
devices = [0]
|
||||
|
||||
# Text-to-image (#204): the 12 GB tier serves the q8_0-quantized DiT
|
||||
# (8.7 GB resident incl. VAE). Ceiling is 768² — naive f32 attention
|
||||
# transients OOM 1024² beside the resident model; lifts to 1024² if
|
||||
# #95 extends flash-attn to the ampere flavour. Load spec: quant =
|
||||
# "q8_0" on the load request / catalogue.
|
||||
[harness.candle.image]
|
||||
te_device = "cpu"
|
||||
max_dim = 768
|
||||
@@ -1,16 +0,0 @@
|
||||
# Shared proxy parameters for the angels.helexa.ai vhost.
|
||||
#
|
||||
# Installed to /etc/nginx/conf.d/angels-proxy-params.conf and included by
|
||||
# each location block, so the four of them cannot drift apart.
|
||||
#
|
||||
# X-Forwarded-For is load-bearing rather than decorative here: the access
|
||||
# record stores the client IP, and without this every row would read as
|
||||
# the edge proxy's address.
|
||||
|
||||
proxy_pass_header Server;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_read_timeout 60s;
|
||||
@@ -1,19 +0,0 @@
|
||||
# Bootstrap vhost for angels.helexa.ai — http-only, used ONLY to obtain
|
||||
# the initial Let's Encrypt certificate (the TLS vhost cannot load before
|
||||
# the cert file exists). script/infra-setup.sh installs this, runs
|
||||
# certbot, then swaps in angels.helexa.ai.conf.
|
||||
#
|
||||
# Serves nothing but the ACME challenge: the portal itself must never be
|
||||
# reachable over plaintext, even for the minute this file is in place.
|
||||
server {
|
||||
listen 80;
|
||||
listen [::]:80;
|
||||
server_name angels.helexa.ai;
|
||||
|
||||
location /.well-known/acme-challenge/ {
|
||||
root /var/www/angels.helexa.ai;
|
||||
}
|
||||
location / {
|
||||
return 404;
|
||||
}
|
||||
}
|
||||
@@ -1,106 +0,0 @@
|
||||
# Confidential investor portal at https://angels.helexa.ai.
|
||||
#
|
||||
# Unlike helexa.ai and bench.helexa.ai there is NO static root here and
|
||||
# nothing is served from disk: every byte comes from helexa-angels on
|
||||
# gallumbits, which assembles each page server-side after checking the
|
||||
# session. That is the whole reason the portal is a separate service — a
|
||||
# static bundle cannot keep a secret, and a client-side route guard gates
|
||||
# navigation rather than access.
|
||||
#
|
||||
# Installed on BOTH edge proxies (oolon.kosherinata.internal and
|
||||
# hanzalova.internal), matching helexa.ai, so the Cloudflare A records
|
||||
# can point at either site.
|
||||
#
|
||||
# The real client address arrives via PROXY protocol from the SNI router
|
||||
# and is recovered by conf.d/proxy-protocol.conf's real_ip directives —
|
||||
# which is what makes $proxy_add_x_forwarded_for, and therefore the IP in
|
||||
# every access record, the visitor's rather than the router's.
|
||||
#
|
||||
# TLS via Let's Encrypt (certbot, Cloudflare DNS-01, ECDSA) —
|
||||
# bootstrapped one-time in script/infra-setup.sh.
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
listen [::]:80;
|
||||
server_name angels.helexa.ai;
|
||||
|
||||
location /.well-known/acme-challenge/ {
|
||||
root /var/www/angels.helexa.ai;
|
||||
}
|
||||
location / {
|
||||
return 301 https://$host$request_uri;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
# TCP 443 on the edge is owned by the stream SNI router
|
||||
# (streams-enabled/sni-router.conf, architecture reverse-proxies.md
|
||||
# §5), which preads the SNI and hands every local name to this http
|
||||
# tier over PROXY protocol. Listening on :443 directly here would
|
||||
# never be reached — the router would answer first with whichever
|
||||
# vhost happened to be default, which is exactly the symptom seen
|
||||
# when this file was first written against the older pattern.
|
||||
listen 127.0.0.1:14443 ssl proxy_protocol;
|
||||
http2 on;
|
||||
server_name angels.helexa.ai;
|
||||
|
||||
ssl_certificate /etc/letsencrypt/live/angels.helexa.ai/fullchain.pem;
|
||||
ssl_certificate_key /etc/letsencrypt/live/angels.helexa.ai/privkey.pem;
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
ssl_ciphers HIGH:!aNULL:!MD5;
|
||||
ssl_prefer_server_ciphers on;
|
||||
ssl_session_cache shared:SSL:10m;
|
||||
|
||||
# ── Confidentiality headers ─────────────────────────────────────
|
||||
# Also emitted by the application in each page's <head>. Both, on
|
||||
# purpose: either alone is a single point of failure, and the cost of
|
||||
# duplication is nil.
|
||||
#
|
||||
# `always` matters — without it these are dropped on error responses,
|
||||
# which is exactly when a crawler is most likely to be looking.
|
||||
add_header X-Robots-Tag "noindex, nofollow, noarchive, nosnippet" always;
|
||||
add_header Referrer-Policy "no-referrer" always;
|
||||
add_header X-Content-Type-Options "nosniff" always;
|
||||
add_header X-Frame-Options "DENY" always;
|
||||
# The portal loads nothing from anywhere else — no CDN, no fonts, no
|
||||
# analytics — so the policy can be this tight, and the privacy note
|
||||
# says as much in terms a visitor can verify in their network tab.
|
||||
add_header Content-Security-Policy
|
||||
"default-src 'none'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; form-action 'self'; base-uri 'none'; frame-ancestors 'none'" always;
|
||||
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
|
||||
|
||||
# Keep renewals working.
|
||||
location /.well-known/acme-challenge/ {
|
||||
root /var/www/angels.helexa.ai;
|
||||
}
|
||||
|
||||
# ── Rate limits ─────────────────────────────────────────────────
|
||||
# Zones are declared in conf.d/helexa-ratelimit.conf.
|
||||
#
|
||||
# The invite path is the one an attacker would grind: codes are 32
|
||||
# base62 characters, so guessing is not a realistic threat, but there
|
||||
# is no reason to serve the attempts. Sign-in is limited because it
|
||||
# is the only endpoint that costs real CPU (argon2, deliberately).
|
||||
location /i/ {
|
||||
limit_req zone=angels_invite burst=5 nodelay;
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8092;
|
||||
include /etc/nginx/conf.d/angels-proxy-params.conf;
|
||||
}
|
||||
|
||||
location /signin {
|
||||
limit_req zone=angels_auth burst=5 nodelay;
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8092;
|
||||
include /etc/nginx/conf.d/angels-proxy-params.conf;
|
||||
}
|
||||
|
||||
location /register {
|
||||
limit_req zone=angels_auth burst=5 nodelay;
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8092;
|
||||
include /etc/nginx/conf.d/angels-proxy-params.conf;
|
||||
}
|
||||
|
||||
location / {
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8092;
|
||||
include /etc/nginx/conf.d/angels-proxy-params.conf;
|
||||
}
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
# Internal-only vhost for the investor portal (helexa-angels).
|
||||
#
|
||||
# Useful where the public name does not resolve or connect correctly from
|
||||
# inside the network that hosts it — common without NAT reflection or
|
||||
# split-horizon DNS. Optional: skip this file if your public name works
|
||||
# from everywhere.
|
||||
#
|
||||
# Cert: internal CA, renewed by step@angels.timer.
|
||||
#
|
||||
# Enable with a relative symlink into sites-enabled:
|
||||
# ln -sf ../sites-available/angels.internal.conf /etc/nginx/sites-enabled/
|
||||
|
||||
upstream angels_up {
|
||||
server gallumbits.kosherinata.internal:8092;
|
||||
keepalive 16;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 127.0.0.1:14443 ssl proxy_protocol;
|
||||
http2 on;
|
||||
server_name angels.internal;
|
||||
|
||||
ssl_certificate /etc/nginx/tls/cert/angels.internal.pem;
|
||||
ssl_certificate_key /etc/nginx/tls/key/angels.internal.pem;
|
||||
ssl_protocols TLSv1.3;
|
||||
ssl_trusted_certificate /etc/pki/ca-trust/source/anchors/root-internal.pem;
|
||||
|
||||
# The material is confidential whichever name reaches it, so the same
|
||||
# headers as the public vhost. `always` so they survive error responses.
|
||||
add_header X-Robots-Tag "noindex, nofollow, noarchive, nosnippet" always;
|
||||
add_header Referrer-Policy "no-referrer" always;
|
||||
add_header X-Content-Type-Options "nosniff" always;
|
||||
add_header X-Frame-Options "DENY" always;
|
||||
add_header Content-Security-Policy
|
||||
"default-src 'none'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; form-action 'self'; base-uri 'none'; frame-ancestors 'none'" always;
|
||||
|
||||
# Same per-IP limits as the public path: an invite code is equally
|
||||
# guessable from here, and argon2 costs the same CPU whoever spends it.
|
||||
location /i/ {
|
||||
limit_req zone=angels_invite burst=5 nodelay;
|
||||
proxy_pass http://angels_up;
|
||||
include /etc/nginx/conf.d/angels-proxy-params.conf;
|
||||
}
|
||||
location /signin {
|
||||
limit_req zone=angels_auth burst=5 nodelay;
|
||||
proxy_pass http://angels_up;
|
||||
include /etc/nginx/conf.d/angels-proxy-params.conf;
|
||||
}
|
||||
location /register {
|
||||
limit_req zone=angels_auth burst=5 nodelay;
|
||||
proxy_pass http://angels_up;
|
||||
include /etc/nginx/conf.d/angels-proxy-params.conf;
|
||||
}
|
||||
location / {
|
||||
proxy_pass http://angels_up;
|
||||
include /etc/nginx/conf.d/angels-proxy-params.conf;
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
# Bootstrap vhost for bench.helexa.ai — http-only, used ONLY to obtain
|
||||
# the initial Let's Encrypt cert via the webroot challenge (the full TLS
|
||||
# vhost can't load before the cert file exists). script/infra-setup.sh
|
||||
# installs this, runs certbot, then swaps in bench.helexa.ai.conf.
|
||||
server {
|
||||
listen 80;
|
||||
server_name bench.helexa.ai;
|
||||
|
||||
location /.well-known/acme-challenge/ {
|
||||
root /var/www/bench.helexa.ai;
|
||||
}
|
||||
location / {
|
||||
try_files $uri $uri/ =404;
|
||||
}
|
||||
}
|
||||
@@ -1,70 +0,0 @@
|
||||
# Public, auth-less bench UI at https://bench.helexa.ai.
|
||||
#
|
||||
# Serves the static SPA from /var/www/bench.helexa.ai (rsynced by
|
||||
# .gitea/workflows/deploy.yml's deploy-bench-ui job) and reverse-proxies
|
||||
# /api to the helexa-bench read API on bob over the WireGuard mesh — so
|
||||
# the browser stays same-origin (no CORS) and the internal API never
|
||||
# needs to be exposed publicly.
|
||||
#
|
||||
# TLS via Let's Encrypt; the cert is obtained/renewed by certbot
|
||||
# (bootstrapped one-time in script/infra-setup.sh). Mirrors the
|
||||
# dev.swym.hanzalova.internal vhost convention on this host.
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name bench.helexa.ai;
|
||||
|
||||
# Keep serving the ACME webroot so certbot can renew.
|
||||
location /.well-known/acme-challenge/ {
|
||||
root /var/www/bench.helexa.ai;
|
||||
}
|
||||
location / {
|
||||
return 301 https://$host$request_uri;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
# TCP 443 on this host is owned by the stream SNI router
|
||||
# (streams-enabled/sni-router.conf), which preads the server name and
|
||||
# forwards to the local https tier on 127.0.0.1:14443 over PROXY
|
||||
# protocol. A vhost that listens on :443 directly is never reached —
|
||||
# the router answers first, with whichever certificate its default
|
||||
# branch holds. `nginx -t` passes either way, so the mistake shows up
|
||||
# only as the wrong certificate on a working handshake.
|
||||
#
|
||||
# Prerequisites on the host, both owned by the edge/mail
|
||||
# infrastructure rather than this repo (and host-specific — the two
|
||||
# edges route different mail names, so neither is synced from here):
|
||||
# streams-enabled/sni-router.conf — owns :443, preads SNI
|
||||
# conf.d/proxy-protocol.conf — real_ip, so $binary_remote_addr
|
||||
# and X-Real-IP stay truthful
|
||||
listen 127.0.0.1:14443 ssl proxy_protocol;
|
||||
http2 on;
|
||||
server_name bench.helexa.ai;
|
||||
|
||||
ssl_certificate /etc/letsencrypt/live/bench.helexa.ai/fullchain.pem;
|
||||
ssl_certificate_key /etc/letsencrypt/live/bench.helexa.ai/privkey.pem;
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
ssl_ciphers HIGH:!aNULL:!MD5;
|
||||
ssl_prefer_server_ciphers on;
|
||||
ssl_session_cache shared:SSL:10m;
|
||||
|
||||
root /var/www/bench.helexa.ai;
|
||||
index index.html;
|
||||
|
||||
# Bench read API on bob (internal WireGuard); browser stays same-origin.
|
||||
location /api/ {
|
||||
proxy_pass http://bob.hanzalova.internal:13132;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_read_timeout 60s;
|
||||
}
|
||||
|
||||
# SPA fallback — client-side routes (/trends, /runs) resolve to index.html.
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
# Internal bench UI vhost — https://bench.internal, reachable from inside
|
||||
# the WireGuard mesh (the public bench.helexa.ai dead-ends at the OPNsense
|
||||
# LAN interface, which only port-forwards :443 from the WAN). Same SPA +
|
||||
# /api→bob proxy as bench.helexa.ai, but with an internal-CA cert
|
||||
# (smallstep "lair", renewed by step@bench.timer). Mirrors the
|
||||
# *.internal vhost convention on oolon.kosherinata.internal.
|
||||
server {
|
||||
server_name bench.internal;
|
||||
# TCP 443 on this host is owned by the stream SNI router
|
||||
# (streams-enabled/sni-router.conf), which preads the server name and
|
||||
# forwards to the local https tier on 127.0.0.1:14443 over PROXY
|
||||
# protocol. A vhost that listens on :443 directly is never reached —
|
||||
# the router answers first, with whichever certificate its default
|
||||
# branch holds. `nginx -t` passes either way, so the mistake shows up
|
||||
# only as the wrong certificate on a working handshake.
|
||||
#
|
||||
# Prerequisites on the host, both owned by the edge/mail
|
||||
# infrastructure rather than this repo (and host-specific — the two
|
||||
# edges route different mail names, so neither is synced from here):
|
||||
# streams-enabled/sni-router.conf — owns :443, preads SNI
|
||||
# conf.d/proxy-protocol.conf — real_ip, so $binary_remote_addr
|
||||
# and X-Real-IP stay truthful
|
||||
listen 127.0.0.1:14443 ssl proxy_protocol;
|
||||
http2 on;
|
||||
|
||||
ssl_certificate /etc/nginx/tls/cert/bench.internal.pem;
|
||||
ssl_certificate_key /etc/nginx/tls/key/bench.internal.pem;
|
||||
ssl_trusted_certificate /etc/pki/ca-trust/source/anchors/root-internal.pem;
|
||||
ssl_protocols TLSv1.3;
|
||||
|
||||
# Shared webroot with the public vhost — same built SPA.
|
||||
root /var/www/bench.helexa.ai;
|
||||
index index.html;
|
||||
|
||||
location /api/ {
|
||||
proxy_pass http://bob.hanzalova.internal:13132;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_read_timeout 60s;
|
||||
}
|
||||
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
# Per-client-IP request-rate zones for the helexa.ai vhosts, installed to
|
||||
# /etc/nginx/conf.d/ (http context) by script/infra-setup.sh.
|
||||
#
|
||||
# The anonymous inference path deliberately carries no client identifier
|
||||
# (see /privacy) — per-IP limiting at the edge is the compliant abuse
|
||||
# lever. Authenticated users share the same per-IP ceiling; generous
|
||||
# enough for interactive chat, tight enough to stop a loop. Budgets and
|
||||
# per-key fair-share are enforced deeper in the chain (upstream + cortex).
|
||||
limit_req_zone $binary_remote_addr zone=helexa_v1:10m rate=10r/m;
|
||||
limit_req_zone $binary_remote_addr zone=helexa_api:10m rate=60r/m;
|
||||
# Tool calls (#177): an agentic turn can issue a few searches back to
|
||||
# back, so this runs looser than helexa_v1 — but each search rides an
|
||||
# inference round-trip that IS helexa_v1-limited, so the pair can't
|
||||
# runaway-loop.
|
||||
limit_req_zone $binary_remote_addr zone=helexa_tools:10m rate=30r/m;
|
||||
# angels.helexa.ai (the confidential investor portal). Both are
|
||||
# deliberately tight: the audience is a handful of named people, so
|
||||
# anything resembling volume is either an attack or a bug.
|
||||
# angels_invite — the /i/<code> entry path. Codes are 32 base62
|
||||
# characters so guessing is not a realistic threat, but there is no
|
||||
# reason to serve the attempts.
|
||||
# angels_auth — sign-in and registration. The only endpoints that cost
|
||||
# real CPU, since argon2 is expensive on purpose.
|
||||
limit_req_zone $binary_remote_addr zone=angels_invite:1m rate=20r/m;
|
||||
limit_req_zone $binary_remote_addr zone=angels_auth:1m rate=12r/m;
|
||||
limit_req_status 429;
|
||||
|
||||
# Access-log format for the search locations, which must not record what
|
||||
# people searched for.
|
||||
#
|
||||
# The default `main` format logs `$request` — method, full URI *and
|
||||
# query string*. For a search proxy that means every query a user types
|
||||
# is written to /var/log/nginx/access.log next to their IP, for
|
||||
# authenticated and anonymous visitors alike. SearXNG's own web UI
|
||||
# defaults to POST partly to avoid exactly this.
|
||||
#
|
||||
# `$uri` is the path with the query stripped, so the operational signal
|
||||
# survives — who called, when, which endpoint, what status, how big, how
|
||||
# long — and the query itself does not. `$request_method` is kept
|
||||
# explicit since `$uri` drops it.
|
||||
#
|
||||
# Rate-limit debugging and abuse detection both work from this: they
|
||||
# need counts, statuses and client addresses, never the search terms.
|
||||
log_format helexa_noquery '$remote_addr - $remote_user [$time_local] '
|
||||
'"$request_method $uri $server_protocol" '
|
||||
'$status $body_bytes_sent "$http_referer" '
|
||||
'"$http_user_agent" $request_time';
|
||||
@@ -1,17 +0,0 @@
|
||||
# Bootstrap vhost for helexa.ai — http-only, used ONLY until the initial
|
||||
# Let's Encrypt cert exists (the full TLS vhost can't load before the
|
||||
# cert file does — nginx -t fails on a missing ssl_certificate).
|
||||
# script/infra-setup.sh installs this, obtains the cert via Cloudflare
|
||||
# DNS-01, then swaps in helexa.ai.conf.
|
||||
server {
|
||||
listen 80;
|
||||
listen [::]:80;
|
||||
server_name helexa.ai;
|
||||
|
||||
location /.well-known/acme-challenge/ {
|
||||
root /var/www/helexa.ai;
|
||||
}
|
||||
location / {
|
||||
try_files $uri $uri/ =404;
|
||||
}
|
||||
}
|
||||
@@ -1,132 +0,0 @@
|
||||
# Public helexa.ai website — the built SPA (helexa.ai/dist), rsynced to
|
||||
# /var/www/helexa.ai by .gitea/workflows/deploy.yml's deploy-website job.
|
||||
#
|
||||
# Installed on BOTH edge proxies (oolon.kosherinata.internal and
|
||||
# hanzalova.internal); Cloudflare DNS load balancing routes the public
|
||||
# name to the two site WAN IPs. Mesh clients use helexa.internal instead
|
||||
# (public names don't hairpin — see architecture/reverse-proxies.md §2).
|
||||
#
|
||||
# Same-origin backends (#167): /v1 (+ /health) reverse-proxies to
|
||||
# helexa-router and /api to helexa-upstream, both on gallumbits — so the
|
||||
# browser never makes a cross-origin request and bearers ride
|
||||
# first-party. Per-IP rate zones live in conf.d/helexa-ratelimit.conf.
|
||||
#
|
||||
# TLS via Let's Encrypt (certbot, Cloudflare DNS-01, ECDSA) —
|
||||
# bootstrapped one-time in script/infra-setup.sh.
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
listen [::]:80;
|
||||
server_name helexa.ai;
|
||||
|
||||
# Keep serving the ACME webroot so certbot can renew.
|
||||
location /.well-known/acme-challenge/ {
|
||||
root /var/www/helexa.ai;
|
||||
}
|
||||
location / {
|
||||
return 301 https://$host$request_uri;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
# TCP 443 on this host is owned by the stream SNI router
|
||||
# (streams-enabled/sni-router.conf), which preads the server name and
|
||||
# forwards to the local https tier on 127.0.0.1:14443 over PROXY
|
||||
# protocol. A vhost that listens on :443 directly is never reached —
|
||||
# the router answers first, with whichever certificate its default
|
||||
# branch holds. `nginx -t` passes either way, so the mistake shows up
|
||||
# only as the wrong certificate on a working handshake.
|
||||
#
|
||||
# Prerequisites on the host, both owned by the edge/mail
|
||||
# infrastructure rather than this repo (and host-specific — the two
|
||||
# edges route different mail names, so neither is synced from here):
|
||||
# streams-enabled/sni-router.conf — owns :443, preads SNI
|
||||
# conf.d/proxy-protocol.conf — real_ip, so $binary_remote_addr
|
||||
# and X-Real-IP stay truthful
|
||||
listen 127.0.0.1:14443 ssl proxy_protocol;
|
||||
http2 on;
|
||||
server_name helexa.ai;
|
||||
|
||||
# This vhost listens on an internal port behind the edge SNI router,
|
||||
# so nginx's own idea of "where am I" is 127.0.0.1:14443 rather than
|
||||
# the address the client used. Absolute redirects built from that
|
||||
# leak the internal port -- a request for a directory without a
|
||||
# trailing slash answered 301 to https://<host>:14443/<path>/, which
|
||||
# no client outside this machine can reach.
|
||||
#
|
||||
# Relative redirects sidestep the question: the browser resolves the
|
||||
# Location against the URL it actually requested, which is correct
|
||||
# for every deployment shape this vhost has.
|
||||
absolute_redirect off;
|
||||
|
||||
ssl_certificate /etc/letsencrypt/live/helexa.ai/fullchain.pem;
|
||||
ssl_certificate_key /etc/letsencrypt/live/helexa.ai/privkey.pem;
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
ssl_ciphers HIGH:!aNULL:!MD5;
|
||||
ssl_prefer_server_ciphers on;
|
||||
ssl_session_cache shared:SSL:10m;
|
||||
|
||||
root /var/www/helexa.ai;
|
||||
index index.html;
|
||||
|
||||
# Long-cache fingerprinted assets; never cache the HTML shell.
|
||||
location /assets/ {
|
||||
expires 1y;
|
||||
add_header Cache-Control "public, immutable";
|
||||
}
|
||||
|
||||
# Inference data plane → helexa-router on gallumbits. Streaming
|
||||
# (SSE): buffering off so tokens reach the browser as they arrive.
|
||||
location /v1/ {
|
||||
limit_req zone=helexa_v1 burst=10 nodelay;
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8088;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header Connection "";
|
||||
proxy_buffering off;
|
||||
proxy_read_timeout 300s;
|
||||
}
|
||||
location = /health {
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8088;
|
||||
}
|
||||
|
||||
# Account control plane → helexa-upstream /web/v1/ (strip the /api
|
||||
# prefix).
|
||||
# Page-read tool for the chat app (#177) -> helexa-tools on the
|
||||
# federation service host (SSRF-guarded readability fetcher).
|
||||
location = /tools/fetch {
|
||||
limit_req zone=helexa_tools burst=10 nodelay;
|
||||
limit_except GET { deny all; }
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8889/fetch;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_read_timeout 40s;
|
||||
}
|
||||
|
||||
# Web-search tool for the chat app (#177) -> SearXNG on the
|
||||
# federation service host. GET only; the SPA requests format=json.
|
||||
location = /tools/web_search {
|
||||
limit_req zone=helexa_tools burst=10 nodelay;
|
||||
limit_except GET { deny all; }
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8888/search;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_read_timeout 30s;
|
||||
}
|
||||
|
||||
location /api/ {
|
||||
limit_req zone=helexa_api burst=20 nodelay;
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8090/web/v1/;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
|
||||
# SPA history fallback — client-side routes resolve to index.html.
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
}
|
||||
@@ -1,140 +0,0 @@
|
||||
# Internal helexa website vhost — https://helexa.internal, reachable from
|
||||
# inside the WireGuard mesh (the public helexa.ai dead-ends at the
|
||||
# OPNsense LAN interface, which only port-forwards :443 from the WAN —
|
||||
# see architecture/reverse-proxies.md §2). Same SPA webroot as the
|
||||
# public vhost, but with an internal-CA cert (smallstep "lair", renewed
|
||||
# by step@helexa.timer). Installed on hanzalova.internal only; the
|
||||
# split-horizon helexa.internal record points at hanzalova's mesh IP on
|
||||
# both site routers.
|
||||
server {
|
||||
server_name helexa.internal;
|
||||
# TCP 443 on this host is owned by the stream SNI router
|
||||
# (streams-enabled/sni-router.conf), which preads the server name and
|
||||
# forwards to the local https tier on 127.0.0.1:14443 over PROXY
|
||||
# protocol. A vhost that listens on :443 directly is never reached —
|
||||
# the router answers first, with whichever certificate its default
|
||||
# branch holds. `nginx -t` passes either way, so the mistake shows up
|
||||
# only as the wrong certificate on a working handshake.
|
||||
#
|
||||
# Prerequisites on the host, both owned by the edge/mail
|
||||
# infrastructure rather than this repo (and host-specific — the two
|
||||
# edges route different mail names, so neither is synced from here):
|
||||
# streams-enabled/sni-router.conf — owns :443, preads SNI
|
||||
# conf.d/proxy-protocol.conf — real_ip, so $binary_remote_addr
|
||||
# and X-Real-IP stay truthful
|
||||
listen 127.0.0.1:14443 ssl proxy_protocol;
|
||||
http2 on;
|
||||
|
||||
# This vhost listens on an internal port behind the edge SNI router,
|
||||
# so nginx's own idea of "where am I" is 127.0.0.1:14443 rather than
|
||||
# the address the client used. Absolute redirects built from that
|
||||
# leak the internal port -- a request for a directory without a
|
||||
# trailing slash answered 301 to https://<host>:14443/<path>/, which
|
||||
# no client outside this machine can reach.
|
||||
#
|
||||
# Relative redirects sidestep the question: the browser resolves the
|
||||
# Location against the URL it actually requested, which is correct
|
||||
# for every deployment shape this vhost has.
|
||||
absolute_redirect off;
|
||||
|
||||
ssl_certificate /etc/nginx/tls/cert/helexa.internal.pem;
|
||||
ssl_certificate_key /etc/nginx/tls/key/helexa.internal.pem;
|
||||
ssl_trusted_certificate /etc/pki/ca-trust/source/anchors/root-internal.pem;
|
||||
ssl_protocols TLSv1.3;
|
||||
|
||||
# Shared webroot with the public vhost — same built SPA.
|
||||
root /var/www/helexa.ai;
|
||||
index index.html;
|
||||
|
||||
location /assets/ {
|
||||
expires 1y;
|
||||
add_header Cache-Control "public, immutable";
|
||||
}
|
||||
|
||||
# Same backends as the public vhost (#167): /v1 → helexa-router,
|
||||
# /api → helexa-upstream, both on gallumbits over the mesh.
|
||||
location /v1/ {
|
||||
limit_req zone=helexa_v1 burst=10 nodelay;
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8088;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header Connection "";
|
||||
proxy_buffering off;
|
||||
proxy_read_timeout 300s;
|
||||
}
|
||||
location = /health {
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8088;
|
||||
}
|
||||
# Page-read tool for the chat app (#177) -> helexa-tools on the
|
||||
# federation service host (SSRF-guarded readability fetcher).
|
||||
location = /tools/fetch {
|
||||
limit_req zone=helexa_tools burst=10 nodelay;
|
||||
limit_except GET { deny all; }
|
||||
# The `url=` parameter is as revealing as a search query: it is
|
||||
# the page a user is having read to them. Same treatment.
|
||||
access_log /var/log/nginx/access.log helexa_noquery;
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8889/fetch;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_read_timeout 40s;
|
||||
}
|
||||
|
||||
# Web-search tool for the chat app (#177) -> SearXNG on the
|
||||
# federation service host. GET only; the SPA requests format=json.
|
||||
location = /tools/web_search {
|
||||
limit_req zone=helexa_tools burst=10 nodelay;
|
||||
# GET and POST: SearXNG accepts both, and a client that POSTs
|
||||
# keeps its query out of any intermediary's request line. Other
|
||||
# methods have no meaning here.
|
||||
limit_except GET POST { deny all; }
|
||||
# A search query is small. Bounding the body stops this becoming
|
||||
# a way to push arbitrary bytes at the upstream.
|
||||
client_max_body_size 8k;
|
||||
# Never log the query — see helexa_noquery in
|
||||
# conf.d/helexa-ratelimit.conf.
|
||||
access_log /var/log/nginx/access.log helexa_noquery;
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8888/search;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_read_timeout 30s;
|
||||
}
|
||||
|
||||
# The same upstream under SearXNG's own path, for clients that speak
|
||||
# the SearXNG API rather than our app's.
|
||||
#
|
||||
# `/tools/web_search` is our name for it, and a generic SearXNG
|
||||
# client cannot guess it: they take a base URL and append `/search`,
|
||||
# because that is what every SearXNG instance serves. Without this
|
||||
# block such a request falls through to `location /` and is answered
|
||||
# by the SPA — HTTP 200, `text/html`, and a JSON parse error at the
|
||||
# client. A 404 would at least be honest; 200-with-HTML sends people
|
||||
# looking for a bug in their own parser.
|
||||
#
|
||||
# Same upstream, same rate-limit zone, same GET-only restriction:
|
||||
# this is an alias, not a second, laxer door.
|
||||
location = /search {
|
||||
limit_req zone=helexa_tools burst=10 nodelay;
|
||||
limit_except GET POST { deny all; }
|
||||
client_max_body_size 8k;
|
||||
access_log /var/log/nginx/access.log helexa_noquery;
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8888/search;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_read_timeout 30s;
|
||||
}
|
||||
|
||||
location /api/ {
|
||||
limit_req zone=helexa_api burst=20 nodelay;
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8090/web/v1/;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<service>
|
||||
<short>helexa-searxng</short>
|
||||
<description>SearXNG metasearch backing the helexa /tools/web_search API (#177). Reached only by the edge reverse proxies over the site mesh.</description>
|
||||
<port protocol="tcp" port="8888"/>
|
||||
</service>
|
||||
@@ -1,27 +0,0 @@
|
||||
# Quadlet unit for the SearXNG metasearch container backing the helexa
|
||||
# web_search tool (#177). Installed to /etc/containers/systemd/ on the
|
||||
# federation service host by script/infra-setup.sh; podman-systemd
|
||||
# generates searxng.service from it on daemon-reload.
|
||||
#
|
||||
# Reached only by the edge reverse proxies (/tools/web_search) over the
|
||||
# site mesh; the helexa-searxng firewalld service opens tcp/8888.
|
||||
|
||||
[Unit]
|
||||
Description=SearXNG metasearch (helexa web_search tool)
|
||||
Wants=network-online.target
|
||||
After=network-online.target
|
||||
|
||||
[Container]
|
||||
Image=docker.io/searxng/searxng:latest
|
||||
PublishPort=8888:8080
|
||||
# rw: the image entrypoint generates uwsgi.ini beside settings.yml on
|
||||
# first start. The settings file itself is re-synced by infra-setup.
|
||||
Volume=/etc/searxng:/etc/searxng:Z
|
||||
EnvironmentFile=/etc/searxng/searxng.env
|
||||
AutoUpdate=registry
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,24 +0,0 @@
|
||||
# SearXNG settings for the helexa web_search tool backend (#177).
|
||||
# Installed to /etc/searxng/settings.yml on the federation service host
|
||||
# by script/infra-setup.sh; consumed by the quadlet container
|
||||
# (asset/searxng/searxng.container).
|
||||
#
|
||||
# The secret key is NOT here — it comes from SEARXNG_SECRET in the
|
||||
# operator-owned /etc/searxng/searxng.env (generated once by
|
||||
# infra-setup, never in git).
|
||||
use_default_settings: true
|
||||
|
||||
server:
|
||||
# Bot-detection limiter off: the instance is reachable only from the
|
||||
# edge proxies over the site mesh, and per-client-IP rate limiting is
|
||||
# enforced there (helexa-ratelimit.conf helexa_tools zone).
|
||||
limiter: false
|
||||
image_proxy: false
|
||||
method: "GET"
|
||||
|
||||
search:
|
||||
# `json` enables the machine-readable API the chat SPA's tool loop
|
||||
# consumes (GET /search?q=…&format=json).
|
||||
formats:
|
||||
- html
|
||||
- json
|
||||
@@ -1,25 +0,0 @@
|
||||
# Install on the bench host (bob) as /etc/sudoers.d/helexa_gitea_ci
|
||||
# (owner root:root, mode 0440). Required by .gitea/workflows/deploy.yml,
|
||||
# which SSHes as gitea_ci@bob to roll out helexa-bench package upgrades
|
||||
# and config changes.
|
||||
#
|
||||
# Filename convention `helexa_gitea_ci` (vs bare `gitea_ci`) so other
|
||||
# helexa-org apps can drop their own sudoers files on the same host
|
||||
# without overwriting this one.
|
||||
#
|
||||
# helexa-bench polls the neuron fleet (outbound) and serves a read-only
|
||||
# JSON API on tcp/13132 for the bench UI — hence the firewall-cmd grants.
|
||||
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/helexa-bench/helexa-bench.toml
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl start helexa-bench.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl stop helexa-bench.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable --now helexa-bench.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf install --refresh --allowerasing -y helexa-bench
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf upgrade --refresh --allowerasing -y helexa-bench
|
||||
# sudoers reserves `:` and `=` and requires `\` escaping inside command
|
||||
# arguments — without it visudo errors at the first `:` in `https://`.
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf config-manager addrepo --from-repofile\=https\://rpm.lair.cafe/lair-cafe-unstable.repo
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf config-manager setopt lair-cafe-unstable.enabled\=1
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --add-service=helexa-bench --permanent
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --reload
|
||||
@@ -1,37 +0,0 @@
|
||||
# Install on the cortex gateway host as /etc/sudoers.d/helexa_gitea_ci
|
||||
# (owner root:root, mode 0440). Required by .gitea/workflows/deploy.yml,
|
||||
# which SSHes as gitea_ci@<gateway> to roll out cortex package upgrades
|
||||
# and config changes.
|
||||
#
|
||||
# Filename convention `helexa_gitea_ci` (vs bare `gitea_ci`) so other
|
||||
# helexa-org apps can drop their own sudoers files on the same host
|
||||
# without overwriting this one.
|
||||
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/cortex/cortex.toml
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/cortex/models.toml
|
||||
# deploy-bench-ui rsyncs the built bench SPA into the nginx webroot.
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /var/www/bench.helexa.ai/
|
||||
# deploy-website rsyncs the built helexa.ai SPA into the nginx webroot.
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /var/www/helexa.ai/
|
||||
# deploy.yml's "Sync internal nginx vhost" step: write the vhost, make
|
||||
# sites-enabled a symlink to it, validate, reload. Scoped to the one
|
||||
# file and the one service — `nginx -t` is included because a reload
|
||||
# must never happen on a config nginx has rejected.
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/nginx/sites-available/helexa.internal.conf
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/nginx/conf.d/helexa-ratelimit.conf
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/sha256sum /etc/nginx/conf.d/helexa-ratelimit.conf
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/sha256sum /etc/nginx/sites-available/helexa.internal.conf
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/ln -sfn /etc/nginx/sites-available/helexa.internal.conf /etc/nginx/sites-enabled/helexa.internal.conf
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/nginx -t
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl reload nginx
|
||||
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl start cortex.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl stop cortex.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable --now cortex.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf install --refresh --allowerasing -y cortex
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf upgrade --refresh --allowerasing -y cortex
|
||||
# sudoers reserves `:` and `=` and requires `\` escaping inside command
|
||||
# arguments — without it visudo errors at the first `:` in `https://`.
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf config-manager addrepo --from-repofile\=https\://rpm.lair.cafe/lair-cafe-unstable.repo
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf config-manager setopt lair-cafe-unstable.enabled\=1
|
||||
@@ -1,62 +0,0 @@
|
||||
# Install on the federation service host (gallumbits.kosherinata.internal)
|
||||
# as /etc/sudoers.d/helexa_gitea_ci (owner root:root, mode 0440).
|
||||
# Required by .gitea/workflows/deploy.yml's deploy-gallumbits job, which
|
||||
# rolls out helexa-router (data plane), helexa-upstream (account/budget
|
||||
# authority), helexa-tools (grounding fetcher) and helexa-angels (the
|
||||
# confidential investor portal) package upgrades.
|
||||
# The SECRET-BEARING configs are not synced by CI — helexa-upstream.toml
|
||||
# and helexa-angels.toml carry a DB password, a JWT secret and SMTP
|
||||
# credentials, and ride in script/infra-setup.sh under the operator's own
|
||||
# sudo instead (#287 Gap 1, still open).
|
||||
#
|
||||
# helexa-router.toml is different and IS synced here: it holds no secrets,
|
||||
# only a listen address, the product-tier aliases and a mesh endpoint. It
|
||||
# was excluded by association with its neighbours, which is how its
|
||||
# `helexa/balanced` alias came to point at a model retired from the
|
||||
# catalogue and took beast down on 2026-08-27. A config CI cannot write is
|
||||
# a config no check can defend.
|
||||
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl start helexa-router.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl stop helexa-router.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable --now helexa-router.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl start helexa-upstream.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl stop helexa-upstream.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable --now helexa-upstream.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl start helexa-tools.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl stop helexa-tools.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable --now helexa-tools.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl start helexa-angels.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl stop helexa-angels.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable --now helexa-angels.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload
|
||||
|
||||
# deploy-gallumbits' "Sync router config" step. Scoped to the one file:
|
||||
# the neighbouring service configs stay operator-only until #287 Gap 1
|
||||
# gives them a render-from-secrets path.
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/helexa-router/helexa-router.toml
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/sha256sum /etc/helexa-router/helexa-router.toml
|
||||
# Read-back, so the sync can tell a settings change from a comment
|
||||
# change and restart only for the former. Safe to grant and safe to read
|
||||
# into CI because this file holds no secrets — the same property that
|
||||
# lets it be tracked. If this grant is absent the step degrades to
|
||||
# restarting on any byte change, which is why it is not in the step's
|
||||
# fail-hard preflight.
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/cat /etc/helexa-router/helexa-router.toml
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl restart helexa-router.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf install --refresh --allowerasing -y helexa-router
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf upgrade --refresh --allowerasing -y helexa-router
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf install --refresh --allowerasing -y helexa-upstream
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf upgrade --refresh --allowerasing -y helexa-upstream
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf install --refresh --allowerasing -y helexa-tools
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf upgrade --refresh --allowerasing -y helexa-tools
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf install --refresh --allowerasing -y helexa-angels
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf upgrade --refresh --allowerasing -y helexa-angels
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --add-service=helexa-router --permanent
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --add-service=helexa-upstream --permanent
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --add-service=helexa-tools --permanent
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --add-service=helexa-angels --permanent
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --reload
|
||||
# sudoers reserves `:` and `=` and requires `\` escaping inside command
|
||||
# arguments — without it visudo errors at the first `:` in `https://`.
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf config-manager addrepo --from-repofile\=https\://rpm.lair.cafe/lair-cafe-unstable.repo
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf config-manager setopt lair-cafe-unstable.enabled\=1
|
||||
@@ -1,51 +0,0 @@
|
||||
# Install on every neuron host as /etc/sudoers.d/helexa_gitea_ci
|
||||
# (owner root:root, mode 0440). Required by .gitea/workflows/deploy.yml,
|
||||
# which SSHes as gitea_ci@<neuron-host> to roll out helexa-neuron-<flavour>
|
||||
# package upgrades and config changes.
|
||||
#
|
||||
# Filename convention `helexa_gitea_ci` (vs bare `gitea_ci`) so other
|
||||
# helexa-org apps can drop their own sudoers files on the same host
|
||||
# without overwriting this one.
|
||||
#
|
||||
# All three CUDA flavours are listed because a host's flavour can change
|
||||
# (e.g. GPU swap) and we don't want the sudoers file to need to change
|
||||
# in lockstep. Only one flavour can be installed at a time (the packages
|
||||
# Conflict: with each other), so the attack surface is bounded to "wrong
|
||||
# flavour installed" — vandalism, not privilege escalation.
|
||||
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/neuron/neuron.toml
|
||||
# deploy.yml writes the per-model systemd drop-in carrying
|
||||
# NEURON_MAX_PROMPT_TOKENS: gitea_ci stages it in its own dir, then
|
||||
# installs it root-owned. Exact source/dest paths; see doc/context-limits.md.
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/install -o root -g root -m 0644 -D /var/lib/gitea_ci/model.conf /etc/systemd/system/neuron.service.d/model.conf
|
||||
# deploy.yml also writes the registry drop-in, which points every model
|
||||
# fetch at rustingface and carries the client token for it. Mode 0600
|
||||
# rather than model.conf's 0644: this one holds a credential, and a
|
||||
# world-readable systemd drop-in is a world-readable token.
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/install -o root -g root -m 0600 -D /var/lib/gitea_ci/registry.conf /etc/systemd/system/neuron.service.d/registry.conf
|
||||
# Reading it back to decide whether it changed needs root too, since
|
||||
# 0600 excludes gitea_ci. Restricted to the one file.
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/sha256sum /etc/systemd/system/neuron.service.d/registry.conf
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl start neuron.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl stop neuron.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable --now neuron.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf install --refresh --allowerasing -y helexa-neuron-ampere
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf upgrade --refresh --allowerasing -y helexa-neuron-ampere
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf install --refresh --allowerasing -y helexa-neuron-ada
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf upgrade --refresh --allowerasing -y helexa-neuron-ada
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf install --refresh --allowerasing -y helexa-neuron-blackwell
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf upgrade --refresh --allowerasing -y helexa-neuron-blackwell
|
||||
# sudoers reserves `:` and `=` and requires `\` escaping inside command
|
||||
# arguments — without it visudo errors at the first `:` in `https://`.
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf config-manager addrepo --from-repofile\=https\://rpm.lair.cafe/lair-cafe-unstable.repo
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf config-manager setopt lair-cafe-unstable.enabled\=1
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf config-manager addrepo --from-repofile\=https\://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf install -y libcudnn9-cuda-13
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --add-service=helexa-neuron --permanent
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --reload
|
||||
# deploy-dev.yml fast path: install a freshly-built dev binary over the
|
||||
# packaged one. Exact source path + args; the workflow must use this
|
||||
# command form verbatim. The next deploy.yml run reconciles the host
|
||||
# back to the RPM-owned binary.
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/install -o root -g root -m 0755 /var/lib/gitea_ci/neuron-dev /usr/bin/neuron
|
||||
@@ -1,11 +0,0 @@
|
||||
# Install on a website edge proxy that is NOT the cortex gateway
|
||||
# (currently oolon.kosherinata.internal) as
|
||||
# /etc/sudoers.d/helexa_gitea_ci (owner root:root, mode 0440).
|
||||
# Required by .gitea/workflows/deploy.yml's deploy-website job, which
|
||||
# rsyncs the built helexa.ai SPA into the nginx webroot.
|
||||
#
|
||||
# The cortex gateway (hanzalova.internal) gets this grant via
|
||||
# cortex-host.conf instead — one sudoers file per host, and oolon
|
||||
# hosts other projects' gitea_ci grants in their own drop-ins.
|
||||
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /var/www/helexa.ai/
|
||||
@@ -1,20 +0,0 @@
|
||||
# Internal-CA cert renewal for %i.internal, driven by step@%i.timer.
|
||||
# Replicated from oolon.kosherinata.internal (the kosherinata DC proxy).
|
||||
# Renews an EXISTING cert via mTLS (step ca renew) — the initial cert
|
||||
# must be issued once with a provisioner (see script/infra-setup.sh).
|
||||
# Installed to /etc/systemd/system/step@.service.
|
||||
[Unit]
|
||||
Description=step cert renew for %i.internal
|
||||
Documentation=https://smallstep.com/docs/step-ca/renewal
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecCondition=/usr/bin/step certificate needs-renewal \
|
||||
/etc/nginx/tls/cert/%i.internal.pem
|
||||
ExecStart=/usr/bin/step ca renew \
|
||||
--force \
|
||||
--ca-url https://ca.internal \
|
||||
--root /etc/pki/ca-trust/source/anchors/root-internal.pem \
|
||||
/etc/nginx/tls/cert/%i.internal.pem \
|
||||
/etc/nginx/tls/key/%i.internal.pem
|
||||
ExecStartPost=/usr/bin/systemctl reload nginx.service
|
||||
@@ -1,15 +0,0 @@
|
||||
# Periodic internal-cert renewal for %i.internal (every 15 min, jittered).
|
||||
# Replicated from oolon.kosherinata.internal. Installed to
|
||||
# /etc/systemd/system/step@.timer; enable per-cert with
|
||||
# `systemctl enable --now step@bench.timer`.
|
||||
[Unit]
|
||||
Description=step cert renew timer for %i.internal
|
||||
|
||||
[Timer]
|
||||
Persistent=true
|
||||
OnCalendar=*:1/15
|
||||
AccuracySec=1us
|
||||
RandomizedDelaySec=5m
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
3
bench/.gitignore
vendored
3
bench/.gitignore
vendored
@@ -1,3 +0,0 @@
|
||||
node_modules
|
||||
dist
|
||||
*.local
|
||||
@@ -1,45 +0,0 @@
|
||||
# helexa bench UI
|
||||
|
||||
A Vite + React (SWC, TypeScript) app that visualises the fleet benchmark
|
||||
data collected by `helexa-bench`. It reads the read-only JSON API the
|
||||
bench daemon serves (`crates/helexa-bench/src/api.rs`, default
|
||||
`:13132` on bob).
|
||||
|
||||
Stack: React Router, react-bootstrap, Recharts.
|
||||
|
||||
## Pages
|
||||
|
||||
- **Overview** — latest median results per (host, model, scenario) cell.
|
||||
- **Trends** — decode-tok/s and TTFT plotted across neuron build SHAs as
|
||||
releases roll out (the headline view). Pick host / model / scenario.
|
||||
- **Runs** — filterable raw-run explorer.
|
||||
|
||||
## Develop
|
||||
|
||||
```sh
|
||||
cd bench
|
||||
npm install
|
||||
npm run dev # http://localhost:5173
|
||||
```
|
||||
|
||||
`vite.config.ts` proxies `/api` → `http://bob.hanzalova.internal:13132`,
|
||||
so the dev server talks to the live bench API with no CORS fuss. Point
|
||||
the proxy elsewhere (or run a local `helexa-bench serve`) to develop
|
||||
against other data.
|
||||
|
||||
## Production hosting
|
||||
|
||||
Public at **https://bench.helexa.ai** — nginx on the gateway
|
||||
(`hanzalova.internal`) serves the static `dist/` and reverse-proxies
|
||||
`/api` to the bench API on bob over WireGuard, so the SPA is same-origin
|
||||
(no CORS) and the internal API stays off the public internet.
|
||||
|
||||
- `npm run build` is run with **no** `VITE_API_BASE` (the app calls
|
||||
`/api/...` on its own origin; nginx proxies it to bob).
|
||||
- `.gitea/workflows/deploy.yml` (`deploy-bench-ui`) builds and rsyncs
|
||||
`dist/` to `/var/www/bench.helexa.ai` on every deploy.
|
||||
- The nginx vhost (`asset/nginx/bench.helexa.ai.conf`) and the
|
||||
Let's Encrypt cert are one-time host setup in `script/infra-setup.sh`.
|
||||
|
||||
To host elsewhere instead, build with
|
||||
`VITE_API_BASE=<bob-api-origin>` and serve the static `dist/`.
|
||||
@@ -1,12 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>helexa bench</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.tsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
2191
bench/package-lock.json
generated
2191
bench/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -1,28 +0,0 @@
|
||||
{
|
||||
"name": "helexa-bench-ui",
|
||||
"private": true,
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"description": "Visualisation app for helexa-bench fleet benchmark data.",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "tsc && vite build",
|
||||
"preview": "vite preview"
|
||||
},
|
||||
"dependencies": {
|
||||
"bootstrap": "^5.3.3",
|
||||
"react": "^18.3.1",
|
||||
"react-bootstrap": "^2.10.5",
|
||||
"react-dom": "^18.3.1",
|
||||
"react-router-dom": "^6.26.2",
|
||||
"recharts": "^2.12.7"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.14.0",
|
||||
"@types/react": "^18.3.5",
|
||||
"@types/react-dom": "^18.3.0",
|
||||
"@vitejs/plugin-react-swc": "^3.7.0",
|
||||
"typescript": "^5.5.4",
|
||||
"vite": "^5.4.0"
|
||||
}
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
import { Container, Nav, Navbar } from "react-bootstrap";
|
||||
import { NavLink, Outlet } from "react-router-dom";
|
||||
|
||||
export default function App() {
|
||||
return (
|
||||
<>
|
||||
<Navbar bg="dark" variant="dark" expand="md">
|
||||
<Container>
|
||||
<Navbar.Brand as={NavLink} to="/">
|
||||
helexa bench
|
||||
</Navbar.Brand>
|
||||
<Nav className="me-auto">
|
||||
<Nav.Link as={NavLink} to="/" end>
|
||||
Overview
|
||||
</Nav.Link>
|
||||
<Nav.Link as={NavLink} to="/trends">
|
||||
Trends
|
||||
</Nav.Link>
|
||||
<Nav.Link as={NavLink} to="/runs">
|
||||
Runs
|
||||
</Nav.Link>
|
||||
</Nav>
|
||||
</Container>
|
||||
</Navbar>
|
||||
<Container className="py-4">
|
||||
<Outlet />
|
||||
</Container>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
import type {
|
||||
Dimensions,
|
||||
MeasurementRegime,
|
||||
ReportRow,
|
||||
RunRow,
|
||||
SeriesPoint,
|
||||
} from "./types";
|
||||
|
||||
// Empty default → `fetch('/api/...')` hits the dev proxy (vite.config.ts)
|
||||
// or the same origin. For a separately-hosted build, set VITE_API_BASE to
|
||||
// the bob API origin (e.g. http://bob.hanzalova.internal:13132).
|
||||
const BASE = import.meta.env.VITE_API_BASE ?? "";
|
||||
|
||||
async function getJson<T>(path: string): Promise<T> {
|
||||
const res = await fetch(`${BASE}${path}`);
|
||||
if (!res.ok) {
|
||||
throw new Error(`${res.status} ${res.statusText}: ${await res.text()}`);
|
||||
}
|
||||
return res.json() as Promise<T>;
|
||||
}
|
||||
|
||||
export const getDimensions = () => getJson<Dimensions>("/api/dimensions");
|
||||
export const getSummary = () => getJson<ReportRow[]>("/api/summary");
|
||||
export const getRegimes = () => getJson<MeasurementRegime[]>("/api/regimes");
|
||||
|
||||
// host is resolved server-side (each model maps to one host today), so the
|
||||
// public UI selects by model + scenario alone.
|
||||
export const getSeries = (model: string, scenario: string) =>
|
||||
getJson<SeriesPoint[]>(
|
||||
`/api/series?model=${encodeURIComponent(model)}&scenario=${encodeURIComponent(scenario)}`,
|
||||
);
|
||||
|
||||
export interface RunsParams {
|
||||
host?: string;
|
||||
model?: string;
|
||||
scenario?: string;
|
||||
sha?: string;
|
||||
ok?: boolean;
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
export const getRuns = (p: RunsParams = {}) => {
|
||||
const q = new URLSearchParams();
|
||||
if (p.host) q.set("host", p.host);
|
||||
if (p.model) q.set("model", p.model);
|
||||
if (p.scenario) q.set("scenario", p.scenario);
|
||||
if (p.sha) q.set("sha", p.sha);
|
||||
if (p.ok !== undefined) q.set("ok", String(p.ok));
|
||||
if (p.limit) q.set("limit", String(p.limit));
|
||||
const qs = q.toString();
|
||||
return getJson<RunRow[]>(`/api/runs${qs ? `?${qs}` : ""}`);
|
||||
};
|
||||
@@ -1,52 +0,0 @@
|
||||
// Pre-helexa-bench baseline, transcribed verbatim from doc/benchmarks.md.
|
||||
//
|
||||
// IMPORTANT — different measurement regime. These were measured by
|
||||
// script/bench.py *through the cortex gateway* (so TTFT/total include a
|
||||
// proxy hop), reported as medians only, before helexa-bench existed.
|
||||
// helexa-bench measures each neuron *directly*. So these points are an
|
||||
// honest historical anchor, NOT apples-to-apples with the live series —
|
||||
// the Trends view renders them dashed + labelled, never merged into the
|
||||
// live line.
|
||||
//
|
||||
// Host is inferred from the model via the doc's Fleet table
|
||||
// (beast=27B, benjy=8B, quadbrat=1.7B). Timestamps are the two 2026-06-12
|
||||
// snapshots in the doc, ordered (08:00 = pre-#11, 16:00 = post-#11) so
|
||||
// they sort before the bench era on the shared time axis.
|
||||
|
||||
export interface BaselinePoint {
|
||||
host: string;
|
||||
model: string;
|
||||
scenario: string;
|
||||
git_sha: string;
|
||||
build_timestamp: string;
|
||||
ttft_s: number;
|
||||
decode_tps: number;
|
||||
total_s: number;
|
||||
}
|
||||
|
||||
/** Source: bench.py via cortex gateway — see doc/benchmarks.md. */
|
||||
export const BASELINE_SOURCE = "bench.py · via cortex gateway";
|
||||
|
||||
export const BASELINE: BaselinePoint[] = [
|
||||
// ── 8f6f1d3 — baseline (2026-06-12) ────────────────────────────────
|
||||
{ host: "beast", model: "Qwen/Qwen3.6-27B", scenario: "chat:128", git_sha: "8f6f1d3", build_timestamp: "2026-06-12T08:00:00Z", ttft_s: 1.658, decode_tps: 35.0, total_s: 8.981 },
|
||||
{ host: "beast", model: "Qwen/Qwen3.6-27B", scenario: "chat:4096", git_sha: "8f6f1d3", build_timestamp: "2026-06-12T08:00:00Z", ttft_s: 7.067, decode_tps: 33.7, total_s: 14.63 },
|
||||
{ host: "benjy", model: "Qwen/Qwen3-8B", scenario: "chat:128", git_sha: "8f6f1d3", build_timestamp: "2026-06-12T08:00:00Z", ttft_s: 0.884, decode_tps: 62.4, total_s: 4.938 },
|
||||
{ host: "benjy", model: "Qwen/Qwen3-8B", scenario: "chat:4096", git_sha: "8f6f1d3", build_timestamp: "2026-06-12T08:00:00Z", ttft_s: 1.818, decode_tps: 46.5, total_s: 7.27 },
|
||||
{ host: "quadbrat", model: "Qwen/Qwen3-1.7B", scenario: "chat:128", git_sha: "8f6f1d3", build_timestamp: "2026-06-12T08:00:00Z", ttft_s: 0.685, decode_tps: 81.3, total_s: 3.741 },
|
||||
{ host: "quadbrat", model: "Qwen/Qwen3-1.7B", scenario: "chat:4096", git_sha: "8f6f1d3", build_timestamp: "2026-06-12T08:00:00Z", ttft_s: 2.743, decode_tps: 35.4, total_s: 9.884 },
|
||||
// ── a1952a4 — post prefix-KV-cache (#11, 2026-06-12) ───────────────
|
||||
{ host: "beast", model: "Qwen/Qwen3.6-27B", scenario: "chat:128", git_sha: "a1952a4", build_timestamp: "2026-06-12T16:00:00Z", ttft_s: 1.355, decode_tps: 45.8, total_s: 4.147 },
|
||||
{ host: "beast", model: "Qwen/Qwen3.6-27B", scenario: "chat:4096", git_sha: "a1952a4", build_timestamp: "2026-06-12T16:00:00Z", ttft_s: 1.431, decode_tps: 43.3, total_s: 4.387 },
|
||||
{ host: "benjy", model: "Qwen/Qwen3-8B", scenario: "chat:128", git_sha: "a1952a4", build_timestamp: "2026-06-12T16:00:00Z", ttft_s: 0.886, decode_tps: 78.6, total_s: 2.478 },
|
||||
{ host: "benjy", model: "Qwen/Qwen3-8B", scenario: "chat:4096", git_sha: "a1952a4", build_timestamp: "2026-06-12T16:00:00Z", ttft_s: 1.824, decode_tps: 58.3, total_s: 3.969 },
|
||||
{ host: "quadbrat", model: "Qwen/Qwen3-1.7B", scenario: "chat:128", git_sha: "a1952a4", build_timestamp: "2026-06-12T16:00:00Z", ttft_s: 0.702, decode_tps: 104.8, total_s: 1.895 },
|
||||
{ host: "quadbrat", model: "Qwen/Qwen3-1.7B", scenario: "chat:4096", git_sha: "a1952a4", build_timestamp: "2026-06-12T16:00:00Z", ttft_s: 2.749, decode_tps: 44.9, total_s: 5.534 },
|
||||
];
|
||||
|
||||
/** Baseline points for one (model, scenario) cell, oldest first. */
|
||||
export function baselineFor(model: string, scenario: string): BaselinePoint[] {
|
||||
return BASELINE.filter(
|
||||
(b) => b.model === model && b.scenario === scenario,
|
||||
).sort((a, b) => a.build_timestamp.localeCompare(b.build_timestamp));
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
import React from "react";
|
||||
import ReactDOM from "react-dom/client";
|
||||
import { BrowserRouter, Route, Routes } from "react-router-dom";
|
||||
import "bootstrap/dist/css/bootstrap.min.css";
|
||||
import App from "./App";
|
||||
import Overview from "./pages/Overview";
|
||||
import Trends from "./pages/Trends";
|
||||
import Runs from "./pages/Runs";
|
||||
|
||||
ReactDOM.createRoot(document.getElementById("root")!).render(
|
||||
<React.StrictMode>
|
||||
<BrowserRouter>
|
||||
<Routes>
|
||||
<Route path="/" element={<App />}>
|
||||
<Route index element={<Overview />} />
|
||||
<Route path="trends" element={<Trends />} />
|
||||
<Route path="runs" element={<Runs />} />
|
||||
</Route>
|
||||
</Routes>
|
||||
</BrowserRouter>
|
||||
</React.StrictMode>,
|
||||
);
|
||||
@@ -1,64 +0,0 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { Alert, Spinner, Table } from "react-bootstrap";
|
||||
import { getSummary } from "../api";
|
||||
import type { ReportRow } from "../types";
|
||||
|
||||
const f = (n: number | null, p = 2) => (n == null ? "—" : n.toFixed(p));
|
||||
|
||||
export default function Overview() {
|
||||
const [rows, setRows] = useState<ReportRow[]>([]);
|
||||
const [err, setErr] = useState<string | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
|
||||
useEffect(() => {
|
||||
getSummary()
|
||||
.then(setRows)
|
||||
.catch((e) => setErr(String(e)))
|
||||
.finally(() => setLoading(false));
|
||||
}, []);
|
||||
|
||||
if (loading) return <Spinner animation="border" />;
|
||||
if (err) return <Alert variant="danger">{err}</Alert>;
|
||||
|
||||
return (
|
||||
<>
|
||||
<h3 className="mb-3">Latest results per cell</h3>
|
||||
<p className="text-muted">
|
||||
Median of each cell's samples on the most recent build seen for that
|
||||
(host, model, scenario).
|
||||
</p>
|
||||
<Table striped bordered hover responsive size="sm">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>GPU</th>
|
||||
<th>model</th>
|
||||
<th className="text-end">prompt tok</th>
|
||||
<th className="text-end">TTFT (s)</th>
|
||||
<th className="text-end">decode tok/s</th>
|
||||
<th className="text-end">total (s)</th>
|
||||
<th>build</th>
|
||||
<th className="text-end">n</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{rows.map((r, i) => (
|
||||
<tr key={i}>
|
||||
<td>{r.gpu ?? r.target_name}</td>
|
||||
<td>{r.model_id}</td>
|
||||
<td className="text-end">
|
||||
{r.prompt_tokens ?? `~${r.prompt_size_approx}`}
|
||||
</td>
|
||||
<td className="text-end">{f(r.ttft_s_median, 3)}</td>
|
||||
<td className="text-end">{f(r.decode_tps_median, 1)}</td>
|
||||
<td className="text-end">{f(r.total_s_median, 3)}</td>
|
||||
<td>
|
||||
<code>{r.git_sha}</code>
|
||||
</td>
|
||||
<td className="text-end">{r.samples}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</Table>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -1,141 +0,0 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { Alert, Badge, Col, Form, Row, Spinner, Table } from "react-bootstrap";
|
||||
import { getDimensions, getRuns } from "../api";
|
||||
import type { Dimensions, RunRow } from "../types";
|
||||
|
||||
const f = (n: number | null, p = 2) => (n == null ? "—" : n.toFixed(p));
|
||||
|
||||
function Picker({
|
||||
label,
|
||||
value,
|
||||
set,
|
||||
options,
|
||||
}: {
|
||||
label: string;
|
||||
value: string;
|
||||
set: (v: string) => void;
|
||||
options: string[];
|
||||
}) {
|
||||
return (
|
||||
<Form.Group as={Col}>
|
||||
<Form.Label>{label}</Form.Label>
|
||||
<Form.Select value={value} onChange={(e) => set(e.target.value)}>
|
||||
<option value="">(all)</option>
|
||||
{options.map((o) => (
|
||||
<option key={o} value={o}>
|
||||
{o}
|
||||
</option>
|
||||
))}
|
||||
</Form.Select>
|
||||
</Form.Group>
|
||||
);
|
||||
}
|
||||
|
||||
export default function Runs() {
|
||||
const [dims, setDims] = useState<Dimensions | null>(null);
|
||||
const [host, setHost] = useState("");
|
||||
const [model, setModel] = useState("");
|
||||
const [scenario, setScenario] = useState("");
|
||||
const [rows, setRows] = useState<RunRow[]>([]);
|
||||
const [err, setErr] = useState<string | null>(null);
|
||||
const [loading, setLoading] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
getDimensions()
|
||||
.then(setDims)
|
||||
.catch((e) => setErr(String(e)));
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
setLoading(true);
|
||||
getRuns({
|
||||
host: host || undefined,
|
||||
model: model || undefined,
|
||||
scenario: scenario || undefined,
|
||||
limit: 200,
|
||||
})
|
||||
.then(setRows)
|
||||
.catch((e) => setErr(String(e)))
|
||||
.finally(() => setLoading(false));
|
||||
}, [host, model, scenario]);
|
||||
|
||||
if (err) return <Alert variant="danger">{err}</Alert>;
|
||||
|
||||
return (
|
||||
<>
|
||||
<h3 className="mb-3">Runs</h3>
|
||||
{dims && (
|
||||
<Row className="g-3 mb-3">
|
||||
{/* GPU filter — labelled by GPU, but filters by the underlying host. */}
|
||||
<Form.Group as={Col}>
|
||||
<Form.Label>GPU</Form.Label>
|
||||
<Form.Select value={host} onChange={(e) => setHost(e.target.value)}>
|
||||
<option value="">(all)</option>
|
||||
{dims.hosts.map((h) => (
|
||||
<option key={h} value={h}>
|
||||
{dims.host_gpus[h] ?? h}
|
||||
</option>
|
||||
))}
|
||||
</Form.Select>
|
||||
</Form.Group>
|
||||
<Picker
|
||||
label="Model"
|
||||
value={model}
|
||||
set={setModel}
|
||||
options={dims.models}
|
||||
/>
|
||||
<Picker
|
||||
label="Scenario"
|
||||
value={scenario}
|
||||
set={setScenario}
|
||||
options={dims.scenarios}
|
||||
/>
|
||||
</Row>
|
||||
)}
|
||||
{loading ? (
|
||||
<Spinner animation="border" />
|
||||
) : (
|
||||
<Table striped bordered hover responsive size="sm">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>ts</th>
|
||||
<th>GPU</th>
|
||||
<th>model</th>
|
||||
<th>scenario</th>
|
||||
<th>build</th>
|
||||
<th className="text-end">TTFT</th>
|
||||
<th className="text-end">tok/s</th>
|
||||
<th className="text-end">total</th>
|
||||
<th>ok</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{rows.map((r) => (
|
||||
<tr key={r.id}>
|
||||
<td>{r.ts}</td>
|
||||
<td>{r.gpu ?? r.host}</td>
|
||||
<td>{r.model_id}</td>
|
||||
<td>{r.scenario_id}</td>
|
||||
<td>
|
||||
<code>{r.git_sha}</code>
|
||||
</td>
|
||||
<td className="text-end">{f(r.ttft_s, 3)}</td>
|
||||
<td className="text-end">{f(r.decode_tps, 1)}</td>
|
||||
<td className="text-end">{f(r.total_s, 3)}</td>
|
||||
<td>
|
||||
{r.ok ? (
|
||||
<Badge bg="success">ok</Badge>
|
||||
) : (
|
||||
<Badge bg="danger" title={r.error ?? ""}>
|
||||
fail
|
||||
</Badge>
|
||||
)}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</Table>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -1,423 +0,0 @@
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
import { Alert, Col, Form, Row, Spinner } from "react-bootstrap";
|
||||
import {
|
||||
CartesianGrid,
|
||||
Legend,
|
||||
Line,
|
||||
LineChart,
|
||||
ReferenceLine,
|
||||
ResponsiveContainer,
|
||||
Tooltip,
|
||||
XAxis,
|
||||
YAxis,
|
||||
} from "recharts";
|
||||
import { getDimensions, getRegimes, getSeries } from "../api";
|
||||
import type { Dimensions, MeasurementRegime, SeriesPoint } from "../types";
|
||||
import { BASELINE_SOURCE, baselineFor } from "../baseline";
|
||||
|
||||
function Picker({
|
||||
label,
|
||||
value,
|
||||
set,
|
||||
options,
|
||||
}: {
|
||||
label: string;
|
||||
value: string;
|
||||
set: (v: string) => void;
|
||||
options: string[];
|
||||
}) {
|
||||
return (
|
||||
<Form.Group as={Col}>
|
||||
<Form.Label>{label}</Form.Label>
|
||||
<Form.Select value={value} onChange={(e) => set(e.target.value)}>
|
||||
{options.map((o) => (
|
||||
<option key={o} value={o}>
|
||||
{o}
|
||||
</option>
|
||||
))}
|
||||
</Form.Select>
|
||||
</Form.Group>
|
||||
);
|
||||
}
|
||||
|
||||
type SeriesDef = {
|
||||
key: string;
|
||||
name: string;
|
||||
stroke: string;
|
||||
dashed?: boolean;
|
||||
};
|
||||
|
||||
/** One titled chart over the shared build timeline.
|
||||
*
|
||||
* Every panel draws the same x-axis and the same regime divider, so a
|
||||
* reader can line a change up across metrics — which is the whole point
|
||||
* of having more than two of them. */
|
||||
/** A vertical rule at a build where the metric changed meaning. */
|
||||
type Rule = { at: string; label: string; detail?: string };
|
||||
|
||||
function MetricChart({
|
||||
title,
|
||||
hint,
|
||||
data,
|
||||
lines,
|
||||
rules,
|
||||
unit,
|
||||
}: {
|
||||
title: string;
|
||||
hint?: string;
|
||||
data: Record<string, unknown>[];
|
||||
lines: SeriesDef[];
|
||||
rules?: Rule[];
|
||||
unit?: string;
|
||||
}) {
|
||||
const hasAny = data.some((d) => lines.some((l) => d[l.key] != null));
|
||||
if (!hasAny) return null;
|
||||
return (
|
||||
<>
|
||||
<h5 className="mt-4">
|
||||
{title}
|
||||
{unit ? <span className="text-muted fw-normal"> ({unit})</span> : null}
|
||||
</h5>
|
||||
{hint && <p className="text-muted small mb-2">{hint}</p>}
|
||||
<ResponsiveContainer width="100%" height={260}>
|
||||
<LineChart data={data} margin={{ top: 8, right: 24, bottom: 8, left: 0 }}>
|
||||
<CartesianGrid strokeDasharray="3 3" />
|
||||
<XAxis dataKey="label" />
|
||||
<YAxis />
|
||||
<Tooltip />
|
||||
<Legend />
|
||||
{(rules ?? [])
|
||||
// Only draw a rule whose build is actually on this x-axis;
|
||||
// recharts would otherwise place it at the origin and imply
|
||||
// the boundary sits before all the data.
|
||||
.filter((r) => data.some((d) => d.label === r.at))
|
||||
.map((r) => (
|
||||
<ReferenceLine
|
||||
key={`${r.at}-${r.label}`}
|
||||
x={r.at}
|
||||
stroke="#bbb"
|
||||
strokeDasharray="3 3"
|
||||
label={{
|
||||
value: r.label,
|
||||
position: "top",
|
||||
fill: "#999",
|
||||
fontSize: 11,
|
||||
}}
|
||||
/>
|
||||
))}
|
||||
{lines.map((l) => (
|
||||
<Line
|
||||
key={l.key}
|
||||
type="monotone"
|
||||
dataKey={l.key}
|
||||
name={l.name}
|
||||
stroke={l.stroke}
|
||||
strokeDasharray={l.dashed ? "5 5" : undefined}
|
||||
connectNulls
|
||||
dot={false}
|
||||
/>
|
||||
))}
|
||||
</LineChart>
|
||||
</ResponsiveContainer>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
export default function Trends() {
|
||||
const [dims, setDims] = useState<Dimensions | null>(null);
|
||||
const [model, setModel] = useState("");
|
||||
const [scenario, setScenario] = useState("");
|
||||
const [series, setSeries] = useState<SeriesPoint[]>([]);
|
||||
const [regimes, setRegimes] = useState<MeasurementRegime[]>([]);
|
||||
const [err, setErr] = useState<string | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
getDimensions()
|
||||
.then((d) => {
|
||||
setDims(d);
|
||||
if (d.models[0]) setModel(d.models[0]);
|
||||
if (d.scenarios[0]) setScenario(d.scenarios[0]);
|
||||
})
|
||||
.catch((e) => setErr(String(e)));
|
||||
// A missing/older API just means no rules — not a page failure.
|
||||
getRegimes()
|
||||
.then(setRegimes)
|
||||
.catch(() => setRegimes([]));
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (model && scenario) {
|
||||
getSeries(model, scenario)
|
||||
.then(setSeries)
|
||||
.catch((e) => setErr(String(e)));
|
||||
}
|
||||
}, [model, scenario]);
|
||||
|
||||
// Prepend the pre-helexa-bench baseline (dashed, separate keys) so it
|
||||
// anchors the timeline without being merged into the live line. Different
|
||||
// measurement regime — see baseline.ts / doc/benchmarks.md.
|
||||
const base = useMemo(() => baselineFor(model, scenario), [model, scenario]);
|
||||
const data = useMemo(
|
||||
() => [
|
||||
...base.map((p) => ({
|
||||
label: p.git_sha,
|
||||
baseTtft: p.ttft_s,
|
||||
baseDecode: p.decode_tps,
|
||||
baseTotal: p.total_s,
|
||||
})),
|
||||
...series.map((p) => ({
|
||||
label: p.git_sha,
|
||||
ttft: p.ttft_s_median,
|
||||
decode: p.decode_tps_median,
|
||||
total: p.total_s_median,
|
||||
ttftP95: p.ttft_p95_s_median,
|
||||
queueWait: p.queue_wait_ms_median,
|
||||
rejected: p.rejected_median,
|
||||
prefillTps: p.prefill_tps_median,
|
||||
reasoning: p.reasoning_tokens_median,
|
||||
cached: p.cached_tokens_median,
|
||||
completion: p.completion_tokens_median,
|
||||
tpot: p.tpot_p95_ms_median,
|
||||
})),
|
||||
],
|
||||
[series, base],
|
||||
);
|
||||
|
||||
// Divider marking the boundary between the two regimes (drawn at the
|
||||
// first live build, with baseline points to its left).
|
||||
const firstLive = series[0]?.git_sha;
|
||||
const showDivider = base.length > 0 && series.length > 0;
|
||||
|
||||
// The build where the measuring identity changed, derived from the
|
||||
// data rather than declared — a constant would go stale the moment
|
||||
// the principal is reconfigured (#288).
|
||||
const identityShift = useMemo(() => {
|
||||
for (let i = 1; i < series.length; i++) {
|
||||
const prev = series[i - 1].principal ?? "anonymous";
|
||||
const cur = series[i].principal ?? "anonymous";
|
||||
if (prev !== cur) {
|
||||
return {
|
||||
at: series[i].git_sha,
|
||||
label: `measured as ${cur === "anonymous" ? "anonymous" : "identified"}`,
|
||||
detail:
|
||||
"The identity bench measured under changed here. Anonymous " +
|
||||
"samples are subject to the #262 yield policy and understate " +
|
||||
"capacity, so points either side are not comparable (#288).",
|
||||
} as Rule;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}, [series]);
|
||||
|
||||
/** Declared boundaries touching `metric`, plus the regime-independent
|
||||
* ones that apply to every panel. */
|
||||
const rulesFor = (metric: string): Rule[] => {
|
||||
const out: Rule[] = [];
|
||||
if (showDivider && firstLive) {
|
||||
out.push({ at: firstLive, label: "bench.py → helexa-bench" });
|
||||
}
|
||||
for (const r of regimes) {
|
||||
if (r.affects.includes(metric)) {
|
||||
out.push({ at: r.first_sha, label: r.label, detail: r.detail });
|
||||
}
|
||||
}
|
||||
if (identityShift) out.push(identityShift);
|
||||
return out;
|
||||
};
|
||||
|
||||
// Anonymous and identified samples are not comparable once #262 is in
|
||||
// the build: an anonymous caller is capped below max_in_flight and
|
||||
// parked at the class gate, so it characterises the yield policy
|
||||
// rather than serving capacity. Say so rather than letting someone
|
||||
// read a step change as an engine regression (#288).
|
||||
const identities = useMemo(
|
||||
() => new Set(series.map((p) => p.principal ?? "anonymous")),
|
||||
[series],
|
||||
);
|
||||
const mixedIdentity = identities.size > 1;
|
||||
const anyAnonymous = identities.has("anonymous");
|
||||
|
||||
if (err) return <Alert variant="danger">{err}</Alert>;
|
||||
if (!dims) return <Spinner animation="border" />;
|
||||
|
||||
return (
|
||||
<>
|
||||
<h3 className="mb-3">Trends over builds</h3>
|
||||
<Row className="g-3 mb-4">
|
||||
<Picker
|
||||
label="Model"
|
||||
value={model}
|
||||
set={setModel}
|
||||
options={dims.models}
|
||||
/>
|
||||
<Picker
|
||||
label="Scenario"
|
||||
value={scenario}
|
||||
set={setScenario}
|
||||
options={dims.scenarios}
|
||||
/>
|
||||
</Row>
|
||||
|
||||
{dims.model_gpus[model] && (
|
||||
<p className="text-muted mb-3">
|
||||
Measured on <strong>{dims.model_gpus[model]}</strong>.
|
||||
</p>
|
||||
)}
|
||||
|
||||
{mixedIdentity && (
|
||||
<Alert variant="warning" className="py-2">
|
||||
<strong>Mixed measurement identity.</strong> Some builds were
|
||||
sampled anonymously and some under a principal. Since{" "}
|
||||
<code>#262</code> an anonymous caller is capped below{" "}
|
||||
<code>max_in_flight</code> and yields to identified traffic, so
|
||||
those points measure the admission policy rather than serving
|
||||
capacity. A step change across that boundary is the instrument,
|
||||
not the engine — see <code>#288</code>.
|
||||
</Alert>
|
||||
)}
|
||||
{!mixedIdentity && anyAnonymous && (
|
||||
<Alert variant="secondary" className="py-2 small">
|
||||
Sampled anonymously. Since <code>#262</code> anonymous callers are
|
||||
served from leftover capacity, so these numbers understate what an
|
||||
authenticated caller gets (<code>#288</code>).
|
||||
</Alert>
|
||||
)}
|
||||
|
||||
{data.length === 0 ? (
|
||||
<Alert variant="info">No data for this selection yet.</Alert>
|
||||
) : (
|
||||
<>
|
||||
{base.length > 0 && (
|
||||
<p className="text-muted small mb-3">
|
||||
Dashed = pre-helexa-bench baseline ({BASELINE_SOURCE}); solid =
|
||||
helexa-bench (direct to neuron). Different measurement regimes —
|
||||
see <code>doc/benchmarks.md</code>.
|
||||
</p>
|
||||
)}
|
||||
|
||||
<MetricChart
|
||||
title="decode tok/s"
|
||||
unit="higher is better"
|
||||
data={data}
|
||||
rules={rulesFor("decode")}
|
||||
lines={[
|
||||
{ key: "decode", name: "decode tok/s", stroke: "#0d6efd" },
|
||||
...(base.length > 0
|
||||
? [
|
||||
{
|
||||
key: "baseDecode",
|
||||
name: "baseline (bench.py · gateway)",
|
||||
stroke: "#888",
|
||||
dashed: true,
|
||||
},
|
||||
]
|
||||
: []),
|
||||
]}
|
||||
/>
|
||||
|
||||
<MetricChart
|
||||
title="prefill tok/s"
|
||||
unit="higher is better"
|
||||
hint="The other half of serving speed, derived from prefill_tokens / prefill_ms. A prefix-cache hit shortens prefill_ms while the token count stays whole, so a high rate here is itself the cache-hit signal."
|
||||
data={data}
|
||||
rules={rulesFor("prefillTps")}
|
||||
lines={[
|
||||
{ key: "prefillTps", name: "prefill tok/s", stroke: "#20c997" },
|
||||
]}
|
||||
/>
|
||||
|
||||
<MetricChart
|
||||
title="TTFT"
|
||||
unit="seconds, lower is better"
|
||||
hint="Median and p95 together on purpose. Under concurrency the median is dominated by whichever streams were admitted immediately; the p95 is the one that moves when a caller is made to wait. Charting only the median is how a 0.53 s → 11.77 s tail went unnoticed for a week."
|
||||
data={data}
|
||||
rules={rulesFor("ttft")}
|
||||
lines={[
|
||||
{ key: "ttft", name: "TTFT median (s)", stroke: "#dc3545" },
|
||||
{ key: "ttftP95", name: "TTFT p95 (s)", stroke: "#fd7e14" },
|
||||
...(base.length > 0
|
||||
? [
|
||||
{
|
||||
key: "baseTtft",
|
||||
name: "baseline (bench.py · gateway)",
|
||||
stroke: "#888",
|
||||
dashed: true,
|
||||
},
|
||||
]
|
||||
: []),
|
||||
]}
|
||||
/>
|
||||
|
||||
<MetricChart
|
||||
title="inter-token gap p95"
|
||||
unit="ms, lower is better"
|
||||
hint="Stream smoothness. decode tok/s is a mean over the whole window, so a stream that stalls and then catches up is indistinguishable from one that never stalled — this is the number a user feels."
|
||||
data={data}
|
||||
rules={rulesFor("tpot")}
|
||||
lines={[
|
||||
{ key: "tpot", name: "inter-token p95 (ms)", stroke: "#6f42c1" },
|
||||
]}
|
||||
/>
|
||||
|
||||
<MetricChart
|
||||
title="admission"
|
||||
unit="queue wait ms · requests shed"
|
||||
hint="Separates “the server is slow” from “you were queued behind someone”. Queue wait is TTFT minus server-measured prefill; rejected counts honest backpressure rather than silent failures."
|
||||
data={data}
|
||||
rules={rulesFor("queueWait")}
|
||||
lines={[
|
||||
{ key: "queueWait", name: "queue wait (ms)", stroke: "#d63384" },
|
||||
{ key: "rejected", name: "rejected (count)", stroke: "#adb5bd" },
|
||||
]}
|
||||
/>
|
||||
|
||||
<MetricChart
|
||||
title="tokens per sample"
|
||||
unit="counts"
|
||||
hint="Cost, not speed. Reasoning tokens are the dominant driver on a reasoning model and move independently of every rate above — a template or sampling change can double what the model thinks before answering while the speed charts stay flat. Cached tokens are why prefill timing varies between otherwise identical samples."
|
||||
data={data}
|
||||
rules={rulesFor("completion")}
|
||||
lines={[
|
||||
{ key: "completion", name: "completion tokens", stroke: "#0dcaf0" },
|
||||
{ key: "reasoning", name: "reasoning tokens", stroke: "#ffc107" },
|
||||
{ key: "cached", name: "cached prompt tokens", stroke: "#198754" },
|
||||
]}
|
||||
/>
|
||||
|
||||
{(() => {
|
||||
// Explain every rule actually drawn for this selection. The
|
||||
// chart label is a name; without the reason, a reader still
|
||||
// has to go and rediscover what it meant — which is the cost
|
||||
// this whole feature exists to remove.
|
||||
const shown = [
|
||||
...regimes
|
||||
.filter((r) => series.some((p) => p.git_sha === r.first_sha))
|
||||
.map((r) => ({ at: r.first_sha, label: r.label, detail: r.detail })),
|
||||
...(identityShift ? [identityShift] : []),
|
||||
];
|
||||
if (shown.length === 0) return null;
|
||||
return (
|
||||
<div className="mt-4 pt-3 border-top">
|
||||
<p className="text-muted small mb-2">
|
||||
<strong>Dashed rules mark measurement-regime changes</strong> —
|
||||
builds where a number changed meaning rather than value. A step
|
||||
across one is the instrument, not the engine.
|
||||
</p>
|
||||
<dl className="row small text-muted mb-0">
|
||||
{shown.map((r) => (
|
||||
<div key={`${r.at}-${r.label}`} className="mb-2">
|
||||
<dt className="fw-semibold">
|
||||
<code>{r.at}</code> — {r.label}
|
||||
</dt>
|
||||
<dd className="mb-0">{r.detail}</dd>
|
||||
</div>
|
||||
))}
|
||||
</dl>
|
||||
</div>
|
||||
);
|
||||
})()}
|
||||
</>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
// Mirrors the JSON served by helexa-bench's read API (crates/helexa-bench/src/api.rs).
|
||||
|
||||
export interface BuildRef {
|
||||
git_sha: string;
|
||||
build_timestamp: string | null;
|
||||
package_version: string | null;
|
||||
}
|
||||
|
||||
export interface Dimensions {
|
||||
hosts: string[];
|
||||
models: string[];
|
||||
scenarios: string[];
|
||||
builds: BuildRef[];
|
||||
/** host → GPU label, e.g. "2× RTX 5090". */
|
||||
host_gpus: Record<string, string>;
|
||||
/** model → GPU label (model maps to one host today). */
|
||||
model_gpus: Record<string, string>;
|
||||
}
|
||||
|
||||
/** Latest-SHA-per-cell medians (the report table). */
|
||||
export interface ReportRow {
|
||||
target_name: string;
|
||||
model_id: string;
|
||||
scenario_id: string;
|
||||
prompt_size_approx: number;
|
||||
git_sha: string;
|
||||
prompt_tokens: number | null;
|
||||
ttft_s_median: number | null;
|
||||
decode_tps_median: number | null;
|
||||
total_s_median: number | null;
|
||||
samples: number;
|
||||
/** Public-facing resource name (the host's GPU(s)). */
|
||||
gpu: string | null;
|
||||
}
|
||||
|
||||
/** One point in a per-build time-series for a (host, model, scenario) cell. */
|
||||
export interface SeriesPoint {
|
||||
git_sha: string;
|
||||
build_timestamp: string | null;
|
||||
package_version: string | null;
|
||||
ttft_s_median: number | null;
|
||||
decode_tps_median: number | null;
|
||||
total_s_median: number | null;
|
||||
ttft_p95_s_median: number | null;
|
||||
queue_wait_ms_median: number | null;
|
||||
rejected_median: number | null;
|
||||
prefill_tps_median: number | null;
|
||||
reasoning_tokens_median: number | null;
|
||||
cached_tokens_median: number | null;
|
||||
completion_tokens_median: number | null;
|
||||
tpot_p95_ms_median: number | null;
|
||||
/** Identity the samples were taken under, or null if anonymous.
|
||||
* Anonymous and identified points are not comparable (#288). */
|
||||
principal: string | null;
|
||||
samples: number;
|
||||
}
|
||||
|
||||
export interface RunRow {
|
||||
id: number;
|
||||
ts: string;
|
||||
host: string;
|
||||
/** Public-facing resource name (the host's GPU(s)). */
|
||||
gpu: string | null;
|
||||
hostname: string | null;
|
||||
git_sha: string;
|
||||
build_timestamp: string | null;
|
||||
package_version: string;
|
||||
model_id: string;
|
||||
harness: string;
|
||||
scenario_id: string;
|
||||
prompt_size_approx: number;
|
||||
prompt_tokens_actual: number | null;
|
||||
max_tokens: number;
|
||||
ttft_s: number | null;
|
||||
decode_tps: number | null;
|
||||
total_s: number | null;
|
||||
completion_tokens: number | null;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
/** A build at which a metric changed meaning rather than value (#288).
|
||||
* Served from the API so the UI and `report` cite one list. */
|
||||
export interface MeasurementRegime {
|
||||
first_sha: string;
|
||||
label: string;
|
||||
detail: string;
|
||||
/** Series keys this affects, as Trends.tsx names them. */
|
||||
affects: string[];
|
||||
}
|
||||
9
bench/src/vite-env.d.ts
vendored
9
bench/src/vite-env.d.ts
vendored
@@ -1,9 +0,0 @@
|
||||
/// <reference types="vite/client" />
|
||||
|
||||
interface ImportMetaEnv {
|
||||
/** Base origin of the bench API. Empty → use the dev proxy / same origin. */
|
||||
readonly VITE_API_BASE?: string;
|
||||
}
|
||||
interface ImportMeta {
|
||||
readonly env: ImportMetaEnv;
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"useDefineForClassFields": true,
|
||||
"lib": ["ES2022", "DOM", "DOM.Iterable"],
|
||||
"module": "ESNext",
|
||||
"skipLibCheck": true,
|
||||
"moduleResolution": "bundler",
|
||||
"allowImportingTsExtensions": true,
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"moduleDetection": "force",
|
||||
"noEmit": true,
|
||||
"jsx": "react-jsx",
|
||||
"strict": true,
|
||||
"noUnusedLocals": true,
|
||||
"noUnusedParameters": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"types": ["node", "vite/client"]
|
||||
},
|
||||
"include": ["src", "vite.config.ts"]
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
import { defineConfig } from "vite";
|
||||
import react from "@vitejs/plugin-react-swc";
|
||||
|
||||
// Dev server proxies /api to the bench API on bob so `fetch('/api/...')`
|
||||
// works without CORS/mixed-origin fuss during local development.
|
||||
// For a production build hosted elsewhere, set VITE_API_BASE to the bob
|
||||
// API origin (e.g. http://bob.hanzalova.internal:13132) instead.
|
||||
export default defineConfig({
|
||||
plugins: [react()],
|
||||
server: {
|
||||
proxy: {
|
||||
"/api": {
|
||||
target: "http://bob.hanzalova.internal:13132",
|
||||
changeOrigin: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -5,31 +5,20 @@
|
||||
# Environment variable overrides use CORTEX_ prefix with __ separators:
|
||||
# CORTEX_GATEWAY__LISTEN=0.0.0.0:31313
|
||||
|
||||
# Path to the model catalogue (limits, cost, pinning, aliases, feasibility).
|
||||
# Defaults to the packaged location below; uncomment to override for a
|
||||
# non-packaged / local run.
|
||||
# models_config = "/etc/cortex/models.toml"
|
||||
|
||||
# NOTE: the deployed copy of this file carries secrets — the inference
|
||||
# API keys under [[entitlements.keys]].
|
||||
# It must be installed root:cortex 0640 (the package's %post and
|
||||
# script/infra-setup.sh both enforce this); a world-readable 0644
|
||||
# hands every local account those credentials.
|
||||
|
||||
[gateway]
|
||||
listen = "0.0.0.0:31313"
|
||||
metrics_listen = "0.0.0.0:31314"
|
||||
|
||||
[eviction]
|
||||
strategy = "lru"
|
||||
# Restart neurons after this many load/unload cycles to defragment VRAM.
|
||||
# Restart mistralrs after this many load/unload cycles to defragment VRAM.
|
||||
# Set to 0 to disable.
|
||||
defrag_after_cycles = 50
|
||||
|
||||
# -- Nodes ---------------------------------------------------------------
|
||||
# Each [[nodes]] entry declares a neuron daemon in the fleet.
|
||||
# Models are discovered by polling the neuron's /models endpoint.
|
||||
# Pinned models (see models.toml) are never evicted.
|
||||
# Each [[nodes]] entry declares a mistral.rs instance in the fleet.
|
||||
# Models are discovered by polling the node's /v1/models endpoint.
|
||||
# Pinned models are never evicted.
|
||||
|
||||
[[nodes]]
|
||||
name = "gpu-large"
|
||||
@@ -54,62 +43,3 @@ vram_mb = 12288 # e.g. RTX 3060 (12 GB)
|
||||
pinned = [
|
||||
"your-org/embedding-model",
|
||||
]
|
||||
|
||||
# -- Entitlements (multi-tenant governance, #47) -------------------------
|
||||
# Identity + per-key token budgets. Omit this section entirely for the
|
||||
# legacy single-operator behaviour: requests are anonymous and uncapped.
|
||||
#
|
||||
# The local/static provider below is the source of truth for accounts,
|
||||
# keys, and hard caps until the upstream clearing house exists. Identity
|
||||
# rides standard bearer auth only — clients send
|
||||
# Authorization: Bearer <key>
|
||||
# no custom headers or body fields.
|
||||
|
||||
[entitlements]
|
||||
# Reject unauthenticated requests with 401 invalid_api_key. Leave false
|
||||
# (allow-anonymous) during rollout; flip to true once keys are issued.
|
||||
require_auth = false
|
||||
|
||||
# One entry per API key.
|
||||
[[entitlements.keys]]
|
||||
key = "sk-example-rolling" # the bearer token the client sends
|
||||
account_id = "team-research" # billable account (keys may share one)
|
||||
key_id = "research-ci" # stable label for ledger/metrics (optional)
|
||||
hard_cap = 5_000_000 # hard token cap over the window
|
||||
# Rolling window that resets — over-cap requests get 429 rate_limit_exceeded
|
||||
# + Retry-After, so well-behaved clients (opencode/AI SDK) back off and retry.
|
||||
window = { kind = "rolling", seconds = 3600 }
|
||||
|
||||
[[entitlements.keys]]
|
||||
key = "sk-example-balance"
|
||||
account_id = "team-research"
|
||||
key_id = "research-prepaid"
|
||||
hard_cap = 20_000_000
|
||||
# Hard balance, no reset — exhaustion returns 429 insufficient_quota
|
||||
# (the client surfaces and stops). This is the default when `window` is
|
||||
# omitted. Never 402.
|
||||
window = { kind = "balance" }
|
||||
|
||||
[[entitlements.keys]]
|
||||
key = "sk-example-infra"
|
||||
account_id = "operator"
|
||||
key_id = "infra"
|
||||
# No hard_cap → uncapped operator infra key (own fleet, own use). Still
|
||||
# metered for visibility.
|
||||
|
||||
# -- Upstream (helexa mesh) entitlements client (#57) --------------------
|
||||
# When enabled, a bearer key NOT found in [[entitlements.keys]] above is
|
||||
# validated against the helexa-upstream authority (mesh accounts), and its
|
||||
# budget is reserved/settled there. Operator-local keys (incl. the infra
|
||||
# key) never leave this process. Fail-closed: if upstream is unreachable a
|
||||
# request is refused (503 + Retry-After), never served un-authorized.
|
||||
# Disabled by default — a standalone operator runs purely local.
|
||||
[upstream]
|
||||
enabled = false
|
||||
# url = "https://upstream.helexa.ai"
|
||||
# Shared client bearer this cortex presents (maps to an operator_id
|
||||
# upstream). Override via CORTEX_UPSTREAM__BEARER in prod.
|
||||
# bearer = "replace-with-operator-client-secret"
|
||||
# timeout_secs = 5
|
||||
# How often to flush served-usage counters to upstream for reconciliation (#58).
|
||||
# served_usage_report_interval_secs = 60
|
||||
|
||||
53
cortex.spec
53
cortex.spec
@@ -1,10 +1,10 @@
|
||||
Name: cortex
|
||||
Version: 0.1.16
|
||||
Version: 0.1.12
|
||||
Release: 1%{?dist}
|
||||
Summary: Inference gateway for multi-node GPU clusters
|
||||
|
||||
License: GPL-3.0-or-later
|
||||
URL: https://git.lair.cafe/helexa/helexa
|
||||
URL: https://git.lair.cafe/helexa/cortex
|
||||
Source0: %{name}-%{version}.tar.gz
|
||||
Source1: %{name}-%{version}-vendor.tar.gz
|
||||
|
||||
@@ -21,7 +21,6 @@ BuildRequires: systemd-rpm-macros
|
||||
|
||||
Requires(pre): shadow-utils
|
||||
Requires: systemd
|
||||
Requires: firewalld-filesystem
|
||||
|
||||
# systemd-rpm-macros ships a unit dep generator that parses User=/Group=
|
||||
# from our .service file and emits Requires: user(cortex)/group(cortex).
|
||||
@@ -57,7 +56,6 @@ cargo build --release -p cortex-cli
|
||||
install -Dm755 target/release/cortex %{buildroot}%{_bindir}/cortex
|
||||
install -Dm644 data/cortex.service %{buildroot}%{_unitdir}/cortex.service
|
||||
install -Dm644 data/cortex-sysusers.conf %{buildroot}%{_sysusersdir}/cortex.conf
|
||||
install -Dm644 data/cortex-firewalld.xml %{buildroot}%{_prefix}/lib/firewalld/services/cortex.xml
|
||||
install -dm755 %{buildroot}%{_sysconfdir}/cortex
|
||||
install -Dm644 cortex.example.toml %{buildroot}%{_sysconfdir}/cortex/cortex.toml
|
||||
install -Dm644 models.example.toml %{buildroot}%{_sysconfdir}/cortex/models.toml
|
||||
@@ -67,14 +65,6 @@ install -Dm644 models.example.toml %{buildroot}%{_sysconfdir}/cortex/models.toml
|
||||
|
||||
%post
|
||||
%systemd_post cortex.service
|
||||
# The config carries secrets. %config(noreplace) keeps an existing
|
||||
# file as-is on upgrade — including a too-permissive mode from an
|
||||
# older package — so converge it here. The cortex user comes from
|
||||
# sysusers.d, applied before %post.
|
||||
if [ -f %{_sysconfdir}/cortex/cortex.toml ]; then
|
||||
chgrp cortex %{_sysconfdir}/cortex/cortex.toml >/dev/null 2>&1 || :
|
||||
chmod 0640 %{_sysconfdir}/cortex/cortex.toml >/dev/null 2>&1 || :
|
||||
fi
|
||||
|
||||
%preun
|
||||
%systemd_preun cortex.service
|
||||
@@ -82,53 +72,16 @@ fi
|
||||
%postun
|
||||
%systemd_postun_with_restart cortex.service
|
||||
|
||||
%posttrans
|
||||
# Migration: older cortex packages shipped the firewalld service as
|
||||
# `helexa-cortex` and (in some build streams) with wrong port numbers
|
||||
# (9301/9302/9304). Operators who enabled that legacy service in their
|
||||
# zone end up with the wrong-port override taking precedence over the
|
||||
# vendor `cortex.xml` now in /usr/lib/firewalld/services/. Clean up the
|
||||
# stale /etc/ override here and migrate any zone bindings to the new
|
||||
# service name.
|
||||
if [ -f /etc/firewalld/services/helexa-cortex.xml ]; then
|
||||
rm -f /etc/firewalld/services/helexa-cortex.xml
|
||||
fi
|
||||
if [ -x /usr/bin/firewall-cmd ] && /usr/bin/firewall-cmd --state >/dev/null 2>&1; then
|
||||
# Drop the legacy service name from every zone where it was enabled
|
||||
# and add the new `cortex` service in its place. Operators who never
|
||||
# ran firewall-cmd against either name see no zone change.
|
||||
for zone in $(/usr/bin/firewall-cmd --get-active-zones 2>/dev/null \
|
||||
| awk '!/^[[:space:]]/ {print $1}'); do
|
||||
if /usr/bin/firewall-cmd --permanent --zone="$zone" --query-service=helexa-cortex >/dev/null 2>&1; then
|
||||
/usr/bin/firewall-cmd --permanent --zone="$zone" --remove-service=helexa-cortex >/dev/null 2>&1 || :
|
||||
/usr/bin/firewall-cmd --permanent --zone="$zone" --add-service=cortex >/dev/null 2>&1 || :
|
||||
fi
|
||||
done
|
||||
/usr/bin/firewall-cmd --reload >/dev/null 2>&1 || :
|
||||
fi
|
||||
:
|
||||
|
||||
%files
|
||||
%license LICENSE
|
||||
%doc README.md
|
||||
%{_bindir}/cortex
|
||||
%{_unitdir}/cortex.service
|
||||
%{_sysusersdir}/cortex.conf
|
||||
%{_prefix}/lib/firewalld/services/cortex.xml
|
||||
%dir %{_sysconfdir}/cortex
|
||||
%config(noreplace) %attr(0640,root,cortex) %{_sysconfdir}/cortex/cortex.toml
|
||||
%config(noreplace) %{_sysconfdir}/cortex/cortex.toml
|
||||
%config(noreplace) %{_sysconfdir}/cortex/models.toml
|
||||
|
||||
%changelog
|
||||
* Thu Apr 16 2026 Gitea Actions <actions@git.lair.cafe> - 0.1.16-1
|
||||
- chore: ignore local deploy script
|
||||
- chore: move default ports out of common-collision ranges
|
||||
- ci: drop actions/cache for cargo registry and target
|
||||
|
||||
* Thu Apr 16 2026 Gitea Actions <actions@git.lair.cafe> - 0.1.14-1
|
||||
- ci: publish both packages to a single helexa/helexa COPR project
|
||||
- fix(rpm): rename neuron package to helexa-neuron
|
||||
- ci: commit generated %changelog entries back to main
|
||||
|
||||
* Wed Apr 15 2026 Rob Thijssen <grenade@rob.tn> - 0.1.0-1
|
||||
- Initial package
|
||||
|
||||
@@ -5,7 +5,7 @@ use tracing_subscriber::EnvFilter;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "cortex")]
|
||||
#[command(about = "Unified inference gateway for multi-node GPU clusters")]
|
||||
#[command(about = "Unified inference gateway for multi-node mistral.rs clusters")]
|
||||
#[command(version)]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
//!
|
||||
//! These mirror the `/v1/messages` format used by the Anthropic API.
|
||||
//! The gateway accepts these, translates to OpenAI format, proxies to
|
||||
//! the inference backend (neuron), then translates the response back.
|
||||
//! mistral.rs, then translates the response back.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
@@ -20,11 +20,6 @@ pub struct MessagesRequest {
|
||||
pub temperature: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub top_p: Option<f64>,
|
||||
/// Anthropic's Messages API carries `top_k` natively, so it maps
|
||||
/// straight onto the OpenAI-side field rather than being dropped in
|
||||
/// translation (#272).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub top_k: Option<usize>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stream: Option<bool>,
|
||||
#[serde(flatten)]
|
||||
|
||||
@@ -1,119 +0,0 @@
|
||||
//! Build/version metadata shared between cortex and neuron.
|
||||
//!
|
||||
//! neuron captures these facts at compile time in its `build.rs`
|
||||
//! (git SHA, enabled cargo features, rustc/candle versions, …) and
|
||||
//! serves them from `GET /version`. cortex and `helexa-bench`
|
||||
//! deserialize the same struct so a benchmark run can be attributed to
|
||||
//! the exact daemon build that produced it — not just the host's CUDA
|
||||
//! and driver versions that `/discovery` already reports.
|
||||
//!
|
||||
//! Every field beyond the always-present package version is
|
||||
//! `#[serde(default)]` so a newer reader stays compatible with an
|
||||
//! older neuron that omits a field (and vice versa) — the same
|
||||
//! forward/backward-compat discipline as
|
||||
//! [`crate::discovery::ActivationStatus`].
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Build-time identity of a neuron daemon.
|
||||
///
|
||||
/// Returned by `GET /version`. The `git_sha` is the canonical "which
|
||||
/// build is live" key — benchmark records are bucketed by it, so a
|
||||
/// regression can be pinned to a daemon change rather than a host
|
||||
/// change. When neuron is built from a source tarball with no git
|
||||
/// metadata available (and no `HELEXA_BUILD_SHA` injected by CI/RPM),
|
||||
/// `git_sha` is the string `"unknown"`.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct BuildInfo {
|
||||
/// Crate version from `CARGO_PKG_VERSION` (e.g. `"0.1.16"`).
|
||||
pub package_version: String,
|
||||
/// Short git SHA, or `"unknown"` when unavailable at build time.
|
||||
#[serde(default = "unknown")]
|
||||
pub git_sha: String,
|
||||
/// Full 40-char git SHA when available.
|
||||
#[serde(default)]
|
||||
pub git_sha_long: Option<String>,
|
||||
/// Whether the working tree had uncommitted changes at build time.
|
||||
/// `false` when the SHA is unknown (tarball build).
|
||||
#[serde(default)]
|
||||
pub git_dirty: bool,
|
||||
/// RFC3339 build timestamp.
|
||||
#[serde(default)]
|
||||
pub build_timestamp: Option<String>,
|
||||
/// `rustc --version` output of the compiler used.
|
||||
#[serde(default)]
|
||||
pub rustc_version: Option<String>,
|
||||
/// Cargo build profile: `"release"` or `"debug"`.
|
||||
#[serde(default)]
|
||||
pub profile: Option<String>,
|
||||
/// Target triple the binary was compiled for.
|
||||
#[serde(default)]
|
||||
pub target: Option<String>,
|
||||
/// Enabled cargo features (e.g. `["cuda", "cudnn"]`). These define
|
||||
/// the performance envelope, so they are recorded against every
|
||||
/// benchmark run.
|
||||
#[serde(default)]
|
||||
pub features: Vec<String>,
|
||||
/// Locked `candle-core` version, best-effort from `Cargo.lock`.
|
||||
#[serde(default)]
|
||||
pub candle_version: Option<String>,
|
||||
}
|
||||
|
||||
fn unknown() -> String {
|
||||
"unknown".to_string()
|
||||
}
|
||||
|
||||
impl BuildInfo {
|
||||
/// A placeholder used by non-neuron benchmark targets (and tests)
|
||||
/// that have no build metadata to report.
|
||||
pub fn unknown() -> Self {
|
||||
BuildInfo {
|
||||
package_version: env!("CARGO_PKG_VERSION").to_string(),
|
||||
git_sha: unknown(),
|
||||
git_sha_long: None,
|
||||
git_dirty: false,
|
||||
build_timestamp: None,
|
||||
rustc_version: None,
|
||||
profile: None,
|
||||
target: None,
|
||||
features: Vec::new(),
|
||||
candle_version: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn round_trips_full() {
|
||||
let info = BuildInfo {
|
||||
package_version: "0.1.16".into(),
|
||||
git_sha: "30d50d6".into(),
|
||||
git_sha_long: Some("30d50d6abc123".into()),
|
||||
git_dirty: true,
|
||||
build_timestamp: Some("2026-06-13T10:00:00+00:00".into()),
|
||||
rustc_version: Some("rustc 1.85.0".into()),
|
||||
profile: Some("release".into()),
|
||||
target: Some("x86_64-unknown-linux-gnu".into()),
|
||||
features: vec!["cuda".into(), "cudnn".into()],
|
||||
candle_version: Some("0.10.2".into()),
|
||||
};
|
||||
let json = serde_json::to_string(&info).unwrap();
|
||||
let back: BuildInfo = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(info, back);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deserializes_minimal_payload() {
|
||||
// An older neuron might send only the package version; every
|
||||
// other field must default rather than fail.
|
||||
let back: BuildInfo = serde_json::from_str(r#"{"package_version":"0.1.0"}"#).unwrap();
|
||||
assert_eq!(back.package_version, "0.1.0");
|
||||
assert_eq!(back.git_sha, "unknown");
|
||||
assert!(!back.git_dirty);
|
||||
assert!(back.features.is_empty());
|
||||
assert!(back.candle_version.is_none());
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,6 @@
|
||||
//! Model catalogue — profiles describing how to serve each model.
|
||||
|
||||
use crate::discovery::DeviceInfo;
|
||||
use crate::harness::{ModelCost, ModelLimit, SamplingOverride};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
|
||||
/// A model serving profile loaded from models.toml.
|
||||
@@ -22,104 +19,20 @@ pub struct ModelProfile {
|
||||
/// Minimum VRAM per device in MB.
|
||||
#[serde(default)]
|
||||
pub min_device_vram_mb: Option<u64>,
|
||||
/// Neurons this model is allowed to run on. Empty = anywhere its
|
||||
/// device constraints are satisfied.
|
||||
///
|
||||
/// This is an *affinity* constraint — where the model may be placed.
|
||||
/// It says nothing about whether the model may be evicted once
|
||||
/// resident; that is [`ModelProfile::residency_priority`]. The two
|
||||
/// were a single field once, which made "run only here" and "never
|
||||
/// evict here" impossible to ask for separately.
|
||||
/// Neurons where this model should never be evicted.
|
||||
#[serde(default)]
|
||||
pub pinned_on: Vec<String>,
|
||||
/// Which residency class this model belongs to when a node runs out
|
||||
/// of VRAM. A model may displace residents of its own class or
|
||||
/// below, and none above it — so models that should take turns on a
|
||||
/// node share a number, and a model is protected by being ranked
|
||||
/// *above* whatever must not evict it.
|
||||
///
|
||||
/// Unset means [`DEFAULT_RESIDENCY_PRIORITY`], except for profiles
|
||||
/// carrying `pinned_on`, which default to
|
||||
/// [`PINNED_RESIDENCY_PRIORITY`] — before these were separate
|
||||
/// fields, `pinned_on` implied immunity from eviction, and a
|
||||
/// catalogue written against that meaning must not silently start
|
||||
/// allowing its flagship to be evicted.
|
||||
#[serde(default)]
|
||||
pub residency_priority: Option<u32>,
|
||||
/// Source scheme this profile's weights come from. When set, the
|
||||
/// router prefixes `id` with `scheme:` before forwarding the load
|
||||
/// request to neuron, ensuring the daemon fetches from the right
|
||||
/// registry regardless of which entry happens to match `id`.
|
||||
///
|
||||
/// `None` lets neuron substitute its own `default_source` (typically
|
||||
/// `huggingface`). Set to `"helexa"` when the model is hosted in
|
||||
/// the helexa registry — operator-procurement-grade audit relies
|
||||
/// on this being explicit per model rather than implicit.
|
||||
#[serde(default)]
|
||||
pub source: Option<String>,
|
||||
/// Operator sampling override forwarded to neuron on cold load
|
||||
/// (#283). The counterpart to the same field in a host's
|
||||
/// `[[default_models]]` — a model that can be both cold-loaded by
|
||||
/// cortex and resident on a host must carry the same value in both,
|
||||
/// which `script/check-config-consistency.py` enforces in CI.
|
||||
///
|
||||
/// This is the #252 shape of hazard: two config files describing one
|
||||
/// model, with nothing forcing them to agree.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub sampling: Option<SamplingOverride>,
|
||||
/// The model's own `preserve_thinking` template control, forwarded
|
||||
/// on cold load so a model this gateway loads behaves the same as
|
||||
/// one the host was configured to hold resident. Two config files
|
||||
/// describing one model is how #252 happened; this rides along for
|
||||
/// the same reason `sampling` does.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub preserve_thinking: Option<bool>,
|
||||
|
||||
// ── Enrichment (issue #62) ────────────────────────────────
|
||||
/// Per-model token budget. When present, advertised in `/v1/models`
|
||||
/// so clients can size and compact their context automatically.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub limit: Option<ModelLimit>,
|
||||
/// Operator-set pricing (USD per 1M tokens). `0.0` for self-hosted.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cost: Option<ModelCost>,
|
||||
/// Static capability flags the operator wants to advertise even
|
||||
/// before the model is loaded on any neuron (e.g. `"reasoning"`,
|
||||
/// `"tool_call"`). Runtime-detected capabilities from the harness
|
||||
/// are unioned with this set in the gateway's `/v1/models` response.
|
||||
#[serde(default)]
|
||||
pub capabilities: Vec<String>,
|
||||
}
|
||||
|
||||
fn default_min_devices() -> u32 {
|
||||
1
|
||||
}
|
||||
|
||||
/// Residency priority for a model that declares none. Deliberately not
|
||||
/// zero: an operator needs room to rank something *below* the ordinary
|
||||
/// case (a scratch or experimental model that should yield to anything)
|
||||
/// without editing every other entry.
|
||||
pub const DEFAULT_RESIDENCY_PRIORITY: u32 = 100;
|
||||
|
||||
/// Residency priority assumed for a profile that carries `pinned_on` but
|
||||
/// no explicit priority. High enough that nothing with a default
|
||||
/// priority can evict it, preserving the immunity `pinned_on` used to
|
||||
/// grant on its own.
|
||||
pub const PINNED_RESIDENCY_PRIORITY: u32 = 1000;
|
||||
|
||||
/// The full model catalogue.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct ModelCatalogue {
|
||||
#[serde(default)]
|
||||
pub models: Vec<ModelProfile>,
|
||||
/// Tier aliases — clients can send a request with `model: "helexa/small"`
|
||||
/// and the gateway transparently rewrites + routes to the concrete
|
||||
/// model id this maps to. Lets operators define latency/quality
|
||||
/// tiers (`small`/`balanced`/`large`, `fast`/`thinking`, etc.)
|
||||
/// without imposing knowledge of specific model ids on clients.
|
||||
/// Loaded from the `[aliases]` table in models.toml.
|
||||
#[serde(default)]
|
||||
pub aliases: HashMap<String, String>,
|
||||
}
|
||||
|
||||
impl ModelCatalogue {
|
||||
@@ -145,368 +58,10 @@ impl ModelCatalogue {
|
||||
}
|
||||
}
|
||||
|
||||
/// How strongly `model_id` holds its place. Models absent from the
|
||||
/// catalogue rank at the default — a model can be resident on a
|
||||
/// neuron without a profile (loaded directly, or left over from an
|
||||
/// earlier catalogue), and treating those as unevictable would let
|
||||
/// an unlisted model wedge a node permanently.
|
||||
pub fn residency_priority(&self, model_id: &str) -> u32 {
|
||||
self.get(model_id)
|
||||
.map(|p| {
|
||||
p.residency_priority.unwrap_or({
|
||||
if p.pinned_on.is_empty() {
|
||||
DEFAULT_RESIDENCY_PRIORITY
|
||||
} else {
|
||||
PINNED_RESIDENCY_PRIORITY
|
||||
}
|
||||
})
|
||||
})
|
||||
.unwrap_or(DEFAULT_RESIDENCY_PRIORITY)
|
||||
}
|
||||
|
||||
/// May `incoming` take VRAM from `resident` when a node cannot hold
|
||||
/// both?
|
||||
///
|
||||
/// Greater-or-equal, which makes the priority a *class* rather than
|
||||
/// a strict order: a model may displace anything in its own class or
|
||||
/// below, and nothing above it.
|
||||
///
|
||||
/// Equal rank has to permit displacement, because mutual
|
||||
/// displacement is what cold-swap *is*. Two models that share a node
|
||||
/// and take turns on it — an image generator and a mid-tier text
|
||||
/// model, or two generations of the same flagship being compared —
|
||||
/// each need to evict the other on demand. Under a strict
|
||||
/// greater-than the first one to arrive would win permanently and
|
||||
/// the other could never come back, which reads as "the model
|
||||
/// vanished after we generated an image".
|
||||
///
|
||||
/// Protection therefore comes from ranking a model *above* its
|
||||
/// would-be evictor, not from ranking every model differently.
|
||||
///
|
||||
/// This governs only *whether* a displacement is permitted, never
|
||||
/// whether one is needed. A node with room for both evicts nothing,
|
||||
/// however the two rank.
|
||||
pub fn may_displace(&self, incoming_id: &str, resident_id: &str) -> bool {
|
||||
self.residency_priority(incoming_id) >= self.residency_priority(resident_id)
|
||||
}
|
||||
|
||||
/// Find a profile by model id.
|
||||
pub fn get(&self, model_id: &str) -> Option<&ModelProfile> {
|
||||
self.models.iter().find(|p| p.id == model_id)
|
||||
}
|
||||
|
||||
/// Resolve an alias to its concrete model id. Returns `id` verbatim
|
||||
/// when it isn't an alias. Aliases never chain — operator config
|
||||
/// is treated as flat — so this is a single lookup.
|
||||
pub fn resolve_alias<'a>(&'a self, id: &'a str) -> &'a str {
|
||||
self.aliases.get(id).map(String::as_str).unwrap_or(id)
|
||||
}
|
||||
}
|
||||
|
||||
impl ModelProfile {
|
||||
/// True iff this profile's placement constraints can be satisfied
|
||||
/// by the named neuron with the given device topology.
|
||||
///
|
||||
/// Constraints checked:
|
||||
/// - `pinned_on`: non-empty → neuron must be on the list.
|
||||
/// - `min_devices`: neuron must have at least this many devices.
|
||||
/// - `min_device_vram_mb`: at least `min_devices` of the neuron's
|
||||
/// devices must each meet this VRAM floor.
|
||||
pub fn is_feasible_on(&self, neuron_name: &str, devices: &[DeviceInfo]) -> bool {
|
||||
if !self.pinned_on.is_empty() && !self.pinned_on.iter().any(|n| n == neuron_name) {
|
||||
return false;
|
||||
}
|
||||
if (devices.len() as u32) < self.min_devices {
|
||||
return false;
|
||||
}
|
||||
if let Some(min_vram) = self.min_device_vram_mb {
|
||||
let big_enough = devices
|
||||
.iter()
|
||||
.filter(|d| d.vram_total_mb >= min_vram)
|
||||
.count() as u32;
|
||||
if big_enough < self.min_devices {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::discovery::DeviceInfo;
|
||||
|
||||
fn device(idx: u32, vram_mb: u64) -> DeviceInfo {
|
||||
DeviceInfo {
|
||||
index: idx,
|
||||
name: format!("DEV-{idx}"),
|
||||
vram_total_mb: vram_mb,
|
||||
compute_capability: "8.6".into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn profile() -> ModelProfile {
|
||||
ModelProfile {
|
||||
id: "Qwen/Qwen3.6-27B".into(),
|
||||
harness: "candle".into(),
|
||||
quant: None,
|
||||
vram_mb: Some(45_000),
|
||||
min_devices: 2,
|
||||
min_device_vram_mb: Some(24_000),
|
||||
pinned_on: vec![],
|
||||
residency_priority: None,
|
||||
source: None,
|
||||
sampling: None,
|
||||
preserve_thinking: None,
|
||||
limit: None,
|
||||
cost: None,
|
||||
capabilities: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn feasible_when_two_devices_meet_vram_floor() {
|
||||
let p = profile();
|
||||
let devices = [device(0, 32_000), device(1, 32_000)];
|
||||
assert!(p.is_feasible_on("beast", &devices));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn infeasible_when_only_one_device() {
|
||||
let p = profile();
|
||||
let devices = [device(0, 64_000)];
|
||||
assert!(!p.is_feasible_on("benjy", &devices));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn infeasible_when_one_device_underspec() {
|
||||
let p = profile();
|
||||
let devices = [device(0, 32_000), device(1, 12_000)];
|
||||
assert!(!p.is_feasible_on("mixed", &devices));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pinned_on_excludes_other_neurons() {
|
||||
let mut p = profile();
|
||||
p.pinned_on = vec!["beast".into()];
|
||||
let devices = [device(0, 32_000), device(1, 32_000)];
|
||||
assert!(p.is_feasible_on("beast", &devices));
|
||||
assert!(!p.is_feasible_on("benjy", &devices));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_vram_floor_just_needs_min_devices() {
|
||||
let mut p = profile();
|
||||
p.min_device_vram_mb = None;
|
||||
let devices = [device(0, 1_000), device(1, 1_000)];
|
||||
assert!(p.is_feasible_on("anywhere", &devices));
|
||||
}
|
||||
|
||||
/// A catalogue shaped like a real fleet. Two residency classes: the
|
||||
/// big-node class holds a flagship and a frontier model that take
|
||||
/// turns on the one machine large enough for either; the everyday
|
||||
/// class holds an image generator and a mid-tier text model that
|
||||
/// take turns on a smaller one. Nothing in the everyday class may
|
||||
/// touch the big-node class.
|
||||
fn tiered_catalogue() -> ModelCatalogue {
|
||||
toml::from_str(
|
||||
r#"
|
||||
[[models]]
|
||||
id = "flagship"
|
||||
harness = "candle"
|
||||
pinned_on = ["big-node"]
|
||||
residency_priority = 300
|
||||
|
||||
[[models]]
|
||||
id = "frontier"
|
||||
harness = "candle"
|
||||
residency_priority = 300
|
||||
|
||||
[[models]]
|
||||
id = "image"
|
||||
harness = "candle"
|
||||
residency_priority = 200
|
||||
|
||||
[[models]]
|
||||
id = "mid"
|
||||
harness = "candle"
|
||||
residency_priority = 200
|
||||
|
||||
[[models]]
|
||||
id = "tiny"
|
||||
harness = "candle"
|
||||
residency_priority = 100
|
||||
"#,
|
||||
)
|
||||
.expect("parse tiered catalogue")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn image_generation_displaces_the_mid_tier() {
|
||||
assert!(tiered_catalogue().may_displace("image", "mid"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_mid_tier_comes_back_after_an_image_takes_its_node() {
|
||||
// The swap-back half. An image request evicts the text model;
|
||||
// the next text request must be able to evict the image model,
|
||||
// or the text tier disappears until someone restarts something.
|
||||
assert!(tiered_catalogue().may_displace("mid", "image"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_generations_of_a_flagship_can_swap_in_both_directions() {
|
||||
// Comparing a new flagship against the incumbent needs traffic
|
||||
// to move each way on demand. A strict order would let whichever
|
||||
// arrived first hold the node permanently.
|
||||
let cat = tiered_catalogue();
|
||||
assert!(cat.may_displace("frontier", "flagship"));
|
||||
assert!(cat.may_displace("flagship", "frontier"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn image_generation_never_displaces_the_flagship() {
|
||||
// The image generator's device constraints alone would let it
|
||||
// land on the flagship's node, so this is the case priority
|
||||
// exists to prevent -- not a hypothetical one.
|
||||
assert!(!tiered_catalogue().may_displace("image", "flagship"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_frontier_tier_displaces_the_flagship() {
|
||||
// The requirement a boolean pin cannot express: the flagship is
|
||||
// protected from one model and not from another.
|
||||
assert!(tiered_catalogue().may_displace("frontier", "flagship"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_lower_class_cannot_displace_a_higher_one() {
|
||||
let cat = tiered_catalogue();
|
||||
assert!(!cat.may_displace("tiny", "mid"));
|
||||
assert!(!cat.may_displace("tiny", "flagship"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_higher_class_can_displace_a_lower_one() {
|
||||
assert!(tiered_catalogue().may_displace("flagship", "tiny"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pinned_on_alone_still_protects_a_catalogue_written_before_priorities() {
|
||||
// `pinned_on` used to mean "never evict here". A catalogue that
|
||||
// predates the split says nothing about priority, and must not
|
||||
// silently start allowing its flagship to be evicted.
|
||||
let cat: ModelCatalogue = toml::from_str(
|
||||
r#"
|
||||
[[models]]
|
||||
id = "flagship"
|
||||
harness = "candle"
|
||||
pinned_on = ["big-node"]
|
||||
|
||||
[[models]]
|
||||
id = "ordinary"
|
||||
harness = "candle"
|
||||
"#,
|
||||
)
|
||||
.expect("parse legacy catalogue");
|
||||
assert!(!cat.may_displace("ordinary", "flagship"));
|
||||
assert!(cat.may_displace("flagship", "ordinary"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unlisted_resident_is_displaceable_by_a_ranked_model() {
|
||||
// A model can be resident without a profile. Treating it as
|
||||
// unevictable would let an unlisted model wedge a node forever.
|
||||
let cat = tiered_catalogue();
|
||||
assert_eq!(
|
||||
cat.residency_priority("never-heard-of-it"),
|
||||
DEFAULT_RESIDENCY_PRIORITY
|
||||
);
|
||||
assert!(cat.may_displace("image", "never-heard-of-it"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn affinity_and_immunity_are_independently_expressible() {
|
||||
// The whole point of the split: confine a model to a node
|
||||
// without protecting it there, and protect one without
|
||||
// confining it anywhere.
|
||||
let cat: ModelCatalogue = toml::from_str(
|
||||
r#"
|
||||
[[models]]
|
||||
id = "confined-but-evictable"
|
||||
harness = "candle"
|
||||
pinned_on = ["big-node"]
|
||||
residency_priority = 50
|
||||
|
||||
[[models]]
|
||||
id = "roaming-but-protected"
|
||||
harness = "candle"
|
||||
residency_priority = 900
|
||||
"#,
|
||||
)
|
||||
.expect("parse catalogue");
|
||||
let devices = [device(0, 32_000)];
|
||||
|
||||
let confined = cat.get("confined-but-evictable").unwrap();
|
||||
assert!(confined.is_feasible_on("big-node", &devices));
|
||||
assert!(!confined.is_feasible_on("other-node", &devices));
|
||||
|
||||
let roaming = cat.get("roaming-but-protected").unwrap();
|
||||
assert!(roaming.is_feasible_on("other-node", &devices));
|
||||
|
||||
assert!(cat.may_displace("roaming-but-protected", "confined-but-evictable"));
|
||||
assert!(!cat.may_displace("confined-but-evictable", "roaming-but-protected"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_alias_returns_target_when_alias_present() {
|
||||
let mut cat = ModelCatalogue::default();
|
||||
cat.aliases
|
||||
.insert("helexa/small".into(), "Qwen/Qwen3-1.7B".into());
|
||||
assert_eq!(cat.resolve_alias("helexa/small"), "Qwen/Qwen3-1.7B");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_alias_passes_through_when_not_an_alias() {
|
||||
let mut cat = ModelCatalogue::default();
|
||||
cat.aliases
|
||||
.insert("helexa/small".into(), "Qwen/Qwen3-1.7B".into());
|
||||
assert_eq!(cat.resolve_alias("Qwen/Qwen3-8B"), "Qwen/Qwen3-8B");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_defaults_to_none_when_absent_from_toml() {
|
||||
let src = r#"
|
||||
[[models]]
|
||||
id = "Qwen/Qwen3-30B"
|
||||
harness = "candle"
|
||||
"#;
|
||||
let cat: ModelCatalogue = toml::from_str(src).expect("parse models table");
|
||||
assert!(cat.models[0].source.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_round_trips_through_toml() {
|
||||
let src = r#"
|
||||
[[models]]
|
||||
id = "Helexa/Qwen3.6-27B-Uncensored"
|
||||
harness = "candle"
|
||||
source = "helexa"
|
||||
"#;
|
||||
let cat: ModelCatalogue = toml::from_str(src).expect("parse models table");
|
||||
assert_eq!(cat.models[0].source.as_deref(), Some("helexa"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aliases_table_round_trips_through_toml() {
|
||||
let src = r#"
|
||||
[aliases]
|
||||
"helexa/small" = "Qwen/Qwen3-1.7B"
|
||||
"helexa/large" = "Qwen/Qwen3.6-27B"
|
||||
"#;
|
||||
let cat: ModelCatalogue = toml::from_str(src).expect("parse aliases table");
|
||||
assert_eq!(cat.resolve_alias("helexa/small"), "Qwen/Qwen3-1.7B");
|
||||
assert_eq!(cat.resolve_alias("helexa/large"), "Qwen/Qwen3.6-27B");
|
||||
/// Check if a model is pinned on a given neuron.
|
||||
pub fn is_pinned(&self, model_id: &str, neuron_name: &str) -> bool {
|
||||
self.models
|
||||
.iter()
|
||||
.any(|p| p.id == model_id && p.pinned_on.contains(&neuron_name.to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
use crate::entitlements::CapWindow;
|
||||
use figment::{
|
||||
Figment,
|
||||
providers::{Env, Format, Toml},
|
||||
@@ -12,98 +11,13 @@ pub struct GatewayConfig {
|
||||
pub eviction: EvictionSettings,
|
||||
/// Neuron endpoints (replaces old NodeConfig with static vram_mb/pinned).
|
||||
pub neurons: Vec<NeuronEndpoint>,
|
||||
/// Path to the model catalogue file. Defaults to the packaged
|
||||
/// location (`/etc/cortex/models.toml`); set explicitly for
|
||||
/// non-packaged / local runs.
|
||||
/// Path to the model catalogue file (default: "models.toml").
|
||||
#[serde(default = "default_models_path")]
|
||||
pub models_config: String,
|
||||
/// Multi-tenant governance: auth + per-key token budgets (#47). Empty
|
||||
/// by default — anonymous, uncapped — so existing single-operator
|
||||
/// setups keep working until keys are configured.
|
||||
#[serde(default)]
|
||||
pub entitlements: EntitlementsConfig,
|
||||
/// helexa-upstream client (#57). When enabled, keys not found in the
|
||||
/// local `[entitlements]` config are validated against the mesh
|
||||
/// authority, and budget is reserved/settled there. Disabled by default
|
||||
/// — a single operator runs purely local.
|
||||
#[serde(default)]
|
||||
pub upstream: UpstreamClientConfig,
|
||||
}
|
||||
|
||||
/// `[upstream]` — the helexa-upstream authority client (#57). Locally
|
||||
/// unrecognised bearer keys are resolved against `url`'s `/authz/v1` surface
|
||||
/// (mesh accounts); local keys (operator + infra) never leave the process.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct UpstreamClientConfig {
|
||||
/// Enable the upstream fallthrough. Off → purely local entitlements.
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
/// Base URL of helexa-upstream (e.g. "https://upstream.helexa.ai").
|
||||
#[serde(default)]
|
||||
pub url: String,
|
||||
/// Shared client bearer this cortex presents to `/authz/v1` (maps to an
|
||||
/// operator_id upstream). Sent as `Authorization: Bearer <bearer>`.
|
||||
#[serde(default)]
|
||||
pub bearer: String,
|
||||
/// Per-call timeout (seconds) to upstream.
|
||||
#[serde(default = "default_upstream_timeout")]
|
||||
pub timeout_secs: u64,
|
||||
/// How often (seconds) to flush served-usage counters to upstream for
|
||||
/// reconciliation (#58).
|
||||
#[serde(default = "default_served_usage_interval")]
|
||||
pub served_usage_report_interval_secs: u64,
|
||||
}
|
||||
|
||||
fn default_upstream_timeout() -> u64 {
|
||||
5
|
||||
}
|
||||
fn default_served_usage_interval() -> u64 {
|
||||
60
|
||||
}
|
||||
|
||||
/// `[entitlements]` — the local/static [`crate::entitlements::EntitlementProvider`]
|
||||
/// source of truth (#50). Accounts, keys, and hard caps live here; the
|
||||
/// future upstream client (#57) ignores this section.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct EntitlementsConfig {
|
||||
/// Reject unauthenticated requests with `401 invalid_api_key` when
|
||||
/// true. Default `false` (allow-anonymous) for dev / single-operator
|
||||
/// continuity.
|
||||
#[serde(default)]
|
||||
pub require_auth: bool,
|
||||
/// Static API keys and their budgets, consumed by the local provider.
|
||||
#[serde(default)]
|
||||
pub keys: Vec<ApiKeyConfig>,
|
||||
}
|
||||
|
||||
/// One configured API key: the bearer token, the account it bills to, and
|
||||
/// its hard cap. `[[entitlements.keys]]` in TOML.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ApiKeyConfig {
|
||||
/// The bearer token clients send in `Authorization: Bearer <key>`.
|
||||
pub key: String,
|
||||
/// Billable account. Multiple keys may share one account.
|
||||
pub account_id: String,
|
||||
/// Stable per-key identifier for ledger/metrics labels. Defaults to
|
||||
/// `account_id` when omitted, so the secret is never used as a label.
|
||||
#[serde(default)]
|
||||
pub key_id: Option<String>,
|
||||
/// Hard token cap. `None`/omitted = uncapped (e.g. operator infra key).
|
||||
#[serde(default)]
|
||||
pub hard_cap: Option<u64>,
|
||||
/// Cap-window semantics. Default: a non-resetting [`CapWindow::Balance`].
|
||||
#[serde(default)]
|
||||
pub window: CapWindow,
|
||||
}
|
||||
|
||||
fn default_models_path() -> String {
|
||||
// Absolute, so the systemd-launched binary finds the catalogue
|
||||
// regardless of its working directory. The RPM installs the catalogue
|
||||
// here (`cortex.spec`); a relative "models.toml" silently resolved to
|
||||
// the service cwd and left the catalogue empty in production
|
||||
// (pinning / aliases / limits all no-ops). Override via `models_config`
|
||||
// in cortex.toml for local runs.
|
||||
"/etc/cortex/models.toml".into()
|
||||
"models.toml".into()
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
@@ -143,48 +57,15 @@ pub struct NeuronEndpoint {
|
||||
impl GatewayConfig {
|
||||
/// Load configuration from a TOML file, with environment variable overrides.
|
||||
/// Env vars are prefixed with `CORTEX_` and use `__` as a separator.
|
||||
///
|
||||
/// A sibling `secrets.toml`, if present, is merged **after** the main
|
||||
/// file and therefore wins. It exists so the two have different
|
||||
/// owners: CI writes the main config, and only the operator writes
|
||||
/// the secrets — see [`secrets_path`].
|
||||
pub fn load(path: impl AsRef<Path>) -> Result<Self, Box<figment::Error>> {
|
||||
let path = path.as_ref();
|
||||
Figment::new()
|
||||
.merge(Toml::file(path))
|
||||
.merge(Toml::file(secrets_path(path)))
|
||||
.merge(Env::prefixed("CORTEX_").split("__"))
|
||||
.extract()
|
||||
.map_err(Box::new)
|
||||
}
|
||||
}
|
||||
|
||||
/// `secrets.toml` beside the main config.
|
||||
///
|
||||
/// The split exists to make a class of outage impossible rather than
|
||||
/// merely unlikely. Config that CI cannot write is config no check can
|
||||
/// defend: `helexa-router.toml` was excluded from git because it sat
|
||||
/// beside secret-bearing files, and its `helexa/balanced` alias then
|
||||
/// pointed at a retired model for long enough to take a node down
|
||||
/// (2026-08-27). The fix is not to hand CI the credentials — it is to
|
||||
/// stop mixing the two in one file.
|
||||
///
|
||||
/// So `cortex.toml` holds structure and is deployed by CI, while
|
||||
/// `secrets.toml` holds API keys and the upstream bearer, is written
|
||||
/// only by the operator, and is never read, written or diffed by the
|
||||
/// pipeline. Because it merges last it also wins, so a value present in
|
||||
/// both is the operator's.
|
||||
///
|
||||
/// A missing file is not an error — figment treats an absent
|
||||
/// `Toml::file` as an empty source — so a host that keeps everything in
|
||||
/// one file, and every existing deployment, keeps working unchanged.
|
||||
fn secrets_path(config: &Path) -> std::path::PathBuf {
|
||||
config
|
||||
.parent()
|
||||
.unwrap_or_else(|| Path::new("."))
|
||||
.join("secrets.toml")
|
||||
}
|
||||
|
||||
impl Default for GatewayConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
@@ -198,116 +79,6 @@ impl Default for GatewayConfig {
|
||||
},
|
||||
neurons: vec![],
|
||||
models_config: default_models_path(),
|
||||
entitlements: EntitlementsConfig::default(),
|
||||
upstream: UpstreamClientConfig::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod secrets_layer_tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
|
||||
/// Write `main` (and optionally `secrets.toml`) into a fresh directory
|
||||
/// under the target dir, and load through the real code path.
|
||||
fn load_with(dir: &str, main: &str, secrets: Option<&str>) -> GatewayConfig {
|
||||
// `CARGO_TARGET_TMPDIR` is only set for integration tests, not for
|
||||
// unit tests inside src/, so derive a unique directory instead.
|
||||
let base = std::env::temp_dir().join(format!("helexa-cortex-config-{dir}"));
|
||||
let _ = std::fs::remove_dir_all(&base);
|
||||
std::fs::create_dir_all(&base).expect("create test dir");
|
||||
let cfg = base.join("cortex.toml");
|
||||
write!(
|
||||
std::fs::File::create(&cfg).expect("create config"),
|
||||
"{main}"
|
||||
)
|
||||
.expect("write");
|
||||
if let Some(s) = secrets {
|
||||
let p = base.join("secrets.toml");
|
||||
write!(std::fs::File::create(&p).expect("create secrets"), "{s}").expect("write");
|
||||
}
|
||||
GatewayConfig::load(&cfg).expect("config loads")
|
||||
}
|
||||
|
||||
const MAIN: &str = r#"
|
||||
[gateway]
|
||||
listen = "0.0.0.0:31313"
|
||||
metrics_listen = "0.0.0.0:31314"
|
||||
[eviction]
|
||||
strategy = "lru"
|
||||
defrag_after_cycles = 50
|
||||
[entitlements]
|
||||
require_auth = true
|
||||
|
||||
[[neurons]]
|
||||
name = "beast"
|
||||
endpoint = "http://beast.invalid:13131"
|
||||
"#;
|
||||
|
||||
/// The whole point: a host with no secrets.toml behaves exactly as
|
||||
/// before. Every current deployment is in this state, so if this
|
||||
/// regressed the change would take the fleet down on rollout rather
|
||||
/// than on adoption.
|
||||
#[test]
|
||||
fn an_absent_secrets_file_is_not_an_error() {
|
||||
let cfg = load_with("secrets_absent", MAIN, None);
|
||||
assert!(cfg.entitlements.require_auth);
|
||||
assert!(
|
||||
cfg.entitlements.keys.is_empty(),
|
||||
"no keys configured anywhere means no keys"
|
||||
);
|
||||
}
|
||||
|
||||
/// The array-of-tables case that ruled out env-var overrides:
|
||||
/// `[[entitlements.keys]]` cannot be expressed as `CORTEX_*` env vars,
|
||||
/// so the second TOML layer is what carries it.
|
||||
#[test]
|
||||
fn secrets_file_supplies_the_entitlement_keys() {
|
||||
let cfg = load_with(
|
||||
"secrets_keys",
|
||||
MAIN,
|
||||
Some(
|
||||
r#"
|
||||
[[entitlements.keys]]
|
||||
key = "sk-test-aaa"
|
||||
account_id = "acct-a"
|
||||
key_id = "a"
|
||||
[[entitlements.keys]]
|
||||
key = "sk-test-bbb"
|
||||
account_id = "acct-b"
|
||||
key_id = "b"
|
||||
"#,
|
||||
),
|
||||
);
|
||||
assert_eq!(cfg.entitlements.keys.len(), 2);
|
||||
assert_eq!(cfg.entitlements.keys[0].key, "sk-test-aaa");
|
||||
assert_eq!(cfg.entitlements.keys[1].account_id, "acct-b");
|
||||
assert!(
|
||||
cfg.entitlements.require_auth,
|
||||
"structure from the CI-owned file must survive the merge"
|
||||
);
|
||||
}
|
||||
|
||||
/// Ordering is the safety property. If CI ever ships a placeholder in
|
||||
/// the main file, the operator's real value must still win — otherwise
|
||||
/// a deploy silently swaps a live credential for a dummy.
|
||||
#[test]
|
||||
fn the_operators_value_wins_over_the_deployed_one() {
|
||||
let cfg = load_with(
|
||||
"secrets_precedence",
|
||||
&format!(
|
||||
"{MAIN}\n[upstream]\nenabled = true\nurl = \"https://example.invalid\"\nbearer = \"PLACEHOLDER\"\n"
|
||||
),
|
||||
Some("[upstream]\nbearer = \"real-token\"\n"),
|
||||
);
|
||||
assert_eq!(
|
||||
cfg.upstream.bearer, "real-token",
|
||||
"secrets.toml merges last, so it wins"
|
||||
);
|
||||
assert_eq!(
|
||||
cfg.upstream.url, "https://example.invalid",
|
||||
"fields the operator does not override keep the deployed value"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,23 +22,6 @@ pub struct DiscoveryResponse {
|
||||
pub driver_version: Option<String>,
|
||||
pub devices: Vec<DeviceInfo>,
|
||||
pub harnesses: Vec<String>,
|
||||
/// Set when the host has an NVIDIA stack that is currently
|
||||
/// unusable — specifically the userspace↔kernel-module version
|
||||
/// skew after an un-rebooted driver update ("Driver/library
|
||||
/// version mismatch"), where every CUDA call including nvidia-smi
|
||||
/// fails (#19). `None` on healthy hosts AND on hosts with no
|
||||
/// NVIDIA stack at all (CPU-only is not an error). Carries an
|
||||
/// operator-actionable description; cortex can read it to route
|
||||
/// around the node instead of cold-loading into a guaranteed
|
||||
/// failure.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cuda_unavailable_reason: Option<String>,
|
||||
/// The neuron's effective maximum prompt size in tokens
|
||||
/// (`NEURON_MAX_PROMPT_TOKENS`) — the enforced prompt cap on this
|
||||
/// host. `#[serde(default)]` (→ 0) for forward-compat with neurons
|
||||
/// that predate this field; cortex treats 0 as "unknown".
|
||||
#[serde(default)]
|
||||
pub max_prompt_tokens: u64,
|
||||
}
|
||||
|
||||
/// Runtime health metrics for a single GPU device.
|
||||
@@ -53,224 +36,8 @@ pub struct DeviceHealth {
|
||||
|
||||
/// Runtime health response from a neuron endpoint.
|
||||
/// Returned by `GET /health`.
|
||||
///
|
||||
/// `activation` was added in 2026-05-26 to distinguish "process is up
|
||||
/// and reachable" from "process is ready to serve traffic". A `Type=simple`
|
||||
/// systemd unit reports `active` the moment the binary starts — but a
|
||||
/// neuron whose `default_models` list takes minutes to materialise
|
||||
/// won't bind its listener (or, in the new flow, won't have any models
|
||||
/// loaded) until pre-warm completes. The new field is `#[serde(default)]`
|
||||
/// so a pre-2026-05-26 gateway polling a new neuron — or vice versa —
|
||||
/// keeps working.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct HealthResponse {
|
||||
pub uptime_secs: u64,
|
||||
pub devices: Vec<DeviceHealth>,
|
||||
#[serde(default)]
|
||||
pub activation: ActivationStatus,
|
||||
/// Per-model admission load (#53): how many requests are running vs.
|
||||
/// queued on each loaded model right now. Cortex's load-aware router
|
||||
/// (#55) reads this to spread traffic across replicas and to propagate
|
||||
/// honest backpressure. `#[serde(default)]` keeps older gateways/neurons
|
||||
/// interoperable (absent → empty → treated as no load info).
|
||||
#[serde(default)]
|
||||
pub models: Vec<ModelLoad>,
|
||||
}
|
||||
|
||||
/// Live admission load for one loaded model (#53).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ModelLoad {
|
||||
pub id: String,
|
||||
/// Requests currently running (batch-1 → 0 or 1).
|
||||
pub in_flight: usize,
|
||||
/// Requests waiting in the bounded admission queue.
|
||||
pub queue_depth: usize,
|
||||
/// Admission concurrency ceiling (#137) — the denominator for
|
||||
/// saturation = `in_flight / max_in_flight`. `#[serde(default)]` (→ 0)
|
||||
/// for pre-#137 neurons; cortex treats 0 as "unknown" and skips the
|
||||
/// ceiling gauge so a rolling deploy doesn't publish a bogus 0.
|
||||
#[serde(default)]
|
||||
pub max_in_flight: usize,
|
||||
/// Admission queue capacity (#137): how many requests may wait beyond
|
||||
/// the in-flight slots before the model sheds load. `#[serde(default)]`
|
||||
/// for back-compat with pre-#137 neurons.
|
||||
#[serde(default)]
|
||||
pub max_queue_depth: usize,
|
||||
/// Cumulative requests rejected because the admission queue was full
|
||||
/// (#137), since this model loaded. The load-shedding signal; cortex
|
||||
/// publishes it as a counter. `#[serde(default)]` for back-compat.
|
||||
#[serde(default)]
|
||||
pub rejected_queue_full: u64,
|
||||
/// Cumulative requests rejected because the in-flight slot didn't free
|
||||
/// within `max_wait` (#137). `#[serde(default)]` for back-compat.
|
||||
#[serde(default)]
|
||||
pub rejected_timeout: u64,
|
||||
/// Cumulative requests rejected by the per-principal fair-share cap
|
||||
/// (#54/#137). `#[serde(default)]` for back-compat.
|
||||
#[serde(default)]
|
||||
pub rejected_per_principal: u64,
|
||||
/// Anonymous requests currently holding a seat (#262). Read against
|
||||
/// `in_flight` it says how much of this model's load is unattributable;
|
||||
/// against `anon_max_in_flight`, how close that traffic is to the
|
||||
/// ceiling that keeps it from starving identified callers.
|
||||
#[serde(default)]
|
||||
pub anon_in_flight: usize,
|
||||
/// Seats anonymous traffic may hold at once (#262); `0` = refused
|
||||
/// outright. `#[serde(default)]` (→ 0) for pre-#262 neurons, which
|
||||
/// applied no ceiling at all — cortex skips the gauge rather than
|
||||
/// publish a 0 that reads as the opposite of the truth.
|
||||
#[serde(default)]
|
||||
pub anon_max_in_flight: usize,
|
||||
/// Cumulative anonymous requests refused for want of leftover capacity
|
||||
/// (#262). Rising here with a flat `rejected_timeout` means the
|
||||
/// reservation is working, not that the model is overloaded.
|
||||
#[serde(default)]
|
||||
pub rejected_anon_yield: u64,
|
||||
/// Cumulative requests that waited for KV budget and never got it
|
||||
/// (#257) — the model is saturated by long-context sequences rather
|
||||
/// than by request count. `#[serde(default)]` for back-compat.
|
||||
#[serde(default)]
|
||||
pub rejected_kv_timeout: u64,
|
||||
/// Cumulative requests whose KV reservation exceeded the model's entire
|
||||
/// budget (#257) — prompts too long for this node at any load, not a
|
||||
/// load-shedding signal. `#[serde(default)]` for back-compat.
|
||||
#[serde(default)]
|
||||
pub rejected_kv_unservable: u64,
|
||||
/// This model's total KV budget in MiB and how much of it is currently
|
||||
/// unreserved (#257). `kv_budget_mb == 0` means the gate is disabled
|
||||
/// (CPU load, or an arch with no context profile).
|
||||
#[serde(default)]
|
||||
pub kv_budget_mb: u64,
|
||||
#[serde(default)]
|
||||
pub kv_available_mb: u64,
|
||||
/// Live prefill throughput EMA in tokens/sec (#137) — prompt tokens
|
||||
/// processed per second. `0.0` before the first sample. `#[serde(default)]`
|
||||
/// for back-compat with pre-#137 neurons.
|
||||
#[serde(default)]
|
||||
pub tok_s_prefill: f64,
|
||||
/// Live decode throughput EMA in tokens/sec (#137) — generation tokens
|
||||
/// per second, the headline capacity number. `0.0` before the first
|
||||
/// sample. `#[serde(default)]` for back-compat.
|
||||
#[serde(default)]
|
||||
pub tok_s_decode: f64,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod health_load_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn health_response_without_models_field_still_deserializes() {
|
||||
// A pre-#53 neuron's /health payload omits `models`; the gateway
|
||||
// must still parse it (serde default → empty).
|
||||
let json = r#"{"uptime_secs":42,"devices":[]}"#;
|
||||
let resp: HealthResponse = serde_json::from_str(json).expect("back-compat parse");
|
||||
assert_eq!(resp.uptime_secs, 42);
|
||||
assert!(resp.models.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn health_response_round_trips_model_load() {
|
||||
let resp = HealthResponse {
|
||||
uptime_secs: 1,
|
||||
devices: vec![],
|
||||
activation: ActivationStatus::default(),
|
||||
models: vec![ModelLoad {
|
||||
id: "Qwen/Qwen3.6-27B".into(),
|
||||
in_flight: 1,
|
||||
queue_depth: 3,
|
||||
max_in_flight: 8,
|
||||
max_queue_depth: 8,
|
||||
rejected_queue_full: 0,
|
||||
rejected_kv_timeout: 0,
|
||||
rejected_kv_unservable: 0,
|
||||
kv_budget_mb: 0,
|
||||
kv_available_mb: 0,
|
||||
rejected_timeout: 0,
|
||||
rejected_per_principal: 0,
|
||||
anon_in_flight: 2,
|
||||
anon_max_in_flight: 7,
|
||||
rejected_anon_yield: 0,
|
||||
tok_s_prefill: 0.0,
|
||||
tok_s_decode: 0.0,
|
||||
}],
|
||||
};
|
||||
let s = serde_json::to_string(&resp).unwrap();
|
||||
let back: HealthResponse = serde_json::from_str(&s).unwrap();
|
||||
assert_eq!(back.models.len(), 1);
|
||||
assert_eq!(back.models[0].in_flight, 1);
|
||||
assert_eq!(back.models[0].queue_depth, 3);
|
||||
assert_eq!(back.models[0].max_in_flight, 8);
|
||||
assert_eq!(back.models[0].max_queue_depth, 8);
|
||||
assert_eq!(back.models[0].anon_in_flight, 2);
|
||||
assert_eq!(back.models[0].anon_max_in_flight, 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_load_without_ceiling_fields_defaults_to_zero() {
|
||||
// A pre-#137 neuron omits max_in_flight/max_queue_depth; cortex must
|
||||
// still parse (serde default → 0, treated as "unknown").
|
||||
let json = r#"{"id":"m","in_flight":2,"queue_depth":0}"#;
|
||||
let m: ModelLoad = serde_json::from_str(json).expect("back-compat parse");
|
||||
assert_eq!(m.in_flight, 2);
|
||||
assert_eq!(m.max_in_flight, 0);
|
||||
assert_eq!(m.max_queue_depth, 0);
|
||||
// Likewise a pre-#262 neuron, which enforced no anonymous ceiling.
|
||||
assert_eq!(m.anon_in_flight, 0);
|
||||
assert_eq!(m.anon_max_in_flight, 0);
|
||||
assert_eq!(m.rejected_anon_yield, 0);
|
||||
}
|
||||
}
|
||||
|
||||
/// High-level activation state of the neuron daemon. The HTTP listener
|
||||
/// is bound during both states; what differs is whether the configured
|
||||
/// `default_models` have finished loading.
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ActivationState {
|
||||
/// At least one `default_models` entry is still loading. The
|
||||
/// neuron's other endpoints work, but inference against
|
||||
/// not-yet-loaded models will 404.
|
||||
PreWarming,
|
||||
/// Every `default_models` entry has either loaded or failed; the
|
||||
/// neuron is steady-state. Subsequent on-demand loads via
|
||||
/// `/models/load` don't flip back to PreWarming — that field
|
||||
/// reflects the activation-time set only.
|
||||
#[default]
|
||||
Ready,
|
||||
}
|
||||
|
||||
/// Per-model failure record surfaced in [`ActivationStatus::failed`].
|
||||
/// The error string is the rendered anyhow chain at the time of the
|
||||
/// failure; operators read it from `/health` to decide whether to
|
||||
/// retry, edit the spec, or unload+reload.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct PreWarmFailure {
|
||||
pub model_id: String,
|
||||
pub error: String,
|
||||
}
|
||||
|
||||
/// Activation-time progress snapshot. All four lists are populated by
|
||||
/// the neuron's pre-warm task and read by the `/health` handler. The
|
||||
/// snapshot is consistent: a model id appears in exactly one of
|
||||
/// `pending`, `in_progress` (as `Option<String>`), `completed`, or
|
||||
/// `failed` at any point in time.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct ActivationStatus {
|
||||
pub state: ActivationState,
|
||||
/// Model ids queued but not yet started. Empty in `Ready` state.
|
||||
#[serde(default)]
|
||||
pub pending: Vec<String>,
|
||||
/// Model id currently materialising. None when between models or
|
||||
/// in `Ready` state.
|
||||
#[serde(default)]
|
||||
pub in_progress: Option<String>,
|
||||
/// Model ids that finished loading successfully during this
|
||||
/// activation. Cleared on process restart.
|
||||
#[serde(default)]
|
||||
pub completed: Vec<String>,
|
||||
/// Model ids that failed during this activation, with the rendered
|
||||
/// error chain. Cleared on process restart.
|
||||
#[serde(default)]
|
||||
pub failed: Vec<PreWarmFailure>,
|
||||
}
|
||||
|
||||
@@ -1,152 +0,0 @@
|
||||
//! Identity and entitlement primitives for multi-tenant governance (#47).
|
||||
//!
|
||||
//! Identity is the shared substrate the whole epic hangs off:
|
||||
//! `identity (principal) → accounting (spend) → policy → enforcement`. This
|
||||
//! module defines the seam — the [`EntitlementProvider`] trait and its data
|
||||
//! types — so the local/static provider (operator-config caps, in
|
||||
//! cortex-gateway) can land the auth + per-key-cap + amplification fix
|
||||
//! *before* any upstream clearing house exists. The future helexa-upstream
|
||||
//! client (#57) is just another impl of this trait.
|
||||
//!
|
||||
//! The provider owns three jobs:
|
||||
//! 1. **resolve** a bearer key to a [`Principal`] (drives auth, #49);
|
||||
//! 2. **reserve → settle/release** token budget around a request so spend
|
||||
//! can never overshoot a hard cap under concurrency (drives budget
|
||||
//! enforcement, #52);
|
||||
//! 3. expose a [`BudgetSnapshot`] for metering/metrics (#51).
|
||||
//!
|
||||
//! [`BudgetError`] carries the cap-window semantics so the caller can pick
|
||||
//! the correct #63 rejection (`rate_limit_exceeded` + `Retry-After` for a
|
||||
//! resetting window vs `insufficient_quota` for a hard balance) without the
|
||||
//! provider knowing anything about HTTP.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Internal header carrying the resolved account id from cortex to neuron.
|
||||
/// neuron trusts these over the WireGuard link (#54); cortex **strips** any
|
||||
/// client-supplied copy before stamping the authoritative value, so a client
|
||||
/// can never assert a principal directly.
|
||||
pub const HEADER_ACCOUNT_ID: &str = "x-helexa-account-id";
|
||||
/// Internal header carrying the resolved key id from cortex to neuron.
|
||||
pub const HEADER_KEY_ID: &str = "x-helexa-key-id";
|
||||
|
||||
/// Who a request is for. Resolved once at the edge from the bearer key and
|
||||
/// carried through the request context. `account_id` is the billable owner
|
||||
/// (spendable at any operator, by decision); `key_id` identifies the
|
||||
/// specific API key for per-key hard caps and ledger/metrics labels.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct Principal {
|
||||
pub account_id: String,
|
||||
pub key_id: String,
|
||||
}
|
||||
|
||||
/// Cap-window semantics for a key's hard cap. Determines which #63 code an
|
||||
/// over-cap reservation maps to.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub enum CapWindow {
|
||||
/// Hard balance — the cap never resets. Exhaustion is permanent
|
||||
/// (`429 insufficient_quota`, no `Retry-After`).
|
||||
#[default]
|
||||
Balance,
|
||||
/// Rolling window of `seconds` that resets. Exhaustion is transient
|
||||
/// (`429 rate_limit_exceeded` + `Retry-After` until reset).
|
||||
Rolling { seconds: u64 },
|
||||
}
|
||||
|
||||
/// An outstanding budget reservation. The caller holds this opaque handle
|
||||
/// between [`EntitlementProvider::reserve`] and exactly one of
|
||||
/// [`EntitlementProvider::settle`] / [`EntitlementProvider::release`]. Not
|
||||
/// `Clone` — a reservation is consumed once.
|
||||
#[derive(Debug)]
|
||||
pub struct Reservation {
|
||||
/// Provider-local handle; opaque to the caller.
|
||||
pub id: u64,
|
||||
/// The principal this reservation belongs to.
|
||||
pub principal: Principal,
|
||||
/// Tokens reserved against the cap.
|
||||
pub reserved: u64,
|
||||
}
|
||||
|
||||
/// A point-in-time view of a key's budget, for metering and metrics (#51).
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct BudgetSnapshot {
|
||||
/// Hard cap in tokens. `None` means uncapped (e.g. an operator infra
|
||||
/// key, #58).
|
||||
pub hard_cap: Option<u64>,
|
||||
/// Settled spend in the current window.
|
||||
pub spent: u64,
|
||||
/// Sum of outstanding (un-settled) reservations.
|
||||
pub reserved: u64,
|
||||
}
|
||||
|
||||
/// Authentication failure — the bearer key could not be resolved.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum AuthError {
|
||||
/// The key is genuinely unknown → `401 invalid_api_key` (#49/#63).
|
||||
#[error("invalid or unknown API key")]
|
||||
InvalidKey,
|
||||
/// The authority that could resolve the key is unreachable (e.g. the
|
||||
/// helexa-upstream client failed, #57). Fail **closed** but distinctly:
|
||||
/// a transient outage must surface as `503 service_unavailable` +
|
||||
/// `Retry-After`, never `401` — a real key must not be rejected as
|
||||
/// invalid during an upstream blip.
|
||||
#[error("entitlement authority unavailable; retry in {retry_after_secs}s")]
|
||||
Unavailable { retry_after_secs: u64 },
|
||||
}
|
||||
|
||||
/// Why a reservation was refused. Carries enough for the caller to build the
|
||||
/// correct #63 envelope without the provider touching HTTP.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum BudgetError {
|
||||
/// A resetting window is exhausted → `429 rate_limit_exceeded` +
|
||||
/// `Retry-After: retry_after_secs`.
|
||||
#[error(
|
||||
"rolling-window budget exhausted ({requested} requested, {available} available); \
|
||||
resets in {retry_after_secs}s"
|
||||
)]
|
||||
RateLimited {
|
||||
requested: u64,
|
||||
available: u64,
|
||||
retry_after_secs: u64,
|
||||
},
|
||||
/// A hard balance is exhausted → `429 insufficient_quota` (no
|
||||
/// `Retry-After`; the client surfaces and stops). Never `402`.
|
||||
#[error("hard balance exhausted ({requested} requested, {available} available)")]
|
||||
InsufficientQuota { requested: u64, available: u64 },
|
||||
}
|
||||
|
||||
/// The seam between cortex's enforcement and whatever decides entitlement —
|
||||
/// a local/static config provider today (#50), the helexa-upstream client
|
||||
/// later (#57). All methods are async so the upstream impl can do network
|
||||
/// I/O; the local impl resolves in-process.
|
||||
#[async_trait]
|
||||
pub trait EntitlementProvider: Send + Sync {
|
||||
/// Resolve a bearer API key to its principal. `Err(InvalidKey)` for an
|
||||
/// unknown/empty key.
|
||||
async fn resolve(&self, api_key: &str) -> Result<Principal, AuthError>;
|
||||
|
||||
/// Reserve up to `max_tokens` against the principal's cap. Returns a
|
||||
/// handle on success, or a [`BudgetError`] (which the caller maps to a
|
||||
/// #63 `429`) if the reservation would exceed the cap. Reserving the
|
||||
/// *maximum* a request could consume before dispatch is what prevents
|
||||
/// overshoot under concurrency.
|
||||
async fn reserve(
|
||||
&self,
|
||||
principal: &Principal,
|
||||
max_tokens: u64,
|
||||
) -> Result<Reservation, BudgetError>;
|
||||
|
||||
/// Settle a reservation with the tokens actually consumed, releasing the
|
||||
/// unused remainder back to the cap.
|
||||
async fn settle(&self, reservation: Reservation, actual_tokens: u64);
|
||||
|
||||
/// Release a reservation in full — e.g. dispatch failed before any
|
||||
/// tokens were consumed.
|
||||
async fn release(&self, reservation: Reservation);
|
||||
|
||||
/// Current budget snapshot for a principal, for metering/metrics.
|
||||
/// `None` if the provider doesn't track this principal.
|
||||
async fn snapshot(&self, principal: &Principal) -> Option<BudgetSnapshot>;
|
||||
}
|
||||
@@ -1,257 +0,0 @@
|
||||
//! The OpenAI-standard error envelope (#60) and the rejection contract
|
||||
//! that rides on it (#63).
|
||||
//!
|
||||
//! Every non-2xx response cortex and neuron emit uses the shape
|
||||
//!
|
||||
//! ```json
|
||||
//! { "error": { "message": "...", "type": "...", "code": "...", "param": null } }
|
||||
//! ```
|
||||
//!
|
||||
//! because OpenAI-compatible clients (opencode, the AI SDK, litellm, the
|
||||
//! OpenAI SDKs) read `error.type` / `error.code` to decide what to do —
|
||||
//! most importantly `code == "context_length_exceeded"` triggers
|
||||
//! auto-compaction, and a `429` with `Retry-After` makes them back off and
|
||||
//! retry rather than surfacing an opaque failure. A flat `{"error":"..."}`
|
||||
//! string is invisible to that logic.
|
||||
//!
|
||||
//! This module is the single source of truth for that envelope. It is
|
||||
//! deliberately **axum-agnostic** — cortex-core is a pure types crate — so
|
||||
//! it carries the response as data (`status`, `body()`, `retry_after_secs`)
|
||||
//! and each HTTP crate (cortex-gateway, neuron) owns a tiny adapter that
|
||||
//! turns an [`OpenAiError`] into its framework's response type, setting the
|
||||
//! `Retry-After` header when present.
|
||||
//!
|
||||
//! Retryable conditions **must** carry `Retry-After` (per #63). The named
|
||||
//! constructors below encode that: [`OpenAiError::rate_limit_exceeded`] and
|
||||
//! [`OpenAiError::service_unavailable`] take a retry hint;
|
||||
//! [`OpenAiError::insufficient_quota`] (hard balance, no reset) and
|
||||
//! [`OpenAiError::context_length_exceeded`] / [`OpenAiError::invalid_api_key`]
|
||||
//! (permanent) do not. `402 Payment Required` is banned by the contract — use
|
||||
//! `429 insufficient_quota` for hard budget exhaustion.
|
||||
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
/// A rejection rendered in the OpenAI error envelope.
|
||||
///
|
||||
/// Build with [`OpenAiError::new`] (or a named constructor), refine with the
|
||||
/// `with_*` builders, then hand to the consuming crate's adapter to turn into
|
||||
/// an HTTP response.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct OpenAiError {
|
||||
/// HTTP status code (e.g. `401`, `429`, `503`).
|
||||
pub status: u16,
|
||||
/// Broad OpenAI category — `"invalid_request_error"`, `"api_error"`,
|
||||
/// `"rate_limit_error"`, …
|
||||
pub error_type: String,
|
||||
/// Specific machine-readable code clients key on (`"invalid_api_key"`,
|
||||
/// `"rate_limit_exceeded"`, `"context_length_exceeded"`, …). `None`
|
||||
/// renders as JSON `null`.
|
||||
pub code: Option<String>,
|
||||
/// Human-readable, actionable message.
|
||||
pub message: String,
|
||||
/// OpenAI's `param` field — the offending request parameter, if any.
|
||||
pub param: Option<String>,
|
||||
/// Seconds to advertise in the `Retry-After` header. Set only on
|
||||
/// retryable conditions; `None` means no header.
|
||||
pub retry_after_secs: Option<u64>,
|
||||
/// Diagnostic fields merged *inside* the `error` object (e.g.
|
||||
/// `prompt_len`, `max`, `free_mb`) so they don't break the envelope
|
||||
/// shape. Clients ignore unknown keys.
|
||||
pub extra: Map<String, Value>,
|
||||
}
|
||||
|
||||
impl OpenAiError {
|
||||
/// Construct an envelope with an explicit code. For a `null` code use
|
||||
/// [`OpenAiError::without_code`].
|
||||
pub fn new(
|
||||
status: u16,
|
||||
error_type: impl Into<String>,
|
||||
code: impl Into<String>,
|
||||
message: impl Into<String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
status,
|
||||
error_type: error_type.into(),
|
||||
code: Some(code.into()),
|
||||
message: message.into(),
|
||||
param: None,
|
||||
retry_after_secs: None,
|
||||
extra: Map::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Construct an envelope whose `code` is `null` (e.g. an unclassified
|
||||
/// internal error).
|
||||
pub fn without_code(
|
||||
status: u16,
|
||||
error_type: impl Into<String>,
|
||||
message: impl Into<String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
status,
|
||||
error_type: error_type.into(),
|
||||
code: None,
|
||||
message: message.into(),
|
||||
param: None,
|
||||
retry_after_secs: None,
|
||||
extra: Map::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Advertise a `Retry-After` (seconds). Use on retryable rejections.
|
||||
pub fn with_retry_after(mut self, secs: u64) -> Self {
|
||||
self.retry_after_secs = Some(secs);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the OpenAI `param` field.
|
||||
pub fn with_param(mut self, param: impl Into<String>) -> Self {
|
||||
self.param = Some(param.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Merge one diagnostic field into the error object.
|
||||
pub fn with_extra(mut self, key: impl Into<String>, value: Value) -> Self {
|
||||
self.extra.insert(key.into(), value);
|
||||
self
|
||||
}
|
||||
|
||||
/// Merge a bag of diagnostic fields into the error object.
|
||||
pub fn with_extras(mut self, extras: Map<String, Value>) -> Self {
|
||||
for (k, v) in extras {
|
||||
self.extra.insert(k, v);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Render the `{ "error": { … } }` body. Field order is irrelevant to
|
||||
/// clients (they parse JSON); the standard keys come first, then any
|
||||
/// diagnostic extras.
|
||||
pub fn body(&self) -> Value {
|
||||
let mut error = Map::new();
|
||||
error.insert("message".into(), Value::String(self.message.clone()));
|
||||
error.insert("type".into(), Value::String(self.error_type.clone()));
|
||||
error.insert(
|
||||
"code".into(),
|
||||
self.code.clone().map(Value::String).unwrap_or(Value::Null),
|
||||
);
|
||||
error.insert(
|
||||
"param".into(),
|
||||
self.param.clone().map(Value::String).unwrap_or(Value::Null),
|
||||
);
|
||||
for (k, v) in &self.extra {
|
||||
error.insert(k.clone(), v.clone());
|
||||
}
|
||||
json!({ "error": Value::Object(error) })
|
||||
}
|
||||
|
||||
// ── Named constructors for the #63 standard codes ──────────────────
|
||||
|
||||
/// `401 invalid_api_key` — missing/invalid bearer token (#49). Permanent.
|
||||
pub fn invalid_api_key(message: impl Into<String>) -> Self {
|
||||
Self::new(401, "invalid_request_error", "invalid_api_key", message)
|
||||
}
|
||||
|
||||
/// `429 rate_limit_exceeded` + `Retry-After` — transient overload,
|
||||
/// fair-share/in-flight cap, admission rejection, or a rolling budget
|
||||
/// window that resets (#52/#53/#54/#55). Clients back off and retry.
|
||||
pub fn rate_limit_exceeded(message: impl Into<String>, retry_after_secs: u64) -> Self {
|
||||
Self::new(429, "rate_limit_error", "rate_limit_exceeded", message)
|
||||
.with_retry_after(retry_after_secs)
|
||||
}
|
||||
|
||||
/// `429 insufficient_quota` — hard balance exhausted, no reset (#52).
|
||||
/// No `Retry-After`; the client surfaces and stops. (Never `402`.)
|
||||
pub fn insufficient_quota(message: impl Into<String>) -> Self {
|
||||
Self::new(429, "insufficient_quota", "insufficient_quota", message)
|
||||
}
|
||||
|
||||
/// `400 context_length_exceeded` — prompt exceeds the model's context
|
||||
/// window (#56/#60). Permanent for this request; opencode auto-compacts.
|
||||
pub fn context_length_exceeded(message: impl Into<String>) -> Self {
|
||||
Self::new(
|
||||
400,
|
||||
"invalid_request_error",
|
||||
"context_length_exceeded",
|
||||
message,
|
||||
)
|
||||
}
|
||||
|
||||
/// `503 service_unavailable` + optional `Retry-After` — transient
|
||||
/// backend unavailability (no healthy nodes, recovery, fail-closed
|
||||
/// upstream). Retryable when a hint is given.
|
||||
pub fn service_unavailable(message: impl Into<String>, retry_after_secs: Option<u64>) -> Self {
|
||||
let mut err = Self::new(503, "api_error", "service_unavailable", message);
|
||||
err.retry_after_secs = retry_after_secs;
|
||||
err
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn body_has_standard_envelope_shape() {
|
||||
let env = OpenAiError::new(429, "rate_limit_error", "rate_limit_exceeded", "slow down");
|
||||
let body = env.body();
|
||||
let error = body.get("error").and_then(Value::as_object).unwrap();
|
||||
assert_eq!(error["message"], "slow down");
|
||||
assert_eq!(error["type"], "rate_limit_error");
|
||||
assert_eq!(error["code"], "rate_limit_exceeded");
|
||||
assert_eq!(error["param"], Value::Null);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn without_code_renders_null_code() {
|
||||
let env = OpenAiError::without_code(500, "api_error", "kaboom");
|
||||
assert_eq!(env.body()["error"]["code"], Value::Null);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extras_ride_inside_the_error_object() {
|
||||
let env = OpenAiError::context_length_exceeded("too long")
|
||||
.with_extra("prompt_len", json!(60_000))
|
||||
.with_extra("max", json!(49_152));
|
||||
let error = &env.body()["error"];
|
||||
assert_eq!(error["prompt_len"], 60_000);
|
||||
assert_eq!(error["max"], 49_152);
|
||||
assert_eq!(error["code"], "context_length_exceeded");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rolling_window_rejection_carries_retry_after() {
|
||||
let env = OpenAiError::rate_limit_exceeded("budget window", 30);
|
||||
assert_eq!(env.status, 429);
|
||||
assert_eq!(env.retry_after_secs, Some(30));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hard_balance_rejection_has_no_retry_after() {
|
||||
let env = OpenAiError::insufficient_quota("out of credit");
|
||||
assert_eq!(env.status, 429);
|
||||
assert_eq!(env.code.as_deref(), Some("insufficient_quota"));
|
||||
assert_eq!(env.retry_after_secs, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn permanent_rejections_have_no_retry_after() {
|
||||
assert_eq!(OpenAiError::invalid_api_key("nope").retry_after_secs, None);
|
||||
assert_eq!(
|
||||
OpenAiError::context_length_exceeded("too long").retry_after_secs,
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn service_unavailable_retry_after_is_optional() {
|
||||
assert_eq!(
|
||||
OpenAiError::service_unavailable("recovering", Some(5)).retry_after_secs,
|
||||
Some(5)
|
||||
);
|
||||
assert_eq!(
|
||||
OpenAiError::service_unavailable("gone", None).retry_after_secs,
|
||||
None
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -9,13 +9,13 @@ use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Configuration for a harness instance on a neuron.
|
||||
///
|
||||
/// All current harnesses are in-process (candle); per-harness tuning
|
||||
/// (cache paths, device policies, etc.) lives in dedicated config
|
||||
/// blocks rather than on this struct.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct HarnessConfig {
|
||||
pub name: String,
|
||||
/// Base URL of the harness (e.g. "http://localhost:8080" for mistral.rs).
|
||||
pub endpoint: Option<String>,
|
||||
/// Systemd unit name, if the harness is managed via systemd.
|
||||
pub systemd_unit: Option<String>,
|
||||
}
|
||||
|
||||
/// Health status of a harness process.
|
||||
@@ -26,115 +26,7 @@ pub struct HarnessHealth {
|
||||
pub uptime_secs: Option<u64>,
|
||||
}
|
||||
|
||||
/// Operator-set sampling defaults for one model (#283).
|
||||
///
|
||||
/// A model's `generation_config.json` is the model authors' statement of
|
||||
/// intent, and honouring it (#272) is right by default. It is not always
|
||||
/// right for the workload the operator serves: `Qwen/Qwen3.8-27B`
|
||||
/// publishes `temperature = 1.0`, which measured a 20% structural-defect
|
||||
/// rate on ~2k-token code generation against 0/60 at `<= 0.6`
|
||||
/// (#283, p = 0.0031).
|
||||
///
|
||||
/// This is the operator's explicit, visible correction to that default —
|
||||
/// deliberately *not* a heuristic that inspects the model and guesses.
|
||||
/// A temperature guesser is a footgun; a number in a config file the
|
||||
/// operator wrote is not.
|
||||
///
|
||||
/// Precedence is **request > operator > model > built-in fallback**: the
|
||||
/// override replaces what the model published, but a caller that names a
|
||||
/// value still wins, because an explicit API parameter that is silently
|
||||
/// ignored is a contract break.
|
||||
///
|
||||
/// Every field is optional and overlays independently — setting
|
||||
/// `temperature` alone leaves the model's `top_p`/`top_k` in force.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SamplingOverride {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub temperature: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub top_p: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub top_k: Option<usize>,
|
||||
/// Multiplier applied to recently-generated tokens before sampling.
|
||||
/// `1.0` disables it; above that, recently-emitted tokens get less
|
||||
/// likely.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub repeat_penalty: Option<f32>,
|
||||
/// How many recently-generated tokens the penalty considers.
|
||||
///
|
||||
/// The reason this is operator-settable at all: the built-in default
|
||||
/// is 64, inherited from candle's chat example, and a reasoning
|
||||
/// model's think block runs to tens of thousands of tokens. Anything
|
||||
/// restated more than 64 tokens later is invisible to the penalty,
|
||||
/// which is how a 28,672-token think block can contain the same
|
||||
/// derivation five times over.
|
||||
///
|
||||
/// Raising it is a real trade, not a free win: the penalty is
|
||||
/// token-level, and source code legitimately repeats tokens
|
||||
/// constantly (`const`, `function`, brace runs). A wide window at a
|
||||
/// meaningful penalty distorts exactly the output this fleet is for.
|
||||
/// Sequence-level repetition wants a sequence-level instrument
|
||||
/// (frequency/presence penalties, or a DRY-style sampler); this knob
|
||||
/// exists so the trade can be *measured* per model rather than
|
||||
/// guessed once and frozen into a constant.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub repeat_last_n: Option<usize>,
|
||||
/// Sequence-wide one-time penalty for already-seen tokens.
|
||||
///
|
||||
/// This, not `repeat_last_n`, is the remedy Qwen prescribe for the
|
||||
/// endless repetition their models fall into during long reasoning
|
||||
/// (0..2, 1.5 when severe). It is scored over everything generated
|
||||
/// so far, so a passage restated 5,000 tokens later is still
|
||||
/// penalised — the case a trailing window cannot see.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub presence_penalty: Option<f32>,
|
||||
/// Sequence-wide penalty proportional to a token's existing count.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub frequency_penalty: Option<f32>,
|
||||
/// Pin the sampler's RNG for every request to this model.
|
||||
///
|
||||
/// Exists because parameter search needs a control: comparing two
|
||||
/// `presence_penalty` values across unseeded runs cannot separate
|
||||
/// the parameter's effect from sampling variance, and the clients
|
||||
/// that would otherwise pin it cannot — pi's request builder sends
|
||||
/// no `seed` at all, so a caller-side seed is not available on the
|
||||
/// path that matters.
|
||||
///
|
||||
/// Note what this does and does not give. It removes sampling noise
|
||||
/// as a confounder; it does not make two runs comparable
|
||||
/// token-by-token, because changing a penalty changes the logits
|
||||
/// and the sequences diverge at the first step where a token choice
|
||||
/// flips.
|
||||
///
|
||||
/// Leaving this set is a product decision, not just a testing one:
|
||||
/// every identical prompt then yields an identical answer for every
|
||||
/// caller. That is defensible — predictable and cacheable — but it
|
||||
/// removes the variation a user may expect from retrying. A request
|
||||
/// that names its own seed still wins.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub seed: Option<u64>,
|
||||
}
|
||||
|
||||
impl SamplingOverride {
|
||||
/// True when the operator set nothing — an empty `[sampling]` table
|
||||
/// must behave exactly as no table at all.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.temperature.is_none()
|
||||
&& self.top_p.is_none()
|
||||
&& self.top_k.is_none()
|
||||
&& self.repeat_penalty.is_none()
|
||||
&& self.repeat_last_n.is_none()
|
||||
&& self.presence_penalty.is_none()
|
||||
&& self.frequency_penalty.is_none()
|
||||
&& self.seed.is_none()
|
||||
}
|
||||
}
|
||||
|
||||
/// Specification for loading a model through a harness.
|
||||
///
|
||||
/// Doubles as the `[[default_models]]` entry shape in `neuron.toml`, so
|
||||
/// a field added here is available to both the load API and the
|
||||
/// operator's per-host config without a second type to keep in step.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ModelSpec {
|
||||
pub model_id: String,
|
||||
@@ -142,174 +34,6 @@ pub struct ModelSpec {
|
||||
pub quant: Option<String>,
|
||||
pub tensor_parallel: Option<u32>,
|
||||
pub devices: Option<Vec<u32>>,
|
||||
/// Operator sampling override (#283). Travels with the *model*, not
|
||||
/// the request: neuron is the only place sampling is resolved, so
|
||||
/// the override has to reach it whether the load came from cortex's
|
||||
/// catalogue or from this host's own `[[default_models]]`.
|
||||
///
|
||||
/// `#[serde(default)]` so a cortex or neuron predating this field
|
||||
/// still round-trips the spec.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub sampling: Option<SamplingOverride>,
|
||||
/// Whether prior assistant turns keep their reasoning when the chat
|
||||
/// template re-renders the conversation.
|
||||
///
|
||||
/// This is the model's own `preserve_thinking` template control, not
|
||||
/// an invention of ours. `Qwen/Qwen3.8-27B` defaults it to *true* —
|
||||
/// the whole transcript's think blocks are replayed — and `false`
|
||||
/// keeps reasoning only for turns after the last user query, i.e.
|
||||
/// the turn in progress.
|
||||
///
|
||||
/// `None` leaves the kwarg unset so the template's own default
|
||||
/// applies, which is the only safe default: the model's authors
|
||||
/// chose it, and overriding that fleet-wide on a hunch is how you
|
||||
/// ship a regression you cannot see.
|
||||
///
|
||||
/// Exposed as operator config so the premise can be A/B'd on a real
|
||||
/// workload — full replay is the larger prompt and pushes against
|
||||
/// the prefix-cache budget and the throughput-derived context
|
||||
/// ceiling, but whether it *helps* the model is a measurement, not
|
||||
/// a belief. A request's own `chat_template_kwargs` still wins.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub preserve_thinking: Option<bool>,
|
||||
}
|
||||
|
||||
/// Per-model token budget advertised by the catalogue or neuron.
|
||||
///
|
||||
/// `context` is the hard wall (the served max-seq-len). `input` is the
|
||||
/// compaction trigger — when set, opencode treats it as "usable context =
|
||||
/// input − reserved". When omitted, clients fall back to `context − output`.
|
||||
/// `output` is the maximum number of generation tokens.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ModelLimit {
|
||||
/// Hard wall — served max-seq-len in tokens.
|
||||
pub context: usize,
|
||||
/// Compaction trigger / usable input budget. When absent clients fall
|
||||
/// back to `context − output`.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input: Option<usize>,
|
||||
/// The generation budget a request gets when it names none, and the
|
||||
/// figure `input = context − output` is derived from.
|
||||
///
|
||||
/// A *reserve*, not a ceiling — a request may ask for more and be
|
||||
/// served it. Named `output` for the shape's history; the number a
|
||||
/// client should plan against is [`ModelLimit::output_ceiling`].
|
||||
pub output: usize,
|
||||
/// The largest output a request may name and be served (#278).
|
||||
///
|
||||
/// Distinct from `output`, which is the default and the KV-planning
|
||||
/// reserve. Publishing the reserve as the ceiling meant a client
|
||||
/// that trusted the advertisement asked for a fraction of what the
|
||||
/// model would happily generate — on a reasoning model, often less
|
||||
/// than its own think block costs.
|
||||
///
|
||||
/// `#[serde(default)]` so a neuron predating this field still
|
||||
/// deserializes; zero means "not advertised", and consumers fall
|
||||
/// back to `output`.
|
||||
#[serde(default, skip_serializing_if = "is_zero")]
|
||||
pub output_ceiling: usize,
|
||||
}
|
||||
|
||||
fn is_zero(v: &usize) -> bool {
|
||||
*v == 0
|
||||
}
|
||||
|
||||
/// What each reasoning effort level costs, in reasoning tokens (#223).
|
||||
///
|
||||
/// OpenAI-shaped clients can only send `minimal|low|medium|high` — the
|
||||
/// ladder is their entire vocabulary, and they cannot express a token
|
||||
/// count. So the server picks the numbers, and has to say what they are:
|
||||
/// a client choosing `low` with no idea whether that buys 2k tokens or
|
||||
/// 20k is guessing, which is the failure #274 exists to end.
|
||||
///
|
||||
/// Ordered rungs rather than a map, so the ladder reads in the order a
|
||||
/// client would climb it.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ReasoningBudgetRung {
|
||||
/// The effort level as the caller spells it on the wire.
|
||||
///
|
||||
/// Since #290 these are the rungs the **model's own chat template**
|
||||
/// accepts, discovered by rendering it, rather than a ladder the
|
||||
/// operator invented. `Qwen/Qwen3.8-27B` accepts exactly `low`,
|
||||
/// `medium` and `xhigh` and raises on anything else; we previously
|
||||
/// advertised `minimal`/`low`/`medium`/`high`, two of which the
|
||||
/// template rejects and which omitted the model's own default.
|
||||
///
|
||||
/// Advertise only rungs the template accepts: an invented name is a
|
||||
/// name a caller can send and the model will reject. Note that a
|
||||
/// client need not read this at all — pi-ai takes its levels from a
|
||||
/// static `thinkingLevelMap` in the operator's config — so this is
|
||||
/// a statement of truth, not a control surface we can rely on.
|
||||
pub effort: String,
|
||||
/// Backstop reasoning-token cap for this rung, when the deployment
|
||||
/// sets one.
|
||||
///
|
||||
/// `None` is the normal case and means "named rung, no cap": effort
|
||||
/// is expressed to the model through its template, and the number of
|
||||
/// tokens it then spends is the model's business. A cap is a safety
|
||||
/// net against a runaway think block (#223), not the mechanism by
|
||||
/// which effort is selected — enforcing effort by truncation is what
|
||||
/// #290 fixed.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub tokens: Option<usize>,
|
||||
/// Whether the model applies this rung when the caller names none.
|
||||
///
|
||||
/// Taken from the template's own `|default(...)`. Without it a
|
||||
/// caller that says nothing cannot tell what it is getting — on
|
||||
/// Qwen3.8 that silently means `xhigh`.
|
||||
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||
pub default: bool,
|
||||
/// Why this rung is not being offered right now, or `None` when it
|
||||
/// is. Absent from the wire in the normal case.
|
||||
///
|
||||
/// Availability is a *capacity* statement, not a permission: the
|
||||
/// request path does not reject an unavailable rung. A caller that
|
||||
/// insists still gets it, which keeps this from becoming a new
|
||||
/// refusal path for every existing API client. It exists so a UI can
|
||||
/// avoid offering an option that will serve the user badly, and so
|
||||
/// an integrator reading `/v1/models` can see the same thing.
|
||||
///
|
||||
/// Today the only reason is concurrency: on a node running many
|
||||
/// slots, the longest rung is reported unavailable. See
|
||||
/// `reasoning_budget::availability` for the argument and the knob.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub unavailable_reason: Option<String>,
|
||||
}
|
||||
|
||||
/// Operator-set pricing, **USD per 1,000,000 tokens, as JSON numbers**
|
||||
/// (`float`) — the models.dev/opencode `cost` convention, which is what
|
||||
/// helexa's primary client reads. NOT per-token, NOT decimal strings (that
|
||||
/// is OpenRouter's `pricing` shape, which helexa deliberately does not emit
|
||||
/// — see #68). A client must not rescale by 10⁶.
|
||||
///
|
||||
/// `cost` is sourced from the operator's `models.toml` catalogue profile and
|
||||
/// surfaced verbatim on `/v1/models`. The *absent* vs *zero* distinction is
|
||||
/// intentional and load-bearing (#68):
|
||||
/// - **`cost` absent** (the whole object omitted) — the model is **not
|
||||
/// priced**: the operator has not declared a rate. Clients should treat
|
||||
/// spend as unknown, not free.
|
||||
/// - **`cost` present with `input`/`output` = `0.0`** — the model is
|
||||
/// **intentionally free** (self-hosted, no charge). opencode renders `$0`.
|
||||
///
|
||||
/// Cache fields are optional — set them only when the backend supports a
|
||||
/// prefix-cache discount tier (relevant once cache-token reporting, #64,
|
||||
/// lands). The advertised rate here must equal the rate metering (#51) and
|
||||
/// reconciliation (#58/#59) bill against; today both read this catalogue
|
||||
/// value.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ModelCost {
|
||||
/// USD per 1M input (prompt) tokens.
|
||||
#[serde(default)]
|
||||
pub input: f64,
|
||||
/// USD per 1M output (completion) tokens.
|
||||
#[serde(default)]
|
||||
pub output: f64,
|
||||
/// USD per 1M cache-hit tokens (optional).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_read: Option<f64>,
|
||||
/// USD per 1M cache-write tokens (optional).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_write: Option<f64>,
|
||||
}
|
||||
|
||||
/// A model as reported by a harness.
|
||||
@@ -320,101 +44,19 @@ pub struct ModelInfo {
|
||||
pub status: String,
|
||||
pub devices: Vec<u32>,
|
||||
pub vram_used_mb: Option<u64>,
|
||||
/// Modalities this loaded model supports. Today: `["text"]` for
|
||||
/// text-only checkpoints, `["text", "vision"]` for vision-capable
|
||||
/// ones (Stage B7). Clients like litellm / agent0 can gate
|
||||
/// `image_url` submission on the advertised set.
|
||||
///
|
||||
/// Optional in the wire format so older clients that don't read
|
||||
/// it stay compatible. Default-empty for absent/older data, which
|
||||
/// callers can interpret as "text".
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub capabilities: Vec<String>,
|
||||
|
||||
// ── Enrichment (issue #62) ────────────────────────────────
|
||||
/// Token budget advertised by the catalogue or discovered at load time.
|
||||
/// `None` when neither the catalogue nor the loaded model can provide it.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub limit: Option<ModelLimit>,
|
||||
/// Operator-set pricing — see [`ModelCost`] for units and the
|
||||
/// absent (not priced) vs `0.0` (intentionally free) distinction.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cost: Option<ModelCost>,
|
||||
/// `true` when the model's tokenizer contains recognised tool-call
|
||||
/// marker tokens (`<tool_call>` / `<\/tool_call>` convention).
|
||||
#[serde(default)]
|
||||
pub tool_call: bool,
|
||||
/// `true` when the model's tokenizer contains recognised reasoning
|
||||
/// marker tokens (`<think>` / `<\/think>` or similar).
|
||||
#[serde(default)]
|
||||
pub reasoning: bool,
|
||||
/// The operator's `preserve_thinking` for this loaded model, or
|
||||
/// `None` when unset and the template's own default applies.
|
||||
///
|
||||
/// Advertised so a run can be *stamped* with the value it ran
|
||||
/// under. This is an A/B knob whose whole purpose is comparison
|
||||
/// between runs, and a comparison whose arms cannot be told apart
|
||||
/// afterwards is not a comparison. Recovering it from deploy
|
||||
/// history is guesswork — a config sync can land mid-session.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub preserve_thinking: Option<bool>,
|
||||
|
||||
/// Whether this node can actually serve a request for this model
|
||||
/// right now (#245).
|
||||
///
|
||||
/// `status == "loaded"` only says the weights are resident. A node
|
||||
/// can hold a model and still reject every request — most commonly
|
||||
/// because something outside neuron has eaten the device's free VRAM
|
||||
/// and the prefill floor check will fail before any device work.
|
||||
/// That happened in production: a leaked desktop compositor took a
|
||||
/// whole tier down while the node answered every poll correctly and
|
||||
/// the fleet reported itself fully healthy.
|
||||
///
|
||||
/// `None` means the node did not say — an older neuron, or a state
|
||||
/// it cannot evaluate. Callers must treat that as "assume servable",
|
||||
/// never as "unservable", or a version skew would evict the fleet
|
||||
/// from its own routing table.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub servable: Option<ModelServability>,
|
||||
/// The reasoning-effort ladder this deployment honours (#223).
|
||||
/// Absent when the model does not reason, or the harness has no
|
||||
/// budget to offer.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub reasoning_budget: Vec<ReasoningBudgetRung>,
|
||||
}
|
||||
|
||||
/// Why a loaded model can or cannot be served right now (#245).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ModelServability {
|
||||
pub ok: bool,
|
||||
/// Machine-readable cause when `ok == false` — e.g.
|
||||
/// `"insufficient_vram"`, matching the error the request would get.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub reason: Option<String>,
|
||||
/// Human-facing detail, for an operator reading a health page.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub detail: Option<String>,
|
||||
}
|
||||
|
||||
/// What an inference harness must do, from neuron's perspective.
|
||||
///
|
||||
/// All current harnesses are in-process — they share neuron's address
|
||||
/// space and lifecycle. `start`/`stop` therefore default to no-ops; a
|
||||
/// future process-supervising harness would override them.
|
||||
#[async_trait]
|
||||
pub trait Harness: Send + Sync {
|
||||
/// Human-readable name (e.g. "candle").
|
||||
/// Human-readable name (e.g. "mistralrs", "llamacpp", "comfyui").
|
||||
fn name(&self) -> &str;
|
||||
|
||||
/// Start the harness. Default no-op for in-process harnesses.
|
||||
async fn start(&self, _config: &HarnessConfig) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
/// Start the harness process if it is not already running.
|
||||
async fn start(&self, config: &HarnessConfig) -> Result<()>;
|
||||
|
||||
/// Stop the harness. Default no-op for in-process harnesses.
|
||||
async fn stop(&self) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
/// Stop the harness process gracefully.
|
||||
async fn stop(&self) -> Result<()>;
|
||||
|
||||
/// Health check. Returns the harness process status.
|
||||
async fn health(&self) -> HarnessHealth;
|
||||
|
||||
@@ -1,161 +0,0 @@
|
||||
//! OpenAI-compatible images API envelope (#200).
|
||||
//!
|
||||
//! Shared between neuron (which serves `/v1/images/generations`) and
|
||||
//! cortex (which proxies it). The shape follows OpenAI's images API —
|
||||
//! `model`/`prompt`/`n`/`size`/`response_format` — with helexa
|
||||
//! extensions (`seed`, `negative_prompt`, `guidance_scale`,
|
||||
//! `num_steps`) as sibling fields, mirroring how the chat surface
|
||||
//! carries `helexa_timing` inside `usage`.
|
||||
//!
|
||||
//! v1 constraints, enforced at the neuron: `n` must be 1,
|
||||
//! `response_format` must be `b64_json` (neuron has no object store to
|
||||
//! host `url` responses), output is always PNG.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// `POST /v1/images/generations` request body.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ImagesGenerationRequest {
|
||||
pub model: String,
|
||||
pub prompt: String,
|
||||
/// Number of images. v1 serves exactly 1; larger values are
|
||||
/// rejected with a clear error rather than silently truncated.
|
||||
#[serde(default = "default_n")]
|
||||
pub n: u32,
|
||||
/// `"WIDTHxHEIGHT"`, e.g. `"1024x1024"`. Defaults to 1024².
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub size: Option<String>,
|
||||
/// Only `"b64_json"` is served (the default). `"url"` requires an
|
||||
/// object store the fleet doesn't run.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub response_format: Option<String>,
|
||||
/// Only `"png"` is served (the default).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_format: Option<String>,
|
||||
|
||||
// ── helexa extensions ──────────────────────────────────────
|
||||
/// Fixed RNG seed for reproducible generations.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub seed: Option<u64>,
|
||||
/// Negative prompt; enables classifier-free guidance, which doubles
|
||||
/// per-step cost (and therefore the metered units, #202).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub negative_prompt: Option<String>,
|
||||
/// CFG scale, meaningful only with `negative_prompt`.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub guidance_scale: Option<f64>,
|
||||
/// Denoise steps. Defaults to the model profile's step count
|
||||
/// (9 for Z-Image-Turbo).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub num_steps: Option<usize>,
|
||||
}
|
||||
|
||||
fn default_n() -> u32 {
|
||||
1
|
||||
}
|
||||
|
||||
impl ImagesGenerationRequest {
|
||||
/// Parse `size` into `(width, height)`; defaults to 1024².
|
||||
/// Dimension *validation* (alignment, ceiling) stays server-side —
|
||||
/// this only parses the syntax.
|
||||
pub fn parse_size(&self) -> Result<(usize, usize), String> {
|
||||
let raw = self.size.as_deref().unwrap_or("1024x1024");
|
||||
let (w, h) = raw
|
||||
.split_once(['x', 'X'])
|
||||
.ok_or_else(|| format!("size '{raw}' is not of the form WIDTHxHEIGHT"))?;
|
||||
let width: usize = w
|
||||
.trim()
|
||||
.parse()
|
||||
.map_err(|_| format!("size '{raw}': width is not a number"))?;
|
||||
let height: usize = h
|
||||
.trim()
|
||||
.parse()
|
||||
.map_err(|_| format!("size '{raw}': height is not a number"))?;
|
||||
Ok((width, height))
|
||||
}
|
||||
}
|
||||
|
||||
/// `POST /v1/images/generations` response body.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ImagesGenerationResponse {
|
||||
/// Unix seconds.
|
||||
pub created: u64,
|
||||
pub data: Vec<ImageData>,
|
||||
/// Metering + timing, in the same spirit as chat's
|
||||
/// `usage.helexa_timing`.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub usage: Option<ImagesUsage>,
|
||||
}
|
||||
|
||||
/// One generated image.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ImageData {
|
||||
/// Base64-encoded PNG bytes.
|
||||
pub b64_json: String,
|
||||
}
|
||||
|
||||
/// Usage block for the images surface. `helexa_image_units` is the
|
||||
/// metered work in megapixel-steps (#202): `w × h × steps / 1e6`,
|
||||
/// doubled under CFG. cortex reads it for budget settlement; clients
|
||||
/// can read it to anticipate spend.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ImagesUsage {
|
||||
pub helexa_image_units: f64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub helexa_timing: Option<ImageTiming>,
|
||||
}
|
||||
|
||||
/// Phase timing for one generation, mirrored from the neuron's
|
||||
/// worker-side measurement.
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
|
||||
pub struct ImageTiming {
|
||||
pub encode_ms: u64,
|
||||
pub denoise_ms: u64,
|
||||
pub decode_ms: u64,
|
||||
pub steps: usize,
|
||||
/// True when classifier-free guidance ran (two forwards per step).
|
||||
pub cfg: bool,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn req(size: Option<&str>) -> ImagesGenerationRequest {
|
||||
ImagesGenerationRequest {
|
||||
model: "m".into(),
|
||||
prompt: "p".into(),
|
||||
n: 1,
|
||||
size: size.map(String::from),
|
||||
response_format: None,
|
||||
output_format: None,
|
||||
seed: None,
|
||||
negative_prompt: None,
|
||||
guidance_scale: None,
|
||||
num_steps: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_size_default_and_explicit() {
|
||||
assert_eq!(req(None).parse_size().unwrap(), (1024, 1024));
|
||||
assert_eq!(req(Some("512x768")).parse_size().unwrap(), (512, 768));
|
||||
assert_eq!(req(Some("2048X1024")).parse_size().unwrap(), (2048, 1024));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_size_rejects_garbage() {
|
||||
assert!(req(Some("1024")).parse_size().is_err());
|
||||
assert!(req(Some("axb")).parse_size().is_err());
|
||||
assert!(req(Some("1024x")).parse_size().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_deserializes_with_defaults() {
|
||||
let r: ImagesGenerationRequest =
|
||||
serde_json::from_str(r#"{"model": "m", "prompt": "a cat"}"#).unwrap();
|
||||
assert_eq!(r.n, 1);
|
||||
assert!(r.size.is_none());
|
||||
assert!(r.seed.is_none());
|
||||
}
|
||||
}
|
||||
@@ -1,15 +1,9 @@
|
||||
pub mod anthropic;
|
||||
pub mod build_info;
|
||||
pub mod catalogue;
|
||||
pub mod config;
|
||||
pub mod discovery;
|
||||
pub mod entitlements;
|
||||
pub mod error_envelope;
|
||||
pub mod harness;
|
||||
pub mod images;
|
||||
pub mod metrics;
|
||||
pub mod node;
|
||||
pub mod openai;
|
||||
pub mod responses;
|
||||
pub mod source;
|
||||
pub mod translate;
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
use crate::discovery::{ActivationStatus, DiscoveryResponse, ModelLoad};
|
||||
use crate::harness::{ModelCost, ModelLimit, ModelServability};
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
@@ -15,41 +13,6 @@ pub struct NodeState {
|
||||
/// Number of load/unload cycles since last process restart.
|
||||
pub lifecycle_cycles: u32,
|
||||
pub last_poll: Option<DateTime<Utc>>,
|
||||
/// Result of the most recent successful `GET /discovery` against
|
||||
/// this neuron. Cached forever once obtained — device topology is
|
||||
/// invariant for a given neuron process. `None` until the first
|
||||
/// successful poll. Used by the router and `/v1/models` to do
|
||||
/// catalogue × topology feasibility checks.
|
||||
pub discovery: Option<DiscoveryResponse>,
|
||||
/// Last-seen pre-warm progress from this neuron's `/health`
|
||||
/// endpoint. `None` until the first /health poll succeeds. The
|
||||
/// `/v1/models` handler reads `in_progress` + `pending` from here
|
||||
/// to synthesize `Loading` locations so clients see a catalogued
|
||||
/// model that's mid-prewarm as "loading", not "missing".
|
||||
pub activation: Option<ActivationStatus>,
|
||||
/// Last-seen per-model admission load from this neuron's `/health`
|
||||
/// (#53), keyed by model id. The router (#55) reads it to pick the
|
||||
/// least-busy replica when a model is loaded on more than one neuron.
|
||||
/// Empty until the first /health poll reports load.
|
||||
pub model_load: HashMap<String, ModelLoad>,
|
||||
/// Consecutive failed `/models` polls. The poller marks a node
|
||||
/// unhealthy only once this crosses a threshold, so a single transient
|
||||
/// miss (e.g. a neuron momentarily slow to answer while busy) doesn't
|
||||
/// yank the node — and all its models — out of routing. Reset to 0 on
|
||||
/// any successful poll.
|
||||
pub consecutive_poll_failures: u32,
|
||||
/// Last-seen per-device VRAM/utilisation readings from `/health`
|
||||
/// (#203). The router's cold-load placement reads free VRAM here so
|
||||
/// a model lands on a node that can actually hold it *now*, not
|
||||
/// just one whose total topology could. Empty until the first
|
||||
/// /health poll.
|
||||
pub device_health: Vec<crate::discovery::DeviceHealth>,
|
||||
/// The reasoning-effort ladder this neuron honours (#223), copied
|
||||
/// from its `/models` reply at poll time. Per-node rather than
|
||||
/// per-model because it is host configuration — every model on a
|
||||
/// neuron reports the same rungs — and duplicating it onto each
|
||||
/// model entry would invite the two to disagree.
|
||||
pub reasoning_budget: Vec<crate::harness::ReasoningBudgetRung>,
|
||||
}
|
||||
|
||||
/// A model registered on a node, with its runtime status.
|
||||
@@ -61,211 +24,25 @@ pub struct ModelEntry {
|
||||
pub last_accessed: Option<DateTime<Utc>>,
|
||||
/// Estimated VRAM usage in MB when loaded.
|
||||
pub vram_estimate_mb: Option<u64>,
|
||||
/// Modalities the loaded model advertises (e.g. `["text", "vision"]`),
|
||||
/// copied verbatim from the neuron's `ModelInfo.capabilities` at poll
|
||||
/// time. Empty when the neuron reports none. `#[serde(default)]` keeps
|
||||
/// older persisted/serialised entries deserialisable.
|
||||
#[serde(default)]
|
||||
pub capabilities: Vec<String>,
|
||||
/// The reasoning ladder this *model* offers, with any rung the node
|
||||
/// is currently withholding marked unavailable.
|
||||
///
|
||||
/// Per model, not per node. It was held on `NodeState` on the
|
||||
/// reasoning that "every model on a neuron reports the same rungs",
|
||||
/// which was true while a rung was only a name from the model's
|
||||
/// template. It stopped being true once availability began depending
|
||||
/// on the model's own `max_in_flight`: two models on one host can
|
||||
/// have different admission settings, and the node-level copy would
|
||||
/// attribute one model's withheld rung to the other.
|
||||
#[serde(default)]
|
||||
pub reasoning_budget: Vec<crate::harness::ReasoningBudgetRung>,
|
||||
/// Runtime-detected capability flags from the neuron's `/models`
|
||||
/// response (`ModelInfo`). `false` when the neuron predates these
|
||||
/// fields or hasn't reported them yet.
|
||||
#[serde(default)]
|
||||
pub tool_call: bool,
|
||||
#[serde(default)]
|
||||
pub reasoning: bool,
|
||||
/// Self-derived token budget the neuron computed for this loaded
|
||||
/// model (#67), copied from `ModelInfo.limit` at poll time. `None`
|
||||
/// when the neuron doesn't compute one (arch without a context
|
||||
/// profile, or derivation disabled). This is the authoritative
|
||||
/// source the gateway advertises — operator-declared catalogue
|
||||
/// limits are no longer consulted.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub limit: Option<ModelLimit>,
|
||||
/// Whether this node could actually serve the model right now
|
||||
/// (#245), from `ModelInfo.servable` at poll time.
|
||||
///
|
||||
/// `None` means the neuron expressed no opinion — an older build, a
|
||||
/// CPU load, an image model, or a cache not yet seeded. Absent is
|
||||
/// **not** unservable: see [`ModelEntry::is_servable`].
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub servable: Option<ModelServability>,
|
||||
}
|
||||
|
||||
impl ModelEntry {
|
||||
/// Whether the router should treat this location as a candidate.
|
||||
///
|
||||
/// Absent evidence means yes. A neuron that predates the field, or
|
||||
/// one that cannot evaluate its own state, must not be read as
|
||||
/// broken — a version skew would otherwise empty the routing table
|
||||
/// and take the fleet down far more comprehensively than the fault
|
||||
/// this guards against.
|
||||
pub fn is_servable(&self) -> bool {
|
||||
self.servable.as_ref().is_none_or(|s| s.ok)
|
||||
}
|
||||
}
|
||||
|
||||
/// Model lifecycle status.
|
||||
///
|
||||
/// `Loading` is a gateway-side synthetic status: neurons never emit it
|
||||
/// on `/models` (that endpoint only knows about already-loaded handles).
|
||||
/// The gateway populates it from a neuron's `/health` activation
|
||||
/// snapshot so the unified `/v1/models` can distinguish "model is
|
||||
/// catalogued but no one has it" from "model is materialising on
|
||||
/// neuron N right now". Other status values are reported verbatim by
|
||||
/// neurons.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum ModelStatus {
|
||||
Loaded,
|
||||
Unloaded,
|
||||
Reloading,
|
||||
Loading,
|
||||
/// Reported by neuron while a poisoned model auto-recovers via
|
||||
/// unload→reload (#17/#20). Temporarily unservable but NOT
|
||||
/// evicted: the gateway holds the route, answers with a transient
|
||||
/// retry error instead of 404, and must not race a second
|
||||
/// placement elsewhere.
|
||||
Recovering,
|
||||
}
|
||||
|
||||
/// Unified model entry as exposed by the gateway's `/v1/models` endpoint.
|
||||
///
|
||||
/// The first four fields (`id`, `object`, `created`, `owned_by`) match
|
||||
/// OpenAI's `/v1/models` shape verbatim, so existing OpenAI-aware
|
||||
/// tooling deserialises this without custom code. The remaining fields
|
||||
/// are helexa-specific extensions — OpenAI clients ignore unknown
|
||||
/// fields and other consumers can read them for placement / debugging.
|
||||
/// Includes which node(s) host this model and their status.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct CortexModelEntry {
|
||||
pub id: String,
|
||||
/// Always `"model"` per OpenAI's contract.
|
||||
pub object: String,
|
||||
/// Unix-second timestamp; cortex stamps this at response time.
|
||||
pub created: u64,
|
||||
/// OpenAI's "publisher" field — `"helexa"` for everything we serve.
|
||||
pub owned_by: String,
|
||||
/// True if any neuron currently has this model loaded. False for
|
||||
/// catalogue entries that are feasible but not yet loaded.
|
||||
pub loaded: bool,
|
||||
/// Neurons whose discovered topology can satisfy this model's
|
||||
/// catalogue placement constraints. Empty for models that are
|
||||
/// loaded somewhere but not present in the catalogue (cortex has
|
||||
/// no feasibility opinion on those).
|
||||
pub feasible_on: Vec<String>,
|
||||
/// Where this model is actually loaded right now. Subset of (or
|
||||
/// disjoint from) `feasible_on` depending on whether the catalogue
|
||||
/// covers this model.
|
||||
/// Which nodes have this model (and their status).
|
||||
pub locations: Vec<ModelLocation>,
|
||||
/// Union of the modalities advertised by every neuron that has this
|
||||
/// model loaded (e.g. `["text", "vision"]`). Empty for catalogue-only
|
||||
/// entries with no loaded location — filled from catalogue profile
|
||||
/// capabilities when available, then unioned with runtime-detected
|
||||
/// values from loaded neurons.
|
||||
#[serde(default)]
|
||||
pub capabilities: Vec<String>,
|
||||
// ── Enrichment (issue #62) ────────────────────────────────
|
||||
/// Per-model token budget from the catalogue profile or discovered
|
||||
/// at load time. `None` when neither source provides it.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub limit: Option<ModelLimit>,
|
||||
/// Operator-set pricing from the catalogue profile — see
|
||||
/// [`cortex_core::harness::ModelCost`] for units (USD per 1M tokens) and
|
||||
/// the absent (not priced) vs `0.0` (intentionally free) distinction.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cost: Option<ModelCost>,
|
||||
/// `true` when any neuron reports this model supports tool calls.
|
||||
#[serde(default)]
|
||||
pub tool_call: bool,
|
||||
/// `true` when any neuron reports this model supports reasoning tokens.
|
||||
#[serde(default)]
|
||||
pub reasoning: bool,
|
||||
// ── Flat ecosystem context-window fields (issue #78) ──────
|
||||
// Duplicates of `limit` under the flat, vLLM-convention key names
|
||||
// (`max_model_len` et al.) that OpenAI-ecosystem clients (Hermes
|
||||
// Agent, vLLM tooling) probe for — they cannot see `limit.context`.
|
||||
// Additive: `limit` stays the opencode-oriented source of truth.
|
||||
// Derived, never set directly — call [`sync_flat_limit`] after the
|
||||
// final `limit` value is known. Omitted (not `0`) when the window
|
||||
// is unknown; absent-vs-zero is load-bearing, as with `cost`.
|
||||
//
|
||||
// [`sync_flat_limit`]: CortexModelEntry::sync_flat_limit
|
||||
/// Served max-seq-len in tokens — mirrors `limit.context`.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_model_len: Option<usize>,
|
||||
/// The same window under the two names generic OpenAI clients look
|
||||
/// for (#278) — a client that probes for one of these rather than
|
||||
/// `max_model_len` would otherwise find no window at all and fall
|
||||
/// back to a number its operator had to invent by hand.
|
||||
///
|
||||
/// Publishing them is cheap and harmless; do not assume any
|
||||
/// *specific* client consumes them. An earlier version of this
|
||||
/// comment credited pi-ai's provider discovery, which was wrong:
|
||||
/// pi requires `contextWindow` as a static field on its model type
|
||||
/// and reads none of these keys at runtime (checked against 0.82.1
|
||||
/// and 0.84.3). The misattribution mattered — it made an operator's
|
||||
/// hand-set 80k window look like our advertisement failing to land,
|
||||
/// when in fact nothing was reading it and the number had to be
|
||||
/// maintained by hand regardless.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub context_window: Option<usize>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub context_length: Option<usize>,
|
||||
/// Usable input budget — mirrors `limit.input` when present.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_input_tokens: Option<usize>,
|
||||
/// The largest output a request may name — mirrors
|
||||
/// `limit.output_ceiling`, falling back to `limit.output` (#278).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_output_tokens: Option<usize>,
|
||||
/// What each effort level buys, in reasoning tokens (#223) — so a
|
||||
/// client picking `low` knows what it asked for.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub reasoning_budget: Vec<crate::harness::ReasoningBudgetRung>,
|
||||
}
|
||||
|
||||
impl CortexModelEntry {
|
||||
/// Re-derive the flat ecosystem fields (#78) from `limit`.
|
||||
///
|
||||
/// Must run after the final `limit` is known (post merge/tightening),
|
||||
/// immediately before serialization. Fully overwrites: a `None` limit
|
||||
/// clears the flat fields, so stale values can't survive a merge that
|
||||
/// dropped the limit.
|
||||
pub fn sync_flat_limit(&mut self) {
|
||||
self.max_model_len = self.limit.as_ref().map(|l| l.context);
|
||||
self.context_window = self.limit.as_ref().map(|l| l.context);
|
||||
self.context_length = self.limit.as_ref().map(|l| l.context);
|
||||
self.max_input_tokens = self.limit.as_ref().and_then(|l| l.input);
|
||||
// The ceiling, not the reserve (#278): a client that reads this
|
||||
// field turns it into the cap it sends on each request, so
|
||||
// advertising the reserve handed a reasoning model less budget
|
||||
// than its own think block needs. Falls back to the reserve for
|
||||
// a neuron too old to publish a ceiling.
|
||||
//
|
||||
// Observed with dsh. pi is *not* an example: its `maxTokens` is
|
||||
// a required static field in models.json and it reads nothing
|
||||
// from here — an operator has to copy this number across by
|
||||
// hand, and gets no error if they copy it wrong.
|
||||
self.max_output_tokens = self.limit.as_ref().map(|l| {
|
||||
if l.output_ceiling > 0 {
|
||||
l.output_ceiling
|
||||
} else {
|
||||
l.output
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
//! These are a subset sufficient for chat completions (streaming + non-streaming).
|
||||
//! Fields not relevant to proxying are captured as `serde_json::Value` via
|
||||
//! `#[serde(flatten)]` so we forward them without needing to enumerate every
|
||||
//! extension field a backend might support.
|
||||
//! extension field mistral.rs supports.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
@@ -18,105 +18,29 @@ pub struct ChatCompletionRequest {
|
||||
pub temperature: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub top_p: Option<f64>,
|
||||
/// Truncate sampling to the `k` most likely tokens (#272).
|
||||
///
|
||||
/// Not in the OpenAI schema but universal among serving stacks
|
||||
/// (vLLM, SGLang, llama.cpp, Ollama), and required by models that
|
||||
/// publish it — Qwen3 specifies `top_k = 20`. A named field rather
|
||||
/// than `extra`, because a sampling parameter that lands in `extra`
|
||||
/// is accepted and silently discarded.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub top_k: Option<usize>,
|
||||
/// Pin the sampler's RNG for reproducible output (OpenAI core).
|
||||
///
|
||||
/// Previously ignored on every text path, which called
|
||||
/// `unix_subsec_nanos()` regardless — a caller asking for
|
||||
/// determinism got fresh randomness and a `200`.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub seed: Option<u64>,
|
||||
/// Penalty applied to recently-generated tokens; `1.0` disables.
|
||||
/// Defaults to 1.1 when absent (#272).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub repetition_penalty: Option<f32>,
|
||||
/// How many recent tokens `repetition_penalty` considers. Defaults
|
||||
/// to 64 — fine for chat, short for a long reasoning block, which is
|
||||
/// why it is a knob rather than a constant (#272).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub repeat_last_n: Option<usize>,
|
||||
/// OpenAI core, `-2.0..=2.0`. Subtracted once from any token that
|
||||
/// has already appeared, regardless of how often.
|
||||
///
|
||||
/// Unlike `repetition_penalty` this is scored over the **whole**
|
||||
/// generated sequence, not a trailing window — which is what a
|
||||
/// reasoning model needs, and what Qwen recommend (0..2, 1.5 for
|
||||
/// severe cases) for the endless-repetition failure their own
|
||||
/// models exhibit during long thinking.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub presence_penalty: Option<f32>,
|
||||
/// OpenAI core, `-2.0..=2.0`. Subtracted in proportion to how many
|
||||
/// times a token has already appeared.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub frequency_penalty: Option<f32>,
|
||||
/// The deprecated spelling of the output cap. Still what most
|
||||
/// clients send; see [`ChatCompletionRequest::effective_max_tokens`].
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub max_tokens: Option<u64>,
|
||||
/// OpenAI's current spelling of the output cap, which deprecated
|
||||
/// `max_tokens`. Newer SDKs and clients send only this one — it has
|
||||
/// to be a named field rather than falling into `extra`, or the cap
|
||||
/// is silently ignored and generation runs to the server default.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_completion_tokens: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stream: Option<bool>,
|
||||
/// All other fields (tools, response_format, backend extensions, etc.)
|
||||
/// All other fields (tools, response_format, mistral.rs extensions, etc.)
|
||||
#[serde(flatten)]
|
||||
pub extra: Value,
|
||||
}
|
||||
|
||||
impl ChatCompletionRequest {
|
||||
/// The output cap the caller asked for, under either spelling.
|
||||
///
|
||||
/// `max_completion_tokens` wins when both are present — it is the
|
||||
/// current OpenAI field, and cortex's metering already reserves
|
||||
/// against it (`metering::requested_max_output`), so preferring it
|
||||
/// here keeps what we bill and what we generate in agreement.
|
||||
pub fn effective_max_tokens(&self) -> Option<u64> {
|
||||
self.max_completion_tokens.or(self.max_tokens)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ChatMessage {
|
||||
pub role: String,
|
||||
/// Absent on an assistant turn that carries only `tool_calls` —
|
||||
/// see [`MessageContent::Null`]. `#[serde(default)]` so a message
|
||||
/// with no `content` key at all deserializes rather than being
|
||||
/// rejected.
|
||||
#[serde(default)]
|
||||
pub content: MessageContent,
|
||||
#[serde(flatten)]
|
||||
pub extra: Value,
|
||||
}
|
||||
|
||||
/// Content can be a simple string, an array of content parts (for
|
||||
/// vision), or absent.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
/// Content can be a simple string or an array of content parts (for vision).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum MessageContent {
|
||||
Text(String),
|
||||
Parts(Vec<Value>),
|
||||
/// `"content": null`, or no `content` key at all.
|
||||
///
|
||||
/// This is the OpenAI-canonical shape for an assistant turn whose
|
||||
/// only payload is `tool_calls`, and agentic clients replay the
|
||||
/// assistant turn verbatim on the follow-up request that carries
|
||||
/// the tool result — so any client doing OpenAI-native tool
|
||||
/// calling sends it on its second turn. HF chat templates model it
|
||||
/// the same way (`content is none` renders as empty), so it maps
|
||||
/// straight through to the prompt.
|
||||
#[default]
|
||||
Null,
|
||||
}
|
||||
|
||||
// ── Chat completion response (non-streaming) ─────────────────────────
|
||||
@@ -147,18 +71,10 @@ pub struct ChatCompletionChoice {
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ChatCompletionChunk {
|
||||
#[serde(default)]
|
||||
pub id: String,
|
||||
#[serde(default)]
|
||||
pub object: String,
|
||||
#[serde(default)]
|
||||
pub created: u64,
|
||||
// Lenient deserialization throughout: the gateway parses chunks
|
||||
// from arbitrary OpenAI-compatible upstreams, and some engines
|
||||
// omit fields on special frames (e.g. usage-only final chunks).
|
||||
#[serde(default)]
|
||||
pub model: String,
|
||||
#[serde(default)]
|
||||
pub choices: Vec<ChunkChoice>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub usage: Option<Usage>,
|
||||
@@ -182,50 +98,6 @@ pub struct Usage {
|
||||
pub prompt_tokens: u64,
|
||||
pub completion_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
/// OpenAI-standard breakdown of `completion_tokens`. Optional and
|
||||
/// additive — clients that don't read it are unaffected. Carries
|
||||
/// `reasoning_tokens` for reasoning models (a sub-count of
|
||||
/// `completion_tokens`, never added into `total_tokens`).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub completion_tokens_details: Option<CompletionTokensDetails>,
|
||||
/// OpenAI-standard breakdown of `prompt_tokens`, carrying
|
||||
/// `cached_tokens` (#269). Omitted when nothing was reused, so
|
||||
/// "absent" stays distinguishable from "measured as zero".
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub prompt_tokens_details: Option<PromptTokensDetails>,
|
||||
/// helexa extension (non-OpenAI): server-measured prefill/decode
|
||||
/// timing, so the bench harness can compute true prefill vs decode
|
||||
/// tok/s instead of inferring both from client-side SSE arrival
|
||||
/// (#85). Additive and optional — standard OpenAI clients ignore
|
||||
/// it; cortex forwards usage verbatim so it survives proxying.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub helexa_timing: Option<HelexaTiming>,
|
||||
}
|
||||
|
||||
/// helexa extension carried on [`Usage::helexa_timing`]. Mirrors
|
||||
/// neuron's internal `FinishTiming`. All fields are server-measured;
|
||||
/// `prefill_tokens` is the prefill-rate denominator.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct HelexaTiming {
|
||||
pub prefill_ms: u64,
|
||||
pub decode_ms: u64,
|
||||
pub prefill_tokens: u64,
|
||||
}
|
||||
|
||||
/// Sub-counts of `Usage::completion_tokens`.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct CompletionTokensDetails {
|
||||
/// Tokens generated inside the model's reasoning span.
|
||||
pub reasoning_tokens: u64,
|
||||
}
|
||||
|
||||
/// Sub-counts of `Usage::prompt_tokens`.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct PromptTokensDetails {
|
||||
/// Prompt tokens served from neuron's prefix KV cache (#11), so a
|
||||
/// caller can see the saving rather than assume none (#269). A
|
||||
/// sub-count of `prompt_tokens`, never added into `total_tokens`.
|
||||
pub cached_tokens: u64,
|
||||
}
|
||||
|
||||
// ── Models list response ─────────────────────────────────────────────
|
||||
@@ -248,133 +120,3 @@ pub struct ModelObject {
|
||||
#[serde(flatten)]
|
||||
pub extra: Value,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The follow-up request an OpenAI-native agentic client sends
|
||||
/// after running a tool: it replays its own assistant turn, which
|
||||
/// OpenAI emits with `"content": null` because the payload was
|
||||
/// only `tool_calls`. Rejecting this shape breaks every such
|
||||
/// client on its second turn.
|
||||
#[test]
|
||||
fn assistant_tool_call_turn_with_null_content_deserializes() {
|
||||
let req: ChatCompletionRequest = serde_json::from_str(
|
||||
r#"{
|
||||
"model": "m",
|
||||
"messages": [
|
||||
{"role": "user", "content": "list the files"},
|
||||
{"role": "assistant", "content": null, "tool_calls": [
|
||||
{"id": "call_0", "type": "function",
|
||||
"function": {"name": "bash", "arguments": "{\"command\":\"ls\"}"}}
|
||||
]},
|
||||
{"role": "tool", "content": "a.txt", "tool_call_id": "call_0"}
|
||||
]
|
||||
}"#,
|
||||
)
|
||||
.expect("null assistant content is a valid OpenAI request");
|
||||
|
||||
assert!(matches!(req.messages[1].content, MessageContent::Null));
|
||||
// The tool calls survive in `extra`, where the chat template
|
||||
// reads them from.
|
||||
assert_eq!(
|
||||
req.messages[1].extra["tool_calls"][0]["function"]["name"],
|
||||
"bash"
|
||||
);
|
||||
assert_eq!(req.messages[2].extra["tool_call_id"], "call_0");
|
||||
}
|
||||
|
||||
/// Some clients omit the key entirely rather than sending null.
|
||||
#[test]
|
||||
fn assistant_turn_with_no_content_key_deserializes() {
|
||||
let msg: ChatMessage = serde_json::from_str(
|
||||
r#"{"role": "assistant", "tool_calls": [
|
||||
{"id": "c", "type": "function",
|
||||
"function": {"name": "bash", "arguments": "{}"}}]}"#,
|
||||
)
|
||||
.expect("absent content is a valid OpenAI message");
|
||||
assert!(matches!(msg.content, MessageContent::Null));
|
||||
}
|
||||
|
||||
/// Absent content round-trips back onto the wire as null, so a
|
||||
/// translated or re-serialized request stays OpenAI-shaped.
|
||||
#[test]
|
||||
fn null_content_round_trips() {
|
||||
let msg = ChatMessage {
|
||||
role: "assistant".into(),
|
||||
content: MessageContent::Null,
|
||||
extra: Value::Null,
|
||||
};
|
||||
let v = serde_json::to_value(&msg).expect("serialize");
|
||||
assert_eq!(v["content"], Value::Null);
|
||||
assert!(v.get("content").is_some(), "content key must be present");
|
||||
}
|
||||
|
||||
/// Newer OpenAI SDKs send only `max_completion_tokens`; the cap
|
||||
/// must survive as a named field rather than falling into `extra`,
|
||||
/// where every generation loop would miss it.
|
||||
#[test]
|
||||
fn max_completion_tokens_is_the_effective_cap() {
|
||||
let req: ChatCompletionRequest = serde_json::from_str(
|
||||
r#"{"model": "m", "max_completion_tokens": 240,
|
||||
"messages": [{"role": "user", "content": "hi"}]}"#,
|
||||
)
|
||||
.expect("deserialize");
|
||||
assert_eq!(req.max_completion_tokens, Some(240));
|
||||
assert_eq!(req.max_tokens, None);
|
||||
assert_eq!(req.effective_max_tokens(), Some(240));
|
||||
assert!(
|
||||
req.extra.get("max_completion_tokens").is_none(),
|
||||
"must be a named field, not swept into extra"
|
||||
);
|
||||
}
|
||||
|
||||
/// The legacy spelling keeps working on its own.
|
||||
#[test]
|
||||
fn legacy_max_tokens_still_caps() {
|
||||
let req: ChatCompletionRequest = serde_json::from_str(
|
||||
r#"{"model": "m", "max_tokens": 128,
|
||||
"messages": [{"role": "user", "content": "hi"}]}"#,
|
||||
)
|
||||
.expect("deserialize");
|
||||
assert_eq!(req.effective_max_tokens(), Some(128));
|
||||
}
|
||||
|
||||
/// Clients that send both for compatibility must not be rejected,
|
||||
/// and the current field wins — matching what cortex meters
|
||||
/// against, so billed and generated caps agree.
|
||||
#[test]
|
||||
fn both_spellings_resolve_to_the_current_field() {
|
||||
let req: ChatCompletionRequest = serde_json::from_str(
|
||||
r#"{"model": "m", "max_tokens": 99, "max_completion_tokens": 256,
|
||||
"messages": [{"role": "user", "content": "hi"}]}"#,
|
||||
)
|
||||
.expect("sending both must not be an error");
|
||||
assert_eq!(req.effective_max_tokens(), Some(256));
|
||||
}
|
||||
|
||||
/// Neither spelling present leaves the cap to the server default.
|
||||
#[test]
|
||||
fn absent_cap_is_none() {
|
||||
let req: ChatCompletionRequest = serde_json::from_str(
|
||||
r#"{"model": "m", "messages": [{"role": "user", "content": "hi"}]}"#,
|
||||
)
|
||||
.expect("deserialize");
|
||||
assert_eq!(req.effective_max_tokens(), None);
|
||||
}
|
||||
|
||||
/// The string and array forms keep working unchanged.
|
||||
#[test]
|
||||
fn text_and_parts_content_still_deserialize() {
|
||||
let text: ChatMessage =
|
||||
serde_json::from_str(r#"{"role": "user", "content": "hi"}"#).expect("text");
|
||||
assert!(matches!(text.content, MessageContent::Text(ref t) if t == "hi"));
|
||||
|
||||
let parts: ChatMessage = serde_json::from_str(
|
||||
r#"{"role": "user", "content": [{"type": "text", "text": "hi"}]}"#,
|
||||
)
|
||||
.expect("parts");
|
||||
assert!(matches!(parts.content, MessageContent::Parts(ref p) if p.len() == 1));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,664 +0,0 @@
|
||||
//! OpenAI Responses API (`POST /v1/responses`) envelope types.
|
||||
//!
|
||||
//! This is OpenAI's newer chat surface, distinct from
|
||||
//! `/v1/chat/completions` in three ways that matter for us:
|
||||
//!
|
||||
//! 1. **Input shape**. Instead of a `messages` array, the request
|
||||
//! carries `input` — either a plain string (single user turn)
|
||||
//! or an array of typed items (messages, function calls,
|
||||
//! function-call outputs, reasoning blocks, …).
|
||||
//! 2. **Output shape**. The response carries a single `output`
|
||||
//! array of items, each typed. We always emit one
|
||||
//! `OutputItem::Message` containing the assistant's reply (plus,
|
||||
//! when we get there, separate `function_call` items).
|
||||
//! 3. **Streaming events**. Where chat completions stream
|
||||
//! structurally-identical `chat.completion.chunk` frames over
|
||||
//! `data:` lines, Responses streams *named* events
|
||||
//! (`response.created`, `response.output_text.delta`,
|
||||
//! `response.completed`, …) over `event:` + `data:` SSE pairs.
|
||||
//! The wire projector in `neuron::wire::openai_responses` builds
|
||||
//! these from the same [`crate::openai`]-shaped
|
||||
//! `InferenceEvent` stream the chat projector consumes.
|
||||
//!
|
||||
//! Scope cuts for this first cut:
|
||||
//!
|
||||
//! - **`previous_response_id` is rejected at parse time**. Stateful
|
||||
//! chained conversations need a persistence layer we don't have.
|
||||
//! - **Reasoning items are accepted-and-ignored** (no Qwen3
|
||||
//! `<think>` routing yet). Audio and embedded resources are
|
||||
//! rejected as unsupported.
|
||||
//! - **Tool calls** (function_call / function_call_output) are
|
||||
//! carried as round-trip types but the candle harness doesn't
|
||||
//! emit them yet — wired so the surface is in place for the
|
||||
//! day we add proper tool-call extraction.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
// ── Request ──────────────────────────────────────────────────────────
|
||||
|
||||
/// Body of a `POST /v1/responses` request.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ResponsesRequest {
|
||||
pub model: String,
|
||||
pub input: ResponsesInput,
|
||||
/// System-prompt-style instructions. The Responses API
|
||||
/// separates these from input so a caller doesn't have to
|
||||
/// build a `system` message item by hand.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub instructions: Option<String>,
|
||||
#[serde(default)]
|
||||
pub stream: bool,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_output_tokens: Option<u64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub temperature: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub top_p: Option<f64>,
|
||||
/// Truncate sampling to the `k` most likely tokens (#272). Not in
|
||||
/// the Responses schema, but a sampling parameter that lands in
|
||||
/// `extra` is accepted and silently discarded, which is the defect
|
||||
/// this closes.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub top_k: Option<usize>,
|
||||
/// Pin the sampler's RNG for reproducible output (#272).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub seed: Option<u64>,
|
||||
/// Penalty on recently-generated tokens; `1.0` disables (#272).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub repetition_penalty: Option<f32>,
|
||||
/// Window `repetition_penalty` considers (#272).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub repeat_last_n: Option<usize>,
|
||||
/// OpenAI core; forwarded to the chat path unchanged.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub presence_penalty: Option<f32>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub frequency_penalty: Option<f32>,
|
||||
/// Chained-conversation identifier. We don't store responses
|
||||
/// server-side yet; if this is `Some`, the handler returns 400.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub previous_response_id: Option<String>,
|
||||
/// Catch-all for anything we don't model yet (tools, tool_choice,
|
||||
/// reasoning, response_format, …). Lets a client send a
|
||||
/// forward-compatible request without our parser rejecting it.
|
||||
#[serde(flatten)]
|
||||
pub extra: Value,
|
||||
}
|
||||
|
||||
/// `input` is either a single string or an array of items.
|
||||
/// `#[serde(untagged)]` so the wire shape `"input": "hi"` and
|
||||
/// `"input": [{...}]` both deserialize.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum ResponsesInput {
|
||||
Text(String),
|
||||
Items(Vec<ResponsesInputElement>),
|
||||
}
|
||||
|
||||
/// One element of an `input` array.
|
||||
///
|
||||
/// OpenAI's Responses API accepts three shapes here, and real clients
|
||||
/// use all of them — most notably agent-zero (via litellm), which
|
||||
/// sends the bare "easy message" form. We must tolerate every shape,
|
||||
/// because `input` is an `#[serde(untagged)]` array: a single element
|
||||
/// that matches no variant fails the *entire* request with a 422
|
||||
/// (`did not match any variant of untagged enum ResponsesInput`).
|
||||
///
|
||||
/// 1. [`Self::Typed`] — an item carrying an explicit `"type"`
|
||||
/// discriminant (`message`, `function_call`, `function_call_output`,
|
||||
/// `reasoning`).
|
||||
/// 2. [`Self::EasyMessage`] — a bare `{role, content}` with **no**
|
||||
/// `type` field. This is OpenAI's `EasyInputMessage` and what
|
||||
/// litellm emits for every turn. `content` is optional so an
|
||||
/// assistant turn carrying only tool calls (`content: null`) still
|
||||
/// parses.
|
||||
/// 3. [`Self::Other`] — anything else, captured as raw JSON and
|
||||
/// dropped during translation. This is the forward-compat escape
|
||||
/// hatch that mirrors [`ResponsesRequest::extra`] at the item
|
||||
/// level: an unmodeled item type can never again reject the whole
|
||||
/// request.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum ResponsesInputElement {
|
||||
Typed(ResponsesInputItem),
|
||||
EasyMessage {
|
||||
role: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
content: Option<ResponsesMessageContent>,
|
||||
},
|
||||
Other(Value),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum ResponsesInputItem {
|
||||
/// A user / assistant / system turn.
|
||||
Message {
|
||||
role: String,
|
||||
content: ResponsesMessageContent,
|
||||
},
|
||||
/// Assistant emitted a tool call. Round-trip only — neuron
|
||||
/// doesn't synthesise these yet.
|
||||
FunctionCall {
|
||||
call_id: String,
|
||||
name: String,
|
||||
arguments: String,
|
||||
},
|
||||
/// User is feeding a tool result back into the model. `output`
|
||||
/// is a `Value` because OpenAI allows it to be either a plain
|
||||
/// string or an array of content parts; the translator renders
|
||||
/// either form to text rather than losing the tool result.
|
||||
FunctionCallOutput { call_id: String, output: Value },
|
||||
/// A reasoning item the model emitted on an earlier turn, replayed
|
||||
/// by the client so the model can continue its own train of thought.
|
||||
///
|
||||
/// Both spellings are modelled because both are in the wild and one
|
||||
/// of them is ours: OpenAI's o-series carries the text in `content`
|
||||
/// as `reasoning_text` parts, while neuron emits `summary` with a
|
||||
/// `summary_text` part. Clients round-trip the completed item
|
||||
/// verbatim — pi-ai stores our `response.output_item.done` payload
|
||||
/// and replays it unchanged — so whichever field we emit is the
|
||||
/// field we get back, and reading only one of them silently loses
|
||||
/// the turn's reasoning.
|
||||
Reasoning {
|
||||
#[serde(default)]
|
||||
content: Vec<Value>,
|
||||
#[serde(default)]
|
||||
summary: Vec<Value>,
|
||||
},
|
||||
}
|
||||
|
||||
/// Inside a `Message` item, content is either a plain string or an
|
||||
/// array of typed parts. Mirrors the chat-completions Parts shape.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum ResponsesMessageContent {
|
||||
Text(String),
|
||||
Parts(Vec<ResponsesContentPart>),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum ResponsesContentPart {
|
||||
/// Plain text inside a user / system turn.
|
||||
InputText { text: String },
|
||||
/// An image. `image_url` is either a remote URL or a
|
||||
/// `data:image/png;base64,…` URI; the request translator just
|
||||
/// forwards the string.
|
||||
InputImage {
|
||||
image_url: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
detail: Option<String>,
|
||||
},
|
||||
/// Returned text inside an assistant turn — only relevant when
|
||||
/// the caller is feeding an assistant turn back in to continue
|
||||
/// a conversation manually (no `previous_response_id`).
|
||||
OutputText {
|
||||
text: String,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
annotations: Vec<Value>,
|
||||
},
|
||||
/// Any content-part type we don't model (e.g. `refusal`, audio).
|
||||
/// Captured as a unit so an unknown part can't reject the whole
|
||||
/// request; dropped during translation.
|
||||
#[serde(other)]
|
||||
Unknown,
|
||||
}
|
||||
|
||||
// ── Response (non-streaming) ─────────────────────────────────────────
|
||||
|
||||
/// Body of a `POST /v1/responses` response.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ResponsesResponse {
|
||||
pub id: String,
|
||||
/// Always `"response"`.
|
||||
pub object: String,
|
||||
pub created_at: u64,
|
||||
/// `"completed"`, `"incomplete"`, or — for the initial event of
|
||||
/// a streaming response — `"in_progress"`.
|
||||
pub status: String,
|
||||
pub model: String,
|
||||
pub output: Vec<ResponsesOutputItem>,
|
||||
/// Why a `status: "incomplete"` response stopped short. Required in
|
||||
/// practice, not optional decoration: a client cannot tell an
|
||||
/// honest truncation from a protocol fault without it, and at least
|
||||
/// one treats the difference as fatal.
|
||||
///
|
||||
/// pi-ai's `mapStopReason` maps
|
||||
/// `incomplete` + `reason == "max_output_tokens"` to
|
||||
/// `stopReason: "length"`, and `incomplete` with no reason to
|
||||
/// `stopReason: "error"` ("Response incomplete without a provider
|
||||
/// reason"). Its agent loop then either continues — failing any
|
||||
/// truncated tool calls safely and letting the model recover on the
|
||||
/// next turn — or halts outright. Omitting this field cost a live
|
||||
/// agentic session its whole run.
|
||||
///
|
||||
/// `None` for `completed` and `in_progress`.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub incomplete_details: Option<IncompleteDetails>,
|
||||
/// Populated on completion; `None` while streaming.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub usage: Option<ResponsesUsage>,
|
||||
}
|
||||
|
||||
/// Why a response is `incomplete`.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct IncompleteDetails {
|
||||
/// OpenAI spells the output-cap case `"max_output_tokens"`; that is
|
||||
/// the only reason we produce today. `content_filter` is the other
|
||||
/// value the upstream API defines.
|
||||
pub reason: String,
|
||||
}
|
||||
|
||||
impl IncompleteDetails {
|
||||
/// The response ran into the caller's output budget.
|
||||
pub fn max_output_tokens() -> Self {
|
||||
Self {
|
||||
reason: "max_output_tokens".into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum ResponsesOutputItem {
|
||||
Message {
|
||||
id: String,
|
||||
/// Always `"assistant"` for model output.
|
||||
role: String,
|
||||
/// Output content parts. We always emit a single
|
||||
/// `OutputText` today; multi-part output would land here
|
||||
/// once we have e.g. image generation.
|
||||
content: Vec<ResponsesOutputContent>,
|
||||
/// Item-level status. `"in_progress"` while streaming the
|
||||
/// content parts, `"completed"` when done.
|
||||
#[serde(default = "default_item_status")]
|
||||
status: String,
|
||||
},
|
||||
/// A think block, as its own output item ahead of the message —
|
||||
/// the same shape the streaming projector emits, so a client sees
|
||||
/// one contract whether or not it streams (#300).
|
||||
Reasoning {
|
||||
id: String,
|
||||
summary: Vec<ResponsesSummaryPart>,
|
||||
#[serde(default = "default_item_status")]
|
||||
status: String,
|
||||
},
|
||||
/// Reserved for the day tool-call extraction lands. The wire
|
||||
/// shape mirrors `ResponsesInputItem::FunctionCall`.
|
||||
FunctionCall {
|
||||
id: String,
|
||||
call_id: String,
|
||||
name: String,
|
||||
arguments: String,
|
||||
#[serde(default = "default_item_status")]
|
||||
status: String,
|
||||
},
|
||||
}
|
||||
|
||||
/// One `summary_text` part of a reasoning output item.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ResponsesSummaryPart {
|
||||
/// Always `"summary_text"`.
|
||||
#[serde(rename = "type")]
|
||||
pub part_type: String,
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
impl ResponsesSummaryPart {
|
||||
pub fn summary_text(text: impl Into<String>) -> Self {
|
||||
Self {
|
||||
part_type: "summary_text".into(),
|
||||
text: text.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn default_item_status() -> String {
|
||||
"completed".into()
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum ResponsesOutputContent {
|
||||
OutputText {
|
||||
text: String,
|
||||
/// Citations / inline annotations. Empty today; reserved
|
||||
/// for the day we wire in web search / file search.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
annotations: Vec<Value>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ResponsesUsage {
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
/// OpenAI-standard breakdown of `output_tokens`. Optional and
|
||||
/// additive. Carries `reasoning_tokens` for reasoning models (a
|
||||
/// sub-count of `output_tokens`, never added into `total_tokens`).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_tokens_details: Option<OutputTokensDetails>,
|
||||
/// OpenAI-standard breakdown of `input_tokens`, carrying
|
||||
/// `cached_tokens` (#269). Omitted when nothing was reused.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_tokens_details: Option<InputTokensDetails>,
|
||||
}
|
||||
|
||||
/// Sub-counts of `ResponsesUsage::output_tokens`.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct OutputTokensDetails {
|
||||
/// Tokens generated inside the model's reasoning span.
|
||||
pub reasoning_tokens: u64,
|
||||
}
|
||||
|
||||
/// Sub-counts of `ResponsesUsage::input_tokens`.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct InputTokensDetails {
|
||||
/// Input tokens served from neuron's prefix KV cache (#11), so a
|
||||
/// caller can see the saving rather than assume none (#269).
|
||||
pub cached_tokens: u64,
|
||||
}
|
||||
|
||||
// ── Streaming event names ────────────────────────────────────────────
|
||||
|
||||
/// Event names the SSE projector emits, hoisted as constants so
|
||||
/// the projector and the wire shape stay in sync without
|
||||
/// string-typos. The strings are dictated by OpenAI's published
|
||||
/// Responses API.
|
||||
pub mod events {
|
||||
pub const CREATED: &str = "response.created";
|
||||
/// Fired between `response.created` and the first output-item
|
||||
/// event. Marks "request validated, model is generating" —
|
||||
/// some clients use it to differentiate the "warming up" state
|
||||
/// from "streaming tokens" in their UI.
|
||||
pub const IN_PROGRESS: &str = "response.in_progress";
|
||||
pub const OUTPUT_ITEM_ADDED: &str = "response.output_item.added";
|
||||
pub const CONTENT_PART_ADDED: &str = "response.content_part.added";
|
||||
pub const OUTPUT_TEXT_DELTA: &str = "response.output_text.delta";
|
||||
pub const OUTPUT_TEXT_DONE: &str = "response.output_text.done";
|
||||
pub const CONTENT_PART_DONE: &str = "response.content_part.done";
|
||||
pub const OUTPUT_ITEM_DONE: &str = "response.output_item.done";
|
||||
/// Reasoning-item event family. A reasoning model's think block is
|
||||
/// its own output item, emitted ahead of the message item, with a
|
||||
/// summary part carrying the text. Streaming these is what keeps the
|
||||
/// connection observably alive while a model thinks — clients time
|
||||
/// out on silence, and SSE comment keep-alives don't count because
|
||||
/// idle timers reset on parsed events, not comments.
|
||||
pub const REASONING_SUMMARY_PART_ADDED: &str = "response.reasoning_summary_part.added";
|
||||
pub const REASONING_SUMMARY_TEXT_DELTA: &str = "response.reasoning_summary_text.delta";
|
||||
pub const REASONING_SUMMARY_TEXT_DONE: &str = "response.reasoning_summary_text.done";
|
||||
pub const REASONING_SUMMARY_PART_DONE: &str = "response.reasoning_summary_part.done";
|
||||
pub const FUNCTION_CALL_ARGUMENTS_DELTA: &str = "response.function_call_arguments.delta";
|
||||
pub const FUNCTION_CALL_ARGUMENTS_DONE: &str = "response.function_call_arguments.done";
|
||||
pub const COMPLETED: &str = "response.completed";
|
||||
/// Terminal frame for a response that stopped short. Carries the
|
||||
/// same `response` payload as [`COMPLETED`], with
|
||||
/// `status: "incomplete"` and `incomplete_details.reason`.
|
||||
///
|
||||
/// A client keyed on the event name — rather than on
|
||||
/// `response.status` — sees no terminal event at all if a truncated
|
||||
/// response is announced as `response.completed`, and waits out its
|
||||
/// idle timeout.
|
||||
pub const INCOMPLETE: &str = "response.incomplete";
|
||||
/// Terminal frame for a response that did not finish because the
|
||||
/// server failed — a poisoned device, an OOM, a producer that died
|
||||
/// mid-stream. Carries `status: "failed"` and an `error` object.
|
||||
///
|
||||
/// The alternative is worse than it looks: defaulting a missing
|
||||
/// finish to a clean stop reports a crash as a complete answer, and
|
||||
/// the caller believes a truncated or empty reply was the model's
|
||||
/// considered response.
|
||||
pub const FAILED: &str = "response.failed";
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn deserialises_input_string_form() {
|
||||
let raw = r#"{"model": "m", "input": "hello"}"#;
|
||||
let req: ResponsesRequest = serde_json::from_str(raw).unwrap();
|
||||
match req.input {
|
||||
ResponsesInput::Text(s) => assert_eq!(s, "hello"),
|
||||
other => panic!("expected Text, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deserialises_input_items_form() {
|
||||
let raw = r#"{
|
||||
"model": "m",
|
||||
"input": [
|
||||
{"type": "message", "role": "user", "content": "hi"}
|
||||
]
|
||||
}"#;
|
||||
let req: ResponsesRequest = serde_json::from_str(raw).unwrap();
|
||||
match req.input {
|
||||
ResponsesInput::Items(items) => {
|
||||
assert_eq!(items.len(), 1);
|
||||
match &items[0] {
|
||||
ResponsesInputElement::Typed(ResponsesInputItem::Message { role, content }) => {
|
||||
assert_eq!(role, "user");
|
||||
match content {
|
||||
ResponsesMessageContent::Text(t) => assert_eq!(t, "hi"),
|
||||
other => panic!("expected Text content, got {other:?}"),
|
||||
}
|
||||
}
|
||||
other => panic!("expected typed Message item, got {other:?}"),
|
||||
}
|
||||
}
|
||||
other => panic!("expected Items, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deserialises_bare_easy_message_without_type() {
|
||||
// The shape agent-zero (via litellm) actually sends: `input`
|
||||
// items are bare `{role, content}` with NO `type` field. This
|
||||
// is the exact payload that was returning 422.
|
||||
let raw = r#"{
|
||||
"model": "Qwen/Qwen3.6-27B",
|
||||
"store": true,
|
||||
"tools": [{"type": "function", "name": "x", "description": "d", "parameters": {}}],
|
||||
"input": [
|
||||
{"role": "system", "content": "you are helpful"},
|
||||
{"role": "assistant", "content": "{\"tool_name\":\"response\"}"},
|
||||
{"role": "user", "content": "hi"}
|
||||
]
|
||||
}"#;
|
||||
let req: ResponsesRequest = serde_json::from_str(raw).unwrap();
|
||||
let items = match req.input {
|
||||
ResponsesInput::Items(i) => i,
|
||||
other => panic!("expected Items, got {other:?}"),
|
||||
};
|
||||
assert_eq!(items.len(), 3);
|
||||
for el in &items {
|
||||
assert!(
|
||||
matches!(el, ResponsesInputElement::EasyMessage { .. }),
|
||||
"expected EasyMessage, got {el:?}"
|
||||
);
|
||||
}
|
||||
// `tools` / `store` ride through `extra`, not `input`.
|
||||
assert!(req.extra.get("tools").is_some());
|
||||
assert_eq!(req.extra.get("store"), Some(&Value::Bool(true)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tolerates_null_content_and_unknown_item_types() {
|
||||
// An assistant turn carrying only tool calls has `content: null`;
|
||||
// and a future/unmodeled item type must not 422 the request.
|
||||
let raw = r#"{
|
||||
"model": "m",
|
||||
"input": [
|
||||
{"role": "assistant", "content": null},
|
||||
{"type": "item_reference", "id": "abc"},
|
||||
{"type": "function_call_output", "call_id": "c1",
|
||||
"output": [{"type": "output_text", "text": "result"}]},
|
||||
{"role": "user", "content": "go"}
|
||||
]
|
||||
}"#;
|
||||
let req: ResponsesRequest = serde_json::from_str(raw).unwrap();
|
||||
let items = match req.input {
|
||||
ResponsesInput::Items(i) => i,
|
||||
other => panic!("expected Items, got {other:?}"),
|
||||
};
|
||||
assert_eq!(items.len(), 4);
|
||||
assert!(matches!(
|
||||
&items[0],
|
||||
ResponsesInputElement::EasyMessage { content: None, .. }
|
||||
));
|
||||
assert!(matches!(&items[1], ResponsesInputElement::Other(_)));
|
||||
assert!(matches!(
|
||||
&items[2],
|
||||
ResponsesInputElement::Typed(ResponsesInputItem::FunctionCallOutput { .. })
|
||||
));
|
||||
assert!(matches!(
|
||||
&items[3],
|
||||
ResponsesInputElement::EasyMessage { .. }
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tolerates_unknown_content_part_type() {
|
||||
// A `refusal` (or any unmodeled) content part must parse, not 422.
|
||||
let raw = r#"{
|
||||
"model": "m",
|
||||
"input": [
|
||||
{"role": "assistant", "content": [
|
||||
{"type": "refusal", "refusal": "no"},
|
||||
{"type": "output_text", "text": "ok"}
|
||||
]}
|
||||
]
|
||||
}"#;
|
||||
let req: ResponsesRequest = serde_json::from_str(raw).unwrap();
|
||||
let items = match req.input {
|
||||
ResponsesInput::Items(i) => i,
|
||||
other => panic!("expected Items, got {other:?}"),
|
||||
};
|
||||
let parts = match &items[0] {
|
||||
ResponsesInputElement::EasyMessage {
|
||||
content: Some(ResponsesMessageContent::Parts(p)),
|
||||
..
|
||||
} => p,
|
||||
other => panic!("expected EasyMessage with Parts, got {other:?}"),
|
||||
};
|
||||
assert_eq!(parts.len(), 2);
|
||||
assert!(matches!(&parts[0], ResponsesContentPart::Unknown));
|
||||
assert!(matches!(&parts[1], ResponsesContentPart::OutputText { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deserialises_input_with_image() {
|
||||
let raw = r#"{
|
||||
"model": "m",
|
||||
"input": [
|
||||
{"type": "message", "role": "user", "content": [
|
||||
{"type": "input_text", "text": "what is this"},
|
||||
{"type": "input_image", "image_url": "data:image/png;base64,AAA="}
|
||||
]}
|
||||
]
|
||||
}"#;
|
||||
let req: ResponsesRequest = serde_json::from_str(raw).unwrap();
|
||||
let items = match req.input {
|
||||
ResponsesInput::Items(i) => i,
|
||||
other => panic!("expected Items, got {other:?}"),
|
||||
};
|
||||
let parts = match &items[0] {
|
||||
ResponsesInputElement::Typed(ResponsesInputItem::Message {
|
||||
content: ResponsesMessageContent::Parts(p),
|
||||
..
|
||||
}) => p,
|
||||
other => panic!("expected Parts, got {other:?}"),
|
||||
};
|
||||
assert_eq!(parts.len(), 2);
|
||||
assert!(matches!(
|
||||
&parts[0],
|
||||
ResponsesContentPart::InputText { text } if text == "what is this"
|
||||
));
|
||||
assert!(matches!(
|
||||
&parts[1],
|
||||
ResponsesContentPart::InputImage { image_url, .. }
|
||||
if image_url == "data:image/png;base64,AAA="
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_fields_round_trip_via_extra() {
|
||||
let raw = r#"{
|
||||
"model": "m",
|
||||
"input": "hi",
|
||||
"tools": [{"type": "web_search"}],
|
||||
"reasoning": {"effort": "medium"}
|
||||
}"#;
|
||||
let req: ResponsesRequest = serde_json::from_str(raw).unwrap();
|
||||
assert!(req.extra.get("tools").is_some());
|
||||
assert!(req.extra.get("reasoning").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_round_trips_through_serde() {
|
||||
let r = ResponsesResponse {
|
||||
id: "resp_1".into(),
|
||||
object: "response".into(),
|
||||
created_at: 1700,
|
||||
status: "completed".into(),
|
||||
model: "m".into(),
|
||||
output: vec![ResponsesOutputItem::Message {
|
||||
id: "msg_1".into(),
|
||||
role: "assistant".into(),
|
||||
content: vec![ResponsesOutputContent::OutputText {
|
||||
text: "hi there".into(),
|
||||
annotations: vec![],
|
||||
}],
|
||||
status: "completed".into(),
|
||||
}],
|
||||
incomplete_details: None,
|
||||
usage: Some(ResponsesUsage {
|
||||
input_tokens: 5,
|
||||
output_tokens: 3,
|
||||
total_tokens: 8,
|
||||
output_tokens_details: None,
|
||||
input_tokens_details: None,
|
||||
}),
|
||||
};
|
||||
let json = serde_json::to_string(&r).unwrap();
|
||||
let parsed: ResponsesResponse = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(parsed.id, "resp_1");
|
||||
assert_eq!(parsed.output.len(), 1);
|
||||
assert!(
|
||||
!json.contains("incomplete_details"),
|
||||
"a completed response must not carry an incomplete reason"
|
||||
);
|
||||
}
|
||||
|
||||
/// The field pi-ai's `mapStopReason` keys on. Absent, it maps
|
||||
/// `incomplete` to `stopReason: "error"` and its agent loop halts;
|
||||
/// present with `max_output_tokens`, it maps to `"length"` and the
|
||||
/// loop continues.
|
||||
#[test]
|
||||
fn an_incomplete_response_serialises_its_reason() {
|
||||
let r = ResponsesResponse {
|
||||
id: "resp_2".into(),
|
||||
object: "response".into(),
|
||||
created_at: 1700,
|
||||
status: "incomplete".into(),
|
||||
model: "m".into(),
|
||||
output: vec![],
|
||||
incomplete_details: Some(IncompleteDetails::max_output_tokens()),
|
||||
usage: None,
|
||||
};
|
||||
let v: serde_json::Value = serde_json::to_value(&r).unwrap();
|
||||
assert_eq!(v["status"], "incomplete");
|
||||
assert_eq!(v["incomplete_details"]["reason"], "max_output_tokens");
|
||||
|
||||
let parsed: ResponsesResponse = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(
|
||||
parsed.incomplete_details,
|
||||
Some(IncompleteDetails::max_output_tokens())
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,267 +0,0 @@
|
||||
//! Scheme-qualified model identifiers.
|
||||
//!
|
||||
//! cortex/neuron historically resolves every model id through hf-hub
|
||||
//! against `https://huggingface.co`. Helexa is adding an EU-hosted
|
||||
//! registry (`registry.helexa.ai`) alongside HF — both speak the same
|
||||
//! HF-compatible wire format, but the bytes, jurisdiction, and trust
|
||||
//! root differ. Model ids therefore need a scheme:
|
||||
//!
|
||||
//! - `huggingface:Qwen/Qwen3.6-27B` — HF-hosted bytes
|
||||
//! - `helexa:Qwen/Qwen3.6-27B-Uncensored` — helexa registry bytes
|
||||
//! - `helexa:SomeOperator/CustomFinetune` — operator publishing
|
||||
//! under the helexa namespace; same scheme handles all `org/name`
|
||||
//! pairs hosted in that registry.
|
||||
//!
|
||||
//! Bare `org/name` parses with an empty scheme; the caller (typically
|
||||
//! a harness) substitutes its configured default scheme so existing
|
||||
//! configs keep working through the transition.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::fmt;
|
||||
use std::str::FromStr;
|
||||
|
||||
/// Parsed `scheme:org/name`. Bare `org/name` produces an empty scheme
|
||||
/// — call `with_default_scheme` (or check `is_scheme_unset`) to
|
||||
/// resolve before using.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub struct ModelSourceId {
|
||||
pub scheme: String,
|
||||
pub org: String,
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
/// Errors from `ModelSourceId::from_str`. Carries the offending input
|
||||
/// so log lines / API errors can echo what the operator typed.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum ParseError {
|
||||
#[error("empty model id")]
|
||||
Empty,
|
||||
#[error("model id '{0}' is missing the '/' between org and name")]
|
||||
MissingSlash(String),
|
||||
#[error("model id '{0}' has an empty scheme before ':'")]
|
||||
EmptyScheme(String),
|
||||
#[error("model id '{0}' has an empty org")]
|
||||
EmptyOrg(String),
|
||||
#[error("model id '{0}' has an empty name")]
|
||||
EmptyName(String),
|
||||
#[error("model id '{0}' has a scheme containing '/' which is reserved for org/name")]
|
||||
SchemeContainsSlash(String),
|
||||
#[error("model id '{0}' has a name containing ':' which is reserved for the scheme prefix")]
|
||||
NameContainsColon(String),
|
||||
}
|
||||
|
||||
impl ModelSourceId {
|
||||
/// Construct directly from already-validated parts. Used by tests
|
||||
/// and call sites that have the fields separately; the public API
|
||||
/// for parsing user input is `FromStr`.
|
||||
pub fn new(scheme: impl Into<String>, org: impl Into<String>, name: impl Into<String>) -> Self {
|
||||
Self {
|
||||
scheme: scheme.into(),
|
||||
org: org.into(),
|
||||
name: name.into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// True when this id parsed from a bare `org/name` (no scheme
|
||||
/// prefix). The harness substitutes its configured default in
|
||||
/// `with_default_scheme` before resolving against a registry.
|
||||
pub fn is_scheme_unset(&self) -> bool {
|
||||
self.scheme.is_empty()
|
||||
}
|
||||
|
||||
/// Substitute `default` for an empty scheme. No-op when the scheme
|
||||
/// is already set. Returns self by value so it composes neatly:
|
||||
/// `id.parse::<ModelSourceId>()?.with_default_scheme("huggingface")`.
|
||||
pub fn with_default_scheme(mut self, default: &str) -> Self {
|
||||
if self.scheme.is_empty() {
|
||||
self.scheme = default.to_string();
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// The `org/name` half — what an hf-hub `Api::model(...)` call
|
||||
/// expects regardless of which scheme/endpoint we're hitting.
|
||||
pub fn repo_path(&self) -> String {
|
||||
format!("{}/{}", self.org, self.name)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ModelSourceId {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
if self.scheme.is_empty() {
|
||||
write!(f, "{}/{}", self.org, self.name)
|
||||
} else {
|
||||
write!(f, "{}:{}/{}", self.scheme, self.org, self.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for ModelSourceId {
|
||||
type Err = ParseError;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
if s.is_empty() {
|
||||
return Err(ParseError::Empty);
|
||||
}
|
||||
// Scheme split. Only the *first* colon counts — anything after
|
||||
// belongs to org/name (and would be rejected separately because
|
||||
// `:` isn't allowed there).
|
||||
let (scheme, rest) = match s.split_once(':') {
|
||||
Some((scheme, rest)) => {
|
||||
if scheme.is_empty() {
|
||||
return Err(ParseError::EmptyScheme(s.to_string()));
|
||||
}
|
||||
if scheme.contains('/') {
|
||||
return Err(ParseError::SchemeContainsSlash(s.to_string()));
|
||||
}
|
||||
(scheme.to_string(), rest)
|
||||
}
|
||||
None => (String::new(), s),
|
||||
};
|
||||
let (org, name) = rest
|
||||
.split_once('/')
|
||||
.ok_or_else(|| ParseError::MissingSlash(s.to_string()))?;
|
||||
if org.is_empty() {
|
||||
return Err(ParseError::EmptyOrg(s.to_string()));
|
||||
}
|
||||
if name.is_empty() {
|
||||
return Err(ParseError::EmptyName(s.to_string()));
|
||||
}
|
||||
if name.contains(':') {
|
||||
return Err(ParseError::NameContainsColon(s.to_string()));
|
||||
}
|
||||
Ok(Self {
|
||||
scheme,
|
||||
org: org.to_string(),
|
||||
name: name.to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parses_qualified() {
|
||||
let id: ModelSourceId = "huggingface:Qwen/Qwen3.6-27B".parse().unwrap();
|
||||
assert_eq!(id.scheme, "huggingface");
|
||||
assert_eq!(id.org, "Qwen");
|
||||
assert_eq!(id.name, "Qwen3.6-27B");
|
||||
assert_eq!(id.repo_path(), "Qwen/Qwen3.6-27B");
|
||||
assert!(!id.is_scheme_unset());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_helexa_scheme() {
|
||||
let id: ModelSourceId = "helexa:SomeOperator/Qwen3.6-27B-Uncensored"
|
||||
.parse()
|
||||
.unwrap();
|
||||
assert_eq!(id.scheme, "helexa");
|
||||
assert_eq!(id.org, "SomeOperator");
|
||||
assert_eq!(id.name, "Qwen3.6-27B-Uncensored");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_bare_id_with_empty_scheme() {
|
||||
let id: ModelSourceId = "Qwen/Qwen3-30B-A3B-Instruct".parse().unwrap();
|
||||
assert_eq!(id.scheme, "");
|
||||
assert_eq!(id.org, "Qwen");
|
||||
assert_eq!(id.name, "Qwen3-30B-A3B-Instruct");
|
||||
assert!(id.is_scheme_unset());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn substitutes_default_scheme_only_when_unset() {
|
||||
let id: ModelSourceId = "Qwen/Q3".parse().unwrap();
|
||||
assert_eq!(id.with_default_scheme("huggingface").scheme, "huggingface");
|
||||
|
||||
let id: ModelSourceId = "helexa:Qwen/Q3".parse().unwrap();
|
||||
assert_eq!(
|
||||
id.with_default_scheme("huggingface").scheme,
|
||||
"helexa",
|
||||
"default substitution must not override an explicit scheme"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn display_roundtrips_qualified_id() {
|
||||
let s = "helexa:Helexa/Qwen3.6-27B";
|
||||
let id: ModelSourceId = s.parse().unwrap();
|
||||
assert_eq!(id.to_string(), s);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn display_roundtrips_bare_id() {
|
||||
let s = "Qwen/Q3";
|
||||
let id: ModelSourceId = s.parse().unwrap();
|
||||
assert_eq!(id.to_string(), s);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_empty() {
|
||||
assert_eq!("".parse::<ModelSourceId>().unwrap_err(), ParseError::Empty);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_missing_slash() {
|
||||
match "Qwen".parse::<ModelSourceId>().unwrap_err() {
|
||||
ParseError::MissingSlash(s) => assert_eq!(s, "Qwen"),
|
||||
other => panic!("expected MissingSlash, got {other:?}"),
|
||||
}
|
||||
match "huggingface:Qwen".parse::<ModelSourceId>().unwrap_err() {
|
||||
ParseError::MissingSlash(s) => assert_eq!(s, "huggingface:Qwen"),
|
||||
other => panic!("expected MissingSlash, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_empty_scheme() {
|
||||
match ":Qwen/Q3".parse::<ModelSourceId>().unwrap_err() {
|
||||
ParseError::EmptyScheme(s) => assert_eq!(s, ":Qwen/Q3"),
|
||||
other => panic!("expected EmptyScheme, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_scheme_with_slash() {
|
||||
match "hugg/ingface:Q/N".parse::<ModelSourceId>().unwrap_err() {
|
||||
ParseError::SchemeContainsSlash(s) => assert_eq!(s, "hugg/ingface:Q/N"),
|
||||
other => panic!("expected SchemeContainsSlash, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_empty_org_or_name() {
|
||||
match "huggingface:/N".parse::<ModelSourceId>().unwrap_err() {
|
||||
ParseError::EmptyOrg(_) => {}
|
||||
other => panic!("expected EmptyOrg, got {other:?}"),
|
||||
}
|
||||
match "huggingface:Q/".parse::<ModelSourceId>().unwrap_err() {
|
||||
ParseError::EmptyName(_) => {}
|
||||
other => panic!("expected EmptyName, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_name_with_colon() {
|
||||
match "huggingface:Q/N:weird"
|
||||
.parse::<ModelSourceId>()
|
||||
.unwrap_err()
|
||||
{
|
||||
ParseError::NameContainsColon(s) => assert_eq!(s, "huggingface:Q/N:weird"),
|
||||
other => panic!("expected NameContainsColon, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serde_roundtrips_via_struct() {
|
||||
// We serialize as a struct (scheme/org/name fields) so the
|
||||
// shape is self-describing in API payloads. Callers that want
|
||||
// the compact `scheme:org/name` string use `Display`/`FromStr`.
|
||||
let id = ModelSourceId::new("helexa", "Helexa", "Qwen3.6-27B");
|
||||
let json = serde_json::to_string(&id).unwrap();
|
||||
let back: ModelSourceId = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(back, id);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -6,8 +6,6 @@ license.workspace = true
|
||||
|
||||
[dependencies]
|
||||
cortex-core.workspace = true
|
||||
helexa-stream = { path = "../helexa-stream" }
|
||||
async-trait.workspace = true
|
||||
tokio.workspace = true
|
||||
axum.workspace = true
|
||||
tower.workspace = true
|
||||
@@ -26,7 +24,6 @@ tokio-stream.workspace = true
|
||||
eventsource-stream.workspace = true
|
||||
bytes = "1"
|
||||
urlencoding = "2"
|
||||
url = "2"
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { workspace = true, features = ["test-util"] }
|
||||
|
||||
@@ -1,235 +0,0 @@
|
||||
//! Streaming Anthropic SSE translation (#24).
|
||||
//!
|
||||
//! The `/v1/messages` handler translates the request envelope to
|
||||
//! OpenAI before proxying (see `cortex_core::translate`); this module
|
||||
//! completes the round trip for `stream: true` — the upstream OpenAI
|
||||
//! SSE stream is re-framed, event by event, into Anthropic's
|
||||
//! `message_start` / `content_block_*` / `message_delta` /
|
||||
//! `message_stop` sequence as it arrives. True streaming: each
|
||||
//! upstream chunk is translated and forwarded immediately; nothing is
|
||||
//! buffered beyond the current SSE event's bytes.
|
||||
//!
|
||||
//! The translation state machine itself is pure and lives in
|
||||
//! [`cortex_core::translate::AnthropicStreamTranslator`]; this module
|
||||
//! owns the wire concerns — splitting the upstream byte stream into
|
||||
//! SSE events, parsing `data:` payloads, and framing the translated
|
||||
//! events as `event: <name>\ndata: <json>\n\n`.
|
||||
|
||||
use axum::body::Body;
|
||||
use axum::http::StatusCode;
|
||||
use axum::response::Response;
|
||||
use bytes::Bytes;
|
||||
use cortex_core::openai::ChatCompletionChunk;
|
||||
use cortex_core::translate::AnthropicStreamTranslator;
|
||||
use futures::StreamExt;
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
|
||||
/// Forward the translated OpenAI request to the upstream node and
|
||||
/// return the response translated to Anthropic SSE framing.
|
||||
pub async fn stream_translated(
|
||||
client: &reqwest::Client,
|
||||
endpoint: &str,
|
||||
openai_body: axum::body::Bytes,
|
||||
model_id: &str,
|
||||
node_name: &str,
|
||||
inbound_headers: &axum::http::HeaderMap,
|
||||
usage_sink: Option<crate::metering::UsageSink>,
|
||||
) -> Response {
|
||||
let url = format!("{endpoint}/v1/chat/completions");
|
||||
tracing::info!(
|
||||
handler = "anthropic_messages",
|
||||
model = %model_id,
|
||||
node = %node_name,
|
||||
url = %url,
|
||||
"proxying streaming request (anthropic SSE translation)"
|
||||
);
|
||||
|
||||
let request = crate::auth::forward_principal_headers(
|
||||
client
|
||||
.post(&url)
|
||||
.header("content-type", "application/json")
|
||||
.body(openai_body),
|
||||
inbound_headers,
|
||||
);
|
||||
let upstream = match request.send().await {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
handler = "anthropic_messages",
|
||||
node = %node_name,
|
||||
url = %url,
|
||||
error = %e,
|
||||
"anthropic stream: upstream request failed"
|
||||
);
|
||||
return anthropic_error(StatusCode::BAD_GATEWAY, "upstream request failed");
|
||||
}
|
||||
};
|
||||
|
||||
let status = upstream.status();
|
||||
if !status.is_success() {
|
||||
tracing::warn!(
|
||||
handler = "anthropic_messages",
|
||||
node = %node_name,
|
||||
url = %url,
|
||||
status = status.as_u16(),
|
||||
"anthropic stream: upstream returned non-2xx"
|
||||
);
|
||||
return anthropic_error(
|
||||
StatusCode::from_u16(status.as_u16()).unwrap_or(StatusCode::BAD_GATEWAY),
|
||||
"upstream returned an error",
|
||||
);
|
||||
}
|
||||
|
||||
// Bounded channel: a slow client back-pressures the pump task,
|
||||
// which back-pressures the upstream read — same propagation
|
||||
// discipline as neuron's own projectors.
|
||||
let (tx, rx) = tokio::sync::mpsc::channel::<Result<Bytes, std::convert::Infallible>>(32);
|
||||
let node = node_name.to_string();
|
||||
let model = model_id.to_string();
|
||||
tokio::spawn(async move {
|
||||
let mut upstream = upstream.bytes_stream();
|
||||
let mut translator = AnthropicStreamTranslator::new();
|
||||
let mut buf: Vec<u8> = Vec::new();
|
||||
let mut done = false;
|
||||
// Wire-debug accounting for the stream summary emitted at the
|
||||
// end: did the model emit a structured tool call, what was the
|
||||
// final finish_reason, and how many upstream frames did we see.
|
||||
let mut saw_tool_call = false;
|
||||
let mut last_finish: Option<String> = None;
|
||||
let mut frames = 0u64;
|
||||
// Engine-truth usage for metering (#51), scanned from the upstream
|
||||
// frames (neuron emits a final `usage` object on the stream, #48).
|
||||
let mut usage_prompt = 0u64;
|
||||
let mut usage_completion = 0u64;
|
||||
|
||||
'outer: while let Some(block) = upstream.next().await {
|
||||
let block = match block {
|
||||
Ok(b) => b,
|
||||
Err(e) => {
|
||||
tracing::warn!(node = %node, error = %e, "anthropic stream: upstream read failed mid-stream");
|
||||
break;
|
||||
}
|
||||
};
|
||||
buf.extend_from_slice(&block);
|
||||
// SSE events are separated by a blank line.
|
||||
while let Some(pos) = find_event_boundary(&buf) {
|
||||
let event: Vec<u8> = buf.drain(..pos + 2).collect();
|
||||
let text = String::from_utf8_lossy(&event);
|
||||
for line in text.lines() {
|
||||
let Some(data) = line.strip_prefix("data:") else {
|
||||
continue;
|
||||
};
|
||||
let data = data.trim();
|
||||
if data == "[DONE]" {
|
||||
done = true;
|
||||
if !send_frames(&tx, translator.finish()).await {
|
||||
break 'outer;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
tracing::trace!(node = %node, frame = %data, "anthropic stream: upstream frame");
|
||||
// Capture usage for metering before translation — the
|
||||
// usage object rides on a late frame (often after the
|
||||
// last content delta).
|
||||
if let Some(p) = crate::proxy::last_count_for(data, "prompt_tokens") {
|
||||
usage_prompt = p;
|
||||
}
|
||||
if let Some(c) = crate::proxy::last_count_for(data, "completion_tokens") {
|
||||
usage_completion = c;
|
||||
}
|
||||
let Ok(chunk) = serde_json::from_str::<ChatCompletionChunk>(data) else {
|
||||
tracing::debug!(node = %node, "anthropic stream: unparsable upstream frame skipped");
|
||||
continue;
|
||||
};
|
||||
frames += 1;
|
||||
if chunk
|
||||
.choices
|
||||
.iter()
|
||||
.any(|c| c.delta.get("tool_calls").is_some())
|
||||
{
|
||||
saw_tool_call = true;
|
||||
}
|
||||
if let Some(fr) = chunk.choices.iter().find_map(|c| c.finish_reason.clone()) {
|
||||
last_finish = Some(fr);
|
||||
}
|
||||
if !send_frames(&tx, translator.on_chunk(&chunk)).await {
|
||||
break 'outer;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Upstream ended without [DONE] (error or truncation): still
|
||||
// close the Anthropic event sequence so clients aren't left
|
||||
// with an unterminated message.
|
||||
if !done {
|
||||
let _ = send_frames(&tx, translator.finish()).await;
|
||||
}
|
||||
// Stream summary: the streaming counterpart to the non-streaming
|
||||
// handler's "upstream response" line. `upstream_tool_calls =
|
||||
// false` on a tools-bearing request is the fingerprint of the
|
||||
// model improvising an unparsed tool-call format.
|
||||
tracing::debug!(
|
||||
wire = "anthropic",
|
||||
model = %model,
|
||||
node = %node,
|
||||
frames,
|
||||
upstream_tool_calls = saw_tool_call,
|
||||
finish_reason = ?last_finish,
|
||||
terminated = done,
|
||||
"anthropic stream complete"
|
||||
);
|
||||
|
||||
// Settle metering with the observed usage (#51). Runs on every exit
|
||||
// path of the pump — clean end, early break, or upstream error — so
|
||||
// the reservation is always resolved. `(0, 0)` when no usage frame
|
||||
// was seen, which releases without recording spend.
|
||||
if let Some(sink) = usage_sink {
|
||||
sink(usage_prompt, usage_completion);
|
||||
}
|
||||
});
|
||||
|
||||
Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header("content-type", "text/event-stream")
|
||||
.header("cache-control", "no-cache")
|
||||
.body(Body::from_stream(ReceiverStream::new(rx)))
|
||||
.unwrap_or_else(|_| {
|
||||
anthropic_error(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"failed to build response",
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// `\n\n` boundary of the first complete SSE event in `buf`, if any.
|
||||
fn find_event_boundary(buf: &[u8]) -> Option<usize> {
|
||||
buf.windows(2).position(|w| w == b"\n\n")
|
||||
}
|
||||
|
||||
/// Render translated events as SSE frames and send them. Returns
|
||||
/// `false` when the client has gone away (receiver dropped).
|
||||
async fn send_frames(
|
||||
tx: &tokio::sync::mpsc::Sender<Result<Bytes, std::convert::Infallible>>,
|
||||
events: Vec<(String, serde_json::Value)>,
|
||||
) -> bool {
|
||||
for (name, payload) in events {
|
||||
let frame = format!("event: {name}\ndata: {payload}\n\n");
|
||||
if tx.send(Ok(Bytes::from(frame))).await.is_err() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Anthropic-shaped error body (`{"type":"error","error":{...}}`).
|
||||
fn anthropic_error(status: StatusCode, message: &str) -> Response {
|
||||
let body = serde_json::json!({
|
||||
"type": "error",
|
||||
"error": { "type": "api_error", "message": message }
|
||||
});
|
||||
Response::builder()
|
||||
.status(status)
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(body.to_string()))
|
||||
.expect("static error response must build")
|
||||
}
|
||||
@@ -1,144 +0,0 @@
|
||||
//! API-key authentication + principal resolution (#49).
|
||||
//!
|
||||
//! Identity rides standard bearer auth only — `Authorization: Bearer <key>`
|
||||
//! — which is what keeps every tier OpenAI-compatible by construction (no
|
||||
//! custom required headers or body fields, per #47). The middleware resolves
|
||||
//! the key to a [`Principal`] via the [`EntitlementProvider`], carries it in
|
||||
//! the request extensions for cortex-side metering/enforcement (#51/#52), and
|
||||
//! stamps it as internal headers on the request so it reaches neuron, which
|
||||
//! trusts cortex's assertion over WireGuard (#54).
|
||||
//!
|
||||
//! Anti-spoofing: any client-supplied principal header is **stripped** before
|
||||
//! the authoritative value is stamped, so a client can never assert a
|
||||
//! principal it didn't authenticate as.
|
||||
//!
|
||||
//! Rejection contract (#63): missing key under `require_auth`, or any present
|
||||
//! but unresolvable key, yields `401 invalid_api_key` in the #60 envelope.
|
||||
|
||||
use crate::error::envelope_response;
|
||||
use crate::state::CortexState;
|
||||
use axum::extract::{Request, State};
|
||||
use axum::http::header::AUTHORIZATION;
|
||||
use axum::http::{HeaderMap, HeaderValue};
|
||||
use axum::middleware::Next;
|
||||
use axum::response::Response;
|
||||
use cortex_core::entitlements::{AuthError, HEADER_ACCOUNT_ID, HEADER_KEY_ID};
|
||||
use cortex_core::error_envelope::OpenAiError;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Endpoints that never require auth: liveness/readiness probes. Everything
|
||||
/// else flows through resolution.
|
||||
fn is_public(path: &str) -> bool {
|
||||
path == "/health" || path == "/"
|
||||
}
|
||||
|
||||
/// Extract the bearer token from an `Authorization` header value, if present
|
||||
/// and well-formed. Scheme match is case-insensitive per RFC 7235.
|
||||
fn parse_bearer(headers: &HeaderMap) -> Option<String> {
|
||||
let raw = headers.get(AUTHORIZATION)?.to_str().ok()?;
|
||||
let (scheme, token) = raw.split_once(' ')?;
|
||||
if scheme.eq_ignore_ascii_case("bearer") {
|
||||
let token = token.trim();
|
||||
(!token.is_empty()).then(|| token.to_string())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Axum middleware: resolve the bearer key, attach the principal, stamp the
|
||||
/// internal headers. Wired in `build_app` via `from_fn_with_state`.
|
||||
pub async fn require_principal(
|
||||
State(fleet): State<Arc<CortexState>>,
|
||||
mut req: Request,
|
||||
next: Next,
|
||||
) -> Response {
|
||||
if is_public(req.uri().path()) {
|
||||
return next.run(req).await;
|
||||
}
|
||||
|
||||
// Anti-spoof: drop any client-supplied principal headers up front.
|
||||
{
|
||||
let headers = req.headers_mut();
|
||||
headers.remove(HEADER_ACCOUNT_ID);
|
||||
headers.remove(HEADER_KEY_ID);
|
||||
}
|
||||
|
||||
match parse_bearer(req.headers()) {
|
||||
Some(key) => match fleet.entitlements.resolve(&key).await {
|
||||
Ok(principal) => {
|
||||
// Stamp the authoritative principal for neuron. Account/key
|
||||
// ids come from operator config, so they're valid header
|
||||
// values; guard anyway and skip a malformed one rather than
|
||||
// panic.
|
||||
if let (Ok(account), Ok(key_id)) = (
|
||||
HeaderValue::from_str(&principal.account_id),
|
||||
HeaderValue::from_str(&principal.key_id),
|
||||
) {
|
||||
let headers = req.headers_mut();
|
||||
headers.insert(HEADER_ACCOUNT_ID, account);
|
||||
headers.insert(HEADER_KEY_ID, key_id);
|
||||
}
|
||||
// Carry the typed principal for cortex-side metering (#51)
|
||||
// and budget enforcement (#52).
|
||||
req.extensions_mut().insert(principal);
|
||||
next.run(req).await
|
||||
}
|
||||
// The entitlement authority is unreachable (upstream client
|
||||
// blip, #57). Fail **closed but distinct**: a transient outage
|
||||
// must not reject a real key as `401 invalid_api_key` — it's a
|
||||
// retryable `503`. This holds regardless of require_auth: we
|
||||
// can't safely serve a key we couldn't authorize.
|
||||
Err(AuthError::Unavailable { retry_after_secs }) => {
|
||||
envelope_response(OpenAiError::service_unavailable(
|
||||
"entitlement authority temporarily unavailable",
|
||||
Some(retry_after_secs),
|
||||
))
|
||||
}
|
||||
// A genuinely unrecognized key only hard-fails when auth is
|
||||
// *required*. In allow-anonymous mode (the default) we IGNORE it
|
||||
// and serve unauthenticated — otherwise the placeholder keys that
|
||||
// OpenAI-compatible clients send by default (opencode, Open WebUI,
|
||||
// Agent Zero, litellm) would all break though the operator never
|
||||
// opted into auth. Pre-#49 the bearer was never inspected; this
|
||||
// preserves that for require_auth=false.
|
||||
Err(AuthError::InvalidKey) => {
|
||||
if fleet.require_auth {
|
||||
unauthorized("invalid API key")
|
||||
} else {
|
||||
tracing::debug!(
|
||||
"ignoring unrecognized bearer token (require_auth=false): serving anonymously"
|
||||
);
|
||||
next.run(req).await
|
||||
}
|
||||
}
|
||||
},
|
||||
None => {
|
||||
if fleet.require_auth {
|
||||
unauthorized("missing API key; supply 'Authorization: Bearer <key>'")
|
||||
} else {
|
||||
next.run(req).await
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `401 invalid_api_key` in the standard envelope (#63).
|
||||
fn unauthorized(message: &str) -> Response {
|
||||
envelope_response(OpenAiError::invalid_api_key(message))
|
||||
}
|
||||
|
||||
/// Copy the cortex-stamped principal headers from an inbound [`HeaderMap`]
|
||||
/// onto an outbound reqwest builder. Used by the Anthropic proxy paths,
|
||||
/// which construct their own upstream requests instead of going through
|
||||
/// [`crate::proxy::forward_request`] (which forwards all headers verbatim).
|
||||
pub fn forward_principal_headers(
|
||||
mut builder: reqwest::RequestBuilder,
|
||||
headers: &HeaderMap,
|
||||
) -> reqwest::RequestBuilder {
|
||||
for name in [HEADER_ACCOUNT_ID, HEADER_KEY_ID] {
|
||||
if let Some(value) = headers.get(name) {
|
||||
builder = builder.header(name, value);
|
||||
}
|
||||
}
|
||||
builder
|
||||
}
|
||||
@@ -1,112 +0,0 @@
|
||||
//! Chained entitlement provider (#57): operator-local keys first, mesh
|
||||
//! upstream for everything else.
|
||||
//!
|
||||
//! `resolve` tries the [`LocalEntitlementProvider`] (operator + infra keys —
|
||||
//! never a network hop); only a locally-unknown key falls through to
|
||||
//! [`UpstreamEntitlementProvider`]. Because the local provider treats an
|
||||
//! unconfigured principal as uncapped, reserve/settle/release/snapshot must
|
||||
//! **not** blindly hit local — they dispatch to whichever backend resolved
|
||||
//! that account, remembered in a map keyed by `account_id` (populated at
|
||||
//! resolve time).
|
||||
|
||||
use crate::entitlements_local::LocalEntitlementProvider;
|
||||
use crate::entitlements_upstream::UpstreamEntitlementProvider;
|
||||
use async_trait::async_trait;
|
||||
use cortex_core::entitlements::{
|
||||
AuthError, BudgetError, BudgetSnapshot, EntitlementProvider, Principal, Reservation,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum Backend {
|
||||
Local,
|
||||
Upstream,
|
||||
}
|
||||
|
||||
pub struct ChainedEntitlementProvider {
|
||||
local: LocalEntitlementProvider,
|
||||
upstream: UpstreamEntitlementProvider,
|
||||
/// account_id → which backend owns it, learned at resolve time.
|
||||
backends: RwLock<HashMap<String, Backend>>,
|
||||
}
|
||||
|
||||
impl ChainedEntitlementProvider {
|
||||
pub fn new(local: LocalEntitlementProvider, upstream: UpstreamEntitlementProvider) -> Self {
|
||||
Self {
|
||||
local,
|
||||
upstream,
|
||||
backends: RwLock::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn record(&self, account_id: &str, backend: Backend) {
|
||||
self.backends
|
||||
.write()
|
||||
.await
|
||||
.insert(account_id.to_string(), backend);
|
||||
}
|
||||
|
||||
/// The backend that owns `account_id`. Defaults to `Upstream` for an
|
||||
/// account never resolved this process-lifetime (a resolve always
|
||||
/// precedes reserve in a request, so this is just a safe fallback —
|
||||
/// upstream fails closed if the account is bogus).
|
||||
async fn backend_for(&self, account_id: &str) -> Backend {
|
||||
self.backends
|
||||
.read()
|
||||
.await
|
||||
.get(account_id)
|
||||
.copied()
|
||||
.unwrap_or(Backend::Upstream)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl EntitlementProvider for ChainedEntitlementProvider {
|
||||
async fn resolve(&self, api_key: &str) -> Result<Principal, AuthError> {
|
||||
match self.local.resolve(api_key).await {
|
||||
Ok(p) => {
|
||||
self.record(&p.account_id, Backend::Local).await;
|
||||
Ok(p)
|
||||
}
|
||||
Err(AuthError::InvalidKey) => {
|
||||
let p = self.upstream.resolve(api_key).await?;
|
||||
self.record(&p.account_id, Backend::Upstream).await;
|
||||
Ok(p)
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
async fn reserve(
|
||||
&self,
|
||||
principal: &Principal,
|
||||
max_tokens: u64,
|
||||
) -> Result<Reservation, BudgetError> {
|
||||
match self.backend_for(&principal.account_id).await {
|
||||
Backend::Local => self.local.reserve(principal, max_tokens).await,
|
||||
Backend::Upstream => self.upstream.reserve(principal, max_tokens).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn settle(&self, reservation: Reservation, actual_tokens: u64) {
|
||||
match self.backend_for(&reservation.principal.account_id).await {
|
||||
Backend::Local => self.local.settle(reservation, actual_tokens).await,
|
||||
Backend::Upstream => self.upstream.settle(reservation, actual_tokens).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn release(&self, reservation: Reservation) {
|
||||
match self.backend_for(&reservation.principal.account_id).await {
|
||||
Backend::Local => self.local.release(reservation).await,
|
||||
Backend::Upstream => self.upstream.release(reservation).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn snapshot(&self, principal: &Principal) -> Option<BudgetSnapshot> {
|
||||
match self.backend_for(&principal.account_id).await {
|
||||
Backend::Local => self.local.snapshot(principal).await,
|
||||
Backend::Upstream => self.upstream.snapshot(principal).await,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,317 +0,0 @@
|
||||
//! The local/static [`EntitlementProvider`] (#50).
|
||||
//!
|
||||
//! Accounts, keys, and hard caps come from operator config
|
||||
//! ([`cortex_core::config::EntitlementsConfig`]); reservations and settled
|
||||
//! spend are tracked in-process. This lands auth + per-key caps + the
|
||||
//! amplification fix before any upstream clearing house exists; the future
|
||||
//! helexa-upstream client (#57) implements the same trait.
|
||||
//!
|
||||
//! Budget math is serialized under a single [`std::sync::Mutex`] so
|
||||
//! reserve/settle/release are atomic — a key's `spent + reserved` can never
|
||||
//! exceed its hard cap even under concurrent requests (the #52 guarantee).
|
||||
//! The lock is held only for the in-memory arithmetic, never across an
|
||||
//! await.
|
||||
|
||||
use cortex_core::config::{ApiKeyConfig, EntitlementsConfig};
|
||||
use cortex_core::entitlements::{
|
||||
AuthError, BudgetError, BudgetSnapshot, CapWindow, EntitlementProvider, Principal, Reservation,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::Instant;
|
||||
|
||||
/// Per-key budget configuration (resolved from [`ApiKeyConfig`]).
|
||||
struct Budget {
|
||||
hard_cap: Option<u64>,
|
||||
window: CapWindow,
|
||||
}
|
||||
|
||||
/// Live, mutable accounting for one key over its current window.
|
||||
#[derive(Default)]
|
||||
struct Ledger {
|
||||
/// Settled spend in the current window.
|
||||
spent: u64,
|
||||
/// Sum of outstanding (un-settled) reservations.
|
||||
reserved: u64,
|
||||
/// Start of the current rolling window; `None` until the first reserve.
|
||||
/// Unused for [`CapWindow::Balance`].
|
||||
window_start: Option<Instant>,
|
||||
}
|
||||
|
||||
pub struct LocalEntitlementProvider {
|
||||
/// Bearer token → principal.
|
||||
keys: HashMap<String, Principal>,
|
||||
/// `key_id` → budget config.
|
||||
budgets: HashMap<String, Budget>,
|
||||
/// `key_id` → live ledger.
|
||||
ledgers: Mutex<HashMap<String, Ledger>>,
|
||||
/// Monotonic source of opaque reservation handles.
|
||||
next_id: AtomicU64,
|
||||
}
|
||||
|
||||
impl LocalEntitlementProvider {
|
||||
/// Build from the `[entitlements]` config. A key without an explicit
|
||||
/// `key_id` is tracked at `account_id` granularity (its secret is never
|
||||
/// used as a label).
|
||||
pub fn from_config(config: &EntitlementsConfig) -> Self {
|
||||
let mut keys = HashMap::new();
|
||||
let mut budgets = HashMap::new();
|
||||
for ApiKeyConfig {
|
||||
key,
|
||||
account_id,
|
||||
key_id,
|
||||
hard_cap,
|
||||
window,
|
||||
} in &config.keys
|
||||
{
|
||||
let key_id = key_id.clone().unwrap_or_else(|| account_id.clone());
|
||||
keys.insert(
|
||||
key.clone(),
|
||||
Principal {
|
||||
account_id: account_id.clone(),
|
||||
key_id: key_id.clone(),
|
||||
},
|
||||
);
|
||||
budgets.insert(
|
||||
key_id,
|
||||
Budget {
|
||||
hard_cap: *hard_cap,
|
||||
window: window.clone(),
|
||||
},
|
||||
);
|
||||
}
|
||||
Self {
|
||||
keys,
|
||||
budgets,
|
||||
ledgers: Mutex::new(HashMap::new()),
|
||||
next_id: AtomicU64::new(1),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Tokens still available under `cap` given current `spent`/`reserved`.
|
||||
/// `None` cap = unlimited.
|
||||
fn available(cap: Option<u64>, spent: u64, reserved: u64) -> Option<u64> {
|
||||
cap.map(|c| c.saturating_sub(spent).saturating_sub(reserved))
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl EntitlementProvider for LocalEntitlementProvider {
|
||||
async fn resolve(&self, api_key: &str) -> Result<Principal, AuthError> {
|
||||
self.keys.get(api_key).cloned().ok_or(AuthError::InvalidKey)
|
||||
}
|
||||
|
||||
async fn reserve(
|
||||
&self,
|
||||
principal: &Principal,
|
||||
max_tokens: u64,
|
||||
) -> Result<Reservation, BudgetError> {
|
||||
// A principal with no configured budget (or an uncapped one) always
|
||||
// reserves; we still track spend for metrics.
|
||||
let budget = self.budgets.get(&principal.key_id);
|
||||
let (cap, window) = match budget {
|
||||
Some(b) => (b.hard_cap, b.window.clone()),
|
||||
None => (None, CapWindow::Balance),
|
||||
};
|
||||
|
||||
let mut ledgers = self.ledgers.lock().expect("ledger mutex poisoned");
|
||||
let ledger = ledgers.entry(principal.key_id.clone()).or_default();
|
||||
|
||||
// Lazily reset a rolling window that has elapsed before checking.
|
||||
let mut retry_after_secs = 0;
|
||||
if let CapWindow::Rolling { seconds } = window {
|
||||
let now = Instant::now();
|
||||
match ledger.window_start {
|
||||
Some(start) if now.duration_since(start).as_secs() < seconds => {
|
||||
retry_after_secs = seconds - now.duration_since(start).as_secs();
|
||||
}
|
||||
_ => {
|
||||
// First reserve, or the window has fully elapsed: reset.
|
||||
ledger.spent = 0;
|
||||
ledger.window_start = Some(now);
|
||||
retry_after_secs = seconds;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(avail) = available(cap, ledger.spent, ledger.reserved)
|
||||
&& max_tokens > avail
|
||||
{
|
||||
return Err(match window {
|
||||
CapWindow::Rolling { .. } => BudgetError::RateLimited {
|
||||
requested: max_tokens,
|
||||
available: avail,
|
||||
// At least 1s so clients don't hot-loop on a sub-second
|
||||
// remainder.
|
||||
retry_after_secs: retry_after_secs.max(1),
|
||||
},
|
||||
CapWindow::Balance => BudgetError::InsufficientQuota {
|
||||
requested: max_tokens,
|
||||
available: avail,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
ledger.reserved += max_tokens;
|
||||
Ok(Reservation {
|
||||
id: self.next_id.fetch_add(1, Ordering::Relaxed),
|
||||
principal: principal.clone(),
|
||||
reserved: max_tokens,
|
||||
})
|
||||
}
|
||||
|
||||
async fn settle(&self, reservation: Reservation, actual_tokens: u64) {
|
||||
let mut ledgers = self.ledgers.lock().expect("ledger mutex poisoned");
|
||||
if let Some(ledger) = ledgers.get_mut(&reservation.principal.key_id) {
|
||||
ledger.reserved = ledger.reserved.saturating_sub(reservation.reserved);
|
||||
ledger.spent += actual_tokens;
|
||||
}
|
||||
}
|
||||
|
||||
async fn release(&self, reservation: Reservation) {
|
||||
let mut ledgers = self.ledgers.lock().expect("ledger mutex poisoned");
|
||||
if let Some(ledger) = ledgers.get_mut(&reservation.principal.key_id) {
|
||||
ledger.reserved = ledger.reserved.saturating_sub(reservation.reserved);
|
||||
}
|
||||
}
|
||||
|
||||
async fn snapshot(&self, principal: &Principal) -> Option<BudgetSnapshot> {
|
||||
let ledgers = self.ledgers.lock().expect("ledger mutex poisoned");
|
||||
let (spent, reserved) = ledgers
|
||||
.get(&principal.key_id)
|
||||
.map(|l| (l.spent, l.reserved))
|
||||
.unwrap_or((0, 0));
|
||||
let hard_cap = self.budgets.get(&principal.key_id).and_then(|b| b.hard_cap);
|
||||
Some(BudgetSnapshot {
|
||||
hard_cap,
|
||||
spent,
|
||||
reserved,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn provider() -> LocalEntitlementProvider {
|
||||
let config = EntitlementsConfig {
|
||||
require_auth: true,
|
||||
keys: vec![
|
||||
ApiKeyConfig {
|
||||
key: "sk-balance".into(),
|
||||
account_id: "acct-a".into(),
|
||||
key_id: Some("key-balance".into()),
|
||||
hard_cap: Some(1_000),
|
||||
window: CapWindow::Balance,
|
||||
},
|
||||
ApiKeyConfig {
|
||||
key: "sk-rolling".into(),
|
||||
account_id: "acct-b".into(),
|
||||
key_id: Some("key-rolling".into()),
|
||||
hard_cap: Some(500),
|
||||
window: CapWindow::Rolling { seconds: 3_600 },
|
||||
},
|
||||
ApiKeyConfig {
|
||||
key: "sk-infra".into(),
|
||||
account_id: "operator".into(),
|
||||
key_id: Some("key-infra".into()),
|
||||
hard_cap: None,
|
||||
window: CapWindow::Balance,
|
||||
},
|
||||
],
|
||||
};
|
||||
LocalEntitlementProvider::from_config(&config)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resolves_configured_key_to_principal() {
|
||||
let p = provider();
|
||||
let principal = p.resolve("sk-balance").await.expect("known key resolves");
|
||||
assert_eq!(principal.account_id, "acct-a");
|
||||
assert_eq!(principal.key_id, "key-balance");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_key_is_invalid() {
|
||||
let p = provider();
|
||||
assert!(matches!(
|
||||
p.resolve("sk-nope").await,
|
||||
Err(AuthError::InvalidKey)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reserve_settle_release_round_trip() {
|
||||
let p = provider();
|
||||
let principal = p.resolve("sk-balance").await.unwrap();
|
||||
|
||||
let r = p.reserve(&principal, 400).await.expect("within cap");
|
||||
// Reserved, not yet spent.
|
||||
let snap = p.snapshot(&principal).await.unwrap();
|
||||
assert_eq!(snap.hard_cap, Some(1_000));
|
||||
assert_eq!(snap.reserved, 400);
|
||||
assert_eq!(snap.spent, 0);
|
||||
|
||||
// Used fewer tokens than reserved → remainder released, spend exact.
|
||||
p.settle(r, 250).await;
|
||||
let snap = p.snapshot(&principal).await.unwrap();
|
||||
assert_eq!(snap.reserved, 0);
|
||||
assert_eq!(snap.spent, 250);
|
||||
|
||||
// A reservation that is released contributes no spend.
|
||||
let r2 = p.reserve(&principal, 100).await.unwrap();
|
||||
p.release(r2).await;
|
||||
let snap = p.snapshot(&principal).await.unwrap();
|
||||
assert_eq!(snap.reserved, 0);
|
||||
assert_eq!(snap.spent, 250);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn balance_over_cap_is_insufficient_quota() {
|
||||
let p = provider();
|
||||
let principal = p.resolve("sk-balance").await.unwrap();
|
||||
// Reserve most of the cap, then ask for more than remains.
|
||||
let _r = p.reserve(&principal, 900).await.unwrap();
|
||||
let err = p.reserve(&principal, 200).await.expect_err("over cap");
|
||||
match err {
|
||||
BudgetError::InsufficientQuota {
|
||||
requested,
|
||||
available,
|
||||
} => {
|
||||
assert_eq!(requested, 200);
|
||||
assert_eq!(available, 100);
|
||||
}
|
||||
other => panic!("expected InsufficientQuota, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rolling_over_cap_is_rate_limited_with_retry_after() {
|
||||
let p = provider();
|
||||
let principal = p.resolve("sk-rolling").await.unwrap();
|
||||
let _r = p.reserve(&principal, 500).await.unwrap();
|
||||
let err = p.reserve(&principal, 1).await.expect_err("over cap");
|
||||
match err {
|
||||
BudgetError::RateLimited {
|
||||
retry_after_secs, ..
|
||||
} => {
|
||||
assert!(retry_after_secs >= 1, "must advertise a retry hint");
|
||||
assert!(retry_after_secs <= 3_600);
|
||||
}
|
||||
other => panic!("expected RateLimited, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn uncapped_infra_key_never_refuses() {
|
||||
let p = provider();
|
||||
let principal = p.resolve("sk-infra").await.unwrap();
|
||||
let r = p.reserve(&principal, 10_000_000).await.expect("uncapped");
|
||||
p.settle(r, 10_000_000).await;
|
||||
let snap = p.snapshot(&principal).await.unwrap();
|
||||
assert_eq!(snap.hard_cap, None);
|
||||
assert_eq!(snap.spent, 10_000_000);
|
||||
}
|
||||
}
|
||||
@@ -1,246 +0,0 @@
|
||||
//! helexa-upstream client (#57): an [`EntitlementProvider`] that resolves
|
||||
//! keys and reserves/settles budget against the mesh authority's
|
||||
//! `/authz/v1` surface (B2). It is "just another impl of the trait" — cortex
|
||||
//! enforcement (`auth.rs`, `metering.rs`) is unchanged.
|
||||
//!
|
||||
//! **Fail closed.** When upstream is unreachable, `resolve` returns
|
||||
//! [`AuthError::Unavailable`] (→ `503`, never `401`) and `reserve` refuses
|
||||
//! with a retryable [`BudgetError::RateLimited`] — a request is never served
|
||||
//! on an un-authorized key, and a real key is never rejected as invalid
|
||||
//! during a blip.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use cortex_core::config::UpstreamClientConfig;
|
||||
use cortex_core::entitlements::{
|
||||
AuthError, BudgetError, BudgetSnapshot, EntitlementProvider, Principal, Reservation,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use std::time::Duration;
|
||||
|
||||
/// Retry-After (seconds) advertised when we fail closed on an upstream
|
||||
/// outage.
|
||||
const FAIL_CLOSED_RETRY_SECS: u64 = 5;
|
||||
|
||||
pub struct UpstreamEntitlementProvider {
|
||||
client: reqwest::Client,
|
||||
base_url: String,
|
||||
bearer: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct PrincipalDto {
|
||||
account_id: String,
|
||||
key_id: String,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct SnapshotDto {
|
||||
hard_cap: Option<u64>,
|
||||
spent: u64,
|
||||
reserved: u64,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct ResolveResp {
|
||||
principal: PrincipalDto,
|
||||
#[allow(dead_code)]
|
||||
snapshot: Option<SnapshotDto>,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct ReserveResp {
|
||||
reservation_id: Option<i64>,
|
||||
rejected: Option<Rejection>,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
enum Rejection {
|
||||
InsufficientQuota {
|
||||
requested: u64,
|
||||
available: u64,
|
||||
},
|
||||
RateLimited {
|
||||
requested: u64,
|
||||
available: u64,
|
||||
retry_after_secs: u64,
|
||||
},
|
||||
}
|
||||
|
||||
impl UpstreamEntitlementProvider {
|
||||
pub fn new(cfg: &UpstreamClientConfig) -> Self {
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(cfg.timeout_secs))
|
||||
.build()
|
||||
.expect("failed to build upstream HTTP client");
|
||||
Self {
|
||||
client,
|
||||
base_url: cfg.url.trim_end_matches('/').to_string(),
|
||||
bearer: cfg.bearer.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
fn url(&self, path: &str) -> String {
|
||||
format!("{}{}", self.base_url, path)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl EntitlementProvider for UpstreamEntitlementProvider {
|
||||
async fn resolve(&self, api_key: &str) -> Result<Principal, AuthError> {
|
||||
let resp = self
|
||||
.client
|
||||
.post(self.url("/authz/v1/resolve"))
|
||||
.bearer_auth(&self.bearer)
|
||||
.json(&serde_json::json!({ "api_key": api_key }))
|
||||
.send()
|
||||
.await;
|
||||
let resp = match resp {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "upstream resolve unreachable; failing closed");
|
||||
return Err(AuthError::Unavailable {
|
||||
retry_after_secs: FAIL_CLOSED_RETRY_SECS,
|
||||
});
|
||||
}
|
||||
};
|
||||
if resp.status().as_u16() == 401 {
|
||||
return Err(AuthError::InvalidKey);
|
||||
}
|
||||
if !resp.status().is_success() {
|
||||
return Err(AuthError::Unavailable {
|
||||
retry_after_secs: FAIL_CLOSED_RETRY_SECS,
|
||||
});
|
||||
}
|
||||
match resp.json::<ResolveResp>().await {
|
||||
Ok(r) => Ok(Principal {
|
||||
account_id: r.principal.account_id,
|
||||
key_id: r.principal.key_id,
|
||||
}),
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "upstream resolve: bad body; failing closed");
|
||||
Err(AuthError::Unavailable {
|
||||
retry_after_secs: FAIL_CLOSED_RETRY_SECS,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn reserve(
|
||||
&self,
|
||||
principal: &Principal,
|
||||
max_tokens: u64,
|
||||
) -> Result<Reservation, BudgetError> {
|
||||
let fail_closed = || BudgetError::RateLimited {
|
||||
requested: max_tokens,
|
||||
available: 0,
|
||||
retry_after_secs: FAIL_CLOSED_RETRY_SECS,
|
||||
};
|
||||
let resp = self
|
||||
.client
|
||||
.post(self.url("/authz/v1/reserve"))
|
||||
.bearer_auth(&self.bearer)
|
||||
.json(&serde_json::json!({
|
||||
"account_id": principal.account_id,
|
||||
"key_id": principal.key_id,
|
||||
"max_tokens": max_tokens,
|
||||
}))
|
||||
.send()
|
||||
.await;
|
||||
let resp = match resp {
|
||||
Ok(r) if r.status().is_success() => r,
|
||||
Ok(r) => {
|
||||
tracing::warn!(status = %r.status(), "upstream reserve non-2xx; failing closed");
|
||||
return Err(fail_closed());
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "upstream reserve unreachable; failing closed");
|
||||
return Err(fail_closed());
|
||||
}
|
||||
};
|
||||
match resp.json::<ReserveResp>().await {
|
||||
Ok(ReserveResp {
|
||||
reservation_id: Some(id),
|
||||
..
|
||||
}) => Ok(Reservation {
|
||||
id: id as u64,
|
||||
principal: principal.clone(),
|
||||
reserved: max_tokens,
|
||||
}),
|
||||
Ok(ReserveResp {
|
||||
rejected:
|
||||
Some(Rejection::InsufficientQuota {
|
||||
requested,
|
||||
available,
|
||||
}),
|
||||
..
|
||||
}) => Err(BudgetError::InsufficientQuota {
|
||||
requested,
|
||||
available,
|
||||
}),
|
||||
Ok(ReserveResp {
|
||||
rejected:
|
||||
Some(Rejection::RateLimited {
|
||||
requested,
|
||||
available,
|
||||
retry_after_secs,
|
||||
}),
|
||||
..
|
||||
}) => Err(BudgetError::RateLimited {
|
||||
requested,
|
||||
available,
|
||||
retry_after_secs,
|
||||
}),
|
||||
_ => Err(fail_closed()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn settle(&self, reservation: Reservation, actual_tokens: u64) {
|
||||
// Best-effort; a lost settle is reaped by the upstream sweeper (B2).
|
||||
let _ = self
|
||||
.client
|
||||
.post(self.url("/authz/v1/settle"))
|
||||
.bearer_auth(&self.bearer)
|
||||
.json(&serde_json::json!({
|
||||
"reservation_id": reservation.id as i64,
|
||||
"actual_tokens": actual_tokens,
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.inspect_err(
|
||||
|e| tracing::warn!(error = %e, "upstream settle failed (sweeper will reap)"),
|
||||
);
|
||||
}
|
||||
|
||||
async fn release(&self, reservation: Reservation) {
|
||||
let _ = self
|
||||
.client
|
||||
.post(self.url("/authz/v1/release"))
|
||||
.bearer_auth(&self.bearer)
|
||||
.json(&serde_json::json!({ "reservation_id": reservation.id as i64 }))
|
||||
.send()
|
||||
.await
|
||||
.inspect_err(
|
||||
|e| tracing::warn!(error = %e, "upstream release failed (sweeper will reap)"),
|
||||
);
|
||||
}
|
||||
|
||||
async fn snapshot(&self, principal: &Principal) -> Option<BudgetSnapshot> {
|
||||
let resp = self
|
||||
.client
|
||||
.post(self.url("/authz/v1/snapshot"))
|
||||
.bearer_auth(&self.bearer)
|
||||
.json(&serde_json::json!({
|
||||
"account_id": principal.account_id,
|
||||
"key_id": principal.key_id,
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.ok()?;
|
||||
if !resp.status().is_success() {
|
||||
return None;
|
||||
}
|
||||
let dto = resp.json::<SnapshotDto>().await.ok()?;
|
||||
Some(BudgetSnapshot {
|
||||
hard_cap: dto.hard_cap,
|
||||
spent: dto.spent,
|
||||
reserved: dto.reserved,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
//! Gateway adapter that turns the shared, axum-agnostic
|
||||
//! [`cortex_core::error_envelope::OpenAiError`] into an axum [`Response`],
|
||||
//! setting the `Retry-After` header when the envelope carries one.
|
||||
//!
|
||||
//! cortex-core owns the envelope shape and the rejection contract (#60/#63);
|
||||
//! this is the only place the gateway crosses from that data into axum.
|
||||
|
||||
use axum::http::{HeaderValue, StatusCode, header};
|
||||
use axum::response::{IntoResponse, Json, Response};
|
||||
use cortex_core::error_envelope::OpenAiError;
|
||||
|
||||
/// Render an [`OpenAiError`] as an axum response (status + JSON envelope +
|
||||
/// optional `Retry-After`).
|
||||
pub fn envelope_response(err: OpenAiError) -> Response {
|
||||
let status = StatusCode::from_u16(err.status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
|
||||
let retry_after = err.retry_after_secs;
|
||||
let mut response = (status, Json(err.body())).into_response();
|
||||
if let Some(secs) = retry_after
|
||||
&& let Ok(value) = HeaderValue::from_str(&secs.to_string())
|
||||
{
|
||||
response.headers_mut().insert(header::RETRY_AFTER, value);
|
||||
}
|
||||
response
|
||||
}
|
||||
@@ -17,23 +17,11 @@ pub async fn eviction_loop(fleet: Arc<CortexState>) {
|
||||
}
|
||||
}
|
||||
|
||||
/// Evict the least-recently-used model on a given node that `incoming`
|
||||
/// is permitted to displace.
|
||||
///
|
||||
/// `incoming` is the model the caller is trying to make room for. It is
|
||||
/// load-bearing, not diagnostic: whether a displacement is allowed is a
|
||||
/// question about the *pair*, so an evictor that only knows the victim
|
||||
/// cannot answer it. Passing `None` means "no particular model" and
|
||||
/// permits displacing anything at or below the default priority, which
|
||||
/// is the right reading for maintenance-driven eviction rather than a
|
||||
/// cold-load making room for itself.
|
||||
///
|
||||
/// Returns the model ID that was evicted, or None when nothing on the
|
||||
/// node may be displaced.
|
||||
/// Evict the least-recently-used model on a given node.
|
||||
/// Returns the model ID that was evicted, or None if nothing could be evicted.
|
||||
pub async fn evict_lru_on_node(
|
||||
fleet: &CortexState,
|
||||
node_name: &str,
|
||||
incoming: Option<&str>,
|
||||
) -> anyhow::Result<Option<String>> {
|
||||
let (neuron_endpoint, candidate) = {
|
||||
let nodes = fleet.nodes.read().await;
|
||||
@@ -41,18 +29,13 @@ pub async fn evict_lru_on_node(
|
||||
anyhow::bail!("node '{node_name}' not found");
|
||||
};
|
||||
|
||||
// Oldest first, among the models this incoming model outranks.
|
||||
// Find the loaded model with the oldest last_accessed,
|
||||
// excluding models pinned on this neuron (from catalogue).
|
||||
let candidate = node
|
||||
.models
|
||||
.values()
|
||||
.filter(|m| m.status == ModelStatus::Loaded)
|
||||
.filter(|m| match incoming {
|
||||
Some(inc) => fleet.catalogue.may_displace(inc, &m.id),
|
||||
None => {
|
||||
fleet.catalogue.residency_priority(&m.id)
|
||||
<= cortex_core::catalogue::DEFAULT_RESIDENCY_PRIORITY
|
||||
}
|
||||
})
|
||||
.filter(|m| !fleet.catalogue.is_pinned(&m.id, node_name))
|
||||
.min_by_key(|m| m.last_accessed)
|
||||
.map(|m| m.id.clone());
|
||||
|
||||
@@ -60,11 +43,7 @@ pub async fn evict_lru_on_node(
|
||||
};
|
||||
|
||||
let Some(model_id) = candidate else {
|
||||
tracing::info!(
|
||||
node = node_name,
|
||||
incoming = incoming.unwrap_or("(none)"),
|
||||
"no displaceable models found"
|
||||
);
|
||||
tracing::info!(node = node_name, "no evictable models found");
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,41 +1,22 @@
|
||||
pub mod anthropic_sse;
|
||||
pub mod auth;
|
||||
pub mod entitlements_chain;
|
||||
pub mod entitlements_local;
|
||||
pub mod entitlements_upstream;
|
||||
pub mod error;
|
||||
pub mod evictor;
|
||||
pub mod handlers;
|
||||
pub mod metering;
|
||||
pub mod metrics;
|
||||
pub mod poller;
|
||||
pub mod proxy;
|
||||
pub mod router;
|
||||
pub mod served_usage;
|
||||
pub mod state;
|
||||
|
||||
use anyhow::Result;
|
||||
use axum::Router;
|
||||
use axum::middleware::from_fn_with_state;
|
||||
use cortex_core::config::GatewayConfig;
|
||||
use std::sync::Arc;
|
||||
use tower_http::cors::CorsLayer;
|
||||
use tower_http::trace::TraceLayer;
|
||||
|
||||
/// Build the Axum application router with all routes wired up.
|
||||
///
|
||||
/// Layer order (outermost first): trace → CORS → auth → handlers. CORS is
|
||||
/// outer to auth so preflight `OPTIONS` short-circuits before resolution;
|
||||
/// auth (`require_principal`) resolves the bearer key, attaches the
|
||||
/// principal, and stamps the internal principal headers before any handler
|
||||
/// runs.
|
||||
pub fn build_app(fleet: Arc<state::CortexState>) -> Router {
|
||||
Router::new()
|
||||
.merge(handlers::api_routes())
|
||||
.layer(from_fn_with_state(
|
||||
Arc::clone(&fleet),
|
||||
auth::require_principal,
|
||||
))
|
||||
.layer(CorsLayer::permissive())
|
||||
.layer(TraceLayer::new_for_http())
|
||||
.with_state(fleet)
|
||||
@@ -58,28 +39,6 @@ pub async fn run(config: GatewayConfig) -> Result<()> {
|
||||
evictor::eviction_loop(evictor_fleet).await;
|
||||
});
|
||||
|
||||
// Served-usage reporter (#58): when this operator is part of the mesh,
|
||||
// periodically flush absolute per-principal served-token counters to
|
||||
// upstream for reconciliation.
|
||||
if config.upstream.enabled {
|
||||
let su_fleet = Arc::clone(&fleet);
|
||||
let url = config.upstream.url.clone();
|
||||
let bearer = config.upstream.bearer.clone();
|
||||
let interval =
|
||||
std::time::Duration::from_secs(config.upstream.served_usage_report_interval_secs);
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::time::sleep(interval).await;
|
||||
let rows = su_fleet.served_usage.snapshot();
|
||||
if let Err(e) =
|
||||
served_usage::report(&su_fleet.http_client, &url, &bearer, &rows).await
|
||||
{
|
||||
tracing::warn!(error = %e, "served-usage report failed (will retry)");
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let app = build_app(Arc::clone(&fleet));
|
||||
|
||||
let listen_addr = config.gateway.listen.parse::<std::net::SocketAddr>()?;
|
||||
|
||||
@@ -1,307 +0,0 @@
|
||||
//! Per-request token metering (#51).
|
||||
//!
|
||||
//! Captures the real `(prompt, completion)` usage of every request and feeds
|
||||
//! it to two places: the [`EntitlementProvider`] spend ledger (via
|
||||
//! reserve→settle) and per-principal Prometheus counters. The principal is
|
||||
//! reconstructed from the internal headers the auth middleware stamped (#49),
|
||||
//! so this works uniformly across every proxy path without threading the
|
||||
//! typed principal through each handler.
|
||||
//!
|
||||
//! The reserve→settle lifecycle is established here but, in this phase,
|
||||
//! reserves **zero** tokens — metering only, no enforcement. Budget
|
||||
//! enforcement (#52) flips the reserved amount to the real
|
||||
//! `prompt + max_output` and handles the [`BudgetError`] rejection; the
|
||||
//! settle/release plumbing is identical, so that change is localized.
|
||||
//!
|
||||
//! [`ReservationGuard`] makes leaks impossible: settling records actual
|
||||
//! spend and releases the unused remainder; dropping a guard that was never
|
||||
//! settled releases the whole reservation. So an early return, error path,
|
||||
//! or dropped stream can't strand a reservation.
|
||||
|
||||
use axum::http::HeaderMap;
|
||||
use cortex_core::entitlements::{
|
||||
BudgetError, EntitlementProvider, HEADER_ACCOUNT_ID, HEADER_KEY_ID, Principal,
|
||||
};
|
||||
use cortex_core::error_envelope::OpenAiError;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Fallback output-token budget when neither the request nor the model's
|
||||
/// advertised limit gives one. Bounds the reservation so a capped key is
|
||||
/// still gated even on under-specified requests (#52).
|
||||
pub const FALLBACK_MAX_OUTPUT: u64 = 4096;
|
||||
|
||||
/// Invoked exactly once at request completion with best-effort
|
||||
/// `(prompt_tokens, completion_tokens)`. When no usage could be observed
|
||||
/// (e.g. a pre-dispatch failure or a dropped stream) it is dropped unused —
|
||||
/// which releases the held reservation via [`ReservationGuard`]'s `Drop`.
|
||||
pub type UsageSink = Box<dyn FnOnce(u64, u64) + Send>;
|
||||
|
||||
/// Reconstruct the principal from the cortex-stamped internal headers. The
|
||||
/// auth middleware strips any client copy and stamps the authoritative value,
|
||||
/// so these headers are trustworthy within cortex. `None` for anonymous
|
||||
/// (unauthenticated) requests.
|
||||
pub fn principal_from_headers(headers: &HeaderMap) -> Option<Principal> {
|
||||
let account_id = headers.get(HEADER_ACCOUNT_ID)?.to_str().ok()?.to_string();
|
||||
let key_id = headers.get(HEADER_KEY_ID)?.to_str().ok()?.to_string();
|
||||
Some(Principal { account_id, key_id })
|
||||
}
|
||||
|
||||
/// Emit per-principal spend counters (#51). Labelled by account/key only —
|
||||
/// both are operator-bounded, so cardinality is controlled.
|
||||
pub fn record_spend(principal: &Principal, prompt: u64, completion: u64) {
|
||||
let labels = [
|
||||
("account", principal.account_id.clone()),
|
||||
("key", principal.key_id.clone()),
|
||||
];
|
||||
metrics::counter!("cortex_spend_tokens_total", &labels).increment(prompt + completion);
|
||||
metrics::counter!("cortex_spend_prompt_tokens_total", &labels).increment(prompt);
|
||||
metrics::counter!("cortex_spend_completion_tokens_total", &labels).increment(completion);
|
||||
}
|
||||
|
||||
/// Holds a budget reservation for the life of a request. [`settle`] records
|
||||
/// actual spend and releases the remainder; an un-settled guard releases the
|
||||
/// whole reservation when dropped. Anonymous requests carry an empty guard,
|
||||
/// where every operation is a no-op.
|
||||
///
|
||||
/// [`settle`]: ReservationGuard::settle
|
||||
pub struct ReservationGuard {
|
||||
provider: Arc<dyn EntitlementProvider>,
|
||||
reservation: Option<cortex_core::entitlements::Reservation>,
|
||||
}
|
||||
|
||||
impl ReservationGuard {
|
||||
/// An empty guard for an anonymous request — no reservation to resolve.
|
||||
pub fn anonymous(provider: Arc<dyn EntitlementProvider>) -> Self {
|
||||
Self {
|
||||
provider,
|
||||
reservation: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Wrap an already-acquired reservation.
|
||||
fn held(
|
||||
provider: Arc<dyn EntitlementProvider>,
|
||||
reservation: cortex_core::entitlements::Reservation,
|
||||
) -> Self {
|
||||
Self {
|
||||
provider,
|
||||
reservation: Some(reservation),
|
||||
}
|
||||
}
|
||||
|
||||
/// Settle with the tokens actually consumed, disarming the drop-release.
|
||||
/// Spawns the (fast, in-process for the local provider) settle so the
|
||||
/// caller — which may be a sync stream-completion callback — needn't
|
||||
/// await.
|
||||
pub fn settle(mut self, actual_tokens: u64) {
|
||||
if let Some(reservation) = self.reservation.take() {
|
||||
let provider = Arc::clone(&self.provider);
|
||||
tokio::spawn(async move {
|
||||
provider.settle(reservation, actual_tokens).await;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ReservationGuard {
|
||||
fn drop(&mut self) {
|
||||
if let Some(reservation) = self.reservation.take() {
|
||||
let provider = Arc::clone(&self.provider);
|
||||
tokio::spawn(async move {
|
||||
provider.release(reservation).await;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the completion sink for an authenticated request: record spend and
|
||||
/// settle the reservation with the observed total. Dropping it unused (no
|
||||
/// usage observed) releases the reservation via the guard.
|
||||
pub fn usage_sink(
|
||||
principal: Principal,
|
||||
guard: ReservationGuard,
|
||||
served_usage: std::sync::Arc<crate::served_usage::ServedUsage>,
|
||||
) -> UsageSink {
|
||||
Box::new(move |prompt, completion| {
|
||||
record_spend(&principal, prompt, completion);
|
||||
// Per-principal served-usage tally for #58 reconciliation. Recorded
|
||||
// for every metered (authenticated) request; the flush task reports
|
||||
// it to upstream when the operator is part of the mesh.
|
||||
served_usage.add(
|
||||
&principal.account_id,
|
||||
&principal.key_id,
|
||||
prompt + completion,
|
||||
);
|
||||
guard.settle(prompt + completion);
|
||||
})
|
||||
}
|
||||
|
||||
/// Reserve the request's upper-bound token cost for the principal, refusing
|
||||
/// *before* dispatch if it would exceed the hard cap (#52). On success
|
||||
/// returns a guard the caller settles with actual usage; on refusal returns
|
||||
/// the #63 envelope (`rate_limit_exceeded` + `Retry-After` for a resetting
|
||||
/// window, `insufficient_quota` for a hard balance — never `402`).
|
||||
// The `Err` variant is the #63 envelope itself, which is a data-carrying
|
||||
// struct (status, type, code, message, param, retry hint, diagnostics) and
|
||||
// is passed by value everywhere it is built, matched and rendered. Boxing it
|
||||
// at this one boundary would push `Box` conversions through three handlers
|
||||
// and both HTTP adapters to save an allocation on a path taken once per
|
||||
// rejected request. clippy 1.98 lowered what it considers large enough to
|
||||
// flag; the shape is deliberate. Same call as the existing allows on the
|
||||
// figment-returning config loaders.
|
||||
#[allow(clippy::result_large_err)]
|
||||
pub async fn reserve_or_reject(
|
||||
provider: Arc<dyn EntitlementProvider>,
|
||||
principal: &Principal,
|
||||
max_tokens: u64,
|
||||
) -> Result<ReservationGuard, OpenAiError> {
|
||||
match provider.reserve(principal, max_tokens).await {
|
||||
Ok(reservation) => Ok(ReservationGuard::held(provider, reservation)),
|
||||
Err(err) => Err(budget_error_to_envelope(err)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Budget-equivalence rate for image generation (#202): how many
|
||||
/// token-budget units one megapixel-step consumes. The metered unit
|
||||
/// IS megapixel-steps (`usage.helexa_image_units`, reported end to
|
||||
/// end); this rate only bridges it into the single-currency budget
|
||||
/// ledger the clearing house runs today, so per-key caps and
|
||||
/// fail-closed enforcement apply to images without an upstream
|
||||
/// protocol change. A native image unit in the clearing-house
|
||||
/// contract is the follow-up recorded on #202.
|
||||
///
|
||||
/// 1000 tokens/Mp-step ≈ a 9-step 1024² image (9.44 units) costing
|
||||
/// ~9.4k tokens of budget — the same order as a large chat turn.
|
||||
pub const TOKENS_PER_IMAGE_UNIT: u64 = 1000;
|
||||
|
||||
/// Upper-bound megapixel-steps for an images request (#202): parsed
|
||||
/// size (default 1024²) × steps (default 9), doubled when a negative
|
||||
/// prompt requests CFG, floored at one unit. Over-reserving is safe —
|
||||
/// settle corrects to the actual `usage.helexa_image_units`.
|
||||
pub fn image_reservation_units(body: &[u8]) -> f64 {
|
||||
let (mut width, mut height, mut steps, mut cfg) = (1024usize, 1024usize, 9usize, false);
|
||||
if let Ok(v) = serde_json::from_slice::<serde_json::Value>(body) {
|
||||
if let Some(size) = v.get("size").and_then(serde_json::Value::as_str)
|
||||
&& let Some((w, h)) = size.split_once(['x', 'X'])
|
||||
&& let (Ok(w), Ok(h)) = (w.trim().parse(), h.trim().parse())
|
||||
{
|
||||
width = w;
|
||||
height = h;
|
||||
}
|
||||
if let Some(n) = v.get("num_steps").and_then(serde_json::Value::as_u64) {
|
||||
steps = n as usize;
|
||||
}
|
||||
cfg = v
|
||||
.get("negative_prompt")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.is_some_and(|s| !s.is_empty());
|
||||
}
|
||||
let mp = (width as f64 * height as f64) / 1_000_000.0;
|
||||
let units = mp * steps as f64 * if cfg { 2.0 } else { 1.0 };
|
||||
units.max(1.0)
|
||||
}
|
||||
|
||||
/// Convert megapixel-steps to budget tokens, rounding up so fractional
|
||||
/// units still cost at least their share.
|
||||
pub fn image_units_to_tokens(units: f64) -> u64 {
|
||||
(units * TOKENS_PER_IMAGE_UNIT as f64).ceil() as u64
|
||||
}
|
||||
|
||||
/// Record per-principal image spend (#202) and settle the reservation
|
||||
/// with the actual generated units.
|
||||
pub fn settle_image_usage(
|
||||
principal: &Principal,
|
||||
guard: ReservationGuard,
|
||||
served_usage: &crate::served_usage::ServedUsage,
|
||||
actual_units: f64,
|
||||
) {
|
||||
let labels = [
|
||||
("account", principal.account_id.clone()),
|
||||
("key", principal.key_id.clone()),
|
||||
];
|
||||
// Milli-units keep the counter integral without losing precision
|
||||
// that matters at reconciliation scale.
|
||||
metrics::counter!("cortex_spend_image_milliunits_total", &labels)
|
||||
.increment((actual_units * 1000.0).round() as u64);
|
||||
let tokens = image_units_to_tokens(actual_units);
|
||||
served_usage.add(&principal.account_id, &principal.key_id, tokens);
|
||||
guard.settle(tokens);
|
||||
}
|
||||
|
||||
/// Map a [`BudgetError`] to the #63 envelope. The provider chose the window
|
||||
/// semantics; this only translates them to HTTP.
|
||||
fn budget_error_to_envelope(err: BudgetError) -> OpenAiError {
|
||||
match err {
|
||||
BudgetError::RateLimited {
|
||||
retry_after_secs, ..
|
||||
} => OpenAiError::rate_limit_exceeded(err.to_string(), retry_after_secs),
|
||||
BudgetError::InsufficientQuota { .. } => OpenAiError::insufficient_quota(err.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Upper-bound tokens to reserve for a request (#52): an over-estimate of
|
||||
/// the prompt plus the maximum output. `advertised_output` is the model's
|
||||
/// `limit.output` (#62), used when the request omits `max_(completion_)tokens`.
|
||||
/// Over-reserving is safe — settle corrects spend to the actual usage.
|
||||
pub fn reservation_estimate(body: &[u8], advertised_output: Option<u64>) -> u64 {
|
||||
let max_output = requested_max_output(body)
|
||||
.or(advertised_output)
|
||||
.unwrap_or(FALLBACK_MAX_OUTPUT);
|
||||
estimate_prompt_tokens(body).saturating_add(max_output)
|
||||
}
|
||||
|
||||
/// The client's requested output cap, from `max_completion_tokens` (or the
|
||||
/// legacy `max_tokens`). `None` when unspecified.
|
||||
fn requested_max_output(body: &[u8]) -> Option<u64> {
|
||||
let v: serde_json::Value = serde_json::from_slice(body).ok()?;
|
||||
v.get("max_completion_tokens")
|
||||
.or_else(|| v.get("max_tokens"))
|
||||
.and_then(serde_json::Value::as_u64)
|
||||
}
|
||||
|
||||
/// Rough prompt-token estimate at ~4 chars/token over the whole body. cortex
|
||||
/// has no tokenizer; JSON overhead makes this a conservative over-estimate,
|
||||
/// and neuron remains the exact context wall (#56/#60). Settle reconciles to
|
||||
/// the real usage afterward.
|
||||
fn estimate_prompt_tokens(body: &[u8]) -> u64 {
|
||||
(body.len() as u64 / 4).max(1)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn requested_max_output_prefers_max_completion_tokens() {
|
||||
let body = br#"{"model":"m","max_completion_tokens":256,"max_tokens":99}"#;
|
||||
assert_eq!(requested_max_output(body), Some(256));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn requested_max_output_falls_back_to_legacy_max_tokens() {
|
||||
let body = br#"{"model":"m","max_tokens":128}"#;
|
||||
assert_eq!(requested_max_output(body), Some(128));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn estimate_uses_requested_output_when_present() {
|
||||
// Requested output dominates; prompt estimate is small for a tiny body.
|
||||
let body = br#"{"model":"m","max_tokens":1000}"#;
|
||||
let est = reservation_estimate(body, Some(8192));
|
||||
assert!((1000..1100).contains(&est), "est was {est}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn estimate_uses_advertised_output_when_request_omits_it() {
|
||||
let body = br#"{"model":"m","messages":[]}"#;
|
||||
let est = reservation_estimate(body, Some(8192));
|
||||
assert!(est >= 8192, "est was {est}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn estimate_falls_back_when_nothing_advertised() {
|
||||
let body = br#"{"model":"m"}"#;
|
||||
let est = reservation_estimate(body, None);
|
||||
assert!(est >= FALLBACK_MAX_OUTPUT, "est was {est}");
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,7 @@
|
||||
//! in Prometheus text format.
|
||||
|
||||
use anyhow::Result;
|
||||
use metrics_exporter_prometheus::{Matcher, PrometheusBuilder};
|
||||
use metrics_exporter_prometheus::PrometheusBuilder;
|
||||
use std::net::SocketAddr;
|
||||
|
||||
/// Install the Prometheus metrics recorder and return a handle.
|
||||
@@ -12,7 +12,8 @@ use std::net::SocketAddr;
|
||||
pub fn install(listen: &str) -> Result<()> {
|
||||
let addr: SocketAddr = listen.parse()?;
|
||||
|
||||
with_buckets(PrometheusBuilder::new().with_http_listener(addr))?
|
||||
PrometheusBuilder::new()
|
||||
.with_http_listener(addr)
|
||||
.install()
|
||||
.map_err(|e| anyhow::anyhow!("failed to install Prometheus exporter: {e}"))?;
|
||||
|
||||
@@ -24,77 +25,13 @@ pub fn install(listen: &str) -> Result<()> {
|
||||
/// Install a recorder for testing (no HTTP listener). Returns a handle
|
||||
/// that can render the current metrics as Prometheus text.
|
||||
pub fn install_test_recorder() -> Result<metrics_exporter_prometheus::PrometheusHandle> {
|
||||
let handle = with_buckets(PrometheusBuilder::new())?
|
||||
let handle = PrometheusBuilder::new()
|
||||
.install_recorder()
|
||||
.map_err(|e| anyhow::anyhow!("failed to install test recorder: {e}"))?;
|
||||
describe_metrics();
|
||||
Ok(handle)
|
||||
}
|
||||
|
||||
/// Give every histogram explicit buckets, so it is exported as a
|
||||
/// Prometheus **histogram** rather than a summary.
|
||||
///
|
||||
/// Without this, `metrics-exporter-prometheus` renders every
|
||||
/// `histogram!` as a summary: `{quantile="0.95"}` series and no
|
||||
/// `_bucket` series at all. Three things break as a result, and the
|
||||
/// third is what made it visible.
|
||||
///
|
||||
/// 1. `histogram_quantile()` has nothing to read, so any dashboard
|
||||
/// panel written the idiomatic way returns *No data* forever. The
|
||||
/// fleet dashboard's TTFT panel did exactly that while cortex was
|
||||
/// serving thousands of requests.
|
||||
/// 2. Summary quantiles are computed per process over a rolling
|
||||
/// window, so they **cannot be aggregated**. Averaging p95 across
|
||||
/// two gateways is not p95 of anything.
|
||||
/// 3. They decay: with no samples in the window every quantile reads
|
||||
/// `0`, which is indistinguishable from "genuinely instant" on a
|
||||
/// graph.
|
||||
///
|
||||
/// Buckets are per-metric because the quantities differ by orders of
|
||||
/// magnitude — a request lasting minutes and a TTFT of milliseconds
|
||||
/// have no useful shared scale. Ranges are chosen from observed fleet
|
||||
/// behaviour rather than round numbers: decode runs for minutes on a
|
||||
/// long turn, prefill is seconds on a long prompt, and decode
|
||||
/// throughput sits in the tens of tokens/sec.
|
||||
fn with_buckets(builder: PrometheusBuilder) -> Result<PrometheusBuilder> {
|
||||
let seconds_short = &[
|
||||
0.05, 0.1, 0.25, 0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 30.0, 60.0, 120.0,
|
||||
];
|
||||
let seconds_long = &[
|
||||
0.1, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0, 1200.0,
|
||||
];
|
||||
let tokens_per_second = &[
|
||||
1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0, 40.0, 60.0, 80.0, 120.0, 200.0,
|
||||
];
|
||||
builder
|
||||
// Whole-request latency: a long agentic turn legitimately runs
|
||||
// for many minutes, so the tail has to reach there.
|
||||
.set_buckets_for_metric(
|
||||
Matcher::Full("cortex_request_duration_seconds".into()),
|
||||
seconds_long,
|
||||
)?
|
||||
// Prefill: sub-second for a short prompt, seconds for a long
|
||||
// one. Anything past a minute is pathological and belongs in
|
||||
// the overflow bucket.
|
||||
.set_buckets_for_metric(
|
||||
Matcher::Full("cortex_time_to_first_token_seconds".into()),
|
||||
seconds_short,
|
||||
)?
|
||||
// Image generation is inherently slower than a text turn and
|
||||
// scales with resolution and step count.
|
||||
.set_buckets_for_metric(
|
||||
Matcher::Full("cortex_images_generation_seconds".into()),
|
||||
seconds_long,
|
||||
)?
|
||||
// Not a duration — decode throughput, tens of tokens/sec on
|
||||
// this fleet.
|
||||
.set_buckets_for_metric(
|
||||
Matcher::Full("cortex_tokens_per_second".into()),
|
||||
tokens_per_second,
|
||||
)
|
||||
.map_err(|e| anyhow::anyhow!("failed to configure histogram buckets: {e}"))
|
||||
}
|
||||
|
||||
fn describe_metrics() {
|
||||
metrics::describe_histogram!(
|
||||
"cortex_request_duration_seconds",
|
||||
@@ -109,131 +46,13 @@ fn describe_metrics() {
|
||||
"Generation throughput in tokens per second"
|
||||
);
|
||||
metrics::describe_counter!("cortex_requests_total", "Total number of proxied requests");
|
||||
metrics::describe_counter!(
|
||||
"cortex_prompt_tokens_total",
|
||||
"Total prompt tokens reported by upstream usage objects"
|
||||
);
|
||||
metrics::describe_counter!(
|
||||
"cortex_completion_tokens_total",
|
||||
"Total completion tokens reported by upstream usage objects"
|
||||
);
|
||||
metrics::describe_counter!(
|
||||
"cortex_request_errors_total",
|
||||
"Total number of failed proxy requests"
|
||||
);
|
||||
metrics::describe_counter!(
|
||||
"cortex_cached_prompt_tokens_total",
|
||||
"Prompt tokens served from neuron's prefix KV cache; over cortex_prompt_tokens_total this is the cache-hit rate (#269)"
|
||||
);
|
||||
metrics::describe_counter!("cortex_evictions_total", "Total number of model evictions");
|
||||
metrics::describe_counter!(
|
||||
"cortex_cold_starts_total",
|
||||
"Total number of cold-start model loads"
|
||||
);
|
||||
metrics::describe_counter!(
|
||||
"cortex_spend_tokens_total",
|
||||
"Total metered tokens (prompt + completion) per principal, labelled by account/key (#51)"
|
||||
);
|
||||
metrics::describe_counter!(
|
||||
"cortex_spend_prompt_tokens_total",
|
||||
"Metered prompt tokens per principal, labelled by account/key (#51)"
|
||||
);
|
||||
metrics::describe_counter!(
|
||||
"cortex_spend_completion_tokens_total",
|
||||
"Metered completion tokens per principal, labelled by account/key (#51)"
|
||||
);
|
||||
// Live capacity signals polled from neuron /health (#137), {node,model}.
|
||||
metrics::describe_gauge!(
|
||||
"cortex_model_in_flight",
|
||||
"Requests currently running on a neuron:model (#137)"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
"cortex_model_queue_depth",
|
||||
"Requests queued in admission for a neuron:model (#137)"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
"cortex_model_max_in_flight",
|
||||
"Configured concurrency ceiling; saturation = in_flight / max_in_flight (#137)"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
"cortex_model_max_queue_depth",
|
||||
"Configured admission queue capacity before a neuron:model sheds load (#137)"
|
||||
);
|
||||
// Per-device GPU headroom polled from neuron /health (#137), {node,device}.
|
||||
metrics::describe_gauge!(
|
||||
"cortex_device_vram_used_mb",
|
||||
"Per-device VRAM used, MB (#137)"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
"cortex_device_vram_free_mb",
|
||||
"Per-device VRAM free, MB (#137)"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
"cortex_device_utilization_pct",
|
||||
"Per-device GPU utilization, percent (#137)"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
"cortex_device_temp_c",
|
||||
"Per-device GPU temperature, Celsius (#137)"
|
||||
);
|
||||
metrics::describe_counter!(
|
||||
"cortex_model_rejections_total",
|
||||
"Admission rejections per neuron:model by reason: queue_full / wait_timeout / per_principal / anon_yield — the load-shedding signal (#137, #262)"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
"cortex_model_anon_in_flight",
|
||||
"Anonymous (unattributable) requests holding a seat on a neuron:model (#262)"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
"cortex_model_anon_max_in_flight",
|
||||
"Seats anonymous traffic may hold at once, so it cannot starve identified callers (#262)"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
"cortex_model_tok_s_decode",
|
||||
"Live decode throughput per neuron:model, tokens/sec EMA — the headline capacity number (#137)"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
"cortex_model_tok_s_prefill",
|
||||
"Live prefill throughput per neuron:model, tokens/sec EMA (#137)"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod bucket_tests {
|
||||
/// Histograms must export as Prometheus **histograms**, with
|
||||
/// `_bucket` series — not as summaries.
|
||||
///
|
||||
/// This asserts the exported *shape*, not the recording call,
|
||||
/// because the recording was never the problem: cortex measured
|
||||
/// TTFT correctly for months while the fleet dashboard's panel sat
|
||||
/// on "No data", since `histogram_quantile()` reads `_bucket`
|
||||
/// series and a summary has none. A metric that is collected but
|
||||
/// cannot be queried is indistinguishable from one that was never
|
||||
/// collected — from the graph, and from the operator's chair.
|
||||
#[test]
|
||||
fn histograms_export_buckets_not_summary_quantiles() {
|
||||
let handle = match super::install_test_recorder() {
|
||||
Ok(h) => h,
|
||||
// Another test in this binary owns the global recorder;
|
||||
// it installs the same buckets, so skipping is honest.
|
||||
Err(_) => return,
|
||||
};
|
||||
metrics::histogram!("cortex_time_to_first_token_seconds", "node" => "n", "model" => "m")
|
||||
.record(0.42);
|
||||
let rendered = handle.render();
|
||||
|
||||
assert!(
|
||||
rendered.contains("cortex_time_to_first_token_seconds_bucket"),
|
||||
"no _bucket series — histogram_quantile() cannot read this:\n{rendered}"
|
||||
);
|
||||
assert!(
|
||||
rendered.contains("# TYPE cortex_time_to_first_token_seconds histogram"),
|
||||
"exported as the wrong metric type:\n{rendered}"
|
||||
);
|
||||
assert!(
|
||||
!rendered
|
||||
.contains(r#"cortex_time_to_first_token_seconds{node="n",model="m",quantile="#),
|
||||
"still exporting summary quantiles, which cannot be aggregated across gateways"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,38 +3,13 @@
|
||||
|
||||
use crate::state::CortexState;
|
||||
use chrono::Utc;
|
||||
use cortex_core::discovery::{DiscoveryResponse, HealthResponse};
|
||||
use cortex_core::harness::ModelInfo;
|
||||
use cortex_core::node::{ModelEntry, ModelStatus, NodeState};
|
||||
use metrics::{counter, gauge};
|
||||
use cortex_core::node::{ModelEntry, ModelStatus};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
const POLL_INTERVAL: Duration = Duration::from_secs(10);
|
||||
|
||||
/// Consecutive failed `/models` polls before a node is marked unhealthy.
|
||||
/// Debounces transient misses (a busy neuron briefly slow to answer) so a
|
||||
/// single blip can't yank a node — and its models — out of routing. At the
|
||||
/// 10s poll interval this tolerates ~20s of flapping before evicting.
|
||||
const POLL_FAILURE_THRESHOLD: u32 = 3;
|
||||
|
||||
/// How long before re-asking the fleet about a model whose capabilities
|
||||
/// nobody could derive (#241). Long, because the usual cause — no node
|
||||
/// has the weights cached — is resolved by a deliberate operator action
|
||||
/// (a load, a prefetch) rather than by time passing.
|
||||
const CAPABILITY_RETRY: Duration = Duration::from_secs(600);
|
||||
|
||||
/// Record a failed poll for `node`, marking it unhealthy only once failures
|
||||
/// reach [`POLL_FAILURE_THRESHOLD`]. Below the threshold the node keeps its
|
||||
/// last-known health, riding over transient misses. A successful poll resets
|
||||
/// the counter (see the success arm in `poll_once`).
|
||||
fn record_poll_failure(node: &mut NodeState) {
|
||||
node.consecutive_poll_failures = node.consecutive_poll_failures.saturating_add(1);
|
||||
if node.consecutive_poll_failures >= POLL_FAILURE_THRESHOLD {
|
||||
node.healthy = false;
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs forever, polling all neurons on a fixed interval.
|
||||
pub async fn poll_loop(fleet: Arc<CortexState>) {
|
||||
loop {
|
||||
@@ -50,68 +25,7 @@ pub async fn poll_once(fleet: &CortexState) {
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch `GET /discovery` and cache it on the NodeState — topology is
|
||||
/// invariant for a given neuron process, so a successful fetch is kept.
|
||||
/// Re-polled only while `max_prompt_tokens` is still unknown (0): on a
|
||||
/// rolling deploy cortex can win the race and cache a neuron's discovery
|
||||
/// before that neuron reports the field (it deserialises to 0). Re-polling
|
||||
/// until a real cap arrives self-heals that without periodic polling.
|
||||
async fn maybe_poll_discovery(fleet: &CortexState, name: &str, endpoint: &str) {
|
||||
{
|
||||
let nodes = fleet.nodes.read().await;
|
||||
match nodes.get(name) {
|
||||
Some(n)
|
||||
if n.discovery
|
||||
.as_ref()
|
||||
.is_some_and(|d| d.max_prompt_tokens > 0) =>
|
||||
{
|
||||
return;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let url = format!("{endpoint}/discovery");
|
||||
let resp = match fleet
|
||||
.http_client
|
||||
.get(&url)
|
||||
.timeout(Duration::from_secs(5))
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(r) if r.status().is_success() => r,
|
||||
Ok(r) => {
|
||||
tracing::debug!(node = name, status = %r.status(), "discovery probe non-success");
|
||||
return;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::debug!(node = name, error = %e, "discovery probe unreachable");
|
||||
return;
|
||||
}
|
||||
};
|
||||
match resp.json::<DiscoveryResponse>().await {
|
||||
Ok(d) => {
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
if let Some(node) = nodes.get_mut(name) {
|
||||
tracing::info!(
|
||||
node = name,
|
||||
hostname = %d.hostname,
|
||||
devices = d.devices.len(),
|
||||
"discovery cached"
|
||||
);
|
||||
node.discovery = Some(d);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(node = name, error = %e, "failed to parse /discovery response");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn poll_neuron(fleet: &CortexState, name: &str, endpoint: &str) {
|
||||
// Topology first — cheap once cached, and the router needs it to
|
||||
// route requests against catalogue entries that aren't loaded yet.
|
||||
maybe_poll_discovery(fleet, name, endpoint).await;
|
||||
|
||||
let url = format!("{endpoint}/models");
|
||||
|
||||
let result = fleet
|
||||
@@ -130,18 +44,6 @@ async fn poll_neuron(fleet: &CortexState, name: &str, endpoint: &str) {
|
||||
Ok(resp) if resp.status().is_success() => {
|
||||
match resp.json::<Vec<ModelInfo>>().await {
|
||||
Ok(models) => {
|
||||
// Node-level ladder, kept only as a fallback for a
|
||||
// model whose own entry carries none. It is NOT the
|
||||
// source of truth any more: availability depends on
|
||||
// the model's `max_in_flight`, so two models on one
|
||||
// host can legitimately offer different rungs and a
|
||||
// single node-level copy would attribute one's
|
||||
// withheld rung to the other. Per-model is set below.
|
||||
node.reasoning_budget = models
|
||||
.iter()
|
||||
.map(|m| m.reasoning_budget.clone())
|
||||
.find(|rungs| !rungs.is_empty())
|
||||
.unwrap_or_default();
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
for upstream in &models {
|
||||
seen.insert(upstream.id.clone());
|
||||
@@ -152,40 +54,25 @@ async fn poll_neuron(fleet: &CortexState, name: &str, endpoint: &str) {
|
||||
.and_modify(|e| {
|
||||
e.status = status;
|
||||
e.vram_estimate_mb = upstream.vram_used_mb;
|
||||
e.capabilities = upstream.capabilities.clone();
|
||||
e.tool_call = upstream.tool_call;
|
||||
e.reasoning = upstream.reasoning;
|
||||
// Neuron's self-derived limit (#67) — the
|
||||
// authoritative source the gateway advertises.
|
||||
e.limit = upstream.limit.clone();
|
||||
e.servable = upstream.servable.clone();
|
||||
e.reasoning_budget = upstream.reasoning_budget.clone();
|
||||
})
|
||||
.or_insert_with(|| ModelEntry {
|
||||
id: upstream.id.clone(),
|
||||
status,
|
||||
last_accessed: None,
|
||||
vram_estimate_mb: upstream.vram_used_mb,
|
||||
capabilities: upstream.capabilities.clone(),
|
||||
tool_call: upstream.tool_call,
|
||||
reasoning: upstream.reasoning,
|
||||
limit: upstream.limit.clone(),
|
||||
servable: upstream.servable.clone(),
|
||||
reasoning_budget: upstream.reasoning_budget.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
// Remove models no longer reported by the neuron.
|
||||
node.models.retain(|id, _| seen.contains(id));
|
||||
|
||||
node.consecutive_poll_failures = 0;
|
||||
node.healthy = true;
|
||||
node.last_poll = Some(Utc::now());
|
||||
tracing::debug!(node = name, models = models.len(), "poll ok");
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(node = name, error = %e, "failed to parse /models response");
|
||||
record_poll_failure(node);
|
||||
node.healthy = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -195,232 +82,13 @@ async fn poll_neuron(fleet: &CortexState, name: &str, endpoint: &str) {
|
||||
status = %resp.status(),
|
||||
"neuron returned non-success status"
|
||||
);
|
||||
record_poll_failure(node);
|
||||
node.healthy = false;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(node = name, error = %e, "failed to reach neuron");
|
||||
record_poll_failure(node);
|
||||
node.healthy = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Release the write lock before the next HTTP call.
|
||||
drop(nodes);
|
||||
|
||||
discover_capabilities(fleet, name, endpoint).await;
|
||||
|
||||
// Poll /health for the activation snapshot. We don't want this to
|
||||
// flip the node to unhealthy on its own — a neuron that's serving
|
||||
// /models fine is still operational even if /health is briefly
|
||||
// unavailable — so failures are debug-level and leave the existing
|
||||
// activation reading in place.
|
||||
poll_health(fleet, name, endpoint).await;
|
||||
}
|
||||
|
||||
/// Ask a neuron what the catalogue models it has never loaded can do
|
||||
/// (#241).
|
||||
///
|
||||
/// A loaded model reports its modalities on every `/models` poll, but a
|
||||
/// cold one reported nothing at all — which is why image generation was
|
||||
/// invisible on `/v1/models`, the image model being evicted almost all
|
||||
/// of the time. The neuron derives the answer from its local model cache
|
||||
/// without loading anything.
|
||||
///
|
||||
/// Only ever asks about ids it has no answer for, so a steady fleet
|
||||
/// costs nothing after the first cycle: what a given model id can do is
|
||||
/// a property of the model, not of the moment. A 404 means this node has
|
||||
/// no local evidence — another node may still have the weights cached,
|
||||
/// so nothing is recorded and the id is retried against the next node.
|
||||
///
|
||||
/// Ids that stay unresolved back off to [`CAPABILITY_RETRY`] rather than
|
||||
/// being re-asked every cycle. A catalogue entry whose weights nobody has
|
||||
/// downloaded can never be answered, and at the poll interval that would
|
||||
/// be a permanent trickle of requests and 404s. The retry still exists
|
||||
/// because the answer *can* change: the first node to pull those weights
|
||||
/// starts being able to reply.
|
||||
async fn discover_capabilities(fleet: &CortexState, name: &str, endpoint: &str) {
|
||||
let unknown: Vec<String> = {
|
||||
let known = fleet.discovered_capabilities.read().await;
|
||||
let attempts = fleet.capability_probe_attempts.read().await;
|
||||
fleet
|
||||
.catalogue
|
||||
.models
|
||||
.iter()
|
||||
.map(|p| p.id.clone())
|
||||
.filter(|id| !known.contains_key(id))
|
||||
.filter(|id| {
|
||||
attempts
|
||||
.get(&(name.to_string(), id.clone()))
|
||||
.is_none_or(|at| at.elapsed() >= CAPABILITY_RETRY)
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
if unknown.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
{
|
||||
let now = std::time::Instant::now();
|
||||
let mut attempts = fleet.capability_probe_attempts.write().await;
|
||||
for id in &unknown {
|
||||
attempts.insert((name.to_string(), id.clone()), now);
|
||||
}
|
||||
}
|
||||
|
||||
for model_id in unknown {
|
||||
let url = format!(
|
||||
"{endpoint}/models/{}/capabilities",
|
||||
urlencoding::encode(&model_id)
|
||||
);
|
||||
let resp = fleet
|
||||
.http_client
|
||||
.get(&url)
|
||||
.timeout(Duration::from_secs(5))
|
||||
.send()
|
||||
.await;
|
||||
let Ok(resp) = resp else { continue };
|
||||
if !resp.status().is_success() {
|
||||
continue;
|
||||
}
|
||||
let Ok(body) = resp.json::<serde_json::Value>().await else {
|
||||
continue;
|
||||
};
|
||||
let Some(caps) = body.get("capabilities").and_then(|c| c.as_array()) else {
|
||||
continue;
|
||||
};
|
||||
let caps: Vec<String> = caps
|
||||
.iter()
|
||||
.filter_map(|c| c.as_str().map(str::to_string))
|
||||
.collect();
|
||||
// An empty list is a real answer ("cached, serves nothing"), but
|
||||
// recording it would pin a model to no capabilities on the word
|
||||
// of one node. Leave it unknown so a node with real weights can
|
||||
// still speak up.
|
||||
if caps.is_empty() {
|
||||
continue;
|
||||
}
|
||||
tracing::debug!(node = name, model = %model_id, ?caps, "discovered capabilities");
|
||||
fleet
|
||||
.discovered_capabilities
|
||||
.write()
|
||||
.await
|
||||
.insert(model_id, caps);
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch `/health` and stash the activation snapshot on NodeState.
|
||||
/// Decoupled from the /models poll so a /health glitch doesn't mark
|
||||
/// the neuron unhealthy or evict the model list.
|
||||
async fn poll_health(fleet: &CortexState, name: &str, endpoint: &str) {
|
||||
let url = format!("{endpoint}/health");
|
||||
let resp = match fleet
|
||||
.http_client
|
||||
.get(&url)
|
||||
.timeout(Duration::from_secs(5))
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(r) if r.status().is_success() => r,
|
||||
Ok(r) => {
|
||||
tracing::debug!(node = name, status = %r.status(), "/health probe non-success");
|
||||
return;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::debug!(node = name, error = %e, "/health probe failed");
|
||||
return;
|
||||
}
|
||||
};
|
||||
match resp.json::<HealthResponse>().await {
|
||||
Ok(h) => {
|
||||
// Export the live load + device health to Prometheus (#137).
|
||||
// These values are already in hand from the routing scrape, so
|
||||
// publishing them adds no polling. Emitted as gauges (last-write
|
||||
// wins, refreshed every ~10s poll) outside the state lock.
|
||||
export_health_metrics(name, &h);
|
||||
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
if let Some(node) = nodes.get_mut(name) {
|
||||
node.activation = Some(h.activation);
|
||||
// Per-model admission load (#53) → keyed by id for the
|
||||
// load-aware router (#55).
|
||||
node.model_load = h.models.into_iter().map(|m| (m.id.clone(), m)).collect();
|
||||
// Per-device VRAM readings (#203) for free-fit
|
||||
// cold-load placement.
|
||||
node.device_health = h.devices.clone();
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::debug!(node = name, error = %e, "failed to parse /health response");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Publish a neuron's `/health` snapshot to Prometheus (#137): live
|
||||
/// per-model admission load + configured ceiling, and per-device GPU
|
||||
/// headroom. Gauges are `{node,model}` / `{node,device}` labelled to match
|
||||
/// the existing `cortex_*` set. Called on every successful poll so values
|
||||
/// track the ~10s cadence; a model that unloads simply stops being
|
||||
/// refreshed (its last gauge value goes stale — acceptable for the bounded
|
||||
/// fleet cardinality here).
|
||||
fn export_health_metrics(node: &str, h: &HealthResponse) {
|
||||
for m in &h.models {
|
||||
gauge!("cortex_model_in_flight", "node" => node.to_string(), "model" => m.id.clone())
|
||||
.set(m.in_flight as f64);
|
||||
gauge!("cortex_model_queue_depth", "node" => node.to_string(), "model" => m.id.clone())
|
||||
.set(m.queue_depth as f64);
|
||||
// Ceiling is the saturation denominator. 0 = pre-#137 neuron that
|
||||
// doesn't advertise it yet — skip rather than publish a bogus 0.
|
||||
if m.max_in_flight > 0 {
|
||||
gauge!("cortex_model_max_in_flight", "node" => node.to_string(), "model" => m.id.clone())
|
||||
.set(m.max_in_flight as f64);
|
||||
gauge!("cortex_model_max_queue_depth", "node" => node.to_string(), "model" => m.id.clone())
|
||||
.set(m.max_queue_depth as f64);
|
||||
}
|
||||
// Live throughput EMAs (#137) — decode tok/s is the headline
|
||||
// capacity number. Emitted unconditionally (0.0 = no sample yet).
|
||||
gauge!("cortex_model_tok_s_prefill", "node" => node.to_string(), "model" => m.id.clone())
|
||||
.set(m.tok_s_prefill);
|
||||
gauge!("cortex_model_tok_s_decode", "node" => node.to_string(), "model" => m.id.clone())
|
||||
.set(m.tok_s_decode);
|
||||
// Cumulative rejections by reason (#137) — the shedding signal.
|
||||
// Neuron reports counts-since-load; `.absolute` mirrors them onto a
|
||||
// counter (a model reload resets to 0, which Prometheus reads as a
|
||||
// normal counter reset).
|
||||
counter!("cortex_model_rejections_total",
|
||||
"node" => node.to_string(), "model" => m.id.clone(), "reason" => "queue_full")
|
||||
.absolute(m.rejected_queue_full);
|
||||
counter!("cortex_model_rejections_total",
|
||||
"node" => node.to_string(), "model" => m.id.clone(), "reason" => "wait_timeout")
|
||||
.absolute(m.rejected_timeout);
|
||||
counter!("cortex_model_rejections_total",
|
||||
"node" => node.to_string(), "model" => m.id.clone(), "reason" => "per_principal")
|
||||
.absolute(m.rejected_per_principal);
|
||||
counter!("cortex_model_rejections_total",
|
||||
"node" => node.to_string(), "model" => m.id.clone(), "reason" => "anon_yield")
|
||||
.absolute(m.rejected_anon_yield);
|
||||
// How much of this model's load is unattributable (#262), and the
|
||||
// ceiling that keeps it from starving identified callers. 0 for a
|
||||
// pre-#262 neuron, which had no ceiling — skip rather than publish
|
||||
// a 0 that reads as "anonymous is refused" when it means the
|
||||
// opposite.
|
||||
gauge!("cortex_model_anon_in_flight", "node" => node.to_string(), "model" => m.id.clone())
|
||||
.set(m.anon_in_flight as f64);
|
||||
if m.anon_max_in_flight > 0 {
|
||||
gauge!("cortex_model_anon_max_in_flight",
|
||||
"node" => node.to_string(), "model" => m.id.clone())
|
||||
.set(m.anon_max_in_flight as f64);
|
||||
}
|
||||
}
|
||||
for d in &h.devices {
|
||||
let device = d.index.to_string();
|
||||
gauge!("cortex_device_vram_used_mb", "node" => node.to_string(), "device" => device.clone())
|
||||
.set(d.vram_used_mb as f64);
|
||||
gauge!("cortex_device_vram_free_mb", "node" => node.to_string(), "device" => device.clone())
|
||||
.set(d.vram_free_mb as f64);
|
||||
gauge!("cortex_device_utilization_pct", "node" => node.to_string(), "device" => device.clone())
|
||||
.set(d.utilization_pct as f64);
|
||||
gauge!("cortex_device_temp_c", "node" => node.to_string(), "device" => device.clone())
|
||||
.set(d.temp_c as f64);
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_status(s: &str) -> ModelStatus {
|
||||
@@ -428,8 +96,6 @@ fn parse_status(s: &str) -> ModelStatus {
|
||||
"loaded" => ModelStatus::Loaded,
|
||||
"unloaded" => ModelStatus::Unloaded,
|
||||
"reloading" => ModelStatus::Reloading,
|
||||
"loading" => ModelStatus::Loading,
|
||||
"recovering" => ModelStatus::Recovering,
|
||||
_ => ModelStatus::Loaded,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,45 +1,24 @@
|
||||
//! Streaming HTTP reverse proxy to neuron backends.
|
||||
//! Streaming HTTP reverse proxy to mistral.rs backends.
|
||||
//!
|
||||
//! The streaming *mechanism* — forward an SSE body chunk-for-chunk without
|
||||
//! buffering, observing the bytes for metrics — lives in the shared
|
||||
//! [`helexa_stream`] crate (#71), so cortex and helexa-router use one
|
||||
//! implementation. This module supplies cortex's *policy*: the
|
||||
//! [`CortexMetrics`] observer (per-request token metrics + per-principal
|
||||
//! reservation settle), cortex's logging contract, and the cortex error
|
||||
//! envelope. The usage-extraction helper is re-exported from the shared
|
||||
//! crate so existing call sites keep working.
|
||||
//! For streaming requests, SSE chunks are forwarded as they arrive.
|
||||
//! The proxy captures timing information for metrics but does not
|
||||
//! buffer the full response.
|
||||
|
||||
use crate::router::RouteDecision;
|
||||
use axum::http::HeaderMap;
|
||||
use axum::http::StatusCode;
|
||||
use anyhow::Result;
|
||||
use axum::body::Body;
|
||||
use axum::http::{HeaderMap, StatusCode};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use helexa_stream::{BodyTail, ChunkObserver, StreamError};
|
||||
use reqwest::Client;
|
||||
use std::time::Instant;
|
||||
|
||||
/// Re-export the shared usage-extraction helper. Several cortex modules
|
||||
/// (`handlers`, `anthropic_sse`) pull token counts out of a buffered body
|
||||
/// tail via this function; it lives in `helexa-stream` now.
|
||||
pub use helexa_stream::last_count_for;
|
||||
|
||||
/// Proxy a request body to the resolved backend node and stream the response.
|
||||
///
|
||||
/// Logging contract: every call emits exactly one structured event at
|
||||
/// info / warn level for operator visibility, regardless of outcome.
|
||||
/// Network-level failures and non-2xx upstream statuses are warn'd here
|
||||
/// (closest to the wire); the user-facing response carries only the
|
||||
/// status code and a generic message — implementation detail (body,
|
||||
/// error chain) lives in the log, never in the API surface.
|
||||
pub async fn forward_request(
|
||||
client: &Client,
|
||||
route: &RouteDecision,
|
||||
path: &str,
|
||||
headers: HeaderMap,
|
||||
body: bytes::Bytes,
|
||||
model_id: &str,
|
||||
usage_sink: Option<crate::metering::UsageSink>,
|
||||
) -> Result<Response, ProxyError> {
|
||||
let request_start = Instant::now();
|
||||
let url = format!("{}{}", route.endpoint, path);
|
||||
tracing::info!(
|
||||
node = %route.node_name,
|
||||
@@ -48,240 +27,56 @@ pub async fn forward_request(
|
||||
"proxying request"
|
||||
);
|
||||
|
||||
let observer = CortexMetrics::new(model_id, &route.node_name, request_start, usage_sink);
|
||||
let mut req_builder = client.post(&url).body(body);
|
||||
|
||||
let response = helexa_stream::forward_streaming(client, &url, headers, body, observer)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
match &e {
|
||||
StreamError::Upstream(err) => tracing::warn!(
|
||||
node = %route.node_name,
|
||||
url = %url,
|
||||
error = %err,
|
||||
"proxy: upstream request failed (network)"
|
||||
),
|
||||
StreamError::ResponseBuild(err) => tracing::warn!(
|
||||
node = %route.node_name,
|
||||
url = %url,
|
||||
error = %err,
|
||||
"proxy: failed to build response"
|
||||
),
|
||||
}
|
||||
ProxyError::from(e)
|
||||
})?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
// Buffer the error body so the journal records WHY, then hand the
|
||||
// same bytes on. Streaming is what a success body needs; an error
|
||||
// body is a few hundred bytes and streaming it only guarantees
|
||||
// that nobody ever sees the reason.
|
||||
//
|
||||
// This was a real dead end: neuron answered 503 `insufficient_vram`
|
||||
// with the deficit in the body, and cortex logged only
|
||||
// `status=503`. Neither journal held the cause, and the guard
|
||||
// responsible could only be found by reading neuron's source
|
||||
// (#257). The body is upstream-generated (our own #63 envelope),
|
||||
// not caller content, and it is truncated in the log.
|
||||
const MAX_ERROR_BODY: usize = 64 * 1024;
|
||||
let status = response.status();
|
||||
let (parts, body) = response.into_parts();
|
||||
return match axum::body::to_bytes(body, MAX_ERROR_BODY).await {
|
||||
Ok(bytes) => {
|
||||
let snippet: String = String::from_utf8_lossy(&bytes).chars().take(600).collect();
|
||||
tracing::warn!(
|
||||
node = %route.node_name,
|
||||
url = %url,
|
||||
status = status.as_u16(),
|
||||
body = %snippet,
|
||||
"proxy: upstream returned non-2xx"
|
||||
);
|
||||
Ok(Response::from_parts(parts, axum::body::Body::from(bytes)))
|
||||
}
|
||||
// Over the cap or a read failure: the body is consumed and
|
||||
// cannot be handed on. Say so rather than passing an empty
|
||||
// body off as the upstream's answer.
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
node = %route.node_name,
|
||||
url = %url,
|
||||
status = status.as_u16(),
|
||||
error = %e,
|
||||
"proxy: upstream returned non-2xx (body unreadable, dropped)"
|
||||
);
|
||||
Ok(Response::from_parts(parts, axum::body::Body::empty()))
|
||||
}
|
||||
};
|
||||
// Forward relevant headers.
|
||||
for (key, value) in headers.iter() {
|
||||
if key == "host" || key == "content-length" {
|
||||
continue; // reqwest sets these
|
||||
}
|
||||
req_builder = req_builder.header(key, value);
|
||||
}
|
||||
|
||||
Ok(response)
|
||||
let upstream_resp = req_builder.send().await.map_err(ProxyError::Upstream)?;
|
||||
|
||||
let status =
|
||||
StatusCode::from_u16(upstream_resp.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY);
|
||||
|
||||
let resp_headers = upstream_resp.headers().clone();
|
||||
let stream = upstream_resp.bytes_stream();
|
||||
|
||||
let body = Body::from_stream(stream);
|
||||
|
||||
let mut response = Response::builder().status(status);
|
||||
for (key, value) in resp_headers.iter() {
|
||||
response = response.header(key, value);
|
||||
}
|
||||
|
||||
response
|
||||
.body(body)
|
||||
.map_err(|e| ProxyError::ResponseBuild(e.to_string()))
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ProxyError {
|
||||
#[error("upstream request failed")]
|
||||
#[error("upstream request failed: {0}")]
|
||||
Upstream(reqwest::Error),
|
||||
#[error("failed to build response")]
|
||||
#[error("failed to build response: {0}")]
|
||||
ResponseBuild(String),
|
||||
}
|
||||
|
||||
impl From<StreamError> for ProxyError {
|
||||
fn from(e: StreamError) -> Self {
|
||||
match e {
|
||||
StreamError::Upstream(err) => ProxyError::Upstream(err),
|
||||
StreamError::ResponseBuild(msg) => ProxyError::ResponseBuild(msg),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl IntoResponse for ProxyError {
|
||||
fn into_response(self) -> Response {
|
||||
let (status, code, message) = match &self {
|
||||
ProxyError::Upstream(_) => (
|
||||
StatusCode::BAD_GATEWAY,
|
||||
"upstream_connection_error",
|
||||
"upstream request failed",
|
||||
),
|
||||
ProxyError::ResponseBuild(_) => (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"internal_server_error",
|
||||
"failed to build response",
|
||||
),
|
||||
let status = match &self {
|
||||
ProxyError::Upstream(_) => StatusCode::BAD_GATEWAY,
|
||||
ProxyError::ResponseBuild(_) => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
};
|
||||
crate::error::envelope_response(cortex_core::error_envelope::OpenAiError::new(
|
||||
status.as_u16(),
|
||||
"api_error",
|
||||
code,
|
||||
message,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
// ── Per-request token metrics (#21) ─────────────────────────────────
|
||||
//
|
||||
// The proxy never buffers or re-serialises the upstream body — chunks
|
||||
// are forwarded verbatim. For metrics it observes each chunk's arrival
|
||||
// time and keeps a bounded tail of the body text (via the shared
|
||||
// `helexa_stream::BodyTail`), from which the final OpenAI `usage` object
|
||||
// (present on the last SSE chunk and on non-streaming JSON bodies alike)
|
||||
// yields engine-truth token counts.
|
||||
//
|
||||
// Emitted per request, labelled {model, node}:
|
||||
// cortex_time_to_first_token_seconds (histogram) — first body chunk
|
||||
// cortex_tokens_per_second (histogram) — completion tokens
|
||||
// over the decode window (first→last chunk); falls back to the
|
||||
// full request duration for single-chunk (non-streaming) bodies
|
||||
// cortex_prompt_tokens_total / cortex_completion_tokens_total (counters)
|
||||
// cortex_cached_prompt_tokens_total (counter) — prompt tokens served
|
||||
// from neuron's prefix cache; over prompt_tokens_total this is the
|
||||
// fleet's cache-hit rate (#269)
|
||||
|
||||
/// Cap on the retained body tail. The usage object rides on the final
|
||||
/// chunk, so a generous tail is plenty; the cap bounds memory on huge
|
||||
/// non-streaming bodies.
|
||||
const TAIL_CAP_BYTES: usize = 64 * 1024;
|
||||
|
||||
/// cortex's [`ChunkObserver`]: per-request token metrics plus the
|
||||
/// per-principal reservation settle. Drives cortex policy over the shared
|
||||
/// streaming mechanism.
|
||||
struct CortexMetrics {
|
||||
labels: [(&'static str, String); 2],
|
||||
request_start: Instant,
|
||||
first_chunk: Option<Instant>,
|
||||
last_chunk: Option<Instant>,
|
||||
tail: BodyTail,
|
||||
finished: bool,
|
||||
/// Per-principal metering hook (#51). Invoked exactly once in `finish`
|
||||
/// with the observed `(prompt, completion)` so the reservation can be
|
||||
/// settled and spend recorded. `None` for anonymous requests.
|
||||
usage_sink: Option<crate::metering::UsageSink>,
|
||||
}
|
||||
|
||||
impl CortexMetrics {
|
||||
fn new(
|
||||
model_id: &str,
|
||||
node_name: &str,
|
||||
request_start: Instant,
|
||||
usage_sink: Option<crate::metering::UsageSink>,
|
||||
) -> Self {
|
||||
Self {
|
||||
labels: [
|
||||
("model", model_id.to_string()),
|
||||
("node", node_name.to_string()),
|
||||
],
|
||||
request_start,
|
||||
first_chunk: None,
|
||||
last_chunk: None,
|
||||
tail: BodyTail::new(TAIL_CAP_BYTES),
|
||||
finished: false,
|
||||
usage_sink,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ChunkObserver for CortexMetrics {
|
||||
fn observe(&mut self, chunk: &[u8]) {
|
||||
let now = Instant::now();
|
||||
self.first_chunk.get_or_insert(now);
|
||||
self.last_chunk = Some(now);
|
||||
self.tail.push(chunk);
|
||||
}
|
||||
|
||||
/// Emit the metrics exactly once — called on clean stream end and
|
||||
/// from Drop (client disconnect mid-stream still records what we
|
||||
/// saw).
|
||||
fn finish(&mut self) {
|
||||
if self.finished {
|
||||
return;
|
||||
}
|
||||
self.finished = true;
|
||||
|
||||
let prompt = last_count_for(self.tail.as_str(), "prompt_tokens");
|
||||
let completion = last_count_for(self.tail.as_str(), "completion_tokens");
|
||||
|
||||
// Per-model metrics — only when body chunks actually arrived.
|
||||
if let Some(first) = self.first_chunk {
|
||||
let ttft = first.duration_since(self.request_start).as_secs_f64();
|
||||
metrics::histogram!("cortex_time_to_first_token_seconds", &self.labels).record(ttft);
|
||||
|
||||
if let Some(prompt) = prompt {
|
||||
metrics::counter!("cortex_prompt_tokens_total", &self.labels).increment(prompt);
|
||||
let body = serde_json::json!({
|
||||
"error": {
|
||||
"message": self.to_string(),
|
||||
"type": "proxy_error",
|
||||
}
|
||||
// Prefix-cache reuse (#269), read from the same usage object.
|
||||
// Against `cortex_prompt_tokens_total` this gives fleet-wide
|
||||
// cache-hit rate as a ratio, which was previously only
|
||||
// obtainable by grepping `reused=` out of a neuron journal —
|
||||
// per-host, unaggregated, and gone on log rotation.
|
||||
if let Some(cached) = last_count_for(self.tail.as_str(), "cached_tokens") {
|
||||
metrics::counter!("cortex_cached_prompt_tokens_total", &self.labels)
|
||||
.increment(cached);
|
||||
}
|
||||
if let Some(completion) = completion.filter(|c| *c > 0) {
|
||||
metrics::counter!("cortex_completion_tokens_total", &self.labels)
|
||||
.increment(completion);
|
||||
|
||||
let last = self.last_chunk.unwrap_or(first);
|
||||
let decode_window = last.duration_since(first).as_secs_f64();
|
||||
// Streaming: rate over the decode window (first→last chunk).
|
||||
// Non-streaming bodies arrive as ~one chunk (window ≈ 0),
|
||||
// where the only honest denominator is the full request
|
||||
// duration.
|
||||
let secs = if decode_window >= 0.1 {
|
||||
decode_window
|
||||
} else {
|
||||
last.duration_since(self.request_start).as_secs_f64()
|
||||
};
|
||||
if secs > 0.0 {
|
||||
metrics::histogram!("cortex_tokens_per_second", &self.labels)
|
||||
.record(completion as f64 / secs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Per-principal metering + reservation settle (#51). Always runs so
|
||||
// the reservation is resolved even when no usage/body was observed
|
||||
// (sink with (0, 0) → settle 0 → release).
|
||||
if let Some(sink) = self.usage_sink.take() {
|
||||
sink(prompt.unwrap_or(0), completion.unwrap_or(0));
|
||||
}
|
||||
});
|
||||
(status, axum::Json(body)).into_response()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,21 +2,13 @@
|
||||
//!
|
||||
//! Given a model ID from an inbound request, determine which node should
|
||||
//! handle it. Priority:
|
||||
//! 1. Node where the model is currently `Loaded` → use it.
|
||||
//! 2. Node where the model is `Unloaded` → use it; neuron's existing
|
||||
//! lazy-load behaviour will reload before serving the request.
|
||||
//! 3. Model is in the catalogue → pick a feasible neuron, call
|
||||
//! `POST /models/load`, wait for the load to complete, then
|
||||
//! proxy. First-request cold-load latency is acceptable per the
|
||||
//! unified-endpoint contract.
|
||||
//! 4. Not in catalogue, not loaded anywhere → 404.
|
||||
//! 1. Node where the model is currently `Loaded`
|
||||
//! 2. Node where the model is `Unloaded` (will lazy-load on request)
|
||||
//! 3. Error: model not found on any node
|
||||
|
||||
use crate::state::CortexState;
|
||||
use cortex_core::catalogue::ModelProfile;
|
||||
use cortex_core::harness::ModelSpec;
|
||||
use cortex_core::node::ModelStatus;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
/// The routing decision: which node endpoint to proxy the request to.
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -24,556 +16,62 @@ pub struct RouteDecision {
|
||||
pub node_name: String,
|
||||
/// The inference endpoint to proxy to (from neuron's /models/{id}/endpoint).
|
||||
pub endpoint: String,
|
||||
/// Whether the model will need to load (cold start). Set to true
|
||||
/// when we proxied to an `Unloaded` node (lazy load on neuron) or
|
||||
/// when we just triggered an explicit cold-load via the catalogue
|
||||
/// path.
|
||||
/// Whether the model will need to load (cold start).
|
||||
pub cold_start: bool,
|
||||
/// The concrete model id we actually routed to. Equal to the
|
||||
/// caller's requested id unless an alias was resolved (e.g. caller
|
||||
/// asked for `helexa/small`, this carries `Qwen/Qwen3-1.7B`). The
|
||||
/// handler uses this to rewrite the request body's `model` field
|
||||
/// before proxying — neurons reject requests where the body's
|
||||
/// model name doesn't match a loaded model.
|
||||
pub resolved_model_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum RouteError {
|
||||
#[error("model '{0}' not found on any node and not in catalogue")]
|
||||
#[error("model '{0}' not found on any node")]
|
||||
ModelNotFound(String),
|
||||
#[error("no healthy nodes available")]
|
||||
NoHealthyNodes,
|
||||
#[error("failed to resolve inference endpoint for model '{0}' on node '{1}'")]
|
||||
EndpointResolveFailed(String, String),
|
||||
#[error(
|
||||
"model '{model_id}' is in the catalogue but no healthy neuron's topology satisfies its constraints"
|
||||
)]
|
||||
NoFeasibleNeuron { model_id: String },
|
||||
#[error(
|
||||
"model '{model_id}' is feasible on a neuron that is currently unhealthy — retry shortly"
|
||||
)]
|
||||
FeasibleNodeUnhealthy { model_id: String },
|
||||
#[error("cold-load of '{model_id}' on '{node}' failed: {message}")]
|
||||
ColdLoadFailed {
|
||||
model_id: String,
|
||||
node: String,
|
||||
message: String,
|
||||
},
|
||||
#[error(
|
||||
"model '{model_id}' is recovering on node '{node}' (device context rebuild in progress) — retry shortly"
|
||||
)]
|
||||
ModelRecovering { model_id: String, node: String },
|
||||
}
|
||||
|
||||
impl RouteError {
|
||||
/// HTTP status the gateway should answer with. `NoHealthyNodes` and
|
||||
/// `ModelRecovering` are the transient cases (503 service_unavailable,
|
||||
/// safe to retry the same request); everything else is 404.
|
||||
pub fn http_status(&self) -> u16 {
|
||||
match self {
|
||||
RouteError::NoHealthyNodes
|
||||
| RouteError::ModelRecovering { .. }
|
||||
| RouteError::FeasibleNodeUnhealthy { .. } => 503,
|
||||
_ => 404,
|
||||
}
|
||||
}
|
||||
|
||||
/// Broad OpenAI error category for the JSON envelope.
|
||||
pub fn broad_type(&self) -> &'static str {
|
||||
match self {
|
||||
RouteError::ModelNotFound(_) => "invalid_request_error",
|
||||
RouteError::NoHealthyNodes
|
||||
| RouteError::EndpointResolveFailed(_, _)
|
||||
| RouteError::NoFeasibleNeuron { .. }
|
||||
| RouteError::ColdLoadFailed { .. }
|
||||
| RouteError::ModelRecovering { .. }
|
||||
| RouteError::FeasibleNodeUnhealthy { .. } => "api_error",
|
||||
}
|
||||
}
|
||||
|
||||
/// Specific machine-readable error code.
|
||||
pub fn code(&self) -> &'static str {
|
||||
match self {
|
||||
RouteError::ModelNotFound(_) => "model_not_found",
|
||||
RouteError::NoHealthyNodes => "service_unavailable",
|
||||
RouteError::EndpointResolveFailed(_, _) => "service_unavailable",
|
||||
RouteError::NoFeasibleNeuron { .. } => "service_unavailable",
|
||||
RouteError::ColdLoadFailed { .. } => "service_unavailable",
|
||||
RouteError::ModelRecovering { .. } => "service_unavailable",
|
||||
RouteError::FeasibleNodeUnhealthy { .. } => "service_unavailable",
|
||||
}
|
||||
}
|
||||
|
||||
/// Seconds to advertise in `Retry-After` for the transient variants
|
||||
/// (#63). `NoHealthyNodes` may clear once the poller re-marks a node
|
||||
/// healthy; `ModelRecovering` clears once the device context finishes
|
||||
/// rebuilding — both are safe to retry. Everything else is permanent
|
||||
/// for this request (404) and carries no hint.
|
||||
pub fn retry_after_secs(&self) -> Option<u64> {
|
||||
match self {
|
||||
RouteError::ModelRecovering { .. } => Some(2),
|
||||
RouteError::FeasibleNodeUnhealthy { .. } => Some(3),
|
||||
RouteError::NoHealthyNodes => Some(5),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve which node should serve a request for the given model.
|
||||
/// Asks the neuron for the inference endpoint after selecting a node.
|
||||
pub async fn resolve(
|
||||
fleet: &Arc<CortexState>,
|
||||
requested_model_id: &str,
|
||||
model_id: &str,
|
||||
) -> Result<RouteDecision, RouteError> {
|
||||
// Alias resolution first — swap `helexa/small` (etc.) for the
|
||||
// concrete id before any node lookups so the rest of routing,
|
||||
// loading, and metrics deal in concrete ids only. `resolve_alias`
|
||||
// returns the input verbatim when it isn't an alias.
|
||||
let model_id = fleet.catalogue.resolve_alias(requested_model_id);
|
||||
if model_id != requested_model_id {
|
||||
tracing::debug!(
|
||||
requested = requested_model_id,
|
||||
resolved = model_id,
|
||||
"alias resolved"
|
||||
);
|
||||
}
|
||||
// Snapshot loaded / unloaded / recovering state from the poller cache.
|
||||
let (loaded_route, unloaded_route, recovering_node, any_healthy) = {
|
||||
let (node_name, neuron_endpoint, cold_start) = {
|
||||
let nodes = fleet.nodes.read().await;
|
||||
// All healthy nodes with the model loaded, each with its current
|
||||
// admission load (#53) so we can pick the least-busy replica (#55).
|
||||
let mut loaded_candidates: Vec<(String, String, usize)> = Vec::new();
|
||||
let mut unloaded_route = None;
|
||||
let mut recovering_node = None;
|
||||
let mut any_healthy = false;
|
||||
|
||||
let mut loaded_candidate = None;
|
||||
let mut unloaded_candidate = None;
|
||||
|
||||
for node in nodes.values() {
|
||||
if !node.healthy {
|
||||
continue;
|
||||
}
|
||||
any_healthy = true;
|
||||
if let Some(entry) = node.models.get(model_id) {
|
||||
match entry.status {
|
||||
ModelStatus::Loaded | ModelStatus::Reloading => {
|
||||
// Resident is not servable (#245). A node whose
|
||||
// device has been squeezed below the prefill
|
||||
// floor rejects every request before doing any
|
||||
// work — routing there converts a recoverable
|
||||
// placement problem into a 503 for the client.
|
||||
// Skipping it lets the rest of this function do
|
||||
// what it already does well: pick another
|
||||
// replica, or cold-load somewhere that fits.
|
||||
if !entry.is_servable() {
|
||||
tracing::warn!(
|
||||
node = %node.name,
|
||||
model = %model_id,
|
||||
reason = entry
|
||||
.servable
|
||||
.as_ref()
|
||||
.and_then(|s| s.reason.as_deref())
|
||||
.unwrap_or("unknown"),
|
||||
"skipping loaded-but-unservable location"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
// Least-busy score: in-flight + queued from the
|
||||
// neuron's last /health (#53). Unknown load (no poll
|
||||
// yet) scores 0 so the replica stays eligible.
|
||||
let score = node
|
||||
.model_load
|
||||
.get(model_id)
|
||||
.map(|l| l.in_flight + l.queue_depth)
|
||||
.unwrap_or(0);
|
||||
loaded_candidates.push((node.name.clone(), node.endpoint.clone(), score));
|
||||
loaded_candidate = Some((node.name.clone(), node.endpoint.clone(), false));
|
||||
break;
|
||||
}
|
||||
ModelStatus::Unloaded => {
|
||||
if unloaded_route.is_none() {
|
||||
unloaded_route = Some((node.name.clone(), node.endpoint.clone(), true));
|
||||
if unloaded_candidate.is_none() {
|
||||
unloaded_candidate =
|
||||
Some((node.name.clone(), node.endpoint.clone(), true));
|
||||
}
|
||||
}
|
||||
// Auto-recovering (#17/#20): the model is rebuilding
|
||||
// its device context on this node. Hold the route —
|
||||
// answer "retry shortly" rather than 404, and do NOT
|
||||
// fall through to the catalogue cold-load, which
|
||||
// would race a second placement (and a second copy's
|
||||
// worth of VRAM) against the in-flight recovery.
|
||||
ModelStatus::Recovering => {
|
||||
if recovering_node.is_none() {
|
||||
recovering_node = Some(node.name.clone());
|
||||
}
|
||||
}
|
||||
// Loading is gateway-synthesised from neuron's
|
||||
// activation snapshot; it never appears on the
|
||||
// wire from neuron's `/models`. Skip — the model
|
||||
// isn't actually servable yet. The pre-existing
|
||||
// race (catalogue cold_load fires a parallel
|
||||
// /models/load against the in-flight load) is no
|
||||
// worse than before; fixing it needs neuron-side
|
||||
// in-flight tracking on /models/load itself.
|
||||
ModelStatus::Loading => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Pick the least-busy loaded replica; ties break by node name for
|
||||
// deterministic routing. `false` = not a cold start.
|
||||
let loaded_route = loaded_candidates
|
||||
.into_iter()
|
||||
.min_by(|a, b| a.2.cmp(&b.2).then_with(|| a.0.cmp(&b.0)))
|
||||
.map(|(name, endpoint, _score)| (name, endpoint, false));
|
||||
(loaded_route, unloaded_route, recovering_node, any_healthy)
|
||||
};
|
||||
|
||||
if !any_healthy {
|
||||
return Err(RouteError::NoHealthyNodes);
|
||||
}
|
||||
|
||||
// Priority 1: already loaded.
|
||||
if let Some((node_name, neuron_endpoint, cold_start)) = loaded_route {
|
||||
return finish(fleet, &node_name, &neuron_endpoint, model_id, cold_start).await;
|
||||
}
|
||||
|
||||
// Priority 2: recovering somewhere — transient hold, not a reroute.
|
||||
if let Some(node) = recovering_node {
|
||||
return Err(RouteError::ModelRecovering {
|
||||
model_id: model_id.to_string(),
|
||||
node,
|
||||
});
|
||||
}
|
||||
|
||||
// Priority 3: known to neuron but unloaded (neuron's lazy load).
|
||||
if let Some((node_name, neuron_endpoint, cold_start)) = unloaded_route {
|
||||
return finish(fleet, &node_name, &neuron_endpoint, model_id, cold_start).await;
|
||||
}
|
||||
|
||||
// Priority 4: catalogue × topology cold-load.
|
||||
if let Some(profile) = fleet.catalogue.get(model_id) {
|
||||
let (node_name, neuron_endpoint, fits_free) = pick_feasible_neuron(fleet, profile).await?;
|
||||
// Cold-swap (#203): when the chosen node's devices don't have the
|
||||
// free VRAM *right now*, evict unpinned LRU models there until
|
||||
// the profile fits (bounded — a node whose evictable set can't
|
||||
// free enough was ranked below one that can).
|
||||
if !fits_free {
|
||||
for _ in 0..3 {
|
||||
match crate::evictor::evict_lru_on_node(fleet, &node_name, Some(&profile.id)).await
|
||||
{
|
||||
Ok(Some(evicted)) => {
|
||||
tracing::info!(
|
||||
model = %profile.id,
|
||||
node = %node_name,
|
||||
evicted = %evicted,
|
||||
"cold-swap: evicted LRU model to make room"
|
||||
);
|
||||
if node_fits_free(fleet, &node_name, profile).await {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(e) => {
|
||||
tracing::warn!(node = %node_name, error = %e, "cold-swap eviction failed");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
cold_load(fleet, &node_name, &neuron_endpoint, profile).await?;
|
||||
return finish(fleet, &node_name, &neuron_endpoint, model_id, true).await;
|
||||
}
|
||||
|
||||
Err(RouteError::ModelNotFound(model_id.to_string()))
|
||||
}
|
||||
|
||||
/// True when `node_name`'s live device readings show enough free VRAM
|
||||
/// on some device for the profile. Conservative: unknown health (no
|
||||
/// /health poll yet) counts as not-fitting so the ranking prefers
|
||||
/// nodes we can actually see.
|
||||
async fn node_fits_free(fleet: &Arc<CortexState>, node_name: &str, profile: &ModelProfile) -> bool {
|
||||
let nodes = fleet.nodes.read().await;
|
||||
nodes.get(node_name).is_some_and(|node| {
|
||||
node.device_health
|
||||
.iter()
|
||||
.any(|d| d.vram_free_mb >= profile.vram_mb.unwrap_or(0))
|
||||
})
|
||||
}
|
||||
|
||||
/// Pick a healthy neuron whose discovered topology satisfies the
|
||||
/// profile. Preference order (#203):
|
||||
/// 1. A neuron from `profile.pinned_on` that is healthy + feasible.
|
||||
/// 2. One whose live free VRAM already fits the profile.
|
||||
/// 3. One whose *displaceable* models — loaded, and outranked by this
|
||||
/// profile's residency priority — plus free VRAM could fit it
|
||||
/// after a cold-swap eviction.
|
||||
/// 4. Any healthy + feasible neuron, stable by name.
|
||||
///
|
||||
/// Free-fit outranks evict-fit, so a model never displaces anything
|
||||
/// while a node with room exists. Priority decides who may be
|
||||
/// displaced, never whether a displacement is needed.
|
||||
///
|
||||
/// Returns `(name, endpoint, fits_free)` — `fits_free = false` tells
|
||||
/// the caller a cold-swap eviction is needed before loading.
|
||||
async fn pick_feasible_neuron(
|
||||
fleet: &Arc<CortexState>,
|
||||
profile: &ModelProfile,
|
||||
) -> Result<(String, String, bool), RouteError> {
|
||||
let nodes = fleet.nodes.read().await;
|
||||
let mut candidates: Vec<(String, String, bool, bool, bool, u64)> = Vec::new();
|
||||
for node in nodes.values() {
|
||||
if !node.healthy {
|
||||
continue;
|
||||
}
|
||||
let Some(disc) = node.discovery.as_ref() else {
|
||||
continue;
|
||||
};
|
||||
if !profile.is_feasible_on(&node.name, &disc.devices) {
|
||||
continue;
|
||||
}
|
||||
let pinned = profile.pinned_on.iter().any(|n| n == &node.name);
|
||||
let max_free = node
|
||||
.device_health
|
||||
.iter()
|
||||
.map(|d| d.vram_free_mb)
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
let need = profile.vram_mb.unwrap_or(0);
|
||||
let fits_free = max_free >= need;
|
||||
// Evictable estimate: free + the VRAM of loaded models this
|
||||
// profile is permitted to displace. The predicate must be the
|
||||
// same one the evictor applies, or a node ranks as able to make
|
||||
// room and then declines to make any.
|
||||
let evictable: u64 = node
|
||||
.models
|
||||
.values()
|
||||
.filter(|m| {
|
||||
matches!(m.status, cortex_core::node::ModelStatus::Loaded)
|
||||
&& fleet.catalogue.may_displace(&profile.id, &m.id)
|
||||
})
|
||||
// neuron reports vram_used_mb: null today; fall back to the
|
||||
// catalogue's declared footprint so the evictable estimate
|
||||
// isn't silently zero (which made every node rank equal and
|
||||
// sent the first live image cold-load to beast).
|
||||
.filter_map(|m| {
|
||||
m.vram_estimate_mb
|
||||
.or_else(|| fleet.catalogue.get(&m.id).and_then(|p| p.vram_mb))
|
||||
})
|
||||
.sum();
|
||||
let fits_after_evict = max_free.saturating_add(evictable) >= need;
|
||||
candidates.push((
|
||||
node.name.clone(),
|
||||
node.endpoint.clone(),
|
||||
pinned,
|
||||
fits_free,
|
||||
fits_after_evict,
|
||||
max_free,
|
||||
));
|
||||
}
|
||||
candidates.sort_by(|a, b| {
|
||||
b.2.cmp(&a.2) // pinned first
|
||||
.then(b.3.cmp(&a.3)) // then free-fit
|
||||
.then(b.4.cmp(&a.4)) // then evictable-fit
|
||||
.then(b.5.cmp(&a.5)) // then most free VRAM
|
||||
.then(a.0.cmp(&b.0)) // then stable by name
|
||||
});
|
||||
if let Some((n, e, _, fits_free, _, _)) = candidates.into_iter().next() {
|
||||
return Ok((n, e, fits_free));
|
||||
}
|
||||
|
||||
// No *healthy* feasible neuron. Distinguish a transient outage from a
|
||||
// permanent misconfiguration: if some neuron is topologically feasible
|
||||
// but currently unhealthy (e.g. it briefly missed polls while busy),
|
||||
// this is retryable — return 503 + Retry-After so the client backs off
|
||||
// and retries instead of treating a 404 as a hard failure. Only when no
|
||||
// neuron could *ever* satisfy the topology is it a permanent 404.
|
||||
let feasible_but_unhealthy = nodes.values().any(|node| {
|
||||
!node.healthy
|
||||
&& node
|
||||
.discovery
|
||||
.as_ref()
|
||||
.is_some_and(|disc| profile.is_feasible_on(&node.name, &disc.devices))
|
||||
});
|
||||
if feasible_but_unhealthy {
|
||||
Err(RouteError::FeasibleNodeUnhealthy {
|
||||
model_id: profile.id.clone(),
|
||||
})
|
||||
} else {
|
||||
Err(RouteError::NoFeasibleNeuron {
|
||||
model_id: profile.id.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Issue `POST {endpoint}/models/load` for this profile on this neuron,
|
||||
/// blocking until the load completes (neuron's load endpoint is
|
||||
/// synchronous — it returns 200 once VRAM is materialised). On success
|
||||
/// also inserts a `Loaded` entry into the local NodeState cache so the
|
||||
/// caller's subsequent endpoint lookup sees the new model without
|
||||
/// waiting for the next poll cycle.
|
||||
async fn cold_load(
|
||||
fleet: &Arc<CortexState>,
|
||||
node_name: &str,
|
||||
neuron_endpoint: &str,
|
||||
profile: &ModelProfile,
|
||||
) -> Result<(), RouteError> {
|
||||
let spec = profile_to_spec(fleet, node_name, profile).await;
|
||||
let url = format!("{neuron_endpoint}/models/load");
|
||||
tracing::info!(model = %profile.id, node = node_name, "cold-loading via /models/load");
|
||||
|
||||
// Generous timeout: a fresh download + safetensors mmap + device
|
||||
// copy for a 30B-class dense model can comfortably exceed 5 min on
|
||||
// a slow link. The HTTP client's own default already covers most
|
||||
// of this; pin a longer per-request bound just here.
|
||||
let resp = match fleet
|
||||
.http_client
|
||||
.post(&url)
|
||||
.timeout(Duration::from_secs(1800))
|
||||
.json(&spec)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
return Err(RouteError::ColdLoadFailed {
|
||||
model_id: profile.id.clone(),
|
||||
node: node_name.to_string(),
|
||||
message: format!("HTTP request failed: {e}"),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
let status = resp.status();
|
||||
if !status.is_success() {
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
// Neuron returns 400 "already loaded" when two concurrent
|
||||
// requests race the same model. Treat that as success — both
|
||||
// requests effectively achieved the same end state.
|
||||
if body.contains("already loaded") {
|
||||
tracing::info!(
|
||||
model = %profile.id,
|
||||
node = node_name,
|
||||
"cold-load saw 'already loaded' — treating as success"
|
||||
);
|
||||
} else {
|
||||
return Err(RouteError::ColdLoadFailed {
|
||||
model_id: profile.id.clone(),
|
||||
node: node_name.to_string(),
|
||||
message: format!("HTTP {status}: {body}"),
|
||||
});
|
||||
}
|
||||
} else {
|
||||
tracing::info!(model = %profile.id, node = node_name, "cold-load returned 200");
|
||||
}
|
||||
|
||||
// Warm the cache: insert a Loaded ModelEntry so the next
|
||||
// resolve() finds the model without waiting for the poll loop.
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
if let Some(node) = nodes.get_mut(node_name) {
|
||||
node.models.insert(
|
||||
profile.id.clone(),
|
||||
cortex_core::node::ModelEntry {
|
||||
id: profile.id.clone(),
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: Some(chrono::Utc::now()),
|
||||
vram_estimate_mb: profile.vram_mb,
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
// A model we just loaded is presumed servable until
|
||||
// the next poll says otherwise. Seeding `false` here
|
||||
// would make the cold-load path immediately
|
||||
// un-route what it had just placed.
|
||||
servable: None,
|
||||
// Same reasoning: the ladder and its availability are
|
||||
// the neuron's to report, and the next poll (~10s)
|
||||
// fills it. Guessing here risks advertising a rung
|
||||
// this host is withholding.
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Translate a `ModelProfile` to a `ModelSpec` neuron's /models/load
|
||||
/// accepts. Devices are picked from the neuron's discovered topology —
|
||||
/// the first `min_devices` indices that meet `min_device_vram_mb`.
|
||||
async fn profile_to_spec(
|
||||
fleet: &Arc<CortexState>,
|
||||
node_name: &str,
|
||||
profile: &ModelProfile,
|
||||
) -> ModelSpec {
|
||||
let devices = {
|
||||
let nodes = fleet.nodes.read().await;
|
||||
let mut picked: Vec<u32> = Vec::new();
|
||||
if let Some(node) = nodes.get(node_name)
|
||||
&& let Some(disc) = &node.discovery
|
||||
{
|
||||
let min_vram = profile.min_device_vram_mb.unwrap_or(0);
|
||||
for d in &disc.devices {
|
||||
if d.vram_total_mb >= min_vram {
|
||||
picked.push(d.index);
|
||||
if picked.len() as u32 >= profile.min_devices {
|
||||
break;
|
||||
}
|
||||
}
|
||||
loaded_candidate.or(unloaded_candidate).ok_or_else(|| {
|
||||
if nodes.values().any(|n| n.healthy) {
|
||||
RouteError::ModelNotFound(model_id.to_string())
|
||||
} else {
|
||||
RouteError::NoHealthyNodes
|
||||
}
|
||||
}
|
||||
if picked.is_empty() {
|
||||
// Fall back to a 0..min_devices default; pick_feasible_neuron
|
||||
// already verified the topology satisfies the constraints,
|
||||
// so this only fires if discovery raced or was lost.
|
||||
(0..profile.min_devices).collect()
|
||||
} else {
|
||||
picked
|
||||
}
|
||||
})?
|
||||
};
|
||||
|
||||
let tensor_parallel = if profile.min_devices > 1 {
|
||||
Some(profile.min_devices)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
ModelSpec {
|
||||
model_id: qualified_model_id(profile),
|
||||
harness: profile.harness.clone(),
|
||||
quant: profile.quant.clone(),
|
||||
tensor_parallel,
|
||||
devices: Some(devices),
|
||||
// The catalogue's operator override rides along with the load
|
||||
// (#283), so a cold-loaded model samples the same way as one
|
||||
// this host was configured to hold resident.
|
||||
sampling: profile.sampling.clone(),
|
||||
preserve_thinking: profile.preserve_thinking,
|
||||
}
|
||||
}
|
||||
|
||||
/// Prefix the catalogue id with the scheme when one is declared, so
|
||||
/// neuron resolves the load against the right registry. Without this,
|
||||
/// a profile pointing at the helexa registry would resolve via
|
||||
/// neuron's `default_source` (typically `huggingface`) and fetch
|
||||
/// bytes from the wrong place. Profiles that omit `source` continue
|
||||
/// to pass the bare id through, preserving the pre-Phase-3 contract.
|
||||
///
|
||||
/// Stays at module scope (not nested in `profile_to_spec`) so the unit
|
||||
/// tests can exercise it without spinning up CortexState topology.
|
||||
fn qualified_model_id(profile: &ModelProfile) -> String {
|
||||
match profile.source.as_deref() {
|
||||
Some(scheme) if !scheme.is_empty() => format!("{scheme}:{}", profile.id),
|
||||
_ => profile.id.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve neuron's `/models/{id}/endpoint` to its inference URL and
|
||||
/// build the final `RouteDecision`. Shared by all three priority
|
||||
/// branches above.
|
||||
async fn finish(
|
||||
fleet: &Arc<CortexState>,
|
||||
node_name: &str,
|
||||
neuron_endpoint: &str,
|
||||
model_id: &str,
|
||||
cold_start: bool,
|
||||
) -> Result<RouteDecision, RouteError> {
|
||||
// Ask the neuron for the inference endpoint for this model.
|
||||
let endpoint_url = format!(
|
||||
"{}/models/{}/endpoint",
|
||||
neuron_endpoint,
|
||||
@@ -591,125 +89,13 @@ async fn finish(
|
||||
_ => None,
|
||||
};
|
||||
|
||||
let raw = inference_endpoint.ok_or_else(|| {
|
||||
RouteError::EndpointResolveFailed(model_id.to_string(), node_name.to_string())
|
||||
let endpoint = inference_endpoint.ok_or_else(|| {
|
||||
RouteError::EndpointResolveFailed(model_id.to_string(), node_name.clone())
|
||||
})?;
|
||||
|
||||
// Rewrite loopback inference URLs to use the configured neuron host.
|
||||
// Neuron's default bind_url is `http://localhost:13131` (it can't
|
||||
// reliably know its own externally-resolvable name). Cortex sees a
|
||||
// URL that's only meaningful from the neuron host's own perspective;
|
||||
// proxying directly to localhost from a different cortex host would
|
||||
// hit nothing. Keep neuron's port and path (a future harness could
|
||||
// serve inference on a different port than the management API), but
|
||||
// swap the host for the one in cortex.toml.
|
||||
let endpoint = rewrite_loopback_host(&raw, neuron_endpoint).unwrap_or(raw);
|
||||
|
||||
Ok(RouteDecision {
|
||||
node_name: node_name.to_string(),
|
||||
node_name,
|
||||
endpoint,
|
||||
cold_start,
|
||||
resolved_model_id: model_id.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
/// If `inference_url`'s host is a loopback name (localhost / 127.0.0.1 /
|
||||
/// 0.0.0.0 / ::1), return a copy with the host replaced by
|
||||
/// `neuron_endpoint`'s host. Otherwise return None and the caller falls
|
||||
/// back to the inference URL as-is.
|
||||
fn rewrite_loopback_host(inference_url: &str, neuron_endpoint: &str) -> Option<String> {
|
||||
let inf = url::Url::parse(inference_url).ok()?;
|
||||
let inf_host = inf.host_str()?;
|
||||
let is_loopback = matches!(inf_host, "localhost" | "127.0.0.1" | "0.0.0.0" | "::1");
|
||||
if !is_loopback {
|
||||
return None;
|
||||
}
|
||||
let neuron = url::Url::parse(neuron_endpoint).ok()?;
|
||||
let new_host = neuron.host_str()?;
|
||||
let mut out = inf.clone();
|
||||
out.set_host(Some(new_host)).ok()?;
|
||||
// url::Url::to_string normalises an empty path to "/", which then
|
||||
// breaks downstream callers that do format!("{endpoint}/v1/...")
|
||||
// and produce a double slash. The proxy URL is treated as a base
|
||||
// string that the caller appends paths to, so strip the trailing
|
||||
// slash here.
|
||||
let s = out.to_string();
|
||||
Some(s.trim_end_matches('/').to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{ModelProfile, qualified_model_id, rewrite_loopback_host};
|
||||
|
||||
fn bare_profile(id: &str, source: Option<&str>) -> ModelProfile {
|
||||
ModelProfile {
|
||||
id: id.into(),
|
||||
harness: "candle".into(),
|
||||
quant: None,
|
||||
vram_mb: None,
|
||||
min_devices: 1,
|
||||
min_device_vram_mb: None,
|
||||
pinned_on: vec![],
|
||||
residency_priority: None,
|
||||
source: source.map(String::from),
|
||||
sampling: None,
|
||||
preserve_thinking: None,
|
||||
limit: None,
|
||||
cost: None,
|
||||
capabilities: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qualified_id_passes_through_when_source_absent() {
|
||||
let p = bare_profile("Qwen/Qwen3-30B", None);
|
||||
assert_eq!(qualified_model_id(&p), "Qwen/Qwen3-30B");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qualified_id_prefixes_when_source_set() {
|
||||
let p = bare_profile("Helexa/Qwen3.6-27B-Uncensored", Some("helexa"));
|
||||
assert_eq!(
|
||||
qualified_model_id(&p),
|
||||
"helexa:Helexa/Qwen3.6-27B-Uncensored"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qualified_id_passes_through_when_source_is_empty_string() {
|
||||
// An empty scheme is treated as absent — neuron's default_source
|
||||
// substitution kicks in.
|
||||
let p = bare_profile("Qwen/Qwen3-30B", Some(""));
|
||||
assert_eq!(qualified_model_id(&p), "Qwen/Qwen3-30B");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rewrites_localhost_keeps_port_and_path() {
|
||||
let out = rewrite_loopback_host(
|
||||
"http://localhost:13131",
|
||||
"http://beast.hanzalova.internal:13131",
|
||||
);
|
||||
assert_eq!(
|
||||
out.as_deref(),
|
||||
Some("http://beast.hanzalova.internal:13131")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rewrites_loopback_with_distinct_inference_port() {
|
||||
let out = rewrite_loopback_host("http://127.0.0.1:8080", "http://beast.lan:13131");
|
||||
assert_eq!(out.as_deref(), Some("http://beast.lan:8080"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn leaves_non_loopback_alone() {
|
||||
let out = rewrite_loopback_host("http://other.host:1234", "http://beast.lan:13131");
|
||||
assert_eq!(out, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_inference_url_returns_none() {
|
||||
let out = rewrite_loopback_host("not a url", "http://beast.lan:13131");
|
||||
assert_eq!(out, None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,105 +0,0 @@
|
||||
//! Served-usage ledger (#58): cortex meters, per principal and per UTC day,
|
||||
//! the tokens it has served on behalf of mesh accounts, and periodically
|
||||
//! reports **absolute** cumulative counters to helexa-upstream for
|
||||
//! reconciliation (operators are compensated for served tokens).
|
||||
//!
|
||||
//! Counters are cumulative-since-process-start for the current period;
|
||||
//! upstream upserts them monotonically (GREATEST), so re-sending the same
|
||||
//! value is idempotent and a flush that races another is harmless. (A
|
||||
//! process restart resets the in-memory counter; the monotonic upsert keeps
|
||||
//! upstream from regressing — at most it under-counts the restarted window,
|
||||
//! acceptable for beta. One cortex per operator token is assumed.)
|
||||
|
||||
use serde::Serialize;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Mutex;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
|
||||
pub struct ServedRow {
|
||||
pub account_id: String,
|
||||
pub key_id: String,
|
||||
pub period: String, // YYYY-MM-DD (UTC)
|
||||
pub served_tokens: u64,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct ServedUsage {
|
||||
inner: Mutex<HashMap<(String, String, String), u64>>,
|
||||
}
|
||||
|
||||
impl ServedUsage {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Add served tokens for a principal in today's (UTC) period.
|
||||
pub fn add(&self, account_id: &str, key_id: &str, tokens: u64) {
|
||||
if tokens == 0 {
|
||||
return;
|
||||
}
|
||||
let period = chrono::Utc::now().format("%Y-%m-%d").to_string();
|
||||
let mut m = self.inner.lock().expect("served-usage lock");
|
||||
*m.entry((account_id.to_string(), key_id.to_string(), period))
|
||||
.or_insert(0) += tokens;
|
||||
}
|
||||
|
||||
/// Absolute cumulative counters, for a flush to upstream.
|
||||
pub fn snapshot(&self) -> Vec<ServedRow> {
|
||||
let m = self.inner.lock().expect("served-usage lock");
|
||||
m.iter()
|
||||
.map(|((account_id, key_id, period), &served_tokens)| ServedRow {
|
||||
account_id: account_id.clone(),
|
||||
key_id: key_id.clone(),
|
||||
period: period.clone(),
|
||||
served_tokens,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// POST the absolute counters to upstream's `/authz/v1/served-usage`.
|
||||
pub async fn report(
|
||||
client: &reqwest::Client,
|
||||
base_url: &str,
|
||||
bearer: &str,
|
||||
rows: &[ServedRow],
|
||||
) -> Result<(), reqwest::Error> {
|
||||
if rows.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let url = format!("{}/authz/v1/served-usage", base_url.trim_end_matches('/'));
|
||||
client
|
||||
.post(url)
|
||||
.bearer_auth(bearer)
|
||||
.json(&serde_json::json!({ "rows": rows }))
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn accumulates_per_principal_and_period() {
|
||||
let su = ServedUsage::new();
|
||||
su.add("acct", "key", 10);
|
||||
su.add("acct", "key", 5);
|
||||
su.add("acct", "other", 7);
|
||||
su.add("acct", "key", 0); // no-op
|
||||
let mut rows = su.snapshot();
|
||||
rows.sort_by(|a, b| a.key_id.cmp(&b.key_id));
|
||||
assert_eq!(rows.len(), 2);
|
||||
let key_row = rows.iter().find(|r| r.key_id == "key").unwrap();
|
||||
assert_eq!(key_row.served_tokens, 15);
|
||||
assert_eq!(
|
||||
rows.iter()
|
||||
.find(|r| r.key_id == "other")
|
||||
.unwrap()
|
||||
.served_tokens,
|
||||
7
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,7 @@
|
||||
use crate::entitlements_chain::ChainedEntitlementProvider;
|
||||
use crate::entitlements_local::LocalEntitlementProvider;
|
||||
use crate::entitlements_upstream::UpstreamEntitlementProvider;
|
||||
use cortex_core::catalogue::ModelCatalogue;
|
||||
use cortex_core::config::{EvictionSettings, GatewayConfig, NeuronEndpoint};
|
||||
use cortex_core::entitlements::EntitlementProvider;
|
||||
use cortex_core::node::NodeState;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
/// Shared fleet state, protected by a RwLock for concurrent reader access.
|
||||
@@ -16,42 +11,6 @@ pub struct CortexState {
|
||||
pub eviction: EvictionSettings,
|
||||
pub catalogue: ModelCatalogue,
|
||||
pub http_client: reqwest::Client,
|
||||
/// Resolves bearer keys to principals and enforces token budgets (#47).
|
||||
/// A local/static provider today (#50); the upstream client later (#57).
|
||||
pub entitlements: Arc<dyn EntitlementProvider>,
|
||||
/// Whether to reject unauthenticated requests (#49). Read by the auth
|
||||
/// middleware once it lands.
|
||||
pub require_auth: bool,
|
||||
/// Per-principal served-token tally (#58), reported to upstream for
|
||||
/// operator reconciliation by the flush task when upstream is enabled.
|
||||
pub served_usage: Arc<crate::served_usage::ServedUsage>,
|
||||
/// Modalities discovered for models that are not currently loaded
|
||||
/// (#241), keyed by model id.
|
||||
///
|
||||
/// A loaded model reports its own capabilities every poll, so this
|
||||
/// only carries the cold ones — which is the case that mattered,
|
||||
/// since the image model spends nearly all its time evicted and was
|
||||
/// therefore advertising nothing. Neurons derive these from their
|
||||
/// local model cache; cortex asks once and keeps the answer, because
|
||||
/// what a given model id can do does not change.
|
||||
pub discovered_capabilities: RwLock<HashMap<String, Vec<String>>>,
|
||||
/// When each `(neuron, model id)` pair was last probed for
|
||||
/// capabilities.
|
||||
///
|
||||
/// A model no node has cached can never be answered, and the poll
|
||||
/// loop runs every few seconds — without this it would ask every
|
||||
/// node about every such model forever, turning a one-off lookup
|
||||
/// into steady background traffic and a stream of 404s in the logs.
|
||||
/// Resolved ids stop being probed entirely.
|
||||
///
|
||||
/// Keyed per node, not per model: nodes are polled in config order,
|
||||
/// so a model-only key let whichever node came first record the
|
||||
/// attempt and then, if it answered 404, suppress the question to
|
||||
/// every other node for the whole back-off window. Weights are not
|
||||
/// distributed evenly across the fleet, so that is the common case
|
||||
/// rather than an edge — it hid a GGUF that two of three nodes could
|
||||
/// describe perfectly well.
|
||||
pub capability_probe_attempts: RwLock<HashMap<(String, String), std::time::Instant>>,
|
||||
}
|
||||
|
||||
impl CortexState {
|
||||
@@ -61,76 +20,27 @@ impl CortexState {
|
||||
nodes.insert(
|
||||
nc.name.clone(),
|
||||
NodeState {
|
||||
device_health: Vec::new(),
|
||||
name: nc.name.clone(),
|
||||
endpoint: nc.endpoint.clone(),
|
||||
healthy: false,
|
||||
models: HashMap::new(),
|
||||
// Filled from the neuron's /models reply at the
|
||||
// first poll (#223).
|
||||
reasoning_budget: Vec::new(),
|
||||
lifecycle_cycles: 0,
|
||||
last_poll: None,
|
||||
discovery: None,
|
||||
activation: None,
|
||||
model_load: HashMap::new(),
|
||||
consecutive_poll_failures: 0,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let catalogue = ModelCatalogue::load(&config.models_config);
|
||||
|
||||
// Local provider always handles operator + infra keys. When the
|
||||
// upstream client is enabled (#57), wrap it in the chain so locally
|
||||
// unknown keys fall through to the mesh authority; otherwise stay
|
||||
// purely local.
|
||||
let local = LocalEntitlementProvider::from_config(&config.entitlements);
|
||||
let entitlements: Arc<dyn EntitlementProvider> = if config.upstream.enabled {
|
||||
tracing::info!(url = %config.upstream.url, "upstream entitlement client enabled");
|
||||
Arc::new(ChainedEntitlementProvider::new(
|
||||
local,
|
||||
UpstreamEntitlementProvider::new(&config.upstream),
|
||||
))
|
||||
} else {
|
||||
Arc::new(local)
|
||||
};
|
||||
|
||||
Self {
|
||||
nodes: RwLock::new(nodes),
|
||||
neuron_configs: config.neurons.clone(),
|
||||
eviction: config.eviction.clone(),
|
||||
catalogue,
|
||||
// READ timeout, not a total one. reqwest's `.timeout()` is a
|
||||
// total deadline — "from when the request starts connecting
|
||||
// until the response body has finished" — which for a streamed
|
||||
// completion is a hard cap on how long a model may generate,
|
||||
// not a liveness check. At 300s and ~13 tok/s that capped every
|
||||
// stream at roughly 3.9k tokens: cortex severed the connection
|
||||
// mid-answer, the client reported the stream as terminated, and
|
||||
// neuron carried on generating into a socket nobody was reading.
|
||||
// A long agentic turn could therefore never complete through the
|
||||
// gateway however healthy the fleet was, and the failure looked
|
||||
// like a model fault because the model was blameless.
|
||||
//
|
||||
// `.read_timeout()` resets on every successful read, so it times
|
||||
// out a stalled upstream while letting a producing one run as
|
||||
// long as it needs. The same 300s is kept deliberately: it is
|
||||
// the headroom a cold model load needs before its first byte,
|
||||
// which is what the original total timeout was reaching for.
|
||||
//
|
||||
// This pairs with neuron's 1s SSE keep-alive: a healthy stream
|
||||
// puts bytes on the wire every second, so idle here means idle,
|
||||
// and the deadline finally measures what it was meant to.
|
||||
http_client: reqwest::Client::builder()
|
||||
.read_timeout(std::time::Duration::from_secs(300))
|
||||
.timeout(std::time::Duration::from_secs(300))
|
||||
.build()
|
||||
.expect("failed to build HTTP client"),
|
||||
entitlements,
|
||||
require_auth: config.entitlements.require_auth,
|
||||
served_usage: Arc::new(crate::served_usage::ServedUsage::new()),
|
||||
discovered_capabilities: RwLock::new(HashMap::new()),
|
||||
capability_probe_attempts: RwLock::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,318 +0,0 @@
|
||||
//! Alias resolution: a client request with `model: "helexa/small"`
|
||||
//! routes to the concrete model id (e.g. `Qwen/Qwen3-1.7B`), with the
|
||||
//! proxied request body rewritten so the upstream neuron sees a model
|
||||
//! name that matches its loaded handle.
|
||||
|
||||
mod common;
|
||||
|
||||
use cortex_core::config::{
|
||||
EvictionSettings, EvictionStrategy, GatewayConfig, GatewaySettings, NeuronEndpoint,
|
||||
};
|
||||
use cortex_core::node::{ModelEntry, ModelStatus};
|
||||
use cortex_gateway::state::CortexState;
|
||||
use serde_json::json;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU32, Ordering};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
/// Write a `models.toml` with one alias into a directory of its own
|
||||
/// under the system temp dir, and prove the bytes landed before
|
||||
/// returning. The file is reaped by the OS at exit.
|
||||
///
|
||||
/// Each call gets its own directory rather than a distinct filename in
|
||||
/// a shared one. Two things make that worth the extra syscall:
|
||||
///
|
||||
/// - The tests in this file run concurrently in one binary and write
|
||||
/// *different* alias tables. A shared name is one collision away from
|
||||
/// a test loading another's catalogue.
|
||||
/// - The failure is silent. `ModelCatalogue::load` maps a missing,
|
||||
/// unreadable, unparseable **or empty** file onto an empty catalogue,
|
||||
/// and `resolve_alias` then returns the alias unchanged — so the
|
||||
/// mismatch surfaces at whatever assertion happens to touch it, tens
|
||||
/// of lines from the cause. Observed on CI 2026-08-30 as
|
||||
/// `left: "helexa/small", right: "test-model"` while the same commit
|
||||
/// passed on its branch and 60/60 locally.
|
||||
///
|
||||
/// The read-back is for the same reason: an empty catalogue is
|
||||
/// indistinguishable from a catalogue that never loaded, so a test may
|
||||
/// not simply assume its fixture is on disk.
|
||||
fn write_models_toml(alias: &str, target: &str) -> PathBuf {
|
||||
static SEQ: AtomicU32 = AtomicU32::new(0);
|
||||
let contents = format!(
|
||||
r#"
|
||||
[aliases]
|
||||
"{alias}" = "{target}"
|
||||
"#
|
||||
);
|
||||
let mut dir = std::env::temp_dir();
|
||||
let pid = std::process::id();
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos();
|
||||
let seq = SEQ.fetch_add(1, Ordering::Relaxed);
|
||||
dir.push(format!("cortex-test-models-{pid}-{now}-{seq}"));
|
||||
std::fs::create_dir_all(&dir).expect("create temp catalogue dir");
|
||||
let path = dir.join("models.toml");
|
||||
std::fs::write(&path, &contents).expect("write temp models.toml");
|
||||
let seen = std::fs::read_to_string(&path).expect("read back temp models.toml");
|
||||
assert_eq!(
|
||||
seen,
|
||||
contents,
|
||||
"temp catalogue did not round-trip at {}",
|
||||
path.display()
|
||||
);
|
||||
path
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_alias_resolves_in_chat_completions() {
|
||||
let mock_url = common::spawn_mock_neuron().await;
|
||||
let models_path = write_models_toml("helexa/small", "test-model");
|
||||
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![NeuronEndpoint {
|
||||
name: "mock-node".into(),
|
||||
endpoint: mock_url,
|
||||
}],
|
||||
models_config: models_path.to_string_lossy().to_string(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
|
||||
// Seed the node as healthy with the concrete model loaded under
|
||||
// the target id. The poller doesn't run in this test; we just
|
||||
// populate state manually.
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("mock-node").expect("node must exist");
|
||||
node.healthy = true;
|
||||
node.models.insert(
|
||||
"test-model".into(),
|
||||
ModelEntry {
|
||||
id: "test-model".into(),
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: None,
|
||||
vram_estimate_mb: None,
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
// Sanity: the catalogue actually picked up the alias.
|
||||
assert_eq!(
|
||||
fleet.catalogue.resolve_alias("helexa/small"),
|
||||
"test-model",
|
||||
"alias should resolve to target id"
|
||||
);
|
||||
|
||||
// Spawn the gateway against this fleet.
|
||||
let app = cortex_gateway::build_app(Arc::clone(&fleet));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let gateway_addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
let gateway_url = format!("http://{gateway_addr}");
|
||||
|
||||
// Send a chat completion against the alias. The mock backend
|
||||
// echoes back the `model` field it received — so a body whose
|
||||
// model wasn't rewritten would come back as "helexa/small", and a
|
||||
// properly-rewritten one as "test-model".
|
||||
let client = reqwest::Client::new();
|
||||
let resp = client
|
||||
.post(format!("{gateway_url}/v1/chat/completions"))
|
||||
.json(&json!({
|
||||
"model": "helexa/small",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.expect("gateway should respond");
|
||||
|
||||
assert!(resp.status().is_success(), "gateway returned non-2xx");
|
||||
let body: serde_json::Value = resp.json().await.expect("response is JSON");
|
||||
assert_eq!(
|
||||
body.get("model").and_then(|m| m.as_str()),
|
||||
Some("test-model"),
|
||||
"mock backend should have seen the resolved model id, not the alias"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_aliases_surface_in_v1_models() {
|
||||
let mock_url = common::spawn_mock_neuron().await;
|
||||
let models_path = write_models_toml("helexa/small", "test-model");
|
||||
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![NeuronEndpoint {
|
||||
name: "mock-node".into(),
|
||||
endpoint: mock_url,
|
||||
}],
|
||||
models_config: models_path.to_string_lossy().to_string(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
|
||||
// Seed the target as loaded so the alias's mirrored entry shows
|
||||
// loaded=true.
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("mock-node").expect("node must exist");
|
||||
node.healthy = true;
|
||||
node.models.insert(
|
||||
"test-model".into(),
|
||||
ModelEntry {
|
||||
id: "test-model".into(),
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: None,
|
||||
vram_estimate_mb: Some(2000),
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let app = cortex_gateway::build_app(Arc::clone(&fleet));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let gateway_addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
let gateway_url = format!("http://{gateway_addr}");
|
||||
|
||||
let resp = reqwest::get(format!("{gateway_url}/v1/models"))
|
||||
.await
|
||||
.expect("gateway should respond");
|
||||
let body: serde_json::Value = resp.json().await.unwrap();
|
||||
let entries = body
|
||||
.get("data")
|
||||
.and_then(|d| d.as_array())
|
||||
.expect("data array");
|
||||
|
||||
// Both the alias and the target should be present.
|
||||
let ids: Vec<&str> = entries
|
||||
.iter()
|
||||
.filter_map(|e| e.get("id").and_then(|v| v.as_str()))
|
||||
.collect();
|
||||
assert!(ids.contains(&"test-model"), "target should be listed");
|
||||
assert!(ids.contains(&"helexa/small"), "alias should be listed");
|
||||
|
||||
// The alias's `loaded` flag and locations should mirror the target.
|
||||
let alias_entry = entries
|
||||
.iter()
|
||||
.find(|e| e.get("id").and_then(|v| v.as_str()) == Some("helexa/small"))
|
||||
.expect("alias entry");
|
||||
assert_eq!(alias_entry.get("loaded"), Some(&json!(true)));
|
||||
let locations = alias_entry
|
||||
.get("locations")
|
||||
.and_then(|l| l.as_array())
|
||||
.expect("locations array");
|
||||
assert_eq!(locations.len(), 1);
|
||||
assert_eq!(
|
||||
locations[0].get("node").and_then(|n| n.as_str()),
|
||||
Some("mock-node")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_alias_falls_through_for_unmapped_model() {
|
||||
// Catalogue has an alias for some-other-thing but the request
|
||||
// model "test-model" isn't an alias; resolution should be a no-op.
|
||||
let mock_url = common::spawn_mock_neuron().await;
|
||||
let models_path = write_models_toml("helexa/large", "definitely-not-loaded");
|
||||
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![NeuronEndpoint {
|
||||
name: "mock-node".into(),
|
||||
endpoint: mock_url,
|
||||
}],
|
||||
models_config: models_path.to_string_lossy().to_string(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("mock-node").expect("node must exist");
|
||||
node.healthy = true;
|
||||
node.models.insert(
|
||||
"test-model".into(),
|
||||
ModelEntry {
|
||||
id: "test-model".into(),
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: None,
|
||||
vram_estimate_mb: None,
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let app = cortex_gateway::build_app(Arc::clone(&fleet));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let gateway_addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
let gateway_url = format!("http://{gateway_addr}");
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway_url}/v1/chat/completions"))
|
||||
.json(&json!({
|
||||
"model": "test-model",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(resp.status().is_success());
|
||||
let body: serde_json::Value = resp.json().await.unwrap();
|
||||
assert_eq!(
|
||||
body.get("model").and_then(|m| m.as_str()),
|
||||
Some("test-model")
|
||||
);
|
||||
}
|
||||
@@ -123,212 +123,3 @@ async fn test_anthropic_invalid_request() {
|
||||
|
||||
assert_eq!(resp.status(), 400);
|
||||
}
|
||||
|
||||
/// Tool round-trip: an Anthropic `/v1/messages` request carrying tools
|
||||
/// (the Claude Code shape: `{name, description, input_schema}`) must
|
||||
/// reach the upstream neuron reshaped into OpenAI function-tool form,
|
||||
/// and tool history (`tool_use` / `tool_result` blocks) must become
|
||||
/// `tool_calls` / `role:"tool"` messages. This is the fix for the
|
||||
/// failure where the model received malformed tool defs and improvised
|
||||
/// an unparseable `<tool_use_name>` format.
|
||||
#[tokio::test]
|
||||
async fn test_anthropic_tools_reshaped_for_upstream() {
|
||||
let (mock_url, captured) = common::spawn_capturing_mock_neuron().await;
|
||||
let gw_url = common::spawn_gateway(&mock_url).await;
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let resp = client
|
||||
.post(format!("{gw_url}/v1/messages"))
|
||||
.header("content-type", "application/json")
|
||||
.json(&json!({
|
||||
"model": "test-model",
|
||||
"max_tokens": 100,
|
||||
"tools": [{
|
||||
"name": "Read",
|
||||
"description": "Read a file from disk",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {"path": {"type": "string"}},
|
||||
"required": ["path"]
|
||||
}
|
||||
}],
|
||||
"tool_choice": {"type": "auto"},
|
||||
"messages": [
|
||||
{"role": "user", "content": "read /etc/hosts"},
|
||||
{"role": "assistant", "content": [
|
||||
{"type": "text", "text": "Reading it."},
|
||||
{"type": "tool_use", "id": "toolu_42", "name": "Read",
|
||||
"input": {"path": "/etc/hosts"}}
|
||||
]},
|
||||
{"role": "user", "content": [
|
||||
{"type": "tool_result", "tool_use_id": "toolu_42",
|
||||
"content": "127.0.0.1 localhost"}
|
||||
]}
|
||||
]
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.expect("request should succeed");
|
||||
assert_eq!(resp.status(), 200);
|
||||
|
||||
let forwarded = {
|
||||
let guard = captured.lock().unwrap();
|
||||
guard.last().cloned().expect("upstream received a request")
|
||||
};
|
||||
|
||||
// Tool definitions reshaped to OpenAI function form.
|
||||
let tools = forwarded["tools"].as_array().expect("tools array");
|
||||
assert_eq!(tools[0]["type"], "function");
|
||||
assert_eq!(tools[0]["function"]["name"], "Read");
|
||||
assert_eq!(
|
||||
tools[0]["function"]["parameters"]["properties"]["path"]["type"],
|
||||
"string"
|
||||
);
|
||||
assert!(tools[0]["function"].get("input_schema").is_none());
|
||||
|
||||
// tool_choice mapped.
|
||||
assert_eq!(forwarded["tool_choice"], "auto");
|
||||
|
||||
// Message history: user, assistant(+tool_calls), tool, user.
|
||||
let msgs = forwarded["messages"].as_array().expect("messages array");
|
||||
let assistant = msgs
|
||||
.iter()
|
||||
.find(|m| m["role"] == "assistant")
|
||||
.expect("assistant turn");
|
||||
assert_eq!(assistant["tool_calls"][0]["id"], "toolu_42");
|
||||
assert_eq!(assistant["tool_calls"][0]["function"]["name"], "Read");
|
||||
// arguments is the parsed object, not a JSON string — the Qwen3.6
|
||||
// chat template iterates `tool_call.arguments | items`.
|
||||
assert_eq!(
|
||||
assistant["tool_calls"][0]["function"]["arguments"],
|
||||
json!({"path": "/etc/hosts"})
|
||||
);
|
||||
|
||||
let tool_msg = msgs
|
||||
.iter()
|
||||
.find(|m| m["role"] == "tool")
|
||||
.expect("tool result turn");
|
||||
assert_eq!(tool_msg["tool_call_id"], "toolu_42");
|
||||
assert_eq!(tool_msg["content"], "127.0.0.1 localhost");
|
||||
}
|
||||
|
||||
/// #24: a streaming Anthropic request gets a translated Anthropic SSE
|
||||
/// stream — not raw OpenAI frames. Verifies the full event sequence,
|
||||
/// text reassembly, and the content type.
|
||||
#[tokio::test]
|
||||
async fn test_anthropic_streaming_sse_translation() {
|
||||
let mock_url =
|
||||
common::spawn_streaming_mock_neuron(4, std::time::Duration::from_millis(20)).await;
|
||||
let gw_url = common::spawn_gateway(&mock_url).await;
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let resp = client
|
||||
.post(format!("{gw_url}/v1/messages"))
|
||||
.header("content-type", "application/json")
|
||||
.json(&json!({
|
||||
"model": "test-model",
|
||||
"max_tokens": 64,
|
||||
"stream": true,
|
||||
"messages": [{"role": "user", "content": "Hi"}]
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.expect("request should succeed");
|
||||
|
||||
assert_eq!(resp.status(), 200);
|
||||
assert!(
|
||||
resp.headers()
|
||||
.get("content-type")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or("")
|
||||
.starts_with("text/event-stream"),
|
||||
"anthropic stream must be SSE"
|
||||
);
|
||||
|
||||
let body = resp.text().await.expect("stream should complete");
|
||||
assert!(
|
||||
!body.contains("chat.completion.chunk"),
|
||||
"raw OpenAI frames must not leak through:\n{body}"
|
||||
);
|
||||
|
||||
let event_names: Vec<&str> = body
|
||||
.lines()
|
||||
.filter_map(|l| l.strip_prefix("event: "))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
event_names,
|
||||
vec![
|
||||
"message_start",
|
||||
"content_block_start",
|
||||
"content_block_delta",
|
||||
"content_block_delta",
|
||||
"content_block_delta",
|
||||
"content_block_delta",
|
||||
"content_block_stop",
|
||||
"message_delta",
|
||||
"message_stop",
|
||||
],
|
||||
"unexpected event sequence:\n{body}"
|
||||
);
|
||||
|
||||
// Reassemble the text deltas: the mock emits token0..token3.
|
||||
let text: String = body
|
||||
.lines()
|
||||
.filter_map(|l| l.strip_prefix("data: "))
|
||||
.filter_map(|d| serde_json::from_str::<serde_json::Value>(d).ok())
|
||||
.filter(|v| v["type"] == "content_block_delta")
|
||||
.filter_map(|v| v["delta"]["text"].as_str().map(String::from))
|
||||
.collect();
|
||||
assert_eq!(text, "token0token1token2token3");
|
||||
|
||||
// The mock sends no finish_reason — stop_reason defaults to
|
||||
// end_turn, and output_tokens falls back to the delta count.
|
||||
let message_delta = body
|
||||
.lines()
|
||||
.filter_map(|l| l.strip_prefix("data: "))
|
||||
.filter_map(|d| serde_json::from_str::<serde_json::Value>(d).ok())
|
||||
.find(|v| v["type"] == "message_delta")
|
||||
.expect("message_delta event present");
|
||||
assert_eq!(message_delta["delta"]["stop_reason"], "end_turn");
|
||||
assert_eq!(message_delta["usage"]["output_tokens"], 4);
|
||||
}
|
||||
|
||||
/// #24: an upstream usage frame (stream_options include_usage shape)
|
||||
/// rides into message_delta as input/output token counts.
|
||||
#[tokio::test]
|
||||
async fn test_anthropic_streaming_usage_propagation() {
|
||||
let mock_url = common::spawn_streaming_mock_neuron_with_usage(
|
||||
3,
|
||||
std::time::Duration::from_millis(10),
|
||||
225,
|
||||
42,
|
||||
)
|
||||
.await;
|
||||
let gw_url = common::spawn_gateway(&mock_url).await;
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let body = client
|
||||
.post(format!("{gw_url}/v1/messages"))
|
||||
.header("content-type", "application/json")
|
||||
.json(&json!({
|
||||
"model": "test-model",
|
||||
"max_tokens": 64,
|
||||
"stream": true,
|
||||
"messages": [{"role": "user", "content": "Hi"}]
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.expect("request should succeed")
|
||||
.text()
|
||||
.await
|
||||
.expect("stream should complete");
|
||||
|
||||
let message_delta = body
|
||||
.lines()
|
||||
.filter_map(|l| l.strip_prefix("data: "))
|
||||
.filter_map(|d| serde_json::from_str::<serde_json::Value>(d).ok())
|
||||
.find(|v| v["type"] == "message_delta")
|
||||
.expect("message_delta event present");
|
||||
assert_eq!(message_delta["usage"]["output_tokens"], 42);
|
||||
assert_eq!(message_delta["usage"]["input_tokens"], 225);
|
||||
}
|
||||
|
||||
@@ -1,275 +0,0 @@
|
||||
//! Integration tests for API-key auth + principal resolution (#49).
|
||||
//!
|
||||
//! Verifies the #63 rejection contract (401 invalid_api_key via the #60
|
||||
//! envelope) and that an authenticated request reaches neuron carrying the
|
||||
//! internal principal headers — while a client-supplied principal header is
|
||||
//! stripped (anti-spoofing).
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::Path;
|
||||
use axum::http::HeaderMap;
|
||||
use axum::routing::{get, post};
|
||||
use cortex_core::config::{
|
||||
ApiKeyConfig, EntitlementsConfig, EvictionSettings, EvictionStrategy, GatewayConfig,
|
||||
GatewaySettings, NeuronEndpoint,
|
||||
};
|
||||
use cortex_core::entitlements::{CapWindow, HEADER_ACCOUNT_ID, HEADER_KEY_ID};
|
||||
use cortex_core::node::{ModelEntry, ModelStatus};
|
||||
use cortex_gateway::state::CortexState;
|
||||
use serde_json::{Value, json};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
/// What the mock neuron observed on the inbound `/v1/chat/completions`
|
||||
/// request: the principal headers cortex stamped (or didn't).
|
||||
#[derive(Default)]
|
||||
struct Seen {
|
||||
account_id: Option<String>,
|
||||
key_id: Option<String>,
|
||||
}
|
||||
|
||||
/// Spawn a mock neuron that records the principal headers it receives and
|
||||
/// returns a trivial chat completion. Returns (base_url, observed).
|
||||
async fn spawn_capturing_neuron() -> (String, Arc<Mutex<Seen>>) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let base_url = format!("http://{addr}");
|
||||
let inference_url = base_url.clone();
|
||||
let seen: Arc<Mutex<Seen>> = Arc::new(Mutex::new(Seen::default()));
|
||||
let sink = Arc::clone(&seen);
|
||||
|
||||
let app = axum::Router::new()
|
||||
.route(
|
||||
"/models/{model_id}/endpoint",
|
||||
get(move |Path(_): Path<String>| {
|
||||
let url = inference_url.clone();
|
||||
async move { Json(json!({ "url": url })) }
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/v1/chat/completions",
|
||||
post(move |headers: HeaderMap, Json(body): Json<Value>| {
|
||||
let sink = Arc::clone(&sink);
|
||||
async move {
|
||||
{
|
||||
let mut s = sink.lock().unwrap();
|
||||
s.account_id = headers
|
||||
.get(HEADER_ACCOUNT_ID)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(str::to_string);
|
||||
s.key_id = headers
|
||||
.get(HEADER_KEY_ID)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(str::to_string);
|
||||
}
|
||||
let model = body.get("model").and_then(Value::as_str).unwrap_or("m");
|
||||
Json(json!({
|
||||
"id": "chatcmpl-auth-001",
|
||||
"object": "chat.completion",
|
||||
"created": 1700000000_u64,
|
||||
"model": model,
|
||||
"choices": [{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "ok"},
|
||||
"finish_reason": "stop"
|
||||
}],
|
||||
"usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}
|
||||
}))
|
||||
}
|
||||
}),
|
||||
)
|
||||
.with_state(());
|
||||
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
|
||||
(base_url, seen)
|
||||
}
|
||||
|
||||
/// Spawn a gateway with the given entitlements config, a single neuron, and
|
||||
/// `test-model` seeded as loaded (build_app spawns no poller).
|
||||
async fn spawn_gateway(neuron_url: &str, entitlements: EntitlementsConfig) -> String {
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![NeuronEndpoint {
|
||||
name: "mock-node".into(),
|
||||
endpoint: neuron_url.to_string(),
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements,
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("mock-node").unwrap();
|
||||
node.healthy = true;
|
||||
node.models.insert(
|
||||
"test-model".into(),
|
||||
ModelEntry {
|
||||
id: "test-model".into(),
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: None,
|
||||
vram_estimate_mb: Some(8000),
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let app = cortex_gateway::build_app(Arc::clone(&fleet));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
format!("http://{addr}")
|
||||
}
|
||||
|
||||
fn one_key_config(require_auth: bool) -> EntitlementsConfig {
|
||||
EntitlementsConfig {
|
||||
require_auth,
|
||||
keys: vec![ApiKeyConfig {
|
||||
key: "sk-good".into(),
|
||||
account_id: "acct-1".into(),
|
||||
key_id: Some("key-1".into()),
|
||||
hard_cap: None,
|
||||
window: CapWindow::Balance,
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
fn chat_body() -> Value {
|
||||
json!({
|
||||
"model": "test-model",
|
||||
"messages": [{"role": "user", "content": "hi"}]
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn missing_key_when_required_is_401_invalid_api_key() {
|
||||
let (neuron, _seen) = spawn_capturing_neuron().await;
|
||||
let gateway = spawn_gateway(&neuron, one_key_config(true)).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.json(&chat_body())
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::UNAUTHORIZED);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["error"]["code"], "invalid_api_key");
|
||||
assert_eq!(body["error"]["type"], "invalid_request_error");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unrecognized_key_is_ignored_when_auth_not_required() {
|
||||
let (neuron, seen) = spawn_capturing_neuron().await;
|
||||
// allow-anonymous mode: a placeholder/unknown bearer (as opencode,
|
||||
// Open WebUI, Agent Zero, litellm all send by default) must NOT be
|
||||
// rejected — it's ignored and the request is served anonymously.
|
||||
let gateway = spawn_gateway(&neuron, one_key_config(false)).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.bearer_auth("sk-dummy-placeholder")
|
||||
.json(&chat_body())
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::OK);
|
||||
let _ = resp.bytes().await.unwrap();
|
||||
// Served, but anonymous — no principal stamped from the bogus key.
|
||||
assert!(seen.lock().unwrap().account_id.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_key_is_401_when_auth_required() {
|
||||
let (neuron, seen) = spawn_capturing_neuron().await;
|
||||
// With auth required, a present-but-wrong credential is rejected.
|
||||
let gateway = spawn_gateway(&neuron, one_key_config(true)).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.bearer_auth("sk-wrong")
|
||||
.json(&chat_body())
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::UNAUTHORIZED);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["error"]["code"], "invalid_api_key");
|
||||
// Rejected before dispatch — neuron never saw the request.
|
||||
assert!(seen.lock().unwrap().account_id.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn valid_key_reaches_neuron_with_principal_headers() {
|
||||
let (neuron, seen) = spawn_capturing_neuron().await;
|
||||
let gateway = spawn_gateway(&neuron, one_key_config(true)).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.bearer_auth("sk-good")
|
||||
// A spoofed principal header must be stripped, not forwarded.
|
||||
.header(HEADER_ACCOUNT_ID, "attacker")
|
||||
.json(&chat_body())
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::OK);
|
||||
let s = seen.lock().unwrap();
|
||||
assert_eq!(s.account_id.as_deref(), Some("acct-1"));
|
||||
assert_eq!(s.key_id.as_deref(), Some("key-1"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn anonymous_allowed_when_auth_not_required() {
|
||||
let (neuron, seen) = spawn_capturing_neuron().await;
|
||||
let gateway = spawn_gateway(&neuron, EntitlementsConfig::default()).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.json(&chat_body())
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::OK);
|
||||
// No principal resolved → no principal headers stamped.
|
||||
let s = seen.lock().unwrap();
|
||||
assert!(s.account_id.is_none());
|
||||
assert!(s.key_id.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn health_is_public_even_when_auth_required() {
|
||||
let (neuron, _seen) = spawn_capturing_neuron().await;
|
||||
let gateway = spawn_gateway(&neuron, one_key_config(true)).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.get(format!("{gateway}/health"))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::OK);
|
||||
}
|
||||
@@ -1,256 +0,0 @@
|
||||
//! Integration tests for budget enforcement (#52) — the A0 seatbelt.
|
||||
//!
|
||||
//! A reservation over the key's hard cap is refused *before* neuron is hit,
|
||||
//! with the #63 code matching the cap-window semantics: `rate_limit_exceeded`
|
||||
//! plus `Retry-After` for a resetting window, `insufficient_quota` for a hard
|
||||
//! balance. Spend never exceeds the cap. No 402, ever.
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::Path;
|
||||
use axum::routing::{get, post};
|
||||
use cortex_core::config::{
|
||||
ApiKeyConfig, EntitlementsConfig, EvictionSettings, EvictionStrategy, GatewayConfig,
|
||||
GatewaySettings, NeuronEndpoint,
|
||||
};
|
||||
use cortex_core::entitlements::{CapWindow, Principal};
|
||||
use cortex_core::node::{ModelEntry, ModelStatus};
|
||||
use cortex_gateway::state::CortexState;
|
||||
use serde_json::{Value, json};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
/// Mock neuron with a hit counter on the inference path, so a test can prove
|
||||
/// a request was (or wasn't) dispatched.
|
||||
async fn spawn_counting_neuron() -> (String, Arc<AtomicU64>) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let base_url = format!("http://{addr}");
|
||||
let inference_url = base_url.clone();
|
||||
let hits = Arc::new(AtomicU64::new(0));
|
||||
let sink = Arc::clone(&hits);
|
||||
|
||||
let app = axum::Router::new()
|
||||
.route(
|
||||
"/models/{model_id}/endpoint",
|
||||
get(move |Path(_): Path<String>| {
|
||||
let url = inference_url.clone();
|
||||
async move { Json(json!({ "url": url })) }
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/v1/chat/completions",
|
||||
post(move |Json(body): Json<Value>| {
|
||||
let sink = Arc::clone(&sink);
|
||||
async move {
|
||||
sink.fetch_add(1, Ordering::SeqCst);
|
||||
let model = body.get("model").and_then(Value::as_str).unwrap_or("m");
|
||||
Json(json!({
|
||||
"id": "chatcmpl-budget",
|
||||
"object": "chat.completion",
|
||||
"created": 1700000000_u64,
|
||||
"model": model,
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}
|
||||
}))
|
||||
}
|
||||
}),
|
||||
);
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
(base_url, hits)
|
||||
}
|
||||
|
||||
async fn spawn_gateway(neuron_url: &str, key: ApiKeyConfig) -> (Arc<CortexState>, String) {
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![NeuronEndpoint {
|
||||
name: "mock-node".into(),
|
||||
endpoint: neuron_url.to_string(),
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: EntitlementsConfig {
|
||||
require_auth: true,
|
||||
keys: vec![key],
|
||||
},
|
||||
upstream: Default::default(),
|
||||
};
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("mock-node").unwrap();
|
||||
node.healthy = true;
|
||||
node.models.insert(
|
||||
"test-model".into(),
|
||||
ModelEntry {
|
||||
id: "test-model".into(),
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: None,
|
||||
vram_estimate_mb: Some(8000),
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
let app = cortex_gateway::build_app(Arc::clone(&fleet));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
(fleet, format!("http://{addr}"))
|
||||
}
|
||||
|
||||
fn key(window: CapWindow, hard_cap: u64) -> ApiKeyConfig {
|
||||
ApiKeyConfig {
|
||||
key: "sk-cap".into(),
|
||||
account_id: "acct-cap".into(),
|
||||
key_id: Some("key-cap".into()),
|
||||
hard_cap: Some(hard_cap),
|
||||
window,
|
||||
}
|
||||
}
|
||||
|
||||
fn chat(max_tokens: u64) -> Value {
|
||||
json!({
|
||||
"model": "test-model",
|
||||
"max_tokens": max_tokens,
|
||||
"messages": [{"role": "user", "content": "hi"}]
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn balance_over_cap_is_429_insufficient_quota_before_dispatch() {
|
||||
let (neuron, hits) = spawn_counting_neuron().await;
|
||||
// Cap far below a single request's reservation (max_tokens 1000).
|
||||
let (_fleet, gateway) = spawn_gateway(&neuron, key(CapWindow::Balance, 10)).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.bearer_auth("sk-cap")
|
||||
.json(&chat(1000))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::TOO_MANY_REQUESTS);
|
||||
// Hard balance → no Retry-After.
|
||||
assert!(resp.headers().get(reqwest::header::RETRY_AFTER).is_none());
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["error"]["code"], "insufficient_quota");
|
||||
// Refused before dispatch — neuron never saw it.
|
||||
assert_eq!(hits.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rolling_over_cap_is_429_rate_limited_with_retry_after() {
|
||||
let (neuron, hits) = spawn_counting_neuron().await;
|
||||
let (_fleet, gateway) =
|
||||
spawn_gateway(&neuron, key(CapWindow::Rolling { seconds: 3600 }, 10)).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.bearer_auth("sk-cap")
|
||||
.json(&chat(1000))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::TOO_MANY_REQUESTS);
|
||||
let retry = resp
|
||||
.headers()
|
||||
.get(reqwest::header::RETRY_AFTER)
|
||||
.expect("rolling-window rejection must carry Retry-After");
|
||||
assert!(retry.to_str().unwrap().parse::<u64>().unwrap() >= 1);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["error"]["code"], "rate_limit_exceeded");
|
||||
assert_eq!(hits.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn within_cap_is_served() {
|
||||
let (neuron, hits) = spawn_counting_neuron().await;
|
||||
let (_fleet, gateway) = spawn_gateway(&neuron, key(CapWindow::Balance, 1_000_000)).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.bearer_auth("sk-cap")
|
||||
.json(&chat(50))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::OK);
|
||||
let _ = resp.bytes().await.unwrap();
|
||||
assert_eq!(hits.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a0_seatbelt_caps_a_runaway_fan_out() {
|
||||
// An Agent-Zero-style key with a modest cap: a burst of requests drains
|
||||
// it, then further requests are refused — the account stops draining and
|
||||
// spend never exceeds the cap.
|
||||
let (neuron, hits) = spawn_counting_neuron().await;
|
||||
let (fleet, gateway) = spawn_gateway(&neuron, key(CapWindow::Balance, 100)).await;
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let mut ok = 0;
|
||||
let mut refused = 0;
|
||||
for _ in 0..20 {
|
||||
let resp = client
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.bearer_auth("sk-cap")
|
||||
.json(&chat(20))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
match resp.status() {
|
||||
reqwest::StatusCode::OK => {
|
||||
ok += 1;
|
||||
let _ = resp.bytes().await.unwrap();
|
||||
}
|
||||
reqwest::StatusCode::TOO_MANY_REQUESTS => {
|
||||
refused += 1;
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["error"]["code"], "insufficient_quota");
|
||||
}
|
||||
other => panic!("unexpected status {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
assert!(ok >= 1, "some requests should be served");
|
||||
assert!(refused >= 1, "the cap must eventually refuse the fan-out");
|
||||
assert_eq!(
|
||||
hits.load(Ordering::SeqCst),
|
||||
ok,
|
||||
"refused requests never dispatched"
|
||||
);
|
||||
|
||||
// Spend never exceeded the hard cap (reservation prevents overshoot).
|
||||
// Poll briefly for in-flight settles to land.
|
||||
let principal = Principal {
|
||||
account_id: "acct-cap".into(),
|
||||
key_id: "key-cap".into(),
|
||||
};
|
||||
for _ in 0..50 {
|
||||
let snap = fleet.entitlements.snapshot(&principal).await.unwrap();
|
||||
if snap.reserved == 0 {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
|
||||
}
|
||||
let snap = fleet.entitlements.snapshot(&principal).await.unwrap();
|
||||
assert!(snap.spent <= 100, "spent {} exceeded cap", snap.spent);
|
||||
}
|
||||
@@ -1,208 +0,0 @@
|
||||
//! Cold models advertise what they can do (#241).
|
||||
//!
|
||||
//! Before this, `/v1/models` reported `capabilities: []` for anything not
|
||||
//! currently loaded, because the only source of capabilities was a loaded
|
||||
//! neuron reporting its own handle. That made image generation
|
||||
//! undiscoverable in practice: the image model is evicted almost all of
|
||||
//! the time, so a client listing our models saw an entry that looked like
|
||||
//! a text model with nothing to say about itself.
|
||||
//!
|
||||
//! The fix is discovery, not configuration — neurons derive modalities
|
||||
//! from their local model cache and cortex asks. These tests pin the two
|
||||
//! properties that matter: a cold model's capabilities reach `/v1/models`,
|
||||
//! and a node with no local evidence cannot erase them.
|
||||
|
||||
mod common;
|
||||
|
||||
use cortex_core::config::{
|
||||
EvictionSettings, EvictionStrategy, GatewayConfig, GatewaySettings, NeuronEndpoint,
|
||||
};
|
||||
use cortex_gateway::state::CortexState;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
/// A catalogue holding one image model that nothing has loaded.
|
||||
fn write_catalogue() -> std::path::PathBuf {
|
||||
static SEQ: AtomicU64 = AtomicU64::new(0);
|
||||
let toml = r#"
|
||||
[[models]]
|
||||
id = "Tongyi-MAI/Z-Image-Turbo"
|
||||
harness = "candle"
|
||||
min_devices = 1
|
||||
|
||||
[aliases]
|
||||
"helexa/image" = "Tongyi-MAI/Z-Image-Turbo"
|
||||
"#;
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"cortex_test_capability_models.{}.{}.toml",
|
||||
std::process::id(),
|
||||
SEQ.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
std::fs::write(&path, toml).unwrap();
|
||||
path
|
||||
}
|
||||
|
||||
fn fleet_for(endpoint: String) -> Arc<CortexState> {
|
||||
fleet_of(vec![("gpu", endpoint)])
|
||||
}
|
||||
|
||||
fn fleet_of(neurons: Vec<(&str, String)>) -> Arc<CortexState> {
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: neurons
|
||||
.into_iter()
|
||||
.map(|(name, endpoint)| NeuronEndpoint {
|
||||
name: name.into(),
|
||||
endpoint,
|
||||
})
|
||||
.collect(),
|
||||
models_config: write_catalogue().to_string_lossy().into_owned(),
|
||||
..Default::default()
|
||||
};
|
||||
Arc::new(CortexState::from_config(&config))
|
||||
}
|
||||
|
||||
/// Serve `fleet` and return its base URL, so a test can assert the
|
||||
/// actual HTTP surface rather than the state behind it.
|
||||
async fn serve(fleet: Arc<CortexState>) -> String {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let app = cortex_gateway::build_app(fleet);
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
format!("http://{addr}")
|
||||
}
|
||||
|
||||
/// The headline case: nothing loaded anywhere, and `/v1/models` still
|
||||
/// says the image model generates images. This is the acceptance
|
||||
/// criterion from #241 — asserted on the response a client actually
|
||||
/// receives, because that is where the bug was visible.
|
||||
#[tokio::test]
|
||||
async fn cold_image_model_advertises_image_capability() {
|
||||
let neuron = common::spawn_mock_neuron_with_capabilities(
|
||||
serde_json::json!([]),
|
||||
&[("Tongyi-MAI/Z-Image-Turbo", vec!["image"])],
|
||||
)
|
||||
.await;
|
||||
let fleet = fleet_for(neuron);
|
||||
|
||||
cortex_gateway::poller::poll_once(&fleet).await;
|
||||
|
||||
let gateway = serve(fleet).await;
|
||||
let body: serde_json::Value = reqwest::get(format!("{gateway}/v1/models"))
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let entry = body["data"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|m| m["id"] == "Tongyi-MAI/Z-Image-Turbo")
|
||||
.expect("the catalogued image model should be listed");
|
||||
assert_eq!(
|
||||
entry["loaded"], false,
|
||||
"precondition: this asserts the *cold* path"
|
||||
);
|
||||
assert_eq!(
|
||||
entry["capabilities"],
|
||||
serde_json::json!(["image"]),
|
||||
"a cold image model must still advertise that it generates images"
|
||||
);
|
||||
|
||||
// The alias is the id most clients will actually send, so it has to
|
||||
// carry the capability too — advertising it only on the concrete id
|
||||
// would leave the public-facing name looking like a text model.
|
||||
let alias = body["data"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|m| m["id"] == "helexa/image")
|
||||
.expect("the tier alias should be listed");
|
||||
assert_eq!(
|
||||
alias["capabilities"],
|
||||
serde_json::json!(["image"]),
|
||||
"the alias must advertise the same capabilities as its target"
|
||||
);
|
||||
}
|
||||
|
||||
/// A node that has never cached the weights answers 404. That must leave
|
||||
/// the id unknown rather than recording an empty list, or the first node
|
||||
/// polled would permanently pin a model to no capabilities — the exact
|
||||
/// failure this work removes, reintroduced from a different direction.
|
||||
#[tokio::test]
|
||||
async fn node_without_local_evidence_records_nothing() {
|
||||
let neuron = common::spawn_mock_neuron_with_capabilities(serde_json::json!([]), &[]).await;
|
||||
let fleet = fleet_for(neuron);
|
||||
|
||||
cortex_gateway::poller::poll_once(&fleet).await;
|
||||
|
||||
assert!(
|
||||
fleet.discovered_capabilities.read().await.is_empty(),
|
||||
"a 404 means 'no local evidence', which must not be cached as an answer"
|
||||
);
|
||||
}
|
||||
|
||||
/// Weights are not spread evenly across a fleet, so the node polled
|
||||
/// first routinely lacks a model that a later one has. The back-off that
|
||||
/// stops unanswerable ids being re-probed every cycle must not let that
|
||||
/// first node's 404 suppress the question to everyone else.
|
||||
///
|
||||
/// Found live: a GGUF that two of three neurons could describe reported
|
||||
/// no capabilities at all, because the one node without it was polled
|
||||
/// first and claimed the attempt for the whole window.
|
||||
#[tokio::test]
|
||||
async fn a_node_that_cannot_answer_does_not_starve_one_that_can() {
|
||||
let empty_handed =
|
||||
common::spawn_mock_neuron_with_capabilities(serde_json::json!([]), &[]).await;
|
||||
let has_weights = common::spawn_mock_neuron_with_capabilities(
|
||||
serde_json::json!([]),
|
||||
&[("Tongyi-MAI/Z-Image-Turbo", vec!["image"])],
|
||||
)
|
||||
.await;
|
||||
// Config order is poll order, so the node that 404s goes first.
|
||||
let fleet = fleet_of(vec![("first", empty_handed), ("second", has_weights)]);
|
||||
|
||||
cortex_gateway::poller::poll_once(&fleet).await;
|
||||
|
||||
assert_eq!(
|
||||
fleet
|
||||
.discovered_capabilities
|
||||
.read()
|
||||
.await
|
||||
.get("Tongyi-MAI/Z-Image-Turbo")
|
||||
.cloned(),
|
||||
Some(vec!["image".to_string()]),
|
||||
"the second node's answer must survive the first node's 404, \
|
||||
in the same poll cycle"
|
||||
);
|
||||
}
|
||||
|
||||
/// An empty list is likewise not an answer worth keeping: a repo cached
|
||||
/// without weights says nothing about what other nodes can serve.
|
||||
#[tokio::test]
|
||||
async fn empty_capability_list_is_not_recorded() {
|
||||
let neuron = common::spawn_mock_neuron_with_capabilities(
|
||||
serde_json::json!([]),
|
||||
&[("Tongyi-MAI/Z-Image-Turbo", vec![])],
|
||||
)
|
||||
.await;
|
||||
let fleet = fleet_for(neuron);
|
||||
|
||||
cortex_gateway::poller::poll_once(&fleet).await;
|
||||
|
||||
assert!(
|
||||
fleet.discovered_capabilities.read().await.is_empty(),
|
||||
"an empty list must not be cached as this model's capabilities"
|
||||
);
|
||||
}
|
||||
@@ -3,7 +3,7 @@
|
||||
use axum::body::Body;
|
||||
use axum::extract::Path;
|
||||
use axum::http::header;
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum::response::Response;
|
||||
use axum::routing::{get, post};
|
||||
use axum::{Json, Router};
|
||||
use cortex_core::config::{
|
||||
@@ -22,7 +22,6 @@ use tokio::net::TcpListener;
|
||||
/// - GET /models/:id/endpoint (returns the inference URL)
|
||||
/// - POST /models/unload (accepts unload requests)
|
||||
/// - GET /v1/chat/completions + POST /v1/chat/completions (inference)
|
||||
///
|
||||
/// Returns the neuron base URL.
|
||||
pub async fn spawn_mock_neuron() -> String {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
@@ -44,7 +43,6 @@ pub async fn spawn_mock_neuron() -> String {
|
||||
post(|Json(_body): Json<Value>| async { Json(json!({"status": "unloaded"})) }),
|
||||
)
|
||||
.route("/v1/chat/completions", post(mock_chat_completions))
|
||||
.route("/v1/responses", post(mock_responses))
|
||||
.route("/v1/models", get(mock_v1_models));
|
||||
|
||||
tokio::spawn(async move {
|
||||
@@ -54,64 +52,9 @@ pub async fn spawn_mock_neuron() -> String {
|
||||
base_url
|
||||
}
|
||||
|
||||
/// Like [`spawn_mock_neuron`] but captures the JSON body of every
|
||||
/// `POST /v1/chat/completions` it receives into the returned handle, so
|
||||
/// a test can assert what the gateway *actually forwarded upstream*
|
||||
/// (e.g. that Anthropic-shaped tools were reshaped to OpenAI form).
|
||||
pub async fn spawn_capturing_mock_neuron() -> (String, Arc<std::sync::Mutex<Vec<Value>>>) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let base_url = format!("http://{addr}");
|
||||
let inference_url = base_url.clone();
|
||||
let captured: Arc<std::sync::Mutex<Vec<Value>>> = Arc::new(std::sync::Mutex::new(Vec::new()));
|
||||
let sink = captured.clone();
|
||||
|
||||
let app = Router::new()
|
||||
.route("/models", get(mock_neuron_list_models))
|
||||
.route(
|
||||
"/models/{model_id}/endpoint",
|
||||
get(move |Path(_): Path<String>| {
|
||||
let url = inference_url.clone();
|
||||
async move { Json(json!({"url": url})) }
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/v1/chat/completions",
|
||||
post(move |Json(body): Json<Value>| {
|
||||
let sink = sink.clone();
|
||||
async move {
|
||||
let model = body
|
||||
.get("model")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("unknown");
|
||||
let resp = json!({
|
||||
"id": "chatcmpl-capture-001",
|
||||
"object": "chat.completion",
|
||||
"created": 1700000000_u64,
|
||||
"model": model,
|
||||
"choices": [{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "Hello from mock backend"},
|
||||
"finish_reason": "stop"
|
||||
}],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}
|
||||
});
|
||||
sink.lock().unwrap().push(body);
|
||||
Json(resp)
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
|
||||
(base_url, captured)
|
||||
}
|
||||
|
||||
async fn mock_neuron_list_models() -> Json<Value> {
|
||||
Json(json!([
|
||||
{"id": "test-model", "harness": "candle", "status": "loaded", "devices": [0], "vram_used_mb": 8000, "capabilities": ["text"], "tool_call": false, "reasoning": false}
|
||||
{"id": "test-model", "harness": "mistralrs", "status": "loaded", "devices": [0], "vram_used_mb": 8000}
|
||||
]))
|
||||
}
|
||||
|
||||
@@ -149,39 +92,6 @@ async fn mock_chat_completions(Json(body): Json<Value>) -> Json<Value> {
|
||||
}))
|
||||
}
|
||||
|
||||
async fn mock_responses(Json(body): Json<Value>) -> Json<Value> {
|
||||
let model = body
|
||||
.get("model")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("unknown");
|
||||
// Echo the model field back and synthesise a tiny ResponsesResponse.
|
||||
// Mirrors the shape neuron's /v1/responses handler emits so the
|
||||
// gateway test only needs to assert the proxy round-tripped it.
|
||||
Json(json!({
|
||||
"id": "resp-test-001",
|
||||
"object": "response",
|
||||
"created_at": 1700000000_u64,
|
||||
"status": "completed",
|
||||
"model": model,
|
||||
"output": [{
|
||||
"type": "message",
|
||||
"id": "msg-test-001",
|
||||
"role": "assistant",
|
||||
"content": [{
|
||||
"type": "output_text",
|
||||
"text": "Hello from mock backend",
|
||||
"annotations": []
|
||||
}],
|
||||
"status": "completed"
|
||||
}],
|
||||
"usage": {
|
||||
"input_tokens": 5,
|
||||
"output_tokens": 5,
|
||||
"total_tokens": 10
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
/// Spawns a mock neuron that returns SSE streaming responses for chat completions.
|
||||
pub async fn spawn_streaming_mock_neuron(chunk_count: usize, chunk_delay: Duration) -> String {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
@@ -251,120 +161,8 @@ pub async fn spawn_streaming_mock_neuron(chunk_count: usize, chunk_delay: Durati
|
||||
base_url
|
||||
}
|
||||
|
||||
/// Like `spawn_streaming_mock_neuron`, but the stream ends with an
|
||||
/// OpenAI `stream_options.include_usage`-style final chunk (empty
|
||||
/// choices + usage object) before `[DONE]` — the shape the gateway's
|
||||
/// token metrics (#21) extract counts from.
|
||||
pub async fn spawn_streaming_mock_neuron_with_usage(
|
||||
chunk_count: usize,
|
||||
chunk_delay: Duration,
|
||||
prompt_tokens: u64,
|
||||
completion_tokens: u64,
|
||||
) -> String {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let base_url = format!("http://{addr}");
|
||||
let inference_url = base_url.clone();
|
||||
|
||||
let app = Router::new()
|
||||
.route("/models", get(mock_neuron_list_models))
|
||||
.route(
|
||||
"/models/{model_id}/endpoint",
|
||||
get(move |Path(_model_id): Path<String>| {
|
||||
let url = inference_url.clone();
|
||||
async move { Json(json!({"url": url})) }
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/v1/chat/completions",
|
||||
post(move |Json(body): Json<Value>| async move {
|
||||
let model = body
|
||||
.get("model")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("unknown")
|
||||
.to_string();
|
||||
|
||||
let mut chunks: Vec<String> = (0..chunk_count)
|
||||
.map(|i| {
|
||||
let chunk = json!({
|
||||
"id": "chatcmpl-stream-002",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1700000000_u64,
|
||||
"model": model,
|
||||
"choices": [{
|
||||
"index": 0,
|
||||
"delta": { "content": format!("token{i}") },
|
||||
"finish_reason": null
|
||||
}]
|
||||
});
|
||||
format!("data: {chunk}\n\n")
|
||||
})
|
||||
.collect();
|
||||
let usage_chunk = json!({
|
||||
"id": "chatcmpl-stream-002",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1700000000_u64,
|
||||
"model": model,
|
||||
"choices": [],
|
||||
"usage": {
|
||||
"prompt_tokens": prompt_tokens,
|
||||
"completion_tokens": completion_tokens,
|
||||
"total_tokens": prompt_tokens + completion_tokens
|
||||
}
|
||||
});
|
||||
chunks.push(format!("data: {usage_chunk}\n\n"));
|
||||
chunks.push("data: [DONE]\n\n".to_string());
|
||||
|
||||
let delay = chunk_delay;
|
||||
let stream = stream::iter(chunks).then(move |chunk| async move {
|
||||
tokio::time::sleep(delay).await;
|
||||
Ok::<_, std::convert::Infallible>(chunk)
|
||||
});
|
||||
|
||||
Response::builder()
|
||||
.header(header::CONTENT_TYPE, "text/event-stream")
|
||||
.header(header::CACHE_CONTROL, "no-cache")
|
||||
.body(Body::from_stream(stream))
|
||||
.unwrap()
|
||||
}),
|
||||
);
|
||||
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
|
||||
base_url
|
||||
}
|
||||
|
||||
/// Spawns a mock neuron with a custom models list.
|
||||
pub async fn spawn_mock_neuron_with_models(models_response: Value) -> String {
|
||||
spawn_mock_neuron_with_models_and_health(models_response, default_health_response()).await
|
||||
}
|
||||
|
||||
/// Default `/health` response used by mocks that don't care about the
|
||||
/// activation field — empty devices, no in-flight pre-warm, state=ready.
|
||||
pub fn default_health_response() -> Value {
|
||||
json!({
|
||||
"uptime_secs": 0,
|
||||
"devices": [],
|
||||
"activation": {
|
||||
"state": "ready",
|
||||
"pending": [],
|
||||
"in_progress": null,
|
||||
"completed": [],
|
||||
"failed": []
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Variant of `spawn_mock_neuron_with_models` that also serves a
|
||||
/// `/health` body. Used by tests that drive the gateway's activation
|
||||
/// surface (poller reading /health, /v1/models synthesising Loading
|
||||
/// locations from in_progress / pending).
|
||||
pub async fn spawn_mock_neuron_with_models_and_health(
|
||||
models_response: Value,
|
||||
health_response: Value,
|
||||
) -> String {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let base_url = format!("http://{addr}");
|
||||
@@ -378,13 +176,6 @@ pub async fn spawn_mock_neuron_with_models_and_health(
|
||||
async move { Json(resp) }
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/health",
|
||||
get(move || {
|
||||
let resp = health_response.clone();
|
||||
async move { Json(resp) }
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/models/{model_id}/endpoint",
|
||||
get(move |Path(_model_id): Path<String>| {
|
||||
@@ -405,73 +196,6 @@ pub async fn spawn_mock_neuron_with_models_and_health(
|
||||
base_url
|
||||
}
|
||||
|
||||
/// Variant that also serves `GET /models/{id}/capabilities` (#241), the
|
||||
/// route a neuron answers from its local model cache for models nothing
|
||||
/// has loaded.
|
||||
///
|
||||
/// `capabilities` maps model id → modalities. Any id not listed gets a
|
||||
/// 404, which is how a real neuron says "these weights were never cached
|
||||
/// here" — distinct from an empty list, and the distinction is load-
|
||||
/// bearing, so the mock has to be able to express both.
|
||||
#[allow(dead_code)]
|
||||
pub async fn spawn_mock_neuron_with_capabilities(
|
||||
models_response: Value,
|
||||
capabilities: &[(&str, Vec<&str>)],
|
||||
) -> String {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let base_url = format!("http://{addr}");
|
||||
let health_response = default_health_response();
|
||||
let table: std::collections::HashMap<String, Vec<String>> = capabilities
|
||||
.iter()
|
||||
.map(|(id, caps)| {
|
||||
(
|
||||
(*id).to_string(),
|
||||
caps.iter().map(|c| (*c).to_string()).collect(),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
let table = Arc::new(table);
|
||||
|
||||
let app = Router::new()
|
||||
.route(
|
||||
"/models",
|
||||
get(move || {
|
||||
let resp = models_response.clone();
|
||||
async move { Json(resp) }
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/health",
|
||||
get(move || {
|
||||
let resp = health_response.clone();
|
||||
async move { Json(resp) }
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/models/{model_id}/capabilities",
|
||||
get(move |Path(model_id): Path<String>| {
|
||||
let table = table.clone();
|
||||
async move {
|
||||
match table.get(&model_id) {
|
||||
Some(caps) => Json(json!({ "capabilities": caps })).into_response(),
|
||||
None => (
|
||||
axum::http::StatusCode::NOT_FOUND,
|
||||
Json(json!({"error": "no local evidence"})),
|
||||
)
|
||||
.into_response(),
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
|
||||
base_url
|
||||
}
|
||||
|
||||
/// Spawns the cortex gateway with a single neuron pointing at `mock_url`.
|
||||
/// The node is pre-seeded as healthy with one loaded model ("test-model").
|
||||
/// Returns the gateway's base URL.
|
||||
@@ -496,8 +220,6 @@ pub async fn spawn_gateway_with_state(mock_url: &str) -> (Arc<CortexState>, Stri
|
||||
endpoint: mock_url.to_string(),
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
@@ -514,12 +236,6 @@ pub async fn spawn_gateway_with_state(mock_url: &str) -> (Arc<CortexState>, Stri
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: None,
|
||||
vram_estimate_mb: Some(8000),
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -534,31 +250,3 @@ pub async fn spawn_gateway_with_state(mock_url: &str) -> (Arc<CortexState>, Stri
|
||||
|
||||
(fleet, format!("http://{addr}"))
|
||||
}
|
||||
|
||||
/// Build a gateway state over two mock neurons (no poller; we seed state).
|
||||
pub async fn two_neuron_fleet(endpoint_a: &str, endpoint_b: &str) -> Arc<CortexState> {
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![
|
||||
NeuronEndpoint {
|
||||
name: "node-a".into(),
|
||||
endpoint: endpoint_a.to_string(),
|
||||
},
|
||||
NeuronEndpoint {
|
||||
name: "node-b".into(),
|
||||
endpoint: endpoint_b.to_string(),
|
||||
},
|
||||
],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
Arc::new(CortexState::from_config(&config))
|
||||
}
|
||||
|
||||
@@ -1,141 +0,0 @@
|
||||
mod common;
|
||||
|
||||
use serde_json::json;
|
||||
|
||||
#[tokio::test]
|
||||
async fn error_response_model_not_found() {
|
||||
let neuron_url = common::spawn_mock_neuron().await;
|
||||
let gateway_url = common::spawn_gateway(&neuron_url).await;
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
// Request a model that isn't loaded on the mock neuron.
|
||||
let resp = client
|
||||
.post(format!("{gateway_url}/v1/chat/completions"))
|
||||
.header("Content-Type", "application/json")
|
||||
.json(&json!({
|
||||
"model": "nonexistent-model",
|
||||
"messages": [{"role": "user", "content": "hi"}]
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.expect("request should succeed");
|
||||
|
||||
assert_eq!(resp.status(), axum::http::StatusCode::NOT_FOUND);
|
||||
|
||||
let body: serde_json::Value = resp.json().await.expect("valid json");
|
||||
let err = body.get("error").expect("response has error object");
|
||||
|
||||
// Broad type categorization
|
||||
assert_eq!(err.get("type").unwrap(), "invalid_request_error");
|
||||
// Specific machine-readable code
|
||||
assert_eq!(
|
||||
err.get("code").unwrap().as_str().unwrap(),
|
||||
"model_not_found"
|
||||
);
|
||||
// param is always null
|
||||
assert!(err.get("param").unwrap().is_null());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn error_response_missing_model_field() {
|
||||
let neuron_url = common::spawn_mock_neuron().await;
|
||||
let gateway_url = common::spawn_gateway(&neuron_url).await;
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
// Request without the required `model` field.
|
||||
let resp = client
|
||||
.post(format!("{gateway_url}/v1/chat/completions"))
|
||||
.header("Content-Type", "application/json")
|
||||
.json(&json!({
|
||||
"messages": [{"role": "user", "content": "hi"}]
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.expect("request should succeed");
|
||||
|
||||
assert_eq!(resp.status(), axum::http::StatusCode::BAD_REQUEST);
|
||||
|
||||
let body: serde_json::Value = resp.json().await.expect("valid json");
|
||||
let err = body.get("error").expect("response has error object");
|
||||
|
||||
assert_eq!(err.get("type").unwrap(), "invalid_request_error");
|
||||
assert_eq!(
|
||||
err.get("code").unwrap().as_str().unwrap(),
|
||||
"missing_model_field"
|
||||
);
|
||||
assert!(err.get("param").unwrap().is_null());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn error_response_no_healthy_nodes() {
|
||||
use cortex_core::config::{EvictionSettings, GatewayConfig, GatewaySettings, NeuronEndpoint};
|
||||
use std::sync::Arc;
|
||||
|
||||
// Create a gateway config with a neuron pointing at an unreachable port so no node is ever healthy.
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: cortex_core::config::EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![NeuronEndpoint {
|
||||
name: "dead-node".into(),
|
||||
endpoint: "http://127.0.0.1:1".into(),
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(cortex_gateway::state::CortexState::from_config(&config));
|
||||
|
||||
let app = cortex_gateway::build_app(fleet);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
|
||||
// Allow the poller a moment to mark the node unhealthy.
|
||||
tokio::time::sleep(std::time::Duration::from_millis(200)).await;
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let resp = client
|
||||
.post(format!("http://{addr}/v1/chat/completions"))
|
||||
.header("Content-Type", "application/json")
|
||||
.json(&json!({
|
||||
"model": "any-model",
|
||||
"messages": [{"role": "user", "content": "hi"}]
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.expect("request should succeed");
|
||||
|
||||
assert_eq!(resp.status(), axum::http::StatusCode::SERVICE_UNAVAILABLE);
|
||||
|
||||
// Transient 503 — the gateway advertises Retry-After so OpenAI-compatible
|
||||
// clients back off and retry rather than surfacing an opaque error (#63).
|
||||
let retry_after = resp
|
||||
.headers()
|
||||
.get(reqwest::header::RETRY_AFTER)
|
||||
.expect("transient 503 must carry Retry-After")
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_string();
|
||||
assert_eq!(retry_after, "5");
|
||||
|
||||
let body: serde_json::Value = resp.json().await.expect("valid json");
|
||||
let err = body.get("error").expect("response has error object");
|
||||
|
||||
assert_eq!(err.get("type").unwrap(), "api_error");
|
||||
assert_eq!(
|
||||
err.get("code").unwrap().as_str().unwrap(),
|
||||
"service_unavailable"
|
||||
);
|
||||
assert!(err.get("param").unwrap().is_null());
|
||||
}
|
||||
@@ -56,77 +56,6 @@ async fn spawn_eviction_mock() -> (String, Arc<tokio::sync::Mutex<Vec<String>>>)
|
||||
(base_url, unloaded)
|
||||
}
|
||||
|
||||
/// A fleet whose catalogue ranks models, so the displacement rules are
|
||||
/// exercised rather than defaulted. Writes the catalogue to a temp file
|
||||
/// because `CortexState` loads it from a path.
|
||||
fn make_fleet_with_catalogue(
|
||||
endpoint: &str,
|
||||
catalogue_toml: &str,
|
||||
tag: &str,
|
||||
) -> (Arc<CortexState>, std::path::PathBuf) {
|
||||
let path = std::env::temp_dir().join(format!("cortex-evict-catalogue-{tag}.toml"));
|
||||
std::fs::write(&path, catalogue_toml).expect("write test catalogue");
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![NeuronEndpoint {
|
||||
name: "gpu-node".into(),
|
||||
endpoint: endpoint.to_string(),
|
||||
}],
|
||||
models_config: path.to_string_lossy().into_owned(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
(Arc::new(CortexState::from_config(&config)), path)
|
||||
}
|
||||
|
||||
fn loaded(id: &str, age_secs: i64) -> ModelEntry {
|
||||
ModelEntry {
|
||||
id: id.into(),
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: Some(Utc::now() - chrono::Duration::seconds(age_secs)),
|
||||
vram_estimate_mb: Some(8000),
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The fleet policy under test. Two residency classes: image generation
|
||||
/// and the mid tier share a node and take turns on it; the flagship and
|
||||
/// the frontier model share a bigger one and take turns on that. Nothing
|
||||
/// in the everyday class may touch the big-node class.
|
||||
const TIERED: &str = r#"
|
||||
[[models]]
|
||||
id = "flagship"
|
||||
harness = "candle"
|
||||
residency_priority = 300
|
||||
|
||||
[[models]]
|
||||
id = "frontier"
|
||||
harness = "candle"
|
||||
residency_priority = 300
|
||||
|
||||
[[models]]
|
||||
id = "image"
|
||||
harness = "candle"
|
||||
residency_priority = 200
|
||||
|
||||
[[models]]
|
||||
id = "mid"
|
||||
harness = "candle"
|
||||
residency_priority = 200
|
||||
"#;
|
||||
|
||||
fn make_fleet(endpoint: &str, defrag_after: u32) -> Arc<CortexState> {
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
@@ -142,8 +71,6 @@ fn make_fleet(endpoint: &str, defrag_after: u32) -> Arc<CortexState> {
|
||||
endpoint: endpoint.to_string(),
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
Arc::new(CortexState::from_config(&config))
|
||||
}
|
||||
@@ -164,12 +91,6 @@ async fn test_evict_lru_model() {
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: Some(Utc::now() - chrono::Duration::hours(2)),
|
||||
vram_estimate_mb: Some(8000),
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
node.models.insert(
|
||||
@@ -179,17 +100,11 @@ async fn test_evict_lru_model() {
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: Some(Utc::now()),
|
||||
vram_estimate_mb: Some(8000),
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let evicted = cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node", None)
|
||||
let evicted = cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node")
|
||||
.await
|
||||
.expect("eviction should succeed");
|
||||
|
||||
@@ -222,7 +137,7 @@ async fn test_eviction_nothing_to_evict() {
|
||||
nodes.get_mut("gpu-node").unwrap().healthy = true;
|
||||
}
|
||||
|
||||
let evicted = cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node", None)
|
||||
let evicted = cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node")
|
||||
.await
|
||||
.expect("eviction should succeed");
|
||||
|
||||
@@ -248,17 +163,11 @@ async fn test_eviction_increments_lifecycle_cycles() {
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: None,
|
||||
vram_estimate_mb: None,
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node", None)
|
||||
cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node")
|
||||
.await
|
||||
.expect("eviction should succeed");
|
||||
|
||||
@@ -305,129 +214,3 @@ async fn test_last_accessed_updated_on_request() {
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
|
||||
/// Image generation must take the mid tier's node when it needs it —
|
||||
/// this is existing fleet behaviour and the change must preserve it.
|
||||
#[tokio::test]
|
||||
async fn image_generation_evicts_the_mid_tier() {
|
||||
let (mock_url, unloaded) = spawn_eviction_mock().await;
|
||||
let (fleet, path) = make_fleet_with_catalogue(&mock_url, TIERED, "image-takes-mid");
|
||||
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("gpu-node").unwrap();
|
||||
node.healthy = true;
|
||||
node.models.insert("mid".into(), loaded("mid", 60));
|
||||
}
|
||||
|
||||
let evicted = cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node", Some("image"))
|
||||
.await
|
||||
.expect("eviction should succeed");
|
||||
|
||||
assert_eq!(evicted, Some("mid".to_string()));
|
||||
assert_eq!(unloaded.lock().await.as_slice(), ["mid"]);
|
||||
std::fs::remove_file(path).ok();
|
||||
}
|
||||
|
||||
/// Image generation must never take the flagship's node. Its device
|
||||
/// constraints alone would let it land there, so nothing but priority
|
||||
/// stops this.
|
||||
#[tokio::test]
|
||||
async fn image_generation_cannot_evict_the_flagship() {
|
||||
let (mock_url, unloaded) = spawn_eviction_mock().await;
|
||||
let (fleet, path) = make_fleet_with_catalogue(&mock_url, TIERED, "image-spares-flagship");
|
||||
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("gpu-node").unwrap();
|
||||
node.healthy = true;
|
||||
node.models
|
||||
.insert("flagship".into(), loaded("flagship", 9999));
|
||||
}
|
||||
|
||||
let evicted = cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node", Some("image"))
|
||||
.await
|
||||
.expect("eviction should succeed");
|
||||
|
||||
assert_eq!(
|
||||
evicted, None,
|
||||
"the flagship outranks image generation, however stale it is"
|
||||
);
|
||||
assert!(unloaded.lock().await.is_empty());
|
||||
std::fs::remove_file(path).ok();
|
||||
}
|
||||
|
||||
/// The frontier tier may cold-swap the flagship off its node.
|
||||
#[tokio::test]
|
||||
async fn the_frontier_tier_evicts_the_flagship() {
|
||||
let (mock_url, unloaded) = spawn_eviction_mock().await;
|
||||
let (fleet, path) = make_fleet_with_catalogue(&mock_url, TIERED, "frontier-takes-flagship");
|
||||
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("gpu-node").unwrap();
|
||||
node.healthy = true;
|
||||
node.models
|
||||
.insert("flagship".into(), loaded("flagship", 10));
|
||||
}
|
||||
|
||||
let evicted = cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node", Some("frontier"))
|
||||
.await
|
||||
.expect("eviction should succeed");
|
||||
|
||||
assert_eq!(evicted, Some("flagship".to_string()));
|
||||
assert_eq!(unloaded.lock().await.as_slice(), ["flagship"]);
|
||||
std::fs::remove_file(path).ok();
|
||||
}
|
||||
|
||||
/// LRU still decides *which* victim, but only among the models the
|
||||
/// incoming one outranks. Here the flagship is by far the stalest, so a
|
||||
/// purely age-ordered evictor would take it.
|
||||
#[tokio::test]
|
||||
async fn lru_picks_the_oldest_displaceable_model_not_the_oldest_model() {
|
||||
let (mock_url, unloaded) = spawn_eviction_mock().await;
|
||||
let (fleet, path) = make_fleet_with_catalogue(&mock_url, TIERED, "lru-within-rank");
|
||||
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("gpu-node").unwrap();
|
||||
node.healthy = true;
|
||||
node.models
|
||||
.insert("flagship".into(), loaded("flagship", 9999));
|
||||
node.models.insert("mid".into(), loaded("mid", 60));
|
||||
}
|
||||
|
||||
let evicted = cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node", Some("image"))
|
||||
.await
|
||||
.expect("eviction should succeed");
|
||||
|
||||
assert_eq!(evicted, Some("mid".to_string()));
|
||||
assert_eq!(unloaded.lock().await.as_slice(), ["mid"]);
|
||||
std::fs::remove_file(path).ok();
|
||||
}
|
||||
|
||||
/// The other half of the cold-swap: after an image generation has taken
|
||||
/// the node, the next text request must be able to take it back. A
|
||||
/// strict priority order would let whichever model arrived first hold
|
||||
/// the node forever, which looks to a user like the text tier vanishing
|
||||
/// once somebody generated an image.
|
||||
#[tokio::test]
|
||||
async fn the_mid_tier_takes_its_node_back_from_image_generation() {
|
||||
let (mock_url, unloaded) = spawn_eviction_mock().await;
|
||||
let (fleet, path) = make_fleet_with_catalogue(&mock_url, TIERED, "mid-swaps-back");
|
||||
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("gpu-node").unwrap();
|
||||
node.healthy = true;
|
||||
node.models.insert("image".into(), loaded("image", 30));
|
||||
}
|
||||
|
||||
let evicted = cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node", Some("mid"))
|
||||
.await
|
||||
.expect("eviction should succeed");
|
||||
|
||||
assert_eq!(evicted, Some("image".to_string()));
|
||||
assert_eq!(unloaded.lock().await.as_slice(), ["image"]);
|
||||
std::fs::remove_file(path).ok();
|
||||
}
|
||||
|
||||
@@ -1,138 +0,0 @@
|
||||
//! Router: a catalogued model whose only topologically-feasible neuron is
|
||||
//! currently unhealthy is a *transient* condition (retryable 503), not a
|
||||
//! permanent 404. This is the exact shape of the beast incident: benjy/
|
||||
//! quadbrat (1 GPU, healthy) can't host the 27B, and beast (2 GPU) — the
|
||||
//! sole feasible node — briefly drops out → clients must back off and retry,
|
||||
//! not hard-fail.
|
||||
|
||||
use cortex_core::config::{
|
||||
EvictionSettings, EvictionStrategy, GatewayConfig, GatewaySettings, NeuronEndpoint,
|
||||
};
|
||||
use cortex_core::discovery::{DeviceInfo, DiscoveryResponse};
|
||||
use cortex_gateway::router::{self, RouteError};
|
||||
use cortex_gateway::state::CortexState;
|
||||
use std::sync::Arc;
|
||||
|
||||
fn devices(n: usize) -> Vec<DeviceInfo> {
|
||||
(0..n)
|
||||
.map(|i| DeviceInfo {
|
||||
index: i as u32,
|
||||
name: "RTX 5090".into(),
|
||||
vram_total_mb: 32_768,
|
||||
compute_capability: "9.0".into(),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn discovery(host: &str, n_devices: usize) -> DiscoveryResponse {
|
||||
DiscoveryResponse {
|
||||
hostname: host.into(),
|
||||
os: "Linux".into(),
|
||||
kernel: "7.0".into(),
|
||||
cuda_version: Some("13.0".into()),
|
||||
driver_version: Some("999".into()),
|
||||
devices: devices(n_devices),
|
||||
harnesses: vec!["candle".into()],
|
||||
cuda_unavailable_reason: None,
|
||||
max_prompt_tokens: 49_152,
|
||||
}
|
||||
}
|
||||
|
||||
/// Catalogue with one model needing 2 devices. Returns a temp path.
|
||||
///
|
||||
/// The filename is unique per call: tests in this binary run concurrently,
|
||||
/// and a shared path let one test truncate the file (`fs::write` truncates
|
||||
/// before writing) while another was loading it — yielding an empty
|
||||
/// catalogue and a spurious `ModelNotFound` instead of the routing error
|
||||
/// under test.
|
||||
fn write_catalogue() -> std::path::PathBuf {
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
static SEQ: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
let toml = r#"
|
||||
[[models]]
|
||||
id = "big-model"
|
||||
harness = "candle"
|
||||
min_devices = 2
|
||||
"#;
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"cortex_test_feasibility_models.{}.{}.toml",
|
||||
std::process::id(),
|
||||
SEQ.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
std::fs::write(&path, toml).unwrap();
|
||||
path
|
||||
}
|
||||
|
||||
async fn fleet_with(big_healthy: bool, big_devices: usize) -> Arc<CortexState> {
|
||||
let cat = write_catalogue();
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![
|
||||
NeuronEndpoint {
|
||||
name: "small".into(),
|
||||
endpoint: "http://127.0.0.1:1".into(),
|
||||
},
|
||||
NeuronEndpoint {
|
||||
name: "big".into(),
|
||||
endpoint: "http://127.0.0.1:2".into(),
|
||||
},
|
||||
],
|
||||
models_config: cat.to_string_lossy().into_owned(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
// "small" is healthy but only has 1 GPU → not feasible for the model.
|
||||
let small = nodes.get_mut("small").unwrap();
|
||||
small.healthy = true;
|
||||
small.discovery = Some(discovery("small", 1));
|
||||
// "big" has enough GPUs but its health is the variable under test.
|
||||
let big = nodes.get_mut("big").unwrap();
|
||||
big.healthy = big_healthy;
|
||||
big.discovery = Some(discovery("big", big_devices));
|
||||
}
|
||||
fleet
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn feasible_node_unhealthy_is_transient_503() {
|
||||
// big (2 GPU, the only feasible node) is unhealthy; small (1 GPU) is
|
||||
// healthy but can't host the model → retryable, not a permanent 404.
|
||||
let fleet = fleet_with(false, 2).await;
|
||||
let err = router::resolve(&fleet, "big-model")
|
||||
.await
|
||||
.expect_err("model can't be served right now");
|
||||
assert!(
|
||||
matches!(err, RouteError::FeasibleNodeUnhealthy { .. }),
|
||||
"expected FeasibleNodeUnhealthy, got {err:?}"
|
||||
);
|
||||
assert_eq!(err.http_status(), 503);
|
||||
assert_eq!(err.retry_after_secs(), Some(3));
|
||||
assert_eq!(err.code(), "service_unavailable");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn no_node_can_ever_satisfy_is_permanent_404() {
|
||||
// big is healthy but only has 1 GPU now (e.g. topology genuinely can't
|
||||
// satisfy min_devices=2 anywhere) → permanent, non-retryable 404.
|
||||
let fleet = fleet_with(true, 1).await;
|
||||
let err = router::resolve(&fleet, "big-model")
|
||||
.await
|
||||
.expect_err("no feasible topology");
|
||||
assert!(
|
||||
matches!(err, RouteError::NoFeasibleNeuron { .. }),
|
||||
"expected NoFeasibleNeuron, got {err:?}"
|
||||
);
|
||||
assert_eq!(err.http_status(), 404);
|
||||
assert_eq!(err.retry_after_secs(), None);
|
||||
}
|
||||
@@ -1,267 +0,0 @@
|
||||
//! `/v1/images/generations` proxy tests (#201).
|
||||
//!
|
||||
//! The mock neuron answers the images endpoint with a tiny valid
|
||||
//! envelope; the gateway must route by model, forward verbatim, and
|
||||
//! pass the response (including `usage.helexa_image_units`) through
|
||||
//! untouched.
|
||||
|
||||
mod common;
|
||||
|
||||
use axum::Router;
|
||||
use axum::extract::Path;
|
||||
use axum::response::Json;
|
||||
use axum::routing::{get, post};
|
||||
use common::spawn_gateway;
|
||||
use serde_json::{Value, json};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
/// Mock neuron that serves the images endpoint for "test-model".
|
||||
async fn spawn_images_mock_neuron() -> String {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let base_url = format!("http://{addr}");
|
||||
let inference_url = base_url.clone();
|
||||
|
||||
let app = Router::new()
|
||||
.route(
|
||||
"/models",
|
||||
get(|| async {
|
||||
Json(json!([
|
||||
{"id": "test-model", "harness": "candle", "status": "loaded",
|
||||
"devices": [0], "vram_used_mb": 14000,
|
||||
"capabilities": ["image"], "tool_call": false, "reasoning": false}
|
||||
]))
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/models/{model_id}/endpoint",
|
||||
get(move |Path(_): Path<String>| {
|
||||
let url = inference_url.clone();
|
||||
async move { Json(json!({"url": url})) }
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/v1/images/generations",
|
||||
post(|Json(body): Json<Value>| async move {
|
||||
// Echo enough of the request to prove verbatim forwarding.
|
||||
let seed = body.get("seed").cloned().unwrap_or(Value::Null);
|
||||
Json(json!({
|
||||
"created": 1700000000_u64,
|
||||
"data": [{"b64_json": "aGVsZXhh"}],
|
||||
"usage": {
|
||||
"helexa_image_units": 9.437184,
|
||||
"helexa_timing": {
|
||||
"encode_ms": 1673, "denoise_ms": 7894,
|
||||
"decode_ms": 1835, "steps": 9, "cfg": false
|
||||
}
|
||||
},
|
||||
"echo_seed": seed
|
||||
}))
|
||||
}),
|
||||
);
|
||||
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
base_url
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_images_proxy_round_trip() {
|
||||
let mock_url = spawn_images_mock_neuron().await;
|
||||
let gateway_url = spawn_gateway(&mock_url).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway_url}/v1/images/generations"))
|
||||
.json(&json!({
|
||||
"model": "test-model",
|
||||
"prompt": "a neon sign reading HELEXA",
|
||||
"size": "1024x1024",
|
||||
"seed": 42
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.expect("request should succeed");
|
||||
|
||||
assert_eq!(resp.status(), 200);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["data"][0]["b64_json"], "aGVsZXhh");
|
||||
// Metering unit passthrough (#202 reads this at the gateway).
|
||||
assert!(
|
||||
(body["usage"]["helexa_image_units"].as_f64().unwrap() - 9.437184).abs() < 1e-9,
|
||||
"helexa_image_units must pass through untouched"
|
||||
);
|
||||
assert_eq!(body["usage"]["helexa_timing"]["steps"], 9);
|
||||
// The request body reached the neuron verbatim.
|
||||
assert_eq!(body["echo_seed"], 42);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_images_missing_model_field() {
|
||||
let mock_url = spawn_images_mock_neuron().await;
|
||||
let gateway_url = spawn_gateway(&mock_url).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway_url}/v1/images/generations"))
|
||||
.json(&json!({"prompt": "no model here"}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), 400);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["error"]["code"], "missing_model_field");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_images_unknown_model_404() {
|
||||
let mock_url = spawn_images_mock_neuron().await;
|
||||
let gateway_url = spawn_gateway(&mock_url).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway_url}/v1/images/generations"))
|
||||
.json(&json!({"model": "no-such-model", "prompt": "x"}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), 404);
|
||||
}
|
||||
|
||||
// ── image budget enforcement (#202) ──────────────────────────────
|
||||
|
||||
use cortex_core::config::{
|
||||
ApiKeyConfig, EntitlementsConfig, EvictionSettings, EvictionStrategy, GatewayConfig,
|
||||
GatewaySettings, NeuronEndpoint,
|
||||
};
|
||||
use cortex_core::entitlements::CapWindow;
|
||||
use cortex_core::node::{ModelEntry, ModelStatus};
|
||||
use cortex_gateway::state::CortexState;
|
||||
use std::sync::Arc;
|
||||
|
||||
async fn spawn_keyed_gateway(neuron_url: &str, hard_cap: u64) -> String {
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![NeuronEndpoint {
|
||||
name: "mock-node".into(),
|
||||
endpoint: neuron_url.to_string(),
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: EntitlementsConfig {
|
||||
require_auth: true,
|
||||
keys: vec![ApiKeyConfig {
|
||||
key: "sk-img".into(),
|
||||
account_id: "acct-img".into(),
|
||||
key_id: Some("key-img".into()),
|
||||
hard_cap: Some(hard_cap),
|
||||
window: CapWindow::Balance,
|
||||
}],
|
||||
},
|
||||
upstream: Default::default(),
|
||||
};
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("mock-node").unwrap();
|
||||
node.healthy = true;
|
||||
node.models.insert(
|
||||
"test-model".into(),
|
||||
ModelEntry {
|
||||
id: "test-model".into(),
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: None,
|
||||
vram_estimate_mb: Some(14000),
|
||||
capabilities: vec!["image".into()],
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
reasoning_budget: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
let app = cortex_gateway::build_app(Arc::clone(&fleet));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
format!("http://{addr}")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_images_budget_rejected_before_dispatch() {
|
||||
let mock_url = spawn_images_mock_neuron().await;
|
||||
// A 1024²/9-step image reserves ~9.44 units ≈ 9438 tokens; a cap of
|
||||
// 100 tokens must fail-close before the neuron is touched.
|
||||
let gateway_url = spawn_keyed_gateway(&mock_url, 100).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway_url}/v1/images/generations"))
|
||||
.bearer_auth("sk-img")
|
||||
.json(&json!({"model": "test-model", "prompt": "a cat"}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::TOO_MANY_REQUESTS);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["error"]["code"], "insufficient_quota");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_images_within_budget_succeeds_and_settles() {
|
||||
let mock_url = spawn_images_mock_neuron().await;
|
||||
// Plenty of budget: reservation ~9438 tokens, settle at actual
|
||||
// 9.437184 units ≈ 9438 tokens.
|
||||
let gateway_url = spawn_keyed_gateway(&mock_url, 1_000_000).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway_url}/v1/images/generations"))
|
||||
.bearer_auth("sk-img")
|
||||
.json(&json!({"model": "test-model", "prompt": "a cat", "seed": 42}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), 200);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["data"][0]["b64_json"], "aGVsZXhh");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_images_cfg_doubles_reservation() {
|
||||
let mock_url = spawn_images_mock_neuron().await;
|
||||
// Cap sits between the plain (9.44u ≈ 9438t) and CFG (18.88u ≈
|
||||
// 18875t) reservations: plain passes, CFG fail-closes.
|
||||
let gateway_url = spawn_keyed_gateway(&mock_url, 12_000).await;
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let cfg_resp = client
|
||||
.post(format!("{gateway_url}/v1/images/generations"))
|
||||
.bearer_auth("sk-img")
|
||||
.json(&json!({
|
||||
"model": "test-model", "prompt": "a cat",
|
||||
"negative_prompt": "blurry"
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(cfg_resp.status(), reqwest::StatusCode::TOO_MANY_REQUESTS);
|
||||
|
||||
let plain_resp = client
|
||||
.post(format!("{gateway_url}/v1/images/generations"))
|
||||
.bearer_auth("sk-img")
|
||||
.json(&json!({"model": "test-model", "prompt": "a cat"}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(plain_resp.status(), 200);
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user