Compare commits
2 Commits
fix/stream
...
v0.1.15
| Author | SHA1 | Date | |
|---|---|---|---|
0184ccab28
|
|||
|
471b9b7629
|
@@ -1,984 +0,0 @@
|
||||
name: build-prerelease
|
||||
|
||||
# Builds CUDA-flavoured neuron binaries (and a single cortex binary),
|
||||
# packages each as a Fedora RPM, signs them, and publishes to the
|
||||
# `unstable` channel at rpm.lair.cafe.
|
||||
#
|
||||
# Change-aware: the `prepare` job diffs HEAD against the git sha
|
||||
# embedded in the most recently *published* unstable RPM (per package)
|
||||
# and skips builds whose inputs didn't change. Docs-only commits build
|
||||
# nothing; gateway-only commits skip the 3 CUDA builds (and, via
|
||||
# deploy.yml's own check-update gate, the neuron restarts + model
|
||||
# cold-loads). Diffing against the published sha — not the previous
|
||||
# push — means a failed run can never cause a change to be missed.
|
||||
#
|
||||
# Lint (fmt+clippy) and test run here as parallel jobs and gate
|
||||
# `publish`; ci.yml no longer runs on pushes to main (see its trigger
|
||||
# comment), so the two workflows stop competing for the same runners.
|
||||
#
|
||||
# The published packages are versioned as e.g.
|
||||
# helexa-neuron-blackwell-0.1.16-0.1.20260518T140530.gitabcdef0.fc43.x86_64
|
||||
# ^^^^^^^^^^^^^^^^^^ ^^^^^^^^
|
||||
# commit time (s) commit sha
|
||||
# so they sort BELOW the eventual 0.1.16-1 stable release, and so two
|
||||
# commits on the same day are still strictly ordered by their commit
|
||||
# timestamps (rather than by RPM-vercmp's alpha-vs-digit precedence
|
||||
# on the SHA fragment).
|
||||
|
||||
on:
|
||||
# Auto-build on every push to main so the unstable channel tracks
|
||||
# head without a manual dispatch step.
|
||||
push:
|
||||
branches: [main]
|
||||
# Manual dispatch still available to build from a non-main ref.
|
||||
# Dispatched runs skip change detection and build everything.
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
ref:
|
||||
description: "Git ref to build (branch / tag / commit). Defaults to the workflow's branch."
|
||||
required: false
|
||||
default: ""
|
||||
|
||||
# Coalesce same-ref pushes: a newer push cancels the older in-flight
|
||||
# run — the newest commit is the one we want on the fleet. The publish
|
||||
# job keeps its own `rpm-publish` group (cancel=false) so an in-flight
|
||||
# repo update is never interrupted. Runners are ephemeral (one VM per
|
||||
# job) so concurrent runs no longer race on a shared workspace; the
|
||||
# old shared `cortex-runner-pool` group with ci.yml is gone.
|
||||
concurrency:
|
||||
group: build-prerelease-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
CARGO_INCREMENTAL: "0"
|
||||
CARGO_TERM_COLOR: "always"
|
||||
|
||||
jobs:
|
||||
prepare:
|
||||
name: Resolve version stamps + change detection
|
||||
timeout-minutes: 10
|
||||
runs-on: rust
|
||||
outputs:
|
||||
version: ${{ steps.info.outputs.version }}
|
||||
release: ${{ steps.info.outputs.release }}
|
||||
short_sha: ${{ steps.info.outputs.short_sha }}
|
||||
commit_timestamp: ${{ steps.info.outputs.commit_timestamp }}
|
||||
build_cortex: ${{ steps.changes.outputs.build_cortex }}
|
||||
build_neuron: ${{ steps.changes.outputs.build_neuron }}
|
||||
build_bench: ${{ steps.changes.outputs.build_bench }}
|
||||
build_upstream: ${{ steps.changes.outputs.build_upstream }}
|
||||
build_router: ${{ steps.changes.outputs.build_router }}
|
||||
build_tools: ${{ steps.changes.outputs.build_tools }}
|
||||
build_angels: ${{ steps.changes.outputs.build_angels }}
|
||||
check_rust: ${{ steps.changes.outputs.check_rust }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
fetch-depth: 0
|
||||
|
||||
- id: info
|
||||
run: |
|
||||
set -eux
|
||||
VERSION=$(awk -F\" '/^version[[:space:]]*=/ { print $2; exit }' Cargo.toml)
|
||||
SHORT_SHA=$(git rev-parse --short=7 HEAD)
|
||||
# Second-precise commit timestamp gives the release stamp a
|
||||
# strictly monotonic numeric prefix. The earlier %Y%m%d-only
|
||||
# form let same-day builds be ordered by RPM's rpmvercmp
|
||||
# rules over the SHA, which is non-chronological — e.g.
|
||||
# "git602e8e1" sorts newer than "gitf9f5fa4" purely because
|
||||
# rpmvercmp ranks digit-prefixed segments above alpha ones.
|
||||
# The SHA stays only as a debug identifier; sort order is
|
||||
# decided entirely by the timestamp.
|
||||
# format-local + TZ=UTC0 renders the stamp in UTC regardless
|
||||
# of the committer's recorded timezone. Plain `format:` uses
|
||||
# each commit's own TZ offset — local commits (UTC+3) and
|
||||
# Gitea server-side merge commits (UTC) interleaved
|
||||
# non-monotonically, letting an older build out-rank newer
|
||||
# ones in RPM EVR comparison (the 2026-07-01 "235959" stamp
|
||||
# that froze the fleet on a stale build).
|
||||
COMMIT_TIMESTAMP=$(TZ=UTC0 git log -1 --format=%cd --date=format-local:%Y%m%d%H%M%S HEAD)
|
||||
RELEASE="0.1.${COMMIT_TIMESTAMP}.git${SHORT_SHA}"
|
||||
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
echo "release=${RELEASE}" >> "$GITHUB_OUTPUT"
|
||||
echo "short_sha=${SHORT_SHA}" >> "$GITHUB_OUTPUT"
|
||||
echo "commit_timestamp=${COMMIT_TIMESTAMP}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- id: changes
|
||||
env:
|
||||
# The push's own range. Empty on workflow_dispatch and all
|
||||
# zeroes for a new branch; `decide` treats both as unusable and
|
||||
# falls back to the full gate.
|
||||
PUSH_BEFORE: ${{ github.event.before }}
|
||||
run: |
|
||||
set -ux
|
||||
# Default: build everything. Detection only ever narrows
|
||||
# this, and any failure along the way (manifest unreachable,
|
||||
# unparsable, sha not in history after a force-push) leaves
|
||||
# the full build in place. Manual dispatches always build
|
||||
# everything — predictable when building odd refs.
|
||||
BUILD_CORTEX=true
|
||||
BUILD_NEURON=true
|
||||
BUILD_BENCH=true
|
||||
BUILD_UPSTREAM=true
|
||||
BUILD_ROUTER=true
|
||||
BUILD_TOOLS=true
|
||||
BUILD_ANGELS=true
|
||||
CHECK_RUST=true
|
||||
|
||||
if [ "${GITHUB_EVENT_NAME}" = "push" ]; then
|
||||
MANIFEST_URL="https://rpm.lair.cafe/fedora/43/x86_64/unstable/packages.json"
|
||||
if curl -fsS --max-time 20 -o /tmp/packages.json "$MANIFEST_URL"; then
|
||||
# Latest published sha per package, by buildTime.
|
||||
base_for() {
|
||||
python3 - "$1" <<'PY'
|
||||
import json, re, sys
|
||||
name = sys.argv[1]
|
||||
try:
|
||||
with open("/tmp/packages.json") as f:
|
||||
pkgs = json.load(f)["packages"]
|
||||
cands = [p for p in pkgs if p.get("name") == name]
|
||||
if cands:
|
||||
latest = max(cands, key=lambda p: p.get("buildTime", 0))
|
||||
m = re.search(r"git\.?([0-9a-f]{7,40})", latest.get("release", ""))
|
||||
if m:
|
||||
print(m.group(1))
|
||||
except Exception:
|
||||
pass
|
||||
PY
|
||||
}
|
||||
|
||||
# true if no usable base, else true iff the diff since
|
||||
# the published sha touches the given path pattern.
|
||||
decide() {
|
||||
local base="$1" pattern="$2"
|
||||
if [ -z "$base" ] \
|
||||
|| ! git cat-file -e "${base}^{commit}" 2>/dev/null \
|
||||
|| ! git merge-base --is-ancestor "$base" HEAD 2>/dev/null; then
|
||||
echo true; return
|
||||
fi
|
||||
if git diff --name-only "${base}..HEAD" | grep -qE "$pattern"; then
|
||||
echo true
|
||||
else
|
||||
echo false
|
||||
fi
|
||||
}
|
||||
|
||||
# cortex-core is shared by both binaries; Cargo.{toml,lock}
|
||||
# affect both; this workflow file affects both.
|
||||
NEURON_RE='^crates/neuron/|^crates/cortex-core/|^Cargo\.toml$|^Cargo\.lock$|^rpm/helexa-neuron-prerelease\.spec$|^data/neuron|^neuron\.example\.toml$|^\.gitea/workflows/build-prerelease\.yml$'
|
||||
CORTEX_RE='^crates/cortex-gateway/|^crates/cortex-cli/|^crates/cortex-core/|^Cargo\.toml$|^Cargo\.lock$|^rpm/cortex-prerelease\.spec$|^data/cortex|^cortex\.example\.toml$|^models\.example\.toml$|^\.gitea/workflows/build-prerelease\.yml$'
|
||||
BENCH_RE='^crates/helexa-bench/|^crates/cortex-core/|^Cargo\.toml$|^Cargo\.lock$|^rpm/helexa-bench-prerelease\.spec$|^data/helexa-bench|^helexa-bench\.example\.toml$|^\.gitea/workflows/build-prerelease\.yml$'
|
||||
UPSTREAM_RE='^crates/helexa-upstream/|^crates/cortex-core/|^Cargo\.toml$|^Cargo\.lock$|^rpm/helexa-upstream-prerelease\.spec$|^data/helexa-upstream|^helexa-upstream\.example\.toml$|^\.gitea/workflows/build-prerelease\.yml$'
|
||||
ROUTER_RE='^crates/helexa-router/|^crates/helexa-stream/|^crates/cortex-core/|^Cargo\.toml$|^Cargo\.lock$|^rpm/helexa-router-prerelease\.spec$|^data/helexa-router|^helexa-router\.example\.toml$|^\.gitea/workflows/build-prerelease\.yml$'
|
||||
TOOLS_RE='^crates/helexa-tools/|^Cargo\.toml$|^Cargo\.lock$|^rpm/helexa-tools-prerelease\.spec$|^data/helexa-tools|^helexa-tools\.example\.toml$|^\.gitea/workflows/build-prerelease\.yml$'
|
||||
# Templates are compiled into the binary with include_str!,
|
||||
# so a template edit is a code change and must rebuild.
|
||||
ANGELS_RE='^crates/helexa-angels/|^Cargo\.toml$|^Cargo\.lock$|^rpm/helexa-angels-prerelease\.spec$|^data/helexa-angels|^helexa-angels\.example\.toml$|^\.gitea/workflows/build-prerelease\.yml$'
|
||||
# Any Rust change (incl. crates not packaged here, e.g.
|
||||
# helexa-acp) still needs lint+test on main.
|
||||
RUST_RE='\.rs$|^crates/|Cargo\.toml$|^Cargo\.lock$'
|
||||
|
||||
CORTEX_BASE=$(base_for cortex)
|
||||
NEURON_BASE=$(base_for helexa-neuron-blackwell)
|
||||
BENCH_BASE=$(base_for helexa-bench)
|
||||
UPSTREAM_BASE=$(base_for helexa-upstream)
|
||||
ROUTER_BASE=$(base_for helexa-router)
|
||||
TOOLS_BASE=$(base_for helexa-tools)
|
||||
ANGELS_BASE=$(base_for helexa-angels)
|
||||
BUILD_CORTEX=$(decide "$CORTEX_BASE" "$CORTEX_RE")
|
||||
BUILD_NEURON=$(decide "$NEURON_BASE" "$NEURON_RE")
|
||||
BUILD_BENCH=$(decide "$BENCH_BASE" "$BENCH_RE")
|
||||
BUILD_UPSTREAM=$(decide "$UPSTREAM_BASE" "$UPSTREAM_RE")
|
||||
BUILD_ROUTER=$(decide "$ROUTER_BASE" "$ROUTER_RE")
|
||||
BUILD_TOOLS=$(decide "$TOOLS_BASE" "$TOOLS_RE")
|
||||
BUILD_ANGELS=$(decide "$ANGELS_BASE" "$ANGELS_RE")
|
||||
if [ "$BUILD_CORTEX" = "true" ] || [ "$BUILD_NEURON" = "true" ] || [ "$BUILD_BENCH" = "true" ] || [ "$BUILD_UPSTREAM" = "true" ] || [ "$BUILD_ROUTER" = "true" ] || [ "$BUILD_TOOLS" = "true" ] || [ "$BUILD_ANGELS" = "true" ]; then
|
||||
# Anything we are about to build gets linted and tested.
|
||||
# This is the invariant that matters: no artefact reaches
|
||||
# the fleet unchecked, however the branch below behaves.
|
||||
CHECK_RUST=true
|
||||
else
|
||||
# Nothing is being built, so nothing will be published.
|
||||
# The only question left is whether THIS PUSH touched
|
||||
# Rust — and the honest base for that is the push's own
|
||||
# range, not a package's publication lag.
|
||||
#
|
||||
# This used to ask "has anything Rust changed since
|
||||
# cortex was last published". Cortex is only rebuilt when
|
||||
# cortex paths change, so after a run of neuron-only work
|
||||
# its base sits days behind and the diff from it sweeps in
|
||||
# every unrelated Rust commit since. A one-character
|
||||
# whitespace fix to a doc then re-ran fmt, clippy and the
|
||||
# full test suite over a delta that had already been
|
||||
# linted on the branch that introduced it (2026-08-18,
|
||||
# base fec5b33).
|
||||
#
|
||||
# `decide` returns true for an unusable base, so a new
|
||||
# branch (all-zero before-sha) or a force-push still gets
|
||||
# the full gate.
|
||||
CHECK_RUST=$(decide "$PUSH_BEFORE" "$RUST_RE")
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "build_cortex=${BUILD_CORTEX}" >> "$GITHUB_OUTPUT"
|
||||
echo "build_neuron=${BUILD_NEURON}" >> "$GITHUB_OUTPUT"
|
||||
echo "build_bench=${BUILD_BENCH}" >> "$GITHUB_OUTPUT"
|
||||
echo "build_upstream=${BUILD_UPSTREAM}" >> "$GITHUB_OUTPUT"
|
||||
echo "build_router=${BUILD_ROUTER}" >> "$GITHUB_OUTPUT"
|
||||
echo "build_tools=${BUILD_TOOLS}" >> "$GITHUB_OUTPUT"
|
||||
echo "build_angels=${BUILD_ANGELS}" >> "$GITHUB_OUTPUT"
|
||||
echo "check_rust=${CHECK_RUST}" >> "$GITHUB_OUTPUT"
|
||||
echo "### change detection: build_cortex=${BUILD_CORTEX} build_neuron=${BUILD_NEURON} build_bench=${BUILD_BENCH} build_upstream=${BUILD_UPSTREAM} build_router=${BUILD_ROUTER} build_tools=${BUILD_TOOLS} build_angels=${BUILD_ANGELS} check_rust=${CHECK_RUST}"
|
||||
|
||||
# fmt + clippy + test moved here from ci.yml for main pushes so the
|
||||
# two workflows stop queueing against each other (ci.yml's checks
|
||||
# used to delay build-cortex by ~12 minutes on the shared runner
|
||||
# pool). They run in parallel with the builds and gate `publish`,
|
||||
# not the builds themselves — a clippy warning still can't reach the
|
||||
# fleet, but it also doesn't serialize the pipeline.
|
||||
lint:
|
||||
name: Lint (fmt + clippy)
|
||||
timeout-minutes: 25
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.check_rust == 'true'
|
||||
runs-on: rust
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_BUCKET: sccache
|
||||
SCCACHE_ENDPOINT: http://caveman.kosherinata.internal:9000
|
||||
SCCACHE_REGION: auto
|
||||
SCCACHE_S3_USE_SSL: "false"
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.SCCACHE_S3_ACCESS_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_S3_SECRET_KEY }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
- run: cargo fmt --check --all
|
||||
# Failure-aware sccache escalation lives in the shared script: a
|
||||
# signal death (rustc SIGSEGV / OOM-kill) keeps the cache and fails
|
||||
# fast instead of triggering a slower uncached rebuild; only a real
|
||||
# sccache fault drops the cache. See script/ci-cargo-escalate.sh.
|
||||
- name: Clippy (sccache escalation)
|
||||
run: script/ci-cargo-escalate.sh cargo clippy --workspace -- -D warnings
|
||||
|
||||
test:
|
||||
name: Test
|
||||
timeout-minutes: 25
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.check_rust == 'true'
|
||||
runs-on: rust
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_BUCKET: sccache
|
||||
SCCACHE_ENDPOINT: http://caveman.kosherinata.internal:9000
|
||||
SCCACHE_REGION: auto
|
||||
SCCACHE_S3_USE_SSL: "false"
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.SCCACHE_S3_ACCESS_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_S3_SECRET_KEY }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
# See script/ci-cargo-escalate.sh for the escalation rationale.
|
||||
- name: Test (sccache escalation)
|
||||
run: script/ci-cargo-escalate.sh cargo test --workspace
|
||||
|
||||
build-cortex:
|
||||
name: Build cortex binary
|
||||
timeout-minutes: 25
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.build_cortex == 'true'
|
||||
# runner-rust image already provides rust/cargo/clippy/rustfmt via
|
||||
# dnf — no rustup install step needed.
|
||||
runs-on: rust
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_BUCKET: sccache
|
||||
SCCACHE_ENDPOINT: http://caveman.kosherinata.internal:9000
|
||||
SCCACHE_REGION: auto
|
||||
SCCACHE_S3_USE_SSL: "false"
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.SCCACHE_S3_ACCESS_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_S3_SECRET_KEY }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
# See script/ci-cargo-escalate.sh for the escalation rationale.
|
||||
- name: Build cortex (release, sccache escalation)
|
||||
run: script/ci-cargo-escalate.sh cargo build --release -p cortex-cli
|
||||
|
||||
- name: Stage binary
|
||||
run: |
|
||||
mkdir --parents artifacts
|
||||
cp target/release/cortex artifacts/cortex
|
||||
./artifacts/cortex --version || true
|
||||
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: cortex-fc43
|
||||
path: artifacts/cortex
|
||||
retention-days: 1
|
||||
|
||||
build-bench:
|
||||
name: Build helexa-bench binary
|
||||
timeout-minutes: 25
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.build_bench == 'true'
|
||||
# Pure-Rust, non-CUDA binary — same runner as cortex.
|
||||
runs-on: rust
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_BUCKET: sccache
|
||||
SCCACHE_ENDPOINT: http://caveman.kosherinata.internal:9000
|
||||
SCCACHE_REGION: auto
|
||||
SCCACHE_S3_USE_SSL: "false"
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.SCCACHE_S3_ACCESS_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_S3_SECRET_KEY }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- name: Build helexa-bench (release, sccache escalation)
|
||||
run: |
|
||||
# Stamp the SHA helexa-bench records as bench_sha against every
|
||||
# run (option_env! in sweep.rs reads it at compile time).
|
||||
export HELEXA_BUILD_SHA="$(git rev-parse HEAD)"
|
||||
script/ci-cargo-escalate.sh cargo build --release -p helexa-bench
|
||||
|
||||
- name: Stage binary
|
||||
run: |
|
||||
mkdir --parents artifacts
|
||||
cp target/release/helexa-bench artifacts/helexa-bench
|
||||
./artifacts/helexa-bench --version || true
|
||||
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: bench-fc43
|
||||
path: artifacts/helexa-bench
|
||||
retention-days: 1
|
||||
|
||||
build-upstream:
|
||||
name: Build helexa-upstream binary
|
||||
timeout-minutes: 25
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.build_upstream == 'true'
|
||||
# Pure-Rust, non-CUDA binary — same runner as cortex/bench.
|
||||
runs-on: rust
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_BUCKET: sccache
|
||||
SCCACHE_ENDPOINT: http://caveman.kosherinata.internal:9000
|
||||
SCCACHE_REGION: auto
|
||||
SCCACHE_S3_USE_SSL: "false"
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.SCCACHE_S3_ACCESS_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_S3_SECRET_KEY }}
|
||||
# helexa-upstream uses the sqlx runtime query API (no compile-time
|
||||
# query macros), so it builds without a database or a .sqlx cache.
|
||||
# Set OFFLINE defensively so a stray macro can never reach for a DB.
|
||||
SQLX_OFFLINE: "true"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- name: Build helexa-upstream (release, sccache escalation)
|
||||
run: script/ci-cargo-escalate.sh cargo build --release -p helexa-upstream
|
||||
|
||||
- name: Stage binary
|
||||
run: |
|
||||
mkdir --parents artifacts
|
||||
cp target/release/helexa-upstream artifacts/helexa-upstream
|
||||
./artifacts/helexa-upstream --version || true
|
||||
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: upstream-fc43
|
||||
path: artifacts/helexa-upstream
|
||||
retention-days: 1
|
||||
|
||||
build-angels:
|
||||
name: Build helexa-angels binary
|
||||
timeout-minutes: 25
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.build_angels == 'true'
|
||||
# Pure-Rust, non-CUDA binary — same runner as cortex/upstream.
|
||||
runs-on: rust
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_BUCKET: sccache
|
||||
SCCACHE_ENDPOINT: http://caveman.kosherinata.internal:9000
|
||||
SCCACHE_REGION: auto
|
||||
SCCACHE_S3_USE_SSL: "false"
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.SCCACHE_S3_ACCESS_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_S3_SECRET_KEY }}
|
||||
# Same posture as helexa-upstream: sqlx runtime query API only, so
|
||||
# the crate builds without a database or a .sqlx cache. Set OFFLINE
|
||||
# defensively so a stray macro can never reach for a DB.
|
||||
SQLX_OFFLINE: "true"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- name: Build helexa-angels (release, sccache escalation)
|
||||
run: script/ci-cargo-escalate.sh cargo build --release -p helexa-angels
|
||||
|
||||
- name: Stage binary
|
||||
run: |
|
||||
mkdir --parents artifacts
|
||||
cp target/release/helexa-angels artifacts/helexa-angels
|
||||
./artifacts/helexa-angels --version || true
|
||||
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: angels-fc43
|
||||
path: artifacts/helexa-angels
|
||||
retention-days: 1
|
||||
|
||||
build-router:
|
||||
name: Build helexa-router binary
|
||||
timeout-minutes: 25
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.build_router == 'true'
|
||||
# Pure-Rust, non-CUDA binary — same runner as cortex/bench.
|
||||
runs-on: rust
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_BUCKET: sccache
|
||||
SCCACHE_ENDPOINT: http://caveman.kosherinata.internal:9000
|
||||
SCCACHE_REGION: auto
|
||||
SCCACHE_S3_USE_SSL: "false"
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.SCCACHE_S3_ACCESS_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_S3_SECRET_KEY }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- name: Build helexa-router (release, sccache escalation)
|
||||
run: script/ci-cargo-escalate.sh cargo build --release -p helexa-router
|
||||
|
||||
- name: Stage binary
|
||||
run: |
|
||||
mkdir --parents artifacts
|
||||
cp target/release/helexa-router artifacts/helexa-router
|
||||
./artifacts/helexa-router --version || true
|
||||
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: router-fc43
|
||||
path: artifacts/helexa-router
|
||||
retention-days: 1
|
||||
|
||||
build-tools:
|
||||
name: Build helexa-tools binary
|
||||
timeout-minutes: 25
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.build_tools == 'true'
|
||||
# Pure-Rust, non-CUDA binary — same runner as cortex/bench.
|
||||
runs-on: rust
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_BUCKET: sccache
|
||||
SCCACHE_ENDPOINT: http://caveman.kosherinata.internal:9000
|
||||
SCCACHE_REGION: auto
|
||||
SCCACHE_S3_USE_SSL: "false"
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.SCCACHE_S3_ACCESS_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_S3_SECRET_KEY }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- name: Build helexa-tools (release, sccache escalation)
|
||||
run: script/ci-cargo-escalate.sh cargo build --release -p helexa-tools
|
||||
|
||||
- name: Stage binary
|
||||
run: |
|
||||
mkdir --parents artifacts
|
||||
cp target/release/helexa-tools artifacts/helexa-tools
|
||||
./artifacts/helexa-tools --version || true
|
||||
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: tools-fc43
|
||||
path: artifacts/helexa-tools
|
||||
retention-days: 1
|
||||
|
||||
build-neuron:
|
||||
name: Build neuron-${{ matrix.flavour }}
|
||||
timeout-minutes: 35
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.build_neuron == 'true'
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- flavour: ampere
|
||||
compute_cap: "86"
|
||||
runner: cuda-13.0
|
||||
cuda_home: /usr/local/cuda-13.0
|
||||
build_jobs: 8
|
||||
nvcc_threads: 4
|
||||
cargo_features: "cuda cudnn"
|
||||
- flavour: ada
|
||||
compute_cap: "89"
|
||||
runner: cuda-13.0
|
||||
cuda_home: /usr/local/cuda-13.0
|
||||
build_jobs: 8
|
||||
nvcc_threads: 4
|
||||
cargo_features: "cuda cudnn"
|
||||
- flavour: blackwell
|
||||
compute_cap: "120"
|
||||
runner: cuda-13.0
|
||||
cuda_home: /usr/local/cuda-13.0
|
||||
build_jobs: 8
|
||||
nvcc_threads: 4
|
||||
# flash-attn on blackwell first (#95): beast carries the
|
||||
# agentic prefill pain; ada/ampere follow once the win is
|
||||
# measured. NEURON_FLASH_ATTN=0 is the runtime rollback.
|
||||
cargo_features: "cuda cudnn flash-attn"
|
||||
runs-on: ${{ matrix.runner }}
|
||||
env:
|
||||
SCCACHE_BUCKET: sccache
|
||||
SCCACHE_ENDPOINT: http://caveman.kosherinata.internal:9000
|
||||
SCCACHE_REGION: auto
|
||||
SCCACHE_S3_USE_SSL: "false"
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.SCCACHE_S3_ACCESS_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_S3_SECRET_KEY }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
# sccache handling + failure classification lives in
|
||||
# script/ci-cargo-escalate.sh: it probes for sccache (the CUDA
|
||||
# image may not ship it — a missing binary degrades to an uncached
|
||||
# build rather than failing at `sccache rustc -vV`), and a rustc
|
||||
# SIGSEGV / OOM-kill keeps the cache and fails fast instead of
|
||||
# escalating to a slower uncached rebuild. The cache covers the
|
||||
# ~600-crate host-side dep tree (the bulk of the 10-14 min build),
|
||||
# shared across all three flavours, so even one run seeds the next.
|
||||
- name: Build neuron with CUDA (${{ matrix.flavour }})
|
||||
run: |
|
||||
export PATH="${{ matrix.cuda_home }}/bin:${PATH}"
|
||||
export LD_LIBRARY_PATH="${{ matrix.cuda_home }}/targets/x86_64-linux/lib:${{ matrix.cuda_home }}/lib64:${LD_LIBRARY_PATH:-}"
|
||||
export LIBRARY_PATH="${{ matrix.cuda_home }}/targets/x86_64-linux/lib:${{ matrix.cuda_home }}/lib64:${LIBRARY_PATH:-}"
|
||||
# Pin the build SHA neuron reports from GET /version. The git
|
||||
# fallback in build.rs would also work on a full checkout, but
|
||||
# injecting the exact checked-out commit is unambiguous under
|
||||
# shallow/detached states and makes the artifact self-describing.
|
||||
export HELEXA_BUILD_SHA="$(git rev-parse HEAD)"
|
||||
script/ci-cargo-escalate.sh cargo build --release -p neuron --features "${{ matrix.cargo_features }}"
|
||||
env:
|
||||
CUDA_COMPUTE_CAP: ${{ matrix.compute_cap }}
|
||||
CARGO_BUILD_JOBS: ${{ matrix.build_jobs }}
|
||||
NVCC_THREADS: ${{ matrix.nvcc_threads }}
|
||||
|
||||
- name: Stage binary
|
||||
run: |
|
||||
mkdir --parents artifacts
|
||||
cp target/release/neuron artifacts/neuron-${{ matrix.flavour }}
|
||||
file "artifacts/neuron-${{ matrix.flavour }}"
|
||||
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: neuron-${{ matrix.flavour }}-fc43
|
||||
path: artifacts/neuron-${{ matrix.flavour }}
|
||||
retention-days: 1
|
||||
|
||||
package-cortex:
|
||||
name: Package cortex RPM
|
||||
timeout-minutes: 20
|
||||
needs: [prepare, build-cortex]
|
||||
runs-on: rpm
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: cortex-fc43
|
||||
path: artifacts/
|
||||
|
||||
- name: Build RPM
|
||||
run: |
|
||||
set -eux
|
||||
rm -f ~/.rpmmacros
|
||||
rpmdev-setuptree
|
||||
cp artifacts/cortex ~/rpmbuild/SOURCES/
|
||||
cp data/cortex.service ~/rpmbuild/SOURCES/
|
||||
cp data/cortex-sysusers.conf ~/rpmbuild/SOURCES/
|
||||
cp data/cortex-firewalld.xml ~/rpmbuild/SOURCES/
|
||||
cp cortex.example.toml ~/rpmbuild/SOURCES/
|
||||
cp models.example.toml ~/rpmbuild/SOURCES/
|
||||
cp LICENSE ~/rpmbuild/SOURCES/
|
||||
rpmbuild -bb rpm/cortex-prerelease.spec \
|
||||
--define "cortex_version ${{ needs.prepare.outputs.version }}" \
|
||||
--define "cortex_prerelease ${{ needs.prepare.outputs.release }}" \
|
||||
--undefine dist \
|
||||
--define "dist .fc43"
|
||||
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: rpm-cortex-fc43
|
||||
path: ~/rpmbuild/RPMS/x86_64/*.rpm
|
||||
retention-days: 7
|
||||
|
||||
package-bench:
|
||||
name: Package helexa-bench RPM
|
||||
timeout-minutes: 20
|
||||
needs: [prepare, build-bench]
|
||||
runs-on: rpm
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: bench-fc43
|
||||
path: artifacts/
|
||||
|
||||
- name: Build RPM
|
||||
run: |
|
||||
set -eux
|
||||
rm -f ~/.rpmmacros
|
||||
rpmdev-setuptree
|
||||
cp artifacts/helexa-bench ~/rpmbuild/SOURCES/
|
||||
cp data/helexa-bench.service ~/rpmbuild/SOURCES/
|
||||
cp data/helexa-bench-sysusers.conf ~/rpmbuild/SOURCES/
|
||||
cp data/helexa-bench-firewalld.xml ~/rpmbuild/SOURCES/
|
||||
cp helexa-bench.example.toml ~/rpmbuild/SOURCES/
|
||||
cp LICENSE ~/rpmbuild/SOURCES/
|
||||
rpmbuild -bb rpm/helexa-bench-prerelease.spec \
|
||||
--define "bench_version ${{ needs.prepare.outputs.version }}" \
|
||||
--define "bench_prerelease ${{ needs.prepare.outputs.release }}" \
|
||||
--undefine dist \
|
||||
--define "dist .fc43"
|
||||
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: rpm-bench-fc43
|
||||
path: ~/rpmbuild/RPMS/x86_64/*.rpm
|
||||
retention-days: 7
|
||||
|
||||
package-upstream:
|
||||
name: Package helexa-upstream RPM
|
||||
timeout-minutes: 20
|
||||
needs: [prepare, build-upstream]
|
||||
runs-on: rpm
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: upstream-fc43
|
||||
path: artifacts/
|
||||
|
||||
- name: Build RPM
|
||||
run: |
|
||||
set -eux
|
||||
rm -f ~/.rpmmacros
|
||||
rpmdev-setuptree
|
||||
cp artifacts/helexa-upstream ~/rpmbuild/SOURCES/
|
||||
cp data/helexa-upstream.service ~/rpmbuild/SOURCES/
|
||||
cp data/helexa-upstream-sysusers.conf ~/rpmbuild/SOURCES/
|
||||
cp data/helexa-upstream-firewalld.xml ~/rpmbuild/SOURCES/
|
||||
cp helexa-upstream.example.toml ~/rpmbuild/SOURCES/
|
||||
cp LICENSE ~/rpmbuild/SOURCES/
|
||||
rpmbuild -bb rpm/helexa-upstream-prerelease.spec \
|
||||
--define "upstream_version ${{ needs.prepare.outputs.version }}" \
|
||||
--define "upstream_prerelease ${{ needs.prepare.outputs.release }}" \
|
||||
--undefine dist \
|
||||
--define "dist .fc43"
|
||||
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: rpm-upstream-fc43
|
||||
path: ~/rpmbuild/RPMS/x86_64/*.rpm
|
||||
retention-days: 7
|
||||
|
||||
package-angels:
|
||||
name: Package helexa-angels RPM
|
||||
timeout-minutes: 20
|
||||
needs: [prepare, build-angels]
|
||||
runs-on: rpm
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: angels-fc43
|
||||
path: artifacts/
|
||||
|
||||
- name: Build RPM
|
||||
run: |
|
||||
set -eux
|
||||
rm -f ~/.rpmmacros
|
||||
rpmdev-setuptree
|
||||
cp artifacts/helexa-angels ~/rpmbuild/SOURCES/
|
||||
cp data/helexa-angels.service ~/rpmbuild/SOURCES/
|
||||
cp data/helexa-angels-sysusers.conf ~/rpmbuild/SOURCES/
|
||||
cp data/helexa-angels-firewalld.xml ~/rpmbuild/SOURCES/
|
||||
cp helexa-angels.example.toml ~/rpmbuild/SOURCES/
|
||||
cp LICENSE ~/rpmbuild/SOURCES/
|
||||
rpmbuild -bb rpm/helexa-angels-prerelease.spec \
|
||||
--define "upstream_version ${{ needs.prepare.outputs.version }}" \
|
||||
--define "upstream_prerelease ${{ needs.prepare.outputs.release }}" \
|
||||
--undefine dist \
|
||||
--define "dist .fc43"
|
||||
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: rpm-angels-fc43
|
||||
path: ~/rpmbuild/RPMS/x86_64/*.rpm
|
||||
retention-days: 7
|
||||
|
||||
package-router:
|
||||
name: Package helexa-router RPM
|
||||
timeout-minutes: 20
|
||||
needs: [prepare, build-router]
|
||||
runs-on: rpm
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: router-fc43
|
||||
path: artifacts/
|
||||
|
||||
- name: Build RPM
|
||||
run: |
|
||||
set -eux
|
||||
rm -f ~/.rpmmacros
|
||||
rpmdev-setuptree
|
||||
cp artifacts/helexa-router ~/rpmbuild/SOURCES/
|
||||
cp data/helexa-router.service ~/rpmbuild/SOURCES/
|
||||
cp data/helexa-router-sysusers.conf ~/rpmbuild/SOURCES/
|
||||
cp data/helexa-router-firewalld.xml ~/rpmbuild/SOURCES/
|
||||
cp helexa-router.example.toml ~/rpmbuild/SOURCES/
|
||||
cp LICENSE ~/rpmbuild/SOURCES/
|
||||
rpmbuild -bb rpm/helexa-router-prerelease.spec \
|
||||
--define "router_version ${{ needs.prepare.outputs.version }}" \
|
||||
--define "router_prerelease ${{ needs.prepare.outputs.release }}" \
|
||||
--undefine dist \
|
||||
--define "dist .fc43"
|
||||
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: rpm-router-fc43
|
||||
path: ~/rpmbuild/RPMS/x86_64/*.rpm
|
||||
retention-days: 7
|
||||
|
||||
package-tools:
|
||||
name: Package helexa-tools RPM
|
||||
timeout-minutes: 20
|
||||
needs: [prepare, build-tools]
|
||||
runs-on: rpm
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: tools-fc43
|
||||
path: artifacts/
|
||||
|
||||
- name: Build RPM
|
||||
run: |
|
||||
set -eux
|
||||
rm -f ~/.rpmmacros
|
||||
rpmdev-setuptree
|
||||
cp artifacts/helexa-tools ~/rpmbuild/SOURCES/
|
||||
cp data/helexa-tools.service ~/rpmbuild/SOURCES/
|
||||
cp data/helexa-tools-sysusers.conf ~/rpmbuild/SOURCES/
|
||||
cp data/helexa-tools-firewalld.xml ~/rpmbuild/SOURCES/
|
||||
cp helexa-tools.example.toml ~/rpmbuild/SOURCES/
|
||||
cp LICENSE ~/rpmbuild/SOURCES/
|
||||
rpmbuild -bb rpm/helexa-tools-prerelease.spec \
|
||||
--define "tools_version ${{ needs.prepare.outputs.version }}" \
|
||||
--define "tools_prerelease ${{ needs.prepare.outputs.release }}" \
|
||||
--undefine dist \
|
||||
--define "dist .fc43"
|
||||
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: rpm-tools-fc43
|
||||
path: ~/rpmbuild/RPMS/x86_64/*.rpm
|
||||
retention-days: 7
|
||||
|
||||
package-neuron:
|
||||
name: Package helexa-neuron-${{ matrix.flavour }} RPM
|
||||
timeout-minutes: 20
|
||||
needs: [prepare, build-neuron]
|
||||
runs-on: rpm
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- flavour: ampere
|
||||
- flavour: ada
|
||||
- flavour: blackwell
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: neuron-${{ matrix.flavour }}-fc43
|
||||
path: artifacts/
|
||||
|
||||
- name: Build RPM
|
||||
run: |
|
||||
set -eux
|
||||
rm -f ~/.rpmmacros
|
||||
rpmdev-setuptree
|
||||
cp artifacts/neuron-${{ matrix.flavour }} ~/rpmbuild/SOURCES/
|
||||
cp data/neuron.service ~/rpmbuild/SOURCES/
|
||||
cp data/neuron-sysusers.conf ~/rpmbuild/SOURCES/
|
||||
cp data/neuron-firewalld.xml ~/rpmbuild/SOURCES/
|
||||
cp neuron.example.toml ~/rpmbuild/SOURCES/
|
||||
cp LICENSE ~/rpmbuild/SOURCES/
|
||||
rpmbuild -bb rpm/helexa-neuron-prerelease.spec \
|
||||
--define "neuron_version ${{ needs.prepare.outputs.version }}" \
|
||||
--define "neuron_flavour ${{ matrix.flavour }}" \
|
||||
--define "neuron_prerelease ${{ needs.prepare.outputs.release }}" \
|
||||
--undefine dist \
|
||||
--define "dist .fc43"
|
||||
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: rpm-neuron-${{ matrix.flavour }}-fc43
|
||||
path: ~/rpmbuild/RPMS/x86_64/*.rpm
|
||||
retention-days: 7
|
||||
|
||||
publish:
|
||||
name: Publish to rpm.lair.cafe (unstable)
|
||||
timeout-minutes: 25
|
||||
needs: [lint, test, package-cortex, package-neuron, package-bench, package-upstream, package-router, package-tools, package-angels]
|
||||
# Runs when at least one package was built and nothing failed.
|
||||
# lint/test may be skipped (docs-only refs never get here because
|
||||
# no packages build), but a real failure in any blocks the
|
||||
# fleet from receiving the RPMs.
|
||||
if: >-
|
||||
${{
|
||||
!cancelled()
|
||||
&& (needs.lint.result == 'success' || needs.lint.result == 'skipped')
|
||||
&& (needs.test.result == 'success' || needs.test.result == 'skipped')
|
||||
&& (needs.package-cortex.result == 'success' || needs.package-neuron.result == 'success' || needs.package-bench.result == 'success' || needs.package-upstream.result == 'success' || needs.package-router.result == 'success' || needs.package-tools.result == 'success' || needs.package-angels.result == 'success')
|
||||
&& needs.package-cortex.result != 'failure'
|
||||
&& needs.package-neuron.result != 'failure'
|
||||
&& needs.package-bench.result != 'failure'
|
||||
&& needs.package-upstream.result != 'failure'
|
||||
&& needs.package-router.result != 'failure'
|
||||
&& needs.package-tools.result != 'failure'
|
||||
&& needs.package-angels.result != 'failure'
|
||||
}}
|
||||
runs-on: rpm
|
||||
concurrency:
|
||||
group: rpm-publish
|
||||
cancel-in-progress: false
|
||||
env:
|
||||
RPM_REPO_HOST: oolon.kosherinata.internal
|
||||
FEDORA_VERSION: "43"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- name: Download all built RPMs
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
path: rpms/
|
||||
pattern: rpm-*-fc43
|
||||
|
||||
- name: Flatten RPM artifacts
|
||||
run: |
|
||||
set -eux
|
||||
find rpms/ -name '*.rpm' -exec mv --target-directory=rpms/ {} +
|
||||
find rpms/ -mindepth 1 -type d -empty -delete
|
||||
ls -la rpms/
|
||||
|
||||
- name: Check for sequoia-sq
|
||||
run: |
|
||||
if ! command -v sq &> /dev/null; then
|
||||
echo "ERROR: sequoia-sq is not installed. Install with: sudo dnf install sequoia-sq"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Import signing key
|
||||
env:
|
||||
# Pass secrets via env so values stay out of the rendered shell
|
||||
# script (which Gitea includes in step logs). Template
|
||||
# expansion of ${{ secrets.X }} inside `run:` writes the literal
|
||||
# value into the script and depends on Gitea's log masker to
|
||||
# scrub it — fragile for multi-line keys.
|
||||
RPM_SIGNING_KEY: ${{ secrets.RPM_SIGNING_KEY }}
|
||||
RPM_SIGNING_KEY_ID: ${{ secrets.RPM_SIGNING_KEY_ID }}
|
||||
run: |
|
||||
echo "$RPM_SIGNING_KEY" | gpg --batch --import
|
||||
fpr=$(gpg --batch --with-colons --list-keys "$RPM_SIGNING_KEY_ID" | awk -F: '/^fpr:/ { print $10; exit }')
|
||||
echo "${fpr}:6:" | gpg --batch --import-ownertrust
|
||||
sed "s/@GPG_NAME@/$RPM_SIGNING_KEY_ID/" rpm/rpmmacros > ~/.rpmmacros
|
||||
|
||||
- name: Sign RPMs
|
||||
run: |
|
||||
set -eux
|
||||
for rpm in rpms/*.rpm; do
|
||||
echo "signing ${rpm}..."
|
||||
rpm --addsign "${rpm}"
|
||||
done
|
||||
|
||||
- name: Set up SSH for rsync
|
||||
run: |
|
||||
install --directory --mode 700 ~/.ssh
|
||||
echo "${RSYNC_SSH_KEY}" | install --mode 600 /dev/stdin ~/.ssh/id_ed25519
|
||||
env:
|
||||
RSYNC_SSH_KEY: ${{ secrets.RSYNC_SSH_KEY }}
|
||||
|
||||
- name: Test SSH connectivity
|
||||
run: |
|
||||
ssh -o StrictHostKeyChecking=accept-new "gitea_ci@${RPM_REPO_HOST}" exit
|
||||
|
||||
- name: Ensure unstable repo directory exists
|
||||
run: |
|
||||
ssh "gitea_ci@${RPM_REPO_HOST}" \
|
||||
"mkdir --parents /var/www/rpm/fedora/${FEDORA_VERSION}/x86_64/unstable"
|
||||
|
||||
- name: Sync RPMs to unstable repo
|
||||
run: |
|
||||
rsync \
|
||||
--archive \
|
||||
--verbose \
|
||||
--chmod D755,F644 \
|
||||
rpms/*.rpm \
|
||||
"gitea_ci@${RPM_REPO_HOST}:/var/www/rpm/fedora/${FEDORA_VERSION}/x86_64/unstable/"
|
||||
|
||||
- name: Update unstable repo metadata
|
||||
run: |
|
||||
ssh "gitea_ci@${RPM_REPO_HOST}" \
|
||||
"cd /var/www/rpm/fedora/${FEDORA_VERSION}/x86_64/unstable && createrepo_c --update ."
|
||||
|
||||
- name: Generate packages.json manifest
|
||||
run: |
|
||||
scp script/generate-packages-json.py "gitea_ci@${RPM_REPO_HOST}:/tmp/"
|
||||
ssh "gitea_ci@${RPM_REPO_HOST}" \
|
||||
"python3 /tmp/generate-packages-json.py \
|
||||
--repodata-dir /var/www/rpm/fedora/${FEDORA_VERSION}/x86_64/unstable/repodata \
|
||||
--output /var/www/rpm/fedora/${FEDORA_VERSION}/x86_64/unstable/packages.json \
|
||||
--base-url https://rpm.lair.cafe/fedora/${FEDORA_VERSION}/x86_64/unstable"
|
||||
@@ -1,26 +1,12 @@
|
||||
name: CI
|
||||
|
||||
# Pushes to main are deliberately excluded: build-prerelease.yml runs
|
||||
# its own lint/test jobs there (gating publish), and running both
|
||||
# workflows on the same push made them queue against each other on the
|
||||
# same runner labels — ~12 minutes of added latency per deploy. Feature
|
||||
# branches, PRs to main, and release tags keep the full gate here.
|
||||
on:
|
||||
push:
|
||||
branches-ignore: [main]
|
||||
branches: ["**"]
|
||||
tags: ["v*"]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
# Coalesce same-ref pushes; a newer push supersedes the in-flight run.
|
||||
# (The old shared `cortex-runner-pool` group with build-prerelease.yml
|
||||
# is gone — the workflows no longer trigger on the same refs, and
|
||||
# ephemeral one-VM-per-job runners removed the shared-workspace race
|
||||
# that group existed to serialize.)
|
||||
concurrency:
|
||||
group: ci-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
CARGO_INCREMENTAL: "0"
|
||||
RUSTC_WRAPPER: sccache
|
||||
@@ -30,226 +16,40 @@ env:
|
||||
SCCACHE_S3_USE_SSL: "false"
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.SCCACHE_S3_ACCESS_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_S3_SECRET_KEY }}
|
||||
# fmt, clippy, and test all run in parallel on the same `rust` runner
|
||||
# and would otherwise share /root/.cache/act/<hash>/hostexecutor/target/,
|
||||
# racing each other's cargo temp files (.tmpXXXXXX) and failing builds
|
||||
# mid-compile. Give each job its own target directory so the invocations
|
||||
# don't collide. sccache still backs the actual rustc cache, so the
|
||||
# rebuild penalty is small.
|
||||
CARGO_TARGET_DIR: target-${{ github.job }}
|
||||
|
||||
jobs:
|
||||
# Two independent questions: does this change need the Rust gate, and
|
||||
# does it need the web gate.
|
||||
#
|
||||
# The SPA under helexa.ai/ is built and shipped by its own deploy job and
|
||||
# is not read by any crate, build script or spec file — so a change
|
||||
# confined to it cannot alter a Rust result. Waiting up to 70 minutes for
|
||||
# a CUDA type-check to prove that is pure latency, and the wait is what
|
||||
# tempts people to merge front-end work unchecked. Documentation is inert
|
||||
# the same way, and more obviously: no crate `include_str!`s a markdown
|
||||
# file, no spec packages `doc/`, no build script reads one.
|
||||
#
|
||||
# Conversely a Rust-only change cannot break the SPA, so it need not
|
||||
# spend three minutes on `npm ci` + lint + typecheck + build.
|
||||
#
|
||||
# The classification stays deliberately one-directional: it only ever
|
||||
# narrows the gate when *every* changed path is known-inert, and every
|
||||
# other outcome — a tag, an unknown base, an empty diff, a mixed
|
||||
# changeset — falls through to the full gate. A new top-level directory
|
||||
# is therefore covered on the day it appears, without anyone
|
||||
# remembering to add it here. Adding an inert pattern is the only way to
|
||||
# widen what gets skipped, which is the property worth keeping.
|
||||
changes:
|
||||
name: Classify changes
|
||||
timeout-minutes: 10
|
||||
runs-on: fedora-43
|
||||
outputs:
|
||||
rust: ${{ steps.classify.outputs.rust }}
|
||||
web: ${{ steps.classify.outputs.web }}
|
||||
check:
|
||||
name: Format, lint, build, test
|
||||
runs-on: fedora
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: classify
|
||||
|
||||
- name: Ensure sccache with S3 support
|
||||
env:
|
||||
RUSTC_WRAPPER: ""
|
||||
run: |
|
||||
set -u
|
||||
full() {
|
||||
echo "→ full gate: $1"
|
||||
echo "rust=true" >> "$GITHUB_OUTPUT"
|
||||
echo "web=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
}
|
||||
|
||||
# Tags build the SRPMs from the whole tree, and the srpm jobs
|
||||
# gate on fmt/clippy/test/cuda-check. Never narrow there.
|
||||
case "${GITHUB_REF}" in
|
||||
refs/tags/*) full "tag push" ;;
|
||||
esac
|
||||
|
||||
# merge-base against main, not the push's before-sha: a new
|
||||
# branch has no before-sha, and a force-push's is unreachable.
|
||||
base=""
|
||||
if git fetch --no-tags origin main >/dev/null 2>&1; then
|
||||
base="$(git merge-base FETCH_HEAD HEAD 2>/dev/null || true)"
|
||||
fi
|
||||
[ -n "${base}" ] || full "no merge-base with main"
|
||||
|
||||
files="$(git diff --name-only "${base}" HEAD)"
|
||||
[ -n "${files}" ] || full "empty diff against ${base}"
|
||||
|
||||
echo "changed files:"
|
||||
printf '%s\n' "${files}" | sed 's/^/ /'
|
||||
|
||||
# Inert for Rust: the SPA, and documentation. Markdown is
|
||||
# matched only at the repo root and under doc/ — a .md inside
|
||||
# crates/ is left to fall through, since `include_str!` could
|
||||
# make one load-bearing without this file knowing.
|
||||
rust_relevant="$(printf '%s\n' "${files}" \
|
||||
| grep -v '^helexa\.ai/' \
|
||||
| grep -v '^doc/' \
|
||||
| grep -vE '^[^/]+\.md$' \
|
||||
|| true)"
|
||||
if [ -n "${rust_relevant}" ]; then
|
||||
echo "rust=true" >> "$GITHUB_OUTPUT"
|
||||
echo "→ Rust gate: required"
|
||||
if sccache --version 2>/dev/null && sccache --show-stats 2>/dev/null; then
|
||||
echo "sccache with S3 support already installed"
|
||||
else
|
||||
echo "rust=false" >> "$GITHUB_OUTPUT"
|
||||
echo "→ Rust gate: skipped (front-end and/or docs only)"
|
||||
cargo install sccache --features s3 --locked
|
||||
fi
|
||||
|
||||
# The web gate is needed only when the SPA itself changed.
|
||||
if printf '%s\n' "${files}" | grep -q '^helexa\.ai/'; then
|
||||
echo "web=true" >> "$GITHUB_OUTPUT"
|
||||
echo "→ web gate: required"
|
||||
else
|
||||
echo "web=false" >> "$GITHUB_OUTPUT"
|
||||
echo "→ web gate: skipped (nothing under helexa.ai/)"
|
||||
fi
|
||||
- name: Check formatting
|
||||
run: cargo fmt --check --all
|
||||
|
||||
fmt:
|
||||
name: Format
|
||||
timeout-minutes: 15
|
||||
runs-on: rust
|
||||
needs: changes
|
||||
if: needs.changes.outputs.rust == 'true'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: cargo fmt --check --all
|
||||
- name: Clippy
|
||||
run: cargo clippy --workspace -- -D warnings
|
||||
|
||||
web:
|
||||
name: Web (lint + typecheck + i18n + build)
|
||||
timeout-minutes: 20
|
||||
runs-on: fedora-43
|
||||
needs: changes
|
||||
if: needs.changes.outputs.web == 'true'
|
||||
defaults:
|
||||
run:
|
||||
working-directory: helexa.ai
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "20"
|
||||
- run: npm ci
|
||||
# eslint and tsc previously ran nowhere. The deploy job builds the
|
||||
# SPA, but only on main and only after a merge — so a defect landed
|
||||
# first and broke the deploy afterwards. A setState-in-effect error
|
||||
# reached main exactly that way.
|
||||
- run: npm run lint
|
||||
- run: npm run typecheck
|
||||
# 42 locales with strict key parity is a promise that breaks
|
||||
# quietly. Adding an English string without translating it must fail
|
||||
# here, not in review, and certainly not in front of somebody
|
||||
# evaluating the project in their own language.
|
||||
- run: npm run i18n:check
|
||||
- run: npm run i18n:meta
|
||||
- run: npm run i18n:lang-labels
|
||||
- run: npm run build
|
||||
- name: Test
|
||||
run: cargo test --workspace
|
||||
|
||||
clippy:
|
||||
name: Clippy
|
||||
timeout-minutes: 25
|
||||
runs-on: rust
|
||||
needs: changes
|
||||
if: needs.changes.outputs.rust == 'true'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# Failure-aware sccache escalation lives in the shared script (kept
|
||||
# in sync with build-prerelease.yml): a signal death (rustc SIGSEGV
|
||||
# / OOM-kill) keeps the cache and fails fast instead of an uncached
|
||||
# rebuild; only a real sccache fault drops the cache.
|
||||
- name: Clippy (sccache escalation)
|
||||
run: script/ci-cargo-escalate.sh cargo clippy --workspace -- -D warnings
|
||||
|
||||
test:
|
||||
name: Test
|
||||
timeout-minutes: 25
|
||||
runs-on: rust
|
||||
needs: changes
|
||||
if: needs.changes.outputs.rust == 'true'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# See script/ci-cargo-escalate.sh for the escalation rationale.
|
||||
- name: Test (sccache escalation)
|
||||
run: script/ci-cargo-escalate.sh cargo test --workspace
|
||||
|
||||
# Type-check the CUDA-only code path. Borrow-check-only — we
|
||||
# never run the tests here (the runner has no GPU). This catches
|
||||
# the category of bug where a refactor compiles fine under the
|
||||
# default feature set (which is what the `clippy` and `test` jobs
|
||||
# exercise) but fails inside a `#[cfg(feature = "cuda")]` block.
|
||||
# `runs-on: cuda-13.0` selects the runner that ships nvcc /
|
||||
# cudarc's build prerequisites. The generic `rust` and `rpm`
|
||||
# runners don't have them (the previous label `rpm` was tried
|
||||
# first and tripped cudarc's `nvcc --version` build script —
|
||||
# see commit history).
|
||||
cuda-check:
|
||||
name: CUDA type-check
|
||||
# flash-attn kernel compilation dominates the first uncached run;
|
||||
# sccache + the cargo target cache absorb it afterwards.
|
||||
timeout-minutes: 70
|
||||
runs-on: cuda-13.0
|
||||
needs: changes
|
||||
if: needs.changes.outputs.rust == 'true'
|
||||
# The workflow-level env sets `RUSTC_WRAPPER: sccache`
|
||||
# unconditionally, which hard-fails cargo if the CUDA image
|
||||
# doesn't ship sccache. Clear it at job level; the "Enable
|
||||
# sccache when available" step opts back in only after probing
|
||||
# for the binary. SCCACHE_*/AWS creds stay set — harmless when
|
||||
# the wrapper is off, required when it's on.
|
||||
env:
|
||||
RUSTC_WRAPPER: ""
|
||||
# candle-kernels' build script falls back to `nvidia-smi` for
|
||||
# compute-cap detection when this is unset — and the GPU-less
|
||||
# builder image doesn't ship nvidia-smi. Any valid cap works for
|
||||
# a borrow-check; the real per-flavour caps live in
|
||||
# build-prerelease.yml's matrix.
|
||||
CUDA_COMPUTE_CAP: "86"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# sccache probing + failure classification lives in the shared
|
||||
# script (see build-prerelease.yml's neuron build for the same
|
||||
# pattern). It probes for sccache and, on a rustc SIGSEGV / OOM,
|
||||
# keeps the cache and fails fast rather than rebuilding uncached.
|
||||
- name: cargo check --features cuda (sccache escalation)
|
||||
run: |
|
||||
# act launches the step shell without /etc/profile, so the
|
||||
# gitea_runner user's inherited PATH lacks /usr/local/cuda-13.0/bin.
|
||||
# cudarc's build.rs shells out to `nvcc --version` (the neuron
|
||||
# crate enables cuda-version-from-build-system) and panics with
|
||||
# ENOENT if nvcc isn't resolvable — keep this export in sync
|
||||
# with build-prerelease.yml.
|
||||
export PATH="/usr/local/cuda-13.0/bin:${PATH}"
|
||||
export LD_LIBRARY_PATH="/usr/local/cuda-13.0/targets/x86_64-linux/lib:/usr/local/cuda-13.0/lib64:${LD_LIBRARY_PATH:-}"
|
||||
export LIBRARY_PATH="/usr/local/cuda-13.0/targets/x86_64-linux/lib:/usr/local/cuda-13.0/lib64:${LIBRARY_PATH:-}"
|
||||
script/ci-cargo-escalate.sh cargo check -p neuron --features cuda,flash-attn --all-targets
|
||||
- name: Show sccache stats
|
||||
run: sccache --show-stats
|
||||
|
||||
srpm-cortex:
|
||||
name: Build cortex SRPM
|
||||
timeout-minutes: 25
|
||||
runs-on: rpm
|
||||
needs: [fmt, clippy, test, cuda-check]
|
||||
runs-on: fedora
|
||||
needs: check
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -308,9 +108,8 @@ jobs:
|
||||
|
||||
srpm-neuron:
|
||||
name: Build neuron SRPM
|
||||
timeout-minutes: 25
|
||||
runs-on: rpm
|
||||
needs: [fmt, clippy, test, cuda-check]
|
||||
runs-on: fedora
|
||||
needs: check
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -369,8 +168,7 @@ jobs:
|
||||
|
||||
copr-cortex:
|
||||
name: Publish cortex to COPR
|
||||
timeout-minutes: 60
|
||||
runs-on: fedora-43
|
||||
runs-on: fedora
|
||||
needs: srpm-cortex
|
||||
steps:
|
||||
- name: Download SRPM
|
||||
@@ -387,8 +185,7 @@ jobs:
|
||||
|
||||
copr-neuron:
|
||||
name: Publish neuron to COPR
|
||||
timeout-minutes: 60
|
||||
runs-on: fedora-43
|
||||
runs-on: fedora
|
||||
needs: srpm-neuron
|
||||
steps:
|
||||
- name: Download SRPM
|
||||
@@ -405,8 +202,7 @@ jobs:
|
||||
|
||||
bump-version:
|
||||
name: Bump version in source
|
||||
timeout-minutes: 15
|
||||
runs-on: rust
|
||||
runs-on: fedora
|
||||
needs: [copr-cortex, copr-neuron]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -449,6 +245,6 @@ jobs:
|
||||
echo "Nothing to commit for ${VERSION}"
|
||||
else
|
||||
git commit -m "chore: bump version to ${VERSION}"
|
||||
git remote set-url origin "https://gitea-actions:${GITEA_TOKEN}@git.lair.cafe/${{ github.repository }}.git"
|
||||
git remote set-url origin "https://gitea-actions:${GITEA_TOKEN}@git.lair.cafe/helexa/cortex.git"
|
||||
git push origin HEAD:main
|
||||
fi
|
||||
|
||||
@@ -1,136 +0,0 @@
|
||||
name: deploy-dev
|
||||
|
||||
# Fast-path iteration deploy for a SINGLE neuron host: build one CUDA
|
||||
# flavour, copy the raw binary to the host, restart neuron.service.
|
||||
# Skips the other two flavours, all RPM packaging, signing, repo
|
||||
# publish, and dnf — push-to-testable drops from ~20 min to roughly
|
||||
# one CUDA build plus a service restart.
|
||||
#
|
||||
# This is a DEV convenience, not a release path:
|
||||
# - the binary lands at /usr/bin/neuron *outside* RPM ownership;
|
||||
# the next regular deploy.yml run reconciles the host back to the
|
||||
# packaged binary (dnf sees the newer RPM and reinstalls). `rpm -V
|
||||
# helexa-neuron-<flavour>` flagging a modified /usr/bin/neuron in
|
||||
# the interim is expected.
|
||||
# - nothing is published; other hosts are untouched.
|
||||
# - requires the `install` sudoers rule from
|
||||
# asset/sudoers.d/neuron-host.conf (re-run script/infra-setup.sh
|
||||
# after updating it).
|
||||
#
|
||||
# Trigger from the Gitea UI: Actions → deploy-dev → Run workflow,
|
||||
# pick the target host. Defaults to the ref you dispatch from, so it
|
||||
# works from feature branches without touching main.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
target:
|
||||
description: "neuron host to deploy to"
|
||||
required: true
|
||||
type: choice
|
||||
options: [beast, benjy, quadbrat]
|
||||
default: beast
|
||||
|
||||
# One dev deploy at a time; a newer dispatch for the same host wins.
|
||||
concurrency:
|
||||
group: deploy-dev-${{ inputs.target }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
CARGO_INCREMENTAL: "0"
|
||||
CARGO_TERM_COLOR: "always"
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build neuron (${{ inputs.target }})
|
||||
runs-on: cuda-13.0
|
||||
outputs:
|
||||
flavour: ${{ steps.map.outputs.flavour }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
# host → flavour → compute cap. Keep in sync with the
|
||||
# build-neuron matrix in build-prerelease.yml and the
|
||||
# deploy-neurons matrix in deploy.yml.
|
||||
- id: map
|
||||
run: |
|
||||
case "${{ inputs.target }}" in
|
||||
beast) flavour=blackwell cap=120 ;;
|
||||
benjy) flavour=ada cap=89 ;;
|
||||
quadbrat) flavour=ampere cap=86 ;;
|
||||
*) echo "unknown target ${{ inputs.target }}"; exit 1 ;;
|
||||
esac
|
||||
echo "flavour=${flavour}" >> "$GITHUB_OUTPUT"
|
||||
echo "cap=${cap}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Build neuron with CUDA
|
||||
run: |
|
||||
set -eux
|
||||
export PATH="/usr/local/cuda-13.0/bin:${PATH}"
|
||||
export LD_LIBRARY_PATH="/usr/local/cuda-13.0/targets/x86_64-linux/lib:/usr/local/cuda-13.0/lib64:${LD_LIBRARY_PATH:-}"
|
||||
export LIBRARY_PATH="/usr/local/cuda-13.0/targets/x86_64-linux/lib:/usr/local/cuda-13.0/lib64:${LIBRARY_PATH:-}"
|
||||
cargo build --release -p neuron --features "cuda cudnn"
|
||||
env:
|
||||
CUDA_COMPUTE_CAP: ${{ steps.map.outputs.cap }}
|
||||
CARGO_BUILD_JOBS: "8"
|
||||
NVCC_THREADS: "4"
|
||||
|
||||
- name: Stage binary
|
||||
run: |
|
||||
mkdir --parents artifacts
|
||||
cp target/release/neuron artifacts/neuron-dev
|
||||
file artifacts/neuron-dev
|
||||
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: neuron-dev-${{ inputs.target }}
|
||||
path: artifacts/neuron-dev
|
||||
retention-days: 1
|
||||
|
||||
deploy:
|
||||
name: Deploy to ${{ inputs.target }}
|
||||
needs: build
|
||||
runs-on: fedora-43
|
||||
env:
|
||||
DEPLOY_KEY: |
|
||||
${{ secrets.RSYNC_SSH_KEY }}
|
||||
TARGET_HOST: ${{ inputs.target }}.hanzalova.internal
|
||||
steps:
|
||||
- name: SSH init
|
||||
run: |
|
||||
mkdir -p ~/.ssh
|
||||
echo "${DEPLOY_KEY}" > ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
|
||||
"gitea_ci@${TARGET_HOST}" 'hostname -f'
|
||||
|
||||
- uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: neuron-dev-${{ inputs.target }}
|
||||
path: artifacts/
|
||||
|
||||
- name: Copy binary to host
|
||||
run: |
|
||||
scp artifacts/neuron-dev "gitea_ci@${TARGET_HOST}:/var/lib/gitea_ci/neuron-dev"
|
||||
|
||||
- name: Install binary and restart neuron.service
|
||||
run: |
|
||||
ssh "gitea_ci@${TARGET_HOST}" '
|
||||
set -eu
|
||||
if systemctl is-active --quiet neuron.service; then
|
||||
sudo /usr/bin/systemctl stop neuron.service
|
||||
fi
|
||||
# Exact command form required by the sudoers rule in
|
||||
# asset/sudoers.d/neuron-host.conf — change both together.
|
||||
sudo /usr/bin/install -o root -g root -m 0755 /var/lib/gitea_ci/neuron-dev /usr/bin/neuron
|
||||
# enable --now so a dev deploy also leaves the unit enabled
|
||||
# for boot, consistent with deploy.yml.
|
||||
sudo /usr/bin/systemctl enable --now neuron.service
|
||||
rm -f /var/lib/gitea_ci/neuron-dev'
|
||||
|
||||
- name: Capture neuron.service startup journal
|
||||
if: always()
|
||||
run: |
|
||||
sleep 10
|
||||
ssh "gitea_ci@${TARGET_HOST}" \
|
||||
'journalctl --unit neuron.service -I --no-pager'
|
||||
@@ -1,748 +0,0 @@
|
||||
name: deploy
|
||||
|
||||
# Roll the freshly-published unstable RPMs onto the helexa fleet:
|
||||
# cortex on the gateway, helexa-neuron-<flavour> on each neuron host,
|
||||
# and helexa-bench on bob (the bench host).
|
||||
#
|
||||
# Triggered automatically after `build-prerelease` succeeds (by which
|
||||
# point the new RPMs are live on rpm.lair.cafe/unstable), and also
|
||||
# re-runnable manually from the Gitea UI.
|
||||
#
|
||||
# Each host self-gates: if dnf sees no newer package than what is
|
||||
# installed, the service is left alone — no stop, no restart, no model
|
||||
# cold-load. Combined with build-prerelease's change detection this
|
||||
# means a docs- or gateway-only push never restarts the neurons (a
|
||||
# neuron restart costs ~5 min of 27B cold-load, see issue #1).
|
||||
#
|
||||
# Per-host one-time setup (gitea_ci user, authorized_keys, scoped
|
||||
# sudoers drop-in) lives in script/infra-setup.sh — run that once per
|
||||
# host before this workflow can succeed.
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: [build-prerelease]
|
||||
types: [completed]
|
||||
workflow_dispatch:
|
||||
|
||||
# Serialize deploys. Overlapping runs would race on dnf metadata
|
||||
# refresh and service-restart timing; queueing keeps the fleet
|
||||
# predictable. Don't cancel an in-flight deploy — a half-applied dnf
|
||||
# transaction is worse than a slightly stale deploy.
|
||||
concurrency:
|
||||
group: deploy
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
DEPLOY_KEY: |
|
||||
${{ secrets.RSYNC_SSH_KEY }}
|
||||
|
||||
jobs:
|
||||
deploy-cortex:
|
||||
runs-on: fedora-43
|
||||
# Two trigger paths: manual dispatch always runs; workflow_run
|
||||
# only runs if the upstream `build-prerelease` actually succeeded.
|
||||
if: >-
|
||||
${{
|
||||
github.event_name == 'workflow_dispatch'
|
||||
|| github.event.workflow_run.conclusion == 'success'
|
||||
}}
|
||||
steps:
|
||||
- name: SSH init
|
||||
run: |
|
||||
mkdir -p ~/.ssh
|
||||
echo "${DEPLOY_KEY}" > ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
|
||||
gitea_ci@hanzalova.internal 'hostname -f'
|
||||
|
||||
# The journal-capture step below needs to know whether this run
|
||||
# actually restarted the service. systemd mints a fresh
|
||||
# InvocationID on every start, so comparing before/after answers
|
||||
# that without depending on the deploy step's control flow.
|
||||
- name: Record cortex.service invocation
|
||||
id: pre
|
||||
run: |
|
||||
echo "inv=$(ssh gitea_ci@hanzalova.internal \
|
||||
'systemctl show -P InvocationID cortex.service' || true)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Gating compares `rpm -q` against the packages.json manifest the
|
||||
# publish job maintains — NOT unprivileged `dnf check-update`,
|
||||
# which proved unreliable as the gitea_ci user (hung on metadata
|
||||
# locks on one host, silently reported "no updates" on others).
|
||||
# An unreadable/unparsable manifest fails open: deploy proceeds.
|
||||
- name: Deploy cortex (skips when already current)
|
||||
run: |
|
||||
ssh gitea_ci@hanzalova.internal 'bash -s' <<'DEPLOY'
|
||||
set -eu
|
||||
pkg=cortex
|
||||
# `rpm -q` prints "package X is not installed" on STDOUT and exits
|
||||
# 1, so `|| echo not-installed` never fires — it captures the
|
||||
# message text instead. Test the exit code.
|
||||
if installed=$(rpm -q --qf '%{VERSION}-%{RELEASE}' "${pkg}" 2>/dev/null); then :; else
|
||||
installed="not-installed"
|
||||
fi
|
||||
latest=$(curl -fsS --max-time 15 "https://rpm.lair.cafe/fedora/43/x86_64/unstable/packages.json" 2>/dev/null \
|
||||
| python3 -c '
|
||||
import json, sys
|
||||
name = sys.argv[1]
|
||||
cands = [p for p in json.load(sys.stdin)["packages"] if p.get("name") == name]
|
||||
if cands:
|
||||
p = max(cands, key=lambda p: p.get("buildTime", 0))
|
||||
print(p["version"] + "-" + p["release"])
|
||||
' "${pkg}" 2>/dev/null || true)
|
||||
if [ -n "${latest}" ] && [ "${latest}" = "${installed}" ]; then
|
||||
echo "${pkg}-${installed} already current — leaving service untouched"
|
||||
exit 0
|
||||
fi
|
||||
echo "installed=${installed} published=${latest:-unknown} — deploying"
|
||||
if systemctl is-active --quiet cortex.service; then
|
||||
sudo /usr/bin/systemctl stop cortex.service
|
||||
fi
|
||||
if rpm -q "${pkg}" >/dev/null 2>&1; then
|
||||
sudo /usr/bin/dnf upgrade --refresh --allowerasing -y cortex
|
||||
else
|
||||
sudo /usr/bin/dnf install --refresh --allowerasing -y cortex
|
||||
fi
|
||||
sudo /usr/bin/systemctl daemon-reload
|
||||
# enable --now: start the service AND enable it for boot so the
|
||||
# fleet self-heals after a host reboot.
|
||||
sudo /usr/bin/systemctl enable --now cortex.service
|
||||
DEPLOY
|
||||
|
||||
# Wait for the service to either come up or wedge, then capture
|
||||
# the journal for the invocation this run started. Runs even on
|
||||
# prior failure so a failed start step still leaves a usable
|
||||
# record in the deploy log.
|
||||
- name: Capture cortex.service startup journal
|
||||
if: always()
|
||||
env:
|
||||
PRE_INV: ${{ steps.pre.outputs.inv }}
|
||||
run: |
|
||||
sleep 10
|
||||
ssh gitea_ci@hanzalova.internal bash -s "${PRE_INV}" <<'CAPTURE'
|
||||
set -eu
|
||||
pre=${1:-}
|
||||
unit=cortex.service
|
||||
now=$(systemctl show -P InvocationID "${unit}")
|
||||
if [ -n "${now}" ] && [ "${now}" = "${pre}" ]; then
|
||||
echo "${unit} not restarted this run (invocation ${now}) — nothing to capture"
|
||||
systemctl status --no-pager --lines=0 "${unit}" || true
|
||||
exit 0
|
||||
fi
|
||||
# -I selects the latest invocation, which on a long-lived service
|
||||
# is unbounded — days of logs, hundreds of MB at debug level. Cap
|
||||
# the line count and drop the debug/trace firehose; startup lines
|
||||
# (and panics on stderr) are info and above. An empty invocation
|
||||
# id means the unit never came up, so drop -I and fall back to the
|
||||
# unit's recent tail, which is where the failure will be.
|
||||
timeout 60 journalctl --unit "${unit}" ${now:+-I} \
|
||||
--priority=info --lines=500 --no-pager \
|
||||
|| { echo "(no info-level entries for this invocation, or capture timed out)"
|
||||
systemctl status --no-pager --lines=0 "${unit}" || true; }
|
||||
CAPTURE
|
||||
|
||||
deploy-neurons:
|
||||
needs: [deploy-cortex]
|
||||
runs-on: fedora-43
|
||||
strategy:
|
||||
# One neuron failing must not cancel the others. Cortex is up
|
||||
# already; a partial neuron deploy is strictly better than
|
||||
# rolling back to zero.
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
# load_timeout: how long to wait for default_models to finish
|
||||
# loading after a restart. beast cold-loads Qwen3.6-27B Q6K
|
||||
# TP=2 (~5-6 min typical, see #1); benjy/quadbrat load small
|
||||
# single-GPU models in well under a minute.
|
||||
#
|
||||
# max_prompt_tokens: per-model context cap, written to the
|
||||
# neuron.service.d/model.conf drop-in (NEURON_MAX_PROMPT_TOKENS).
|
||||
# A change here restarts the neuron even with no new RPM. Values
|
||||
# are VRAM-safe ceilings derived per model — see
|
||||
# doc/context-limits.md. beast (Qwen3.6-27B, hybrid linear, 2x
|
||||
# 32GB) has ample KV headroom; benjy (Qwen3-8B dense, ~6GB free)
|
||||
# is VRAM-bound and stays at the default; quadbrat (Qwen3-1.7B)
|
||||
# likewise conservative.
|
||||
- host: beast.hanzalova.internal
|
||||
flavour: blackwell
|
||||
load_timeout: 900
|
||||
max_prompt_tokens: 131072
|
||||
- host: benjy.hanzalova.internal
|
||||
flavour: ada
|
||||
load_timeout: 300
|
||||
max_prompt_tokens: 16384
|
||||
- host: quadbrat.hanzalova.internal
|
||||
flavour: ampere
|
||||
load_timeout: 300
|
||||
max_prompt_tokens: 16384
|
||||
steps:
|
||||
- name: SSH init
|
||||
run: |
|
||||
mkdir -p ~/.ssh
|
||||
echo "${DEPLOY_KEY}" > ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
|
||||
gitea_ci@${{ matrix.host }} 'hostname -f'
|
||||
|
||||
# See deploy-cortex for why the capture step needs this.
|
||||
- name: Record neuron.service invocation
|
||||
id: pre
|
||||
run: |
|
||||
echo "inv=$(ssh gitea_ci@${{ matrix.host }} \
|
||||
'systemctl show -P InvocationID neuron.service' || true)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# See deploy-cortex for why gating uses the publish manifest and
|
||||
# not unprivileged `dnf check-update`.
|
||||
- name: Deploy helexa-neuron-${{ matrix.flavour }} (skips when already current)
|
||||
run: |
|
||||
ssh gitea_ci@${{ matrix.host }} 'bash -s' <<'DEPLOY'
|
||||
set -eu
|
||||
pkg=helexa-neuron-${{ matrix.flavour }}
|
||||
max_prompt_tokens="${{ matrix.max_prompt_tokens }}"
|
||||
|
||||
# ── Desired per-model systemd drop-in ─────────────────────────
|
||||
# model.conf carries NEURON_MAX_PROMPT_TOKENS so the context cap
|
||||
# is deterministic per host and rolled out (with a restart) by
|
||||
# this workflow, not hand-edited. It sorts after local.conf, so a
|
||||
# deploy-managed value wins over any manual local override of the
|
||||
# same variable. See doc/context-limits.md.
|
||||
conf=/etc/systemd/system/neuron.service.d/model.conf
|
||||
config_changed=0
|
||||
if [ -n "${max_prompt_tokens}" ]; then
|
||||
desired=$(printf '%s\n%s\n%s\n%s' \
|
||||
"# Managed by .gitea/workflows/deploy.yml - do not edit by hand." \
|
||||
"# Per-model context cap; see doc/context-limits.md." \
|
||||
"[Service]" \
|
||||
"Environment=NEURON_MAX_PROMPT_TOKENS=${max_prompt_tokens}")
|
||||
[ "${desired}" = "$(cat "${conf}" 2>/dev/null || true)" ] || config_changed=1
|
||||
fi
|
||||
|
||||
# ── Package version gate (manifest rationale: see deploy-cortex) ──
|
||||
# `rpm -q` prints "package X is not installed" on STDOUT and exits
|
||||
# 1, so `|| echo not-installed` never fires — it captures the
|
||||
# message text instead. Test the exit code.
|
||||
if installed=$(rpm -q --qf '%{VERSION}-%{RELEASE}' "${pkg}" 2>/dev/null); then :; else
|
||||
installed="not-installed"
|
||||
fi
|
||||
latest=$(curl -fsS --max-time 15 "https://rpm.lair.cafe/fedora/43/x86_64/unstable/packages.json" 2>/dev/null \
|
||||
| python3 -c '
|
||||
import json, sys
|
||||
name = sys.argv[1]
|
||||
cands = [p for p in json.load(sys.stdin)["packages"] if p.get("name") == name]
|
||||
if cands:
|
||||
p = max(cands, key=lambda p: p.get("buildTime", 0))
|
||||
print(p["version"] + "-" + p["release"])
|
||||
' "${pkg}" 2>/dev/null || true)
|
||||
pkg_changed=1
|
||||
if [ -n "${latest}" ] && [ "${latest}" = "${installed}" ]; then
|
||||
pkg_changed=0
|
||||
fi
|
||||
|
||||
# Skip only when BOTH the package and the drop-in are unchanged —
|
||||
# a context-cap change must restart the neuron even with no new RPM.
|
||||
if [ "${pkg_changed}" -eq 0 ] && [ "${config_changed}" -eq 0 ]; then
|
||||
echo "${pkg}-${installed} current; NEURON_MAX_PROMPT_TOKENS=${max_prompt_tokens:-<unset>} unchanged — leaving service untouched"
|
||||
exit 0
|
||||
fi
|
||||
echo "installed=${installed} published=${latest:-unknown} pkg_changed=${pkg_changed} config_changed=${config_changed} — deploying"
|
||||
|
||||
# Write the drop-in (staged in gitea_ci's dir, installed root-owned).
|
||||
if [ "${config_changed}" -eq 1 ]; then
|
||||
printf '%s\n' "${desired}" > /var/lib/gitea_ci/model.conf
|
||||
sudo /usr/bin/install -o root -g root -m 0644 -D /var/lib/gitea_ci/model.conf "${conf}"
|
||||
rm -f /var/lib/gitea_ci/model.conf
|
||||
echo "applied ${conf}: NEURON_MAX_PROMPT_TOKENS=${max_prompt_tokens}"
|
||||
fi
|
||||
|
||||
if systemctl is-active --quiet neuron.service; then
|
||||
sudo /usr/bin/systemctl stop neuron.service
|
||||
fi
|
||||
if [ "${pkg_changed}" -eq 1 ]; then
|
||||
if rpm -q "${pkg}" >/dev/null 2>&1; then
|
||||
sudo /usr/bin/dnf upgrade --refresh --allowerasing -y "${pkg}"
|
||||
else
|
||||
sudo /usr/bin/dnf install --refresh --allowerasing -y "${pkg}"
|
||||
fi
|
||||
fi
|
||||
# daemon-reload picks up both a new unit (dnf) and the drop-in.
|
||||
sudo /usr/bin/systemctl daemon-reload
|
||||
# enable --now: start the service AND enable it for boot so the
|
||||
# fleet self-heals after a host reboot.
|
||||
sudo /usr/bin/systemctl enable --now neuron.service
|
||||
|
||||
# ── Post-deploy validation ────────────────────────────────
|
||||
# A deploy only goes green if the neuron (a) finishes loading
|
||||
# its default models and (b) answers a trivial prompt like an
|
||||
# LLM should. Catches the class of bug where the binary
|
||||
# starts fine but model load or inference is broken — which
|
||||
# previously surfaced only when a human noticed. The wait
|
||||
# polls /health activation (the structured source of the
|
||||
# "loaded default model" journal line, plus per-model failure
|
||||
# detail); the journal-capture step below still runs for
|
||||
# forensics either way.
|
||||
load_timeout=${{ matrix.load_timeout }}
|
||||
echo "waiting for default models (timeout ${load_timeout}s)"
|
||||
deadline=$(( $(date +%s) + load_timeout ))
|
||||
health=""
|
||||
while :; do
|
||||
health=$(curl -fsS --max-time 5 http://localhost:13131/health 2>/dev/null || true)
|
||||
state=$(printf %s "${health}" | python3 -c '
|
||||
import json, sys
|
||||
try:
|
||||
print(json.load(sys.stdin).get("activation", {}).get("state", ""))
|
||||
except Exception:
|
||||
print("")
|
||||
')
|
||||
if [ "${state}" = "ready" ]; then
|
||||
break
|
||||
fi
|
||||
if [ "$(date +%s)" -ge "${deadline}" ]; then
|
||||
echo "FAIL: activation not ready within ${load_timeout}s (last state: ${state:-unreachable})"
|
||||
exit 1
|
||||
fi
|
||||
sleep 10
|
||||
done
|
||||
|
||||
model=$(printf %s "${health}" | python3 -c '
|
||||
import json, sys
|
||||
a = json.load(sys.stdin).get("activation", {})
|
||||
failed = a.get("failed", [])
|
||||
if failed:
|
||||
for f in failed:
|
||||
msg = "FAILED " + str(f.get("model_id")) + ": " + str(f.get("error", ""))[:400]
|
||||
sys.stderr.write(msg + chr(10))
|
||||
sys.exit(1)
|
||||
completed = a.get("completed", [])
|
||||
print(completed[0] if completed else "")
|
||||
')
|
||||
if [ -z "${model}" ]; then
|
||||
echo "no default models configured — skipping LLM probe"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "LLM probe against ${model}"
|
||||
probe_body=$(printf '{"model":"%s","messages":[{"role":"user","content":"Reply with exactly one word: pineapple"}],"max_tokens":512,"temperature":0}' "${model}")
|
||||
# The probe races real traffic: a deploy publishes a new build
|
||||
# SHA, which is exactly what triggers helexa-bench to re-sweep
|
||||
# every scenario against this neuron — long capability-probe
|
||||
# generations can hold the batch-1 model for minutes. Admission
|
||||
# control answers concurrent requests with 429/503 +
|
||||
# Retry-After (#53/#63); treat those as "busy, try again", not
|
||||
# deploy failure. Overall budget ~6 min.
|
||||
probe_deadline=$(( $(date +%s) + 360 ))
|
||||
attempt=0
|
||||
while :; do
|
||||
attempt=$(( attempt + 1 ))
|
||||
hdrs=$(mktemp)
|
||||
resp=$(curl -sS --max-time 180 -D "${hdrs}" \
|
||||
-H "content-type: application/json" \
|
||||
-d "${probe_body}" http://localhost:13131/v1/chat/completions) || resp=""
|
||||
status=$(awk 'toupper($1) ~ /^HTTP/ {code=$2} END {print code}' "${hdrs}")
|
||||
retry_after=$(awk 'tolower($1) == "retry-after:" {print $2+0; exit}' "${hdrs}")
|
||||
rm -f "${hdrs}"
|
||||
if [ "${status}" = "200" ]; then
|
||||
if printf %s "${resp}" | grep -qi pineapple; then
|
||||
echo "LLM probe passed (attempt ${attempt})"
|
||||
break
|
||||
fi
|
||||
echo "FAIL: probe response missing expected token"
|
||||
printf %s "${resp}" | head -c 2000
|
||||
echo
|
||||
exit 1
|
||||
fi
|
||||
case "${status}" in
|
||||
429|503) ;; # busy/queue-full — retryable
|
||||
*)
|
||||
echo "FAIL: probe request errored (HTTP ${status:-none})"
|
||||
printf %s "${resp}" | head -c 2000
|
||||
echo
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
if [ "$(date +%s)" -ge "${probe_deadline}" ]; then
|
||||
echo "FAIL: model still busy (HTTP ${status}) after probe budget"
|
||||
exit 1
|
||||
fi
|
||||
wait_s=${retry_after:-15}
|
||||
[ "${wait_s}" -ge 5 ] 2>/dev/null || wait_s=15
|
||||
echo "model busy (HTTP ${status}), retrying in ${wait_s}s (attempt ${attempt})"
|
||||
sleep "${wait_s}"
|
||||
done
|
||||
DEPLOY
|
||||
|
||||
- name: Ensure firewalld allows helexa-neuron
|
||||
run: |
|
||||
ssh gitea_ci@${{ matrix.host }} '
|
||||
if ! sudo /usr/bin/firewall-cmd --query-service=helexa-neuron --quiet 2>/dev/null; then
|
||||
sudo /usr/bin/firewall-cmd --add-service=helexa-neuron --permanent
|
||||
sudo /usr/bin/firewall-cmd --reload
|
||||
fi'
|
||||
|
||||
# Wait for the service to either come up or wedge, then capture
|
||||
# the journal for the invocation this run started. Runs even on
|
||||
# prior failure so a failed start step still leaves a usable
|
||||
# record in the deploy log. See deploy-cortex for the bounding
|
||||
# rationale — a neuron that has not restarted in a week is the
|
||||
# most exposed of the fleet to an unbounded -I.
|
||||
- name: Capture neuron.service startup journal
|
||||
if: always()
|
||||
env:
|
||||
PRE_INV: ${{ steps.pre.outputs.inv }}
|
||||
run: |
|
||||
sleep 10
|
||||
ssh gitea_ci@${{ matrix.host }} bash -s "${PRE_INV}" <<'CAPTURE'
|
||||
set -eu
|
||||
pre=${1:-}
|
||||
unit=neuron.service
|
||||
now=$(systemctl show -P InvocationID "${unit}")
|
||||
if [ -n "${now}" ] && [ "${now}" = "${pre}" ]; then
|
||||
echo "${unit} not restarted this run (invocation ${now}) — nothing to capture"
|
||||
systemctl status --no-pager --lines=0 "${unit}" || true
|
||||
exit 0
|
||||
fi
|
||||
timeout 60 journalctl --unit "${unit}" ${now:+-I} \
|
||||
--priority=info --lines=500 --no-pager \
|
||||
|| { echo "(no info-level entries for this invocation, or capture timed out)"
|
||||
systemctl status --no-pager --lines=0 "${unit}" || true; }
|
||||
CAPTURE
|
||||
|
||||
# helexa-bench is a separate package on a separate host (bob), and it
|
||||
# only consumes the fleet's HTTP APIs — it has no deploy-ordering
|
||||
# dependency on cortex or the neurons (the sweep loop is version-aware
|
||||
# and picks up whatever each neuron reports whenever). So it runs
|
||||
# alongside the cortex→neurons chain rather than after it.
|
||||
# Roll helexa-router + helexa-upstream onto the federation service host
|
||||
# (gallumbits). Same manifest-gated dnf pattern as deploy-cortex; the
|
||||
# two services are independent daemons on one host, so one job handles
|
||||
# both. Configs are NOT synced here — they carry secrets and ride in
|
||||
# script/infra-setup.sh under the operator's own sudo.
|
||||
deploy-gallumbits:
|
||||
runs-on: fedora-43
|
||||
if: >-
|
||||
${{
|
||||
github.event_name == 'workflow_dispatch'
|
||||
|| github.event.workflow_run.conclusion == 'success'
|
||||
}}
|
||||
steps:
|
||||
- name: SSH init
|
||||
run: |
|
||||
mkdir -p ~/.ssh
|
||||
echo "${DEPLOY_KEY}" > ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
|
||||
gitea_ci@gallumbits.kosherinata.internal 'hostname -f'
|
||||
|
||||
# See deploy-cortex for why the capture step needs this. One
|
||||
# `unit=invocation` token per service; ssh flattens the list into
|
||||
# separate argv entries on the far side.
|
||||
- name: Record service invocations
|
||||
id: pre
|
||||
run: |
|
||||
echo "inv=$(ssh gitea_ci@gallumbits.kosherinata.internal '
|
||||
for u in helexa-router helexa-upstream helexa-tools helexa-angels; do
|
||||
printf "%s=%s " "${u}" "$(systemctl show -P InvocationID ${u}.service)"
|
||||
done' || true)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Deploy helexa-router + helexa-upstream + helexa-tools + helexa-angels (skips when already current)
|
||||
run: |
|
||||
ssh gitea_ci@gallumbits.kosherinata.internal 'bash -s' <<'DEPLOY'
|
||||
set -eu
|
||||
for pkg in helexa-router helexa-upstream helexa-tools helexa-angels; do
|
||||
# `rpm -q` prints "package X is not installed" on STDOUT and
|
||||
# exits 1, so `|| echo not-installed` never fires and the
|
||||
# sentinel below would never match. Test the exit code.
|
||||
if installed=$(rpm -q --qf '%{VERSION}-%{RELEASE}' "${pkg}" 2>/dev/null); then :; else
|
||||
installed="not-installed"
|
||||
fi
|
||||
latest=$(curl -fsS --max-time 15 "https://rpm.lair.cafe/fedora/43/x86_64/unstable/packages.json" 2>/dev/null \
|
||||
| python3 -c '
|
||||
import json, sys
|
||||
name = sys.argv[1]
|
||||
cands = [p for p in json.load(sys.stdin)["packages"] if p.get("name") == name]
|
||||
if cands:
|
||||
p = max(cands, key=lambda p: p.get("buildTime", 0))
|
||||
print(p["version"] + "-" + p["release"])
|
||||
' "${pkg}" 2>/dev/null || true)
|
||||
if [ -n "${latest}" ] && [ "${latest}" = "${installed}" ]; then
|
||||
echo "${pkg}-${installed} already current — leaving service untouched"
|
||||
continue
|
||||
fi
|
||||
# A package that has never been published AND is not installed
|
||||
# cannot be deployed — most often a new service whose first RPM
|
||||
# has not landed yet. Skip it rather than letting `dnf install`
|
||||
# fail: this loop runs under `set -e`, so one unpublished
|
||||
# newcomer would otherwise abort the whole host's deploy and
|
||||
# strand every package after it in the list.
|
||||
if [ -z "${latest}" ] && [ "${installed}" = "not-installed" ]; then
|
||||
echo "${pkg}: not published yet and not installed — skipping"
|
||||
continue
|
||||
fi
|
||||
echo "${pkg}: installed=${installed} published=${latest:-unknown} — deploying"
|
||||
if systemctl is-active --quiet "${pkg}.service"; then
|
||||
sudo /usr/bin/systemctl stop "${pkg}.service"
|
||||
fi
|
||||
if rpm -q "${pkg}" >/dev/null 2>&1; then
|
||||
sudo /usr/bin/dnf upgrade --refresh --allowerasing -y "${pkg}"
|
||||
else
|
||||
sudo /usr/bin/dnf install --refresh --allowerasing -y "${pkg}"
|
||||
fi
|
||||
sudo /usr/bin/systemctl daemon-reload
|
||||
sudo /usr/bin/systemctl enable --now "${pkg}.service"
|
||||
done
|
||||
DEPLOY
|
||||
|
||||
- name: Ensure firewalld allows helexa-router + helexa-upstream + helexa-tools + helexa-angels
|
||||
run: |
|
||||
ssh gitea_ci@gallumbits.kosherinata.internal '
|
||||
changed=0
|
||||
for svc in helexa-router helexa-upstream helexa-tools helexa-angels; do
|
||||
if ! sudo /usr/bin/firewall-cmd --query-service=${svc} --quiet 2>/dev/null; then
|
||||
sudo /usr/bin/firewall-cmd --add-service=${svc} --permanent
|
||||
changed=1
|
||||
fi
|
||||
done
|
||||
[ "$changed" = "1" ] && sudo /usr/bin/firewall-cmd --reload || true
|
||||
'
|
||||
|
||||
# Per-unit: capture only the invocation this run started, bounded.
|
||||
# See deploy-cortex for the rationale.
|
||||
- name: Capture service startup journals
|
||||
if: always()
|
||||
env:
|
||||
PRE_INV: ${{ steps.pre.outputs.inv }}
|
||||
run: |
|
||||
sleep 10
|
||||
ssh gitea_ci@gallumbits.kosherinata.internal bash -s "${PRE_INV}" <<'CAPTURE'
|
||||
set -eu
|
||||
for spec in "$@"; do
|
||||
u=${spec%%=*}
|
||||
pre=${spec#*=}
|
||||
unit=${u}.service
|
||||
echo "── ${unit} ──"
|
||||
now=$(systemctl show -P InvocationID "${unit}")
|
||||
if [ -n "${now}" ] && [ "${now}" = "${pre}" ]; then
|
||||
echo "${unit} not restarted this run (invocation ${now}) — nothing to capture"
|
||||
systemctl status --no-pager --lines=0 "${unit}" || true
|
||||
continue
|
||||
fi
|
||||
timeout 60 journalctl --unit "${unit}" ${now:+-I} \
|
||||
--priority=info --lines=500 --no-pager \
|
||||
|| { echo "(no info-level entries for this invocation, or capture timed out)"
|
||||
systemctl status --no-pager --lines=0 "${unit}" || true; }
|
||||
done
|
||||
CAPTURE
|
||||
|
||||
deploy-bench:
|
||||
runs-on: fedora-43
|
||||
if: >-
|
||||
${{
|
||||
github.event_name == 'workflow_dispatch'
|
||||
|| github.event.workflow_run.conclusion == 'success'
|
||||
}}
|
||||
steps:
|
||||
- name: SSH init
|
||||
run: |
|
||||
mkdir -p ~/.ssh
|
||||
echo "${DEPLOY_KEY}" > ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
|
||||
gitea_ci@bob.hanzalova.internal 'hostname -f'
|
||||
|
||||
# See deploy-cortex for why the capture step needs this.
|
||||
- name: Record helexa-bench.service invocation
|
||||
id: pre
|
||||
run: |
|
||||
echo "inv=$(ssh gitea_ci@bob.hanzalova.internal \
|
||||
'systemctl show -P InvocationID helexa-bench.service' || true)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# See deploy-cortex for why gating uses the publish manifest and
|
||||
# not unprivileged `dnf check-update`.
|
||||
- name: Deploy helexa-bench (skips when already current)
|
||||
run: |
|
||||
ssh gitea_ci@bob.hanzalova.internal 'bash -s' <<'DEPLOY'
|
||||
set -eu
|
||||
pkg=helexa-bench
|
||||
# `rpm -q` prints "package X is not installed" on STDOUT and exits
|
||||
# 1, so `|| echo not-installed` never fires — it captures the
|
||||
# message text instead. Test the exit code.
|
||||
if installed=$(rpm -q --qf '%{VERSION}-%{RELEASE}' "${pkg}" 2>/dev/null); then :; else
|
||||
installed="not-installed"
|
||||
fi
|
||||
latest=$(curl -fsS --max-time 15 "https://rpm.lair.cafe/fedora/43/x86_64/unstable/packages.json" 2>/dev/null \
|
||||
| python3 -c '
|
||||
import json, sys
|
||||
name = sys.argv[1]
|
||||
cands = [p for p in json.load(sys.stdin)["packages"] if p.get("name") == name]
|
||||
if cands:
|
||||
p = max(cands, key=lambda p: p.get("buildTime", 0))
|
||||
print(p["version"] + "-" + p["release"])
|
||||
' "${pkg}" 2>/dev/null || true)
|
||||
if [ -n "${latest}" ] && [ "${latest}" = "${installed}" ]; then
|
||||
echo "${pkg}-${installed} already current — leaving service untouched"
|
||||
exit 0
|
||||
fi
|
||||
echo "installed=${installed} published=${latest:-unknown} — deploying"
|
||||
if systemctl is-active --quiet helexa-bench.service; then
|
||||
sudo /usr/bin/systemctl stop helexa-bench.service
|
||||
fi
|
||||
if rpm -q "${pkg}" >/dev/null 2>&1; then
|
||||
sudo /usr/bin/dnf upgrade --refresh --allowerasing -y helexa-bench
|
||||
else
|
||||
sudo /usr/bin/dnf install --refresh --allowerasing -y helexa-bench
|
||||
fi
|
||||
sudo /usr/bin/systemctl daemon-reload
|
||||
# enable --now: start the service AND enable it for boot so the
|
||||
# bench resumes collecting after a host reboot.
|
||||
sudo /usr/bin/systemctl enable --now helexa-bench.service
|
||||
|
||||
# ── Post-deploy validation ────────────────────────────────
|
||||
# The bench serves a read-only API on :13132 alongside the
|
||||
# outbound sweep loop. Probe the API over localhost (bypasses
|
||||
# firewalld) — catches a crash-on-start or a bad bind. Bail
|
||||
# early if the unit drops out of active (Restart backoff).
|
||||
echo "waiting for bench API on :13132"
|
||||
deadline=$(( $(date +%s) + 30 ))
|
||||
while :; do
|
||||
if curl -fsS --max-time 5 http://localhost:13132/api/health >/dev/null 2>&1; then
|
||||
echo "bench API healthy"
|
||||
break
|
||||
fi
|
||||
if ! systemctl is-active --quiet helexa-bench.service; then
|
||||
echo "FAIL: helexa-bench.service is not active"
|
||||
systemctl --no-pager status helexa-bench.service | head -20 || true
|
||||
exit 1
|
||||
fi
|
||||
if [ "$(date +%s)" -ge "${deadline}" ]; then
|
||||
echo "FAIL: bench API not healthy within 30s"
|
||||
exit 1
|
||||
fi
|
||||
sleep 3
|
||||
done
|
||||
DEPLOY
|
||||
|
||||
- name: Ensure firewalld allows helexa-bench
|
||||
run: |
|
||||
ssh gitea_ci@bob.hanzalova.internal '
|
||||
if ! sudo /usr/bin/firewall-cmd --query-service=helexa-bench --quiet 2>/dev/null; then
|
||||
sudo /usr/bin/firewall-cmd --add-service=helexa-bench --permanent
|
||||
sudo /usr/bin/firewall-cmd --reload
|
||||
fi'
|
||||
|
||||
# Wait for the service to either come up or wedge, then capture
|
||||
# the journal for the invocation this run started. Runs even on
|
||||
# prior failure so a failed start step still leaves a usable
|
||||
# record in the deploy log. See deploy-cortex for the bounding
|
||||
# rationale.
|
||||
- name: Capture helexa-bench.service startup journal
|
||||
if: always()
|
||||
env:
|
||||
PRE_INV: ${{ steps.pre.outputs.inv }}
|
||||
run: |
|
||||
sleep 10
|
||||
ssh gitea_ci@bob.hanzalova.internal bash -s "${PRE_INV}" <<'CAPTURE'
|
||||
set -eu
|
||||
pre=${1:-}
|
||||
unit=helexa-bench.service
|
||||
now=$(systemctl show -P InvocationID "${unit}")
|
||||
if [ -n "${now}" ] && [ "${now}" = "${pre}" ]; then
|
||||
echo "${unit} not restarted this run (invocation ${now}) — nothing to capture"
|
||||
systemctl status --no-pager --lines=0 "${unit}" || true
|
||||
exit 0
|
||||
fi
|
||||
timeout 60 journalctl --unit "${unit}" ${now:+-I} \
|
||||
--priority=info --lines=500 --no-pager \
|
||||
|| { echo "(no info-level entries for this invocation, or capture timed out)"
|
||||
systemctl status --no-pager --lines=0 "${unit}" || true; }
|
||||
CAPTURE
|
||||
|
||||
# Build the bench UI and publish it to the public nginx vhost on the
|
||||
# gateway (https://bench.helexa.ai). The vhost + Let's Encrypt cert are
|
||||
# one-time host setup (script/infra-setup.sh); this job just refreshes
|
||||
# the static assets. nginx reverse-proxies /api to the bob API, so the
|
||||
# SPA is built same-origin (no VITE_API_BASE). Independent of the other
|
||||
# deploy jobs.
|
||||
deploy-bench-ui:
|
||||
runs-on: fedora-43
|
||||
if: >-
|
||||
${{
|
||||
github.event_name == 'workflow_dispatch'
|
||||
|| github.event.workflow_run.conclusion == 'success'
|
||||
}}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "20"
|
||||
|
||||
- name: Build UI
|
||||
run: |
|
||||
cd bench
|
||||
npm ci
|
||||
npm run build
|
||||
|
||||
- name: SSH init
|
||||
run: |
|
||||
mkdir -p ~/.ssh
|
||||
echo "${DEPLOY_KEY}" > ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
|
||||
gitea_ci@hanzalova.internal 'hostname -f'
|
||||
|
||||
- name: Rsync built UI to gateway webroot
|
||||
run: |
|
||||
rsync --archive --compress --delete \
|
||||
--rsync-path 'sudo rsync' \
|
||||
bench/dist/ \
|
||||
gitea_ci@hanzalova.internal:/var/www/bench.helexa.ai/
|
||||
|
||||
# Build the helexa.ai website and publish it to the public nginx
|
||||
# vhosts on BOTH edge proxies (oolon.kosherinata.internal for the DC
|
||||
# site, hanzalova.internal for the office site). Cloudflare DNS load
|
||||
# balancing routes helexa.ai to the two site WAN IPs; mesh clients use
|
||||
# https://helexa.internal (hanzalova only). The vhosts + certs are
|
||||
# one-time host setup (script/infra-setup.sh); this job just
|
||||
# refreshes the static assets. Independent of the other deploy jobs.
|
||||
deploy-website:
|
||||
runs-on: fedora-43
|
||||
if: >-
|
||||
${{
|
||||
github.event_name == 'workflow_dispatch'
|
||||
|| github.event.workflow_run.conclusion == 'success'
|
||||
}}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "20"
|
||||
|
||||
- name: Build website
|
||||
run: |
|
||||
cd helexa.ai
|
||||
npm ci
|
||||
npm run i18n:check
|
||||
npm run build
|
||||
|
||||
- name: SSH init
|
||||
run: |
|
||||
mkdir -p ~/.ssh
|
||||
echo "${DEPLOY_KEY}" > ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
for host in hanzalova.internal oolon.kosherinata.internal; do
|
||||
ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
|
||||
"gitea_ci@${host}" 'hostname -f'
|
||||
done
|
||||
|
||||
# Both proxies serve the same build; a failed rsync to one host
|
||||
# fails the job (Cloudflare LB health checks paper over a stale
|
||||
# instance, but the job exit code should still tell the truth).
|
||||
- name: Rsync built site to edge webroots
|
||||
run: |
|
||||
for host in hanzalova.internal oolon.kosherinata.internal; do
|
||||
rsync --archive --compress --delete \
|
||||
--rsync-path 'sudo rsync' \
|
||||
helexa.ai/dist/ \
|
||||
"gitea_ci@${host}:/var/www/helexa.ai/"
|
||||
done
|
||||
11
.gitignore
vendored
11
.gitignore
vendored
@@ -1,18 +1,7 @@
|
||||
/target
|
||||
/bench/node_modules
|
||||
/bench/dist
|
||||
/helexa.ai/node_modules
|
||||
/helexa.ai/dist
|
||||
helexa.ai/.env.local
|
||||
*.swp
|
||||
*.swo
|
||||
.idea/
|
||||
.vscode/
|
||||
cortex.toml
|
||||
models.toml
|
||||
doc/plan/*
|
||||
/target-cuda/
|
||||
.claude/
|
||||
helexa-router.toml
|
||||
helexa-upstream.toml
|
||||
helexa-angels.toml
|
||||
|
||||
269
AGENTS.md
269
AGENTS.md
@@ -1,269 +0,0 @@
|
||||
# AGENTS.md — helexa/cortex
|
||||
|
||||
## Project Overview
|
||||
|
||||
helexa is a self-hosted LLM serving stack for multi-node GPU inference clusters. It has two components:
|
||||
|
||||
- **cortex** — the per-operator control plane and LLM proxy. A Rust reverse-proxy that sits in front of the fleet and presents a unified OpenAI + Anthropic compatible API surface. It handles model routing, lifecycle management (load/unload/evict), request translation, and metrics collection.
|
||||
- **neuron** — the per-host LLM harness. One instance runs on every GPU host, serving candle-based in-process inference and managing local hardware discovery and model lifecycle.
|
||||
|
||||
## Repository Layout
|
||||
|
||||
```
|
||||
cortex/
|
||||
├── Cargo.toml # workspace root (Rust 2024 edition, GPL-3.0)
|
||||
├── cortex.example.toml # example gateway config
|
||||
├── models.example.toml # example model catalogue
|
||||
├── neuron.example.toml # example neuron config
|
||||
├── README.md # public-facing documentation
|
||||
├── CLAUDE.md # detailed design rationale and implementation history
|
||||
├── AGENTS.md # ← you are here
|
||||
├── cortex.spec # RPM spec for cortex
|
||||
├── helexa-neuron.spec # RPM spec for neuron (renamed to avoid Fedora collision)
|
||||
├── rpm/ # prerelease RPM specs
|
||||
│ ├── cortex-prerelease.spec
|
||||
│ ├── helexa-neuron-prerelease.spec
|
||||
│ └── helexa-bench-prerelease.spec
|
||||
├── data/ # systemd units and example configs for packaging
|
||||
│ ├── cortex.service
|
||||
│ ├── neuron.service
|
||||
│ ├── cortex.example.toml
|
||||
│ ├── neuron.example.toml
|
||||
│ └── models.example.toml
|
||||
└── crates/
|
||||
├── cortex-core/ # shared types, config, envelopes
|
||||
│ └── src/
|
||||
│ ├── lib.rs
|
||||
│ ├── build_info.rs # BuildInfo type for /version endpoint
|
||||
│ ├── config.rs # figment-based config structs
|
||||
│ ├── catalogue.rs # ModelProfile, placement matching
|
||||
│ ├── discovery.rs # DeviceInfo, DiscoveryResponse
|
||||
│ ├── harness.rs # Harness trait, HarnessConfig, HarnessHealth
|
||||
│ ├── node.rs # NodeState, ModelStatus
|
||||
│ ├── openai.rs # OpenAI request/response types
|
||||
│ ├── anthropic.rs # Anthropic request/response types
|
||||
│ ├── translate.rs # OpenAI <-> Anthropic translation
|
||||
│ └── metrics.rs # RequestMetrics, histogram helpers
|
||||
├── cortex-gateway/ # the HTTP proxy server
|
||||
│ └── src/
|
||||
│ ├── lib.rs
|
||||
│ ├── state.rs # CortexState: Arc<RwLock<...>>
|
||||
│ ├── router.rs # model -> node routing logic
|
||||
│ ├── proxy.rs # streaming HTTP proxy to backends
|
||||
│ ├── evictor.rs # LRU/priority eviction logic
|
||||
│ ├── poller.rs # background task polling neuron status
|
||||
│ ├── handlers.rs # axum handlers (chat, completions, models, etc.)
|
||||
│ └── metrics.rs # prometheus exporter endpoint
|
||||
├── cortex-cli/ # CLI entrypoint
|
||||
│ └── src/main.rs # binary: `cortex`
|
||||
├── neuron/ # per-host LLM daemon (replaces cortex-agent)
|
||||
│ ├── Cargo.toml # features: cuda, cudnn, flash-attn, cuda-integration
|
||||
│ ├── build.rs # compiles CUDA kernels, emits build metadata
|
||||
│ └── src/
|
||||
│ ├── main.rs # binary: `neuron`
|
||||
│ ├── discovery.rs # nvidia-smi parsing, device enumeration
|
||||
│ ├── health.rs # runtime GPU polling
|
||||
│ ├── api.rs # HTTP handlers for /discovery, /models, etc.
|
||||
│ ├── version.rs # GET /version endpoint with BuildInfo
|
||||
│ ├── models.rs # local model lifecycle orchestration
|
||||
│ └── harness/ # in-process candle inference
|
||||
│ ├── device_worker/ # per-device CUDA worker threads
|
||||
│ │ ├── mod.rs # canonical narrative for worker architecture
|
||||
│ │ ├── jobs.rs # Job enum, dispatch handlers
|
||||
│ │ └── dispatch.rs # DeviceWorkerState struct
|
||||
│ ├── candle.rs # candle model implementation
|
||||
│ └── tp/ # tensor parallelism
|
||||
│ └── worker.rs # TP worker subprocesses
|
||||
├── helexa-acp/ # Agent Client Protocol bridge (Apache-2.0)
|
||||
│ └── src/main.rs # binary: `helexa-acp`, self-contained (no workspace deps)
|
||||
└── helexa-bench/ # benchmark harness
|
||||
└── src/main.rs # binary: `helexa-bench`, SQLite-backed, version-aware
|
||||
```
|
||||
|
||||
## Key Design Decisions
|
||||
|
||||
### Architecture
|
||||
- **cortex** is the control plane. It exposes the unified API, routes requests, manages model lifecycle across the fleet, and collects metrics.
|
||||
- **neuron** is the node plane. One instance runs on every GPU host. It discovers local hardware, manages in-process candle inference, handles NCCL tensor parallelism, and reports runtime state.
|
||||
- cortex never shells out to `nvidia-smi`, never touches systemd units, and never talks directly to a harness. It talks only to neurons via HTTP API on port 13131.
|
||||
|
||||
### Per-device worker thread (neuron)
|
||||
Every CUDA device gets one dedicated OS thread that owns its `CudaContext` for the daemon's lifetime. All CUDA operations route through this thread via a `std::sync::mpsc` job channel. Tensors never escape the worker thread alive. Inference replies carry `Vec<f32>` CPU-side logits; sampled tokens come back as `u32`. The opaque `ArchHandle(u64)` and `TpHandle(u64)` are indices into the worker's state slab, not pointers.
|
||||
|
||||
CPU loads (`Device::Cpu` fallback) keep the legacy `tokio::task::spawn_blocking + Arc<Mutex<ModelArch>>` path — there's no context to own and the channel hop would only add latency. Four `spawn_blocking` references in `harness/candle.rs` are deliberate CPU fallback.
|
||||
|
||||
### candle-native (not mistral.rs)
|
||||
neuron builds directly on [candle](https://github.com/huggingface/candle). Every model architecture it serves is implemented in this repository, ported against the HuggingFace reference. No external inference server to babysit. The Harness trait remains as an internal seam for adding future engines (vision/audio/diffusion) but its only implementation is in-process candle.
|
||||
|
||||
### Streaming proxy
|
||||
Chat completions are proxied as SSE streams. The gateway must:
|
||||
1. Parse the inbound request to extract the model name
|
||||
2. Route to the correct backend neuron
|
||||
3. Stream the response back, capturing token timing for metrics
|
||||
4. NOT buffer the full response — true streaming passthrough
|
||||
|
||||
### Anthropic translation
|
||||
When a request arrives at `/v1/messages` (Anthropic format), the gateway translates it to OpenAI format before proxying to neuron, then translates the response back. This is stateless envelope transformation. Non-streaming round-trip is implemented; streaming SSE translation deferred.
|
||||
|
||||
### Eviction
|
||||
The evictor runs as a background task. Before loading a model on a node where VRAM is tight:
|
||||
1. Check if the model is already loaded elsewhere → route there instead
|
||||
2. Find the LRU model on the target node (excluding pinned models)
|
||||
3. Call `POST {neuron}/models/unload` on that model
|
||||
4. The incoming request's lazy-load triggers the new model load
|
||||
|
||||
### Metrics
|
||||
Per-request: model, node, prompt_tokens, completion_tokens, total_tokens, tok_per_sec, time_to_first_token_ms, total_latency_ms. Exposed as Prometheus histograms/counters on a separate port (31314).
|
||||
|
||||
## Tech Stack
|
||||
|
||||
- **Rust 2024 edition** — workspace with 6 crates
|
||||
- **Axum 0.8** — HTTP framework
|
||||
- **reqwest** — HTTP client for proxying to backends
|
||||
- **figment** — config loading (TOML + env vars)
|
||||
- **tokio** — async runtime
|
||||
- **metrics + metrics-exporter-prometheus** — observability
|
||||
- **tracing** — structured logging
|
||||
- **candle** — in-process inference engine (neuron only, with CUDA support)
|
||||
- **cudarc** — patched for neuron's needs (see workspace `[patch]`)
|
||||
- **clap** — CLI parsing
|
||||
- **rusqlite** (bundled) — helexa-bench SQLite system-of-record
|
||||
|
||||
## Build Commands
|
||||
|
||||
```sh
|
||||
cargo build --release # build all crates
|
||||
cargo run -p cortex-cli -- serve # run the gateway
|
||||
cargo test # run all tests
|
||||
cargo clippy --workspace # lint
|
||||
```
|
||||
|
||||
### neuron Features
|
||||
- `cuda`: Enables CUDA acceleration in candle and cudarc/nccl bindings. Without it, falls back to CPU.
|
||||
- `cudnn`: Use cuDNN for convolution/attention kernels (requires `cuda`).
|
||||
- `flash-attn`: FlashAttention kernels (requires `cuda`).
|
||||
- `cuda-integration`: Reserved for GPU-only integration tests (requires multiple CUDA devices + libnccl).
|
||||
|
||||
### Build Scripts
|
||||
- `neuron/build.rs`: Compiles CUDA kernels (`src/cuda/*.cu`) using `cudaforge::KernelBuilder` when `cuda` feature is enabled. Handles compute capability checks (sm_<80 disables bf16 intrinsics). Also captures build metadata: git SHA, dirty flag, timestamp, rustc version, profile, features, candle-core version.
|
||||
|
||||
## CI
|
||||
|
||||
Gitea Actions runs on every push to any branch. All three checks must pass before merging:
|
||||
|
||||
```sh
|
||||
cargo fmt --check --all # formatting
|
||||
cargo clippy --workspace -- -D warnings # lint (warnings are errors)
|
||||
cargo test --workspace # tests
|
||||
```
|
||||
|
||||
Run these locally before pushing. `cargo fmt --all` fixes formatting automatically. Clippy warnings must be resolved, not suppressed with `#[allow(...)]` unless there is a clear rationale.
|
||||
|
||||
Tagged releases (`v*`) build SRPMs for `cortex`, `helexa-neuron`, and `helexa-bench` and publish to COPR (`helexa/helexa`). Build metadata SHA injection: CI sets `HELEXA_BUILD_SHA=$(git rev-parse HEAD)`.
|
||||
|
||||
## Environment
|
||||
|
||||
- Targets Fedora 43 (systemd, SELinux enforcing)
|
||||
- Nodes communicate over a private network (e.g. WireGuard mesh)
|
||||
- cortex listens on port 31313 (API) and 31314 (metrics)
|
||||
- neuron listens on port 13131 on each GPU host
|
||||
- TLS terminated at gateway or via nginx; internal traffic is plaintext over WireGuard
|
||||
|
||||
## Conventions
|
||||
|
||||
- Error handling: `anyhow` for binaries, `thiserror` for library crates
|
||||
- No `unwrap()` in library code; `expect()` only with clear rationale
|
||||
- All public types derive `Debug, Clone, Serialize, Deserialize` where sensible
|
||||
- Config structs use `figment` with TOML as primary source, env vars as override
|
||||
- Prefer `Arc<RwLock<...>>` for shared fleet state; minimize lock duration
|
||||
- SSE streaming uses `tokio_stream` + `eventsource-stream` for parsing
|
||||
- Log at `info` for request routing, `debug` for proxy details, `warn` for eviction and node health, `error` for proxy failures
|
||||
|
||||
## Testing
|
||||
|
||||
### Gateway tests
|
||||
Use mock neurons spawned via axum in `crates/cortex-gateway/tests/common/mod.rs`. Helpers: `spawn_mock_backend()`, `spawn_gateway()`.
|
||||
|
||||
### neuron integration tests
|
||||
- Numerical reference tests (`numerical_reference.rs`) require `NEURON_REF_MODEL_PATH` env var pointing to a HF snapshot directory. Fixtures are f32-based for precision validation against HuggingFace transformers.
|
||||
- CUDA integration tests (`tp_worker_lifecycle_cuda.rs`) gated behind `cuda-integration` feature; requires 2+ CUDA devices (e.g., 2x RTX 5090).
|
||||
|
||||
### Metrics testing
|
||||
Use `install_test_recorder()` in test code to capture metrics without the HTTP listener.
|
||||
|
||||
## helexa-bench
|
||||
|
||||
A continuous, version-aware benchmark harness. Hits each neuron directly on `:13131`, exercises each warm model with a Scenario suite (chat-latency family), and records results into SQLite stamped with the neuron's full `BuildInfo`. The loop is version-aware: skips any (target, build SHA, model, scenario) cell already at `samples_per_version`.
|
||||
|
||||
Packaged as `helexa-bench` RPM (prebuilt-binary spec). One systemd unit, typically on the metrics host.
|
||||
|
||||
## helexa-acp
|
||||
|
||||
Agent Client Protocol bridge — connects ACP editors (Zed, etc.) to any OpenAI-compatible endpoint, cortex by default. Intentionally self-contained: no workspace crate dependencies. Uses `agent-client-protocol` with `unstable_session_model` feature for Zed model picker support. Licensed Apache-2.0 (workspace is GPL-3.0).
|
||||
|
||||
## RPM Packaging
|
||||
|
||||
- `cortex.spec` — installs the `cortex` binary
|
||||
- `helexa-neuron.spec` — installs the `neuron` binary under package name `helexa-neuron` (renamed to avoid Fedora's NEURON neural-simulation package collision)
|
||||
- Systemd units in `data/cortex.service`, `data/neuron.service`
|
||||
- Example configs: `cortex.example.toml`, `neuron.example.toml`, `models.example.toml`
|
||||
|
||||
Install:
|
||||
```sh
|
||||
dnf copr enable helexa/helexa
|
||||
dnf install cortex # gateway host
|
||||
dnf install helexa-neuron # GPU nodes
|
||||
```
|
||||
|
||||
## Configuration Files
|
||||
|
||||
### cortex.toml (gateway)
|
||||
```toml
|
||||
[gateway]
|
||||
listen = "0.0.0.0:31313"
|
||||
metrics_listen = "0.0.0.0:31314"
|
||||
|
||||
[eviction]
|
||||
strategy = "lru" # lru | priority
|
||||
defrag_after_cycles = 50
|
||||
|
||||
[[neurons]]
|
||||
name = "beast"
|
||||
endpoint = "http://beast.internal:13131"
|
||||
```
|
||||
|
||||
### models.toml (catalogue)
|
||||
```toml
|
||||
[[models]]
|
||||
id = "Qwen/Qwen3-Coder-30B-A3B-Instruct"
|
||||
harness = "candle"
|
||||
quant = "Q4_K_M"
|
||||
vram_mb = 19000
|
||||
min_devices = 2
|
||||
min_device_vram_mb = 10000
|
||||
pinned_on = ["beast"] # optional: affinity — run only on these
|
||||
residency_priority = 300 # optional: who may displace whom under VRAM pressure
|
||||
```
|
||||
|
||||
### neuron.toml (per-host)
|
||||
Configured via figment + env override. See `neuron.example.toml` for reference.
|
||||
|
||||
## neuron API Endpoints
|
||||
|
||||
```
|
||||
GET /discovery → hardware discovery (hostname, OS, CUDA, devices, harnesses)
|
||||
GET /health → runtime GPU stats (VRAM, utilization, temperature)
|
||||
GET /models → loaded/unloaded models with VRAM usage
|
||||
POST /models/load → load a model with spec (quant, TP, devices)
|
||||
POST /models/unload → unload a model, freeing device memory
|
||||
GET /models/{id}/endpoint → inference URL for a model
|
||||
GET /version → build metadata (SHA, features, candle version, etc.)
|
||||
```
|
||||
|
||||
## Sources of Truth
|
||||
|
||||
When prose documentation conflicts with code, trust:
|
||||
1. Executable configuration (`*.toml`, `Cargo.toml` features)
|
||||
2. Type definitions in `cortex-core/`
|
||||
3. Test files in `crates/*/tests/` and `*/src/**/*_test.rs`
|
||||
4. `CLAUDE.md` for historical design rationale
|
||||
499
CLAUDE.md
499
CLAUDE.md
@@ -1,26 +1,16 @@
|
||||
# CLAUDE.md — helexa
|
||||
# CLAUDE.md — cortex
|
||||
|
||||
## Project overview
|
||||
|
||||
helexa is a self-hosted LLM serving stack for multi-node GPU inference
|
||||
clusters. It has two components:
|
||||
|
||||
- **cortex** — the per-operator control plane and LLM proxy. A Rust
|
||||
reverse-proxy that sits in front of the fleet and presents a unified
|
||||
OpenAI + Anthropic compatible API surface. It handles model routing,
|
||||
lifecycle management (load/unload/evict), request translation, and
|
||||
metrics collection.
|
||||
- **neuron** — the per-host LLM harness. One instance runs on every GPU
|
||||
host, serving candle-based in-process inference and managing local
|
||||
hardware discovery and model lifecycle.
|
||||
|
||||
(Historical note: cortex originally proxied to mistral.rs nodes; neuron
|
||||
replaced that — see the 2026-05-18 candle-native addendum below.)
|
||||
cortex is a Rust reverse-proxy that sits in front of multiple
|
||||
mistral.rs inference nodes and presents a unified OpenAI + Anthropic
|
||||
compatible API surface. It handles model routing, lifecycle management
|
||||
(load/unload/evict), request translation, and metrics collection.
|
||||
|
||||
## Repository layout
|
||||
|
||||
```
|
||||
helexa/
|
||||
cortex/
|
||||
├── Cargo.toml # workspace root
|
||||
├── cortex.toml # example gateway config
|
||||
├── README.md
|
||||
@@ -94,63 +84,6 @@ Per-request: model, node, prompt_tokens, completion_tokens, total_tokens,
|
||||
tok_per_sec, time_to_first_token_ms, total_latency_ms.
|
||||
Exposed as Prometheus histograms/counters on a separate port.
|
||||
|
||||
### Per-device worker thread (neuron)
|
||||
The neuron daemon dedicates one OS thread per CUDA device it loads
|
||||
onto. That thread binds the device's `CudaContext` once at startup and
|
||||
owns it for the daemon's lifetime; every model load, forward step,
|
||||
KV-cache reset, VRAM query, NCCL init/sanity, NCCL all_reduce, and
|
||||
model drop on that device routes through this thread via a
|
||||
`std::sync::mpsc` job channel. Replies cross back via
|
||||
`tokio::sync::oneshot`.
|
||||
|
||||
Three properties this gives us, in order of weight:
|
||||
|
||||
1. **Context locality.** cudarc binds the CUDA context per OS thread
|
||||
via `cuCtxSetCurrent`. Before this refactor, ad-hoc
|
||||
`tokio::task::spawn_blocking` calls bound the context onto a
|
||||
different thread per request — and `device_vram_mb()` from an
|
||||
async task bound it onto whichever tokio worker happened to be
|
||||
running. Pinning the context to one named thread ends that.
|
||||
2. **Drop safety.** Every `CudaSlice` in a `Tensor`, every
|
||||
`cudarc::nccl::Comm`, and the `CudaContext` itself call `cuMemFree` /
|
||||
`ncclCommDestroy` / `cuCtxDestroy` during `Drop` — and require the
|
||||
right context current. With the worker owning the model slab,
|
||||
`Drop` always runs on the right thread. The cudarc Drop constraint
|
||||
is structurally enforced.
|
||||
3. **Poisoning blast radius.** When a CUDA driver error makes the
|
||||
context unrecoverable, the poison flag lives on the
|
||||
`DeviceWorkerHandle` itself. Subsequent `submit()` calls fast-reject
|
||||
at the channel boundary with a clear "device worker is poisoned"
|
||||
error before any further CUDA work is attempted. The thread doesn't
|
||||
exit (dropping the slab would re-touch the broken context) — it
|
||||
enters a drain-only mode and replies error to everything until the
|
||||
daemon restarts.
|
||||
|
||||
Tensors never escape the worker thread alive. Inference replies carry
|
||||
`Vec<f32>` CPU-side logits; the async caller wraps them in a CPU
|
||||
candle tensor and runs `apply_repeat_penalty` + `LogitsProcessor::sample`
|
||||
without ever rebinding the device context. Sampled tokens come back as
|
||||
`u32`; VRAM queries as `(u64, u64)`. The opaque `ArchHandle(u64)` and
|
||||
`TpHandle(u64)` are the only "references" callers hold to loaded
|
||||
models — they're indices into the worker's state slab, not pointers.
|
||||
|
||||
The TP worker subprocesses in `harness/tp/worker.rs` are the same
|
||||
pattern out-of-process — a dedicated context-owning process per
|
||||
non-zero NCCL rank. The in-process worker in `harness/device_worker/`
|
||||
brings the discipline to rank 0.
|
||||
|
||||
CPU loads (`Device::Cpu` fallback when CUDA is unavailable) keep the
|
||||
legacy `tokio::task::spawn_blocking + Arc<Mutex<ModelArch>>` path —
|
||||
there's no context to own and the channel hop would only add latency.
|
||||
Four `spawn_blocking` references in `harness/candle.rs` are deliberate
|
||||
CPU fallback.
|
||||
|
||||
Canonical narrative lives in
|
||||
`crates/neuron/src/harness/device_worker/mod.rs`'s module
|
||||
doc-comment; touch points (the `Job` enum, the dispatch handlers, the
|
||||
`DeviceWorkerState` struct) are in the sibling `jobs.rs` and
|
||||
`dispatch.rs`.
|
||||
|
||||
## Tech stack
|
||||
|
||||
- **Rust 2024 edition** — workspace with 4 crates
|
||||
@@ -185,95 +118,6 @@ Run these locally before pushing. `cargo fmt --all` fixes formatting
|
||||
automatically. Clippy warnings must be resolved, not suppressed with
|
||||
`#[allow(...)]` unless there is a clear rationale.
|
||||
|
||||
## Development workflow
|
||||
|
||||
Work each change on its own branch; `main` stays releasable.
|
||||
|
||||
1. Implement on a feature branch (`fix/<issue>-…`, `feat/<issue>-…`).
|
||||
2. Run the CI triad locally (`cargo fmt --check --all`,
|
||||
`cargo clippy --workspace -- -D warnings`, `cargo test --workspace`).
|
||||
Local builds are **CPU-only** — the `#[cfg(feature = "cuda")]` neuron/TP
|
||||
paths do NOT compile locally. The branch CI's **CUDA type-check** job is
|
||||
the only thing that validates them, so for any neuron change the push to
|
||||
Gitea is the real gate, not a rubber stamp.
|
||||
3. Push the branch on local-green (no need to ask first), and background-watch
|
||||
its CI run via the gitea-mcp `actions_run_read` tools. Start the next piece
|
||||
of work meanwhile.
|
||||
4. Merge to `main` when the four **validation** jobs are green — Format,
|
||||
Clippy, Test, CUDA type-check. The SRPM / COPR / version-bump jobs are the
|
||||
deploy pipeline (they run on `main`), not validation — don't wait on them.
|
||||
5. Merging/pushing to `main` triggers the auto-deploy pipeline.
|
||||
|
||||
Docs-only changes (no `#[cfg(feature = "cuda")]` impact) can go straight to
|
||||
`main` — there's nothing for the CUDA type-check to prove.
|
||||
|
||||
## Epic execution playbook
|
||||
|
||||
Distilled from the #197 text-to-image epic (2026-07-29/30): seven
|
||||
children — an engine integration, two upstream candle fixes, a new API
|
||||
surface across neuron + cortex + router, metering, fleet rollout, and
|
||||
quantization — planned, implemented, live-validated, and closed in one
|
||||
multi-hour session with no operator input beyond one up-front decision
|
||||
round. This is the template for large, unblocked, well-planned epics;
|
||||
future sessions should reproduce it deliberately.
|
||||
|
||||
1. **De-risk before planning.** Run the throwaway spike on real
|
||||
hardware first. Hand-driving the candle example on beast produced
|
||||
the VRAM/latency numbers and the three findings that became the
|
||||
port spec — the epic was written from evidence, and every design
|
||||
claim in it had already been observed. An epic whose engine risk is
|
||||
retired before filing is an epic that can be finished.
|
||||
2. **Front-load operator decisions, then don't come back.** Identify
|
||||
the genuinely operator-owned decision surfaces (for #197: metering
|
||||
unit, v1 placement, public exposure, optional-scope selection) and
|
||||
settle them in ONE question round before implementation starts.
|
||||
Record the answers as issue comments so they survive context loss.
|
||||
Everything else is implementation detail the session owns.
|
||||
3. **Decisions are goals, not designs.** When live evidence
|
||||
contradicts the planned design, change the design autonomously,
|
||||
keep the decided goal fixed, and record the divergence + rationale
|
||||
on the issue. #197 examples: the text encoder moved to host CPU
|
||||
when the 24 GB tier couldn't hold it beside the DiT; tiled VAE
|
||||
decode was pulled forward a stage when 1024² OOMed on the actual
|
||||
deployment card; allocator-pool trims were added when mixed-size
|
||||
request sequences fragmented into OOMs. The operator decided
|
||||
*benjy cold-swap*, not *how the bytes fit*.
|
||||
4. **Branch-per-child, stack when dependent, never wait idle.** Push
|
||||
on local green, let branch CI (especially the CUDA type-check)
|
||||
validate in the background, and start the next child meanwhile.
|
||||
Merge in dependency order as runs green. CI wall-clock is the
|
||||
session's biggest fixed cost — always have a second workstream
|
||||
(the next child, fleet prep, issue bookkeeping) to overlap it.
|
||||
5. **Validate live at every stage, with guaranteed restore.**
|
||||
Production hosts are the test bench, used through guarded windows:
|
||||
check `in_flight == 0`, evict, test, restore, verify — as one
|
||||
scripted sequence, so a dropped ssh session can't leave the fleet
|
||||
degraded. Every stage's acceptance is an observable behaviour on
|
||||
real hardware; unit tests gate merges, live probes gate closure.
|
||||
6. **The definition of done is external.** #197 closed only when an
|
||||
unmodified OpenAI client got a metered image through the public
|
||||
chain, `script/validate-image.sh` passed against the *deployed RPM
|
||||
build* (not a dev binary), and the cold-swap ran both directions.
|
||||
`/version` probes confirm what is actually live; "merged" is not
|
||||
"done".
|
||||
7. **Write everything down as you go.** Issues get closing comments
|
||||
with real numbers; surprises become documented gotchas the same
|
||||
hour they bite; session memory is checkpointed at each phase
|
||||
boundary so a context break loses nothing. The issue trail should
|
||||
let a cold reader reconstruct both what happened and why.
|
||||
8. **Fix the environment, not just the code.** One-shotting means
|
||||
owning the whole path: rebasing the cudarc fork to unblock a candle
|
||||
bump, forking candle for upstream defects (and filing the upstream
|
||||
PR so the pin is temporary), re-dispatching a flaky CI run, syncing
|
||||
27 GB of weights across the mesh, force-updating a host. Blockers
|
||||
are work items, not stop signs — but production-touching steps get
|
||||
the same guarded-window discipline as tests, and destructive or
|
||||
scope-changing moves still go back to the operator.
|
||||
|
||||
SSH note: the gitea remote host offers multiple agent keys and cuts the
|
||||
connection before reaching the right one. This repo pins the working key via
|
||||
`git config core.sshCommand "ssh -i ~/.ssh/id_grenade -o IdentitiesOnly=yes"`.
|
||||
|
||||
## Environment
|
||||
|
||||
- Targets Fedora 43 (systemd, SELinux enforcing)
|
||||
@@ -620,8 +464,7 @@ quant = "Q4_K_M"
|
||||
vram_mb = 19000
|
||||
min_devices = 2
|
||||
min_device_vram_mb = 10000
|
||||
pinned_on = ["beast"] # optional: affinity — run only on these
|
||||
residency_priority = 300 # optional: who may displace whom under VRAM pressure
|
||||
pinned_on = ["beast"] # optional: never evict from these neurons
|
||||
|
||||
[[models]]
|
||||
id = "Qwen/Qwen3-VL-8B"
|
||||
@@ -648,7 +491,7 @@ and the hardcoded `vram_mb` per node.
|
||||
## Revised repository layout
|
||||
|
||||
```
|
||||
helexa/
|
||||
cortex/
|
||||
├── Cargo.toml
|
||||
├── cortex.toml # gateway config (neurons only)
|
||||
├── models.toml # model catalogue
|
||||
@@ -773,290 +616,58 @@ dnf install cortex # gateway host
|
||||
dnf install helexa-neuron # GPU nodes
|
||||
```
|
||||
|
||||
## 2026-05-18 addendum: candle-native pivot
|
||||
### Phase 11: llama.cpp harness stub
|
||||
|
||||
Phases 11 (llama.cpp harness) and 12 (mistral.rs COPR) below are
|
||||
**superseded**. The project no longer treats mistral.rs or llama.cpp as
|
||||
dependencies — both are conceptually out of scope. neuron becomes a
|
||||
candle-native inference daemon, with `Harness` retained as an
|
||||
internal seam for adding future engines (vision/audio/diffusion) but
|
||||
its only implementation being in-process candle.
|
||||
**Goal:** Prove the harness abstraction works with a second engine.
|
||||
|
||||
The full staged plan for this pivot lives at
|
||||
`~/.claude/plans/create-a-more-aggressive-calm-naur.md`. Summary:
|
||||
**Steps:**
|
||||
1. `crates/neuron/src/harness/llamacpp.rs` — implement the `Harness`
|
||||
trait for llama.cpp's `llama-server`.
|
||||
- `start()` — launch `llama-server` with the correct model path,
|
||||
`--port`, `--n-gpu-layers`, `--tensor-split` args. Track the
|
||||
child process.
|
||||
- `stop()` — send SIGTERM to the child process.
|
||||
- `list_models()` — llama-server serves one model per process, so
|
||||
return a single-element list.
|
||||
- `load_model()` — start a new llama-server process for this model.
|
||||
- `unload_model()` — stop the process.
|
||||
- `inference_endpoint()` — return `http://localhost:{assigned_port}`.
|
||||
2. Port allocation: neuron assigns ports from a range (e.g. 8100-8199)
|
||||
to llama-server instances.
|
||||
3. Register in `HarnessRegistry` when configured:
|
||||
```toml
|
||||
[[harnesses]]
|
||||
name = "llamacpp"
|
||||
binary = "/usr/local/bin/llama-server"
|
||||
port_range = [8100, 8199]
|
||||
```
|
||||
4. Tests: mock llama-server (simple HTTP server returning canned
|
||||
responses), test load/unload/endpoint lifecycle.
|
||||
|
||||
- **Stage 1 (this commit):** delete `mistralrs.rs` and `llamacpp.rs`,
|
||||
scaffold inert `CandleHarness`, drop `endpoint`/`systemd_unit` from
|
||||
`HarnessConfig`, default no-op `start`/`stop` on the `Harness` trait.
|
||||
- **Stages 2–4:** wire up candle model load/unload (quantized Qwen3
|
||||
first), add OpenAI-compatible inference endpoint in neuron, then SSE
|
||||
streaming.
|
||||
- **Stages 5–6:** load-on-activation (default models in config) and
|
||||
unload-on-deactivation (graceful shutdown).
|
||||
- **Stages 7–8:** multi-GPU tensor parallelism and broader model/quant
|
||||
coverage.
|
||||
**Done when:** A model with `harness = "llamacpp"` in `models.toml` can
|
||||
be loaded and served through cortex. Tests pass with mock llama-server.
|
||||
|
||||
Sections of this document that describe mistral.rs HTTP behaviour
|
||||
("mistral.rs API gotchas") are retained as historical context for
|
||||
Phases 1–10 — they document what was true while the project depended
|
||||
on mistral.rs. They do not describe current behaviour.
|
||||
### Phase 12 (lower priority): mistral.rs COPR packaging
|
||||
|
||||
---
|
||||
**Goal:** Fedora RPMs for mistral.rs built against specific CUDA versions.
|
||||
|
||||
### Phase 11 (superseded): llama.cpp harness stub
|
||||
**Steps:**
|
||||
1. `mistralrs-cuda.spec` — RPM spec that clones a pinned mistral.rs git
|
||||
tag, builds with `--features cuda`, links against the system CUDA
|
||||
toolkit. Produces `mistralrs-cuda13-server` (CUDA 13.x / sm_120) and
|
||||
`mistralrs-cuda12-server` (CUDA 12.x / sm_89). Install binary to
|
||||
`/usr/local/bin/mistralrs`.
|
||||
2. COPR build config: enable the NVIDIA CUDA repo as a build dependency.
|
||||
Pin the CUDA toolkit version in `BuildRequires`.
|
||||
3. Gitea Actions or manual workflow: bump the mistral.rs tag in the spec,
|
||||
trigger COPR rebuild.
|
||||
4. neuron's mistralrs harness config references which binary/package
|
||||
provides the mistral.rs binary. neuron could warn at startup if the
|
||||
installed mistral.rs CUDA version doesn't match the discovered driver.
|
||||
|
||||
~~Originally planned as a second engine to prove the harness
|
||||
abstraction.~~ Replaced by the candle harness work in the 2026-05-18
|
||||
addendum above. llama.cpp's any-model/any-hardware breadth is no
|
||||
longer in scope for helexa.
|
||||
**Done when:** `dnf install mistralrs-cuda13-server` on beast provides a
|
||||
working `mistralrs` binary built for Blackwell GPUs. `dnf install
|
||||
mistralrs-cuda12-server` on benjy provides one built for Ada GPUs.
|
||||
|
||||
### Phase 12 (superseded): mistral.rs COPR packaging
|
||||
|
||||
~~Originally planned to ship CUDA-versioned mistral.rs RPMs.~~ Replaced
|
||||
by the candle harness work in the 2026-05-18 addendum above. With
|
||||
mistral.rs out of the dependency tree, there is nothing to package.
|
||||
|
||||
## 2026-05-27 addendum: per-device worker thread
|
||||
|
||||
Replaced the ad-hoc `tokio::task::spawn_blocking` pattern that drove
|
||||
every leader-side CUDA op with one dedicated OS thread per CUDA device,
|
||||
permanently bound to that device's `CudaContext`. All leader-side
|
||||
inference work (GGUF + dense + TP shard load, forward, kv-cache clear,
|
||||
NCCL init/sanity, NCCL all_reduce, VRAM query, model drop) routes
|
||||
through the worker via a `std::sync::mpsc` channel; tensors never
|
||||
escape the worker thread alive. See "Per-device worker thread (neuron)"
|
||||
above and `crates/neuron/src/harness/device_worker/mod.rs` for the
|
||||
canonical narrative.
|
||||
|
||||
Motivated by the 2026-05-26 silent-hang on beast: a CUDA OOM cascade
|
||||
poisoned the device context on whichever spawn_blocking thread caught
|
||||
it, and subsequent requests stalled invisibly on the pool lock. After
|
||||
the refactor, the same failure mode shows up in journalctl as
|
||||
`prefill sample failed; logits unhealthy nan: 248320/248320` followed
|
||||
by `failed, model marked poisoned`. The thread stays alive and rejects
|
||||
subsequent requests at the channel boundary.
|
||||
|
||||
Landed in four PRs:
|
||||
|
||||
- **Phase 1** (`081b532`) — device_worker module + 8 VRAM-query sites
|
||||
route through the worker. CPU build only; smoke on beast confirmed
|
||||
a persistent `cuda-dev-0` thread.
|
||||
- **Phase 2** (`b179204`) — single-GPU forward + clear_kv + drop via
|
||||
the worker. `LoadedModel.arch_handle: Option<ArchHandle>` replaces
|
||||
`Arc<Mutex<ModelArch>>` for CUDA loads. CPU keeps the legacy path.
|
||||
- **Phase 3** (`76ab24d`) — TP forward + NCCL init/sanity + leader
|
||||
KV-clear routed through the worker. `WorkerPool.leader_nccl` moves
|
||||
into the worker's state. `TpLoadedModel.leader_handle: TpHandle`
|
||||
replaces `Arc<Mutex<TpLeaderModel>>`. CUDA-only TP smoke deferred to
|
||||
next deploy.
|
||||
- **Phase 4** (`b4f3576`) — GGUF + dense + TP shard loads move onto
|
||||
the worker. The `Job::TransferIn` / `Job::CloneLeaderComm` bridges
|
||||
from Phases 2/3 deleted; `SendComm` newtype no longer needed in the
|
||||
load path. `grep -rn spawn_blocking crates/neuron/src/harness/`
|
||||
returns only deliberate CPU-fallback hits after this PR.
|
||||
|
||||
## 2026-06-13 addendum: build metadata + helexa-bench
|
||||
|
||||
Two coupled additions so fleet performance can be tracked automatically
|
||||
across neuron updates instead of by hand-running `script/bench.py` and
|
||||
editing `doc/benchmarks.md`.
|
||||
|
||||
**neuron build metadata + `GET /version`.** neuron's `build.rs` now also
|
||||
captures build identity (`HELEXA_GIT_SHA` — preferring a CI/RPM-injected
|
||||
`HELEXA_BUILD_SHA`, falling back to git, else `unknown` — plus dirty
|
||||
flag, build timestamp, rustc version, profile, enabled cargo features,
|
||||
and a best-effort `candle-core` version from `Cargo.lock`). These are
|
||||
exposed as `cortex_core::build_info::BuildInfo` (new module) from a new
|
||||
`GET /version` endpoint (`neuron/src/version.rs`, wired in `api.rs`) and
|
||||
in clap's `--version` long form. The SHA is injected in CI
|
||||
(`build-prerelease.yml` build-neuron step: `export HELEXA_BUILD_SHA=$(git
|
||||
rev-parse HEAD)`) and via `--define helexa_commit` in the source-build
|
||||
spec, so tarball-built RPMs report the real SHA. `/version` is now the
|
||||
canonical "which build is live" probe (supersedes the per-host RPM-sha
|
||||
check in the fleet-validation flow).
|
||||
|
||||
**`crates/helexa-bench`** — a new binary: a continuous, version-aware
|
||||
benchmark harness (one systemd unit, typically on the metrics host). It
|
||||
hits each neuron **directly** on `:13131`, exercises each **warm**
|
||||
(`status == "loaded"`) model with an extensible `Scenario` suite (phase
|
||||
1: the chat-latency family ported verbatim from `bench.py` — synthetic
|
||||
128/4096-tok prompts, `/no_think`, streamed TTFT + decode-window
|
||||
tok/s), and records each run into a SQLite system-of-record stamped with
|
||||
the neuron's full `BuildInfo`. The loop is **version-aware**: it skips
|
||||
any (target, build SHA, model, scenario) cell already at
|
||||
`samples_per_version`, so a steady fleet costs only cheap `/version` +
|
||||
`/models` polls until a new SHA ships. `helexa-bench report` regenerates
|
||||
the `benchmarks.md`-style table from the DB. `kind = "openai"` targets
|
||||
(mistral.rs/llama.cpp comparison) are scaffolded but not yet wired.
|
||||
Packaged as the `helexa-bench` RPM (prebuilt-binary spec, outbound-only
|
||||
so no firewalld service) via the same `build-prerelease.yml` pipeline.
|
||||
|
||||
## 2026-07-09 addendum: concurrency & admission model
|
||||
|
||||
Historical note: earlier docs described inference as "batch-1,
|
||||
serialized per model, no admission control" and the README ruled out
|
||||
continuous batching "permanently." **Both are obsolete.** Each loaded
|
||||
model on a neuron now serves multiple concurrent requests behind bounded
|
||||
admission control, and cortex routes load-aware — not as a pure
|
||||
passthrough.
|
||||
|
||||
**Admission control (#53, #54).** Every loaded model owns an
|
||||
`AdmissionController` (`crates/neuron/src/harness/admission.rs`). It
|
||||
bounds `max_in_flight` running requests plus a `max_queue_depth` waiting
|
||||
queue with a `max_wait_secs` deadline; over-capacity requests fast-reject
|
||||
with `429`/`503` + `Retry-After` rather than hanging. A per-principal
|
||||
`max_per_principal` cap (keyed on the account/key headers cortex stamps)
|
||||
gives fair-share so one client cannot monopolise a model. Defaults live
|
||||
in `config.rs` (`max_in_flight=1`, `max_queue_depth=8`, `max_wait_secs=30`,
|
||||
`max_per_principal=2`); production overrides per host — e.g. beast runs
|
||||
`max_in_flight=8`. A request holds its `AdmissionPermit` for its whole
|
||||
lifetime; the forward itself is still serialized behind the single-GPU
|
||||
`inference_lock` / TP pool mutex.
|
||||
|
||||
**Batched decode (#98).** When `max_in_flight > 1` on a snapshot-capable
|
||||
arch (`qwen3_5` / `qwen3_next`), the lockstep batch engine
|
||||
(`crates/neuron/src/harness/engine.rs`) multiplexes the concurrent
|
||||
sequences into one decode step (ragged prompts, mid-stream joins), so
|
||||
those models are genuinely not batch-1. Runtime kill switch
|
||||
`NEURON_BATCHING=0`. Archs without snapshot support, or
|
||||
`max_in_flight=1`, keep the serialized single-sequence path.
|
||||
|
||||
**Live load surface.** neuron `GET /health` publishes per-model
|
||||
`ModelLoad { in_flight, queue_depth }` and per-device
|
||||
`DeviceHealth { vram_used_mb, vram_free_mb, utilization_pct, temp_c }`.
|
||||
cortex's poller reads these (~10s) into `NodeState.model_load` and picks
|
||||
the least-busy replica in `router.rs` — this is why cortex is load-aware,
|
||||
not first-loaded-node. The concurrency capacity of a neuron:model is
|
||||
therefore a real number today: `max_in_flight` (+ `max_queue_depth`
|
||||
burst absorption), with `in_flight`/`queue_depth` the live utilisation.
|
||||
|
||||
**Observability gap.** That live load — and the device health — is
|
||||
currently consumed only by routing; it is **not** exported to cortex's
|
||||
Prometheus surface, and neuron has no `/metrics` endpoint. Nor is live
|
||||
tok/s aggregated (per-request `FinishTiming` goes to the caller in
|
||||
`usage.helexa_timing` only) or `max_in_flight` advertised. Closing this
|
||||
(export the polled load, advertise the ceiling, roll up tok/s, count
|
||||
rejections) is tracked separately — it is a plumbing gap, not a
|
||||
measurement one, since the signals already exist in memory.
|
||||
|
||||
## Application-owned system prompts (#179)
|
||||
|
||||
System prompts belong to the **calling application**, never to the
|
||||
operator or the serving chain. cortex and helexa-router proxy inference
|
||||
bodies **without adding to them**: no injected prompt, no house style, no
|
||||
default when the caller sends none, no rewriting or reordering of what
|
||||
was sent.
|
||||
|
||||
This is a contract with API consumers, documented in the README
|
||||
("Tailoring model behaviour"), not an incidental behaviour — treat any
|
||||
change that puts content into a proxied request as a breaking change.
|
||||
When adding a surface or a translation path, the system slot must map
|
||||
straight through:
|
||||
|
||||
- `/v1/chat/completions` — `messages[role == "system"]`, all of them, in
|
||||
order (the model sees the last one last, so last-wins is what users
|
||||
observe).
|
||||
- `/v1/responses` — `instructions`, plus `input` items with
|
||||
`role: system`.
|
||||
- `/v1/messages` — top-level `system`, both the string and the
|
||||
content-block-array forms, translated into a system message.
|
||||
|
||||
Pinned by `crates/cortex-gateway/tests/system_prompt.rs` (asserts what
|
||||
cortex *forwarded upstream*, including a control proving nothing is
|
||||
injected when the caller sends no prompt) and the system-prompt cases in
|
||||
`crates/neuron/src/harness/chat_template.rs`.
|
||||
|
||||
## 2026-07-30 addendum: text-to-image serving (epic #197)
|
||||
|
||||
Image generation is a first-class modality. **Tongyi-MAI/Z-Image-Turbo**
|
||||
(6B single-stream DiT + Qwen3-4B text encoder + VAE, Apache 2.0,
|
||||
flow-matching, 9-step turbo) serves candle-natively through the same
|
||||
device-worker discipline as the text archs. Epic #197, PRs #205–#213.
|
||||
|
||||
**Engine (`crates/neuron/src/harness/image.rs`).** `ZImagePipeline`
|
||||
lives in the device worker's image slab (`ImageHandle`; `LoadImage` /
|
||||
`DropImage` / `GenerateImage` jobs — one job per generation, since a
|
||||
denoise loop saturates the device). CPU-side RGB replies preserve the
|
||||
tensors-never-escape invariant; PNG encoding happens async-side.
|
||||
Component placement is the VRAM story:
|
||||
|
||||
- The **text encoder runs on the host CPU** by default (f32, resident
|
||||
in system RAM; `[harness.candle.image] te_device`) — prompt encoding
|
||||
is one short forward and the features are tiny, and a 24 GB card
|
||||
cannot hold an 8 GB GPU TE beside the resident DiT.
|
||||
- **Tiled VAE decode** (64-latent tiles, 16-latent seam overlap,
|
||||
weighted blending) bounds decoder transients at any resolution;
|
||||
full-frame decode OOMs 24 GB at 1024² and 32 GB at ≥1536².
|
||||
- **`quant = "q8_0"`** ISQ-quantizes the DiT in situ from the dense
|
||||
safetensors (no GGUF artifact): ~6.4 GB DiT, f32 activations,
|
||||
fixed-seed output visually identical to BF16. Opens the 12 GB tier.
|
||||
- The cudarc **allocator pool is trimmed after every image job** —
|
||||
mixed-resolution sequences otherwise fragment it until a generation
|
||||
that would fit a clean slate OOMs.
|
||||
|
||||
**candle is pinned to a fork** (`[patch.crates-io]` →
|
||||
`grenade/candle`, branch `z-image-mask-elision`) carrying: all-ones
|
||||
attention-mask elision so flash-attn actually engages (upstream PR
|
||||
huggingface/candle#3798 — drop the pin when it lands), the
|
||||
`QuantizedZImageTransformer2DModel`, and an f32→bf16 cast around
|
||||
flash-attn for the quantized path. The cudarc fork is rebased onto
|
||||
0.19.8 so one cudarc serves both neuron's NCCL layer and candle 0.11.
|
||||
|
||||
**Surface.** `POST /v1/images/generations` on neuron, cortex, and the
|
||||
public router (the router has an explicit route allowlist — new
|
||||
endpoints must be added there; the edge nginx forwards `/v1/*` by
|
||||
prefix). OpenAI envelope: `n=1`, `b64_json`, PNG; extensions `seed`,
|
||||
`negative_prompt` (CFG doubles per-step cost), `guidance_scale`,
|
||||
`num_steps`. Errors conform to #63 (`wrong_modality` 422 for
|
||||
cross-modality requests, `invalid_image_params` 400 pre-admission).
|
||||
Wire types in `cortex-core/src/images.rs`. Admission reuses #53/#54
|
||||
with `max_in_flight = 1`.
|
||||
|
||||
**Metering (#202).** Unit is **megapixel-steps**
|
||||
(`w × h × steps / 1e6`, CFG ×2, 1-unit floor), reported as
|
||||
`usage.helexa_image_units` + `usage.helexa_timing`
|
||||
(encode/denoise/decode ms). Budget enforcement bridges into the token
|
||||
ledger at `TOKENS_PER_IMAGE_UNIT = 1000` — fail-closed pre-dispatch,
|
||||
settled from actual units. A native image unit in the clearing-house
|
||||
contract is the open follow-up on #202.
|
||||
|
||||
**Placement is automatic cold-swap.** The router ranks catalogue
|
||||
cold-load targets pinned → free-fit → evictable-fit → most-free using
|
||||
live per-device VRAM (now stored on `NodeState.device_health`), with a
|
||||
catalogue-`vram_mb` fallback for the evictable estimate (neuron
|
||||
reports `vram_used_mb: null`), and evicts unpinned LRU models on the
|
||||
chosen node before loading. Catalogue pinning steers: beast's 27B is
|
||||
pinned, so image loads land on benjy and swap the 8B out; the next
|
||||
text request swaps it back. Measured: image cold-swap 10.9 s,
|
||||
text restore 2.6 s.
|
||||
|
||||
**Serving tiers** (bench + live numbers, 2026-07-29/30):
|
||||
|
||||
| Host | Precision | Resident | Ceiling | 1024² |
|
||||
|---|---|---|---|---|
|
||||
| beast (5090, flash-attn flavour) | BF16 / q8 | 14.6 / 9.4 GB | 2048² (37.7 u) | q8 denoise 2.8 s |
|
||||
| benjy (4090) — v1 placement | BF16 | 14.6 GB | 1024² | 11.4 s wall |
|
||||
| quadbrat (3060) | q8_0 | 8.7 GB | 768² | OOM (naive f32 attn) |
|
||||
|
||||
The 3060 ceiling lifts to 1024² if #95 extends flash-attn to the
|
||||
ampere flavour.
|
||||
|
||||
**Ops sharp edges.**
|
||||
|
||||
- neuron binaries are **per-sm** (`CUDA_COMPUTE_CAP`; CI builds
|
||||
ampere/ada/blackwell flavours): a blackwell build throws
|
||||
`CUDA_ERROR_INVALID_PTX` on ada. Local hand-builds must also match
|
||||
the driver's CUDA version (13.0), not the `/usr/local/cuda` symlink
|
||||
(13.2 on beast), or loads die with
|
||||
`CUDA_ERROR_UNSUPPORTED_PTX_VERSION`.
|
||||
- The production neuron runs as the `neuron` system user with its
|
||||
**own HF cache** (`/var/lib/neuron/.cache`) — first image load on a
|
||||
fresh host is a one-time ~29 GB fetch (beast and benjy are warm).
|
||||
- Fleet validation: `script/validate-image.sh <host>` (evicts and
|
||||
restores a co-resident text model around the probe window).
|
||||
helexa-bench has an `image:<px>` scenario (default `[1024]`),
|
||||
gated on the `image` capability; text scenarios gate image models
|
||||
out.
|
||||
- Deferred by operator decision: Z-Image base variant, operator
|
||||
fine-tunes, Z-Image-Edit (`images/edits`). Qwen-Image-2512 is the
|
||||
parked "frontier image" tier (own epic if taken).
|
||||
This is a separate repo/spec — not part of the cortex workspace — but
|
||||
tightly coupled operationally. Track it as a sibling project.
|
||||
|
||||
3815
Cargo.lock
generated
3815
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
32
Cargo.toml
32
Cargo.toml
@@ -5,20 +5,13 @@ members = [
|
||||
"crates/cortex-gateway",
|
||||
"crates/cortex-cli",
|
||||
"crates/neuron",
|
||||
"crates/helexa-acp",
|
||||
"crates/helexa-angels",
|
||||
"crates/helexa-bench",
|
||||
"crates/helexa-router",
|
||||
"crates/helexa-stream",
|
||||
"crates/helexa-tools",
|
||||
"crates/helexa-upstream",
|
||||
]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.1.16"
|
||||
version = "0.1.12"
|
||||
edition = "2024"
|
||||
license = "GPL-3.0-or-later"
|
||||
repository = "https://git.lair.cafe/helexa/helexa"
|
||||
repository = "https://git.lair.cafe/helexa/cortex"
|
||||
|
||||
[workspace.dependencies]
|
||||
# async runtime
|
||||
@@ -34,7 +27,7 @@ serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
toml = "0.8"
|
||||
|
||||
# http client (for proxying to neuron backends)
|
||||
# http client (for proxying to mistralrs backends)
|
||||
reqwest = { version = "0.12", features = ["json", "stream"] }
|
||||
|
||||
# observability
|
||||
@@ -67,22 +60,3 @@ eventsource-stream = "0.2"
|
||||
# workspace crates
|
||||
cortex-core = { path = "crates/cortex-core" }
|
||||
cortex-gateway = { path = "crates/cortex-gateway" }
|
||||
|
||||
# Patched cudarc (affects neuron's 0.19.x only; candle's 0.17.x is
|
||||
# untouched since the fork is 0.19.8, rebased for the candle 0.11 bump, and now also serves candle's ^0.19.8 req — one unified cudarc). Adds
|
||||
# Comm::abort / get_async_error / raw comm() — needed for #17 Stage 2 TP
|
||||
# hang-recovery (abort a wedged collective from another thread, then
|
||||
# rebuild the comm). Pinned to a fork revision pending upstream review
|
||||
# (grenade/cudarc @ nccl-comm-abort).
|
||||
[patch.crates-io]
|
||||
cudarc = { git = "https://github.com/grenade/cudarc", rev = "6904cf2e651a0384eac318f003ce6417f47f5d94" }
|
||||
# Patched candle (#199): z_image all-ones attention-mask elision so the
|
||||
# flash-attn path actually engages — unlocks >=1536^2 denoising on the
|
||||
# blackwell (flash-attn) builds. Upstream PR:
|
||||
# https://github.com/huggingface/candle/pull/3798 — drop this pin (and
|
||||
# the fork) when it lands in a release. All four crates pin to the same
|
||||
# rev so candle types unify across the graph.
|
||||
candle-core = { git = "https://github.com/grenade/candle", rev = "f8fa85fd2753b334cb4f42e7357f8ba9d9bb2940" }
|
||||
candle-nn = { git = "https://github.com/grenade/candle", rev = "f8fa85fd2753b334cb4f42e7357f8ba9d9bb2940" }
|
||||
candle-transformers = { git = "https://github.com/grenade/candle", rev = "f8fa85fd2753b334cb4f42e7357f8ba9d9bb2940" }
|
||||
candle-flash-attn = { git = "https://github.com/grenade/candle", rev = "f8fa85fd2753b334cb4f42e7357f8ba9d9bb2940" }
|
||||
|
||||
306
README.md
306
README.md
@@ -1,68 +1,24 @@
|
||||
# helexa
|
||||
# cortex
|
||||
|
||||
**Near-frontier AI for mortals.**
|
||||
A Rust reverse-proxy and fleet management layer for multi-node
|
||||
[mistral.rs](https://github.com/EricLBuehler/mistral.rs) inference clusters.
|
||||
|
||||
helexa is a self-hosted LLM serving stack, written in Rust, for people
|
||||
who run open-weight models on their own consumer GPUs. It has two
|
||||
components:
|
||||
## Problem
|
||||
|
||||
- **cortex** — the per-operator control plane and LLM proxy. It sits in
|
||||
front of your GPU fleet and presents a unified OpenAI + Anthropic
|
||||
compatible API surface, handling model routing, lifecycle management
|
||||
(load / unload / evict), request translation, and metrics.
|
||||
- **neuron** — the per-host LLM harness. One instance runs on every GPU
|
||||
host, serving candle-based in-process inference and managing local
|
||||
hardware discovery and model lifecycle.
|
||||
Running local LLMs across multiple GPU nodes (different VRAM tiers, different
|
||||
model affinities) requires a unified API surface that:
|
||||
|
||||
## Why
|
||||
|
||||
Two principles constrain everything in this repository:
|
||||
|
||||
1. **Frontier or close to it.** helexa serves the open-weight models
|
||||
that get nearest to frontier capability — not every architecture
|
||||
ever published.
|
||||
2. **Consumer hardware.** Everything must run on the cards mortals can
|
||||
actually buy: a 3060 here, a 4090 there, a 5090 if you got lucky.
|
||||
Mixed VRAM tiers across mismatched boxes are the expected topology,
|
||||
not a degraded case.
|
||||
|
||||
GPU acquisition is harder than it was a year ago, and the gap between
|
||||
what cloud providers charge and what your own silicon costs keeps
|
||||
widening. The intersection of those two principles — near-frontier
|
||||
models, squeezed onto hardware you own — is helexa's entire niche.
|
||||
|
||||
The secondary objective is **predictable consumption**. If you own the
|
||||
hardware, your tooling shouldn't break because a cloud provider changed
|
||||
billing, deprecated a model, or reshaped an API. cortex's OpenAI and
|
||||
Anthropic surfaces are a stability contract: point your editor, agent,
|
||||
or CLI at it once, and it keeps working.
|
||||
|
||||
## What helexa is not
|
||||
|
||||
This is an intentionally different path from vLLM, SGLang, and peers —
|
||||
not a smaller version of them. Out of scope, permanently:
|
||||
|
||||
- Any-model breadth. Architectures are ported because they're at or
|
||||
near the frontier, not to complete a compatibility matrix.
|
||||
- Datacenter-class scheduling. No sophisticated continuous-batching /
|
||||
paged-attention machinery — the workload is a handful of operators
|
||||
and their agents, not 200 QPS.
|
||||
- Wrapping external inference engines. neuron builds directly on
|
||||
[candle](https://github.com/huggingface/candle); every model
|
||||
architecture it serves is implemented in this repository, ported
|
||||
against the HuggingFace reference.
|
||||
|
||||
One thing that is *not* a principle: CUDA exclusivity. All high-end
|
||||
consumer hardware is in scope. helexa is CUDA-only today because
|
||||
that's the hardware on the bench — nothing ships untested — and ROCm
|
||||
or other consumer accelerators join as soon as there's real hardware
|
||||
to build against.
|
||||
|
||||
In scope, and where the engineering effort goes: aggressive
|
||||
quantization (GGUF Q4_K_M / Q6_K / Q8_0), NCCL tensor parallelism
|
||||
across heterogeneous consumer GPUs, careful CUDA failure handling, and
|
||||
single-request latency — the performance that one operator at a
|
||||
keyboard actually feels.
|
||||
- Presents a **single `/v1/models` catalogue** merging every model across every
|
||||
node.
|
||||
- **Routes requests** to the correct node based on where a model is loaded (or
|
||||
*can* be loaded).
|
||||
- Manages **model lifecycle** — unload cold models, reload on demand, pin
|
||||
critical ones — using the mistral.rs
|
||||
`/v1/models/{unload,reload,status}` HTTP API (PR #1828+).
|
||||
- Translates between **OpenAI and Anthropic** request/response envelopes so
|
||||
every client in the homelab speaks whichever dialect it prefers.
|
||||
- Captures **per-request metrics** (tokens, tok/s, TTFT, latency) and exposes
|
||||
them as Prometheus counters/histograms.
|
||||
|
||||
## Architecture
|
||||
|
||||
@@ -72,83 +28,65 @@ keyboard actually feels.
|
||||
└──────┬───────┘ └─────┬────┘ └──────┬─────┘ └──────┬─────┘
|
||||
│ │ │ │
|
||||
└────────────────┴──────┬───────┴───────────────┘
|
||||
│ OpenAI + Anthropic APIs
|
||||
│
|
||||
┌──────────▼──────────┐
|
||||
│ cortex │
|
||||
│ (cortex-gateway) │
|
||||
│ cortex │
|
||||
│ (cortex-gateway) │
|
||||
│ │
|
||||
│ Router · Metrics │
|
||||
│ Evictor · Translate│
|
||||
└──┬──────┬────────┬──┘
|
||||
│ │ │
|
||||
┌──────────▼┐ ┌──▼─────┐ ┌▼──────────┐
|
||||
│ neuron │ │ neuron │ │ neuron │
|
||||
│ :13131 │ │ :13131 │ │ :13131 │
|
||||
│ candle │ │ candle │ │ candle │
|
||||
│ gpu-large │ │gpu-med │ │ gpu-small │
|
||||
│ mistralrs │ │mistral │ │ mistralrs │
|
||||
│ serve │ │rs serve│ │ serve │
|
||||
│ :8080 │ │ :8080 │ │ :8080 │
|
||||
└───────────┘ └────────┘ └───────────┘
|
||||
private network (.internal)
|
||||
```
|
||||
|
||||
cortex discovers each neuron's hardware (devices, VRAM, compute
|
||||
capability) at runtime and matches it against a model catalogue
|
||||
(`models.toml`) to decide placement: which models fit where, what to
|
||||
evict when VRAM is tight, where to route a request right now. Adding a
|
||||
GPU host to the fleet is one `[[neurons]]` entry — no device specs in
|
||||
config.
|
||||
|
||||
### Crates
|
||||
|
||||
| Crate | Purpose |
|
||||
|---|---|
|
||||
| `cortex-core` | Shared types: config, node/model state, metrics, OpenAI/Anthropic envelopes, harness trait, discovery types |
|
||||
| `cortex-gateway` | Axum HTTP server: proxy, router, evictor, poller, metrics exporter |
|
||||
| `neuron` | Per-host daemon: GPU discovery, in-process candle inference, NCCL tensor parallelism, model lifecycle API |
|
||||
| `cortex-core` | Shared types: config, node/model state, metrics, OpenAI/Anthropic request/response envelopes |
|
||||
| `cortex-gateway` | Axum HTTP server: proxy, router, evictor, metrics exporter |
|
||||
| `cortex-agent` | Per-node sidecar: polls local mistralrs, reports to gateway, handles restart/defrag |
|
||||
| `cortex-cli` | CLI entrypoint (`cortex serve`, `cortex status`, etc.) |
|
||||
| `helexa-acp` | Agent Client Protocol bridge — connects ACP editors (Zed, etc.) to any OpenAI-compatible endpoint, cortex by default |
|
||||
|
||||
## The engine
|
||||
## Node setup
|
||||
|
||||
neuron runs inference in-process on candle — there is no external
|
||||
inference server to babysit. The parts that earn their keep:
|
||||
Each GPU node runs `mistralrs serve` with a multi-model config. Models are
|
||||
declared but start **unloaded** — mistral.rs lazy-loads on first request and
|
||||
the gateway can explicitly unload/reload via the HTTP API.
|
||||
|
||||
- **Per-device worker threads.** Every CUDA device gets one dedicated
|
||||
OS thread that owns its CUDA context for the daemon's lifetime. All
|
||||
loads, forward passes, KV-cache resets, NCCL collectives, VRAM
|
||||
queries, and unloads route through it; tensors never escape it
|
||||
alive. Context binding is pinned to a known thread, the CUDA `Drop`
|
||||
contract is structurally safe, and a driver error poisons one worker
|
||||
— visibly — instead of hanging the whole process.
|
||||
- **Tensor parallelism on consumer cards.** Megatron-style row/column
|
||||
parallel layers with NCCL all-reduce, spanning the mismatched GPUs
|
||||
you actually have. A step watchdog aborts wedged collectives instead
|
||||
of letting a request hang forever.
|
||||
- **Text-to-image.** Z-Image-Turbo (6B S3-DiT, Apache 2.0) served
|
||||
candle-native through the same device-worker discipline: OpenAI
|
||||
`/v1/images/generations` end to end, ~11 s for a 1024x1024 image on
|
||||
an RTX 4090, metered in megapixel-steps.
|
||||
- **Current model focus: the Qwen3 family** — dense and GGUF-quantized,
|
||||
including the hybrid linear-attention (Gated DeltaNet) generation.
|
||||
Vision support is in progress. Each architecture is ported against
|
||||
its HuggingFace reference implementation.
|
||||
Example node systemd unit:
|
||||
|
||||
See `CLAUDE.md` for design rationale and
|
||||
`crates/neuron/src/harness/device_worker/` for the worker narrative.
|
||||
```ini
|
||||
# /etc/systemd/system/mistralrs.service
|
||||
[Unit]
|
||||
Description=mistral.rs inference server
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
## Install
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/local/bin/mistralrs serve \
|
||||
--from-config /etc/mistralrs/config.toml \
|
||||
--port 8080
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
Environment=CUDA_VISIBLE_DEVICES=0,1
|
||||
|
||||
Pre-built RPMs for Fedora:
|
||||
|
||||
```sh
|
||||
dnf copr enable helexa/helexa
|
||||
dnf install cortex # on the gateway host
|
||||
dnf install helexa-neuron # on each GPU host
|
||||
systemctl enable --now cortex # or neuron, respectively
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
## Configure
|
||||
## Gateway config
|
||||
|
||||
```toml
|
||||
# /etc/cortex/cortex.toml
|
||||
# cortex.toml
|
||||
[gateway]
|
||||
listen = "0.0.0.0:31313"
|
||||
metrics_listen = "0.0.0.0:31314"
|
||||
@@ -157,117 +95,35 @@ metrics_listen = "0.0.0.0:31314"
|
||||
strategy = "lru" # lru | priority
|
||||
defrag_after_cycles = 50
|
||||
|
||||
[[neurons]]
|
||||
name = "beast"
|
||||
endpoint = "http://beast.internal:13131"
|
||||
[[nodes]]
|
||||
name = "gpu-large"
|
||||
endpoint = "http://gpu-large.internal:8080"
|
||||
vram_mb = 49_152 # e.g. 2x RTX 4090
|
||||
pinned = ["your-org/large-model"]
|
||||
|
||||
[[neurons]]
|
||||
name = "benjy"
|
||||
endpoint = "http://benjy.internal:13131"
|
||||
[[nodes]]
|
||||
name = "gpu-medium"
|
||||
endpoint = "http://gpu-medium.internal:8080"
|
||||
vram_mb = 24_576 # e.g. RTX 4090
|
||||
pinned = ["your-org/medium-model"]
|
||||
|
||||
[[nodes]]
|
||||
name = "gpu-small"
|
||||
endpoint = "http://gpu-small.internal:8080"
|
||||
vram_mb = 12_288 # e.g. RTX 3060
|
||||
pinned = ["your-org/embedding-model"]
|
||||
```
|
||||
|
||||
Model placement profiles — VRAM requirements, quant, device minimums,
|
||||
which neurons a model may run on, and what it may displace when one runs
|
||||
out of VRAM — live in `models.toml`. `models.example.toml` is the field
|
||||
reference; [placement & displacement](https://helexa.ai/docs/operating/placement)
|
||||
explains how the two fit together, and is worth reading before you set
|
||||
`residency_priority` on anything.
|
||||
|
||||
Full documentation — using helexa and operating it — is at
|
||||
[helexa.ai/docs](https://helexa.ai/docs); the source lives under
|
||||
`helexa.ai/content/docs/`.
|
||||
|
||||
## Run
|
||||
|
||||
```sh
|
||||
# start the gateway
|
||||
cortex serve --config /etc/cortex/cortex.toml
|
||||
|
||||
# check fleet status
|
||||
cortex status
|
||||
|
||||
# one catalogue across every node
|
||||
curl http://localhost:31313/v1/models
|
||||
```
|
||||
|
||||
## Tailoring model behaviour
|
||||
|
||||
System prompts are **application-owned**. Send yours through the standard
|
||||
field for whichever API you speak, and the serving chain — edge → router →
|
||||
cortex → neuron — passes it to the model **verbatim**.
|
||||
|
||||
```sh
|
||||
# OpenAI chat completions
|
||||
curl http://localhost:31313/v1/chat/completions \
|
||||
-H 'content-type: application/json' \
|
||||
-d '{"model": "helexa/balanced",
|
||||
"messages": [{"role": "system", "content": "Reply only in French."},
|
||||
{"role": "user", "content": "Good morning"}]}'
|
||||
|
||||
# OpenAI responses — the `instructions` field is the system slot
|
||||
curl http://localhost:31313/v1/responses \
|
||||
-H 'content-type: application/json' \
|
||||
-d '{"model": "helexa/balanced",
|
||||
"instructions": "Reply only in French.",
|
||||
"input": "Good morning"}'
|
||||
|
||||
# Anthropic messages — top-level `system`, string or content-block array
|
||||
curl http://localhost:31313/v1/messages \
|
||||
-H 'content-type: application/json' \
|
||||
-d '{"model": "helexa/balanced", "max_tokens": 256,
|
||||
"system": "Reply only in French.",
|
||||
"messages": [{"role": "user", "content": "Good morning"}]}'
|
||||
```
|
||||
|
||||
### The passthrough guarantee
|
||||
|
||||
helexa will never, to any request you proxy through it:
|
||||
|
||||
- inject a system prompt, house style or preamble of its own,
|
||||
- rewrite, truncate or reorder the one you sent,
|
||||
- supply a default when you send none.
|
||||
|
||||
If you send no system prompt, the model receives none. Behaviour you did
|
||||
not ask for is a bug — please report it.
|
||||
|
||||
Streaming and non-streaming behave identically, and the guarantee holds on
|
||||
every surface above. Two details worth knowing:
|
||||
|
||||
- **Several system messages** are all forwarded, unmerged and in order.
|
||||
The model sees the last one last, so in practice the last instruction
|
||||
wins. Send one message if you want certainty.
|
||||
- **`/no_think`** in a prompt is a Qwen-family convention the *model*
|
||||
interprets to skip its reasoning block. It is the model's feature, not
|
||||
ours — helexa neither adds nor strips it. Note it currently suppresses
|
||||
reasoning on `/v1/chat/completions` but **not** on `/v1/responses`,
|
||||
where a reasoning model may think regardless (#223); with a small
|
||||
`max_output_tokens` the whole budget can go on reasoning and the reply
|
||||
comes back empty with `status: "incomplete"`. Give Responses requests
|
||||
room (a few hundred tokens) when the model reasons.
|
||||
|
||||
### Why it works this way
|
||||
|
||||
The ecosystem serves an unenumerable diversity of workloads through
|
||||
OpenAI- and Anthropic-compatible APIs. An operator cannot curate prompts
|
||||
for use cases they will never see, and a proxy that quietly edits your
|
||||
payload makes model behaviour impossible to reason about. So the split is:
|
||||
**applications own the prompt, operators own the fleet.**
|
||||
|
||||
If you specifically want centrally-managed prompts across your own
|
||||
workloads, run your own helexa mesh — it is open source, and that is a
|
||||
different deployment from the shared helexa.ai ecosystem.
|
||||
|
||||
Operators: this is a contract, not a default you may flip. cortex and
|
||||
helexa-router proxy inference bodies without adding to them; nothing in
|
||||
the chain is a place to put prompt content.
|
||||
|
||||
## Build from source
|
||||
## Building
|
||||
|
||||
```sh
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
CI runs on every push; keep it green locally:
|
||||
## CI
|
||||
|
||||
Every push triggers format, lint, and test checks. Ensure these pass
|
||||
locally before pushing:
|
||||
|
||||
```sh
|
||||
cargo fmt --check --all # must be clean
|
||||
@@ -275,18 +131,20 @@ cargo clippy --workspace -- -D warnings # warnings are errors
|
||||
cargo test --workspace # all tests must pass
|
||||
```
|
||||
|
||||
Tagged releases (`v*`) build SRPMs for `cortex` and `helexa-neuron`
|
||||
and publish to COPR.
|
||||
Tagged releases (`v*`) additionally build an SRPM and publish to COPR.
|
||||
|
||||
## Status
|
||||
## Running
|
||||
|
||||
Pre-1.0 and moving fast. The gateway path (routing, eviction,
|
||||
translation, metrics) is stable and tested; the candle-native engine
|
||||
is under active development — expect the supported-model list to track
|
||||
the open-weight frontier, deliberately narrowly.
|
||||
```sh
|
||||
# start the gateway
|
||||
cortex serve --config cortex.toml
|
||||
|
||||
Development happens at <https://git.lair.cafe/helexa/helexa>;
|
||||
<https://github.com/helexa-ai/helexa> is a read-only mirror.
|
||||
# check fleet status
|
||||
cortex status
|
||||
|
||||
# list all models across nodes
|
||||
curl http://localhost:31313/v1/models
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
# helexa-bench config for bob.hanzalova.internal.
|
||||
#
|
||||
# Synced to /etc/helexa-bench/helexa-bench.toml by script/infra-setup.sh
|
||||
# (the helexa-bench RPM ships helexa-bench.example.toml as a
|
||||
# %config(noreplace) default; this per-host file overrides it).
|
||||
#
|
||||
# bob is a client host (it also runs Agent Zero); helexa-bench here hits
|
||||
# every neuron on the fleet directly and records build-stamped results
|
||||
# into the local SQLite store.
|
||||
|
||||
[bench]
|
||||
sweep_interval_secs = 1800
|
||||
samples_per_version = 5
|
||||
iteration_pause_secs = 2
|
||||
request_timeout_secs = 600
|
||||
db_path = "/var/lib/helexa-bench/bench.sqlite"
|
||||
|
||||
[scenarios]
|
||||
prompt_sizes = [128, 4096]
|
||||
max_tokens = 256
|
||||
# Concurrency / agentic-load scenarios (#89), enabled so the 27B baseline
|
||||
# carries p95-under-concurrency data before the F3 A/B gate (#94) needs
|
||||
# it for comparison. Levels mirror the real a0/hermes/opencode fan-out.
|
||||
concurrency_levels = [2, 4, 8]
|
||||
concurrency_prompt_tokens = 512
|
||||
|
||||
# Capability probes (#91) — the reasoning/planning axis the speed
|
||||
# scenarios miss; scored manually via `helexa-bench score` (O7). Enabled
|
||||
# for the same reason: the F3 gate compares 80B-A3B variants against the
|
||||
# 27B on planning quality, so the 27B needs scored artifacts first.
|
||||
[[scenarios.capability_probes]]
|
||||
name = "rust-plan"
|
||||
max_tokens = 4096
|
||||
prompt = """
|
||||
Write an implementation plan for adding rate limiting to an Axum service.
|
||||
Honor existing conventions, call out trade-offs, and sequence the work.
|
||||
"""
|
||||
|
||||
[[scenarios.capability_probes]]
|
||||
name = "debug-reason"
|
||||
max_tokens = 4096
|
||||
prompt = """
|
||||
A Rust axum server streams SSE responses through a reverse proxy. Clients
|
||||
report that streams stall for exactly 60 seconds and then resume, but only
|
||||
when response chunks are small and infrequent. Curling the backend directly
|
||||
never stalls. List the most likely causes in order of probability, explain
|
||||
the mechanism behind each, and describe the smallest experiment that would
|
||||
confirm or eliminate each cause.
|
||||
"""
|
||||
|
||||
# Read-only JSON API consumed by the bench UI (hosted separately) and for
|
||||
# programmatic access. Served alongside the sweep loop.
|
||||
[api]
|
||||
enabled = true
|
||||
listen = "0.0.0.0:13132"
|
||||
|
||||
[[targets]]
|
||||
name = "beast"
|
||||
endpoint = "http://beast.hanzalova.internal:13131"
|
||||
|
||||
[[targets]]
|
||||
name = "benjy"
|
||||
endpoint = "http://benjy.hanzalova.internal:13131"
|
||||
|
||||
[[targets]]
|
||||
name = "quadbrat"
|
||||
endpoint = "http://quadbrat.hanzalova.internal:13131"
|
||||
@@ -1,81 +0,0 @@
|
||||
# Fleet monitoring — Prometheus scrape + Grafana dashboard
|
||||
|
||||
Visibility into the cortex `#137` capacity metrics (per-neuron:model load,
|
||||
saturation, tok/s, load-shedding, and per-device GPU health) beyond what the
|
||||
bench UI shows.
|
||||
|
||||
## Topology
|
||||
|
||||
- **cortex** runs on `hanzalova.internal` (10.6.0.46) and is the **only**
|
||||
Prometheus target — it exposes every `cortex_*` metric on `:31314`, already
|
||||
labelled by `{node,model}` / `{node,device}` from its neuron poller. neuron
|
||||
has no `/metrics` endpoint.
|
||||
- **Prometheus + Grafana** run on `golgafrinchans.kosherinata.internal` as
|
||||
podman quadlets. Host ports (from the registered band — 9090 is Cockpit,
|
||||
3000 was avoided): **Prometheus `:26559`** (container config bind-mounted at
|
||||
`/etc/prometheus/prometheus.yml`, `--web.enable-lifecycle` on), **Grafana
|
||||
`:28767`**. Its mesh IP toward hanzalova is `10.3.101.4`.
|
||||
|
||||
## Apply (three steps, in order)
|
||||
|
||||
### 1. Open the metrics port on the cortex host
|
||||
|
||||
cortex binds `0.0.0.0:31314` but firewalld has no rule for it, so a
|
||||
cross-host scrape times out. Open it to the monitoring host **only**:
|
||||
|
||||
```sh
|
||||
# on hanzalova.internal, as root
|
||||
./firewalld-cortex-metrics.sh
|
||||
```
|
||||
|
||||
### 2. Add the scrape job on the monitoring host
|
||||
|
||||
Append `prometheus-cortex.scrape.yml` into the `scrape_configs:` list of
|
||||
`/etc/prometheus/prometheus.yml` on `golgafrinchans.kosherinata.internal`,
|
||||
then hot-reload (no restart, lifecycle API is enabled):
|
||||
|
||||
```sh
|
||||
curl -X POST http://localhost:26559/-/reload
|
||||
# verify the target is UP:
|
||||
curl -s 'http://localhost:26559/api/v1/targets' | jq '.data.activeTargets[]|select(.labels.job=="cortex")|{health,lastError}'
|
||||
```
|
||||
|
||||
### 3. Import the dashboard
|
||||
|
||||
`grafana-helexa-fleet.json` is a raw dashboard model with a `datasource`
|
||||
template variable, so it binds to whichever Prometheus data source you pick
|
||||
at import. Import via the API (creds in `/etc/grafana/grafana.env`):
|
||||
|
||||
```sh
|
||||
# on golgafrinchans; GF_SECURITY_ADMIN_USER/PASSWORD live in the root-only
|
||||
# env file, Grafana is published on :28767
|
||||
set -a; source <(sudo cat /etc/grafana/grafana.env); set +a
|
||||
jq -n --slurpfile d grafana-helexa-fleet.json \
|
||||
'{dashboard: $d[0], overwrite: true, folderUid: null}' \
|
||||
| curl -s -u "$GF_SECURITY_ADMIN_USER:$GF_SECURITY_ADMIN_PASSWORD" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d @- http://localhost:28767/api/dashboards/db | jq '{status,uid,version}'
|
||||
```
|
||||
|
||||
The dashboard lands at uid `helexa-fleet`. Re-importing with `overwrite:true`
|
||||
updates it in place.
|
||||
|
||||
## What the dashboard shows
|
||||
|
||||
| Row | Panels |
|
||||
|---|---|
|
||||
| Capacity & saturation | saturation % (in_flight ÷ max_in_flight), in-flight vs ceiling, queue depth |
|
||||
| Throughput | decode tok/s (revenue capacity), prefill tok/s |
|
||||
| Backpressure & traffic | rejection rate by reason, request & error rate, TTFT p95/p50 |
|
||||
| GPU health | VRAM used per device, GPU utilization %, temperature |
|
||||
|
||||
Templating: pick the Prometheus data source, then filter by `neuron` (node)
|
||||
and `model`. Values are live at the cortex poll cadence (~10 s) scraped every
|
||||
15 s.
|
||||
|
||||
## Note on the empirical knee
|
||||
|
||||
The live gauges above answer "how loaded is it right now". The
|
||||
**sustainable-concurrency knee** (max N before latency/shedding breaks) is a
|
||||
load-test result, not a live gauge — see `helexa-bench report --concurrency`
|
||||
and `GET /api/concurrency` (#137 T3).
|
||||
@@ -1,28 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Open cortex's Prometheus metrics port (:31314) on the cortex gateway host
|
||||
# (hanzalova.internal, 10.6.0.46) to the monitoring host ONLY
|
||||
# (golgafrinchans.kosherinata.internal, which reaches hanzalova as 10.3.101.4).
|
||||
#
|
||||
# cortex binds 0.0.0.0:31314 but firewalld has no rule for it, so a
|
||||
# cross-host scrape times out. This adds a scoped rich rule — the metrics
|
||||
# stay closed to everything except the Prometheus host.
|
||||
#
|
||||
# Run as root on hanzalova.internal. Idempotent.
|
||||
set -euo pipefail
|
||||
|
||||
MONITOR_IP="10.3.101.4" # golgafrinchans.kosherinata.internal (mesh source)
|
||||
PORT="31314"
|
||||
|
||||
rule="rule family=\"ipv4\" source address=\"${MONITOR_IP}/32\" port port=\"${PORT}\" protocol=\"tcp\" accept"
|
||||
|
||||
if firewall-cmd --permanent --query-rich-rule="${rule}" >/dev/null 2>&1; then
|
||||
echo "rich rule already present; nothing to do"
|
||||
else
|
||||
firewall-cmd --permanent --add-rich-rule="${rule}"
|
||||
firewall-cmd --reload
|
||||
echo "opened tcp/${PORT} to ${MONITOR_IP} and reloaded firewalld"
|
||||
fi
|
||||
|
||||
# Verify from this host; the real check is a scrape from the monitoring host:
|
||||
# curl -s -o /dev/null -w '%{http_code}\n' http://hanzalova.internal:31314/metrics
|
||||
firewall-cmd --list-rich-rules | grep "${PORT}" || true
|
||||
@@ -1,841 +0,0 @@
|
||||
{
|
||||
"uid": "helexa-fleet",
|
||||
"title": "helexa fleet \u2014 capacity & throughput",
|
||||
"tags": [
|
||||
"helexa",
|
||||
"cortex",
|
||||
"capacity"
|
||||
],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "15s",
|
||||
"time": {
|
||||
"from": "now-3h",
|
||||
"to": "now"
|
||||
},
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "datasource",
|
||||
"label": "Prometheus",
|
||||
"type": "datasource",
|
||||
"query": "prometheus",
|
||||
"current": {},
|
||||
"hide": 0
|
||||
},
|
||||
{
|
||||
"name": "node",
|
||||
"label": "neuron",
|
||||
"type": "query",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"query": "label_values(cortex_model_max_in_flight, node)",
|
||||
"refresh": 2,
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"allValue": ".*",
|
||||
"current": {
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "model",
|
||||
"label": "model",
|
||||
"type": "query",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"query": "label_values(cortex_model_max_in_flight{node=~\"$node\"}, model)",
|
||||
"refresh": 2,
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"allValue": ".*",
|
||||
"current": {
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "row",
|
||||
"title": "Capacity & saturation",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "timeseries",
|
||||
"title": "Saturation % (in_flight \u00f7 max_in_flight)",
|
||||
"description": "How full each neuron:model is against its admission ceiling. 100% = every slot busy; sustained >100% headroom pressure means it's about to queue/shed.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 1
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"custom": {
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 70
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 90
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "100 * cortex_model_in_flight{node=~\"$node\", model=~\"$model\"} / clamp_min(cortex_model_max_in_flight{node=~\"$node\", model=~\"$model\"}, 1)",
|
||||
"legendFormat": "{{node}} \u00b7 {{model}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "timeseries",
|
||||
"title": "In-flight vs ceiling",
|
||||
"description": "Live concurrent requests (in_flight) against the configured max_in_flight ceiling.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 1
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "none",
|
||||
"min": 0,
|
||||
"custom": {
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never"
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": ".*ceiling.*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "custom.lineStyle",
|
||||
"value": {
|
||||
"dash": [
|
||||
8,
|
||||
6
|
||||
],
|
||||
"fill": "dash"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "custom.fillOpacity",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "cortex_model_in_flight{node=~\"$node\", model=~\"$model\"}",
|
||||
"legendFormat": "{{node}} \u00b7 {{model}}"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "cortex_model_max_in_flight{node=~\"$node\", model=~\"$model\"}",
|
||||
"legendFormat": "{{node}} \u00b7 {{model}} ceiling"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "timeseries",
|
||||
"title": "Queue depth",
|
||||
"description": "Requests waiting in admission beyond the in-flight slots \u2014 the backpressure building before rejection.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 1
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "none",
|
||||
"min": 0,
|
||||
"custom": {
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "cortex_model_queue_depth{node=~\"$node\", model=~\"$model\"}",
|
||||
"legendFormat": "{{node}} \u00b7 {{model}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "row",
|
||||
"title": "Throughput",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 9
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"type": "timeseries",
|
||||
"title": "Decode tok/s (generation throughput)",
|
||||
"description": "Live decode tokens/sec EMA per neuron:model \u2014 the revenue-capacity number. Honest under batch concurrency (folded in at every serving finish site).",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 10
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "none",
|
||||
"min": 0,
|
||||
"custom": {
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never",
|
||||
"axisLabel": "tok/s"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"mean",
|
||||
"max"
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "cortex_model_tok_s_decode{node=~\"$node\", model=~\"$model\"}",
|
||||
"legendFormat": "{{node}} \u00b7 {{model}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"type": "timeseries",
|
||||
"title": "Prefill tok/s (prompt-processing throughput)",
|
||||
"description": "Live prefill tokens/sec EMA per neuron:model.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 10
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "none",
|
||||
"min": 0,
|
||||
"custom": {
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never",
|
||||
"axisLabel": "tok/s"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"mean",
|
||||
"max"
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "cortex_model_tok_s_prefill{node=~\"$node\", model=~\"$model\"}",
|
||||
"legendFormat": "{{node}} \u00b7 {{model}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 20,
|
||||
"type": "row",
|
||||
"title": "Backpressure & traffic",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 18
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 21,
|
||||
"type": "timeseries",
|
||||
"title": "Rejection rate by reason (load shedding)",
|
||||
"description": "rate() of admission rejections. Any sustained non-zero here means the neuron:model is turning work away \u2014 queue_full/wait_timeout = at capacity, per_principal = a client hit its fair-share cap.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 19
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps",
|
||||
"min": 0,
|
||||
"custom": {
|
||||
"fillOpacity": 15,
|
||||
"showPoints": "never",
|
||||
"stacking": {
|
||||
"mode": "normal"
|
||||
}
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"sum"
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "sum by (node, model, reason) (rate(cortex_model_rejections_total{node=~\"$node\", model=~\"$model\"}[5m]))",
|
||||
"legendFormat": "{{node}} \u00b7 {{model}} \u00b7 {{reason}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 22,
|
||||
"type": "timeseries",
|
||||
"title": "Request & error rate",
|
||||
"description": "Proxied request rate and error rate per neuron:model.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 19
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps",
|
||||
"min": 0,
|
||||
"custom": {
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never"
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": ".*error.*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "red"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "sum by (node, model) (rate(cortex_requests_total{node=~\"$node\", model=~\"$model\"}[5m]))",
|
||||
"legendFormat": "{{node}} \u00b7 {{model}}"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "sum by (node, model) (rate(cortex_request_errors_total{node=~\"$node\", model=~\"$model\"}[5m]))",
|
||||
"legendFormat": "{{node}} \u00b7 {{model}} error"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 23,
|
||||
"type": "timeseries",
|
||||
"title": "TTFT p95 / p50",
|
||||
"description": "Time-to-first-token latency percentiles from the request histogram.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 19
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"min": 0,
|
||||
"custom": {
|
||||
"fillOpacity": 0,
|
||||
"showPoints": "never"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "histogram_quantile(0.95, sum by (le, node, model) (rate(cortex_time_to_first_token_seconds_bucket{node=~\"$node\", model=~\"$model\"}[5m])))",
|
||||
"legendFormat": "p95 {{node}} \u00b7 {{model}}"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "histogram_quantile(0.50, sum by (le, node, model) (rate(cortex_time_to_first_token_seconds_bucket{node=~\"$node\", model=~\"$model\"}[5m])))",
|
||||
"legendFormat": "p50 {{node}} \u00b7 {{model}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 30,
|
||||
"type": "row",
|
||||
"title": "GPU health",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 27
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 31,
|
||||
"type": "timeseries",
|
||||
"title": "VRAM used per device",
|
||||
"description": "Per-device VRAM used (MB). Pair with free to see headroom for loading bigger models.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 28
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "decmbytes",
|
||||
"min": 0,
|
||||
"custom": {
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "cortex_device_vram_used_mb{node=~\"$node\"}",
|
||||
"legendFormat": "{{node}} \u00b7 gpu{{device}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 32,
|
||||
"type": "timeseries",
|
||||
"title": "GPU utilization %",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 28
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"custom": {
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"mean"
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "cortex_device_utilization_pct{node=~\"$node\"}",
|
||||
"legendFormat": "{{node}} \u00b7 gpu{{device}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 33,
|
||||
"type": "timeseries",
|
||||
"title": "GPU temperature \u00b0C",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 28
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "celsius",
|
||||
"min": 0,
|
||||
"custom": {
|
||||
"fillOpacity": 0,
|
||||
"showPoints": "never"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 75
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 87
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"expr": "cortex_device_temp_c{node=~\"$node\"}",
|
||||
"legendFormat": "{{node}} \u00b7 gpu{{device}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 34,
|
||||
"type": "row",
|
||||
"title": "Text-to-image (#203)",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 36
|
||||
},
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 35,
|
||||
"type": "timeseries",
|
||||
"title": "Images generated / hour",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 37
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (model) (increase(cortex_images_generated_total[1h]))",
|
||||
"legendFormat": "{{model}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 36,
|
||||
"type": "timeseries",
|
||||
"title": "Generation latency (p50/p95)",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 37
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(cortex_images_generation_seconds_bucket[10m])))",
|
||||
"legendFormat": "p95",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, sum by (le) (rate(cortex_images_generation_seconds_bucket[10m])))",
|
||||
"legendFormat": "p50",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 37,
|
||||
"type": "timeseries",
|
||||
"title": "Image spend (Mp-steps / hour by account)",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 37
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (account) (increase(cortex_spend_image_milliunits_total[1h])) / 1000",
|
||||
"legendFormat": "{{account}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {},
|
||||
"overrides": []
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
# helexa cortex gateway scrape job — append into the `scrape_configs:` list
|
||||
# of /etc/prometheus/prometheus.yml on golgafrinchans.kosherinata.internal,
|
||||
# then reload (Prometheus is published on host port 26559, NOT 9090 — 9090 is
|
||||
# Cockpit on that box): curl -X POST http://localhost:26559/-/reload
|
||||
#
|
||||
# cortex runs on hanzalova.internal (10.6.0.46) and exposes ALL fleet metrics
|
||||
# on :31314 — it is the only Prometheus target in the stack (neuron has no
|
||||
# /metrics; cortex's poller already relabels everything by {node,model} /
|
||||
# {node,device}). Reaching it cross-DC (kosherinata -> hanzalova) requires the
|
||||
# firewalld rule in asset/monitoring/firewalld-cortex-metrics.sh, which opens
|
||||
# :31314 to this host's mesh IP (10.3.101.4) only.
|
||||
#
|
||||
# Metrics exposed here include (see #137):
|
||||
# cortex_model_in_flight / _queue_depth / _max_in_flight / _max_queue_depth
|
||||
# cortex_model_tok_s_decode / _tok_s_prefill
|
||||
# cortex_model_rejections_total{reason=queue_full|wait_timeout|per_principal}
|
||||
# cortex_device_vram_used_mb / _vram_free_mb / _utilization_pct / _temp_c
|
||||
# cortex_requests_total / _request_errors_total / _cold_starts_total
|
||||
# cortex_request_duration_seconds / _time_to_first_token_seconds (histograms)
|
||||
# cortex_prompt_tokens_total / _completion_tokens_total / cortex_spend_*
|
||||
- job_name: cortex
|
||||
# cortex refreshes the load/device gauges from each neuron's /health on a
|
||||
# ~10s poll; 15s scrape (the global default) tracks it without aliasing.
|
||||
static_configs:
|
||||
- targets: ['hanzalova.internal:31314']
|
||||
labels:
|
||||
fleet: hanzalova
|
||||
@@ -1,37 +0,0 @@
|
||||
# neuron.toml for beast.hanzalova.internal
|
||||
#
|
||||
# 2x RTX 5090 (32 GB each) — TP-2 capable. Pre-warms Qwen3.6-27B with
|
||||
# q5k ISQ across both GPUs at activation, matching the validate-neuron
|
||||
# invocation: `validate-neuron.sh beast.hanzalova.internal
|
||||
# Qwen/Qwen3.6-27B q5k 2`.
|
||||
#
|
||||
# Synced to /etc/neuron/neuron.toml by script/infra-setup.sh. Edits
|
||||
# take effect after the next deploy workflow run restarts the service
|
||||
# (default_models is read at activation).
|
||||
|
||||
port = 13131
|
||||
|
||||
[[harnesses]]
|
||||
name = "candle"
|
||||
|
||||
[harness.candle]
|
||||
|
||||
# Batched decode engine (#98): up to 8 concurrent text streams
|
||||
# multiplex through one lockstep (B,1) forward per decode step.
|
||||
# max_in_flight is both the admission bound and the engine's slot
|
||||
# count. NEURON_BATCHING=0 (systemd drop-in) is the kill switch;
|
||||
# removing this section reverts to batch-1 serialization.
|
||||
[harness.candle.admission]
|
||||
max_in_flight = 8
|
||||
|
||||
[[default_models]]
|
||||
# Qwen3.8-27B is the resident flagship (#251). Architecturally identical
|
||||
# to the 3.6-27B it replaced — same model_type, layer counts and vision
|
||||
# tower — so the load spec is unchanged. The 3.6-27B stays catalogued at
|
||||
# the same residency class, so it can still be cold-loaded on demand and
|
||||
# the two displace each other in either direction.
|
||||
model_id = "Qwen/Qwen3.8-27B"
|
||||
harness = "candle"
|
||||
quant = "q6k"
|
||||
tensor_parallel = 2
|
||||
devices = [0, 1]
|
||||
@@ -1,41 +0,0 @@
|
||||
# neuron.toml for benjy.hanzalova.internal
|
||||
#
|
||||
# 1x RTX 4090 (24 GB) — largest single-GPU host on the fleet. Pre-warms
|
||||
# Qwen3-8B (bf16, ~18 GB), leaving ~6 GB for KV cache + activations on
|
||||
# moderate-length contexts.
|
||||
#
|
||||
# Synced to /etc/neuron/neuron.toml by script/infra-setup.sh.
|
||||
|
||||
port = 13131
|
||||
|
||||
[[harnesses]]
|
||||
name = "candle"
|
||||
|
||||
[harness.candle]
|
||||
|
||||
[[default_models]]
|
||||
model_id = "Qwen/Qwen3-8B"
|
||||
harness = "candle"
|
||||
devices = [0]
|
||||
|
||||
# Admission (#53): Qwen3-8B is the classic qwen3 arch, which has no KV
|
||||
# snapshot support — forwards serialize on the inference lock, so the
|
||||
# batch engine (and any real max_in_flight > 1) doesn't apply here; the
|
||||
# 24 GB card is not the limiter. Capacity for bursts comes from the
|
||||
# QUEUE: ~5-8 s per chat reply means a full queue drains in under a
|
||||
# minute, and waiters beyond max_wait get an honest 429 instead of an
|
||||
# invisible lock wait. Revisit with prom traffic data; the real unlock
|
||||
# is porting KV snapshots to the qwen3 dense arch.
|
||||
[harness.candle.admission]
|
||||
max_in_flight = 1
|
||||
max_queue_depth = 24
|
||||
max_wait_secs = 60
|
||||
|
||||
# Text-to-image (#197/#203). Z-Image-Turbo cold-swaps against Qwen3-8B
|
||||
# on the 4090 (operator decision 2026-07-29): cortex evicts the 8B on
|
||||
# an images request and the 8B cold-loads back on demand. The Qwen3-4B
|
||||
# text encoder runs on the host CPU (the default) — GPU TE + resident
|
||||
# DiT OOMs 24 GB — and stays resident in system RAM.
|
||||
[harness.candle.image]
|
||||
te_device = "cpu"
|
||||
max_dim = 2048
|
||||
@@ -1,28 +0,0 @@
|
||||
# neuron.toml for quadbrat.hanzalova.internal
|
||||
#
|
||||
# 1x RTX 3060 (12 GB) — small / quantised tier. Pre-warms Qwen3-1.7B
|
||||
# (bf16, ~4 GB), leaving ~7 GB for KV cache so long contexts on a small
|
||||
# model still have plenty of room.
|
||||
#
|
||||
# Synced to /etc/neuron/neuron.toml by script/infra-setup.sh.
|
||||
|
||||
port = 13131
|
||||
|
||||
[[harnesses]]
|
||||
name = "candle"
|
||||
|
||||
[harness.candle]
|
||||
|
||||
[[default_models]]
|
||||
model_id = "Qwen/Qwen3-1.7B"
|
||||
harness = "candle"
|
||||
devices = [0]
|
||||
|
||||
# Text-to-image (#204): the 12 GB tier serves the q8_0-quantized DiT
|
||||
# (8.7 GB resident incl. VAE). Ceiling is 768² — naive f32 attention
|
||||
# transients OOM 1024² beside the resident model; lifts to 1024² if
|
||||
# #95 extends flash-attn to the ampere flavour. Load spec: quant =
|
||||
# "q8_0" on the load request / catalogue.
|
||||
[harness.candle.image]
|
||||
te_device = "cpu"
|
||||
max_dim = 768
|
||||
@@ -1,16 +0,0 @@
|
||||
# Shared proxy parameters for the angels.helexa.ai vhost.
|
||||
#
|
||||
# Installed to /etc/nginx/conf.d/angels-proxy-params.conf and included by
|
||||
# each location block, so the four of them cannot drift apart.
|
||||
#
|
||||
# X-Forwarded-For is load-bearing rather than decorative here: the access
|
||||
# record stores the client IP, and without this every row would read as
|
||||
# the edge proxy's address.
|
||||
|
||||
proxy_pass_header Server;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_read_timeout 60s;
|
||||
@@ -1,19 +0,0 @@
|
||||
# Bootstrap vhost for angels.helexa.ai — http-only, used ONLY to obtain
|
||||
# the initial Let's Encrypt certificate (the TLS vhost cannot load before
|
||||
# the cert file exists). script/infra-setup.sh installs this, runs
|
||||
# certbot, then swaps in angels.helexa.ai.conf.
|
||||
#
|
||||
# Serves nothing but the ACME challenge: the portal itself must never be
|
||||
# reachable over plaintext, even for the minute this file is in place.
|
||||
server {
|
||||
listen 80;
|
||||
listen [::]:80;
|
||||
server_name angels.helexa.ai;
|
||||
|
||||
location /.well-known/acme-challenge/ {
|
||||
root /var/www/angels.helexa.ai;
|
||||
}
|
||||
location / {
|
||||
return 404;
|
||||
}
|
||||
}
|
||||
@@ -1,106 +0,0 @@
|
||||
# Confidential investor portal at https://angels.helexa.ai.
|
||||
#
|
||||
# Unlike helexa.ai and bench.helexa.ai there is NO static root here and
|
||||
# nothing is served from disk: every byte comes from helexa-angels on
|
||||
# gallumbits, which assembles each page server-side after checking the
|
||||
# session. That is the whole reason the portal is a separate service — a
|
||||
# static bundle cannot keep a secret, and a client-side route guard gates
|
||||
# navigation rather than access.
|
||||
#
|
||||
# Installed on BOTH edge proxies (oolon.kosherinata.internal and
|
||||
# hanzalova.internal), matching helexa.ai, so the Cloudflare A records
|
||||
# can point at either site.
|
||||
#
|
||||
# The real client address arrives via PROXY protocol from the SNI router
|
||||
# and is recovered by conf.d/proxy-protocol.conf's real_ip directives —
|
||||
# which is what makes $proxy_add_x_forwarded_for, and therefore the IP in
|
||||
# every access record, the visitor's rather than the router's.
|
||||
#
|
||||
# TLS via Let's Encrypt (certbot, Cloudflare DNS-01, ECDSA) —
|
||||
# bootstrapped one-time in script/infra-setup.sh.
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
listen [::]:80;
|
||||
server_name angels.helexa.ai;
|
||||
|
||||
location /.well-known/acme-challenge/ {
|
||||
root /var/www/angels.helexa.ai;
|
||||
}
|
||||
location / {
|
||||
return 301 https://$host$request_uri;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
# TCP 443 on the edge is owned by the stream SNI router
|
||||
# (streams-enabled/sni-router.conf, architecture reverse-proxies.md
|
||||
# §5), which preads the SNI and hands every local name to this http
|
||||
# tier over PROXY protocol. Listening on :443 directly here would
|
||||
# never be reached — the router would answer first with whichever
|
||||
# vhost happened to be default, which is exactly the symptom seen
|
||||
# when this file was first written against the older pattern.
|
||||
listen 127.0.0.1:14443 ssl proxy_protocol;
|
||||
http2 on;
|
||||
server_name angels.helexa.ai;
|
||||
|
||||
ssl_certificate /etc/letsencrypt/live/angels.helexa.ai/fullchain.pem;
|
||||
ssl_certificate_key /etc/letsencrypt/live/angels.helexa.ai/privkey.pem;
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
ssl_ciphers HIGH:!aNULL:!MD5;
|
||||
ssl_prefer_server_ciphers on;
|
||||
ssl_session_cache shared:SSL:10m;
|
||||
|
||||
# ── Confidentiality headers ─────────────────────────────────────
|
||||
# Also emitted by the application in each page's <head>. Both, on
|
||||
# purpose: either alone is a single point of failure, and the cost of
|
||||
# duplication is nil.
|
||||
#
|
||||
# `always` matters — without it these are dropped on error responses,
|
||||
# which is exactly when a crawler is most likely to be looking.
|
||||
add_header X-Robots-Tag "noindex, nofollow, noarchive, nosnippet" always;
|
||||
add_header Referrer-Policy "no-referrer" always;
|
||||
add_header X-Content-Type-Options "nosniff" always;
|
||||
add_header X-Frame-Options "DENY" always;
|
||||
# The portal loads nothing from anywhere else — no CDN, no fonts, no
|
||||
# analytics — so the policy can be this tight, and the privacy note
|
||||
# says as much in terms a visitor can verify in their network tab.
|
||||
add_header Content-Security-Policy
|
||||
"default-src 'none'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; form-action 'self'; base-uri 'none'; frame-ancestors 'none'" always;
|
||||
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
|
||||
|
||||
# Keep renewals working.
|
||||
location /.well-known/acme-challenge/ {
|
||||
root /var/www/angels.helexa.ai;
|
||||
}
|
||||
|
||||
# ── Rate limits ─────────────────────────────────────────────────
|
||||
# Zones are declared in conf.d/helexa-ratelimit.conf.
|
||||
#
|
||||
# The invite path is the one an attacker would grind: codes are 32
|
||||
# base62 characters, so guessing is not a realistic threat, but there
|
||||
# is no reason to serve the attempts. Sign-in is limited because it
|
||||
# is the only endpoint that costs real CPU (argon2, deliberately).
|
||||
location /i/ {
|
||||
limit_req zone=angels_invite burst=5 nodelay;
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8092;
|
||||
include /etc/nginx/conf.d/angels-proxy-params.conf;
|
||||
}
|
||||
|
||||
location /signin {
|
||||
limit_req zone=angels_auth burst=5 nodelay;
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8092;
|
||||
include /etc/nginx/conf.d/angels-proxy-params.conf;
|
||||
}
|
||||
|
||||
location /register {
|
||||
limit_req zone=angels_auth burst=5 nodelay;
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8092;
|
||||
include /etc/nginx/conf.d/angels-proxy-params.conf;
|
||||
}
|
||||
|
||||
location / {
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8092;
|
||||
include /etc/nginx/conf.d/angels-proxy-params.conf;
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
# Bootstrap vhost for bench.helexa.ai — http-only, used ONLY to obtain
|
||||
# the initial Let's Encrypt cert via the webroot challenge (the full TLS
|
||||
# vhost can't load before the cert file exists). script/infra-setup.sh
|
||||
# installs this, runs certbot, then swaps in bench.helexa.ai.conf.
|
||||
server {
|
||||
listen 80;
|
||||
server_name bench.helexa.ai;
|
||||
|
||||
location /.well-known/acme-challenge/ {
|
||||
root /var/www/bench.helexa.ai;
|
||||
}
|
||||
location / {
|
||||
try_files $uri $uri/ =404;
|
||||
}
|
||||
}
|
||||
@@ -1,70 +0,0 @@
|
||||
# Public, auth-less bench UI at https://bench.helexa.ai.
|
||||
#
|
||||
# Serves the static SPA from /var/www/bench.helexa.ai (rsynced by
|
||||
# .gitea/workflows/deploy.yml's deploy-bench-ui job) and reverse-proxies
|
||||
# /api to the helexa-bench read API on bob over the WireGuard mesh — so
|
||||
# the browser stays same-origin (no CORS) and the internal API never
|
||||
# needs to be exposed publicly.
|
||||
#
|
||||
# TLS via Let's Encrypt; the cert is obtained/renewed by certbot
|
||||
# (bootstrapped one-time in script/infra-setup.sh). Mirrors the
|
||||
# dev.swym.hanzalova.internal vhost convention on this host.
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name bench.helexa.ai;
|
||||
|
||||
# Keep serving the ACME webroot so certbot can renew.
|
||||
location /.well-known/acme-challenge/ {
|
||||
root /var/www/bench.helexa.ai;
|
||||
}
|
||||
location / {
|
||||
return 301 https://$host$request_uri;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
# TCP 443 on this host is owned by the stream SNI router
|
||||
# (streams-enabled/sni-router.conf), which preads the server name and
|
||||
# forwards to the local https tier on 127.0.0.1:14443 over PROXY
|
||||
# protocol. A vhost that listens on :443 directly is never reached —
|
||||
# the router answers first, with whichever certificate its default
|
||||
# branch holds. `nginx -t` passes either way, so the mistake shows up
|
||||
# only as the wrong certificate on a working handshake.
|
||||
#
|
||||
# Prerequisites on the host, both owned by the edge/mail
|
||||
# infrastructure rather than this repo (and host-specific — the two
|
||||
# edges route different mail names, so neither is synced from here):
|
||||
# streams-enabled/sni-router.conf — owns :443, preads SNI
|
||||
# conf.d/proxy-protocol.conf — real_ip, so $binary_remote_addr
|
||||
# and X-Real-IP stay truthful
|
||||
listen 127.0.0.1:14443 ssl proxy_protocol;
|
||||
http2 on;
|
||||
server_name bench.helexa.ai;
|
||||
|
||||
ssl_certificate /etc/letsencrypt/live/bench.helexa.ai/fullchain.pem;
|
||||
ssl_certificate_key /etc/letsencrypt/live/bench.helexa.ai/privkey.pem;
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
ssl_ciphers HIGH:!aNULL:!MD5;
|
||||
ssl_prefer_server_ciphers on;
|
||||
ssl_session_cache shared:SSL:10m;
|
||||
|
||||
root /var/www/bench.helexa.ai;
|
||||
index index.html;
|
||||
|
||||
# Bench read API on bob (internal WireGuard); browser stays same-origin.
|
||||
location /api/ {
|
||||
proxy_pass http://bob.hanzalova.internal:13132;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_read_timeout 60s;
|
||||
}
|
||||
|
||||
# SPA fallback — client-side routes (/trends, /runs) resolve to index.html.
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
# Internal bench UI vhost — https://bench.internal, reachable from inside
|
||||
# the WireGuard mesh (the public bench.helexa.ai dead-ends at the OPNsense
|
||||
# LAN interface, which only port-forwards :443 from the WAN). Same SPA +
|
||||
# /api→bob proxy as bench.helexa.ai, but with an internal-CA cert
|
||||
# (smallstep "lair", renewed by step@bench.timer). Mirrors the
|
||||
# *.internal vhost convention on oolon.kosherinata.internal.
|
||||
server {
|
||||
server_name bench.internal;
|
||||
# TCP 443 on this host is owned by the stream SNI router
|
||||
# (streams-enabled/sni-router.conf), which preads the server name and
|
||||
# forwards to the local https tier on 127.0.0.1:14443 over PROXY
|
||||
# protocol. A vhost that listens on :443 directly is never reached —
|
||||
# the router answers first, with whichever certificate its default
|
||||
# branch holds. `nginx -t` passes either way, so the mistake shows up
|
||||
# only as the wrong certificate on a working handshake.
|
||||
#
|
||||
# Prerequisites on the host, both owned by the edge/mail
|
||||
# infrastructure rather than this repo (and host-specific — the two
|
||||
# edges route different mail names, so neither is synced from here):
|
||||
# streams-enabled/sni-router.conf — owns :443, preads SNI
|
||||
# conf.d/proxy-protocol.conf — real_ip, so $binary_remote_addr
|
||||
# and X-Real-IP stay truthful
|
||||
listen 127.0.0.1:14443 ssl proxy_protocol;
|
||||
http2 on;
|
||||
|
||||
ssl_certificate /etc/nginx/tls/cert/bench.internal.pem;
|
||||
ssl_certificate_key /etc/nginx/tls/key/bench.internal.pem;
|
||||
ssl_trusted_certificate /etc/pki/ca-trust/source/anchors/root-internal.pem;
|
||||
ssl_protocols TLSv1.3;
|
||||
|
||||
# Shared webroot with the public vhost — same built SPA.
|
||||
root /var/www/bench.helexa.ai;
|
||||
index index.html;
|
||||
|
||||
location /api/ {
|
||||
proxy_pass http://bob.hanzalova.internal:13132;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_read_timeout 60s;
|
||||
}
|
||||
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
# Per-client-IP request-rate zones for the helexa.ai vhosts, installed to
|
||||
# /etc/nginx/conf.d/ (http context) by script/infra-setup.sh.
|
||||
#
|
||||
# The anonymous inference path deliberately carries no client identifier
|
||||
# (see /privacy) — per-IP limiting at the edge is the compliant abuse
|
||||
# lever. Authenticated users share the same per-IP ceiling; generous
|
||||
# enough for interactive chat, tight enough to stop a loop. Budgets and
|
||||
# per-key fair-share are enforced deeper in the chain (upstream + cortex).
|
||||
limit_req_zone $binary_remote_addr zone=helexa_v1:10m rate=10r/m;
|
||||
limit_req_zone $binary_remote_addr zone=helexa_api:10m rate=60r/m;
|
||||
# Tool calls (#177): an agentic turn can issue a few searches back to
|
||||
# back, so this runs looser than helexa_v1 — but each search rides an
|
||||
# inference round-trip that IS helexa_v1-limited, so the pair can't
|
||||
# runaway-loop.
|
||||
limit_req_zone $binary_remote_addr zone=helexa_tools:10m rate=30r/m;
|
||||
# angels.helexa.ai (the confidential investor portal). Both are
|
||||
# deliberately tight: the audience is a handful of named people, so
|
||||
# anything resembling volume is either an attack or a bug.
|
||||
# angels_invite — the /i/<code> entry path. Codes are 32 base62
|
||||
# characters so guessing is not a realistic threat, but there is no
|
||||
# reason to serve the attempts.
|
||||
# angels_auth — sign-in and registration. The only endpoints that cost
|
||||
# real CPU, since argon2 is expensive on purpose.
|
||||
limit_req_zone $binary_remote_addr zone=angels_invite:1m rate=20r/m;
|
||||
limit_req_zone $binary_remote_addr zone=angels_auth:1m rate=12r/m;
|
||||
limit_req_status 429;
|
||||
@@ -1,17 +0,0 @@
|
||||
# Bootstrap vhost for helexa.ai — http-only, used ONLY until the initial
|
||||
# Let's Encrypt cert exists (the full TLS vhost can't load before the
|
||||
# cert file does — nginx -t fails on a missing ssl_certificate).
|
||||
# script/infra-setup.sh installs this, obtains the cert via Cloudflare
|
||||
# DNS-01, then swaps in helexa.ai.conf.
|
||||
server {
|
||||
listen 80;
|
||||
listen [::]:80;
|
||||
server_name helexa.ai;
|
||||
|
||||
location /.well-known/acme-challenge/ {
|
||||
root /var/www/helexa.ai;
|
||||
}
|
||||
location / {
|
||||
try_files $uri $uri/ =404;
|
||||
}
|
||||
}
|
||||
@@ -1,132 +0,0 @@
|
||||
# Public helexa.ai website — the built SPA (helexa.ai/dist), rsynced to
|
||||
# /var/www/helexa.ai by .gitea/workflows/deploy.yml's deploy-website job.
|
||||
#
|
||||
# Installed on BOTH edge proxies (oolon.kosherinata.internal and
|
||||
# hanzalova.internal); Cloudflare DNS load balancing routes the public
|
||||
# name to the two site WAN IPs. Mesh clients use helexa.internal instead
|
||||
# (public names don't hairpin — see architecture/reverse-proxies.md §2).
|
||||
#
|
||||
# Same-origin backends (#167): /v1 (+ /health) reverse-proxies to
|
||||
# helexa-router and /api to helexa-upstream, both on gallumbits — so the
|
||||
# browser never makes a cross-origin request and bearers ride
|
||||
# first-party. Per-IP rate zones live in conf.d/helexa-ratelimit.conf.
|
||||
#
|
||||
# TLS via Let's Encrypt (certbot, Cloudflare DNS-01, ECDSA) —
|
||||
# bootstrapped one-time in script/infra-setup.sh.
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
listen [::]:80;
|
||||
server_name helexa.ai;
|
||||
|
||||
# Keep serving the ACME webroot so certbot can renew.
|
||||
location /.well-known/acme-challenge/ {
|
||||
root /var/www/helexa.ai;
|
||||
}
|
||||
location / {
|
||||
return 301 https://$host$request_uri;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
# TCP 443 on this host is owned by the stream SNI router
|
||||
# (streams-enabled/sni-router.conf), which preads the server name and
|
||||
# forwards to the local https tier on 127.0.0.1:14443 over PROXY
|
||||
# protocol. A vhost that listens on :443 directly is never reached —
|
||||
# the router answers first, with whichever certificate its default
|
||||
# branch holds. `nginx -t` passes either way, so the mistake shows up
|
||||
# only as the wrong certificate on a working handshake.
|
||||
#
|
||||
# Prerequisites on the host, both owned by the edge/mail
|
||||
# infrastructure rather than this repo (and host-specific — the two
|
||||
# edges route different mail names, so neither is synced from here):
|
||||
# streams-enabled/sni-router.conf — owns :443, preads SNI
|
||||
# conf.d/proxy-protocol.conf — real_ip, so $binary_remote_addr
|
||||
# and X-Real-IP stay truthful
|
||||
listen 127.0.0.1:14443 ssl proxy_protocol;
|
||||
http2 on;
|
||||
server_name helexa.ai;
|
||||
|
||||
# This vhost listens on an internal port behind the edge SNI router,
|
||||
# so nginx's own idea of "where am I" is 127.0.0.1:14443 rather than
|
||||
# the address the client used. Absolute redirects built from that
|
||||
# leak the internal port -- a request for a directory without a
|
||||
# trailing slash answered 301 to https://<host>:14443/<path>/, which
|
||||
# no client outside this machine can reach.
|
||||
#
|
||||
# Relative redirects sidestep the question: the browser resolves the
|
||||
# Location against the URL it actually requested, which is correct
|
||||
# for every deployment shape this vhost has.
|
||||
absolute_redirect off;
|
||||
|
||||
ssl_certificate /etc/letsencrypt/live/helexa.ai/fullchain.pem;
|
||||
ssl_certificate_key /etc/letsencrypt/live/helexa.ai/privkey.pem;
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
ssl_ciphers HIGH:!aNULL:!MD5;
|
||||
ssl_prefer_server_ciphers on;
|
||||
ssl_session_cache shared:SSL:10m;
|
||||
|
||||
root /var/www/helexa.ai;
|
||||
index index.html;
|
||||
|
||||
# Long-cache fingerprinted assets; never cache the HTML shell.
|
||||
location /assets/ {
|
||||
expires 1y;
|
||||
add_header Cache-Control "public, immutable";
|
||||
}
|
||||
|
||||
# Inference data plane → helexa-router on gallumbits. Streaming
|
||||
# (SSE): buffering off so tokens reach the browser as they arrive.
|
||||
location /v1/ {
|
||||
limit_req zone=helexa_v1 burst=10 nodelay;
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8088;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header Connection "";
|
||||
proxy_buffering off;
|
||||
proxy_read_timeout 300s;
|
||||
}
|
||||
location = /health {
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8088;
|
||||
}
|
||||
|
||||
# Account control plane → helexa-upstream /web/v1/ (strip the /api
|
||||
# prefix).
|
||||
# Page-read tool for the chat app (#177) -> helexa-tools on the
|
||||
# federation service host (SSRF-guarded readability fetcher).
|
||||
location = /tools/fetch {
|
||||
limit_req zone=helexa_tools burst=10 nodelay;
|
||||
limit_except GET { deny all; }
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8889/fetch;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_read_timeout 40s;
|
||||
}
|
||||
|
||||
# Web-search tool for the chat app (#177) -> SearXNG on the
|
||||
# federation service host. GET only; the SPA requests format=json.
|
||||
location = /tools/web_search {
|
||||
limit_req zone=helexa_tools burst=10 nodelay;
|
||||
limit_except GET { deny all; }
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8888/search;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_read_timeout 30s;
|
||||
}
|
||||
|
||||
location /api/ {
|
||||
limit_req zone=helexa_api burst=20 nodelay;
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8090/web/v1/;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
|
||||
# SPA history fallback — client-side routes resolve to index.html.
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
}
|
||||
@@ -1,104 +0,0 @@
|
||||
# Internal helexa website vhost — https://helexa.internal, reachable from
|
||||
# inside the WireGuard mesh (the public helexa.ai dead-ends at the
|
||||
# OPNsense LAN interface, which only port-forwards :443 from the WAN —
|
||||
# see architecture/reverse-proxies.md §2). Same SPA webroot as the
|
||||
# public vhost, but with an internal-CA cert (smallstep "lair", renewed
|
||||
# by step@helexa.timer). Installed on hanzalova.internal only; the
|
||||
# split-horizon helexa.internal record points at hanzalova's mesh IP on
|
||||
# both site routers.
|
||||
server {
|
||||
server_name helexa.internal;
|
||||
# TCP 443 on this host is owned by the stream SNI router
|
||||
# (streams-enabled/sni-router.conf), which preads the server name and
|
||||
# forwards to the local https tier on 127.0.0.1:14443 over PROXY
|
||||
# protocol. A vhost that listens on :443 directly is never reached —
|
||||
# the router answers first, with whichever certificate its default
|
||||
# branch holds. `nginx -t` passes either way, so the mistake shows up
|
||||
# only as the wrong certificate on a working handshake.
|
||||
#
|
||||
# Prerequisites on the host, both owned by the edge/mail
|
||||
# infrastructure rather than this repo (and host-specific — the two
|
||||
# edges route different mail names, so neither is synced from here):
|
||||
# streams-enabled/sni-router.conf — owns :443, preads SNI
|
||||
# conf.d/proxy-protocol.conf — real_ip, so $binary_remote_addr
|
||||
# and X-Real-IP stay truthful
|
||||
listen 127.0.0.1:14443 ssl proxy_protocol;
|
||||
http2 on;
|
||||
|
||||
# This vhost listens on an internal port behind the edge SNI router,
|
||||
# so nginx's own idea of "where am I" is 127.0.0.1:14443 rather than
|
||||
# the address the client used. Absolute redirects built from that
|
||||
# leak the internal port -- a request for a directory without a
|
||||
# trailing slash answered 301 to https://<host>:14443/<path>/, which
|
||||
# no client outside this machine can reach.
|
||||
#
|
||||
# Relative redirects sidestep the question: the browser resolves the
|
||||
# Location against the URL it actually requested, which is correct
|
||||
# for every deployment shape this vhost has.
|
||||
absolute_redirect off;
|
||||
|
||||
ssl_certificate /etc/nginx/tls/cert/helexa.internal.pem;
|
||||
ssl_certificate_key /etc/nginx/tls/key/helexa.internal.pem;
|
||||
ssl_trusted_certificate /etc/pki/ca-trust/source/anchors/root-internal.pem;
|
||||
ssl_protocols TLSv1.3;
|
||||
|
||||
# Shared webroot with the public vhost — same built SPA.
|
||||
root /var/www/helexa.ai;
|
||||
index index.html;
|
||||
|
||||
location /assets/ {
|
||||
expires 1y;
|
||||
add_header Cache-Control "public, immutable";
|
||||
}
|
||||
|
||||
# Same backends as the public vhost (#167): /v1 → helexa-router,
|
||||
# /api → helexa-upstream, both on gallumbits over the mesh.
|
||||
location /v1/ {
|
||||
limit_req zone=helexa_v1 burst=10 nodelay;
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8088;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header Connection "";
|
||||
proxy_buffering off;
|
||||
proxy_read_timeout 300s;
|
||||
}
|
||||
location = /health {
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8088;
|
||||
}
|
||||
# Page-read tool for the chat app (#177) -> helexa-tools on the
|
||||
# federation service host (SSRF-guarded readability fetcher).
|
||||
location = /tools/fetch {
|
||||
limit_req zone=helexa_tools burst=10 nodelay;
|
||||
limit_except GET { deny all; }
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8889/fetch;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_read_timeout 40s;
|
||||
}
|
||||
|
||||
# Web-search tool for the chat app (#177) -> SearXNG on the
|
||||
# federation service host. GET only; the SPA requests format=json.
|
||||
location = /tools/web_search {
|
||||
limit_req zone=helexa_tools burst=10 nodelay;
|
||||
limit_except GET { deny all; }
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8888/search;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_read_timeout 30s;
|
||||
}
|
||||
|
||||
location /api/ {
|
||||
limit_req zone=helexa_api burst=20 nodelay;
|
||||
proxy_pass http://gallumbits.kosherinata.internal:8090/web/v1/;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<service>
|
||||
<short>helexa-searxng</short>
|
||||
<description>SearXNG metasearch backing the helexa /tools/web_search API (#177). Reached only by the edge reverse proxies over the site mesh.</description>
|
||||
<port protocol="tcp" port="8888"/>
|
||||
</service>
|
||||
@@ -1,27 +0,0 @@
|
||||
# Quadlet unit for the SearXNG metasearch container backing the helexa
|
||||
# web_search tool (#177). Installed to /etc/containers/systemd/ on the
|
||||
# federation service host by script/infra-setup.sh; podman-systemd
|
||||
# generates searxng.service from it on daemon-reload.
|
||||
#
|
||||
# Reached only by the edge reverse proxies (/tools/web_search) over the
|
||||
# site mesh; the helexa-searxng firewalld service opens tcp/8888.
|
||||
|
||||
[Unit]
|
||||
Description=SearXNG metasearch (helexa web_search tool)
|
||||
Wants=network-online.target
|
||||
After=network-online.target
|
||||
|
||||
[Container]
|
||||
Image=docker.io/searxng/searxng:latest
|
||||
PublishPort=8888:8080
|
||||
# rw: the image entrypoint generates uwsgi.ini beside settings.yml on
|
||||
# first start. The settings file itself is re-synced by infra-setup.
|
||||
Volume=/etc/searxng:/etc/searxng:Z
|
||||
EnvironmentFile=/etc/searxng/searxng.env
|
||||
AutoUpdate=registry
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,24 +0,0 @@
|
||||
# SearXNG settings for the helexa web_search tool backend (#177).
|
||||
# Installed to /etc/searxng/settings.yml on the federation service host
|
||||
# by script/infra-setup.sh; consumed by the quadlet container
|
||||
# (asset/searxng/searxng.container).
|
||||
#
|
||||
# The secret key is NOT here — it comes from SEARXNG_SECRET in the
|
||||
# operator-owned /etc/searxng/searxng.env (generated once by
|
||||
# infra-setup, never in git).
|
||||
use_default_settings: true
|
||||
|
||||
server:
|
||||
# Bot-detection limiter off: the instance is reachable only from the
|
||||
# edge proxies over the site mesh, and per-client-IP rate limiting is
|
||||
# enforced there (helexa-ratelimit.conf helexa_tools zone).
|
||||
limiter: false
|
||||
image_proxy: false
|
||||
method: "GET"
|
||||
|
||||
search:
|
||||
# `json` enables the machine-readable API the chat SPA's tool loop
|
||||
# consumes (GET /search?q=…&format=json).
|
||||
formats:
|
||||
- html
|
||||
- json
|
||||
@@ -1,25 +0,0 @@
|
||||
# Install on the bench host (bob) as /etc/sudoers.d/helexa_gitea_ci
|
||||
# (owner root:root, mode 0440). Required by .gitea/workflows/deploy.yml,
|
||||
# which SSHes as gitea_ci@bob to roll out helexa-bench package upgrades
|
||||
# and config changes.
|
||||
#
|
||||
# Filename convention `helexa_gitea_ci` (vs bare `gitea_ci`) so other
|
||||
# helexa-org apps can drop their own sudoers files on the same host
|
||||
# without overwriting this one.
|
||||
#
|
||||
# helexa-bench polls the neuron fleet (outbound) and serves a read-only
|
||||
# JSON API on tcp/13132 for the bench UI — hence the firewall-cmd grants.
|
||||
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/helexa-bench/helexa-bench.toml
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl start helexa-bench.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl stop helexa-bench.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable --now helexa-bench.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf install --refresh --allowerasing -y helexa-bench
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf upgrade --refresh --allowerasing -y helexa-bench
|
||||
# sudoers reserves `:` and `=` and requires `\` escaping inside command
|
||||
# arguments — without it visudo errors at the first `:` in `https://`.
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf config-manager addrepo --from-repofile\=https\://rpm.lair.cafe/lair-cafe-unstable.repo
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf config-manager setopt lair-cafe-unstable.enabled\=1
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --add-service=helexa-bench --permanent
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --reload
|
||||
@@ -1,25 +0,0 @@
|
||||
# Install on the cortex gateway host as /etc/sudoers.d/helexa_gitea_ci
|
||||
# (owner root:root, mode 0440). Required by .gitea/workflows/deploy.yml,
|
||||
# which SSHes as gitea_ci@<gateway> to roll out cortex package upgrades
|
||||
# and config changes.
|
||||
#
|
||||
# Filename convention `helexa_gitea_ci` (vs bare `gitea_ci`) so other
|
||||
# helexa-org apps can drop their own sudoers files on the same host
|
||||
# without overwriting this one.
|
||||
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/cortex/cortex.toml
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/cortex/models.toml
|
||||
# deploy-bench-ui rsyncs the built bench SPA into the nginx webroot.
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /var/www/bench.helexa.ai/
|
||||
# deploy-website rsyncs the built helexa.ai SPA into the nginx webroot.
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /var/www/helexa.ai/
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl start cortex.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl stop cortex.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable --now cortex.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf install --refresh --allowerasing -y cortex
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf upgrade --refresh --allowerasing -y cortex
|
||||
# sudoers reserves `:` and `=` and requires `\` escaping inside command
|
||||
# arguments — without it visudo errors at the first `:` in `https://`.
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf config-manager addrepo --from-repofile\=https\://rpm.lair.cafe/lair-cafe-unstable.repo
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf config-manager setopt lair-cafe-unstable.enabled\=1
|
||||
@@ -1,40 +0,0 @@
|
||||
# Install on the federation service host (gallumbits.kosherinata.internal)
|
||||
# as /etc/sudoers.d/helexa_gitea_ci (owner root:root, mode 0440).
|
||||
# Required by .gitea/workflows/deploy.yml's deploy-gallumbits job, which
|
||||
# rolls out helexa-router (data plane), helexa-upstream (account/budget
|
||||
# authority), helexa-tools (grounding fetcher) and helexa-angels (the
|
||||
# confidential investor portal) package upgrades.
|
||||
# Configs are NOT synced by CI — they carry secrets (DB password, JWT
|
||||
# secret, operator tokens) and ride in script/infra-setup.sh under the
|
||||
# operator's own sudo instead.
|
||||
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl start helexa-router.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl stop helexa-router.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable --now helexa-router.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl start helexa-upstream.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl stop helexa-upstream.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable --now helexa-upstream.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl start helexa-tools.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl stop helexa-tools.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable --now helexa-tools.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl start helexa-angels.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl stop helexa-angels.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable --now helexa-angels.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf install --refresh --allowerasing -y helexa-router
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf upgrade --refresh --allowerasing -y helexa-router
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf install --refresh --allowerasing -y helexa-upstream
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf upgrade --refresh --allowerasing -y helexa-upstream
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf install --refresh --allowerasing -y helexa-tools
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf upgrade --refresh --allowerasing -y helexa-tools
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf install --refresh --allowerasing -y helexa-angels
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf upgrade --refresh --allowerasing -y helexa-angels
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --add-service=helexa-router --permanent
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --add-service=helexa-upstream --permanent
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --add-service=helexa-tools --permanent
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --add-service=helexa-angels --permanent
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --reload
|
||||
# sudoers reserves `:` and `=` and requires `\` escaping inside command
|
||||
# arguments — without it visudo errors at the first `:` in `https://`.
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf config-manager addrepo --from-repofile\=https\://rpm.lair.cafe/lair-cafe-unstable.repo
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf config-manager setopt lair-cafe-unstable.enabled\=1
|
||||
@@ -1,43 +0,0 @@
|
||||
# Install on every neuron host as /etc/sudoers.d/helexa_gitea_ci
|
||||
# (owner root:root, mode 0440). Required by .gitea/workflows/deploy.yml,
|
||||
# which SSHes as gitea_ci@<neuron-host> to roll out helexa-neuron-<flavour>
|
||||
# package upgrades and config changes.
|
||||
#
|
||||
# Filename convention `helexa_gitea_ci` (vs bare `gitea_ci`) so other
|
||||
# helexa-org apps can drop their own sudoers files on the same host
|
||||
# without overwriting this one.
|
||||
#
|
||||
# All three CUDA flavours are listed because a host's flavour can change
|
||||
# (e.g. GPU swap) and we don't want the sudoers file to need to change
|
||||
# in lockstep. Only one flavour can be installed at a time (the packages
|
||||
# Conflict: with each other), so the attack surface is bounded to "wrong
|
||||
# flavour installed" — vandalism, not privilege escalation.
|
||||
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/neuron/neuron.toml
|
||||
# deploy.yml writes the per-model systemd drop-in carrying
|
||||
# NEURON_MAX_PROMPT_TOKENS: gitea_ci stages it in its own dir, then
|
||||
# installs it root-owned. Exact source/dest paths; see doc/context-limits.md.
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/install -o root -g root -m 0644 -D /var/lib/gitea_ci/model.conf /etc/systemd/system/neuron.service.d/model.conf
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl start neuron.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl stop neuron.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable --now neuron.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf install --refresh --allowerasing -y helexa-neuron-ampere
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf upgrade --refresh --allowerasing -y helexa-neuron-ampere
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf install --refresh --allowerasing -y helexa-neuron-ada
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf upgrade --refresh --allowerasing -y helexa-neuron-ada
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf install --refresh --allowerasing -y helexa-neuron-blackwell
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf upgrade --refresh --allowerasing -y helexa-neuron-blackwell
|
||||
# sudoers reserves `:` and `=` and requires `\` escaping inside command
|
||||
# arguments — without it visudo errors at the first `:` in `https://`.
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf config-manager addrepo --from-repofile\=https\://rpm.lair.cafe/lair-cafe-unstable.repo
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf config-manager setopt lair-cafe-unstable.enabled\=1
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf config-manager addrepo --from-repofile\=https\://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/dnf install -y libcudnn9-cuda-13
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --add-service=helexa-neuron --permanent
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --reload
|
||||
# deploy-dev.yml fast path: install a freshly-built dev binary over the
|
||||
# packaged one. Exact source path + args; the workflow must use this
|
||||
# command form verbatim. The next deploy.yml run reconciles the host
|
||||
# back to the RPM-owned binary.
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/install -o root -g root -m 0755 /var/lib/gitea_ci/neuron-dev /usr/bin/neuron
|
||||
@@ -1,11 +0,0 @@
|
||||
# Install on a website edge proxy that is NOT the cortex gateway
|
||||
# (currently oolon.kosherinata.internal) as
|
||||
# /etc/sudoers.d/helexa_gitea_ci (owner root:root, mode 0440).
|
||||
# Required by .gitea/workflows/deploy.yml's deploy-website job, which
|
||||
# rsyncs the built helexa.ai SPA into the nginx webroot.
|
||||
#
|
||||
# The cortex gateway (hanzalova.internal) gets this grant via
|
||||
# cortex-host.conf instead — one sudoers file per host, and oolon
|
||||
# hosts other projects' gitea_ci grants in their own drop-ins.
|
||||
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /var/www/helexa.ai/
|
||||
@@ -1,20 +0,0 @@
|
||||
# Internal-CA cert renewal for %i.internal, driven by step@%i.timer.
|
||||
# Replicated from oolon.kosherinata.internal (the kosherinata DC proxy).
|
||||
# Renews an EXISTING cert via mTLS (step ca renew) — the initial cert
|
||||
# must be issued once with a provisioner (see script/infra-setup.sh).
|
||||
# Installed to /etc/systemd/system/step@.service.
|
||||
[Unit]
|
||||
Description=step cert renew for %i.internal
|
||||
Documentation=https://smallstep.com/docs/step-ca/renewal
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecCondition=/usr/bin/step certificate needs-renewal \
|
||||
/etc/nginx/tls/cert/%i.internal.pem
|
||||
ExecStart=/usr/bin/step ca renew \
|
||||
--force \
|
||||
--ca-url https://ca.internal \
|
||||
--root /etc/pki/ca-trust/source/anchors/root-internal.pem \
|
||||
/etc/nginx/tls/cert/%i.internal.pem \
|
||||
/etc/nginx/tls/key/%i.internal.pem
|
||||
ExecStartPost=/usr/bin/systemctl reload nginx.service
|
||||
@@ -1,15 +0,0 @@
|
||||
# Periodic internal-cert renewal for %i.internal (every 15 min, jittered).
|
||||
# Replicated from oolon.kosherinata.internal. Installed to
|
||||
# /etc/systemd/system/step@.timer; enable per-cert with
|
||||
# `systemctl enable --now step@bench.timer`.
|
||||
[Unit]
|
||||
Description=step cert renew timer for %i.internal
|
||||
|
||||
[Timer]
|
||||
Persistent=true
|
||||
OnCalendar=*:1/15
|
||||
AccuracySec=1us
|
||||
RandomizedDelaySec=5m
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
3
bench/.gitignore
vendored
3
bench/.gitignore
vendored
@@ -1,3 +0,0 @@
|
||||
node_modules
|
||||
dist
|
||||
*.local
|
||||
@@ -1,45 +0,0 @@
|
||||
# helexa bench UI
|
||||
|
||||
A Vite + React (SWC, TypeScript) app that visualises the fleet benchmark
|
||||
data collected by `helexa-bench`. It reads the read-only JSON API the
|
||||
bench daemon serves (`crates/helexa-bench/src/api.rs`, default
|
||||
`:13132` on bob).
|
||||
|
||||
Stack: React Router, react-bootstrap, Recharts.
|
||||
|
||||
## Pages
|
||||
|
||||
- **Overview** — latest median results per (host, model, scenario) cell.
|
||||
- **Trends** — decode-tok/s and TTFT plotted across neuron build SHAs as
|
||||
releases roll out (the headline view). Pick host / model / scenario.
|
||||
- **Runs** — filterable raw-run explorer.
|
||||
|
||||
## Develop
|
||||
|
||||
```sh
|
||||
cd bench
|
||||
npm install
|
||||
npm run dev # http://localhost:5173
|
||||
```
|
||||
|
||||
`vite.config.ts` proxies `/api` → `http://bob.hanzalova.internal:13132`,
|
||||
so the dev server talks to the live bench API with no CORS fuss. Point
|
||||
the proxy elsewhere (or run a local `helexa-bench serve`) to develop
|
||||
against other data.
|
||||
|
||||
## Production hosting
|
||||
|
||||
Public at **https://bench.helexa.ai** — nginx on the gateway
|
||||
(`hanzalova.internal`) serves the static `dist/` and reverse-proxies
|
||||
`/api` to the bench API on bob over WireGuard, so the SPA is same-origin
|
||||
(no CORS) and the internal API stays off the public internet.
|
||||
|
||||
- `npm run build` is run with **no** `VITE_API_BASE` (the app calls
|
||||
`/api/...` on its own origin; nginx proxies it to bob).
|
||||
- `.gitea/workflows/deploy.yml` (`deploy-bench-ui`) builds and rsyncs
|
||||
`dist/` to `/var/www/bench.helexa.ai` on every deploy.
|
||||
- The nginx vhost (`asset/nginx/bench.helexa.ai.conf`) and the
|
||||
Let's Encrypt cert are one-time host setup in `script/infra-setup.sh`.
|
||||
|
||||
To host elsewhere instead, build with
|
||||
`VITE_API_BASE=<bob-api-origin>` and serve the static `dist/`.
|
||||
@@ -1,12 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>helexa bench</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.tsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
2191
bench/package-lock.json
generated
2191
bench/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -1,28 +0,0 @@
|
||||
{
|
||||
"name": "helexa-bench-ui",
|
||||
"private": true,
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"description": "Visualisation app for helexa-bench fleet benchmark data.",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "tsc && vite build",
|
||||
"preview": "vite preview"
|
||||
},
|
||||
"dependencies": {
|
||||
"bootstrap": "^5.3.3",
|
||||
"react": "^18.3.1",
|
||||
"react-bootstrap": "^2.10.5",
|
||||
"react-dom": "^18.3.1",
|
||||
"react-router-dom": "^6.26.2",
|
||||
"recharts": "^2.12.7"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.14.0",
|
||||
"@types/react": "^18.3.5",
|
||||
"@types/react-dom": "^18.3.0",
|
||||
"@vitejs/plugin-react-swc": "^3.7.0",
|
||||
"typescript": "^5.5.4",
|
||||
"vite": "^5.4.0"
|
||||
}
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
import { Container, Nav, Navbar } from "react-bootstrap";
|
||||
import { NavLink, Outlet } from "react-router-dom";
|
||||
|
||||
export default function App() {
|
||||
return (
|
||||
<>
|
||||
<Navbar bg="dark" variant="dark" expand="md">
|
||||
<Container>
|
||||
<Navbar.Brand as={NavLink} to="/">
|
||||
helexa bench
|
||||
</Navbar.Brand>
|
||||
<Nav className="me-auto">
|
||||
<Nav.Link as={NavLink} to="/" end>
|
||||
Overview
|
||||
</Nav.Link>
|
||||
<Nav.Link as={NavLink} to="/trends">
|
||||
Trends
|
||||
</Nav.Link>
|
||||
<Nav.Link as={NavLink} to="/runs">
|
||||
Runs
|
||||
</Nav.Link>
|
||||
</Nav>
|
||||
</Container>
|
||||
</Navbar>
|
||||
<Container className="py-4">
|
||||
<Outlet />
|
||||
</Container>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
import type { Dimensions, ReportRow, RunRow, SeriesPoint } from "./types";
|
||||
|
||||
// Empty default → `fetch('/api/...')` hits the dev proxy (vite.config.ts)
|
||||
// or the same origin. For a separately-hosted build, set VITE_API_BASE to
|
||||
// the bob API origin (e.g. http://bob.hanzalova.internal:13132).
|
||||
const BASE = import.meta.env.VITE_API_BASE ?? "";
|
||||
|
||||
async function getJson<T>(path: string): Promise<T> {
|
||||
const res = await fetch(`${BASE}${path}`);
|
||||
if (!res.ok) {
|
||||
throw new Error(`${res.status} ${res.statusText}: ${await res.text()}`);
|
||||
}
|
||||
return res.json() as Promise<T>;
|
||||
}
|
||||
|
||||
export const getDimensions = () => getJson<Dimensions>("/api/dimensions");
|
||||
export const getSummary = () => getJson<ReportRow[]>("/api/summary");
|
||||
|
||||
// host is resolved server-side (each model maps to one host today), so the
|
||||
// public UI selects by model + scenario alone.
|
||||
export const getSeries = (model: string, scenario: string) =>
|
||||
getJson<SeriesPoint[]>(
|
||||
`/api/series?model=${encodeURIComponent(model)}&scenario=${encodeURIComponent(scenario)}`,
|
||||
);
|
||||
|
||||
export interface RunsParams {
|
||||
host?: string;
|
||||
model?: string;
|
||||
scenario?: string;
|
||||
sha?: string;
|
||||
ok?: boolean;
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
export const getRuns = (p: RunsParams = {}) => {
|
||||
const q = new URLSearchParams();
|
||||
if (p.host) q.set("host", p.host);
|
||||
if (p.model) q.set("model", p.model);
|
||||
if (p.scenario) q.set("scenario", p.scenario);
|
||||
if (p.sha) q.set("sha", p.sha);
|
||||
if (p.ok !== undefined) q.set("ok", String(p.ok));
|
||||
if (p.limit) q.set("limit", String(p.limit));
|
||||
const qs = q.toString();
|
||||
return getJson<RunRow[]>(`/api/runs${qs ? `?${qs}` : ""}`);
|
||||
};
|
||||
@@ -1,52 +0,0 @@
|
||||
// Pre-helexa-bench baseline, transcribed verbatim from doc/benchmarks.md.
|
||||
//
|
||||
// IMPORTANT — different measurement regime. These were measured by
|
||||
// script/bench.py *through the cortex gateway* (so TTFT/total include a
|
||||
// proxy hop), reported as medians only, before helexa-bench existed.
|
||||
// helexa-bench measures each neuron *directly*. So these points are an
|
||||
// honest historical anchor, NOT apples-to-apples with the live series —
|
||||
// the Trends view renders them dashed + labelled, never merged into the
|
||||
// live line.
|
||||
//
|
||||
// Host is inferred from the model via the doc's Fleet table
|
||||
// (beast=27B, benjy=8B, quadbrat=1.7B). Timestamps are the two 2026-06-12
|
||||
// snapshots in the doc, ordered (08:00 = pre-#11, 16:00 = post-#11) so
|
||||
// they sort before the bench era on the shared time axis.
|
||||
|
||||
export interface BaselinePoint {
|
||||
host: string;
|
||||
model: string;
|
||||
scenario: string;
|
||||
git_sha: string;
|
||||
build_timestamp: string;
|
||||
ttft_s: number;
|
||||
decode_tps: number;
|
||||
total_s: number;
|
||||
}
|
||||
|
||||
/** Source: bench.py via cortex gateway — see doc/benchmarks.md. */
|
||||
export const BASELINE_SOURCE = "bench.py · via cortex gateway";
|
||||
|
||||
export const BASELINE: BaselinePoint[] = [
|
||||
// ── 8f6f1d3 — baseline (2026-06-12) ────────────────────────────────
|
||||
{ host: "beast", model: "Qwen/Qwen3.6-27B", scenario: "chat:128", git_sha: "8f6f1d3", build_timestamp: "2026-06-12T08:00:00Z", ttft_s: 1.658, decode_tps: 35.0, total_s: 8.981 },
|
||||
{ host: "beast", model: "Qwen/Qwen3.6-27B", scenario: "chat:4096", git_sha: "8f6f1d3", build_timestamp: "2026-06-12T08:00:00Z", ttft_s: 7.067, decode_tps: 33.7, total_s: 14.63 },
|
||||
{ host: "benjy", model: "Qwen/Qwen3-8B", scenario: "chat:128", git_sha: "8f6f1d3", build_timestamp: "2026-06-12T08:00:00Z", ttft_s: 0.884, decode_tps: 62.4, total_s: 4.938 },
|
||||
{ host: "benjy", model: "Qwen/Qwen3-8B", scenario: "chat:4096", git_sha: "8f6f1d3", build_timestamp: "2026-06-12T08:00:00Z", ttft_s: 1.818, decode_tps: 46.5, total_s: 7.27 },
|
||||
{ host: "quadbrat", model: "Qwen/Qwen3-1.7B", scenario: "chat:128", git_sha: "8f6f1d3", build_timestamp: "2026-06-12T08:00:00Z", ttft_s: 0.685, decode_tps: 81.3, total_s: 3.741 },
|
||||
{ host: "quadbrat", model: "Qwen/Qwen3-1.7B", scenario: "chat:4096", git_sha: "8f6f1d3", build_timestamp: "2026-06-12T08:00:00Z", ttft_s: 2.743, decode_tps: 35.4, total_s: 9.884 },
|
||||
// ── a1952a4 — post prefix-KV-cache (#11, 2026-06-12) ───────────────
|
||||
{ host: "beast", model: "Qwen/Qwen3.6-27B", scenario: "chat:128", git_sha: "a1952a4", build_timestamp: "2026-06-12T16:00:00Z", ttft_s: 1.355, decode_tps: 45.8, total_s: 4.147 },
|
||||
{ host: "beast", model: "Qwen/Qwen3.6-27B", scenario: "chat:4096", git_sha: "a1952a4", build_timestamp: "2026-06-12T16:00:00Z", ttft_s: 1.431, decode_tps: 43.3, total_s: 4.387 },
|
||||
{ host: "benjy", model: "Qwen/Qwen3-8B", scenario: "chat:128", git_sha: "a1952a4", build_timestamp: "2026-06-12T16:00:00Z", ttft_s: 0.886, decode_tps: 78.6, total_s: 2.478 },
|
||||
{ host: "benjy", model: "Qwen/Qwen3-8B", scenario: "chat:4096", git_sha: "a1952a4", build_timestamp: "2026-06-12T16:00:00Z", ttft_s: 1.824, decode_tps: 58.3, total_s: 3.969 },
|
||||
{ host: "quadbrat", model: "Qwen/Qwen3-1.7B", scenario: "chat:128", git_sha: "a1952a4", build_timestamp: "2026-06-12T16:00:00Z", ttft_s: 0.702, decode_tps: 104.8, total_s: 1.895 },
|
||||
{ host: "quadbrat", model: "Qwen/Qwen3-1.7B", scenario: "chat:4096", git_sha: "a1952a4", build_timestamp: "2026-06-12T16:00:00Z", ttft_s: 2.749, decode_tps: 44.9, total_s: 5.534 },
|
||||
];
|
||||
|
||||
/** Baseline points for one (model, scenario) cell, oldest first. */
|
||||
export function baselineFor(model: string, scenario: string): BaselinePoint[] {
|
||||
return BASELINE.filter(
|
||||
(b) => b.model === model && b.scenario === scenario,
|
||||
).sort((a, b) => a.build_timestamp.localeCompare(b.build_timestamp));
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
import React from "react";
|
||||
import ReactDOM from "react-dom/client";
|
||||
import { BrowserRouter, Route, Routes } from "react-router-dom";
|
||||
import "bootstrap/dist/css/bootstrap.min.css";
|
||||
import App from "./App";
|
||||
import Overview from "./pages/Overview";
|
||||
import Trends from "./pages/Trends";
|
||||
import Runs from "./pages/Runs";
|
||||
|
||||
ReactDOM.createRoot(document.getElementById("root")!).render(
|
||||
<React.StrictMode>
|
||||
<BrowserRouter>
|
||||
<Routes>
|
||||
<Route path="/" element={<App />}>
|
||||
<Route index element={<Overview />} />
|
||||
<Route path="trends" element={<Trends />} />
|
||||
<Route path="runs" element={<Runs />} />
|
||||
</Route>
|
||||
</Routes>
|
||||
</BrowserRouter>
|
||||
</React.StrictMode>,
|
||||
);
|
||||
@@ -1,64 +0,0 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { Alert, Spinner, Table } from "react-bootstrap";
|
||||
import { getSummary } from "../api";
|
||||
import type { ReportRow } from "../types";
|
||||
|
||||
const f = (n: number | null, p = 2) => (n == null ? "—" : n.toFixed(p));
|
||||
|
||||
export default function Overview() {
|
||||
const [rows, setRows] = useState<ReportRow[]>([]);
|
||||
const [err, setErr] = useState<string | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
|
||||
useEffect(() => {
|
||||
getSummary()
|
||||
.then(setRows)
|
||||
.catch((e) => setErr(String(e)))
|
||||
.finally(() => setLoading(false));
|
||||
}, []);
|
||||
|
||||
if (loading) return <Spinner animation="border" />;
|
||||
if (err) return <Alert variant="danger">{err}</Alert>;
|
||||
|
||||
return (
|
||||
<>
|
||||
<h3 className="mb-3">Latest results per cell</h3>
|
||||
<p className="text-muted">
|
||||
Median of each cell's samples on the most recent build seen for that
|
||||
(host, model, scenario).
|
||||
</p>
|
||||
<Table striped bordered hover responsive size="sm">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>GPU</th>
|
||||
<th>model</th>
|
||||
<th className="text-end">prompt tok</th>
|
||||
<th className="text-end">TTFT (s)</th>
|
||||
<th className="text-end">decode tok/s</th>
|
||||
<th className="text-end">total (s)</th>
|
||||
<th>build</th>
|
||||
<th className="text-end">n</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{rows.map((r, i) => (
|
||||
<tr key={i}>
|
||||
<td>{r.gpu ?? r.target_name}</td>
|
||||
<td>{r.model_id}</td>
|
||||
<td className="text-end">
|
||||
{r.prompt_tokens ?? `~${r.prompt_size_approx}`}
|
||||
</td>
|
||||
<td className="text-end">{f(r.ttft_s_median, 3)}</td>
|
||||
<td className="text-end">{f(r.decode_tps_median, 1)}</td>
|
||||
<td className="text-end">{f(r.total_s_median, 3)}</td>
|
||||
<td>
|
||||
<code>{r.git_sha}</code>
|
||||
</td>
|
||||
<td className="text-end">{r.samples}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</Table>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -1,141 +0,0 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { Alert, Badge, Col, Form, Row, Spinner, Table } from "react-bootstrap";
|
||||
import { getDimensions, getRuns } from "../api";
|
||||
import type { Dimensions, RunRow } from "../types";
|
||||
|
||||
const f = (n: number | null, p = 2) => (n == null ? "—" : n.toFixed(p));
|
||||
|
||||
function Picker({
|
||||
label,
|
||||
value,
|
||||
set,
|
||||
options,
|
||||
}: {
|
||||
label: string;
|
||||
value: string;
|
||||
set: (v: string) => void;
|
||||
options: string[];
|
||||
}) {
|
||||
return (
|
||||
<Form.Group as={Col}>
|
||||
<Form.Label>{label}</Form.Label>
|
||||
<Form.Select value={value} onChange={(e) => set(e.target.value)}>
|
||||
<option value="">(all)</option>
|
||||
{options.map((o) => (
|
||||
<option key={o} value={o}>
|
||||
{o}
|
||||
</option>
|
||||
))}
|
||||
</Form.Select>
|
||||
</Form.Group>
|
||||
);
|
||||
}
|
||||
|
||||
export default function Runs() {
|
||||
const [dims, setDims] = useState<Dimensions | null>(null);
|
||||
const [host, setHost] = useState("");
|
||||
const [model, setModel] = useState("");
|
||||
const [scenario, setScenario] = useState("");
|
||||
const [rows, setRows] = useState<RunRow[]>([]);
|
||||
const [err, setErr] = useState<string | null>(null);
|
||||
const [loading, setLoading] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
getDimensions()
|
||||
.then(setDims)
|
||||
.catch((e) => setErr(String(e)));
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
setLoading(true);
|
||||
getRuns({
|
||||
host: host || undefined,
|
||||
model: model || undefined,
|
||||
scenario: scenario || undefined,
|
||||
limit: 200,
|
||||
})
|
||||
.then(setRows)
|
||||
.catch((e) => setErr(String(e)))
|
||||
.finally(() => setLoading(false));
|
||||
}, [host, model, scenario]);
|
||||
|
||||
if (err) return <Alert variant="danger">{err}</Alert>;
|
||||
|
||||
return (
|
||||
<>
|
||||
<h3 className="mb-3">Runs</h3>
|
||||
{dims && (
|
||||
<Row className="g-3 mb-3">
|
||||
{/* GPU filter — labelled by GPU, but filters by the underlying host. */}
|
||||
<Form.Group as={Col}>
|
||||
<Form.Label>GPU</Form.Label>
|
||||
<Form.Select value={host} onChange={(e) => setHost(e.target.value)}>
|
||||
<option value="">(all)</option>
|
||||
{dims.hosts.map((h) => (
|
||||
<option key={h} value={h}>
|
||||
{dims.host_gpus[h] ?? h}
|
||||
</option>
|
||||
))}
|
||||
</Form.Select>
|
||||
</Form.Group>
|
||||
<Picker
|
||||
label="Model"
|
||||
value={model}
|
||||
set={setModel}
|
||||
options={dims.models}
|
||||
/>
|
||||
<Picker
|
||||
label="Scenario"
|
||||
value={scenario}
|
||||
set={setScenario}
|
||||
options={dims.scenarios}
|
||||
/>
|
||||
</Row>
|
||||
)}
|
||||
{loading ? (
|
||||
<Spinner animation="border" />
|
||||
) : (
|
||||
<Table striped bordered hover responsive size="sm">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>ts</th>
|
||||
<th>GPU</th>
|
||||
<th>model</th>
|
||||
<th>scenario</th>
|
||||
<th>build</th>
|
||||
<th className="text-end">TTFT</th>
|
||||
<th className="text-end">tok/s</th>
|
||||
<th className="text-end">total</th>
|
||||
<th>ok</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{rows.map((r) => (
|
||||
<tr key={r.id}>
|
||||
<td>{r.ts}</td>
|
||||
<td>{r.gpu ?? r.host}</td>
|
||||
<td>{r.model_id}</td>
|
||||
<td>{r.scenario_id}</td>
|
||||
<td>
|
||||
<code>{r.git_sha}</code>
|
||||
</td>
|
||||
<td className="text-end">{f(r.ttft_s, 3)}</td>
|
||||
<td className="text-end">{f(r.decode_tps, 1)}</td>
|
||||
<td className="text-end">{f(r.total_s, 3)}</td>
|
||||
<td>
|
||||
{r.ok ? (
|
||||
<Badge bg="success">ok</Badge>
|
||||
) : (
|
||||
<Badge bg="danger" title={r.error ?? ""}>
|
||||
fail
|
||||
</Badge>
|
||||
)}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</Table>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -1,221 +0,0 @@
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
import { Alert, Col, Form, Row, Spinner } from "react-bootstrap";
|
||||
import {
|
||||
CartesianGrid,
|
||||
Legend,
|
||||
Line,
|
||||
LineChart,
|
||||
ReferenceLine,
|
||||
ResponsiveContainer,
|
||||
Tooltip,
|
||||
XAxis,
|
||||
YAxis,
|
||||
} from "recharts";
|
||||
import { getDimensions, getSeries } from "../api";
|
||||
import type { Dimensions, SeriesPoint } from "../types";
|
||||
import { BASELINE_SOURCE, baselineFor } from "../baseline";
|
||||
|
||||
function Picker({
|
||||
label,
|
||||
value,
|
||||
set,
|
||||
options,
|
||||
}: {
|
||||
label: string;
|
||||
value: string;
|
||||
set: (v: string) => void;
|
||||
options: string[];
|
||||
}) {
|
||||
return (
|
||||
<Form.Group as={Col}>
|
||||
<Form.Label>{label}</Form.Label>
|
||||
<Form.Select value={value} onChange={(e) => set(e.target.value)}>
|
||||
{options.map((o) => (
|
||||
<option key={o} value={o}>
|
||||
{o}
|
||||
</option>
|
||||
))}
|
||||
</Form.Select>
|
||||
</Form.Group>
|
||||
);
|
||||
}
|
||||
|
||||
export default function Trends() {
|
||||
const [dims, setDims] = useState<Dimensions | null>(null);
|
||||
const [model, setModel] = useState("");
|
||||
const [scenario, setScenario] = useState("");
|
||||
const [series, setSeries] = useState<SeriesPoint[]>([]);
|
||||
const [err, setErr] = useState<string | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
getDimensions()
|
||||
.then((d) => {
|
||||
setDims(d);
|
||||
if (d.models[0]) setModel(d.models[0]);
|
||||
if (d.scenarios[0]) setScenario(d.scenarios[0]);
|
||||
})
|
||||
.catch((e) => setErr(String(e)));
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (model && scenario) {
|
||||
getSeries(model, scenario)
|
||||
.then(setSeries)
|
||||
.catch((e) => setErr(String(e)));
|
||||
}
|
||||
}, [model, scenario]);
|
||||
|
||||
// Prepend the pre-helexa-bench baseline (dashed, separate keys) so it
|
||||
// anchors the timeline without being merged into the live line. Different
|
||||
// measurement regime — see baseline.ts / doc/benchmarks.md.
|
||||
const base = useMemo(
|
||||
() => baselineFor(model, scenario),
|
||||
[model, scenario],
|
||||
);
|
||||
const data = useMemo(
|
||||
() => [
|
||||
...base.map((p) => ({
|
||||
label: p.git_sha,
|
||||
baseTtft: p.ttft_s,
|
||||
baseDecode: p.decode_tps,
|
||||
baseTotal: p.total_s,
|
||||
})),
|
||||
...series.map((p) => ({
|
||||
label: p.git_sha,
|
||||
ttft: p.ttft_s_median,
|
||||
decode: p.decode_tps_median,
|
||||
total: p.total_s_median,
|
||||
})),
|
||||
],
|
||||
[series, base],
|
||||
);
|
||||
|
||||
// Divider marking the boundary between the two regimes (drawn at the
|
||||
// first live build, with baseline points to its left).
|
||||
const firstLive = series[0]?.git_sha;
|
||||
const showDivider = base.length > 0 && series.length > 0;
|
||||
|
||||
if (err) return <Alert variant="danger">{err}</Alert>;
|
||||
if (!dims) return <Spinner animation="border" />;
|
||||
|
||||
return (
|
||||
<>
|
||||
<h3 className="mb-3">Trends over builds</h3>
|
||||
<Row className="g-3 mb-4">
|
||||
<Picker
|
||||
label="Model"
|
||||
value={model}
|
||||
set={setModel}
|
||||
options={dims.models}
|
||||
/>
|
||||
<Picker
|
||||
label="Scenario"
|
||||
value={scenario}
|
||||
set={setScenario}
|
||||
options={dims.scenarios}
|
||||
/>
|
||||
</Row>
|
||||
|
||||
{dims.model_gpus[model] && (
|
||||
<p className="text-muted mb-3">
|
||||
Measured on <strong>{dims.model_gpus[model]}</strong>.
|
||||
</p>
|
||||
)}
|
||||
|
||||
{data.length === 0 ? (
|
||||
<Alert variant="info">No data for this selection yet.</Alert>
|
||||
) : (
|
||||
<>
|
||||
{base.length > 0 && (
|
||||
<p className="text-muted small mb-3">
|
||||
Dashed = pre-helexa-bench baseline ({BASELINE_SOURCE}); solid =
|
||||
helexa-bench (direct to neuron). Different measurement regimes —
|
||||
see <code>doc/benchmarks.md</code>.
|
||||
</p>
|
||||
)}
|
||||
<h5 className="mt-3">decode tok/s (higher is better)</h5>
|
||||
<ResponsiveContainer width="100%" height={280}>
|
||||
<LineChart data={data} margin={{ top: 8, right: 24, bottom: 8, left: 0 }}>
|
||||
<CartesianGrid strokeDasharray="3 3" />
|
||||
<XAxis dataKey="label" />
|
||||
<YAxis />
|
||||
<Tooltip />
|
||||
<Legend />
|
||||
{showDivider && firstLive && (
|
||||
<ReferenceLine
|
||||
x={firstLive}
|
||||
stroke="#bbb"
|
||||
strokeDasharray="3 3"
|
||||
label={{
|
||||
value: "bench.py → helexa-bench",
|
||||
position: "top",
|
||||
fill: "#999",
|
||||
fontSize: 11,
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
<Line
|
||||
type="monotone"
|
||||
dataKey="decode"
|
||||
name="decode tok/s"
|
||||
stroke="#0d6efd"
|
||||
connectNulls
|
||||
/>
|
||||
{base.length > 0 && (
|
||||
<Line
|
||||
type="monotone"
|
||||
dataKey="baseDecode"
|
||||
name="baseline (bench.py · gateway)"
|
||||
stroke="#888"
|
||||
strokeDasharray="5 5"
|
||||
connectNulls
|
||||
/>
|
||||
)}
|
||||
</LineChart>
|
||||
</ResponsiveContainer>
|
||||
|
||||
<h5 className="mt-4">TTFT seconds (lower is better)</h5>
|
||||
<ResponsiveContainer width="100%" height={280}>
|
||||
<LineChart data={data} margin={{ top: 8, right: 24, bottom: 8, left: 0 }}>
|
||||
<CartesianGrid strokeDasharray="3 3" />
|
||||
<XAxis dataKey="label" />
|
||||
<YAxis />
|
||||
<Tooltip />
|
||||
<Legend />
|
||||
{showDivider && firstLive && (
|
||||
<ReferenceLine
|
||||
x={firstLive}
|
||||
stroke="#bbb"
|
||||
strokeDasharray="3 3"
|
||||
label={{
|
||||
value: "bench.py → helexa-bench",
|
||||
position: "top",
|
||||
fill: "#999",
|
||||
fontSize: 11,
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
<Line
|
||||
type="monotone"
|
||||
dataKey="ttft"
|
||||
name="TTFT (s)"
|
||||
stroke="#dc3545"
|
||||
connectNulls
|
||||
/>
|
||||
{base.length > 0 && (
|
||||
<Line
|
||||
type="monotone"
|
||||
dataKey="baseTtft"
|
||||
name="baseline (bench.py · gateway)"
|
||||
stroke="#888"
|
||||
strokeDasharray="5 5"
|
||||
connectNulls
|
||||
/>
|
||||
)}
|
||||
</LineChart>
|
||||
</ResponsiveContainer>
|
||||
</>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
// Mirrors the JSON served by helexa-bench's read API (crates/helexa-bench/src/api.rs).
|
||||
|
||||
export interface BuildRef {
|
||||
git_sha: string;
|
||||
build_timestamp: string | null;
|
||||
package_version: string | null;
|
||||
}
|
||||
|
||||
export interface Dimensions {
|
||||
hosts: string[];
|
||||
models: string[];
|
||||
scenarios: string[];
|
||||
builds: BuildRef[];
|
||||
/** host → GPU label, e.g. "2× RTX 5090". */
|
||||
host_gpus: Record<string, string>;
|
||||
/** model → GPU label (model maps to one host today). */
|
||||
model_gpus: Record<string, string>;
|
||||
}
|
||||
|
||||
/** Latest-SHA-per-cell medians (the report table). */
|
||||
export interface ReportRow {
|
||||
target_name: string;
|
||||
model_id: string;
|
||||
scenario_id: string;
|
||||
prompt_size_approx: number;
|
||||
git_sha: string;
|
||||
prompt_tokens: number | null;
|
||||
ttft_s_median: number | null;
|
||||
decode_tps_median: number | null;
|
||||
total_s_median: number | null;
|
||||
samples: number;
|
||||
/** Public-facing resource name (the host's GPU(s)). */
|
||||
gpu: string | null;
|
||||
}
|
||||
|
||||
/** One point in a per-build time-series for a (host, model, scenario) cell. */
|
||||
export interface SeriesPoint {
|
||||
git_sha: string;
|
||||
build_timestamp: string | null;
|
||||
package_version: string | null;
|
||||
ttft_s_median: number | null;
|
||||
decode_tps_median: number | null;
|
||||
total_s_median: number | null;
|
||||
samples: number;
|
||||
}
|
||||
|
||||
export interface RunRow {
|
||||
id: number;
|
||||
ts: string;
|
||||
host: string;
|
||||
/** Public-facing resource name (the host's GPU(s)). */
|
||||
gpu: string | null;
|
||||
hostname: string | null;
|
||||
git_sha: string;
|
||||
build_timestamp: string | null;
|
||||
package_version: string;
|
||||
model_id: string;
|
||||
harness: string;
|
||||
scenario_id: string;
|
||||
prompt_size_approx: number;
|
||||
prompt_tokens_actual: number | null;
|
||||
max_tokens: number;
|
||||
ttft_s: number | null;
|
||||
decode_tps: number | null;
|
||||
total_s: number | null;
|
||||
completion_tokens: number | null;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
9
bench/src/vite-env.d.ts
vendored
9
bench/src/vite-env.d.ts
vendored
@@ -1,9 +0,0 @@
|
||||
/// <reference types="vite/client" />
|
||||
|
||||
interface ImportMetaEnv {
|
||||
/** Base origin of the bench API. Empty → use the dev proxy / same origin. */
|
||||
readonly VITE_API_BASE?: string;
|
||||
}
|
||||
interface ImportMeta {
|
||||
readonly env: ImportMetaEnv;
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"useDefineForClassFields": true,
|
||||
"lib": ["ES2022", "DOM", "DOM.Iterable"],
|
||||
"module": "ESNext",
|
||||
"skipLibCheck": true,
|
||||
"moduleResolution": "bundler",
|
||||
"allowImportingTsExtensions": true,
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"moduleDetection": "force",
|
||||
"noEmit": true,
|
||||
"jsx": "react-jsx",
|
||||
"strict": true,
|
||||
"noUnusedLocals": true,
|
||||
"noUnusedParameters": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"types": ["node", "vite/client"]
|
||||
},
|
||||
"include": ["src", "vite.config.ts"]
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
import { defineConfig } from "vite";
|
||||
import react from "@vitejs/plugin-react-swc";
|
||||
|
||||
// Dev server proxies /api to the bench API on bob so `fetch('/api/...')`
|
||||
// works without CORS/mixed-origin fuss during local development.
|
||||
// For a production build hosted elsewhere, set VITE_API_BASE to the bob
|
||||
// API origin (e.g. http://bob.hanzalova.internal:13132) instead.
|
||||
export default defineConfig({
|
||||
plugins: [react()],
|
||||
server: {
|
||||
proxy: {
|
||||
"/api": {
|
||||
target: "http://bob.hanzalova.internal:13132",
|
||||
changeOrigin: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -5,31 +5,20 @@
|
||||
# Environment variable overrides use CORTEX_ prefix with __ separators:
|
||||
# CORTEX_GATEWAY__LISTEN=0.0.0.0:31313
|
||||
|
||||
# Path to the model catalogue (limits, cost, pinning, aliases, feasibility).
|
||||
# Defaults to the packaged location below; uncomment to override for a
|
||||
# non-packaged / local run.
|
||||
# models_config = "/etc/cortex/models.toml"
|
||||
|
||||
# NOTE: the deployed copy of this file carries secrets — the inference
|
||||
# API keys under [[entitlements.keys]].
|
||||
# It must be installed root:cortex 0640 (the package's %post and
|
||||
# script/infra-setup.sh both enforce this); a world-readable 0644
|
||||
# hands every local account those credentials.
|
||||
|
||||
[gateway]
|
||||
listen = "0.0.0.0:31313"
|
||||
metrics_listen = "0.0.0.0:31314"
|
||||
|
||||
[eviction]
|
||||
strategy = "lru"
|
||||
# Restart neurons after this many load/unload cycles to defragment VRAM.
|
||||
# Restart mistralrs after this many load/unload cycles to defragment VRAM.
|
||||
# Set to 0 to disable.
|
||||
defrag_after_cycles = 50
|
||||
|
||||
# -- Nodes ---------------------------------------------------------------
|
||||
# Each [[nodes]] entry declares a neuron daemon in the fleet.
|
||||
# Models are discovered by polling the neuron's /models endpoint.
|
||||
# Pinned models (see models.toml) are never evicted.
|
||||
# Each [[nodes]] entry declares a mistral.rs instance in the fleet.
|
||||
# Models are discovered by polling the node's /v1/models endpoint.
|
||||
# Pinned models are never evicted.
|
||||
|
||||
[[nodes]]
|
||||
name = "gpu-large"
|
||||
@@ -54,62 +43,3 @@ vram_mb = 12288 # e.g. RTX 3060 (12 GB)
|
||||
pinned = [
|
||||
"your-org/embedding-model",
|
||||
]
|
||||
|
||||
# -- Entitlements (multi-tenant governance, #47) -------------------------
|
||||
# Identity + per-key token budgets. Omit this section entirely for the
|
||||
# legacy single-operator behaviour: requests are anonymous and uncapped.
|
||||
#
|
||||
# The local/static provider below is the source of truth for accounts,
|
||||
# keys, and hard caps until the upstream clearing house exists. Identity
|
||||
# rides standard bearer auth only — clients send
|
||||
# Authorization: Bearer <key>
|
||||
# no custom headers or body fields.
|
||||
|
||||
[entitlements]
|
||||
# Reject unauthenticated requests with 401 invalid_api_key. Leave false
|
||||
# (allow-anonymous) during rollout; flip to true once keys are issued.
|
||||
require_auth = false
|
||||
|
||||
# One entry per API key.
|
||||
[[entitlements.keys]]
|
||||
key = "sk-example-rolling" # the bearer token the client sends
|
||||
account_id = "team-research" # billable account (keys may share one)
|
||||
key_id = "research-ci" # stable label for ledger/metrics (optional)
|
||||
hard_cap = 5_000_000 # hard token cap over the window
|
||||
# Rolling window that resets — over-cap requests get 429 rate_limit_exceeded
|
||||
# + Retry-After, so well-behaved clients (opencode/AI SDK) back off and retry.
|
||||
window = { kind = "rolling", seconds = 3600 }
|
||||
|
||||
[[entitlements.keys]]
|
||||
key = "sk-example-balance"
|
||||
account_id = "team-research"
|
||||
key_id = "research-prepaid"
|
||||
hard_cap = 20_000_000
|
||||
# Hard balance, no reset — exhaustion returns 429 insufficient_quota
|
||||
# (the client surfaces and stops). This is the default when `window` is
|
||||
# omitted. Never 402.
|
||||
window = { kind = "balance" }
|
||||
|
||||
[[entitlements.keys]]
|
||||
key = "sk-example-infra"
|
||||
account_id = "operator"
|
||||
key_id = "infra"
|
||||
# No hard_cap → uncapped operator infra key (own fleet, own use). Still
|
||||
# metered for visibility.
|
||||
|
||||
# -- Upstream (helexa mesh) entitlements client (#57) --------------------
|
||||
# When enabled, a bearer key NOT found in [[entitlements.keys]] above is
|
||||
# validated against the helexa-upstream authority (mesh accounts), and its
|
||||
# budget is reserved/settled there. Operator-local keys (incl. the infra
|
||||
# key) never leave this process. Fail-closed: if upstream is unreachable a
|
||||
# request is refused (503 + Retry-After), never served un-authorized.
|
||||
# Disabled by default — a standalone operator runs purely local.
|
||||
[upstream]
|
||||
enabled = false
|
||||
# url = "https://upstream.helexa.ai"
|
||||
# Shared client bearer this cortex presents (maps to an operator_id
|
||||
# upstream). Override via CORTEX_UPSTREAM__BEARER in prod.
|
||||
# bearer = "replace-with-operator-client-secret"
|
||||
# timeout_secs = 5
|
||||
# How often to flush served-usage counters to upstream for reconciliation (#58).
|
||||
# served_usage_report_interval_secs = 60
|
||||
|
||||
53
cortex.spec
53
cortex.spec
@@ -1,10 +1,10 @@
|
||||
Name: cortex
|
||||
Version: 0.1.16
|
||||
Version: 0.1.12
|
||||
Release: 1%{?dist}
|
||||
Summary: Inference gateway for multi-node GPU clusters
|
||||
|
||||
License: GPL-3.0-or-later
|
||||
URL: https://git.lair.cafe/helexa/helexa
|
||||
URL: https://git.lair.cafe/helexa/cortex
|
||||
Source0: %{name}-%{version}.tar.gz
|
||||
Source1: %{name}-%{version}-vendor.tar.gz
|
||||
|
||||
@@ -21,7 +21,6 @@ BuildRequires: systemd-rpm-macros
|
||||
|
||||
Requires(pre): shadow-utils
|
||||
Requires: systemd
|
||||
Requires: firewalld-filesystem
|
||||
|
||||
# systemd-rpm-macros ships a unit dep generator that parses User=/Group=
|
||||
# from our .service file and emits Requires: user(cortex)/group(cortex).
|
||||
@@ -57,7 +56,6 @@ cargo build --release -p cortex-cli
|
||||
install -Dm755 target/release/cortex %{buildroot}%{_bindir}/cortex
|
||||
install -Dm644 data/cortex.service %{buildroot}%{_unitdir}/cortex.service
|
||||
install -Dm644 data/cortex-sysusers.conf %{buildroot}%{_sysusersdir}/cortex.conf
|
||||
install -Dm644 data/cortex-firewalld.xml %{buildroot}%{_prefix}/lib/firewalld/services/cortex.xml
|
||||
install -dm755 %{buildroot}%{_sysconfdir}/cortex
|
||||
install -Dm644 cortex.example.toml %{buildroot}%{_sysconfdir}/cortex/cortex.toml
|
||||
install -Dm644 models.example.toml %{buildroot}%{_sysconfdir}/cortex/models.toml
|
||||
@@ -67,14 +65,6 @@ install -Dm644 models.example.toml %{buildroot}%{_sysconfdir}/cortex/models.toml
|
||||
|
||||
%post
|
||||
%systemd_post cortex.service
|
||||
# The config carries secrets. %config(noreplace) keeps an existing
|
||||
# file as-is on upgrade — including a too-permissive mode from an
|
||||
# older package — so converge it here. The cortex user comes from
|
||||
# sysusers.d, applied before %post.
|
||||
if [ -f %{_sysconfdir}/cortex/cortex.toml ]; then
|
||||
chgrp cortex %{_sysconfdir}/cortex/cortex.toml >/dev/null 2>&1 || :
|
||||
chmod 0640 %{_sysconfdir}/cortex/cortex.toml >/dev/null 2>&1 || :
|
||||
fi
|
||||
|
||||
%preun
|
||||
%systemd_preun cortex.service
|
||||
@@ -82,53 +72,16 @@ fi
|
||||
%postun
|
||||
%systemd_postun_with_restart cortex.service
|
||||
|
||||
%posttrans
|
||||
# Migration: older cortex packages shipped the firewalld service as
|
||||
# `helexa-cortex` and (in some build streams) with wrong port numbers
|
||||
# (9301/9302/9304). Operators who enabled that legacy service in their
|
||||
# zone end up with the wrong-port override taking precedence over the
|
||||
# vendor `cortex.xml` now in /usr/lib/firewalld/services/. Clean up the
|
||||
# stale /etc/ override here and migrate any zone bindings to the new
|
||||
# service name.
|
||||
if [ -f /etc/firewalld/services/helexa-cortex.xml ]; then
|
||||
rm -f /etc/firewalld/services/helexa-cortex.xml
|
||||
fi
|
||||
if [ -x /usr/bin/firewall-cmd ] && /usr/bin/firewall-cmd --state >/dev/null 2>&1; then
|
||||
# Drop the legacy service name from every zone where it was enabled
|
||||
# and add the new `cortex` service in its place. Operators who never
|
||||
# ran firewall-cmd against either name see no zone change.
|
||||
for zone in $(/usr/bin/firewall-cmd --get-active-zones 2>/dev/null \
|
||||
| awk '!/^[[:space:]]/ {print $1}'); do
|
||||
if /usr/bin/firewall-cmd --permanent --zone="$zone" --query-service=helexa-cortex >/dev/null 2>&1; then
|
||||
/usr/bin/firewall-cmd --permanent --zone="$zone" --remove-service=helexa-cortex >/dev/null 2>&1 || :
|
||||
/usr/bin/firewall-cmd --permanent --zone="$zone" --add-service=cortex >/dev/null 2>&1 || :
|
||||
fi
|
||||
done
|
||||
/usr/bin/firewall-cmd --reload >/dev/null 2>&1 || :
|
||||
fi
|
||||
:
|
||||
|
||||
%files
|
||||
%license LICENSE
|
||||
%doc README.md
|
||||
%{_bindir}/cortex
|
||||
%{_unitdir}/cortex.service
|
||||
%{_sysusersdir}/cortex.conf
|
||||
%{_prefix}/lib/firewalld/services/cortex.xml
|
||||
%dir %{_sysconfdir}/cortex
|
||||
%config(noreplace) %attr(0640,root,cortex) %{_sysconfdir}/cortex/cortex.toml
|
||||
%config(noreplace) %{_sysconfdir}/cortex/cortex.toml
|
||||
%config(noreplace) %{_sysconfdir}/cortex/models.toml
|
||||
|
||||
%changelog
|
||||
* Thu Apr 16 2026 Gitea Actions <actions@git.lair.cafe> - 0.1.16-1
|
||||
- chore: ignore local deploy script
|
||||
- chore: move default ports out of common-collision ranges
|
||||
- ci: drop actions/cache for cargo registry and target
|
||||
|
||||
* Thu Apr 16 2026 Gitea Actions <actions@git.lair.cafe> - 0.1.14-1
|
||||
- ci: publish both packages to a single helexa/helexa COPR project
|
||||
- fix(rpm): rename neuron package to helexa-neuron
|
||||
- ci: commit generated %changelog entries back to main
|
||||
|
||||
* Wed Apr 15 2026 Rob Thijssen <grenade@rob.tn> - 0.1.0-1
|
||||
- Initial package
|
||||
|
||||
@@ -5,7 +5,7 @@ use tracing_subscriber::EnvFilter;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "cortex")]
|
||||
#[command(about = "Unified inference gateway for multi-node GPU clusters")]
|
||||
#[command(about = "Unified inference gateway for multi-node mistral.rs clusters")]
|
||||
#[command(version)]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
//!
|
||||
//! These mirror the `/v1/messages` format used by the Anthropic API.
|
||||
//! The gateway accepts these, translates to OpenAI format, proxies to
|
||||
//! the inference backend (neuron), then translates the response back.
|
||||
//! mistral.rs, then translates the response back.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
@@ -1,119 +0,0 @@
|
||||
//! Build/version metadata shared between cortex and neuron.
|
||||
//!
|
||||
//! neuron captures these facts at compile time in its `build.rs`
|
||||
//! (git SHA, enabled cargo features, rustc/candle versions, …) and
|
||||
//! serves them from `GET /version`. cortex and `helexa-bench`
|
||||
//! deserialize the same struct so a benchmark run can be attributed to
|
||||
//! the exact daemon build that produced it — not just the host's CUDA
|
||||
//! and driver versions that `/discovery` already reports.
|
||||
//!
|
||||
//! Every field beyond the always-present package version is
|
||||
//! `#[serde(default)]` so a newer reader stays compatible with an
|
||||
//! older neuron that omits a field (and vice versa) — the same
|
||||
//! forward/backward-compat discipline as
|
||||
//! [`crate::discovery::ActivationStatus`].
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Build-time identity of a neuron daemon.
|
||||
///
|
||||
/// Returned by `GET /version`. The `git_sha` is the canonical "which
|
||||
/// build is live" key — benchmark records are bucketed by it, so a
|
||||
/// regression can be pinned to a daemon change rather than a host
|
||||
/// change. When neuron is built from a source tarball with no git
|
||||
/// metadata available (and no `HELEXA_BUILD_SHA` injected by CI/RPM),
|
||||
/// `git_sha` is the string `"unknown"`.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct BuildInfo {
|
||||
/// Crate version from `CARGO_PKG_VERSION` (e.g. `"0.1.16"`).
|
||||
pub package_version: String,
|
||||
/// Short git SHA, or `"unknown"` when unavailable at build time.
|
||||
#[serde(default = "unknown")]
|
||||
pub git_sha: String,
|
||||
/// Full 40-char git SHA when available.
|
||||
#[serde(default)]
|
||||
pub git_sha_long: Option<String>,
|
||||
/// Whether the working tree had uncommitted changes at build time.
|
||||
/// `false` when the SHA is unknown (tarball build).
|
||||
#[serde(default)]
|
||||
pub git_dirty: bool,
|
||||
/// RFC3339 build timestamp.
|
||||
#[serde(default)]
|
||||
pub build_timestamp: Option<String>,
|
||||
/// `rustc --version` output of the compiler used.
|
||||
#[serde(default)]
|
||||
pub rustc_version: Option<String>,
|
||||
/// Cargo build profile: `"release"` or `"debug"`.
|
||||
#[serde(default)]
|
||||
pub profile: Option<String>,
|
||||
/// Target triple the binary was compiled for.
|
||||
#[serde(default)]
|
||||
pub target: Option<String>,
|
||||
/// Enabled cargo features (e.g. `["cuda", "cudnn"]`). These define
|
||||
/// the performance envelope, so they are recorded against every
|
||||
/// benchmark run.
|
||||
#[serde(default)]
|
||||
pub features: Vec<String>,
|
||||
/// Locked `candle-core` version, best-effort from `Cargo.lock`.
|
||||
#[serde(default)]
|
||||
pub candle_version: Option<String>,
|
||||
}
|
||||
|
||||
fn unknown() -> String {
|
||||
"unknown".to_string()
|
||||
}
|
||||
|
||||
impl BuildInfo {
|
||||
/// A placeholder used by non-neuron benchmark targets (and tests)
|
||||
/// that have no build metadata to report.
|
||||
pub fn unknown() -> Self {
|
||||
BuildInfo {
|
||||
package_version: env!("CARGO_PKG_VERSION").to_string(),
|
||||
git_sha: unknown(),
|
||||
git_sha_long: None,
|
||||
git_dirty: false,
|
||||
build_timestamp: None,
|
||||
rustc_version: None,
|
||||
profile: None,
|
||||
target: None,
|
||||
features: Vec::new(),
|
||||
candle_version: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn round_trips_full() {
|
||||
let info = BuildInfo {
|
||||
package_version: "0.1.16".into(),
|
||||
git_sha: "30d50d6".into(),
|
||||
git_sha_long: Some("30d50d6abc123".into()),
|
||||
git_dirty: true,
|
||||
build_timestamp: Some("2026-06-13T10:00:00+00:00".into()),
|
||||
rustc_version: Some("rustc 1.85.0".into()),
|
||||
profile: Some("release".into()),
|
||||
target: Some("x86_64-unknown-linux-gnu".into()),
|
||||
features: vec!["cuda".into(), "cudnn".into()],
|
||||
candle_version: Some("0.10.2".into()),
|
||||
};
|
||||
let json = serde_json::to_string(&info).unwrap();
|
||||
let back: BuildInfo = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(info, back);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deserializes_minimal_payload() {
|
||||
// An older neuron might send only the package version; every
|
||||
// other field must default rather than fail.
|
||||
let back: BuildInfo = serde_json::from_str(r#"{"package_version":"0.1.0"}"#).unwrap();
|
||||
assert_eq!(back.package_version, "0.1.0");
|
||||
assert_eq!(back.git_sha, "unknown");
|
||||
assert!(!back.git_dirty);
|
||||
assert!(back.features.is_empty());
|
||||
assert!(back.candle_version.is_none());
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,6 @@
|
||||
//! Model catalogue — profiles describing how to serve each model.
|
||||
|
||||
use crate::discovery::DeviceInfo;
|
||||
use crate::harness::{ModelCost, ModelLimit};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
|
||||
/// A model serving profile loaded from models.toml.
|
||||
@@ -22,87 +19,20 @@ pub struct ModelProfile {
|
||||
/// Minimum VRAM per device in MB.
|
||||
#[serde(default)]
|
||||
pub min_device_vram_mb: Option<u64>,
|
||||
/// Neurons this model is allowed to run on. Empty = anywhere its
|
||||
/// device constraints are satisfied.
|
||||
///
|
||||
/// This is an *affinity* constraint — where the model may be placed.
|
||||
/// It says nothing about whether the model may be evicted once
|
||||
/// resident; that is [`ModelProfile::residency_priority`]. The two
|
||||
/// were a single field once, which made "run only here" and "never
|
||||
/// evict here" impossible to ask for separately.
|
||||
/// Neurons where this model should never be evicted.
|
||||
#[serde(default)]
|
||||
pub pinned_on: Vec<String>,
|
||||
/// Which residency class this model belongs to when a node runs out
|
||||
/// of VRAM. A model may displace residents of its own class or
|
||||
/// below, and none above it — so models that should take turns on a
|
||||
/// node share a number, and a model is protected by being ranked
|
||||
/// *above* whatever must not evict it.
|
||||
///
|
||||
/// Unset means [`DEFAULT_RESIDENCY_PRIORITY`], except for profiles
|
||||
/// carrying `pinned_on`, which default to
|
||||
/// [`PINNED_RESIDENCY_PRIORITY`] — before these were separate
|
||||
/// fields, `pinned_on` implied immunity from eviction, and a
|
||||
/// catalogue written against that meaning must not silently start
|
||||
/// allowing its flagship to be evicted.
|
||||
#[serde(default)]
|
||||
pub residency_priority: Option<u32>,
|
||||
/// Source scheme this profile's weights come from. When set, the
|
||||
/// router prefixes `id` with `scheme:` before forwarding the load
|
||||
/// request to neuron, ensuring the daemon fetches from the right
|
||||
/// registry regardless of which entry happens to match `id`.
|
||||
///
|
||||
/// `None` lets neuron substitute its own `default_source` (typically
|
||||
/// `huggingface`). Set to `"helexa"` when the model is hosted in
|
||||
/// the helexa registry — operator-procurement-grade audit relies
|
||||
/// on this being explicit per model rather than implicit.
|
||||
#[serde(default)]
|
||||
pub source: Option<String>,
|
||||
|
||||
// ── Enrichment (issue #62) ────────────────────────────────
|
||||
/// Per-model token budget. When present, advertised in `/v1/models`
|
||||
/// so clients can size and compact their context automatically.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub limit: Option<ModelLimit>,
|
||||
/// Operator-set pricing (USD per 1M tokens). `0.0` for self-hosted.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cost: Option<ModelCost>,
|
||||
/// Static capability flags the operator wants to advertise even
|
||||
/// before the model is loaded on any neuron (e.g. `"reasoning"`,
|
||||
/// `"tool_call"`). Runtime-detected capabilities from the harness
|
||||
/// are unioned with this set in the gateway's `/v1/models` response.
|
||||
#[serde(default)]
|
||||
pub capabilities: Vec<String>,
|
||||
}
|
||||
|
||||
fn default_min_devices() -> u32 {
|
||||
1
|
||||
}
|
||||
|
||||
/// Residency priority for a model that declares none. Deliberately not
|
||||
/// zero: an operator needs room to rank something *below* the ordinary
|
||||
/// case (a scratch or experimental model that should yield to anything)
|
||||
/// without editing every other entry.
|
||||
pub const DEFAULT_RESIDENCY_PRIORITY: u32 = 100;
|
||||
|
||||
/// Residency priority assumed for a profile that carries `pinned_on` but
|
||||
/// no explicit priority. High enough that nothing with a default
|
||||
/// priority can evict it, preserving the immunity `pinned_on` used to
|
||||
/// grant on its own.
|
||||
pub const PINNED_RESIDENCY_PRIORITY: u32 = 1000;
|
||||
|
||||
/// The full model catalogue.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct ModelCatalogue {
|
||||
#[serde(default)]
|
||||
pub models: Vec<ModelProfile>,
|
||||
/// Tier aliases — clients can send a request with `model: "helexa/small"`
|
||||
/// and the gateway transparently rewrites + routes to the concrete
|
||||
/// model id this maps to. Lets operators define latency/quality
|
||||
/// tiers (`small`/`balanced`/`large`, `fast`/`thinking`, etc.)
|
||||
/// without imposing knowledge of specific model ids on clients.
|
||||
/// Loaded from the `[aliases]` table in models.toml.
|
||||
#[serde(default)]
|
||||
pub aliases: HashMap<String, String>,
|
||||
}
|
||||
|
||||
impl ModelCatalogue {
|
||||
@@ -128,366 +58,10 @@ impl ModelCatalogue {
|
||||
}
|
||||
}
|
||||
|
||||
/// How strongly `model_id` holds its place. Models absent from the
|
||||
/// catalogue rank at the default — a model can be resident on a
|
||||
/// neuron without a profile (loaded directly, or left over from an
|
||||
/// earlier catalogue), and treating those as unevictable would let
|
||||
/// an unlisted model wedge a node permanently.
|
||||
pub fn residency_priority(&self, model_id: &str) -> u32 {
|
||||
self.get(model_id)
|
||||
.map(|p| {
|
||||
p.residency_priority.unwrap_or({
|
||||
if p.pinned_on.is_empty() {
|
||||
DEFAULT_RESIDENCY_PRIORITY
|
||||
} else {
|
||||
PINNED_RESIDENCY_PRIORITY
|
||||
}
|
||||
})
|
||||
})
|
||||
.unwrap_or(DEFAULT_RESIDENCY_PRIORITY)
|
||||
}
|
||||
|
||||
/// May `incoming` take VRAM from `resident` when a node cannot hold
|
||||
/// both?
|
||||
///
|
||||
/// Greater-or-equal, which makes the priority a *class* rather than
|
||||
/// a strict order: a model may displace anything in its own class or
|
||||
/// below, and nothing above it.
|
||||
///
|
||||
/// Equal rank has to permit displacement, because mutual
|
||||
/// displacement is what cold-swap *is*. Two models that share a node
|
||||
/// and take turns on it — an image generator and a mid-tier text
|
||||
/// model, or two generations of the same flagship being compared —
|
||||
/// each need to evict the other on demand. Under a strict
|
||||
/// greater-than the first one to arrive would win permanently and
|
||||
/// the other could never come back, which reads as "the model
|
||||
/// vanished after we generated an image".
|
||||
///
|
||||
/// Protection therefore comes from ranking a model *above* its
|
||||
/// would-be evictor, not from ranking every model differently.
|
||||
///
|
||||
/// This governs only *whether* a displacement is permitted, never
|
||||
/// whether one is needed. A node with room for both evicts nothing,
|
||||
/// however the two rank.
|
||||
pub fn may_displace(&self, incoming_id: &str, resident_id: &str) -> bool {
|
||||
self.residency_priority(incoming_id) >= self.residency_priority(resident_id)
|
||||
}
|
||||
|
||||
/// Find a profile by model id.
|
||||
pub fn get(&self, model_id: &str) -> Option<&ModelProfile> {
|
||||
self.models.iter().find(|p| p.id == model_id)
|
||||
}
|
||||
|
||||
/// Resolve an alias to its concrete model id. Returns `id` verbatim
|
||||
/// when it isn't an alias. Aliases never chain — operator config
|
||||
/// is treated as flat — so this is a single lookup.
|
||||
pub fn resolve_alias<'a>(&'a self, id: &'a str) -> &'a str {
|
||||
self.aliases.get(id).map(String::as_str).unwrap_or(id)
|
||||
}
|
||||
}
|
||||
|
||||
impl ModelProfile {
|
||||
/// True iff this profile's placement constraints can be satisfied
|
||||
/// by the named neuron with the given device topology.
|
||||
///
|
||||
/// Constraints checked:
|
||||
/// - `pinned_on`: non-empty → neuron must be on the list.
|
||||
/// - `min_devices`: neuron must have at least this many devices.
|
||||
/// - `min_device_vram_mb`: at least `min_devices` of the neuron's
|
||||
/// devices must each meet this VRAM floor.
|
||||
pub fn is_feasible_on(&self, neuron_name: &str, devices: &[DeviceInfo]) -> bool {
|
||||
if !self.pinned_on.is_empty() && !self.pinned_on.iter().any(|n| n == neuron_name) {
|
||||
return false;
|
||||
}
|
||||
if (devices.len() as u32) < self.min_devices {
|
||||
return false;
|
||||
}
|
||||
if let Some(min_vram) = self.min_device_vram_mb {
|
||||
let big_enough = devices
|
||||
.iter()
|
||||
.filter(|d| d.vram_total_mb >= min_vram)
|
||||
.count() as u32;
|
||||
if big_enough < self.min_devices {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::discovery::DeviceInfo;
|
||||
|
||||
fn device(idx: u32, vram_mb: u64) -> DeviceInfo {
|
||||
DeviceInfo {
|
||||
index: idx,
|
||||
name: format!("DEV-{idx}"),
|
||||
vram_total_mb: vram_mb,
|
||||
compute_capability: "8.6".into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn profile() -> ModelProfile {
|
||||
ModelProfile {
|
||||
id: "Qwen/Qwen3.6-27B".into(),
|
||||
harness: "candle".into(),
|
||||
quant: None,
|
||||
vram_mb: Some(45_000),
|
||||
min_devices: 2,
|
||||
min_device_vram_mb: Some(24_000),
|
||||
pinned_on: vec![],
|
||||
residency_priority: None,
|
||||
source: None,
|
||||
limit: None,
|
||||
cost: None,
|
||||
capabilities: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn feasible_when_two_devices_meet_vram_floor() {
|
||||
let p = profile();
|
||||
let devices = [device(0, 32_000), device(1, 32_000)];
|
||||
assert!(p.is_feasible_on("beast", &devices));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn infeasible_when_only_one_device() {
|
||||
let p = profile();
|
||||
let devices = [device(0, 64_000)];
|
||||
assert!(!p.is_feasible_on("benjy", &devices));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn infeasible_when_one_device_underspec() {
|
||||
let p = profile();
|
||||
let devices = [device(0, 32_000), device(1, 12_000)];
|
||||
assert!(!p.is_feasible_on("mixed", &devices));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pinned_on_excludes_other_neurons() {
|
||||
let mut p = profile();
|
||||
p.pinned_on = vec!["beast".into()];
|
||||
let devices = [device(0, 32_000), device(1, 32_000)];
|
||||
assert!(p.is_feasible_on("beast", &devices));
|
||||
assert!(!p.is_feasible_on("benjy", &devices));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_vram_floor_just_needs_min_devices() {
|
||||
let mut p = profile();
|
||||
p.min_device_vram_mb = None;
|
||||
let devices = [device(0, 1_000), device(1, 1_000)];
|
||||
assert!(p.is_feasible_on("anywhere", &devices));
|
||||
}
|
||||
|
||||
/// A catalogue shaped like a real fleet. Two residency classes: the
|
||||
/// big-node class holds a flagship and a frontier model that take
|
||||
/// turns on the one machine large enough for either; the everyday
|
||||
/// class holds an image generator and a mid-tier text model that
|
||||
/// take turns on a smaller one. Nothing in the everyday class may
|
||||
/// touch the big-node class.
|
||||
fn tiered_catalogue() -> ModelCatalogue {
|
||||
toml::from_str(
|
||||
r#"
|
||||
[[models]]
|
||||
id = "flagship"
|
||||
harness = "candle"
|
||||
pinned_on = ["big-node"]
|
||||
residency_priority = 300
|
||||
|
||||
[[models]]
|
||||
id = "frontier"
|
||||
harness = "candle"
|
||||
residency_priority = 300
|
||||
|
||||
[[models]]
|
||||
id = "image"
|
||||
harness = "candle"
|
||||
residency_priority = 200
|
||||
|
||||
[[models]]
|
||||
id = "mid"
|
||||
harness = "candle"
|
||||
residency_priority = 200
|
||||
|
||||
[[models]]
|
||||
id = "tiny"
|
||||
harness = "candle"
|
||||
residency_priority = 100
|
||||
"#,
|
||||
)
|
||||
.expect("parse tiered catalogue")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn image_generation_displaces_the_mid_tier() {
|
||||
assert!(tiered_catalogue().may_displace("image", "mid"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_mid_tier_comes_back_after_an_image_takes_its_node() {
|
||||
// The swap-back half. An image request evicts the text model;
|
||||
// the next text request must be able to evict the image model,
|
||||
// or the text tier disappears until someone restarts something.
|
||||
assert!(tiered_catalogue().may_displace("mid", "image"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_generations_of_a_flagship_can_swap_in_both_directions() {
|
||||
// Comparing a new flagship against the incumbent needs traffic
|
||||
// to move each way on demand. A strict order would let whichever
|
||||
// arrived first hold the node permanently.
|
||||
let cat = tiered_catalogue();
|
||||
assert!(cat.may_displace("frontier", "flagship"));
|
||||
assert!(cat.may_displace("flagship", "frontier"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn image_generation_never_displaces_the_flagship() {
|
||||
// The image generator's device constraints alone would let it
|
||||
// land on the flagship's node, so this is the case priority
|
||||
// exists to prevent -- not a hypothetical one.
|
||||
assert!(!tiered_catalogue().may_displace("image", "flagship"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_frontier_tier_displaces_the_flagship() {
|
||||
// The requirement a boolean pin cannot express: the flagship is
|
||||
// protected from one model and not from another.
|
||||
assert!(tiered_catalogue().may_displace("frontier", "flagship"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_lower_class_cannot_displace_a_higher_one() {
|
||||
let cat = tiered_catalogue();
|
||||
assert!(!cat.may_displace("tiny", "mid"));
|
||||
assert!(!cat.may_displace("tiny", "flagship"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_higher_class_can_displace_a_lower_one() {
|
||||
assert!(tiered_catalogue().may_displace("flagship", "tiny"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pinned_on_alone_still_protects_a_catalogue_written_before_priorities() {
|
||||
// `pinned_on` used to mean "never evict here". A catalogue that
|
||||
// predates the split says nothing about priority, and must not
|
||||
// silently start allowing its flagship to be evicted.
|
||||
let cat: ModelCatalogue = toml::from_str(
|
||||
r#"
|
||||
[[models]]
|
||||
id = "flagship"
|
||||
harness = "candle"
|
||||
pinned_on = ["big-node"]
|
||||
|
||||
[[models]]
|
||||
id = "ordinary"
|
||||
harness = "candle"
|
||||
"#,
|
||||
)
|
||||
.expect("parse legacy catalogue");
|
||||
assert!(!cat.may_displace("ordinary", "flagship"));
|
||||
assert!(cat.may_displace("flagship", "ordinary"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unlisted_resident_is_displaceable_by_a_ranked_model() {
|
||||
// A model can be resident without a profile. Treating it as
|
||||
// unevictable would let an unlisted model wedge a node forever.
|
||||
let cat = tiered_catalogue();
|
||||
assert_eq!(
|
||||
cat.residency_priority("never-heard-of-it"),
|
||||
DEFAULT_RESIDENCY_PRIORITY
|
||||
);
|
||||
assert!(cat.may_displace("image", "never-heard-of-it"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn affinity_and_immunity_are_independently_expressible() {
|
||||
// The whole point of the split: confine a model to a node
|
||||
// without protecting it there, and protect one without
|
||||
// confining it anywhere.
|
||||
let cat: ModelCatalogue = toml::from_str(
|
||||
r#"
|
||||
[[models]]
|
||||
id = "confined-but-evictable"
|
||||
harness = "candle"
|
||||
pinned_on = ["big-node"]
|
||||
residency_priority = 50
|
||||
|
||||
[[models]]
|
||||
id = "roaming-but-protected"
|
||||
harness = "candle"
|
||||
residency_priority = 900
|
||||
"#,
|
||||
)
|
||||
.expect("parse catalogue");
|
||||
let devices = [device(0, 32_000)];
|
||||
|
||||
let confined = cat.get("confined-but-evictable").unwrap();
|
||||
assert!(confined.is_feasible_on("big-node", &devices));
|
||||
assert!(!confined.is_feasible_on("other-node", &devices));
|
||||
|
||||
let roaming = cat.get("roaming-but-protected").unwrap();
|
||||
assert!(roaming.is_feasible_on("other-node", &devices));
|
||||
|
||||
assert!(cat.may_displace("roaming-but-protected", "confined-but-evictable"));
|
||||
assert!(!cat.may_displace("confined-but-evictable", "roaming-but-protected"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_alias_returns_target_when_alias_present() {
|
||||
let mut cat = ModelCatalogue::default();
|
||||
cat.aliases
|
||||
.insert("helexa/small".into(), "Qwen/Qwen3-1.7B".into());
|
||||
assert_eq!(cat.resolve_alias("helexa/small"), "Qwen/Qwen3-1.7B");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_alias_passes_through_when_not_an_alias() {
|
||||
let mut cat = ModelCatalogue::default();
|
||||
cat.aliases
|
||||
.insert("helexa/small".into(), "Qwen/Qwen3-1.7B".into());
|
||||
assert_eq!(cat.resolve_alias("Qwen/Qwen3-8B"), "Qwen/Qwen3-8B");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_defaults_to_none_when_absent_from_toml() {
|
||||
let src = r#"
|
||||
[[models]]
|
||||
id = "Qwen/Qwen3-30B"
|
||||
harness = "candle"
|
||||
"#;
|
||||
let cat: ModelCatalogue = toml::from_str(src).expect("parse models table");
|
||||
assert!(cat.models[0].source.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_round_trips_through_toml() {
|
||||
let src = r#"
|
||||
[[models]]
|
||||
id = "Helexa/Qwen3.6-27B-Uncensored"
|
||||
harness = "candle"
|
||||
source = "helexa"
|
||||
"#;
|
||||
let cat: ModelCatalogue = toml::from_str(src).expect("parse models table");
|
||||
assert_eq!(cat.models[0].source.as_deref(), Some("helexa"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aliases_table_round_trips_through_toml() {
|
||||
let src = r#"
|
||||
[aliases]
|
||||
"helexa/small" = "Qwen/Qwen3-1.7B"
|
||||
"helexa/large" = "Qwen/Qwen3.6-27B"
|
||||
"#;
|
||||
let cat: ModelCatalogue = toml::from_str(src).expect("parse aliases table");
|
||||
assert_eq!(cat.resolve_alias("helexa/small"), "Qwen/Qwen3-1.7B");
|
||||
assert_eq!(cat.resolve_alias("helexa/large"), "Qwen/Qwen3.6-27B");
|
||||
/// Check if a model is pinned on a given neuron.
|
||||
pub fn is_pinned(&self, model_id: &str, neuron_name: &str) -> bool {
|
||||
self.models
|
||||
.iter()
|
||||
.any(|p| p.id == model_id && p.pinned_on.contains(&neuron_name.to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
use crate::entitlements::CapWindow;
|
||||
use figment::{
|
||||
Figment,
|
||||
providers::{Env, Format, Toml},
|
||||
@@ -12,98 +11,13 @@ pub struct GatewayConfig {
|
||||
pub eviction: EvictionSettings,
|
||||
/// Neuron endpoints (replaces old NodeConfig with static vram_mb/pinned).
|
||||
pub neurons: Vec<NeuronEndpoint>,
|
||||
/// Path to the model catalogue file. Defaults to the packaged
|
||||
/// location (`/etc/cortex/models.toml`); set explicitly for
|
||||
/// non-packaged / local runs.
|
||||
/// Path to the model catalogue file (default: "models.toml").
|
||||
#[serde(default = "default_models_path")]
|
||||
pub models_config: String,
|
||||
/// Multi-tenant governance: auth + per-key token budgets (#47). Empty
|
||||
/// by default — anonymous, uncapped — so existing single-operator
|
||||
/// setups keep working until keys are configured.
|
||||
#[serde(default)]
|
||||
pub entitlements: EntitlementsConfig,
|
||||
/// helexa-upstream client (#57). When enabled, keys not found in the
|
||||
/// local `[entitlements]` config are validated against the mesh
|
||||
/// authority, and budget is reserved/settled there. Disabled by default
|
||||
/// — a single operator runs purely local.
|
||||
#[serde(default)]
|
||||
pub upstream: UpstreamClientConfig,
|
||||
}
|
||||
|
||||
/// `[upstream]` — the helexa-upstream authority client (#57). Locally
|
||||
/// unrecognised bearer keys are resolved against `url`'s `/authz/v1` surface
|
||||
/// (mesh accounts); local keys (operator + infra) never leave the process.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct UpstreamClientConfig {
|
||||
/// Enable the upstream fallthrough. Off → purely local entitlements.
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
/// Base URL of helexa-upstream (e.g. "https://upstream.helexa.ai").
|
||||
#[serde(default)]
|
||||
pub url: String,
|
||||
/// Shared client bearer this cortex presents to `/authz/v1` (maps to an
|
||||
/// operator_id upstream). Sent as `Authorization: Bearer <bearer>`.
|
||||
#[serde(default)]
|
||||
pub bearer: String,
|
||||
/// Per-call timeout (seconds) to upstream.
|
||||
#[serde(default = "default_upstream_timeout")]
|
||||
pub timeout_secs: u64,
|
||||
/// How often (seconds) to flush served-usage counters to upstream for
|
||||
/// reconciliation (#58).
|
||||
#[serde(default = "default_served_usage_interval")]
|
||||
pub served_usage_report_interval_secs: u64,
|
||||
}
|
||||
|
||||
fn default_upstream_timeout() -> u64 {
|
||||
5
|
||||
}
|
||||
fn default_served_usage_interval() -> u64 {
|
||||
60
|
||||
}
|
||||
|
||||
/// `[entitlements]` — the local/static [`crate::entitlements::EntitlementProvider`]
|
||||
/// source of truth (#50). Accounts, keys, and hard caps live here; the
|
||||
/// future upstream client (#57) ignores this section.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct EntitlementsConfig {
|
||||
/// Reject unauthenticated requests with `401 invalid_api_key` when
|
||||
/// true. Default `false` (allow-anonymous) for dev / single-operator
|
||||
/// continuity.
|
||||
#[serde(default)]
|
||||
pub require_auth: bool,
|
||||
/// Static API keys and their budgets, consumed by the local provider.
|
||||
#[serde(default)]
|
||||
pub keys: Vec<ApiKeyConfig>,
|
||||
}
|
||||
|
||||
/// One configured API key: the bearer token, the account it bills to, and
|
||||
/// its hard cap. `[[entitlements.keys]]` in TOML.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ApiKeyConfig {
|
||||
/// The bearer token clients send in `Authorization: Bearer <key>`.
|
||||
pub key: String,
|
||||
/// Billable account. Multiple keys may share one account.
|
||||
pub account_id: String,
|
||||
/// Stable per-key identifier for ledger/metrics labels. Defaults to
|
||||
/// `account_id` when omitted, so the secret is never used as a label.
|
||||
#[serde(default)]
|
||||
pub key_id: Option<String>,
|
||||
/// Hard token cap. `None`/omitted = uncapped (e.g. operator infra key).
|
||||
#[serde(default)]
|
||||
pub hard_cap: Option<u64>,
|
||||
/// Cap-window semantics. Default: a non-resetting [`CapWindow::Balance`].
|
||||
#[serde(default)]
|
||||
pub window: CapWindow,
|
||||
}
|
||||
|
||||
fn default_models_path() -> String {
|
||||
// Absolute, so the systemd-launched binary finds the catalogue
|
||||
// regardless of its working directory. The RPM installs the catalogue
|
||||
// here (`cortex.spec`); a relative "models.toml" silently resolved to
|
||||
// the service cwd and left the catalogue empty in production
|
||||
// (pinning / aliases / limits all no-ops). Override via `models_config`
|
||||
// in cortex.toml for local runs.
|
||||
"/etc/cortex/models.toml".into()
|
||||
"models.toml".into()
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
@@ -165,8 +79,6 @@ impl Default for GatewayConfig {
|
||||
},
|
||||
neurons: vec![],
|
||||
models_config: default_models_path(),
|
||||
entitlements: EntitlementsConfig::default(),
|
||||
upstream: UpstreamClientConfig::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,23 +22,6 @@ pub struct DiscoveryResponse {
|
||||
pub driver_version: Option<String>,
|
||||
pub devices: Vec<DeviceInfo>,
|
||||
pub harnesses: Vec<String>,
|
||||
/// Set when the host has an NVIDIA stack that is currently
|
||||
/// unusable — specifically the userspace↔kernel-module version
|
||||
/// skew after an un-rebooted driver update ("Driver/library
|
||||
/// version mismatch"), where every CUDA call including nvidia-smi
|
||||
/// fails (#19). `None` on healthy hosts AND on hosts with no
|
||||
/// NVIDIA stack at all (CPU-only is not an error). Carries an
|
||||
/// operator-actionable description; cortex can read it to route
|
||||
/// around the node instead of cold-loading into a guaranteed
|
||||
/// failure.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cuda_unavailable_reason: Option<String>,
|
||||
/// The neuron's effective maximum prompt size in tokens
|
||||
/// (`NEURON_MAX_PROMPT_TOKENS`) — the enforced prompt cap on this
|
||||
/// host. `#[serde(default)]` (→ 0) for forward-compat with neurons
|
||||
/// that predate this field; cortex treats 0 as "unknown".
|
||||
#[serde(default)]
|
||||
pub max_prompt_tokens: u64,
|
||||
}
|
||||
|
||||
/// Runtime health metrics for a single GPU device.
|
||||
@@ -53,224 +36,8 @@ pub struct DeviceHealth {
|
||||
|
||||
/// Runtime health response from a neuron endpoint.
|
||||
/// Returned by `GET /health`.
|
||||
///
|
||||
/// `activation` was added in 2026-05-26 to distinguish "process is up
|
||||
/// and reachable" from "process is ready to serve traffic". A `Type=simple`
|
||||
/// systemd unit reports `active` the moment the binary starts — but a
|
||||
/// neuron whose `default_models` list takes minutes to materialise
|
||||
/// won't bind its listener (or, in the new flow, won't have any models
|
||||
/// loaded) until pre-warm completes. The new field is `#[serde(default)]`
|
||||
/// so a pre-2026-05-26 gateway polling a new neuron — or vice versa —
|
||||
/// keeps working.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct HealthResponse {
|
||||
pub uptime_secs: u64,
|
||||
pub devices: Vec<DeviceHealth>,
|
||||
#[serde(default)]
|
||||
pub activation: ActivationStatus,
|
||||
/// Per-model admission load (#53): how many requests are running vs.
|
||||
/// queued on each loaded model right now. Cortex's load-aware router
|
||||
/// (#55) reads this to spread traffic across replicas and to propagate
|
||||
/// honest backpressure. `#[serde(default)]` keeps older gateways/neurons
|
||||
/// interoperable (absent → empty → treated as no load info).
|
||||
#[serde(default)]
|
||||
pub models: Vec<ModelLoad>,
|
||||
}
|
||||
|
||||
/// Live admission load for one loaded model (#53).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ModelLoad {
|
||||
pub id: String,
|
||||
/// Requests currently running (batch-1 → 0 or 1).
|
||||
pub in_flight: usize,
|
||||
/// Requests waiting in the bounded admission queue.
|
||||
pub queue_depth: usize,
|
||||
/// Admission concurrency ceiling (#137) — the denominator for
|
||||
/// saturation = `in_flight / max_in_flight`. `#[serde(default)]` (→ 0)
|
||||
/// for pre-#137 neurons; cortex treats 0 as "unknown" and skips the
|
||||
/// ceiling gauge so a rolling deploy doesn't publish a bogus 0.
|
||||
#[serde(default)]
|
||||
pub max_in_flight: usize,
|
||||
/// Admission queue capacity (#137): how many requests may wait beyond
|
||||
/// the in-flight slots before the model sheds load. `#[serde(default)]`
|
||||
/// for back-compat with pre-#137 neurons.
|
||||
#[serde(default)]
|
||||
pub max_queue_depth: usize,
|
||||
/// Cumulative requests rejected because the admission queue was full
|
||||
/// (#137), since this model loaded. The load-shedding signal; cortex
|
||||
/// publishes it as a counter. `#[serde(default)]` for back-compat.
|
||||
#[serde(default)]
|
||||
pub rejected_queue_full: u64,
|
||||
/// Cumulative requests rejected because the in-flight slot didn't free
|
||||
/// within `max_wait` (#137). `#[serde(default)]` for back-compat.
|
||||
#[serde(default)]
|
||||
pub rejected_timeout: u64,
|
||||
/// Cumulative requests rejected by the per-principal fair-share cap
|
||||
/// (#54/#137). `#[serde(default)]` for back-compat.
|
||||
#[serde(default)]
|
||||
pub rejected_per_principal: u64,
|
||||
/// Anonymous requests currently holding a seat (#262). Read against
|
||||
/// `in_flight` it says how much of this model's load is unattributable;
|
||||
/// against `anon_max_in_flight`, how close that traffic is to the
|
||||
/// ceiling that keeps it from starving identified callers.
|
||||
#[serde(default)]
|
||||
pub anon_in_flight: usize,
|
||||
/// Seats anonymous traffic may hold at once (#262); `0` = refused
|
||||
/// outright. `#[serde(default)]` (→ 0) for pre-#262 neurons, which
|
||||
/// applied no ceiling at all — cortex skips the gauge rather than
|
||||
/// publish a 0 that reads as the opposite of the truth.
|
||||
#[serde(default)]
|
||||
pub anon_max_in_flight: usize,
|
||||
/// Cumulative anonymous requests refused for want of leftover capacity
|
||||
/// (#262). Rising here with a flat `rejected_timeout` means the
|
||||
/// reservation is working, not that the model is overloaded.
|
||||
#[serde(default)]
|
||||
pub rejected_anon_yield: u64,
|
||||
/// Cumulative requests that waited for KV budget and never got it
|
||||
/// (#257) — the model is saturated by long-context sequences rather
|
||||
/// than by request count. `#[serde(default)]` for back-compat.
|
||||
#[serde(default)]
|
||||
pub rejected_kv_timeout: u64,
|
||||
/// Cumulative requests whose KV reservation exceeded the model's entire
|
||||
/// budget (#257) — prompts too long for this node at any load, not a
|
||||
/// load-shedding signal. `#[serde(default)]` for back-compat.
|
||||
#[serde(default)]
|
||||
pub rejected_kv_unservable: u64,
|
||||
/// This model's total KV budget in MiB and how much of it is currently
|
||||
/// unreserved (#257). `kv_budget_mb == 0` means the gate is disabled
|
||||
/// (CPU load, or an arch with no context profile).
|
||||
#[serde(default)]
|
||||
pub kv_budget_mb: u64,
|
||||
#[serde(default)]
|
||||
pub kv_available_mb: u64,
|
||||
/// Live prefill throughput EMA in tokens/sec (#137) — prompt tokens
|
||||
/// processed per second. `0.0` before the first sample. `#[serde(default)]`
|
||||
/// for back-compat with pre-#137 neurons.
|
||||
#[serde(default)]
|
||||
pub tok_s_prefill: f64,
|
||||
/// Live decode throughput EMA in tokens/sec (#137) — generation tokens
|
||||
/// per second, the headline capacity number. `0.0` before the first
|
||||
/// sample. `#[serde(default)]` for back-compat.
|
||||
#[serde(default)]
|
||||
pub tok_s_decode: f64,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod health_load_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn health_response_without_models_field_still_deserializes() {
|
||||
// A pre-#53 neuron's /health payload omits `models`; the gateway
|
||||
// must still parse it (serde default → empty).
|
||||
let json = r#"{"uptime_secs":42,"devices":[]}"#;
|
||||
let resp: HealthResponse = serde_json::from_str(json).expect("back-compat parse");
|
||||
assert_eq!(resp.uptime_secs, 42);
|
||||
assert!(resp.models.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn health_response_round_trips_model_load() {
|
||||
let resp = HealthResponse {
|
||||
uptime_secs: 1,
|
||||
devices: vec![],
|
||||
activation: ActivationStatus::default(),
|
||||
models: vec![ModelLoad {
|
||||
id: "Qwen/Qwen3.6-27B".into(),
|
||||
in_flight: 1,
|
||||
queue_depth: 3,
|
||||
max_in_flight: 8,
|
||||
max_queue_depth: 8,
|
||||
rejected_queue_full: 0,
|
||||
rejected_kv_timeout: 0,
|
||||
rejected_kv_unservable: 0,
|
||||
kv_budget_mb: 0,
|
||||
kv_available_mb: 0,
|
||||
rejected_timeout: 0,
|
||||
rejected_per_principal: 0,
|
||||
anon_in_flight: 2,
|
||||
anon_max_in_flight: 7,
|
||||
rejected_anon_yield: 0,
|
||||
tok_s_prefill: 0.0,
|
||||
tok_s_decode: 0.0,
|
||||
}],
|
||||
};
|
||||
let s = serde_json::to_string(&resp).unwrap();
|
||||
let back: HealthResponse = serde_json::from_str(&s).unwrap();
|
||||
assert_eq!(back.models.len(), 1);
|
||||
assert_eq!(back.models[0].in_flight, 1);
|
||||
assert_eq!(back.models[0].queue_depth, 3);
|
||||
assert_eq!(back.models[0].max_in_flight, 8);
|
||||
assert_eq!(back.models[0].max_queue_depth, 8);
|
||||
assert_eq!(back.models[0].anon_in_flight, 2);
|
||||
assert_eq!(back.models[0].anon_max_in_flight, 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_load_without_ceiling_fields_defaults_to_zero() {
|
||||
// A pre-#137 neuron omits max_in_flight/max_queue_depth; cortex must
|
||||
// still parse (serde default → 0, treated as "unknown").
|
||||
let json = r#"{"id":"m","in_flight":2,"queue_depth":0}"#;
|
||||
let m: ModelLoad = serde_json::from_str(json).expect("back-compat parse");
|
||||
assert_eq!(m.in_flight, 2);
|
||||
assert_eq!(m.max_in_flight, 0);
|
||||
assert_eq!(m.max_queue_depth, 0);
|
||||
// Likewise a pre-#262 neuron, which enforced no anonymous ceiling.
|
||||
assert_eq!(m.anon_in_flight, 0);
|
||||
assert_eq!(m.anon_max_in_flight, 0);
|
||||
assert_eq!(m.rejected_anon_yield, 0);
|
||||
}
|
||||
}
|
||||
|
||||
/// High-level activation state of the neuron daemon. The HTTP listener
|
||||
/// is bound during both states; what differs is whether the configured
|
||||
/// `default_models` have finished loading.
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ActivationState {
|
||||
/// At least one `default_models` entry is still loading. The
|
||||
/// neuron's other endpoints work, but inference against
|
||||
/// not-yet-loaded models will 404.
|
||||
PreWarming,
|
||||
/// Every `default_models` entry has either loaded or failed; the
|
||||
/// neuron is steady-state. Subsequent on-demand loads via
|
||||
/// `/models/load` don't flip back to PreWarming — that field
|
||||
/// reflects the activation-time set only.
|
||||
#[default]
|
||||
Ready,
|
||||
}
|
||||
|
||||
/// Per-model failure record surfaced in [`ActivationStatus::failed`].
|
||||
/// The error string is the rendered anyhow chain at the time of the
|
||||
/// failure; operators read it from `/health` to decide whether to
|
||||
/// retry, edit the spec, or unload+reload.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct PreWarmFailure {
|
||||
pub model_id: String,
|
||||
pub error: String,
|
||||
}
|
||||
|
||||
/// Activation-time progress snapshot. All four lists are populated by
|
||||
/// the neuron's pre-warm task and read by the `/health` handler. The
|
||||
/// snapshot is consistent: a model id appears in exactly one of
|
||||
/// `pending`, `in_progress` (as `Option<String>`), `completed`, or
|
||||
/// `failed` at any point in time.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct ActivationStatus {
|
||||
pub state: ActivationState,
|
||||
/// Model ids queued but not yet started. Empty in `Ready` state.
|
||||
#[serde(default)]
|
||||
pub pending: Vec<String>,
|
||||
/// Model id currently materialising. None when between models or
|
||||
/// in `Ready` state.
|
||||
#[serde(default)]
|
||||
pub in_progress: Option<String>,
|
||||
/// Model ids that finished loading successfully during this
|
||||
/// activation. Cleared on process restart.
|
||||
#[serde(default)]
|
||||
pub completed: Vec<String>,
|
||||
/// Model ids that failed during this activation, with the rendered
|
||||
/// error chain. Cleared on process restart.
|
||||
#[serde(default)]
|
||||
pub failed: Vec<PreWarmFailure>,
|
||||
}
|
||||
|
||||
@@ -1,152 +0,0 @@
|
||||
//! Identity and entitlement primitives for multi-tenant governance (#47).
|
||||
//!
|
||||
//! Identity is the shared substrate the whole epic hangs off:
|
||||
//! `identity (principal) → accounting (spend) → policy → enforcement`. This
|
||||
//! module defines the seam — the [`EntitlementProvider`] trait and its data
|
||||
//! types — so the local/static provider (operator-config caps, in
|
||||
//! cortex-gateway) can land the auth + per-key-cap + amplification fix
|
||||
//! *before* any upstream clearing house exists. The future helexa-upstream
|
||||
//! client (#57) is just another impl of this trait.
|
||||
//!
|
||||
//! The provider owns three jobs:
|
||||
//! 1. **resolve** a bearer key to a [`Principal`] (drives auth, #49);
|
||||
//! 2. **reserve → settle/release** token budget around a request so spend
|
||||
//! can never overshoot a hard cap under concurrency (drives budget
|
||||
//! enforcement, #52);
|
||||
//! 3. expose a [`BudgetSnapshot`] for metering/metrics (#51).
|
||||
//!
|
||||
//! [`BudgetError`] carries the cap-window semantics so the caller can pick
|
||||
//! the correct #63 rejection (`rate_limit_exceeded` + `Retry-After` for a
|
||||
//! resetting window vs `insufficient_quota` for a hard balance) without the
|
||||
//! provider knowing anything about HTTP.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Internal header carrying the resolved account id from cortex to neuron.
|
||||
/// neuron trusts these over the WireGuard link (#54); cortex **strips** any
|
||||
/// client-supplied copy before stamping the authoritative value, so a client
|
||||
/// can never assert a principal directly.
|
||||
pub const HEADER_ACCOUNT_ID: &str = "x-helexa-account-id";
|
||||
/// Internal header carrying the resolved key id from cortex to neuron.
|
||||
pub const HEADER_KEY_ID: &str = "x-helexa-key-id";
|
||||
|
||||
/// Who a request is for. Resolved once at the edge from the bearer key and
|
||||
/// carried through the request context. `account_id` is the billable owner
|
||||
/// (spendable at any operator, by decision); `key_id` identifies the
|
||||
/// specific API key for per-key hard caps and ledger/metrics labels.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct Principal {
|
||||
pub account_id: String,
|
||||
pub key_id: String,
|
||||
}
|
||||
|
||||
/// Cap-window semantics for a key's hard cap. Determines which #63 code an
|
||||
/// over-cap reservation maps to.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub enum CapWindow {
|
||||
/// Hard balance — the cap never resets. Exhaustion is permanent
|
||||
/// (`429 insufficient_quota`, no `Retry-After`).
|
||||
#[default]
|
||||
Balance,
|
||||
/// Rolling window of `seconds` that resets. Exhaustion is transient
|
||||
/// (`429 rate_limit_exceeded` + `Retry-After` until reset).
|
||||
Rolling { seconds: u64 },
|
||||
}
|
||||
|
||||
/// An outstanding budget reservation. The caller holds this opaque handle
|
||||
/// between [`EntitlementProvider::reserve`] and exactly one of
|
||||
/// [`EntitlementProvider::settle`] / [`EntitlementProvider::release`]. Not
|
||||
/// `Clone` — a reservation is consumed once.
|
||||
#[derive(Debug)]
|
||||
pub struct Reservation {
|
||||
/// Provider-local handle; opaque to the caller.
|
||||
pub id: u64,
|
||||
/// The principal this reservation belongs to.
|
||||
pub principal: Principal,
|
||||
/// Tokens reserved against the cap.
|
||||
pub reserved: u64,
|
||||
}
|
||||
|
||||
/// A point-in-time view of a key's budget, for metering and metrics (#51).
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct BudgetSnapshot {
|
||||
/// Hard cap in tokens. `None` means uncapped (e.g. an operator infra
|
||||
/// key, #58).
|
||||
pub hard_cap: Option<u64>,
|
||||
/// Settled spend in the current window.
|
||||
pub spent: u64,
|
||||
/// Sum of outstanding (un-settled) reservations.
|
||||
pub reserved: u64,
|
||||
}
|
||||
|
||||
/// Authentication failure — the bearer key could not be resolved.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum AuthError {
|
||||
/// The key is genuinely unknown → `401 invalid_api_key` (#49/#63).
|
||||
#[error("invalid or unknown API key")]
|
||||
InvalidKey,
|
||||
/// The authority that could resolve the key is unreachable (e.g. the
|
||||
/// helexa-upstream client failed, #57). Fail **closed** but distinctly:
|
||||
/// a transient outage must surface as `503 service_unavailable` +
|
||||
/// `Retry-After`, never `401` — a real key must not be rejected as
|
||||
/// invalid during an upstream blip.
|
||||
#[error("entitlement authority unavailable; retry in {retry_after_secs}s")]
|
||||
Unavailable { retry_after_secs: u64 },
|
||||
}
|
||||
|
||||
/// Why a reservation was refused. Carries enough for the caller to build the
|
||||
/// correct #63 envelope without the provider touching HTTP.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum BudgetError {
|
||||
/// A resetting window is exhausted → `429 rate_limit_exceeded` +
|
||||
/// `Retry-After: retry_after_secs`.
|
||||
#[error(
|
||||
"rolling-window budget exhausted ({requested} requested, {available} available); \
|
||||
resets in {retry_after_secs}s"
|
||||
)]
|
||||
RateLimited {
|
||||
requested: u64,
|
||||
available: u64,
|
||||
retry_after_secs: u64,
|
||||
},
|
||||
/// A hard balance is exhausted → `429 insufficient_quota` (no
|
||||
/// `Retry-After`; the client surfaces and stops). Never `402`.
|
||||
#[error("hard balance exhausted ({requested} requested, {available} available)")]
|
||||
InsufficientQuota { requested: u64, available: u64 },
|
||||
}
|
||||
|
||||
/// The seam between cortex's enforcement and whatever decides entitlement —
|
||||
/// a local/static config provider today (#50), the helexa-upstream client
|
||||
/// later (#57). All methods are async so the upstream impl can do network
|
||||
/// I/O; the local impl resolves in-process.
|
||||
#[async_trait]
|
||||
pub trait EntitlementProvider: Send + Sync {
|
||||
/// Resolve a bearer API key to its principal. `Err(InvalidKey)` for an
|
||||
/// unknown/empty key.
|
||||
async fn resolve(&self, api_key: &str) -> Result<Principal, AuthError>;
|
||||
|
||||
/// Reserve up to `max_tokens` against the principal's cap. Returns a
|
||||
/// handle on success, or a [`BudgetError`] (which the caller maps to a
|
||||
/// #63 `429`) if the reservation would exceed the cap. Reserving the
|
||||
/// *maximum* a request could consume before dispatch is what prevents
|
||||
/// overshoot under concurrency.
|
||||
async fn reserve(
|
||||
&self,
|
||||
principal: &Principal,
|
||||
max_tokens: u64,
|
||||
) -> Result<Reservation, BudgetError>;
|
||||
|
||||
/// Settle a reservation with the tokens actually consumed, releasing the
|
||||
/// unused remainder back to the cap.
|
||||
async fn settle(&self, reservation: Reservation, actual_tokens: u64);
|
||||
|
||||
/// Release a reservation in full — e.g. dispatch failed before any
|
||||
/// tokens were consumed.
|
||||
async fn release(&self, reservation: Reservation);
|
||||
|
||||
/// Current budget snapshot for a principal, for metering/metrics.
|
||||
/// `None` if the provider doesn't track this principal.
|
||||
async fn snapshot(&self, principal: &Principal) -> Option<BudgetSnapshot>;
|
||||
}
|
||||
@@ -1,257 +0,0 @@
|
||||
//! The OpenAI-standard error envelope (#60) and the rejection contract
|
||||
//! that rides on it (#63).
|
||||
//!
|
||||
//! Every non-2xx response cortex and neuron emit uses the shape
|
||||
//!
|
||||
//! ```json
|
||||
//! { "error": { "message": "...", "type": "...", "code": "...", "param": null } }
|
||||
//! ```
|
||||
//!
|
||||
//! because OpenAI-compatible clients (opencode, the AI SDK, litellm, the
|
||||
//! OpenAI SDKs) read `error.type` / `error.code` to decide what to do —
|
||||
//! most importantly `code == "context_length_exceeded"` triggers
|
||||
//! auto-compaction, and a `429` with `Retry-After` makes them back off and
|
||||
//! retry rather than surfacing an opaque failure. A flat `{"error":"..."}`
|
||||
//! string is invisible to that logic.
|
||||
//!
|
||||
//! This module is the single source of truth for that envelope. It is
|
||||
//! deliberately **axum-agnostic** — cortex-core is a pure types crate — so
|
||||
//! it carries the response as data (`status`, `body()`, `retry_after_secs`)
|
||||
//! and each HTTP crate (cortex-gateway, neuron) owns a tiny adapter that
|
||||
//! turns an [`OpenAiError`] into its framework's response type, setting the
|
||||
//! `Retry-After` header when present.
|
||||
//!
|
||||
//! Retryable conditions **must** carry `Retry-After` (per #63). The named
|
||||
//! constructors below encode that: [`OpenAiError::rate_limit_exceeded`] and
|
||||
//! [`OpenAiError::service_unavailable`] take a retry hint;
|
||||
//! [`OpenAiError::insufficient_quota`] (hard balance, no reset) and
|
||||
//! [`OpenAiError::context_length_exceeded`] / [`OpenAiError::invalid_api_key`]
|
||||
//! (permanent) do not. `402 Payment Required` is banned by the contract — use
|
||||
//! `429 insufficient_quota` for hard budget exhaustion.
|
||||
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
/// A rejection rendered in the OpenAI error envelope.
|
||||
///
|
||||
/// Build with [`OpenAiError::new`] (or a named constructor), refine with the
|
||||
/// `with_*` builders, then hand to the consuming crate's adapter to turn into
|
||||
/// an HTTP response.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct OpenAiError {
|
||||
/// HTTP status code (e.g. `401`, `429`, `503`).
|
||||
pub status: u16,
|
||||
/// Broad OpenAI category — `"invalid_request_error"`, `"api_error"`,
|
||||
/// `"rate_limit_error"`, …
|
||||
pub error_type: String,
|
||||
/// Specific machine-readable code clients key on (`"invalid_api_key"`,
|
||||
/// `"rate_limit_exceeded"`, `"context_length_exceeded"`, …). `None`
|
||||
/// renders as JSON `null`.
|
||||
pub code: Option<String>,
|
||||
/// Human-readable, actionable message.
|
||||
pub message: String,
|
||||
/// OpenAI's `param` field — the offending request parameter, if any.
|
||||
pub param: Option<String>,
|
||||
/// Seconds to advertise in the `Retry-After` header. Set only on
|
||||
/// retryable conditions; `None` means no header.
|
||||
pub retry_after_secs: Option<u64>,
|
||||
/// Diagnostic fields merged *inside* the `error` object (e.g.
|
||||
/// `prompt_len`, `max`, `free_mb`) so they don't break the envelope
|
||||
/// shape. Clients ignore unknown keys.
|
||||
pub extra: Map<String, Value>,
|
||||
}
|
||||
|
||||
impl OpenAiError {
|
||||
/// Construct an envelope with an explicit code. For a `null` code use
|
||||
/// [`OpenAiError::without_code`].
|
||||
pub fn new(
|
||||
status: u16,
|
||||
error_type: impl Into<String>,
|
||||
code: impl Into<String>,
|
||||
message: impl Into<String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
status,
|
||||
error_type: error_type.into(),
|
||||
code: Some(code.into()),
|
||||
message: message.into(),
|
||||
param: None,
|
||||
retry_after_secs: None,
|
||||
extra: Map::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Construct an envelope whose `code` is `null` (e.g. an unclassified
|
||||
/// internal error).
|
||||
pub fn without_code(
|
||||
status: u16,
|
||||
error_type: impl Into<String>,
|
||||
message: impl Into<String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
status,
|
||||
error_type: error_type.into(),
|
||||
code: None,
|
||||
message: message.into(),
|
||||
param: None,
|
||||
retry_after_secs: None,
|
||||
extra: Map::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Advertise a `Retry-After` (seconds). Use on retryable rejections.
|
||||
pub fn with_retry_after(mut self, secs: u64) -> Self {
|
||||
self.retry_after_secs = Some(secs);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the OpenAI `param` field.
|
||||
pub fn with_param(mut self, param: impl Into<String>) -> Self {
|
||||
self.param = Some(param.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Merge one diagnostic field into the error object.
|
||||
pub fn with_extra(mut self, key: impl Into<String>, value: Value) -> Self {
|
||||
self.extra.insert(key.into(), value);
|
||||
self
|
||||
}
|
||||
|
||||
/// Merge a bag of diagnostic fields into the error object.
|
||||
pub fn with_extras(mut self, extras: Map<String, Value>) -> Self {
|
||||
for (k, v) in extras {
|
||||
self.extra.insert(k, v);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Render the `{ "error": { … } }` body. Field order is irrelevant to
|
||||
/// clients (they parse JSON); the standard keys come first, then any
|
||||
/// diagnostic extras.
|
||||
pub fn body(&self) -> Value {
|
||||
let mut error = Map::new();
|
||||
error.insert("message".into(), Value::String(self.message.clone()));
|
||||
error.insert("type".into(), Value::String(self.error_type.clone()));
|
||||
error.insert(
|
||||
"code".into(),
|
||||
self.code.clone().map(Value::String).unwrap_or(Value::Null),
|
||||
);
|
||||
error.insert(
|
||||
"param".into(),
|
||||
self.param.clone().map(Value::String).unwrap_or(Value::Null),
|
||||
);
|
||||
for (k, v) in &self.extra {
|
||||
error.insert(k.clone(), v.clone());
|
||||
}
|
||||
json!({ "error": Value::Object(error) })
|
||||
}
|
||||
|
||||
// ── Named constructors for the #63 standard codes ──────────────────
|
||||
|
||||
/// `401 invalid_api_key` — missing/invalid bearer token (#49). Permanent.
|
||||
pub fn invalid_api_key(message: impl Into<String>) -> Self {
|
||||
Self::new(401, "invalid_request_error", "invalid_api_key", message)
|
||||
}
|
||||
|
||||
/// `429 rate_limit_exceeded` + `Retry-After` — transient overload,
|
||||
/// fair-share/in-flight cap, admission rejection, or a rolling budget
|
||||
/// window that resets (#52/#53/#54/#55). Clients back off and retry.
|
||||
pub fn rate_limit_exceeded(message: impl Into<String>, retry_after_secs: u64) -> Self {
|
||||
Self::new(429, "rate_limit_error", "rate_limit_exceeded", message)
|
||||
.with_retry_after(retry_after_secs)
|
||||
}
|
||||
|
||||
/// `429 insufficient_quota` — hard balance exhausted, no reset (#52).
|
||||
/// No `Retry-After`; the client surfaces and stops. (Never `402`.)
|
||||
pub fn insufficient_quota(message: impl Into<String>) -> Self {
|
||||
Self::new(429, "insufficient_quota", "insufficient_quota", message)
|
||||
}
|
||||
|
||||
/// `400 context_length_exceeded` — prompt exceeds the model's context
|
||||
/// window (#56/#60). Permanent for this request; opencode auto-compacts.
|
||||
pub fn context_length_exceeded(message: impl Into<String>) -> Self {
|
||||
Self::new(
|
||||
400,
|
||||
"invalid_request_error",
|
||||
"context_length_exceeded",
|
||||
message,
|
||||
)
|
||||
}
|
||||
|
||||
/// `503 service_unavailable` + optional `Retry-After` — transient
|
||||
/// backend unavailability (no healthy nodes, recovery, fail-closed
|
||||
/// upstream). Retryable when a hint is given.
|
||||
pub fn service_unavailable(message: impl Into<String>, retry_after_secs: Option<u64>) -> Self {
|
||||
let mut err = Self::new(503, "api_error", "service_unavailable", message);
|
||||
err.retry_after_secs = retry_after_secs;
|
||||
err
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn body_has_standard_envelope_shape() {
|
||||
let env = OpenAiError::new(429, "rate_limit_error", "rate_limit_exceeded", "slow down");
|
||||
let body = env.body();
|
||||
let error = body.get("error").and_then(Value::as_object).unwrap();
|
||||
assert_eq!(error["message"], "slow down");
|
||||
assert_eq!(error["type"], "rate_limit_error");
|
||||
assert_eq!(error["code"], "rate_limit_exceeded");
|
||||
assert_eq!(error["param"], Value::Null);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn without_code_renders_null_code() {
|
||||
let env = OpenAiError::without_code(500, "api_error", "kaboom");
|
||||
assert_eq!(env.body()["error"]["code"], Value::Null);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extras_ride_inside_the_error_object() {
|
||||
let env = OpenAiError::context_length_exceeded("too long")
|
||||
.with_extra("prompt_len", json!(60_000))
|
||||
.with_extra("max", json!(49_152));
|
||||
let error = &env.body()["error"];
|
||||
assert_eq!(error["prompt_len"], 60_000);
|
||||
assert_eq!(error["max"], 49_152);
|
||||
assert_eq!(error["code"], "context_length_exceeded");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rolling_window_rejection_carries_retry_after() {
|
||||
let env = OpenAiError::rate_limit_exceeded("budget window", 30);
|
||||
assert_eq!(env.status, 429);
|
||||
assert_eq!(env.retry_after_secs, Some(30));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hard_balance_rejection_has_no_retry_after() {
|
||||
let env = OpenAiError::insufficient_quota("out of credit");
|
||||
assert_eq!(env.status, 429);
|
||||
assert_eq!(env.code.as_deref(), Some("insufficient_quota"));
|
||||
assert_eq!(env.retry_after_secs, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn permanent_rejections_have_no_retry_after() {
|
||||
assert_eq!(OpenAiError::invalid_api_key("nope").retry_after_secs, None);
|
||||
assert_eq!(
|
||||
OpenAiError::context_length_exceeded("too long").retry_after_secs,
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn service_unavailable_retry_after_is_optional() {
|
||||
assert_eq!(
|
||||
OpenAiError::service_unavailable("recovering", Some(5)).retry_after_secs,
|
||||
Some(5)
|
||||
);
|
||||
assert_eq!(
|
||||
OpenAiError::service_unavailable("gone", None).retry_after_secs,
|
||||
None
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -9,13 +9,13 @@ use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Configuration for a harness instance on a neuron.
|
||||
///
|
||||
/// All current harnesses are in-process (candle); per-harness tuning
|
||||
/// (cache paths, device policies, etc.) lives in dedicated config
|
||||
/// blocks rather than on this struct.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct HarnessConfig {
|
||||
pub name: String,
|
||||
/// Base URL of the harness (e.g. "http://localhost:8080" for mistral.rs).
|
||||
pub endpoint: Option<String>,
|
||||
/// Systemd unit name, if the harness is managed via systemd.
|
||||
pub systemd_unit: Option<String>,
|
||||
}
|
||||
|
||||
/// Health status of a harness process.
|
||||
@@ -36,60 +36,6 @@ pub struct ModelSpec {
|
||||
pub devices: Option<Vec<u32>>,
|
||||
}
|
||||
|
||||
/// Per-model token budget advertised by the catalogue or neuron.
|
||||
///
|
||||
/// `context` is the hard wall (the served max-seq-len). `input` is the
|
||||
/// compaction trigger — when set, opencode treats it as "usable context =
|
||||
/// input − reserved". When omitted, clients fall back to `context − output`.
|
||||
/// `output` is the maximum number of generation tokens.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ModelLimit {
|
||||
/// Hard wall — served max-seq-len in tokens.
|
||||
pub context: usize,
|
||||
/// Compaction trigger / usable input budget. When absent clients fall
|
||||
/// back to `context − output`.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input: Option<usize>,
|
||||
/// Maximum number of generation tokens.
|
||||
pub output: usize,
|
||||
}
|
||||
|
||||
/// Operator-set pricing, **USD per 1,000,000 tokens, as JSON numbers**
|
||||
/// (`float`) — the models.dev/opencode `cost` convention, which is what
|
||||
/// helexa's primary client reads. NOT per-token, NOT decimal strings (that
|
||||
/// is OpenRouter's `pricing` shape, which helexa deliberately does not emit
|
||||
/// — see #68). A client must not rescale by 10⁶.
|
||||
///
|
||||
/// `cost` is sourced from the operator's `models.toml` catalogue profile and
|
||||
/// surfaced verbatim on `/v1/models`. The *absent* vs *zero* distinction is
|
||||
/// intentional and load-bearing (#68):
|
||||
/// - **`cost` absent** (the whole object omitted) — the model is **not
|
||||
/// priced**: the operator has not declared a rate. Clients should treat
|
||||
/// spend as unknown, not free.
|
||||
/// - **`cost` present with `input`/`output` = `0.0`** — the model is
|
||||
/// **intentionally free** (self-hosted, no charge). opencode renders `$0`.
|
||||
///
|
||||
/// Cache fields are optional — set them only when the backend supports a
|
||||
/// prefix-cache discount tier (relevant once cache-token reporting, #64,
|
||||
/// lands). The advertised rate here must equal the rate metering (#51) and
|
||||
/// reconciliation (#58/#59) bill against; today both read this catalogue
|
||||
/// value.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ModelCost {
|
||||
/// USD per 1M input (prompt) tokens.
|
||||
#[serde(default)]
|
||||
pub input: f64,
|
||||
/// USD per 1M output (completion) tokens.
|
||||
#[serde(default)]
|
||||
pub output: f64,
|
||||
/// USD per 1M cache-hit tokens (optional).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_read: Option<f64>,
|
||||
/// USD per 1M cache-write tokens (optional).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_write: Option<f64>,
|
||||
}
|
||||
|
||||
/// A model as reported by a harness.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ModelInfo {
|
||||
@@ -98,86 +44,19 @@ pub struct ModelInfo {
|
||||
pub status: String,
|
||||
pub devices: Vec<u32>,
|
||||
pub vram_used_mb: Option<u64>,
|
||||
/// Modalities this loaded model supports. Today: `["text"]` for
|
||||
/// text-only checkpoints, `["text", "vision"]` for vision-capable
|
||||
/// ones (Stage B7). Clients like litellm / agent0 can gate
|
||||
/// `image_url` submission on the advertised set.
|
||||
///
|
||||
/// Optional in the wire format so older clients that don't read
|
||||
/// it stay compatible. Default-empty for absent/older data, which
|
||||
/// callers can interpret as "text".
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub capabilities: Vec<String>,
|
||||
|
||||
// ── Enrichment (issue #62) ────────────────────────────────
|
||||
/// Token budget advertised by the catalogue or discovered at load time.
|
||||
/// `None` when neither the catalogue nor the loaded model can provide it.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub limit: Option<ModelLimit>,
|
||||
/// Operator-set pricing — see [`ModelCost`] for units and the
|
||||
/// absent (not priced) vs `0.0` (intentionally free) distinction.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cost: Option<ModelCost>,
|
||||
/// `true` when the model's tokenizer contains recognised tool-call
|
||||
/// marker tokens (`<tool_call>` / `<\/tool_call>` convention).
|
||||
#[serde(default)]
|
||||
pub tool_call: bool,
|
||||
/// `true` when the model's tokenizer contains recognised reasoning
|
||||
/// marker tokens (`<think>` / `<\/think>` or similar).
|
||||
#[serde(default)]
|
||||
pub reasoning: bool,
|
||||
|
||||
/// Whether this node can actually serve a request for this model
|
||||
/// right now (#245).
|
||||
///
|
||||
/// `status == "loaded"` only says the weights are resident. A node
|
||||
/// can hold a model and still reject every request — most commonly
|
||||
/// because something outside neuron has eaten the device's free VRAM
|
||||
/// and the prefill floor check will fail before any device work.
|
||||
/// That happened in production: a leaked desktop compositor took a
|
||||
/// whole tier down while the node answered every poll correctly and
|
||||
/// the fleet reported itself fully healthy.
|
||||
///
|
||||
/// `None` means the node did not say — an older neuron, or a state
|
||||
/// it cannot evaluate. Callers must treat that as "assume servable",
|
||||
/// never as "unservable", or a version skew would evict the fleet
|
||||
/// from its own routing table.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub servable: Option<ModelServability>,
|
||||
}
|
||||
|
||||
/// Why a loaded model can or cannot be served right now (#245).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ModelServability {
|
||||
pub ok: bool,
|
||||
/// Machine-readable cause when `ok == false` — e.g.
|
||||
/// `"insufficient_vram"`, matching the error the request would get.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub reason: Option<String>,
|
||||
/// Human-facing detail, for an operator reading a health page.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub detail: Option<String>,
|
||||
}
|
||||
|
||||
/// What an inference harness must do, from neuron's perspective.
|
||||
///
|
||||
/// All current harnesses are in-process — they share neuron's address
|
||||
/// space and lifecycle. `start`/`stop` therefore default to no-ops; a
|
||||
/// future process-supervising harness would override them.
|
||||
#[async_trait]
|
||||
pub trait Harness: Send + Sync {
|
||||
/// Human-readable name (e.g. "candle").
|
||||
/// Human-readable name (e.g. "mistralrs", "llamacpp", "comfyui").
|
||||
fn name(&self) -> &str;
|
||||
|
||||
/// Start the harness. Default no-op for in-process harnesses.
|
||||
async fn start(&self, _config: &HarnessConfig) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
/// Start the harness process if it is not already running.
|
||||
async fn start(&self, config: &HarnessConfig) -> Result<()>;
|
||||
|
||||
/// Stop the harness. Default no-op for in-process harnesses.
|
||||
async fn stop(&self) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
/// Stop the harness process gracefully.
|
||||
async fn stop(&self) -> Result<()>;
|
||||
|
||||
/// Health check. Returns the harness process status.
|
||||
async fn health(&self) -> HarnessHealth;
|
||||
|
||||
@@ -1,161 +0,0 @@
|
||||
//! OpenAI-compatible images API envelope (#200).
|
||||
//!
|
||||
//! Shared between neuron (which serves `/v1/images/generations`) and
|
||||
//! cortex (which proxies it). The shape follows OpenAI's images API —
|
||||
//! `model`/`prompt`/`n`/`size`/`response_format` — with helexa
|
||||
//! extensions (`seed`, `negative_prompt`, `guidance_scale`,
|
||||
//! `num_steps`) as sibling fields, mirroring how the chat surface
|
||||
//! carries `helexa_timing` inside `usage`.
|
||||
//!
|
||||
//! v1 constraints, enforced at the neuron: `n` must be 1,
|
||||
//! `response_format` must be `b64_json` (neuron has no object store to
|
||||
//! host `url` responses), output is always PNG.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// `POST /v1/images/generations` request body.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ImagesGenerationRequest {
|
||||
pub model: String,
|
||||
pub prompt: String,
|
||||
/// Number of images. v1 serves exactly 1; larger values are
|
||||
/// rejected with a clear error rather than silently truncated.
|
||||
#[serde(default = "default_n")]
|
||||
pub n: u32,
|
||||
/// `"WIDTHxHEIGHT"`, e.g. `"1024x1024"`. Defaults to 1024².
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub size: Option<String>,
|
||||
/// Only `"b64_json"` is served (the default). `"url"` requires an
|
||||
/// object store the fleet doesn't run.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub response_format: Option<String>,
|
||||
/// Only `"png"` is served (the default).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_format: Option<String>,
|
||||
|
||||
// ── helexa extensions ──────────────────────────────────────
|
||||
/// Fixed RNG seed for reproducible generations.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub seed: Option<u64>,
|
||||
/// Negative prompt; enables classifier-free guidance, which doubles
|
||||
/// per-step cost (and therefore the metered units, #202).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub negative_prompt: Option<String>,
|
||||
/// CFG scale, meaningful only with `negative_prompt`.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub guidance_scale: Option<f64>,
|
||||
/// Denoise steps. Defaults to the model profile's step count
|
||||
/// (9 for Z-Image-Turbo).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub num_steps: Option<usize>,
|
||||
}
|
||||
|
||||
fn default_n() -> u32 {
|
||||
1
|
||||
}
|
||||
|
||||
impl ImagesGenerationRequest {
|
||||
/// Parse `size` into `(width, height)`; defaults to 1024².
|
||||
/// Dimension *validation* (alignment, ceiling) stays server-side —
|
||||
/// this only parses the syntax.
|
||||
pub fn parse_size(&self) -> Result<(usize, usize), String> {
|
||||
let raw = self.size.as_deref().unwrap_or("1024x1024");
|
||||
let (w, h) = raw
|
||||
.split_once(['x', 'X'])
|
||||
.ok_or_else(|| format!("size '{raw}' is not of the form WIDTHxHEIGHT"))?;
|
||||
let width: usize = w
|
||||
.trim()
|
||||
.parse()
|
||||
.map_err(|_| format!("size '{raw}': width is not a number"))?;
|
||||
let height: usize = h
|
||||
.trim()
|
||||
.parse()
|
||||
.map_err(|_| format!("size '{raw}': height is not a number"))?;
|
||||
Ok((width, height))
|
||||
}
|
||||
}
|
||||
|
||||
/// `POST /v1/images/generations` response body.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ImagesGenerationResponse {
|
||||
/// Unix seconds.
|
||||
pub created: u64,
|
||||
pub data: Vec<ImageData>,
|
||||
/// Metering + timing, in the same spirit as chat's
|
||||
/// `usage.helexa_timing`.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub usage: Option<ImagesUsage>,
|
||||
}
|
||||
|
||||
/// One generated image.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ImageData {
|
||||
/// Base64-encoded PNG bytes.
|
||||
pub b64_json: String,
|
||||
}
|
||||
|
||||
/// Usage block for the images surface. `helexa_image_units` is the
|
||||
/// metered work in megapixel-steps (#202): `w × h × steps / 1e6`,
|
||||
/// doubled under CFG. cortex reads it for budget settlement; clients
|
||||
/// can read it to anticipate spend.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ImagesUsage {
|
||||
pub helexa_image_units: f64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub helexa_timing: Option<ImageTiming>,
|
||||
}
|
||||
|
||||
/// Phase timing for one generation, mirrored from the neuron's
|
||||
/// worker-side measurement.
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
|
||||
pub struct ImageTiming {
|
||||
pub encode_ms: u64,
|
||||
pub denoise_ms: u64,
|
||||
pub decode_ms: u64,
|
||||
pub steps: usize,
|
||||
/// True when classifier-free guidance ran (two forwards per step).
|
||||
pub cfg: bool,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn req(size: Option<&str>) -> ImagesGenerationRequest {
|
||||
ImagesGenerationRequest {
|
||||
model: "m".into(),
|
||||
prompt: "p".into(),
|
||||
n: 1,
|
||||
size: size.map(String::from),
|
||||
response_format: None,
|
||||
output_format: None,
|
||||
seed: None,
|
||||
negative_prompt: None,
|
||||
guidance_scale: None,
|
||||
num_steps: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_size_default_and_explicit() {
|
||||
assert_eq!(req(None).parse_size().unwrap(), (1024, 1024));
|
||||
assert_eq!(req(Some("512x768")).parse_size().unwrap(), (512, 768));
|
||||
assert_eq!(req(Some("2048X1024")).parse_size().unwrap(), (2048, 1024));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_size_rejects_garbage() {
|
||||
assert!(req(Some("1024")).parse_size().is_err());
|
||||
assert!(req(Some("axb")).parse_size().is_err());
|
||||
assert!(req(Some("1024x")).parse_size().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_deserializes_with_defaults() {
|
||||
let r: ImagesGenerationRequest =
|
||||
serde_json::from_str(r#"{"model": "m", "prompt": "a cat"}"#).unwrap();
|
||||
assert_eq!(r.n, 1);
|
||||
assert!(r.size.is_none());
|
||||
assert!(r.seed.is_none());
|
||||
}
|
||||
}
|
||||
@@ -1,15 +1,9 @@
|
||||
pub mod anthropic;
|
||||
pub mod build_info;
|
||||
pub mod catalogue;
|
||||
pub mod config;
|
||||
pub mod discovery;
|
||||
pub mod entitlements;
|
||||
pub mod error_envelope;
|
||||
pub mod harness;
|
||||
pub mod images;
|
||||
pub mod metrics;
|
||||
pub mod node;
|
||||
pub mod openai;
|
||||
pub mod responses;
|
||||
pub mod source;
|
||||
pub mod translate;
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
use crate::discovery::{ActivationStatus, DiscoveryResponse, ModelLoad};
|
||||
use crate::harness::{ModelCost, ModelLimit, ModelServability};
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
@@ -15,35 +13,6 @@ pub struct NodeState {
|
||||
/// Number of load/unload cycles since last process restart.
|
||||
pub lifecycle_cycles: u32,
|
||||
pub last_poll: Option<DateTime<Utc>>,
|
||||
/// Result of the most recent successful `GET /discovery` against
|
||||
/// this neuron. Cached forever once obtained — device topology is
|
||||
/// invariant for a given neuron process. `None` until the first
|
||||
/// successful poll. Used by the router and `/v1/models` to do
|
||||
/// catalogue × topology feasibility checks.
|
||||
pub discovery: Option<DiscoveryResponse>,
|
||||
/// Last-seen pre-warm progress from this neuron's `/health`
|
||||
/// endpoint. `None` until the first /health poll succeeds. The
|
||||
/// `/v1/models` handler reads `in_progress` + `pending` from here
|
||||
/// to synthesize `Loading` locations so clients see a catalogued
|
||||
/// model that's mid-prewarm as "loading", not "missing".
|
||||
pub activation: Option<ActivationStatus>,
|
||||
/// Last-seen per-model admission load from this neuron's `/health`
|
||||
/// (#53), keyed by model id. The router (#55) reads it to pick the
|
||||
/// least-busy replica when a model is loaded on more than one neuron.
|
||||
/// Empty until the first /health poll reports load.
|
||||
pub model_load: HashMap<String, ModelLoad>,
|
||||
/// Consecutive failed `/models` polls. The poller marks a node
|
||||
/// unhealthy only once this crosses a threshold, so a single transient
|
||||
/// miss (e.g. a neuron momentarily slow to answer while busy) doesn't
|
||||
/// yank the node — and all its models — out of routing. Reset to 0 on
|
||||
/// any successful poll.
|
||||
pub consecutive_poll_failures: u32,
|
||||
/// Last-seen per-device VRAM/utilisation readings from `/health`
|
||||
/// (#203). The router's cold-load placement reads free VRAM here so
|
||||
/// a model lands on a node that can actually hold it *now*, not
|
||||
/// just one whose total topology could. Empty until the first
|
||||
/// /health poll.
|
||||
pub device_health: Vec<crate::discovery::DeviceHealth>,
|
||||
}
|
||||
|
||||
/// A model registered on a node, with its runtime status.
|
||||
@@ -55,158 +24,25 @@ pub struct ModelEntry {
|
||||
pub last_accessed: Option<DateTime<Utc>>,
|
||||
/// Estimated VRAM usage in MB when loaded.
|
||||
pub vram_estimate_mb: Option<u64>,
|
||||
/// Modalities the loaded model advertises (e.g. `["text", "vision"]`),
|
||||
/// copied verbatim from the neuron's `ModelInfo.capabilities` at poll
|
||||
/// time. Empty when the neuron reports none. `#[serde(default)]` keeps
|
||||
/// older persisted/serialised entries deserialisable.
|
||||
#[serde(default)]
|
||||
pub capabilities: Vec<String>,
|
||||
/// Runtime-detected capability flags from the neuron's `/models`
|
||||
/// response (`ModelInfo`). `false` when the neuron predates these
|
||||
/// fields or hasn't reported them yet.
|
||||
#[serde(default)]
|
||||
pub tool_call: bool,
|
||||
#[serde(default)]
|
||||
pub reasoning: bool,
|
||||
/// Self-derived token budget the neuron computed for this loaded
|
||||
/// model (#67), copied from `ModelInfo.limit` at poll time. `None`
|
||||
/// when the neuron doesn't compute one (arch without a context
|
||||
/// profile, or derivation disabled). This is the authoritative
|
||||
/// source the gateway advertises — operator-declared catalogue
|
||||
/// limits are no longer consulted.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub limit: Option<ModelLimit>,
|
||||
/// Whether this node could actually serve the model right now
|
||||
/// (#245), from `ModelInfo.servable` at poll time.
|
||||
///
|
||||
/// `None` means the neuron expressed no opinion — an older build, a
|
||||
/// CPU load, an image model, or a cache not yet seeded. Absent is
|
||||
/// **not** unservable: see [`ModelEntry::is_servable`].
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub servable: Option<ModelServability>,
|
||||
}
|
||||
|
||||
impl ModelEntry {
|
||||
/// Whether the router should treat this location as a candidate.
|
||||
///
|
||||
/// Absent evidence means yes. A neuron that predates the field, or
|
||||
/// one that cannot evaluate its own state, must not be read as
|
||||
/// broken — a version skew would otherwise empty the routing table
|
||||
/// and take the fleet down far more comprehensively than the fault
|
||||
/// this guards against.
|
||||
pub fn is_servable(&self) -> bool {
|
||||
self.servable.as_ref().is_none_or(|s| s.ok)
|
||||
}
|
||||
}
|
||||
|
||||
/// Model lifecycle status.
|
||||
///
|
||||
/// `Loading` is a gateway-side synthetic status: neurons never emit it
|
||||
/// on `/models` (that endpoint only knows about already-loaded handles).
|
||||
/// The gateway populates it from a neuron's `/health` activation
|
||||
/// snapshot so the unified `/v1/models` can distinguish "model is
|
||||
/// catalogued but no one has it" from "model is materialising on
|
||||
/// neuron N right now". Other status values are reported verbatim by
|
||||
/// neurons.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum ModelStatus {
|
||||
Loaded,
|
||||
Unloaded,
|
||||
Reloading,
|
||||
Loading,
|
||||
/// Reported by neuron while a poisoned model auto-recovers via
|
||||
/// unload→reload (#17/#20). Temporarily unservable but NOT
|
||||
/// evicted: the gateway holds the route, answers with a transient
|
||||
/// retry error instead of 404, and must not race a second
|
||||
/// placement elsewhere.
|
||||
Recovering,
|
||||
}
|
||||
|
||||
/// Unified model entry as exposed by the gateway's `/v1/models` endpoint.
|
||||
///
|
||||
/// The first four fields (`id`, `object`, `created`, `owned_by`) match
|
||||
/// OpenAI's `/v1/models` shape verbatim, so existing OpenAI-aware
|
||||
/// tooling deserialises this without custom code. The remaining fields
|
||||
/// are helexa-specific extensions — OpenAI clients ignore unknown
|
||||
/// fields and other consumers can read them for placement / debugging.
|
||||
/// Includes which node(s) host this model and their status.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct CortexModelEntry {
|
||||
pub id: String,
|
||||
/// Always `"model"` per OpenAI's contract.
|
||||
pub object: String,
|
||||
/// Unix-second timestamp; cortex stamps this at response time.
|
||||
pub created: u64,
|
||||
/// OpenAI's "publisher" field — `"helexa"` for everything we serve.
|
||||
pub owned_by: String,
|
||||
/// True if any neuron currently has this model loaded. False for
|
||||
/// catalogue entries that are feasible but not yet loaded.
|
||||
pub loaded: bool,
|
||||
/// Neurons whose discovered topology can satisfy this model's
|
||||
/// catalogue placement constraints. Empty for models that are
|
||||
/// loaded somewhere but not present in the catalogue (cortex has
|
||||
/// no feasibility opinion on those).
|
||||
pub feasible_on: Vec<String>,
|
||||
/// Where this model is actually loaded right now. Subset of (or
|
||||
/// disjoint from) `feasible_on` depending on whether the catalogue
|
||||
/// covers this model.
|
||||
/// Which nodes have this model (and their status).
|
||||
pub locations: Vec<ModelLocation>,
|
||||
/// Union of the modalities advertised by every neuron that has this
|
||||
/// model loaded (e.g. `["text", "vision"]`). Empty for catalogue-only
|
||||
/// entries with no loaded location — filled from catalogue profile
|
||||
/// capabilities when available, then unioned with runtime-detected
|
||||
/// values from loaded neurons.
|
||||
#[serde(default)]
|
||||
pub capabilities: Vec<String>,
|
||||
// ── Enrichment (issue #62) ────────────────────────────────
|
||||
/// Per-model token budget from the catalogue profile or discovered
|
||||
/// at load time. `None` when neither source provides it.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub limit: Option<ModelLimit>,
|
||||
/// Operator-set pricing from the catalogue profile — see
|
||||
/// [`cortex_core::harness::ModelCost`] for units (USD per 1M tokens) and
|
||||
/// the absent (not priced) vs `0.0` (intentionally free) distinction.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cost: Option<ModelCost>,
|
||||
/// `true` when any neuron reports this model supports tool calls.
|
||||
#[serde(default)]
|
||||
pub tool_call: bool,
|
||||
/// `true` when any neuron reports this model supports reasoning tokens.
|
||||
#[serde(default)]
|
||||
pub reasoning: bool,
|
||||
// ── Flat ecosystem context-window fields (issue #78) ──────
|
||||
// Duplicates of `limit` under the flat, vLLM-convention key names
|
||||
// (`max_model_len` et al.) that OpenAI-ecosystem clients (Hermes
|
||||
// Agent, vLLM tooling) probe for — they cannot see `limit.context`.
|
||||
// Additive: `limit` stays the opencode-oriented source of truth.
|
||||
// Derived, never set directly — call [`sync_flat_limit`] after the
|
||||
// final `limit` value is known. Omitted (not `0`) when the window
|
||||
// is unknown; absent-vs-zero is load-bearing, as with `cost`.
|
||||
//
|
||||
// [`sync_flat_limit`]: CortexModelEntry::sync_flat_limit
|
||||
/// Served max-seq-len in tokens — mirrors `limit.context`.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_model_len: Option<usize>,
|
||||
/// Usable input budget — mirrors `limit.input` when present.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_input_tokens: Option<usize>,
|
||||
/// Maximum generation tokens — mirrors `limit.output`.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_output_tokens: Option<usize>,
|
||||
}
|
||||
|
||||
impl CortexModelEntry {
|
||||
/// Re-derive the flat ecosystem fields (#78) from `limit`.
|
||||
///
|
||||
/// Must run after the final `limit` is known (post merge/tightening),
|
||||
/// immediately before serialization. Fully overwrites: a `None` limit
|
||||
/// clears the flat fields, so stale values can't survive a merge that
|
||||
/// dropped the limit.
|
||||
pub fn sync_flat_limit(&mut self) {
|
||||
self.max_model_len = self.limit.as_ref().map(|l| l.context);
|
||||
self.max_input_tokens = self.limit.as_ref().and_then(|l| l.input);
|
||||
self.max_output_tokens = self.limit.as_ref().map(|l| l.output);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
//! These are a subset sufficient for chat completions (streaming + non-streaming).
|
||||
//! Fields not relevant to proxying are captured as `serde_json::Value` via
|
||||
//! `#[serde(flatten)]` so we forward them without needing to enumerate every
|
||||
//! extension field a backend might support.
|
||||
//! extension field mistral.rs supports.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
@@ -18,66 +18,29 @@ pub struct ChatCompletionRequest {
|
||||
pub temperature: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub top_p: Option<f64>,
|
||||
/// The deprecated spelling of the output cap. Still what most
|
||||
/// clients send; see [`ChatCompletionRequest::effective_max_tokens`].
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub max_tokens: Option<u64>,
|
||||
/// OpenAI's current spelling of the output cap, which deprecated
|
||||
/// `max_tokens`. Newer SDKs and clients send only this one — it has
|
||||
/// to be a named field rather than falling into `extra`, or the cap
|
||||
/// is silently ignored and generation runs to the server default.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_completion_tokens: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stream: Option<bool>,
|
||||
/// All other fields (tools, response_format, backend extensions, etc.)
|
||||
/// All other fields (tools, response_format, mistral.rs extensions, etc.)
|
||||
#[serde(flatten)]
|
||||
pub extra: Value,
|
||||
}
|
||||
|
||||
impl ChatCompletionRequest {
|
||||
/// The output cap the caller asked for, under either spelling.
|
||||
///
|
||||
/// `max_completion_tokens` wins when both are present — it is the
|
||||
/// current OpenAI field, and cortex's metering already reserves
|
||||
/// against it (`metering::requested_max_output`), so preferring it
|
||||
/// here keeps what we bill and what we generate in agreement.
|
||||
pub fn effective_max_tokens(&self) -> Option<u64> {
|
||||
self.max_completion_tokens.or(self.max_tokens)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ChatMessage {
|
||||
pub role: String,
|
||||
/// Absent on an assistant turn that carries only `tool_calls` —
|
||||
/// see [`MessageContent::Null`]. `#[serde(default)]` so a message
|
||||
/// with no `content` key at all deserializes rather than being
|
||||
/// rejected.
|
||||
#[serde(default)]
|
||||
pub content: MessageContent,
|
||||
#[serde(flatten)]
|
||||
pub extra: Value,
|
||||
}
|
||||
|
||||
/// Content can be a simple string, an array of content parts (for
|
||||
/// vision), or absent.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
/// Content can be a simple string or an array of content parts (for vision).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum MessageContent {
|
||||
Text(String),
|
||||
Parts(Vec<Value>),
|
||||
/// `"content": null`, or no `content` key at all.
|
||||
///
|
||||
/// This is the OpenAI-canonical shape for an assistant turn whose
|
||||
/// only payload is `tool_calls`, and agentic clients replay the
|
||||
/// assistant turn verbatim on the follow-up request that carries
|
||||
/// the tool result — so any client doing OpenAI-native tool
|
||||
/// calling sends it on its second turn. HF chat templates model it
|
||||
/// the same way (`content is none` renders as empty), so it maps
|
||||
/// straight through to the prompt.
|
||||
#[default]
|
||||
Null,
|
||||
}
|
||||
|
||||
// ── Chat completion response (non-streaming) ─────────────────────────
|
||||
@@ -108,18 +71,10 @@ pub struct ChatCompletionChoice {
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ChatCompletionChunk {
|
||||
#[serde(default)]
|
||||
pub id: String,
|
||||
#[serde(default)]
|
||||
pub object: String,
|
||||
#[serde(default)]
|
||||
pub created: u64,
|
||||
// Lenient deserialization throughout: the gateway parses chunks
|
||||
// from arbitrary OpenAI-compatible upstreams, and some engines
|
||||
// omit fields on special frames (e.g. usage-only final chunks).
|
||||
#[serde(default)]
|
||||
pub model: String,
|
||||
#[serde(default)]
|
||||
pub choices: Vec<ChunkChoice>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub usage: Option<Usage>,
|
||||
@@ -143,48 +98,6 @@ pub struct Usage {
|
||||
pub prompt_tokens: u64,
|
||||
pub completion_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
/// OpenAI-standard breakdown of `completion_tokens`. Optional and
|
||||
/// additive — clients that don't read it are unaffected. Carries
|
||||
/// `reasoning_tokens` for reasoning models (a sub-count of
|
||||
/// `completion_tokens`, never added into `total_tokens`).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub completion_tokens_details: Option<CompletionTokensDetails>,
|
||||
/// OpenAI-standard breakdown of `prompt_tokens`. Populated once
|
||||
/// prompt caching lands (#11); `None` until then.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub prompt_tokens_details: Option<PromptTokensDetails>,
|
||||
/// helexa extension (non-OpenAI): server-measured prefill/decode
|
||||
/// timing, so the bench harness can compute true prefill vs decode
|
||||
/// tok/s instead of inferring both from client-side SSE arrival
|
||||
/// (#85). Additive and optional — standard OpenAI clients ignore
|
||||
/// it; cortex forwards usage verbatim so it survives proxying.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub helexa_timing: Option<HelexaTiming>,
|
||||
}
|
||||
|
||||
/// helexa extension carried on [`Usage::helexa_timing`]. Mirrors
|
||||
/// neuron's internal `FinishTiming`. All fields are server-measured;
|
||||
/// `prefill_tokens` is the prefill-rate denominator.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct HelexaTiming {
|
||||
pub prefill_ms: u64,
|
||||
pub decode_ms: u64,
|
||||
pub prefill_tokens: u64,
|
||||
}
|
||||
|
||||
/// Sub-counts of `Usage::completion_tokens`.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct CompletionTokensDetails {
|
||||
/// Tokens generated inside the model's reasoning span.
|
||||
pub reasoning_tokens: u64,
|
||||
}
|
||||
|
||||
/// Sub-counts of `Usage::prompt_tokens`.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct PromptTokensDetails {
|
||||
/// Prompt tokens served from cache (cache-read rate). Populated
|
||||
/// once prompt caching lands (#11).
|
||||
pub cached_tokens: u64,
|
||||
}
|
||||
|
||||
// ── Models list response ─────────────────────────────────────────────
|
||||
@@ -207,133 +120,3 @@ pub struct ModelObject {
|
||||
#[serde(flatten)]
|
||||
pub extra: Value,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The follow-up request an OpenAI-native agentic client sends
|
||||
/// after running a tool: it replays its own assistant turn, which
|
||||
/// OpenAI emits with `"content": null` because the payload was
|
||||
/// only `tool_calls`. Rejecting this shape breaks every such
|
||||
/// client on its second turn.
|
||||
#[test]
|
||||
fn assistant_tool_call_turn_with_null_content_deserializes() {
|
||||
let req: ChatCompletionRequest = serde_json::from_str(
|
||||
r#"{
|
||||
"model": "m",
|
||||
"messages": [
|
||||
{"role": "user", "content": "list the files"},
|
||||
{"role": "assistant", "content": null, "tool_calls": [
|
||||
{"id": "call_0", "type": "function",
|
||||
"function": {"name": "bash", "arguments": "{\"command\":\"ls\"}"}}
|
||||
]},
|
||||
{"role": "tool", "content": "a.txt", "tool_call_id": "call_0"}
|
||||
]
|
||||
}"#,
|
||||
)
|
||||
.expect("null assistant content is a valid OpenAI request");
|
||||
|
||||
assert!(matches!(req.messages[1].content, MessageContent::Null));
|
||||
// The tool calls survive in `extra`, where the chat template
|
||||
// reads them from.
|
||||
assert_eq!(
|
||||
req.messages[1].extra["tool_calls"][0]["function"]["name"],
|
||||
"bash"
|
||||
);
|
||||
assert_eq!(req.messages[2].extra["tool_call_id"], "call_0");
|
||||
}
|
||||
|
||||
/// Some clients omit the key entirely rather than sending null.
|
||||
#[test]
|
||||
fn assistant_turn_with_no_content_key_deserializes() {
|
||||
let msg: ChatMessage = serde_json::from_str(
|
||||
r#"{"role": "assistant", "tool_calls": [
|
||||
{"id": "c", "type": "function",
|
||||
"function": {"name": "bash", "arguments": "{}"}}]}"#,
|
||||
)
|
||||
.expect("absent content is a valid OpenAI message");
|
||||
assert!(matches!(msg.content, MessageContent::Null));
|
||||
}
|
||||
|
||||
/// Absent content round-trips back onto the wire as null, so a
|
||||
/// translated or re-serialized request stays OpenAI-shaped.
|
||||
#[test]
|
||||
fn null_content_round_trips() {
|
||||
let msg = ChatMessage {
|
||||
role: "assistant".into(),
|
||||
content: MessageContent::Null,
|
||||
extra: Value::Null,
|
||||
};
|
||||
let v = serde_json::to_value(&msg).expect("serialize");
|
||||
assert_eq!(v["content"], Value::Null);
|
||||
assert!(v.get("content").is_some(), "content key must be present");
|
||||
}
|
||||
|
||||
/// Newer OpenAI SDKs send only `max_completion_tokens`; the cap
|
||||
/// must survive as a named field rather than falling into `extra`,
|
||||
/// where every generation loop would miss it.
|
||||
#[test]
|
||||
fn max_completion_tokens_is_the_effective_cap() {
|
||||
let req: ChatCompletionRequest = serde_json::from_str(
|
||||
r#"{"model": "m", "max_completion_tokens": 240,
|
||||
"messages": [{"role": "user", "content": "hi"}]}"#,
|
||||
)
|
||||
.expect("deserialize");
|
||||
assert_eq!(req.max_completion_tokens, Some(240));
|
||||
assert_eq!(req.max_tokens, None);
|
||||
assert_eq!(req.effective_max_tokens(), Some(240));
|
||||
assert!(
|
||||
req.extra.get("max_completion_tokens").is_none(),
|
||||
"must be a named field, not swept into extra"
|
||||
);
|
||||
}
|
||||
|
||||
/// The legacy spelling keeps working on its own.
|
||||
#[test]
|
||||
fn legacy_max_tokens_still_caps() {
|
||||
let req: ChatCompletionRequest = serde_json::from_str(
|
||||
r#"{"model": "m", "max_tokens": 128,
|
||||
"messages": [{"role": "user", "content": "hi"}]}"#,
|
||||
)
|
||||
.expect("deserialize");
|
||||
assert_eq!(req.effective_max_tokens(), Some(128));
|
||||
}
|
||||
|
||||
/// Clients that send both for compatibility must not be rejected,
|
||||
/// and the current field wins — matching what cortex meters
|
||||
/// against, so billed and generated caps agree.
|
||||
#[test]
|
||||
fn both_spellings_resolve_to_the_current_field() {
|
||||
let req: ChatCompletionRequest = serde_json::from_str(
|
||||
r#"{"model": "m", "max_tokens": 99, "max_completion_tokens": 256,
|
||||
"messages": [{"role": "user", "content": "hi"}]}"#,
|
||||
)
|
||||
.expect("sending both must not be an error");
|
||||
assert_eq!(req.effective_max_tokens(), Some(256));
|
||||
}
|
||||
|
||||
/// Neither spelling present leaves the cap to the server default.
|
||||
#[test]
|
||||
fn absent_cap_is_none() {
|
||||
let req: ChatCompletionRequest = serde_json::from_str(
|
||||
r#"{"model": "m", "messages": [{"role": "user", "content": "hi"}]}"#,
|
||||
)
|
||||
.expect("deserialize");
|
||||
assert_eq!(req.effective_max_tokens(), None);
|
||||
}
|
||||
|
||||
/// The string and array forms keep working unchanged.
|
||||
#[test]
|
||||
fn text_and_parts_content_still_deserialize() {
|
||||
let text: ChatMessage =
|
||||
serde_json::from_str(r#"{"role": "user", "content": "hi"}"#).expect("text");
|
||||
assert!(matches!(text.content, MessageContent::Text(ref t) if t == "hi"));
|
||||
|
||||
let parts: ChatMessage = serde_json::from_str(
|
||||
r#"{"role": "user", "content": [{"type": "text", "text": "hi"}]}"#,
|
||||
)
|
||||
.expect("parts");
|
||||
assert!(matches!(parts.content, MessageContent::Parts(ref p) if p.len() == 1));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,512 +0,0 @@
|
||||
//! OpenAI Responses API (`POST /v1/responses`) envelope types.
|
||||
//!
|
||||
//! This is OpenAI's newer chat surface, distinct from
|
||||
//! `/v1/chat/completions` in three ways that matter for us:
|
||||
//!
|
||||
//! 1. **Input shape**. Instead of a `messages` array, the request
|
||||
//! carries `input` — either a plain string (single user turn)
|
||||
//! or an array of typed items (messages, function calls,
|
||||
//! function-call outputs, reasoning blocks, …).
|
||||
//! 2. **Output shape**. The response carries a single `output`
|
||||
//! array of items, each typed. We always emit one
|
||||
//! `OutputItem::Message` containing the assistant's reply (plus,
|
||||
//! when we get there, separate `function_call` items).
|
||||
//! 3. **Streaming events**. Where chat completions stream
|
||||
//! structurally-identical `chat.completion.chunk` frames over
|
||||
//! `data:` lines, Responses streams *named* events
|
||||
//! (`response.created`, `response.output_text.delta`,
|
||||
//! `response.completed`, …) over `event:` + `data:` SSE pairs.
|
||||
//! The wire projector in `neuron::wire::openai_responses` builds
|
||||
//! these from the same [`crate::openai`]-shaped
|
||||
//! `InferenceEvent` stream the chat projector consumes.
|
||||
//!
|
||||
//! Scope cuts for this first cut:
|
||||
//!
|
||||
//! - **`previous_response_id` is rejected at parse time**. Stateful
|
||||
//! chained conversations need a persistence layer we don't have.
|
||||
//! - **Reasoning items are accepted-and-ignored** (no Qwen3
|
||||
//! `<think>` routing yet). Audio and embedded resources are
|
||||
//! rejected as unsupported.
|
||||
//! - **Tool calls** (function_call / function_call_output) are
|
||||
//! carried as round-trip types but the candle harness doesn't
|
||||
//! emit them yet — wired so the surface is in place for the
|
||||
//! day we add proper tool-call extraction.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
// ── Request ──────────────────────────────────────────────────────────
|
||||
|
||||
/// Body of a `POST /v1/responses` request.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ResponsesRequest {
|
||||
pub model: String,
|
||||
pub input: ResponsesInput,
|
||||
/// System-prompt-style instructions. The Responses API
|
||||
/// separates these from input so a caller doesn't have to
|
||||
/// build a `system` message item by hand.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub instructions: Option<String>,
|
||||
#[serde(default)]
|
||||
pub stream: bool,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_output_tokens: Option<u64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub temperature: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub top_p: Option<f64>,
|
||||
/// Chained-conversation identifier. We don't store responses
|
||||
/// server-side yet; if this is `Some`, the handler returns 400.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub previous_response_id: Option<String>,
|
||||
/// Catch-all for anything we don't model yet (tools, tool_choice,
|
||||
/// reasoning, response_format, …). Lets a client send a
|
||||
/// forward-compatible request without our parser rejecting it.
|
||||
#[serde(flatten)]
|
||||
pub extra: Value,
|
||||
}
|
||||
|
||||
/// `input` is either a single string or an array of items.
|
||||
/// `#[serde(untagged)]` so the wire shape `"input": "hi"` and
|
||||
/// `"input": [{...}]` both deserialize.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum ResponsesInput {
|
||||
Text(String),
|
||||
Items(Vec<ResponsesInputElement>),
|
||||
}
|
||||
|
||||
/// One element of an `input` array.
|
||||
///
|
||||
/// OpenAI's Responses API accepts three shapes here, and real clients
|
||||
/// use all of them — most notably agent-zero (via litellm), which
|
||||
/// sends the bare "easy message" form. We must tolerate every shape,
|
||||
/// because `input` is an `#[serde(untagged)]` array: a single element
|
||||
/// that matches no variant fails the *entire* request with a 422
|
||||
/// (`did not match any variant of untagged enum ResponsesInput`).
|
||||
///
|
||||
/// 1. [`Self::Typed`] — an item carrying an explicit `"type"`
|
||||
/// discriminant (`message`, `function_call`, `function_call_output`,
|
||||
/// `reasoning`).
|
||||
/// 2. [`Self::EasyMessage`] — a bare `{role, content}` with **no**
|
||||
/// `type` field. This is OpenAI's `EasyInputMessage` and what
|
||||
/// litellm emits for every turn. `content` is optional so an
|
||||
/// assistant turn carrying only tool calls (`content: null`) still
|
||||
/// parses.
|
||||
/// 3. [`Self::Other`] — anything else, captured as raw JSON and
|
||||
/// dropped during translation. This is the forward-compat escape
|
||||
/// hatch that mirrors [`ResponsesRequest::extra`] at the item
|
||||
/// level: an unmodeled item type can never again reject the whole
|
||||
/// request.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum ResponsesInputElement {
|
||||
Typed(ResponsesInputItem),
|
||||
EasyMessage {
|
||||
role: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
content: Option<ResponsesMessageContent>,
|
||||
},
|
||||
Other(Value),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum ResponsesInputItem {
|
||||
/// A user / assistant / system turn.
|
||||
Message {
|
||||
role: String,
|
||||
content: ResponsesMessageContent,
|
||||
},
|
||||
/// Assistant emitted a tool call. Round-trip only — neuron
|
||||
/// doesn't synthesise these yet.
|
||||
FunctionCall {
|
||||
call_id: String,
|
||||
name: String,
|
||||
arguments: String,
|
||||
},
|
||||
/// User is feeding a tool result back into the model. `output`
|
||||
/// is a `Value` because OpenAI allows it to be either a plain
|
||||
/// string or an array of content parts; the translator renders
|
||||
/// either form to text rather than losing the tool result.
|
||||
FunctionCallOutput { call_id: String, output: Value },
|
||||
/// Reasoning items emitted by o-series models. Accepted but
|
||||
/// not forwarded to the model — neuron's candle path doesn't
|
||||
/// surface reasoning separately yet.
|
||||
Reasoning {
|
||||
#[serde(default)]
|
||||
content: Vec<Value>,
|
||||
},
|
||||
}
|
||||
|
||||
/// Inside a `Message` item, content is either a plain string or an
|
||||
/// array of typed parts. Mirrors the chat-completions Parts shape.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum ResponsesMessageContent {
|
||||
Text(String),
|
||||
Parts(Vec<ResponsesContentPart>),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum ResponsesContentPart {
|
||||
/// Plain text inside a user / system turn.
|
||||
InputText { text: String },
|
||||
/// An image. `image_url` is either a remote URL or a
|
||||
/// `data:image/png;base64,…` URI; the request translator just
|
||||
/// forwards the string.
|
||||
InputImage {
|
||||
image_url: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
detail: Option<String>,
|
||||
},
|
||||
/// Returned text inside an assistant turn — only relevant when
|
||||
/// the caller is feeding an assistant turn back in to continue
|
||||
/// a conversation manually (no `previous_response_id`).
|
||||
OutputText {
|
||||
text: String,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
annotations: Vec<Value>,
|
||||
},
|
||||
/// Any content-part type we don't model (e.g. `refusal`, audio).
|
||||
/// Captured as a unit so an unknown part can't reject the whole
|
||||
/// request; dropped during translation.
|
||||
#[serde(other)]
|
||||
Unknown,
|
||||
}
|
||||
|
||||
// ── Response (non-streaming) ─────────────────────────────────────────
|
||||
|
||||
/// Body of a `POST /v1/responses` response.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ResponsesResponse {
|
||||
pub id: String,
|
||||
/// Always `"response"`.
|
||||
pub object: String,
|
||||
pub created_at: u64,
|
||||
/// `"completed"`, `"incomplete"`, or — for the initial event of
|
||||
/// a streaming response — `"in_progress"`.
|
||||
pub status: String,
|
||||
pub model: String,
|
||||
pub output: Vec<ResponsesOutputItem>,
|
||||
/// Populated on completion; `None` while streaming.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub usage: Option<ResponsesUsage>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum ResponsesOutputItem {
|
||||
Message {
|
||||
id: String,
|
||||
/// Always `"assistant"` for model output.
|
||||
role: String,
|
||||
/// Output content parts. We always emit a single
|
||||
/// `OutputText` today; multi-part output would land here
|
||||
/// once we have e.g. image generation.
|
||||
content: Vec<ResponsesOutputContent>,
|
||||
/// Item-level status. `"in_progress"` while streaming the
|
||||
/// content parts, `"completed"` when done.
|
||||
#[serde(default = "default_item_status")]
|
||||
status: String,
|
||||
},
|
||||
/// Reserved for the day tool-call extraction lands. The wire
|
||||
/// shape mirrors `ResponsesInputItem::FunctionCall`.
|
||||
FunctionCall {
|
||||
id: String,
|
||||
call_id: String,
|
||||
name: String,
|
||||
arguments: String,
|
||||
#[serde(default = "default_item_status")]
|
||||
status: String,
|
||||
},
|
||||
}
|
||||
|
||||
fn default_item_status() -> String {
|
||||
"completed".into()
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum ResponsesOutputContent {
|
||||
OutputText {
|
||||
text: String,
|
||||
/// Citations / inline annotations. Empty today; reserved
|
||||
/// for the day we wire in web search / file search.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
annotations: Vec<Value>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ResponsesUsage {
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
/// OpenAI-standard breakdown of `output_tokens`. Optional and
|
||||
/// additive. Carries `reasoning_tokens` for reasoning models (a
|
||||
/// sub-count of `output_tokens`, never added into `total_tokens`).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_tokens_details: Option<OutputTokensDetails>,
|
||||
/// OpenAI-standard breakdown of `input_tokens`. Populated once
|
||||
/// prompt caching lands (#11); `None` until then.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_tokens_details: Option<InputTokensDetails>,
|
||||
}
|
||||
|
||||
/// Sub-counts of `ResponsesUsage::output_tokens`.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct OutputTokensDetails {
|
||||
/// Tokens generated inside the model's reasoning span.
|
||||
pub reasoning_tokens: u64,
|
||||
}
|
||||
|
||||
/// Sub-counts of `ResponsesUsage::input_tokens`.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct InputTokensDetails {
|
||||
/// Input tokens served from cache (cache-read rate). Populated
|
||||
/// once prompt caching lands (#11).
|
||||
pub cached_tokens: u64,
|
||||
}
|
||||
|
||||
// ── Streaming event names ────────────────────────────────────────────
|
||||
|
||||
/// Event names the SSE projector emits, hoisted as constants so
|
||||
/// the projector and the wire shape stay in sync without
|
||||
/// string-typos. The strings are dictated by OpenAI's published
|
||||
/// Responses API.
|
||||
pub mod events {
|
||||
pub const CREATED: &str = "response.created";
|
||||
/// Fired between `response.created` and the first output-item
|
||||
/// event. Marks "request validated, model is generating" —
|
||||
/// some clients use it to differentiate the "warming up" state
|
||||
/// from "streaming tokens" in their UI.
|
||||
pub const IN_PROGRESS: &str = "response.in_progress";
|
||||
pub const OUTPUT_ITEM_ADDED: &str = "response.output_item.added";
|
||||
pub const CONTENT_PART_ADDED: &str = "response.content_part.added";
|
||||
pub const OUTPUT_TEXT_DELTA: &str = "response.output_text.delta";
|
||||
pub const OUTPUT_TEXT_DONE: &str = "response.output_text.done";
|
||||
pub const CONTENT_PART_DONE: &str = "response.content_part.done";
|
||||
pub const OUTPUT_ITEM_DONE: &str = "response.output_item.done";
|
||||
pub const FUNCTION_CALL_ARGUMENTS_DELTA: &str = "response.function_call_arguments.delta";
|
||||
pub const FUNCTION_CALL_ARGUMENTS_DONE: &str = "response.function_call_arguments.done";
|
||||
pub const COMPLETED: &str = "response.completed";
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn deserialises_input_string_form() {
|
||||
let raw = r#"{"model": "m", "input": "hello"}"#;
|
||||
let req: ResponsesRequest = serde_json::from_str(raw).unwrap();
|
||||
match req.input {
|
||||
ResponsesInput::Text(s) => assert_eq!(s, "hello"),
|
||||
other => panic!("expected Text, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deserialises_input_items_form() {
|
||||
let raw = r#"{
|
||||
"model": "m",
|
||||
"input": [
|
||||
{"type": "message", "role": "user", "content": "hi"}
|
||||
]
|
||||
}"#;
|
||||
let req: ResponsesRequest = serde_json::from_str(raw).unwrap();
|
||||
match req.input {
|
||||
ResponsesInput::Items(items) => {
|
||||
assert_eq!(items.len(), 1);
|
||||
match &items[0] {
|
||||
ResponsesInputElement::Typed(ResponsesInputItem::Message { role, content }) => {
|
||||
assert_eq!(role, "user");
|
||||
match content {
|
||||
ResponsesMessageContent::Text(t) => assert_eq!(t, "hi"),
|
||||
other => panic!("expected Text content, got {other:?}"),
|
||||
}
|
||||
}
|
||||
other => panic!("expected typed Message item, got {other:?}"),
|
||||
}
|
||||
}
|
||||
other => panic!("expected Items, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deserialises_bare_easy_message_without_type() {
|
||||
// The shape agent-zero (via litellm) actually sends: `input`
|
||||
// items are bare `{role, content}` with NO `type` field. This
|
||||
// is the exact payload that was returning 422.
|
||||
let raw = r#"{
|
||||
"model": "Qwen/Qwen3.6-27B",
|
||||
"store": true,
|
||||
"tools": [{"type": "function", "name": "x", "description": "d", "parameters": {}}],
|
||||
"input": [
|
||||
{"role": "system", "content": "you are helpful"},
|
||||
{"role": "assistant", "content": "{\"tool_name\":\"response\"}"},
|
||||
{"role": "user", "content": "hi"}
|
||||
]
|
||||
}"#;
|
||||
let req: ResponsesRequest = serde_json::from_str(raw).unwrap();
|
||||
let items = match req.input {
|
||||
ResponsesInput::Items(i) => i,
|
||||
other => panic!("expected Items, got {other:?}"),
|
||||
};
|
||||
assert_eq!(items.len(), 3);
|
||||
for el in &items {
|
||||
assert!(
|
||||
matches!(el, ResponsesInputElement::EasyMessage { .. }),
|
||||
"expected EasyMessage, got {el:?}"
|
||||
);
|
||||
}
|
||||
// `tools` / `store` ride through `extra`, not `input`.
|
||||
assert!(req.extra.get("tools").is_some());
|
||||
assert_eq!(req.extra.get("store"), Some(&Value::Bool(true)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tolerates_null_content_and_unknown_item_types() {
|
||||
// An assistant turn carrying only tool calls has `content: null`;
|
||||
// and a future/unmodeled item type must not 422 the request.
|
||||
let raw = r#"{
|
||||
"model": "m",
|
||||
"input": [
|
||||
{"role": "assistant", "content": null},
|
||||
{"type": "item_reference", "id": "abc"},
|
||||
{"type": "function_call_output", "call_id": "c1",
|
||||
"output": [{"type": "output_text", "text": "result"}]},
|
||||
{"role": "user", "content": "go"}
|
||||
]
|
||||
}"#;
|
||||
let req: ResponsesRequest = serde_json::from_str(raw).unwrap();
|
||||
let items = match req.input {
|
||||
ResponsesInput::Items(i) => i,
|
||||
other => panic!("expected Items, got {other:?}"),
|
||||
};
|
||||
assert_eq!(items.len(), 4);
|
||||
assert!(matches!(
|
||||
&items[0],
|
||||
ResponsesInputElement::EasyMessage { content: None, .. }
|
||||
));
|
||||
assert!(matches!(&items[1], ResponsesInputElement::Other(_)));
|
||||
assert!(matches!(
|
||||
&items[2],
|
||||
ResponsesInputElement::Typed(ResponsesInputItem::FunctionCallOutput { .. })
|
||||
));
|
||||
assert!(matches!(
|
||||
&items[3],
|
||||
ResponsesInputElement::EasyMessage { .. }
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tolerates_unknown_content_part_type() {
|
||||
// A `refusal` (or any unmodeled) content part must parse, not 422.
|
||||
let raw = r#"{
|
||||
"model": "m",
|
||||
"input": [
|
||||
{"role": "assistant", "content": [
|
||||
{"type": "refusal", "refusal": "no"},
|
||||
{"type": "output_text", "text": "ok"}
|
||||
]}
|
||||
]
|
||||
}"#;
|
||||
let req: ResponsesRequest = serde_json::from_str(raw).unwrap();
|
||||
let items = match req.input {
|
||||
ResponsesInput::Items(i) => i,
|
||||
other => panic!("expected Items, got {other:?}"),
|
||||
};
|
||||
let parts = match &items[0] {
|
||||
ResponsesInputElement::EasyMessage {
|
||||
content: Some(ResponsesMessageContent::Parts(p)),
|
||||
..
|
||||
} => p,
|
||||
other => panic!("expected EasyMessage with Parts, got {other:?}"),
|
||||
};
|
||||
assert_eq!(parts.len(), 2);
|
||||
assert!(matches!(&parts[0], ResponsesContentPart::Unknown));
|
||||
assert!(matches!(&parts[1], ResponsesContentPart::OutputText { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deserialises_input_with_image() {
|
||||
let raw = r#"{
|
||||
"model": "m",
|
||||
"input": [
|
||||
{"type": "message", "role": "user", "content": [
|
||||
{"type": "input_text", "text": "what is this"},
|
||||
{"type": "input_image", "image_url": "data:image/png;base64,AAA="}
|
||||
]}
|
||||
]
|
||||
}"#;
|
||||
let req: ResponsesRequest = serde_json::from_str(raw).unwrap();
|
||||
let items = match req.input {
|
||||
ResponsesInput::Items(i) => i,
|
||||
other => panic!("expected Items, got {other:?}"),
|
||||
};
|
||||
let parts = match &items[0] {
|
||||
ResponsesInputElement::Typed(ResponsesInputItem::Message {
|
||||
content: ResponsesMessageContent::Parts(p),
|
||||
..
|
||||
}) => p,
|
||||
other => panic!("expected Parts, got {other:?}"),
|
||||
};
|
||||
assert_eq!(parts.len(), 2);
|
||||
assert!(matches!(
|
||||
&parts[0],
|
||||
ResponsesContentPart::InputText { text } if text == "what is this"
|
||||
));
|
||||
assert!(matches!(
|
||||
&parts[1],
|
||||
ResponsesContentPart::InputImage { image_url, .. }
|
||||
if image_url == "data:image/png;base64,AAA="
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_fields_round_trip_via_extra() {
|
||||
let raw = r#"{
|
||||
"model": "m",
|
||||
"input": "hi",
|
||||
"tools": [{"type": "web_search"}],
|
||||
"reasoning": {"effort": "medium"}
|
||||
}"#;
|
||||
let req: ResponsesRequest = serde_json::from_str(raw).unwrap();
|
||||
assert!(req.extra.get("tools").is_some());
|
||||
assert!(req.extra.get("reasoning").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_round_trips_through_serde() {
|
||||
let r = ResponsesResponse {
|
||||
id: "resp_1".into(),
|
||||
object: "response".into(),
|
||||
created_at: 1700,
|
||||
status: "completed".into(),
|
||||
model: "m".into(),
|
||||
output: vec![ResponsesOutputItem::Message {
|
||||
id: "msg_1".into(),
|
||||
role: "assistant".into(),
|
||||
content: vec![ResponsesOutputContent::OutputText {
|
||||
text: "hi there".into(),
|
||||
annotations: vec![],
|
||||
}],
|
||||
status: "completed".into(),
|
||||
}],
|
||||
usage: Some(ResponsesUsage {
|
||||
input_tokens: 5,
|
||||
output_tokens: 3,
|
||||
total_tokens: 8,
|
||||
output_tokens_details: None,
|
||||
input_tokens_details: None,
|
||||
}),
|
||||
};
|
||||
let json = serde_json::to_string(&r).unwrap();
|
||||
let parsed: ResponsesResponse = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(parsed.id, "resp_1");
|
||||
assert_eq!(parsed.output.len(), 1);
|
||||
}
|
||||
}
|
||||
@@ -1,267 +0,0 @@
|
||||
//! Scheme-qualified model identifiers.
|
||||
//!
|
||||
//! cortex/neuron historically resolves every model id through hf-hub
|
||||
//! against `https://huggingface.co`. Helexa is adding an EU-hosted
|
||||
//! registry (`registry.helexa.ai`) alongside HF — both speak the same
|
||||
//! HF-compatible wire format, but the bytes, jurisdiction, and trust
|
||||
//! root differ. Model ids therefore need a scheme:
|
||||
//!
|
||||
//! - `huggingface:Qwen/Qwen3.6-27B` — HF-hosted bytes
|
||||
//! - `helexa:Qwen/Qwen3.6-27B-Uncensored` — helexa registry bytes
|
||||
//! - `helexa:SomeOperator/CustomFinetune` — operator publishing
|
||||
//! under the helexa namespace; same scheme handles all `org/name`
|
||||
//! pairs hosted in that registry.
|
||||
//!
|
||||
//! Bare `org/name` parses with an empty scheme; the caller (typically
|
||||
//! a harness) substitutes its configured default scheme so existing
|
||||
//! configs keep working through the transition.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::fmt;
|
||||
use std::str::FromStr;
|
||||
|
||||
/// Parsed `scheme:org/name`. Bare `org/name` produces an empty scheme
|
||||
/// — call `with_default_scheme` (or check `is_scheme_unset`) to
|
||||
/// resolve before using.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub struct ModelSourceId {
|
||||
pub scheme: String,
|
||||
pub org: String,
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
/// Errors from `ModelSourceId::from_str`. Carries the offending input
|
||||
/// so log lines / API errors can echo what the operator typed.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum ParseError {
|
||||
#[error("empty model id")]
|
||||
Empty,
|
||||
#[error("model id '{0}' is missing the '/' between org and name")]
|
||||
MissingSlash(String),
|
||||
#[error("model id '{0}' has an empty scheme before ':'")]
|
||||
EmptyScheme(String),
|
||||
#[error("model id '{0}' has an empty org")]
|
||||
EmptyOrg(String),
|
||||
#[error("model id '{0}' has an empty name")]
|
||||
EmptyName(String),
|
||||
#[error("model id '{0}' has a scheme containing '/' which is reserved for org/name")]
|
||||
SchemeContainsSlash(String),
|
||||
#[error("model id '{0}' has a name containing ':' which is reserved for the scheme prefix")]
|
||||
NameContainsColon(String),
|
||||
}
|
||||
|
||||
impl ModelSourceId {
|
||||
/// Construct directly from already-validated parts. Used by tests
|
||||
/// and call sites that have the fields separately; the public API
|
||||
/// for parsing user input is `FromStr`.
|
||||
pub fn new(scheme: impl Into<String>, org: impl Into<String>, name: impl Into<String>) -> Self {
|
||||
Self {
|
||||
scheme: scheme.into(),
|
||||
org: org.into(),
|
||||
name: name.into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// True when this id parsed from a bare `org/name` (no scheme
|
||||
/// prefix). The harness substitutes its configured default in
|
||||
/// `with_default_scheme` before resolving against a registry.
|
||||
pub fn is_scheme_unset(&self) -> bool {
|
||||
self.scheme.is_empty()
|
||||
}
|
||||
|
||||
/// Substitute `default` for an empty scheme. No-op when the scheme
|
||||
/// is already set. Returns self by value so it composes neatly:
|
||||
/// `id.parse::<ModelSourceId>()?.with_default_scheme("huggingface")`.
|
||||
pub fn with_default_scheme(mut self, default: &str) -> Self {
|
||||
if self.scheme.is_empty() {
|
||||
self.scheme = default.to_string();
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// The `org/name` half — what an hf-hub `Api::model(...)` call
|
||||
/// expects regardless of which scheme/endpoint we're hitting.
|
||||
pub fn repo_path(&self) -> String {
|
||||
format!("{}/{}", self.org, self.name)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ModelSourceId {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
if self.scheme.is_empty() {
|
||||
write!(f, "{}/{}", self.org, self.name)
|
||||
} else {
|
||||
write!(f, "{}:{}/{}", self.scheme, self.org, self.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for ModelSourceId {
|
||||
type Err = ParseError;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
if s.is_empty() {
|
||||
return Err(ParseError::Empty);
|
||||
}
|
||||
// Scheme split. Only the *first* colon counts — anything after
|
||||
// belongs to org/name (and would be rejected separately because
|
||||
// `:` isn't allowed there).
|
||||
let (scheme, rest) = match s.split_once(':') {
|
||||
Some((scheme, rest)) => {
|
||||
if scheme.is_empty() {
|
||||
return Err(ParseError::EmptyScheme(s.to_string()));
|
||||
}
|
||||
if scheme.contains('/') {
|
||||
return Err(ParseError::SchemeContainsSlash(s.to_string()));
|
||||
}
|
||||
(scheme.to_string(), rest)
|
||||
}
|
||||
None => (String::new(), s),
|
||||
};
|
||||
let (org, name) = rest
|
||||
.split_once('/')
|
||||
.ok_or_else(|| ParseError::MissingSlash(s.to_string()))?;
|
||||
if org.is_empty() {
|
||||
return Err(ParseError::EmptyOrg(s.to_string()));
|
||||
}
|
||||
if name.is_empty() {
|
||||
return Err(ParseError::EmptyName(s.to_string()));
|
||||
}
|
||||
if name.contains(':') {
|
||||
return Err(ParseError::NameContainsColon(s.to_string()));
|
||||
}
|
||||
Ok(Self {
|
||||
scheme,
|
||||
org: org.to_string(),
|
||||
name: name.to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parses_qualified() {
|
||||
let id: ModelSourceId = "huggingface:Qwen/Qwen3.6-27B".parse().unwrap();
|
||||
assert_eq!(id.scheme, "huggingface");
|
||||
assert_eq!(id.org, "Qwen");
|
||||
assert_eq!(id.name, "Qwen3.6-27B");
|
||||
assert_eq!(id.repo_path(), "Qwen/Qwen3.6-27B");
|
||||
assert!(!id.is_scheme_unset());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_helexa_scheme() {
|
||||
let id: ModelSourceId = "helexa:SomeOperator/Qwen3.6-27B-Uncensored"
|
||||
.parse()
|
||||
.unwrap();
|
||||
assert_eq!(id.scheme, "helexa");
|
||||
assert_eq!(id.org, "SomeOperator");
|
||||
assert_eq!(id.name, "Qwen3.6-27B-Uncensored");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_bare_id_with_empty_scheme() {
|
||||
let id: ModelSourceId = "Qwen/Qwen3-30B-A3B-Instruct".parse().unwrap();
|
||||
assert_eq!(id.scheme, "");
|
||||
assert_eq!(id.org, "Qwen");
|
||||
assert_eq!(id.name, "Qwen3-30B-A3B-Instruct");
|
||||
assert!(id.is_scheme_unset());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn substitutes_default_scheme_only_when_unset() {
|
||||
let id: ModelSourceId = "Qwen/Q3".parse().unwrap();
|
||||
assert_eq!(id.with_default_scheme("huggingface").scheme, "huggingface");
|
||||
|
||||
let id: ModelSourceId = "helexa:Qwen/Q3".parse().unwrap();
|
||||
assert_eq!(
|
||||
id.with_default_scheme("huggingface").scheme,
|
||||
"helexa",
|
||||
"default substitution must not override an explicit scheme"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn display_roundtrips_qualified_id() {
|
||||
let s = "helexa:Helexa/Qwen3.6-27B";
|
||||
let id: ModelSourceId = s.parse().unwrap();
|
||||
assert_eq!(id.to_string(), s);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn display_roundtrips_bare_id() {
|
||||
let s = "Qwen/Q3";
|
||||
let id: ModelSourceId = s.parse().unwrap();
|
||||
assert_eq!(id.to_string(), s);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_empty() {
|
||||
assert_eq!("".parse::<ModelSourceId>().unwrap_err(), ParseError::Empty);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_missing_slash() {
|
||||
match "Qwen".parse::<ModelSourceId>().unwrap_err() {
|
||||
ParseError::MissingSlash(s) => assert_eq!(s, "Qwen"),
|
||||
other => panic!("expected MissingSlash, got {other:?}"),
|
||||
}
|
||||
match "huggingface:Qwen".parse::<ModelSourceId>().unwrap_err() {
|
||||
ParseError::MissingSlash(s) => assert_eq!(s, "huggingface:Qwen"),
|
||||
other => panic!("expected MissingSlash, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_empty_scheme() {
|
||||
match ":Qwen/Q3".parse::<ModelSourceId>().unwrap_err() {
|
||||
ParseError::EmptyScheme(s) => assert_eq!(s, ":Qwen/Q3"),
|
||||
other => panic!("expected EmptyScheme, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_scheme_with_slash() {
|
||||
match "hugg/ingface:Q/N".parse::<ModelSourceId>().unwrap_err() {
|
||||
ParseError::SchemeContainsSlash(s) => assert_eq!(s, "hugg/ingface:Q/N"),
|
||||
other => panic!("expected SchemeContainsSlash, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_empty_org_or_name() {
|
||||
match "huggingface:/N".parse::<ModelSourceId>().unwrap_err() {
|
||||
ParseError::EmptyOrg(_) => {}
|
||||
other => panic!("expected EmptyOrg, got {other:?}"),
|
||||
}
|
||||
match "huggingface:Q/".parse::<ModelSourceId>().unwrap_err() {
|
||||
ParseError::EmptyName(_) => {}
|
||||
other => panic!("expected EmptyName, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_name_with_colon() {
|
||||
match "huggingface:Q/N:weird"
|
||||
.parse::<ModelSourceId>()
|
||||
.unwrap_err()
|
||||
{
|
||||
ParseError::NameContainsColon(s) => assert_eq!(s, "huggingface:Q/N:weird"),
|
||||
other => panic!("expected NameContainsColon, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serde_roundtrips_via_struct() {
|
||||
// We serialize as a struct (scheme/org/name fields) so the
|
||||
// shape is self-describing in API payloads. Callers that want
|
||||
// the compact `scheme:org/name` string use `Display`/`FromStr`.
|
||||
let id = ModelSourceId::new("helexa", "Helexa", "Qwen3.6-27B");
|
||||
let json = serde_json::to_string(&id).unwrap();
|
||||
let back: ModelSourceId = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(back, id);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -6,8 +6,6 @@ license.workspace = true
|
||||
|
||||
[dependencies]
|
||||
cortex-core.workspace = true
|
||||
helexa-stream = { path = "../helexa-stream" }
|
||||
async-trait.workspace = true
|
||||
tokio.workspace = true
|
||||
axum.workspace = true
|
||||
tower.workspace = true
|
||||
@@ -26,7 +24,6 @@ tokio-stream.workspace = true
|
||||
eventsource-stream.workspace = true
|
||||
bytes = "1"
|
||||
urlencoding = "2"
|
||||
url = "2"
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { workspace = true, features = ["test-util"] }
|
||||
|
||||
@@ -1,235 +0,0 @@
|
||||
//! Streaming Anthropic SSE translation (#24).
|
||||
//!
|
||||
//! The `/v1/messages` handler translates the request envelope to
|
||||
//! OpenAI before proxying (see `cortex_core::translate`); this module
|
||||
//! completes the round trip for `stream: true` — the upstream OpenAI
|
||||
//! SSE stream is re-framed, event by event, into Anthropic's
|
||||
//! `message_start` / `content_block_*` / `message_delta` /
|
||||
//! `message_stop` sequence as it arrives. True streaming: each
|
||||
//! upstream chunk is translated and forwarded immediately; nothing is
|
||||
//! buffered beyond the current SSE event's bytes.
|
||||
//!
|
||||
//! The translation state machine itself is pure and lives in
|
||||
//! [`cortex_core::translate::AnthropicStreamTranslator`]; this module
|
||||
//! owns the wire concerns — splitting the upstream byte stream into
|
||||
//! SSE events, parsing `data:` payloads, and framing the translated
|
||||
//! events as `event: <name>\ndata: <json>\n\n`.
|
||||
|
||||
use axum::body::Body;
|
||||
use axum::http::StatusCode;
|
||||
use axum::response::Response;
|
||||
use bytes::Bytes;
|
||||
use cortex_core::openai::ChatCompletionChunk;
|
||||
use cortex_core::translate::AnthropicStreamTranslator;
|
||||
use futures::StreamExt;
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
|
||||
/// Forward the translated OpenAI request to the upstream node and
|
||||
/// return the response translated to Anthropic SSE framing.
|
||||
pub async fn stream_translated(
|
||||
client: &reqwest::Client,
|
||||
endpoint: &str,
|
||||
openai_body: axum::body::Bytes,
|
||||
model_id: &str,
|
||||
node_name: &str,
|
||||
inbound_headers: &axum::http::HeaderMap,
|
||||
usage_sink: Option<crate::metering::UsageSink>,
|
||||
) -> Response {
|
||||
let url = format!("{endpoint}/v1/chat/completions");
|
||||
tracing::info!(
|
||||
handler = "anthropic_messages",
|
||||
model = %model_id,
|
||||
node = %node_name,
|
||||
url = %url,
|
||||
"proxying streaming request (anthropic SSE translation)"
|
||||
);
|
||||
|
||||
let request = crate::auth::forward_principal_headers(
|
||||
client
|
||||
.post(&url)
|
||||
.header("content-type", "application/json")
|
||||
.body(openai_body),
|
||||
inbound_headers,
|
||||
);
|
||||
let upstream = match request.send().await {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
handler = "anthropic_messages",
|
||||
node = %node_name,
|
||||
url = %url,
|
||||
error = %e,
|
||||
"anthropic stream: upstream request failed"
|
||||
);
|
||||
return anthropic_error(StatusCode::BAD_GATEWAY, "upstream request failed");
|
||||
}
|
||||
};
|
||||
|
||||
let status = upstream.status();
|
||||
if !status.is_success() {
|
||||
tracing::warn!(
|
||||
handler = "anthropic_messages",
|
||||
node = %node_name,
|
||||
url = %url,
|
||||
status = status.as_u16(),
|
||||
"anthropic stream: upstream returned non-2xx"
|
||||
);
|
||||
return anthropic_error(
|
||||
StatusCode::from_u16(status.as_u16()).unwrap_or(StatusCode::BAD_GATEWAY),
|
||||
"upstream returned an error",
|
||||
);
|
||||
}
|
||||
|
||||
// Bounded channel: a slow client back-pressures the pump task,
|
||||
// which back-pressures the upstream read — same propagation
|
||||
// discipline as neuron's own projectors.
|
||||
let (tx, rx) = tokio::sync::mpsc::channel::<Result<Bytes, std::convert::Infallible>>(32);
|
||||
let node = node_name.to_string();
|
||||
let model = model_id.to_string();
|
||||
tokio::spawn(async move {
|
||||
let mut upstream = upstream.bytes_stream();
|
||||
let mut translator = AnthropicStreamTranslator::new();
|
||||
let mut buf: Vec<u8> = Vec::new();
|
||||
let mut done = false;
|
||||
// Wire-debug accounting for the stream summary emitted at the
|
||||
// end: did the model emit a structured tool call, what was the
|
||||
// final finish_reason, and how many upstream frames did we see.
|
||||
let mut saw_tool_call = false;
|
||||
let mut last_finish: Option<String> = None;
|
||||
let mut frames = 0u64;
|
||||
// Engine-truth usage for metering (#51), scanned from the upstream
|
||||
// frames (neuron emits a final `usage` object on the stream, #48).
|
||||
let mut usage_prompt = 0u64;
|
||||
let mut usage_completion = 0u64;
|
||||
|
||||
'outer: while let Some(block) = upstream.next().await {
|
||||
let block = match block {
|
||||
Ok(b) => b,
|
||||
Err(e) => {
|
||||
tracing::warn!(node = %node, error = %e, "anthropic stream: upstream read failed mid-stream");
|
||||
break;
|
||||
}
|
||||
};
|
||||
buf.extend_from_slice(&block);
|
||||
// SSE events are separated by a blank line.
|
||||
while let Some(pos) = find_event_boundary(&buf) {
|
||||
let event: Vec<u8> = buf.drain(..pos + 2).collect();
|
||||
let text = String::from_utf8_lossy(&event);
|
||||
for line in text.lines() {
|
||||
let Some(data) = line.strip_prefix("data:") else {
|
||||
continue;
|
||||
};
|
||||
let data = data.trim();
|
||||
if data == "[DONE]" {
|
||||
done = true;
|
||||
if !send_frames(&tx, translator.finish()).await {
|
||||
break 'outer;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
tracing::trace!(node = %node, frame = %data, "anthropic stream: upstream frame");
|
||||
// Capture usage for metering before translation — the
|
||||
// usage object rides on a late frame (often after the
|
||||
// last content delta).
|
||||
if let Some(p) = crate::proxy::last_count_for(data, "prompt_tokens") {
|
||||
usage_prompt = p;
|
||||
}
|
||||
if let Some(c) = crate::proxy::last_count_for(data, "completion_tokens") {
|
||||
usage_completion = c;
|
||||
}
|
||||
let Ok(chunk) = serde_json::from_str::<ChatCompletionChunk>(data) else {
|
||||
tracing::debug!(node = %node, "anthropic stream: unparsable upstream frame skipped");
|
||||
continue;
|
||||
};
|
||||
frames += 1;
|
||||
if chunk
|
||||
.choices
|
||||
.iter()
|
||||
.any(|c| c.delta.get("tool_calls").is_some())
|
||||
{
|
||||
saw_tool_call = true;
|
||||
}
|
||||
if let Some(fr) = chunk.choices.iter().find_map(|c| c.finish_reason.clone()) {
|
||||
last_finish = Some(fr);
|
||||
}
|
||||
if !send_frames(&tx, translator.on_chunk(&chunk)).await {
|
||||
break 'outer;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Upstream ended without [DONE] (error or truncation): still
|
||||
// close the Anthropic event sequence so clients aren't left
|
||||
// with an unterminated message.
|
||||
if !done {
|
||||
let _ = send_frames(&tx, translator.finish()).await;
|
||||
}
|
||||
// Stream summary: the streaming counterpart to the non-streaming
|
||||
// handler's "upstream response" line. `upstream_tool_calls =
|
||||
// false` on a tools-bearing request is the fingerprint of the
|
||||
// model improvising an unparsed tool-call format.
|
||||
tracing::debug!(
|
||||
wire = "anthropic",
|
||||
model = %model,
|
||||
node = %node,
|
||||
frames,
|
||||
upstream_tool_calls = saw_tool_call,
|
||||
finish_reason = ?last_finish,
|
||||
terminated = done,
|
||||
"anthropic stream complete"
|
||||
);
|
||||
|
||||
// Settle metering with the observed usage (#51). Runs on every exit
|
||||
// path of the pump — clean end, early break, or upstream error — so
|
||||
// the reservation is always resolved. `(0, 0)` when no usage frame
|
||||
// was seen, which releases without recording spend.
|
||||
if let Some(sink) = usage_sink {
|
||||
sink(usage_prompt, usage_completion);
|
||||
}
|
||||
});
|
||||
|
||||
Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header("content-type", "text/event-stream")
|
||||
.header("cache-control", "no-cache")
|
||||
.body(Body::from_stream(ReceiverStream::new(rx)))
|
||||
.unwrap_or_else(|_| {
|
||||
anthropic_error(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"failed to build response",
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// `\n\n` boundary of the first complete SSE event in `buf`, if any.
|
||||
fn find_event_boundary(buf: &[u8]) -> Option<usize> {
|
||||
buf.windows(2).position(|w| w == b"\n\n")
|
||||
}
|
||||
|
||||
/// Render translated events as SSE frames and send them. Returns
|
||||
/// `false` when the client has gone away (receiver dropped).
|
||||
async fn send_frames(
|
||||
tx: &tokio::sync::mpsc::Sender<Result<Bytes, std::convert::Infallible>>,
|
||||
events: Vec<(String, serde_json::Value)>,
|
||||
) -> bool {
|
||||
for (name, payload) in events {
|
||||
let frame = format!("event: {name}\ndata: {payload}\n\n");
|
||||
if tx.send(Ok(Bytes::from(frame))).await.is_err() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Anthropic-shaped error body (`{"type":"error","error":{...}}`).
|
||||
fn anthropic_error(status: StatusCode, message: &str) -> Response {
|
||||
let body = serde_json::json!({
|
||||
"type": "error",
|
||||
"error": { "type": "api_error", "message": message }
|
||||
});
|
||||
Response::builder()
|
||||
.status(status)
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(body.to_string()))
|
||||
.expect("static error response must build")
|
||||
}
|
||||
@@ -1,144 +0,0 @@
|
||||
//! API-key authentication + principal resolution (#49).
|
||||
//!
|
||||
//! Identity rides standard bearer auth only — `Authorization: Bearer <key>`
|
||||
//! — which is what keeps every tier OpenAI-compatible by construction (no
|
||||
//! custom required headers or body fields, per #47). The middleware resolves
|
||||
//! the key to a [`Principal`] via the [`EntitlementProvider`], carries it in
|
||||
//! the request extensions for cortex-side metering/enforcement (#51/#52), and
|
||||
//! stamps it as internal headers on the request so it reaches neuron, which
|
||||
//! trusts cortex's assertion over WireGuard (#54).
|
||||
//!
|
||||
//! Anti-spoofing: any client-supplied principal header is **stripped** before
|
||||
//! the authoritative value is stamped, so a client can never assert a
|
||||
//! principal it didn't authenticate as.
|
||||
//!
|
||||
//! Rejection contract (#63): missing key under `require_auth`, or any present
|
||||
//! but unresolvable key, yields `401 invalid_api_key` in the #60 envelope.
|
||||
|
||||
use crate::error::envelope_response;
|
||||
use crate::state::CortexState;
|
||||
use axum::extract::{Request, State};
|
||||
use axum::http::header::AUTHORIZATION;
|
||||
use axum::http::{HeaderMap, HeaderValue};
|
||||
use axum::middleware::Next;
|
||||
use axum::response::Response;
|
||||
use cortex_core::entitlements::{AuthError, HEADER_ACCOUNT_ID, HEADER_KEY_ID};
|
||||
use cortex_core::error_envelope::OpenAiError;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Endpoints that never require auth: liveness/readiness probes. Everything
|
||||
/// else flows through resolution.
|
||||
fn is_public(path: &str) -> bool {
|
||||
path == "/health" || path == "/"
|
||||
}
|
||||
|
||||
/// Extract the bearer token from an `Authorization` header value, if present
|
||||
/// and well-formed. Scheme match is case-insensitive per RFC 7235.
|
||||
fn parse_bearer(headers: &HeaderMap) -> Option<String> {
|
||||
let raw = headers.get(AUTHORIZATION)?.to_str().ok()?;
|
||||
let (scheme, token) = raw.split_once(' ')?;
|
||||
if scheme.eq_ignore_ascii_case("bearer") {
|
||||
let token = token.trim();
|
||||
(!token.is_empty()).then(|| token.to_string())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Axum middleware: resolve the bearer key, attach the principal, stamp the
|
||||
/// internal headers. Wired in `build_app` via `from_fn_with_state`.
|
||||
pub async fn require_principal(
|
||||
State(fleet): State<Arc<CortexState>>,
|
||||
mut req: Request,
|
||||
next: Next,
|
||||
) -> Response {
|
||||
if is_public(req.uri().path()) {
|
||||
return next.run(req).await;
|
||||
}
|
||||
|
||||
// Anti-spoof: drop any client-supplied principal headers up front.
|
||||
{
|
||||
let headers = req.headers_mut();
|
||||
headers.remove(HEADER_ACCOUNT_ID);
|
||||
headers.remove(HEADER_KEY_ID);
|
||||
}
|
||||
|
||||
match parse_bearer(req.headers()) {
|
||||
Some(key) => match fleet.entitlements.resolve(&key).await {
|
||||
Ok(principal) => {
|
||||
// Stamp the authoritative principal for neuron. Account/key
|
||||
// ids come from operator config, so they're valid header
|
||||
// values; guard anyway and skip a malformed one rather than
|
||||
// panic.
|
||||
if let (Ok(account), Ok(key_id)) = (
|
||||
HeaderValue::from_str(&principal.account_id),
|
||||
HeaderValue::from_str(&principal.key_id),
|
||||
) {
|
||||
let headers = req.headers_mut();
|
||||
headers.insert(HEADER_ACCOUNT_ID, account);
|
||||
headers.insert(HEADER_KEY_ID, key_id);
|
||||
}
|
||||
// Carry the typed principal for cortex-side metering (#51)
|
||||
// and budget enforcement (#52).
|
||||
req.extensions_mut().insert(principal);
|
||||
next.run(req).await
|
||||
}
|
||||
// The entitlement authority is unreachable (upstream client
|
||||
// blip, #57). Fail **closed but distinct**: a transient outage
|
||||
// must not reject a real key as `401 invalid_api_key` — it's a
|
||||
// retryable `503`. This holds regardless of require_auth: we
|
||||
// can't safely serve a key we couldn't authorize.
|
||||
Err(AuthError::Unavailable { retry_after_secs }) => {
|
||||
envelope_response(OpenAiError::service_unavailable(
|
||||
"entitlement authority temporarily unavailable",
|
||||
Some(retry_after_secs),
|
||||
))
|
||||
}
|
||||
// A genuinely unrecognized key only hard-fails when auth is
|
||||
// *required*. In allow-anonymous mode (the default) we IGNORE it
|
||||
// and serve unauthenticated — otherwise the placeholder keys that
|
||||
// OpenAI-compatible clients send by default (opencode, Open WebUI,
|
||||
// Agent Zero, litellm) would all break though the operator never
|
||||
// opted into auth. Pre-#49 the bearer was never inspected; this
|
||||
// preserves that for require_auth=false.
|
||||
Err(AuthError::InvalidKey) => {
|
||||
if fleet.require_auth {
|
||||
unauthorized("invalid API key")
|
||||
} else {
|
||||
tracing::debug!(
|
||||
"ignoring unrecognized bearer token (require_auth=false): serving anonymously"
|
||||
);
|
||||
next.run(req).await
|
||||
}
|
||||
}
|
||||
},
|
||||
None => {
|
||||
if fleet.require_auth {
|
||||
unauthorized("missing API key; supply 'Authorization: Bearer <key>'")
|
||||
} else {
|
||||
next.run(req).await
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `401 invalid_api_key` in the standard envelope (#63).
|
||||
fn unauthorized(message: &str) -> Response {
|
||||
envelope_response(OpenAiError::invalid_api_key(message))
|
||||
}
|
||||
|
||||
/// Copy the cortex-stamped principal headers from an inbound [`HeaderMap`]
|
||||
/// onto an outbound reqwest builder. Used by the Anthropic proxy paths,
|
||||
/// which construct their own upstream requests instead of going through
|
||||
/// [`crate::proxy::forward_request`] (which forwards all headers verbatim).
|
||||
pub fn forward_principal_headers(
|
||||
mut builder: reqwest::RequestBuilder,
|
||||
headers: &HeaderMap,
|
||||
) -> reqwest::RequestBuilder {
|
||||
for name in [HEADER_ACCOUNT_ID, HEADER_KEY_ID] {
|
||||
if let Some(value) = headers.get(name) {
|
||||
builder = builder.header(name, value);
|
||||
}
|
||||
}
|
||||
builder
|
||||
}
|
||||
@@ -1,112 +0,0 @@
|
||||
//! Chained entitlement provider (#57): operator-local keys first, mesh
|
||||
//! upstream for everything else.
|
||||
//!
|
||||
//! `resolve` tries the [`LocalEntitlementProvider`] (operator + infra keys —
|
||||
//! never a network hop); only a locally-unknown key falls through to
|
||||
//! [`UpstreamEntitlementProvider`]. Because the local provider treats an
|
||||
//! unconfigured principal as uncapped, reserve/settle/release/snapshot must
|
||||
//! **not** blindly hit local — they dispatch to whichever backend resolved
|
||||
//! that account, remembered in a map keyed by `account_id` (populated at
|
||||
//! resolve time).
|
||||
|
||||
use crate::entitlements_local::LocalEntitlementProvider;
|
||||
use crate::entitlements_upstream::UpstreamEntitlementProvider;
|
||||
use async_trait::async_trait;
|
||||
use cortex_core::entitlements::{
|
||||
AuthError, BudgetError, BudgetSnapshot, EntitlementProvider, Principal, Reservation,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum Backend {
|
||||
Local,
|
||||
Upstream,
|
||||
}
|
||||
|
||||
pub struct ChainedEntitlementProvider {
|
||||
local: LocalEntitlementProvider,
|
||||
upstream: UpstreamEntitlementProvider,
|
||||
/// account_id → which backend owns it, learned at resolve time.
|
||||
backends: RwLock<HashMap<String, Backend>>,
|
||||
}
|
||||
|
||||
impl ChainedEntitlementProvider {
|
||||
pub fn new(local: LocalEntitlementProvider, upstream: UpstreamEntitlementProvider) -> Self {
|
||||
Self {
|
||||
local,
|
||||
upstream,
|
||||
backends: RwLock::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn record(&self, account_id: &str, backend: Backend) {
|
||||
self.backends
|
||||
.write()
|
||||
.await
|
||||
.insert(account_id.to_string(), backend);
|
||||
}
|
||||
|
||||
/// The backend that owns `account_id`. Defaults to `Upstream` for an
|
||||
/// account never resolved this process-lifetime (a resolve always
|
||||
/// precedes reserve in a request, so this is just a safe fallback —
|
||||
/// upstream fails closed if the account is bogus).
|
||||
async fn backend_for(&self, account_id: &str) -> Backend {
|
||||
self.backends
|
||||
.read()
|
||||
.await
|
||||
.get(account_id)
|
||||
.copied()
|
||||
.unwrap_or(Backend::Upstream)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl EntitlementProvider for ChainedEntitlementProvider {
|
||||
async fn resolve(&self, api_key: &str) -> Result<Principal, AuthError> {
|
||||
match self.local.resolve(api_key).await {
|
||||
Ok(p) => {
|
||||
self.record(&p.account_id, Backend::Local).await;
|
||||
Ok(p)
|
||||
}
|
||||
Err(AuthError::InvalidKey) => {
|
||||
let p = self.upstream.resolve(api_key).await?;
|
||||
self.record(&p.account_id, Backend::Upstream).await;
|
||||
Ok(p)
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
async fn reserve(
|
||||
&self,
|
||||
principal: &Principal,
|
||||
max_tokens: u64,
|
||||
) -> Result<Reservation, BudgetError> {
|
||||
match self.backend_for(&principal.account_id).await {
|
||||
Backend::Local => self.local.reserve(principal, max_tokens).await,
|
||||
Backend::Upstream => self.upstream.reserve(principal, max_tokens).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn settle(&self, reservation: Reservation, actual_tokens: u64) {
|
||||
match self.backend_for(&reservation.principal.account_id).await {
|
||||
Backend::Local => self.local.settle(reservation, actual_tokens).await,
|
||||
Backend::Upstream => self.upstream.settle(reservation, actual_tokens).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn release(&self, reservation: Reservation) {
|
||||
match self.backend_for(&reservation.principal.account_id).await {
|
||||
Backend::Local => self.local.release(reservation).await,
|
||||
Backend::Upstream => self.upstream.release(reservation).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn snapshot(&self, principal: &Principal) -> Option<BudgetSnapshot> {
|
||||
match self.backend_for(&principal.account_id).await {
|
||||
Backend::Local => self.local.snapshot(principal).await,
|
||||
Backend::Upstream => self.upstream.snapshot(principal).await,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,317 +0,0 @@
|
||||
//! The local/static [`EntitlementProvider`] (#50).
|
||||
//!
|
||||
//! Accounts, keys, and hard caps come from operator config
|
||||
//! ([`cortex_core::config::EntitlementsConfig`]); reservations and settled
|
||||
//! spend are tracked in-process. This lands auth + per-key caps + the
|
||||
//! amplification fix before any upstream clearing house exists; the future
|
||||
//! helexa-upstream client (#57) implements the same trait.
|
||||
//!
|
||||
//! Budget math is serialized under a single [`std::sync::Mutex`] so
|
||||
//! reserve/settle/release are atomic — a key's `spent + reserved` can never
|
||||
//! exceed its hard cap even under concurrent requests (the #52 guarantee).
|
||||
//! The lock is held only for the in-memory arithmetic, never across an
|
||||
//! await.
|
||||
|
||||
use cortex_core::config::{ApiKeyConfig, EntitlementsConfig};
|
||||
use cortex_core::entitlements::{
|
||||
AuthError, BudgetError, BudgetSnapshot, CapWindow, EntitlementProvider, Principal, Reservation,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::Instant;
|
||||
|
||||
/// Per-key budget configuration (resolved from [`ApiKeyConfig`]).
|
||||
struct Budget {
|
||||
hard_cap: Option<u64>,
|
||||
window: CapWindow,
|
||||
}
|
||||
|
||||
/// Live, mutable accounting for one key over its current window.
|
||||
#[derive(Default)]
|
||||
struct Ledger {
|
||||
/// Settled spend in the current window.
|
||||
spent: u64,
|
||||
/// Sum of outstanding (un-settled) reservations.
|
||||
reserved: u64,
|
||||
/// Start of the current rolling window; `None` until the first reserve.
|
||||
/// Unused for [`CapWindow::Balance`].
|
||||
window_start: Option<Instant>,
|
||||
}
|
||||
|
||||
pub struct LocalEntitlementProvider {
|
||||
/// Bearer token → principal.
|
||||
keys: HashMap<String, Principal>,
|
||||
/// `key_id` → budget config.
|
||||
budgets: HashMap<String, Budget>,
|
||||
/// `key_id` → live ledger.
|
||||
ledgers: Mutex<HashMap<String, Ledger>>,
|
||||
/// Monotonic source of opaque reservation handles.
|
||||
next_id: AtomicU64,
|
||||
}
|
||||
|
||||
impl LocalEntitlementProvider {
|
||||
/// Build from the `[entitlements]` config. A key without an explicit
|
||||
/// `key_id` is tracked at `account_id` granularity (its secret is never
|
||||
/// used as a label).
|
||||
pub fn from_config(config: &EntitlementsConfig) -> Self {
|
||||
let mut keys = HashMap::new();
|
||||
let mut budgets = HashMap::new();
|
||||
for ApiKeyConfig {
|
||||
key,
|
||||
account_id,
|
||||
key_id,
|
||||
hard_cap,
|
||||
window,
|
||||
} in &config.keys
|
||||
{
|
||||
let key_id = key_id.clone().unwrap_or_else(|| account_id.clone());
|
||||
keys.insert(
|
||||
key.clone(),
|
||||
Principal {
|
||||
account_id: account_id.clone(),
|
||||
key_id: key_id.clone(),
|
||||
},
|
||||
);
|
||||
budgets.insert(
|
||||
key_id,
|
||||
Budget {
|
||||
hard_cap: *hard_cap,
|
||||
window: window.clone(),
|
||||
},
|
||||
);
|
||||
}
|
||||
Self {
|
||||
keys,
|
||||
budgets,
|
||||
ledgers: Mutex::new(HashMap::new()),
|
||||
next_id: AtomicU64::new(1),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Tokens still available under `cap` given current `spent`/`reserved`.
|
||||
/// `None` cap = unlimited.
|
||||
fn available(cap: Option<u64>, spent: u64, reserved: u64) -> Option<u64> {
|
||||
cap.map(|c| c.saturating_sub(spent).saturating_sub(reserved))
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl EntitlementProvider for LocalEntitlementProvider {
|
||||
async fn resolve(&self, api_key: &str) -> Result<Principal, AuthError> {
|
||||
self.keys.get(api_key).cloned().ok_or(AuthError::InvalidKey)
|
||||
}
|
||||
|
||||
async fn reserve(
|
||||
&self,
|
||||
principal: &Principal,
|
||||
max_tokens: u64,
|
||||
) -> Result<Reservation, BudgetError> {
|
||||
// A principal with no configured budget (or an uncapped one) always
|
||||
// reserves; we still track spend for metrics.
|
||||
let budget = self.budgets.get(&principal.key_id);
|
||||
let (cap, window) = match budget {
|
||||
Some(b) => (b.hard_cap, b.window.clone()),
|
||||
None => (None, CapWindow::Balance),
|
||||
};
|
||||
|
||||
let mut ledgers = self.ledgers.lock().expect("ledger mutex poisoned");
|
||||
let ledger = ledgers.entry(principal.key_id.clone()).or_default();
|
||||
|
||||
// Lazily reset a rolling window that has elapsed before checking.
|
||||
let mut retry_after_secs = 0;
|
||||
if let CapWindow::Rolling { seconds } = window {
|
||||
let now = Instant::now();
|
||||
match ledger.window_start {
|
||||
Some(start) if now.duration_since(start).as_secs() < seconds => {
|
||||
retry_after_secs = seconds - now.duration_since(start).as_secs();
|
||||
}
|
||||
_ => {
|
||||
// First reserve, or the window has fully elapsed: reset.
|
||||
ledger.spent = 0;
|
||||
ledger.window_start = Some(now);
|
||||
retry_after_secs = seconds;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(avail) = available(cap, ledger.spent, ledger.reserved)
|
||||
&& max_tokens > avail
|
||||
{
|
||||
return Err(match window {
|
||||
CapWindow::Rolling { .. } => BudgetError::RateLimited {
|
||||
requested: max_tokens,
|
||||
available: avail,
|
||||
// At least 1s so clients don't hot-loop on a sub-second
|
||||
// remainder.
|
||||
retry_after_secs: retry_after_secs.max(1),
|
||||
},
|
||||
CapWindow::Balance => BudgetError::InsufficientQuota {
|
||||
requested: max_tokens,
|
||||
available: avail,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
ledger.reserved += max_tokens;
|
||||
Ok(Reservation {
|
||||
id: self.next_id.fetch_add(1, Ordering::Relaxed),
|
||||
principal: principal.clone(),
|
||||
reserved: max_tokens,
|
||||
})
|
||||
}
|
||||
|
||||
async fn settle(&self, reservation: Reservation, actual_tokens: u64) {
|
||||
let mut ledgers = self.ledgers.lock().expect("ledger mutex poisoned");
|
||||
if let Some(ledger) = ledgers.get_mut(&reservation.principal.key_id) {
|
||||
ledger.reserved = ledger.reserved.saturating_sub(reservation.reserved);
|
||||
ledger.spent += actual_tokens;
|
||||
}
|
||||
}
|
||||
|
||||
async fn release(&self, reservation: Reservation) {
|
||||
let mut ledgers = self.ledgers.lock().expect("ledger mutex poisoned");
|
||||
if let Some(ledger) = ledgers.get_mut(&reservation.principal.key_id) {
|
||||
ledger.reserved = ledger.reserved.saturating_sub(reservation.reserved);
|
||||
}
|
||||
}
|
||||
|
||||
async fn snapshot(&self, principal: &Principal) -> Option<BudgetSnapshot> {
|
||||
let ledgers = self.ledgers.lock().expect("ledger mutex poisoned");
|
||||
let (spent, reserved) = ledgers
|
||||
.get(&principal.key_id)
|
||||
.map(|l| (l.spent, l.reserved))
|
||||
.unwrap_or((0, 0));
|
||||
let hard_cap = self.budgets.get(&principal.key_id).and_then(|b| b.hard_cap);
|
||||
Some(BudgetSnapshot {
|
||||
hard_cap,
|
||||
spent,
|
||||
reserved,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn provider() -> LocalEntitlementProvider {
|
||||
let config = EntitlementsConfig {
|
||||
require_auth: true,
|
||||
keys: vec![
|
||||
ApiKeyConfig {
|
||||
key: "sk-balance".into(),
|
||||
account_id: "acct-a".into(),
|
||||
key_id: Some("key-balance".into()),
|
||||
hard_cap: Some(1_000),
|
||||
window: CapWindow::Balance,
|
||||
},
|
||||
ApiKeyConfig {
|
||||
key: "sk-rolling".into(),
|
||||
account_id: "acct-b".into(),
|
||||
key_id: Some("key-rolling".into()),
|
||||
hard_cap: Some(500),
|
||||
window: CapWindow::Rolling { seconds: 3_600 },
|
||||
},
|
||||
ApiKeyConfig {
|
||||
key: "sk-infra".into(),
|
||||
account_id: "operator".into(),
|
||||
key_id: Some("key-infra".into()),
|
||||
hard_cap: None,
|
||||
window: CapWindow::Balance,
|
||||
},
|
||||
],
|
||||
};
|
||||
LocalEntitlementProvider::from_config(&config)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resolves_configured_key_to_principal() {
|
||||
let p = provider();
|
||||
let principal = p.resolve("sk-balance").await.expect("known key resolves");
|
||||
assert_eq!(principal.account_id, "acct-a");
|
||||
assert_eq!(principal.key_id, "key-balance");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_key_is_invalid() {
|
||||
let p = provider();
|
||||
assert!(matches!(
|
||||
p.resolve("sk-nope").await,
|
||||
Err(AuthError::InvalidKey)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reserve_settle_release_round_trip() {
|
||||
let p = provider();
|
||||
let principal = p.resolve("sk-balance").await.unwrap();
|
||||
|
||||
let r = p.reserve(&principal, 400).await.expect("within cap");
|
||||
// Reserved, not yet spent.
|
||||
let snap = p.snapshot(&principal).await.unwrap();
|
||||
assert_eq!(snap.hard_cap, Some(1_000));
|
||||
assert_eq!(snap.reserved, 400);
|
||||
assert_eq!(snap.spent, 0);
|
||||
|
||||
// Used fewer tokens than reserved → remainder released, spend exact.
|
||||
p.settle(r, 250).await;
|
||||
let snap = p.snapshot(&principal).await.unwrap();
|
||||
assert_eq!(snap.reserved, 0);
|
||||
assert_eq!(snap.spent, 250);
|
||||
|
||||
// A reservation that is released contributes no spend.
|
||||
let r2 = p.reserve(&principal, 100).await.unwrap();
|
||||
p.release(r2).await;
|
||||
let snap = p.snapshot(&principal).await.unwrap();
|
||||
assert_eq!(snap.reserved, 0);
|
||||
assert_eq!(snap.spent, 250);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn balance_over_cap_is_insufficient_quota() {
|
||||
let p = provider();
|
||||
let principal = p.resolve("sk-balance").await.unwrap();
|
||||
// Reserve most of the cap, then ask for more than remains.
|
||||
let _r = p.reserve(&principal, 900).await.unwrap();
|
||||
let err = p.reserve(&principal, 200).await.expect_err("over cap");
|
||||
match err {
|
||||
BudgetError::InsufficientQuota {
|
||||
requested,
|
||||
available,
|
||||
} => {
|
||||
assert_eq!(requested, 200);
|
||||
assert_eq!(available, 100);
|
||||
}
|
||||
other => panic!("expected InsufficientQuota, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rolling_over_cap_is_rate_limited_with_retry_after() {
|
||||
let p = provider();
|
||||
let principal = p.resolve("sk-rolling").await.unwrap();
|
||||
let _r = p.reserve(&principal, 500).await.unwrap();
|
||||
let err = p.reserve(&principal, 1).await.expect_err("over cap");
|
||||
match err {
|
||||
BudgetError::RateLimited {
|
||||
retry_after_secs, ..
|
||||
} => {
|
||||
assert!(retry_after_secs >= 1, "must advertise a retry hint");
|
||||
assert!(retry_after_secs <= 3_600);
|
||||
}
|
||||
other => panic!("expected RateLimited, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn uncapped_infra_key_never_refuses() {
|
||||
let p = provider();
|
||||
let principal = p.resolve("sk-infra").await.unwrap();
|
||||
let r = p.reserve(&principal, 10_000_000).await.expect("uncapped");
|
||||
p.settle(r, 10_000_000).await;
|
||||
let snap = p.snapshot(&principal).await.unwrap();
|
||||
assert_eq!(snap.hard_cap, None);
|
||||
assert_eq!(snap.spent, 10_000_000);
|
||||
}
|
||||
}
|
||||
@@ -1,246 +0,0 @@
|
||||
//! helexa-upstream client (#57): an [`EntitlementProvider`] that resolves
|
||||
//! keys and reserves/settles budget against the mesh authority's
|
||||
//! `/authz/v1` surface (B2). It is "just another impl of the trait" — cortex
|
||||
//! enforcement (`auth.rs`, `metering.rs`) is unchanged.
|
||||
//!
|
||||
//! **Fail closed.** When upstream is unreachable, `resolve` returns
|
||||
//! [`AuthError::Unavailable`] (→ `503`, never `401`) and `reserve` refuses
|
||||
//! with a retryable [`BudgetError::RateLimited`] — a request is never served
|
||||
//! on an un-authorized key, and a real key is never rejected as invalid
|
||||
//! during a blip.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use cortex_core::config::UpstreamClientConfig;
|
||||
use cortex_core::entitlements::{
|
||||
AuthError, BudgetError, BudgetSnapshot, EntitlementProvider, Principal, Reservation,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use std::time::Duration;
|
||||
|
||||
/// Retry-After (seconds) advertised when we fail closed on an upstream
|
||||
/// outage.
|
||||
const FAIL_CLOSED_RETRY_SECS: u64 = 5;
|
||||
|
||||
pub struct UpstreamEntitlementProvider {
|
||||
client: reqwest::Client,
|
||||
base_url: String,
|
||||
bearer: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct PrincipalDto {
|
||||
account_id: String,
|
||||
key_id: String,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct SnapshotDto {
|
||||
hard_cap: Option<u64>,
|
||||
spent: u64,
|
||||
reserved: u64,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct ResolveResp {
|
||||
principal: PrincipalDto,
|
||||
#[allow(dead_code)]
|
||||
snapshot: Option<SnapshotDto>,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct ReserveResp {
|
||||
reservation_id: Option<i64>,
|
||||
rejected: Option<Rejection>,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
enum Rejection {
|
||||
InsufficientQuota {
|
||||
requested: u64,
|
||||
available: u64,
|
||||
},
|
||||
RateLimited {
|
||||
requested: u64,
|
||||
available: u64,
|
||||
retry_after_secs: u64,
|
||||
},
|
||||
}
|
||||
|
||||
impl UpstreamEntitlementProvider {
|
||||
pub fn new(cfg: &UpstreamClientConfig) -> Self {
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(cfg.timeout_secs))
|
||||
.build()
|
||||
.expect("failed to build upstream HTTP client");
|
||||
Self {
|
||||
client,
|
||||
base_url: cfg.url.trim_end_matches('/').to_string(),
|
||||
bearer: cfg.bearer.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
fn url(&self, path: &str) -> String {
|
||||
format!("{}{}", self.base_url, path)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl EntitlementProvider for UpstreamEntitlementProvider {
|
||||
async fn resolve(&self, api_key: &str) -> Result<Principal, AuthError> {
|
||||
let resp = self
|
||||
.client
|
||||
.post(self.url("/authz/v1/resolve"))
|
||||
.bearer_auth(&self.bearer)
|
||||
.json(&serde_json::json!({ "api_key": api_key }))
|
||||
.send()
|
||||
.await;
|
||||
let resp = match resp {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "upstream resolve unreachable; failing closed");
|
||||
return Err(AuthError::Unavailable {
|
||||
retry_after_secs: FAIL_CLOSED_RETRY_SECS,
|
||||
});
|
||||
}
|
||||
};
|
||||
if resp.status().as_u16() == 401 {
|
||||
return Err(AuthError::InvalidKey);
|
||||
}
|
||||
if !resp.status().is_success() {
|
||||
return Err(AuthError::Unavailable {
|
||||
retry_after_secs: FAIL_CLOSED_RETRY_SECS,
|
||||
});
|
||||
}
|
||||
match resp.json::<ResolveResp>().await {
|
||||
Ok(r) => Ok(Principal {
|
||||
account_id: r.principal.account_id,
|
||||
key_id: r.principal.key_id,
|
||||
}),
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "upstream resolve: bad body; failing closed");
|
||||
Err(AuthError::Unavailable {
|
||||
retry_after_secs: FAIL_CLOSED_RETRY_SECS,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn reserve(
|
||||
&self,
|
||||
principal: &Principal,
|
||||
max_tokens: u64,
|
||||
) -> Result<Reservation, BudgetError> {
|
||||
let fail_closed = || BudgetError::RateLimited {
|
||||
requested: max_tokens,
|
||||
available: 0,
|
||||
retry_after_secs: FAIL_CLOSED_RETRY_SECS,
|
||||
};
|
||||
let resp = self
|
||||
.client
|
||||
.post(self.url("/authz/v1/reserve"))
|
||||
.bearer_auth(&self.bearer)
|
||||
.json(&serde_json::json!({
|
||||
"account_id": principal.account_id,
|
||||
"key_id": principal.key_id,
|
||||
"max_tokens": max_tokens,
|
||||
}))
|
||||
.send()
|
||||
.await;
|
||||
let resp = match resp {
|
||||
Ok(r) if r.status().is_success() => r,
|
||||
Ok(r) => {
|
||||
tracing::warn!(status = %r.status(), "upstream reserve non-2xx; failing closed");
|
||||
return Err(fail_closed());
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "upstream reserve unreachable; failing closed");
|
||||
return Err(fail_closed());
|
||||
}
|
||||
};
|
||||
match resp.json::<ReserveResp>().await {
|
||||
Ok(ReserveResp {
|
||||
reservation_id: Some(id),
|
||||
..
|
||||
}) => Ok(Reservation {
|
||||
id: id as u64,
|
||||
principal: principal.clone(),
|
||||
reserved: max_tokens,
|
||||
}),
|
||||
Ok(ReserveResp {
|
||||
rejected:
|
||||
Some(Rejection::InsufficientQuota {
|
||||
requested,
|
||||
available,
|
||||
}),
|
||||
..
|
||||
}) => Err(BudgetError::InsufficientQuota {
|
||||
requested,
|
||||
available,
|
||||
}),
|
||||
Ok(ReserveResp {
|
||||
rejected:
|
||||
Some(Rejection::RateLimited {
|
||||
requested,
|
||||
available,
|
||||
retry_after_secs,
|
||||
}),
|
||||
..
|
||||
}) => Err(BudgetError::RateLimited {
|
||||
requested,
|
||||
available,
|
||||
retry_after_secs,
|
||||
}),
|
||||
_ => Err(fail_closed()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn settle(&self, reservation: Reservation, actual_tokens: u64) {
|
||||
// Best-effort; a lost settle is reaped by the upstream sweeper (B2).
|
||||
let _ = self
|
||||
.client
|
||||
.post(self.url("/authz/v1/settle"))
|
||||
.bearer_auth(&self.bearer)
|
||||
.json(&serde_json::json!({
|
||||
"reservation_id": reservation.id as i64,
|
||||
"actual_tokens": actual_tokens,
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.inspect_err(
|
||||
|e| tracing::warn!(error = %e, "upstream settle failed (sweeper will reap)"),
|
||||
);
|
||||
}
|
||||
|
||||
async fn release(&self, reservation: Reservation) {
|
||||
let _ = self
|
||||
.client
|
||||
.post(self.url("/authz/v1/release"))
|
||||
.bearer_auth(&self.bearer)
|
||||
.json(&serde_json::json!({ "reservation_id": reservation.id as i64 }))
|
||||
.send()
|
||||
.await
|
||||
.inspect_err(
|
||||
|e| tracing::warn!(error = %e, "upstream release failed (sweeper will reap)"),
|
||||
);
|
||||
}
|
||||
|
||||
async fn snapshot(&self, principal: &Principal) -> Option<BudgetSnapshot> {
|
||||
let resp = self
|
||||
.client
|
||||
.post(self.url("/authz/v1/snapshot"))
|
||||
.bearer_auth(&self.bearer)
|
||||
.json(&serde_json::json!({
|
||||
"account_id": principal.account_id,
|
||||
"key_id": principal.key_id,
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.ok()?;
|
||||
if !resp.status().is_success() {
|
||||
return None;
|
||||
}
|
||||
let dto = resp.json::<SnapshotDto>().await.ok()?;
|
||||
Some(BudgetSnapshot {
|
||||
hard_cap: dto.hard_cap,
|
||||
spent: dto.spent,
|
||||
reserved: dto.reserved,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
//! Gateway adapter that turns the shared, axum-agnostic
|
||||
//! [`cortex_core::error_envelope::OpenAiError`] into an axum [`Response`],
|
||||
//! setting the `Retry-After` header when the envelope carries one.
|
||||
//!
|
||||
//! cortex-core owns the envelope shape and the rejection contract (#60/#63);
|
||||
//! this is the only place the gateway crosses from that data into axum.
|
||||
|
||||
use axum::http::{HeaderValue, StatusCode, header};
|
||||
use axum::response::{IntoResponse, Json, Response};
|
||||
use cortex_core::error_envelope::OpenAiError;
|
||||
|
||||
/// Render an [`OpenAiError`] as an axum response (status + JSON envelope +
|
||||
/// optional `Retry-After`).
|
||||
pub fn envelope_response(err: OpenAiError) -> Response {
|
||||
let status = StatusCode::from_u16(err.status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
|
||||
let retry_after = err.retry_after_secs;
|
||||
let mut response = (status, Json(err.body())).into_response();
|
||||
if let Some(secs) = retry_after
|
||||
&& let Ok(value) = HeaderValue::from_str(&secs.to_string())
|
||||
{
|
||||
response.headers_mut().insert(header::RETRY_AFTER, value);
|
||||
}
|
||||
response
|
||||
}
|
||||
@@ -17,23 +17,11 @@ pub async fn eviction_loop(fleet: Arc<CortexState>) {
|
||||
}
|
||||
}
|
||||
|
||||
/// Evict the least-recently-used model on a given node that `incoming`
|
||||
/// is permitted to displace.
|
||||
///
|
||||
/// `incoming` is the model the caller is trying to make room for. It is
|
||||
/// load-bearing, not diagnostic: whether a displacement is allowed is a
|
||||
/// question about the *pair*, so an evictor that only knows the victim
|
||||
/// cannot answer it. Passing `None` means "no particular model" and
|
||||
/// permits displacing anything at or below the default priority, which
|
||||
/// is the right reading for maintenance-driven eviction rather than a
|
||||
/// cold-load making room for itself.
|
||||
///
|
||||
/// Returns the model ID that was evicted, or None when nothing on the
|
||||
/// node may be displaced.
|
||||
/// Evict the least-recently-used model on a given node.
|
||||
/// Returns the model ID that was evicted, or None if nothing could be evicted.
|
||||
pub async fn evict_lru_on_node(
|
||||
fleet: &CortexState,
|
||||
node_name: &str,
|
||||
incoming: Option<&str>,
|
||||
) -> anyhow::Result<Option<String>> {
|
||||
let (neuron_endpoint, candidate) = {
|
||||
let nodes = fleet.nodes.read().await;
|
||||
@@ -41,18 +29,13 @@ pub async fn evict_lru_on_node(
|
||||
anyhow::bail!("node '{node_name}' not found");
|
||||
};
|
||||
|
||||
// Oldest first, among the models this incoming model outranks.
|
||||
// Find the loaded model with the oldest last_accessed,
|
||||
// excluding models pinned on this neuron (from catalogue).
|
||||
let candidate = node
|
||||
.models
|
||||
.values()
|
||||
.filter(|m| m.status == ModelStatus::Loaded)
|
||||
.filter(|m| match incoming {
|
||||
Some(inc) => fleet.catalogue.may_displace(inc, &m.id),
|
||||
None => {
|
||||
fleet.catalogue.residency_priority(&m.id)
|
||||
<= cortex_core::catalogue::DEFAULT_RESIDENCY_PRIORITY
|
||||
}
|
||||
})
|
||||
.filter(|m| !fleet.catalogue.is_pinned(&m.id, node_name))
|
||||
.min_by_key(|m| m.last_accessed)
|
||||
.map(|m| m.id.clone());
|
||||
|
||||
@@ -60,11 +43,7 @@ pub async fn evict_lru_on_node(
|
||||
};
|
||||
|
||||
let Some(model_id) = candidate else {
|
||||
tracing::info!(
|
||||
node = node_name,
|
||||
incoming = incoming.unwrap_or("(none)"),
|
||||
"no displaceable models found"
|
||||
);
|
||||
tracing::info!(node = node_name, "no evictable models found");
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,41 +1,22 @@
|
||||
pub mod anthropic_sse;
|
||||
pub mod auth;
|
||||
pub mod entitlements_chain;
|
||||
pub mod entitlements_local;
|
||||
pub mod entitlements_upstream;
|
||||
pub mod error;
|
||||
pub mod evictor;
|
||||
pub mod handlers;
|
||||
pub mod metering;
|
||||
pub mod metrics;
|
||||
pub mod poller;
|
||||
pub mod proxy;
|
||||
pub mod router;
|
||||
pub mod served_usage;
|
||||
pub mod state;
|
||||
|
||||
use anyhow::Result;
|
||||
use axum::Router;
|
||||
use axum::middleware::from_fn_with_state;
|
||||
use cortex_core::config::GatewayConfig;
|
||||
use std::sync::Arc;
|
||||
use tower_http::cors::CorsLayer;
|
||||
use tower_http::trace::TraceLayer;
|
||||
|
||||
/// Build the Axum application router with all routes wired up.
|
||||
///
|
||||
/// Layer order (outermost first): trace → CORS → auth → handlers. CORS is
|
||||
/// outer to auth so preflight `OPTIONS` short-circuits before resolution;
|
||||
/// auth (`require_principal`) resolves the bearer key, attaches the
|
||||
/// principal, and stamps the internal principal headers before any handler
|
||||
/// runs.
|
||||
pub fn build_app(fleet: Arc<state::CortexState>) -> Router {
|
||||
Router::new()
|
||||
.merge(handlers::api_routes())
|
||||
.layer(from_fn_with_state(
|
||||
Arc::clone(&fleet),
|
||||
auth::require_principal,
|
||||
))
|
||||
.layer(CorsLayer::permissive())
|
||||
.layer(TraceLayer::new_for_http())
|
||||
.with_state(fleet)
|
||||
@@ -58,28 +39,6 @@ pub async fn run(config: GatewayConfig) -> Result<()> {
|
||||
evictor::eviction_loop(evictor_fleet).await;
|
||||
});
|
||||
|
||||
// Served-usage reporter (#58): when this operator is part of the mesh,
|
||||
// periodically flush absolute per-principal served-token counters to
|
||||
// upstream for reconciliation.
|
||||
if config.upstream.enabled {
|
||||
let su_fleet = Arc::clone(&fleet);
|
||||
let url = config.upstream.url.clone();
|
||||
let bearer = config.upstream.bearer.clone();
|
||||
let interval =
|
||||
std::time::Duration::from_secs(config.upstream.served_usage_report_interval_secs);
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::time::sleep(interval).await;
|
||||
let rows = su_fleet.served_usage.snapshot();
|
||||
if let Err(e) =
|
||||
served_usage::report(&su_fleet.http_client, &url, &bearer, &rows).await
|
||||
{
|
||||
tracing::warn!(error = %e, "served-usage report failed (will retry)");
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let app = build_app(Arc::clone(&fleet));
|
||||
|
||||
let listen_addr = config.gateway.listen.parse::<std::net::SocketAddr>()?;
|
||||
|
||||
@@ -1,298 +0,0 @@
|
||||
//! Per-request token metering (#51).
|
||||
//!
|
||||
//! Captures the real `(prompt, completion)` usage of every request and feeds
|
||||
//! it to two places: the [`EntitlementProvider`] spend ledger (via
|
||||
//! reserve→settle) and per-principal Prometheus counters. The principal is
|
||||
//! reconstructed from the internal headers the auth middleware stamped (#49),
|
||||
//! so this works uniformly across every proxy path without threading the
|
||||
//! typed principal through each handler.
|
||||
//!
|
||||
//! The reserve→settle lifecycle is established here but, in this phase,
|
||||
//! reserves **zero** tokens — metering only, no enforcement. Budget
|
||||
//! enforcement (#52) flips the reserved amount to the real
|
||||
//! `prompt + max_output` and handles the [`BudgetError`] rejection; the
|
||||
//! settle/release plumbing is identical, so that change is localized.
|
||||
//!
|
||||
//! [`ReservationGuard`] makes leaks impossible: settling records actual
|
||||
//! spend and releases the unused remainder; dropping a guard that was never
|
||||
//! settled releases the whole reservation. So an early return, error path,
|
||||
//! or dropped stream can't strand a reservation.
|
||||
|
||||
use axum::http::HeaderMap;
|
||||
use cortex_core::entitlements::{
|
||||
BudgetError, EntitlementProvider, HEADER_ACCOUNT_ID, HEADER_KEY_ID, Principal,
|
||||
};
|
||||
use cortex_core::error_envelope::OpenAiError;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Fallback output-token budget when neither the request nor the model's
|
||||
/// advertised limit gives one. Bounds the reservation so a capped key is
|
||||
/// still gated even on under-specified requests (#52).
|
||||
pub const FALLBACK_MAX_OUTPUT: u64 = 4096;
|
||||
|
||||
/// Invoked exactly once at request completion with best-effort
|
||||
/// `(prompt_tokens, completion_tokens)`. When no usage could be observed
|
||||
/// (e.g. a pre-dispatch failure or a dropped stream) it is dropped unused —
|
||||
/// which releases the held reservation via [`ReservationGuard`]'s `Drop`.
|
||||
pub type UsageSink = Box<dyn FnOnce(u64, u64) + Send>;
|
||||
|
||||
/// Reconstruct the principal from the cortex-stamped internal headers. The
|
||||
/// auth middleware strips any client copy and stamps the authoritative value,
|
||||
/// so these headers are trustworthy within cortex. `None` for anonymous
|
||||
/// (unauthenticated) requests.
|
||||
pub fn principal_from_headers(headers: &HeaderMap) -> Option<Principal> {
|
||||
let account_id = headers.get(HEADER_ACCOUNT_ID)?.to_str().ok()?.to_string();
|
||||
let key_id = headers.get(HEADER_KEY_ID)?.to_str().ok()?.to_string();
|
||||
Some(Principal { account_id, key_id })
|
||||
}
|
||||
|
||||
/// Emit per-principal spend counters (#51). Labelled by account/key only —
|
||||
/// both are operator-bounded, so cardinality is controlled.
|
||||
pub fn record_spend(principal: &Principal, prompt: u64, completion: u64) {
|
||||
let labels = [
|
||||
("account", principal.account_id.clone()),
|
||||
("key", principal.key_id.clone()),
|
||||
];
|
||||
metrics::counter!("cortex_spend_tokens_total", &labels).increment(prompt + completion);
|
||||
metrics::counter!("cortex_spend_prompt_tokens_total", &labels).increment(prompt);
|
||||
metrics::counter!("cortex_spend_completion_tokens_total", &labels).increment(completion);
|
||||
}
|
||||
|
||||
/// Holds a budget reservation for the life of a request. [`settle`] records
|
||||
/// actual spend and releases the remainder; an un-settled guard releases the
|
||||
/// whole reservation when dropped. Anonymous requests carry an empty guard,
|
||||
/// where every operation is a no-op.
|
||||
///
|
||||
/// [`settle`]: ReservationGuard::settle
|
||||
pub struct ReservationGuard {
|
||||
provider: Arc<dyn EntitlementProvider>,
|
||||
reservation: Option<cortex_core::entitlements::Reservation>,
|
||||
}
|
||||
|
||||
impl ReservationGuard {
|
||||
/// An empty guard for an anonymous request — no reservation to resolve.
|
||||
pub fn anonymous(provider: Arc<dyn EntitlementProvider>) -> Self {
|
||||
Self {
|
||||
provider,
|
||||
reservation: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Wrap an already-acquired reservation.
|
||||
fn held(
|
||||
provider: Arc<dyn EntitlementProvider>,
|
||||
reservation: cortex_core::entitlements::Reservation,
|
||||
) -> Self {
|
||||
Self {
|
||||
provider,
|
||||
reservation: Some(reservation),
|
||||
}
|
||||
}
|
||||
|
||||
/// Settle with the tokens actually consumed, disarming the drop-release.
|
||||
/// Spawns the (fast, in-process for the local provider) settle so the
|
||||
/// caller — which may be a sync stream-completion callback — needn't
|
||||
/// await.
|
||||
pub fn settle(mut self, actual_tokens: u64) {
|
||||
if let Some(reservation) = self.reservation.take() {
|
||||
let provider = Arc::clone(&self.provider);
|
||||
tokio::spawn(async move {
|
||||
provider.settle(reservation, actual_tokens).await;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ReservationGuard {
|
||||
fn drop(&mut self) {
|
||||
if let Some(reservation) = self.reservation.take() {
|
||||
let provider = Arc::clone(&self.provider);
|
||||
tokio::spawn(async move {
|
||||
provider.release(reservation).await;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the completion sink for an authenticated request: record spend and
|
||||
/// settle the reservation with the observed total. Dropping it unused (no
|
||||
/// usage observed) releases the reservation via the guard.
|
||||
pub fn usage_sink(
|
||||
principal: Principal,
|
||||
guard: ReservationGuard,
|
||||
served_usage: std::sync::Arc<crate::served_usage::ServedUsage>,
|
||||
) -> UsageSink {
|
||||
Box::new(move |prompt, completion| {
|
||||
record_spend(&principal, prompt, completion);
|
||||
// Per-principal served-usage tally for #58 reconciliation. Recorded
|
||||
// for every metered (authenticated) request; the flush task reports
|
||||
// it to upstream when the operator is part of the mesh.
|
||||
served_usage.add(
|
||||
&principal.account_id,
|
||||
&principal.key_id,
|
||||
prompt + completion,
|
||||
);
|
||||
guard.settle(prompt + completion);
|
||||
})
|
||||
}
|
||||
|
||||
/// Reserve the request's upper-bound token cost for the principal, refusing
|
||||
/// *before* dispatch if it would exceed the hard cap (#52). On success
|
||||
/// returns a guard the caller settles with actual usage; on refusal returns
|
||||
/// the #63 envelope (`rate_limit_exceeded` + `Retry-After` for a resetting
|
||||
/// window, `insufficient_quota` for a hard balance — never `402`).
|
||||
pub async fn reserve_or_reject(
|
||||
provider: Arc<dyn EntitlementProvider>,
|
||||
principal: &Principal,
|
||||
max_tokens: u64,
|
||||
) -> Result<ReservationGuard, OpenAiError> {
|
||||
match provider.reserve(principal, max_tokens).await {
|
||||
Ok(reservation) => Ok(ReservationGuard::held(provider, reservation)),
|
||||
Err(err) => Err(budget_error_to_envelope(err)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Budget-equivalence rate for image generation (#202): how many
|
||||
/// token-budget units one megapixel-step consumes. The metered unit
|
||||
/// IS megapixel-steps (`usage.helexa_image_units`, reported end to
|
||||
/// end); this rate only bridges it into the single-currency budget
|
||||
/// ledger the clearing house runs today, so per-key caps and
|
||||
/// fail-closed enforcement apply to images without an upstream
|
||||
/// protocol change. A native image unit in the clearing-house
|
||||
/// contract is the follow-up recorded on #202.
|
||||
///
|
||||
/// 1000 tokens/Mp-step ≈ a 9-step 1024² image (9.44 units) costing
|
||||
/// ~9.4k tokens of budget — the same order as a large chat turn.
|
||||
pub const TOKENS_PER_IMAGE_UNIT: u64 = 1000;
|
||||
|
||||
/// Upper-bound megapixel-steps for an images request (#202): parsed
|
||||
/// size (default 1024²) × steps (default 9), doubled when a negative
|
||||
/// prompt requests CFG, floored at one unit. Over-reserving is safe —
|
||||
/// settle corrects to the actual `usage.helexa_image_units`.
|
||||
pub fn image_reservation_units(body: &[u8]) -> f64 {
|
||||
let (mut width, mut height, mut steps, mut cfg) = (1024usize, 1024usize, 9usize, false);
|
||||
if let Ok(v) = serde_json::from_slice::<serde_json::Value>(body) {
|
||||
if let Some(size) = v.get("size").and_then(serde_json::Value::as_str)
|
||||
&& let Some((w, h)) = size.split_once(['x', 'X'])
|
||||
&& let (Ok(w), Ok(h)) = (w.trim().parse(), h.trim().parse())
|
||||
{
|
||||
width = w;
|
||||
height = h;
|
||||
}
|
||||
if let Some(n) = v.get("num_steps").and_then(serde_json::Value::as_u64) {
|
||||
steps = n as usize;
|
||||
}
|
||||
cfg = v
|
||||
.get("negative_prompt")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.is_some_and(|s| !s.is_empty());
|
||||
}
|
||||
let mp = (width as f64 * height as f64) / 1_000_000.0;
|
||||
let units = mp * steps as f64 * if cfg { 2.0 } else { 1.0 };
|
||||
units.max(1.0)
|
||||
}
|
||||
|
||||
/// Convert megapixel-steps to budget tokens, rounding up so fractional
|
||||
/// units still cost at least their share.
|
||||
pub fn image_units_to_tokens(units: f64) -> u64 {
|
||||
(units * TOKENS_PER_IMAGE_UNIT as f64).ceil() as u64
|
||||
}
|
||||
|
||||
/// Record per-principal image spend (#202) and settle the reservation
|
||||
/// with the actual generated units.
|
||||
pub fn settle_image_usage(
|
||||
principal: &Principal,
|
||||
guard: ReservationGuard,
|
||||
served_usage: &crate::served_usage::ServedUsage,
|
||||
actual_units: f64,
|
||||
) {
|
||||
let labels = [
|
||||
("account", principal.account_id.clone()),
|
||||
("key", principal.key_id.clone()),
|
||||
];
|
||||
// Milli-units keep the counter integral without losing precision
|
||||
// that matters at reconciliation scale.
|
||||
metrics::counter!("cortex_spend_image_milliunits_total", &labels)
|
||||
.increment((actual_units * 1000.0).round() as u64);
|
||||
let tokens = image_units_to_tokens(actual_units);
|
||||
served_usage.add(&principal.account_id, &principal.key_id, tokens);
|
||||
guard.settle(tokens);
|
||||
}
|
||||
|
||||
/// Map a [`BudgetError`] to the #63 envelope. The provider chose the window
|
||||
/// semantics; this only translates them to HTTP.
|
||||
fn budget_error_to_envelope(err: BudgetError) -> OpenAiError {
|
||||
match err {
|
||||
BudgetError::RateLimited {
|
||||
retry_after_secs, ..
|
||||
} => OpenAiError::rate_limit_exceeded(err.to_string(), retry_after_secs),
|
||||
BudgetError::InsufficientQuota { .. } => OpenAiError::insufficient_quota(err.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Upper-bound tokens to reserve for a request (#52): an over-estimate of
|
||||
/// the prompt plus the maximum output. `advertised_output` is the model's
|
||||
/// `limit.output` (#62), used when the request omits `max_(completion_)tokens`.
|
||||
/// Over-reserving is safe — settle corrects spend to the actual usage.
|
||||
pub fn reservation_estimate(body: &[u8], advertised_output: Option<u64>) -> u64 {
|
||||
let max_output = requested_max_output(body)
|
||||
.or(advertised_output)
|
||||
.unwrap_or(FALLBACK_MAX_OUTPUT);
|
||||
estimate_prompt_tokens(body).saturating_add(max_output)
|
||||
}
|
||||
|
||||
/// The client's requested output cap, from `max_completion_tokens` (or the
|
||||
/// legacy `max_tokens`). `None` when unspecified.
|
||||
fn requested_max_output(body: &[u8]) -> Option<u64> {
|
||||
let v: serde_json::Value = serde_json::from_slice(body).ok()?;
|
||||
v.get("max_completion_tokens")
|
||||
.or_else(|| v.get("max_tokens"))
|
||||
.and_then(serde_json::Value::as_u64)
|
||||
}
|
||||
|
||||
/// Rough prompt-token estimate at ~4 chars/token over the whole body. cortex
|
||||
/// has no tokenizer; JSON overhead makes this a conservative over-estimate,
|
||||
/// and neuron remains the exact context wall (#56/#60). Settle reconciles to
|
||||
/// the real usage afterward.
|
||||
fn estimate_prompt_tokens(body: &[u8]) -> u64 {
|
||||
(body.len() as u64 / 4).max(1)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn requested_max_output_prefers_max_completion_tokens() {
|
||||
let body = br#"{"model":"m","max_completion_tokens":256,"max_tokens":99}"#;
|
||||
assert_eq!(requested_max_output(body), Some(256));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn requested_max_output_falls_back_to_legacy_max_tokens() {
|
||||
let body = br#"{"model":"m","max_tokens":128}"#;
|
||||
assert_eq!(requested_max_output(body), Some(128));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn estimate_uses_requested_output_when_present() {
|
||||
// Requested output dominates; prompt estimate is small for a tiny body.
|
||||
let body = br#"{"model":"m","max_tokens":1000}"#;
|
||||
let est = reservation_estimate(body, Some(8192));
|
||||
assert!(est >= 1000 && est < 1100, "est was {est}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn estimate_uses_advertised_output_when_request_omits_it() {
|
||||
let body = br#"{"model":"m","messages":[]}"#;
|
||||
let est = reservation_estimate(body, Some(8192));
|
||||
assert!(est >= 8192, "est was {est}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn estimate_falls_back_when_nothing_advertised() {
|
||||
let body = br#"{"model":"m"}"#;
|
||||
let est = reservation_estimate(body, None);
|
||||
assert!(est >= FALLBACK_MAX_OUTPUT, "est was {est}");
|
||||
}
|
||||
}
|
||||
@@ -46,14 +46,6 @@ fn describe_metrics() {
|
||||
"Generation throughput in tokens per second"
|
||||
);
|
||||
metrics::describe_counter!("cortex_requests_total", "Total number of proxied requests");
|
||||
metrics::describe_counter!(
|
||||
"cortex_prompt_tokens_total",
|
||||
"Total prompt tokens reported by upstream usage objects"
|
||||
);
|
||||
metrics::describe_counter!(
|
||||
"cortex_completion_tokens_total",
|
||||
"Total completion tokens reported by upstream usage objects"
|
||||
);
|
||||
metrics::describe_counter!(
|
||||
"cortex_request_errors_total",
|
||||
"Total number of failed proxy requests"
|
||||
@@ -63,70 +55,4 @@ fn describe_metrics() {
|
||||
"cortex_cold_starts_total",
|
||||
"Total number of cold-start model loads"
|
||||
);
|
||||
metrics::describe_counter!(
|
||||
"cortex_spend_tokens_total",
|
||||
"Total metered tokens (prompt + completion) per principal, labelled by account/key (#51)"
|
||||
);
|
||||
metrics::describe_counter!(
|
||||
"cortex_spend_prompt_tokens_total",
|
||||
"Metered prompt tokens per principal, labelled by account/key (#51)"
|
||||
);
|
||||
metrics::describe_counter!(
|
||||
"cortex_spend_completion_tokens_total",
|
||||
"Metered completion tokens per principal, labelled by account/key (#51)"
|
||||
);
|
||||
// Live capacity signals polled from neuron /health (#137), {node,model}.
|
||||
metrics::describe_gauge!(
|
||||
"cortex_model_in_flight",
|
||||
"Requests currently running on a neuron:model (#137)"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
"cortex_model_queue_depth",
|
||||
"Requests queued in admission for a neuron:model (#137)"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
"cortex_model_max_in_flight",
|
||||
"Configured concurrency ceiling; saturation = in_flight / max_in_flight (#137)"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
"cortex_model_max_queue_depth",
|
||||
"Configured admission queue capacity before a neuron:model sheds load (#137)"
|
||||
);
|
||||
// Per-device GPU headroom polled from neuron /health (#137), {node,device}.
|
||||
metrics::describe_gauge!(
|
||||
"cortex_device_vram_used_mb",
|
||||
"Per-device VRAM used, MB (#137)"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
"cortex_device_vram_free_mb",
|
||||
"Per-device VRAM free, MB (#137)"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
"cortex_device_utilization_pct",
|
||||
"Per-device GPU utilization, percent (#137)"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
"cortex_device_temp_c",
|
||||
"Per-device GPU temperature, Celsius (#137)"
|
||||
);
|
||||
metrics::describe_counter!(
|
||||
"cortex_model_rejections_total",
|
||||
"Admission rejections per neuron:model by reason: queue_full / wait_timeout / per_principal / anon_yield — the load-shedding signal (#137, #262)"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
"cortex_model_anon_in_flight",
|
||||
"Anonymous (unattributable) requests holding a seat on a neuron:model (#262)"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
"cortex_model_anon_max_in_flight",
|
||||
"Seats anonymous traffic may hold at once, so it cannot starve identified callers (#262)"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
"cortex_model_tok_s_decode",
|
||||
"Live decode throughput per neuron:model, tokens/sec EMA — the headline capacity number (#137)"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
"cortex_model_tok_s_prefill",
|
||||
"Live prefill throughput per neuron:model, tokens/sec EMA (#137)"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -3,38 +3,13 @@
|
||||
|
||||
use crate::state::CortexState;
|
||||
use chrono::Utc;
|
||||
use cortex_core::discovery::{DiscoveryResponse, HealthResponse};
|
||||
use cortex_core::harness::ModelInfo;
|
||||
use cortex_core::node::{ModelEntry, ModelStatus, NodeState};
|
||||
use metrics::{counter, gauge};
|
||||
use cortex_core::node::{ModelEntry, ModelStatus};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
const POLL_INTERVAL: Duration = Duration::from_secs(10);
|
||||
|
||||
/// Consecutive failed `/models` polls before a node is marked unhealthy.
|
||||
/// Debounces transient misses (a busy neuron briefly slow to answer) so a
|
||||
/// single blip can't yank a node — and its models — out of routing. At the
|
||||
/// 10s poll interval this tolerates ~20s of flapping before evicting.
|
||||
const POLL_FAILURE_THRESHOLD: u32 = 3;
|
||||
|
||||
/// How long before re-asking the fleet about a model whose capabilities
|
||||
/// nobody could derive (#241). Long, because the usual cause — no node
|
||||
/// has the weights cached — is resolved by a deliberate operator action
|
||||
/// (a load, a prefetch) rather than by time passing.
|
||||
const CAPABILITY_RETRY: Duration = Duration::from_secs(600);
|
||||
|
||||
/// Record a failed poll for `node`, marking it unhealthy only once failures
|
||||
/// reach [`POLL_FAILURE_THRESHOLD`]. Below the threshold the node keeps its
|
||||
/// last-known health, riding over transient misses. A successful poll resets
|
||||
/// the counter (see the success arm in `poll_once`).
|
||||
fn record_poll_failure(node: &mut NodeState) {
|
||||
node.consecutive_poll_failures = node.consecutive_poll_failures.saturating_add(1);
|
||||
if node.consecutive_poll_failures >= POLL_FAILURE_THRESHOLD {
|
||||
node.healthy = false;
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs forever, polling all neurons on a fixed interval.
|
||||
pub async fn poll_loop(fleet: Arc<CortexState>) {
|
||||
loop {
|
||||
@@ -50,68 +25,7 @@ pub async fn poll_once(fleet: &CortexState) {
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch `GET /discovery` and cache it on the NodeState — topology is
|
||||
/// invariant for a given neuron process, so a successful fetch is kept.
|
||||
/// Re-polled only while `max_prompt_tokens` is still unknown (0): on a
|
||||
/// rolling deploy cortex can win the race and cache a neuron's discovery
|
||||
/// before that neuron reports the field (it deserialises to 0). Re-polling
|
||||
/// until a real cap arrives self-heals that without periodic polling.
|
||||
async fn maybe_poll_discovery(fleet: &CortexState, name: &str, endpoint: &str) {
|
||||
{
|
||||
let nodes = fleet.nodes.read().await;
|
||||
match nodes.get(name) {
|
||||
Some(n)
|
||||
if n.discovery
|
||||
.as_ref()
|
||||
.is_some_and(|d| d.max_prompt_tokens > 0) =>
|
||||
{
|
||||
return;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let url = format!("{endpoint}/discovery");
|
||||
let resp = match fleet
|
||||
.http_client
|
||||
.get(&url)
|
||||
.timeout(Duration::from_secs(5))
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(r) if r.status().is_success() => r,
|
||||
Ok(r) => {
|
||||
tracing::debug!(node = name, status = %r.status(), "discovery probe non-success");
|
||||
return;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::debug!(node = name, error = %e, "discovery probe unreachable");
|
||||
return;
|
||||
}
|
||||
};
|
||||
match resp.json::<DiscoveryResponse>().await {
|
||||
Ok(d) => {
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
if let Some(node) = nodes.get_mut(name) {
|
||||
tracing::info!(
|
||||
node = name,
|
||||
hostname = %d.hostname,
|
||||
devices = d.devices.len(),
|
||||
"discovery cached"
|
||||
);
|
||||
node.discovery = Some(d);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(node = name, error = %e, "failed to parse /discovery response");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn poll_neuron(fleet: &CortexState, name: &str, endpoint: &str) {
|
||||
// Topology first — cheap once cached, and the router needs it to
|
||||
// route requests against catalogue entries that aren't loaded yet.
|
||||
maybe_poll_discovery(fleet, name, endpoint).await;
|
||||
|
||||
let url = format!("{endpoint}/models");
|
||||
|
||||
let result = fleet
|
||||
@@ -140,38 +54,25 @@ async fn poll_neuron(fleet: &CortexState, name: &str, endpoint: &str) {
|
||||
.and_modify(|e| {
|
||||
e.status = status;
|
||||
e.vram_estimate_mb = upstream.vram_used_mb;
|
||||
e.capabilities = upstream.capabilities.clone();
|
||||
e.tool_call = upstream.tool_call;
|
||||
e.reasoning = upstream.reasoning;
|
||||
// Neuron's self-derived limit (#67) — the
|
||||
// authoritative source the gateway advertises.
|
||||
e.limit = upstream.limit.clone();
|
||||
e.servable = upstream.servable.clone();
|
||||
})
|
||||
.or_insert_with(|| ModelEntry {
|
||||
id: upstream.id.clone(),
|
||||
status,
|
||||
last_accessed: None,
|
||||
vram_estimate_mb: upstream.vram_used_mb,
|
||||
capabilities: upstream.capabilities.clone(),
|
||||
tool_call: upstream.tool_call,
|
||||
reasoning: upstream.reasoning,
|
||||
limit: upstream.limit.clone(),
|
||||
servable: upstream.servable.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
// Remove models no longer reported by the neuron.
|
||||
node.models.retain(|id, _| seen.contains(id));
|
||||
|
||||
node.consecutive_poll_failures = 0;
|
||||
node.healthy = true;
|
||||
node.last_poll = Some(Utc::now());
|
||||
tracing::debug!(node = name, models = models.len(), "poll ok");
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(node = name, error = %e, "failed to parse /models response");
|
||||
record_poll_failure(node);
|
||||
node.healthy = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -181,232 +82,13 @@ async fn poll_neuron(fleet: &CortexState, name: &str, endpoint: &str) {
|
||||
status = %resp.status(),
|
||||
"neuron returned non-success status"
|
||||
);
|
||||
record_poll_failure(node);
|
||||
node.healthy = false;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(node = name, error = %e, "failed to reach neuron");
|
||||
record_poll_failure(node);
|
||||
node.healthy = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Release the write lock before the next HTTP call.
|
||||
drop(nodes);
|
||||
|
||||
discover_capabilities(fleet, name, endpoint).await;
|
||||
|
||||
// Poll /health for the activation snapshot. We don't want this to
|
||||
// flip the node to unhealthy on its own — a neuron that's serving
|
||||
// /models fine is still operational even if /health is briefly
|
||||
// unavailable — so failures are debug-level and leave the existing
|
||||
// activation reading in place.
|
||||
poll_health(fleet, name, endpoint).await;
|
||||
}
|
||||
|
||||
/// Ask a neuron what the catalogue models it has never loaded can do
|
||||
/// (#241).
|
||||
///
|
||||
/// A loaded model reports its modalities on every `/models` poll, but a
|
||||
/// cold one reported nothing at all — which is why image generation was
|
||||
/// invisible on `/v1/models`, the image model being evicted almost all
|
||||
/// of the time. The neuron derives the answer from its local model cache
|
||||
/// without loading anything.
|
||||
///
|
||||
/// Only ever asks about ids it has no answer for, so a steady fleet
|
||||
/// costs nothing after the first cycle: what a given model id can do is
|
||||
/// a property of the model, not of the moment. A 404 means this node has
|
||||
/// no local evidence — another node may still have the weights cached,
|
||||
/// so nothing is recorded and the id is retried against the next node.
|
||||
///
|
||||
/// Ids that stay unresolved back off to [`CAPABILITY_RETRY`] rather than
|
||||
/// being re-asked every cycle. A catalogue entry whose weights nobody has
|
||||
/// downloaded can never be answered, and at the poll interval that would
|
||||
/// be a permanent trickle of requests and 404s. The retry still exists
|
||||
/// because the answer *can* change: the first node to pull those weights
|
||||
/// starts being able to reply.
|
||||
async fn discover_capabilities(fleet: &CortexState, name: &str, endpoint: &str) {
|
||||
let unknown: Vec<String> = {
|
||||
let known = fleet.discovered_capabilities.read().await;
|
||||
let attempts = fleet.capability_probe_attempts.read().await;
|
||||
fleet
|
||||
.catalogue
|
||||
.models
|
||||
.iter()
|
||||
.map(|p| p.id.clone())
|
||||
.filter(|id| !known.contains_key(id))
|
||||
.filter(|id| {
|
||||
attempts
|
||||
.get(&(name.to_string(), id.clone()))
|
||||
.is_none_or(|at| at.elapsed() >= CAPABILITY_RETRY)
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
if unknown.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
{
|
||||
let now = std::time::Instant::now();
|
||||
let mut attempts = fleet.capability_probe_attempts.write().await;
|
||||
for id in &unknown {
|
||||
attempts.insert((name.to_string(), id.clone()), now);
|
||||
}
|
||||
}
|
||||
|
||||
for model_id in unknown {
|
||||
let url = format!(
|
||||
"{endpoint}/models/{}/capabilities",
|
||||
urlencoding::encode(&model_id)
|
||||
);
|
||||
let resp = fleet
|
||||
.http_client
|
||||
.get(&url)
|
||||
.timeout(Duration::from_secs(5))
|
||||
.send()
|
||||
.await;
|
||||
let Ok(resp) = resp else { continue };
|
||||
if !resp.status().is_success() {
|
||||
continue;
|
||||
}
|
||||
let Ok(body) = resp.json::<serde_json::Value>().await else {
|
||||
continue;
|
||||
};
|
||||
let Some(caps) = body.get("capabilities").and_then(|c| c.as_array()) else {
|
||||
continue;
|
||||
};
|
||||
let caps: Vec<String> = caps
|
||||
.iter()
|
||||
.filter_map(|c| c.as_str().map(str::to_string))
|
||||
.collect();
|
||||
// An empty list is a real answer ("cached, serves nothing"), but
|
||||
// recording it would pin a model to no capabilities on the word
|
||||
// of one node. Leave it unknown so a node with real weights can
|
||||
// still speak up.
|
||||
if caps.is_empty() {
|
||||
continue;
|
||||
}
|
||||
tracing::debug!(node = name, model = %model_id, ?caps, "discovered capabilities");
|
||||
fleet
|
||||
.discovered_capabilities
|
||||
.write()
|
||||
.await
|
||||
.insert(model_id, caps);
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch `/health` and stash the activation snapshot on NodeState.
|
||||
/// Decoupled from the /models poll so a /health glitch doesn't mark
|
||||
/// the neuron unhealthy or evict the model list.
|
||||
async fn poll_health(fleet: &CortexState, name: &str, endpoint: &str) {
|
||||
let url = format!("{endpoint}/health");
|
||||
let resp = match fleet
|
||||
.http_client
|
||||
.get(&url)
|
||||
.timeout(Duration::from_secs(5))
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(r) if r.status().is_success() => r,
|
||||
Ok(r) => {
|
||||
tracing::debug!(node = name, status = %r.status(), "/health probe non-success");
|
||||
return;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::debug!(node = name, error = %e, "/health probe failed");
|
||||
return;
|
||||
}
|
||||
};
|
||||
match resp.json::<HealthResponse>().await {
|
||||
Ok(h) => {
|
||||
// Export the live load + device health to Prometheus (#137).
|
||||
// These values are already in hand from the routing scrape, so
|
||||
// publishing them adds no polling. Emitted as gauges (last-write
|
||||
// wins, refreshed every ~10s poll) outside the state lock.
|
||||
export_health_metrics(name, &h);
|
||||
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
if let Some(node) = nodes.get_mut(name) {
|
||||
node.activation = Some(h.activation);
|
||||
// Per-model admission load (#53) → keyed by id for the
|
||||
// load-aware router (#55).
|
||||
node.model_load = h.models.into_iter().map(|m| (m.id.clone(), m)).collect();
|
||||
// Per-device VRAM readings (#203) for free-fit
|
||||
// cold-load placement.
|
||||
node.device_health = h.devices.clone();
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::debug!(node = name, error = %e, "failed to parse /health response");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Publish a neuron's `/health` snapshot to Prometheus (#137): live
|
||||
/// per-model admission load + configured ceiling, and per-device GPU
|
||||
/// headroom. Gauges are `{node,model}` / `{node,device}` labelled to match
|
||||
/// the existing `cortex_*` set. Called on every successful poll so values
|
||||
/// track the ~10s cadence; a model that unloads simply stops being
|
||||
/// refreshed (its last gauge value goes stale — acceptable for the bounded
|
||||
/// fleet cardinality here).
|
||||
fn export_health_metrics(node: &str, h: &HealthResponse) {
|
||||
for m in &h.models {
|
||||
gauge!("cortex_model_in_flight", "node" => node.to_string(), "model" => m.id.clone())
|
||||
.set(m.in_flight as f64);
|
||||
gauge!("cortex_model_queue_depth", "node" => node.to_string(), "model" => m.id.clone())
|
||||
.set(m.queue_depth as f64);
|
||||
// Ceiling is the saturation denominator. 0 = pre-#137 neuron that
|
||||
// doesn't advertise it yet — skip rather than publish a bogus 0.
|
||||
if m.max_in_flight > 0 {
|
||||
gauge!("cortex_model_max_in_flight", "node" => node.to_string(), "model" => m.id.clone())
|
||||
.set(m.max_in_flight as f64);
|
||||
gauge!("cortex_model_max_queue_depth", "node" => node.to_string(), "model" => m.id.clone())
|
||||
.set(m.max_queue_depth as f64);
|
||||
}
|
||||
// Live throughput EMAs (#137) — decode tok/s is the headline
|
||||
// capacity number. Emitted unconditionally (0.0 = no sample yet).
|
||||
gauge!("cortex_model_tok_s_prefill", "node" => node.to_string(), "model" => m.id.clone())
|
||||
.set(m.tok_s_prefill);
|
||||
gauge!("cortex_model_tok_s_decode", "node" => node.to_string(), "model" => m.id.clone())
|
||||
.set(m.tok_s_decode);
|
||||
// Cumulative rejections by reason (#137) — the shedding signal.
|
||||
// Neuron reports counts-since-load; `.absolute` mirrors them onto a
|
||||
// counter (a model reload resets to 0, which Prometheus reads as a
|
||||
// normal counter reset).
|
||||
counter!("cortex_model_rejections_total",
|
||||
"node" => node.to_string(), "model" => m.id.clone(), "reason" => "queue_full")
|
||||
.absolute(m.rejected_queue_full);
|
||||
counter!("cortex_model_rejections_total",
|
||||
"node" => node.to_string(), "model" => m.id.clone(), "reason" => "wait_timeout")
|
||||
.absolute(m.rejected_timeout);
|
||||
counter!("cortex_model_rejections_total",
|
||||
"node" => node.to_string(), "model" => m.id.clone(), "reason" => "per_principal")
|
||||
.absolute(m.rejected_per_principal);
|
||||
counter!("cortex_model_rejections_total",
|
||||
"node" => node.to_string(), "model" => m.id.clone(), "reason" => "anon_yield")
|
||||
.absolute(m.rejected_anon_yield);
|
||||
// How much of this model's load is unattributable (#262), and the
|
||||
// ceiling that keeps it from starving identified callers. 0 for a
|
||||
// pre-#262 neuron, which had no ceiling — skip rather than publish
|
||||
// a 0 that reads as "anonymous is refused" when it means the
|
||||
// opposite.
|
||||
gauge!("cortex_model_anon_in_flight", "node" => node.to_string(), "model" => m.id.clone())
|
||||
.set(m.anon_in_flight as f64);
|
||||
if m.anon_max_in_flight > 0 {
|
||||
gauge!("cortex_model_anon_max_in_flight",
|
||||
"node" => node.to_string(), "model" => m.id.clone())
|
||||
.set(m.anon_max_in_flight as f64);
|
||||
}
|
||||
}
|
||||
for d in &h.devices {
|
||||
let device = d.index.to_string();
|
||||
gauge!("cortex_device_vram_used_mb", "node" => node.to_string(), "device" => device.clone())
|
||||
.set(d.vram_used_mb as f64);
|
||||
gauge!("cortex_device_vram_free_mb", "node" => node.to_string(), "device" => device.clone())
|
||||
.set(d.vram_free_mb as f64);
|
||||
gauge!("cortex_device_utilization_pct", "node" => node.to_string(), "device" => device.clone())
|
||||
.set(d.utilization_pct as f64);
|
||||
gauge!("cortex_device_temp_c", "node" => node.to_string(), "device" => device.clone())
|
||||
.set(d.temp_c as f64);
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_status(s: &str) -> ModelStatus {
|
||||
@@ -414,8 +96,6 @@ fn parse_status(s: &str) -> ModelStatus {
|
||||
"loaded" => ModelStatus::Loaded,
|
||||
"unloaded" => ModelStatus::Unloaded,
|
||||
"reloading" => ModelStatus::Reloading,
|
||||
"loading" => ModelStatus::Loading,
|
||||
"recovering" => ModelStatus::Recovering,
|
||||
_ => ModelStatus::Loaded,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,45 +1,24 @@
|
||||
//! Streaming HTTP reverse proxy to neuron backends.
|
||||
//! Streaming HTTP reverse proxy to mistral.rs backends.
|
||||
//!
|
||||
//! The streaming *mechanism* — forward an SSE body chunk-for-chunk without
|
||||
//! buffering, observing the bytes for metrics — lives in the shared
|
||||
//! [`helexa_stream`] crate (#71), so cortex and helexa-router use one
|
||||
//! implementation. This module supplies cortex's *policy*: the
|
||||
//! [`CortexMetrics`] observer (per-request token metrics + per-principal
|
||||
//! reservation settle), cortex's logging contract, and the cortex error
|
||||
//! envelope. The usage-extraction helper is re-exported from the shared
|
||||
//! crate so existing call sites keep working.
|
||||
//! For streaming requests, SSE chunks are forwarded as they arrive.
|
||||
//! The proxy captures timing information for metrics but does not
|
||||
//! buffer the full response.
|
||||
|
||||
use crate::router::RouteDecision;
|
||||
use axum::http::HeaderMap;
|
||||
use axum::http::StatusCode;
|
||||
use anyhow::Result;
|
||||
use axum::body::Body;
|
||||
use axum::http::{HeaderMap, StatusCode};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use helexa_stream::{BodyTail, ChunkObserver, StreamError};
|
||||
use reqwest::Client;
|
||||
use std::time::Instant;
|
||||
|
||||
/// Re-export the shared usage-extraction helper. Several cortex modules
|
||||
/// (`handlers`, `anthropic_sse`) pull token counts out of a buffered body
|
||||
/// tail via this function; it lives in `helexa-stream` now.
|
||||
pub use helexa_stream::last_count_for;
|
||||
|
||||
/// Proxy a request body to the resolved backend node and stream the response.
|
||||
///
|
||||
/// Logging contract: every call emits exactly one structured event at
|
||||
/// info / warn level for operator visibility, regardless of outcome.
|
||||
/// Network-level failures and non-2xx upstream statuses are warn'd here
|
||||
/// (closest to the wire); the user-facing response carries only the
|
||||
/// status code and a generic message — implementation detail (body,
|
||||
/// error chain) lives in the log, never in the API surface.
|
||||
pub async fn forward_request(
|
||||
client: &Client,
|
||||
route: &RouteDecision,
|
||||
path: &str,
|
||||
headers: HeaderMap,
|
||||
body: bytes::Bytes,
|
||||
model_id: &str,
|
||||
usage_sink: Option<crate::metering::UsageSink>,
|
||||
) -> Result<Response, ProxyError> {
|
||||
let request_start = Instant::now();
|
||||
let url = format!("{}{}", route.endpoint, path);
|
||||
tracing::info!(
|
||||
node = %route.node_name,
|
||||
@@ -48,228 +27,56 @@ pub async fn forward_request(
|
||||
"proxying request"
|
||||
);
|
||||
|
||||
let observer = CortexMetrics::new(model_id, &route.node_name, request_start, usage_sink);
|
||||
let mut req_builder = client.post(&url).body(body);
|
||||
|
||||
let response = helexa_stream::forward_streaming(client, &url, headers, body, observer)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
match &e {
|
||||
StreamError::Upstream(err) => tracing::warn!(
|
||||
node = %route.node_name,
|
||||
url = %url,
|
||||
error = %err,
|
||||
"proxy: upstream request failed (network)"
|
||||
),
|
||||
StreamError::ResponseBuild(err) => tracing::warn!(
|
||||
node = %route.node_name,
|
||||
url = %url,
|
||||
error = %err,
|
||||
"proxy: failed to build response"
|
||||
),
|
||||
}
|
||||
ProxyError::from(e)
|
||||
})?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
// Buffer the error body so the journal records WHY, then hand the
|
||||
// same bytes on. Streaming is what a success body needs; an error
|
||||
// body is a few hundred bytes and streaming it only guarantees
|
||||
// that nobody ever sees the reason.
|
||||
//
|
||||
// This was a real dead end: neuron answered 503 `insufficient_vram`
|
||||
// with the deficit in the body, and cortex logged only
|
||||
// `status=503`. Neither journal held the cause, and the guard
|
||||
// responsible could only be found by reading neuron's source
|
||||
// (#257). The body is upstream-generated (our own #63 envelope),
|
||||
// not caller content, and it is truncated in the log.
|
||||
const MAX_ERROR_BODY: usize = 64 * 1024;
|
||||
let status = response.status();
|
||||
let (parts, body) = response.into_parts();
|
||||
return match axum::body::to_bytes(body, MAX_ERROR_BODY).await {
|
||||
Ok(bytes) => {
|
||||
let snippet: String = String::from_utf8_lossy(&bytes).chars().take(600).collect();
|
||||
tracing::warn!(
|
||||
node = %route.node_name,
|
||||
url = %url,
|
||||
status = status.as_u16(),
|
||||
body = %snippet,
|
||||
"proxy: upstream returned non-2xx"
|
||||
);
|
||||
Ok(Response::from_parts(parts, axum::body::Body::from(bytes)))
|
||||
}
|
||||
// Over the cap or a read failure: the body is consumed and
|
||||
// cannot be handed on. Say so rather than passing an empty
|
||||
// body off as the upstream's answer.
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
node = %route.node_name,
|
||||
url = %url,
|
||||
status = status.as_u16(),
|
||||
error = %e,
|
||||
"proxy: upstream returned non-2xx (body unreadable, dropped)"
|
||||
);
|
||||
Ok(Response::from_parts(parts, axum::body::Body::empty()))
|
||||
}
|
||||
};
|
||||
// Forward relevant headers.
|
||||
for (key, value) in headers.iter() {
|
||||
if key == "host" || key == "content-length" {
|
||||
continue; // reqwest sets these
|
||||
}
|
||||
req_builder = req_builder.header(key, value);
|
||||
}
|
||||
|
||||
Ok(response)
|
||||
let upstream_resp = req_builder.send().await.map_err(ProxyError::Upstream)?;
|
||||
|
||||
let status =
|
||||
StatusCode::from_u16(upstream_resp.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY);
|
||||
|
||||
let resp_headers = upstream_resp.headers().clone();
|
||||
let stream = upstream_resp.bytes_stream();
|
||||
|
||||
let body = Body::from_stream(stream);
|
||||
|
||||
let mut response = Response::builder().status(status);
|
||||
for (key, value) in resp_headers.iter() {
|
||||
response = response.header(key, value);
|
||||
}
|
||||
|
||||
response
|
||||
.body(body)
|
||||
.map_err(|e| ProxyError::ResponseBuild(e.to_string()))
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ProxyError {
|
||||
#[error("upstream request failed")]
|
||||
#[error("upstream request failed: {0}")]
|
||||
Upstream(reqwest::Error),
|
||||
#[error("failed to build response")]
|
||||
#[error("failed to build response: {0}")]
|
||||
ResponseBuild(String),
|
||||
}
|
||||
|
||||
impl From<StreamError> for ProxyError {
|
||||
fn from(e: StreamError) -> Self {
|
||||
match e {
|
||||
StreamError::Upstream(err) => ProxyError::Upstream(err),
|
||||
StreamError::ResponseBuild(msg) => ProxyError::ResponseBuild(msg),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl IntoResponse for ProxyError {
|
||||
fn into_response(self) -> Response {
|
||||
let (status, code, message) = match &self {
|
||||
ProxyError::Upstream(_) => (
|
||||
StatusCode::BAD_GATEWAY,
|
||||
"upstream_connection_error",
|
||||
"upstream request failed",
|
||||
),
|
||||
ProxyError::ResponseBuild(_) => (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"internal_server_error",
|
||||
"failed to build response",
|
||||
),
|
||||
let status = match &self {
|
||||
ProxyError::Upstream(_) => StatusCode::BAD_GATEWAY,
|
||||
ProxyError::ResponseBuild(_) => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
};
|
||||
crate::error::envelope_response(cortex_core::error_envelope::OpenAiError::new(
|
||||
status.as_u16(),
|
||||
"api_error",
|
||||
code,
|
||||
message,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
// ── Per-request token metrics (#21) ─────────────────────────────────
|
||||
//
|
||||
// The proxy never buffers or re-serialises the upstream body — chunks
|
||||
// are forwarded verbatim. For metrics it observes each chunk's arrival
|
||||
// time and keeps a bounded tail of the body text (via the shared
|
||||
// `helexa_stream::BodyTail`), from which the final OpenAI `usage` object
|
||||
// (present on the last SSE chunk and on non-streaming JSON bodies alike)
|
||||
// yields engine-truth token counts.
|
||||
//
|
||||
// Emitted per request, labelled {model, node}:
|
||||
// cortex_time_to_first_token_seconds (histogram) — first body chunk
|
||||
// cortex_tokens_per_second (histogram) — completion tokens
|
||||
// over the decode window (first→last chunk); falls back to the
|
||||
// full request duration for single-chunk (non-streaming) bodies
|
||||
// cortex_prompt_tokens_total / cortex_completion_tokens_total (counters)
|
||||
|
||||
/// Cap on the retained body tail. The usage object rides on the final
|
||||
/// chunk, so a generous tail is plenty; the cap bounds memory on huge
|
||||
/// non-streaming bodies.
|
||||
const TAIL_CAP_BYTES: usize = 64 * 1024;
|
||||
|
||||
/// cortex's [`ChunkObserver`]: per-request token metrics plus the
|
||||
/// per-principal reservation settle. Drives cortex policy over the shared
|
||||
/// streaming mechanism.
|
||||
struct CortexMetrics {
|
||||
labels: [(&'static str, String); 2],
|
||||
request_start: Instant,
|
||||
first_chunk: Option<Instant>,
|
||||
last_chunk: Option<Instant>,
|
||||
tail: BodyTail,
|
||||
finished: bool,
|
||||
/// Per-principal metering hook (#51). Invoked exactly once in `finish`
|
||||
/// with the observed `(prompt, completion)` so the reservation can be
|
||||
/// settled and spend recorded. `None` for anonymous requests.
|
||||
usage_sink: Option<crate::metering::UsageSink>,
|
||||
}
|
||||
|
||||
impl CortexMetrics {
|
||||
fn new(
|
||||
model_id: &str,
|
||||
node_name: &str,
|
||||
request_start: Instant,
|
||||
usage_sink: Option<crate::metering::UsageSink>,
|
||||
) -> Self {
|
||||
Self {
|
||||
labels: [
|
||||
("model", model_id.to_string()),
|
||||
("node", node_name.to_string()),
|
||||
],
|
||||
request_start,
|
||||
first_chunk: None,
|
||||
last_chunk: None,
|
||||
tail: BodyTail::new(TAIL_CAP_BYTES),
|
||||
finished: false,
|
||||
usage_sink,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ChunkObserver for CortexMetrics {
|
||||
fn observe(&mut self, chunk: &[u8]) {
|
||||
let now = Instant::now();
|
||||
self.first_chunk.get_or_insert(now);
|
||||
self.last_chunk = Some(now);
|
||||
self.tail.push(chunk);
|
||||
}
|
||||
|
||||
/// Emit the metrics exactly once — called on clean stream end and
|
||||
/// from Drop (client disconnect mid-stream still records what we
|
||||
/// saw).
|
||||
fn finish(&mut self) {
|
||||
if self.finished {
|
||||
return;
|
||||
}
|
||||
self.finished = true;
|
||||
|
||||
let prompt = last_count_for(self.tail.as_str(), "prompt_tokens");
|
||||
let completion = last_count_for(self.tail.as_str(), "completion_tokens");
|
||||
|
||||
// Per-model metrics — only when body chunks actually arrived.
|
||||
if let Some(first) = self.first_chunk {
|
||||
let ttft = first.duration_since(self.request_start).as_secs_f64();
|
||||
metrics::histogram!("cortex_time_to_first_token_seconds", &self.labels).record(ttft);
|
||||
|
||||
if let Some(prompt) = prompt {
|
||||
metrics::counter!("cortex_prompt_tokens_total", &self.labels).increment(prompt);
|
||||
let body = serde_json::json!({
|
||||
"error": {
|
||||
"message": self.to_string(),
|
||||
"type": "proxy_error",
|
||||
}
|
||||
if let Some(completion) = completion.filter(|c| *c > 0) {
|
||||
metrics::counter!("cortex_completion_tokens_total", &self.labels)
|
||||
.increment(completion);
|
||||
|
||||
let last = self.last_chunk.unwrap_or(first);
|
||||
let decode_window = last.duration_since(first).as_secs_f64();
|
||||
// Streaming: rate over the decode window (first→last chunk).
|
||||
// Non-streaming bodies arrive as ~one chunk (window ≈ 0),
|
||||
// where the only honest denominator is the full request
|
||||
// duration.
|
||||
let secs = if decode_window >= 0.1 {
|
||||
decode_window
|
||||
} else {
|
||||
last.duration_since(self.request_start).as_secs_f64()
|
||||
};
|
||||
if secs > 0.0 {
|
||||
metrics::histogram!("cortex_tokens_per_second", &self.labels)
|
||||
.record(completion as f64 / secs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Per-principal metering + reservation settle (#51). Always runs so
|
||||
// the reservation is resolved even when no usage/body was observed
|
||||
// (sink with (0, 0) → settle 0 → release).
|
||||
if let Some(sink) = self.usage_sink.take() {
|
||||
sink(prompt.unwrap_or(0), completion.unwrap_or(0));
|
||||
}
|
||||
});
|
||||
(status, axum::Json(body)).into_response()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,21 +2,13 @@
|
||||
//!
|
||||
//! Given a model ID from an inbound request, determine which node should
|
||||
//! handle it. Priority:
|
||||
//! 1. Node where the model is currently `Loaded` → use it.
|
||||
//! 2. Node where the model is `Unloaded` → use it; neuron's existing
|
||||
//! lazy-load behaviour will reload before serving the request.
|
||||
//! 3. Model is in the catalogue → pick a feasible neuron, call
|
||||
//! `POST /models/load`, wait for the load to complete, then
|
||||
//! proxy. First-request cold-load latency is acceptable per the
|
||||
//! unified-endpoint contract.
|
||||
//! 4. Not in catalogue, not loaded anywhere → 404.
|
||||
//! 1. Node where the model is currently `Loaded`
|
||||
//! 2. Node where the model is `Unloaded` (will lazy-load on request)
|
||||
//! 3. Error: model not found on any node
|
||||
|
||||
use crate::state::CortexState;
|
||||
use cortex_core::catalogue::ModelProfile;
|
||||
use cortex_core::harness::ModelSpec;
|
||||
use cortex_core::node::ModelStatus;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
/// The routing decision: which node endpoint to proxy the request to.
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -24,546 +16,62 @@ pub struct RouteDecision {
|
||||
pub node_name: String,
|
||||
/// The inference endpoint to proxy to (from neuron's /models/{id}/endpoint).
|
||||
pub endpoint: String,
|
||||
/// Whether the model will need to load (cold start). Set to true
|
||||
/// when we proxied to an `Unloaded` node (lazy load on neuron) or
|
||||
/// when we just triggered an explicit cold-load via the catalogue
|
||||
/// path.
|
||||
/// Whether the model will need to load (cold start).
|
||||
pub cold_start: bool,
|
||||
/// The concrete model id we actually routed to. Equal to the
|
||||
/// caller's requested id unless an alias was resolved (e.g. caller
|
||||
/// asked for `helexa/small`, this carries `Qwen/Qwen3-1.7B`). The
|
||||
/// handler uses this to rewrite the request body's `model` field
|
||||
/// before proxying — neurons reject requests where the body's
|
||||
/// model name doesn't match a loaded model.
|
||||
pub resolved_model_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum RouteError {
|
||||
#[error("model '{0}' not found on any node and not in catalogue")]
|
||||
#[error("model '{0}' not found on any node")]
|
||||
ModelNotFound(String),
|
||||
#[error("no healthy nodes available")]
|
||||
NoHealthyNodes,
|
||||
#[error("failed to resolve inference endpoint for model '{0}' on node '{1}'")]
|
||||
EndpointResolveFailed(String, String),
|
||||
#[error(
|
||||
"model '{model_id}' is in the catalogue but no healthy neuron's topology satisfies its constraints"
|
||||
)]
|
||||
NoFeasibleNeuron { model_id: String },
|
||||
#[error(
|
||||
"model '{model_id}' is feasible on a neuron that is currently unhealthy — retry shortly"
|
||||
)]
|
||||
FeasibleNodeUnhealthy { model_id: String },
|
||||
#[error("cold-load of '{model_id}' on '{node}' failed: {message}")]
|
||||
ColdLoadFailed {
|
||||
model_id: String,
|
||||
node: String,
|
||||
message: String,
|
||||
},
|
||||
#[error(
|
||||
"model '{model_id}' is recovering on node '{node}' (device context rebuild in progress) — retry shortly"
|
||||
)]
|
||||
ModelRecovering { model_id: String, node: String },
|
||||
}
|
||||
|
||||
impl RouteError {
|
||||
/// HTTP status the gateway should answer with. `NoHealthyNodes` and
|
||||
/// `ModelRecovering` are the transient cases (503 service_unavailable,
|
||||
/// safe to retry the same request); everything else is 404.
|
||||
pub fn http_status(&self) -> u16 {
|
||||
match self {
|
||||
RouteError::NoHealthyNodes
|
||||
| RouteError::ModelRecovering { .. }
|
||||
| RouteError::FeasibleNodeUnhealthy { .. } => 503,
|
||||
_ => 404,
|
||||
}
|
||||
}
|
||||
|
||||
/// Broad OpenAI error category for the JSON envelope.
|
||||
pub fn broad_type(&self) -> &'static str {
|
||||
match self {
|
||||
RouteError::ModelNotFound(_) => "invalid_request_error",
|
||||
RouteError::NoHealthyNodes
|
||||
| RouteError::EndpointResolveFailed(_, _)
|
||||
| RouteError::NoFeasibleNeuron { .. }
|
||||
| RouteError::ColdLoadFailed { .. }
|
||||
| RouteError::ModelRecovering { .. }
|
||||
| RouteError::FeasibleNodeUnhealthy { .. } => "api_error",
|
||||
}
|
||||
}
|
||||
|
||||
/// Specific machine-readable error code.
|
||||
pub fn code(&self) -> &'static str {
|
||||
match self {
|
||||
RouteError::ModelNotFound(_) => "model_not_found",
|
||||
RouteError::NoHealthyNodes => "service_unavailable",
|
||||
RouteError::EndpointResolveFailed(_, _) => "service_unavailable",
|
||||
RouteError::NoFeasibleNeuron { .. } => "service_unavailable",
|
||||
RouteError::ColdLoadFailed { .. } => "service_unavailable",
|
||||
RouteError::ModelRecovering { .. } => "service_unavailable",
|
||||
RouteError::FeasibleNodeUnhealthy { .. } => "service_unavailable",
|
||||
}
|
||||
}
|
||||
|
||||
/// Seconds to advertise in `Retry-After` for the transient variants
|
||||
/// (#63). `NoHealthyNodes` may clear once the poller re-marks a node
|
||||
/// healthy; `ModelRecovering` clears once the device context finishes
|
||||
/// rebuilding — both are safe to retry. Everything else is permanent
|
||||
/// for this request (404) and carries no hint.
|
||||
pub fn retry_after_secs(&self) -> Option<u64> {
|
||||
match self {
|
||||
RouteError::ModelRecovering { .. } => Some(2),
|
||||
RouteError::FeasibleNodeUnhealthy { .. } => Some(3),
|
||||
RouteError::NoHealthyNodes => Some(5),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve which node should serve a request for the given model.
|
||||
/// Asks the neuron for the inference endpoint after selecting a node.
|
||||
pub async fn resolve(
|
||||
fleet: &Arc<CortexState>,
|
||||
requested_model_id: &str,
|
||||
model_id: &str,
|
||||
) -> Result<RouteDecision, RouteError> {
|
||||
// Alias resolution first — swap `helexa/small` (etc.) for the
|
||||
// concrete id before any node lookups so the rest of routing,
|
||||
// loading, and metrics deal in concrete ids only. `resolve_alias`
|
||||
// returns the input verbatim when it isn't an alias.
|
||||
let model_id = fleet.catalogue.resolve_alias(requested_model_id);
|
||||
if model_id != requested_model_id {
|
||||
tracing::debug!(
|
||||
requested = requested_model_id,
|
||||
resolved = model_id,
|
||||
"alias resolved"
|
||||
);
|
||||
}
|
||||
// Snapshot loaded / unloaded / recovering state from the poller cache.
|
||||
let (loaded_route, unloaded_route, recovering_node, any_healthy) = {
|
||||
let (node_name, neuron_endpoint, cold_start) = {
|
||||
let nodes = fleet.nodes.read().await;
|
||||
// All healthy nodes with the model loaded, each with its current
|
||||
// admission load (#53) so we can pick the least-busy replica (#55).
|
||||
let mut loaded_candidates: Vec<(String, String, usize)> = Vec::new();
|
||||
let mut unloaded_route = None;
|
||||
let mut recovering_node = None;
|
||||
let mut any_healthy = false;
|
||||
|
||||
let mut loaded_candidate = None;
|
||||
let mut unloaded_candidate = None;
|
||||
|
||||
for node in nodes.values() {
|
||||
if !node.healthy {
|
||||
continue;
|
||||
}
|
||||
any_healthy = true;
|
||||
if let Some(entry) = node.models.get(model_id) {
|
||||
match entry.status {
|
||||
ModelStatus::Loaded | ModelStatus::Reloading => {
|
||||
// Resident is not servable (#245). A node whose
|
||||
// device has been squeezed below the prefill
|
||||
// floor rejects every request before doing any
|
||||
// work — routing there converts a recoverable
|
||||
// placement problem into a 503 for the client.
|
||||
// Skipping it lets the rest of this function do
|
||||
// what it already does well: pick another
|
||||
// replica, or cold-load somewhere that fits.
|
||||
if !entry.is_servable() {
|
||||
tracing::warn!(
|
||||
node = %node.name,
|
||||
model = %model_id,
|
||||
reason = entry
|
||||
.servable
|
||||
.as_ref()
|
||||
.and_then(|s| s.reason.as_deref())
|
||||
.unwrap_or("unknown"),
|
||||
"skipping loaded-but-unservable location"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
// Least-busy score: in-flight + queued from the
|
||||
// neuron's last /health (#53). Unknown load (no poll
|
||||
// yet) scores 0 so the replica stays eligible.
|
||||
let score = node
|
||||
.model_load
|
||||
.get(model_id)
|
||||
.map(|l| l.in_flight + l.queue_depth)
|
||||
.unwrap_or(0);
|
||||
loaded_candidates.push((node.name.clone(), node.endpoint.clone(), score));
|
||||
loaded_candidate = Some((node.name.clone(), node.endpoint.clone(), false));
|
||||
break;
|
||||
}
|
||||
ModelStatus::Unloaded => {
|
||||
if unloaded_route.is_none() {
|
||||
unloaded_route = Some((node.name.clone(), node.endpoint.clone(), true));
|
||||
if unloaded_candidate.is_none() {
|
||||
unloaded_candidate =
|
||||
Some((node.name.clone(), node.endpoint.clone(), true));
|
||||
}
|
||||
}
|
||||
// Auto-recovering (#17/#20): the model is rebuilding
|
||||
// its device context on this node. Hold the route —
|
||||
// answer "retry shortly" rather than 404, and do NOT
|
||||
// fall through to the catalogue cold-load, which
|
||||
// would race a second placement (and a second copy's
|
||||
// worth of VRAM) against the in-flight recovery.
|
||||
ModelStatus::Recovering => {
|
||||
if recovering_node.is_none() {
|
||||
recovering_node = Some(node.name.clone());
|
||||
}
|
||||
}
|
||||
// Loading is gateway-synthesised from neuron's
|
||||
// activation snapshot; it never appears on the
|
||||
// wire from neuron's `/models`. Skip — the model
|
||||
// isn't actually servable yet. The pre-existing
|
||||
// race (catalogue cold_load fires a parallel
|
||||
// /models/load against the in-flight load) is no
|
||||
// worse than before; fixing it needs neuron-side
|
||||
// in-flight tracking on /models/load itself.
|
||||
ModelStatus::Loading => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Pick the least-busy loaded replica; ties break by node name for
|
||||
// deterministic routing. `false` = not a cold start.
|
||||
let loaded_route = loaded_candidates
|
||||
.into_iter()
|
||||
.min_by(|a, b| a.2.cmp(&b.2).then_with(|| a.0.cmp(&b.0)))
|
||||
.map(|(name, endpoint, _score)| (name, endpoint, false));
|
||||
(loaded_route, unloaded_route, recovering_node, any_healthy)
|
||||
};
|
||||
|
||||
if !any_healthy {
|
||||
return Err(RouteError::NoHealthyNodes);
|
||||
}
|
||||
|
||||
// Priority 1: already loaded.
|
||||
if let Some((node_name, neuron_endpoint, cold_start)) = loaded_route {
|
||||
return finish(fleet, &node_name, &neuron_endpoint, model_id, cold_start).await;
|
||||
}
|
||||
|
||||
// Priority 2: recovering somewhere — transient hold, not a reroute.
|
||||
if let Some(node) = recovering_node {
|
||||
return Err(RouteError::ModelRecovering {
|
||||
model_id: model_id.to_string(),
|
||||
node,
|
||||
});
|
||||
}
|
||||
|
||||
// Priority 3: known to neuron but unloaded (neuron's lazy load).
|
||||
if let Some((node_name, neuron_endpoint, cold_start)) = unloaded_route {
|
||||
return finish(fleet, &node_name, &neuron_endpoint, model_id, cold_start).await;
|
||||
}
|
||||
|
||||
// Priority 4: catalogue × topology cold-load.
|
||||
if let Some(profile) = fleet.catalogue.get(model_id) {
|
||||
let (node_name, neuron_endpoint, fits_free) = pick_feasible_neuron(fleet, profile).await?;
|
||||
// Cold-swap (#203): when the chosen node's devices don't have the
|
||||
// free VRAM *right now*, evict unpinned LRU models there until
|
||||
// the profile fits (bounded — a node whose evictable set can't
|
||||
// free enough was ranked below one that can).
|
||||
if !fits_free {
|
||||
for _ in 0..3 {
|
||||
match crate::evictor::evict_lru_on_node(fleet, &node_name, Some(&profile.id)).await
|
||||
{
|
||||
Ok(Some(evicted)) => {
|
||||
tracing::info!(
|
||||
model = %profile.id,
|
||||
node = %node_name,
|
||||
evicted = %evicted,
|
||||
"cold-swap: evicted LRU model to make room"
|
||||
);
|
||||
if node_fits_free(fleet, &node_name, profile).await {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(e) => {
|
||||
tracing::warn!(node = %node_name, error = %e, "cold-swap eviction failed");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
cold_load(fleet, &node_name, &neuron_endpoint, profile).await?;
|
||||
return finish(fleet, &node_name, &neuron_endpoint, model_id, true).await;
|
||||
}
|
||||
|
||||
Err(RouteError::ModelNotFound(model_id.to_string()))
|
||||
}
|
||||
|
||||
/// True when `node_name`'s live device readings show enough free VRAM
|
||||
/// on some device for the profile. Conservative: unknown health (no
|
||||
/// /health poll yet) counts as not-fitting so the ranking prefers
|
||||
/// nodes we can actually see.
|
||||
async fn node_fits_free(fleet: &Arc<CortexState>, node_name: &str, profile: &ModelProfile) -> bool {
|
||||
let nodes = fleet.nodes.read().await;
|
||||
nodes.get(node_name).is_some_and(|node| {
|
||||
node.device_health
|
||||
.iter()
|
||||
.any(|d| d.vram_free_mb >= profile.vram_mb.unwrap_or(0))
|
||||
})
|
||||
}
|
||||
|
||||
/// Pick a healthy neuron whose discovered topology satisfies the
|
||||
/// profile. Preference order (#203):
|
||||
/// 1. A neuron from `profile.pinned_on` that is healthy + feasible.
|
||||
/// 2. One whose live free VRAM already fits the profile.
|
||||
/// 3. One whose *displaceable* models — loaded, and outranked by this
|
||||
/// profile's residency priority — plus free VRAM could fit it
|
||||
/// after a cold-swap eviction.
|
||||
/// 4. Any healthy + feasible neuron, stable by name.
|
||||
///
|
||||
/// Free-fit outranks evict-fit, so a model never displaces anything
|
||||
/// while a node with room exists. Priority decides who may be
|
||||
/// displaced, never whether a displacement is needed.
|
||||
///
|
||||
/// Returns `(name, endpoint, fits_free)` — `fits_free = false` tells
|
||||
/// the caller a cold-swap eviction is needed before loading.
|
||||
async fn pick_feasible_neuron(
|
||||
fleet: &Arc<CortexState>,
|
||||
profile: &ModelProfile,
|
||||
) -> Result<(String, String, bool), RouteError> {
|
||||
let nodes = fleet.nodes.read().await;
|
||||
let mut candidates: Vec<(String, String, bool, bool, bool, u64)> = Vec::new();
|
||||
for node in nodes.values() {
|
||||
if !node.healthy {
|
||||
continue;
|
||||
}
|
||||
let Some(disc) = node.discovery.as_ref() else {
|
||||
continue;
|
||||
};
|
||||
if !profile.is_feasible_on(&node.name, &disc.devices) {
|
||||
continue;
|
||||
}
|
||||
let pinned = profile.pinned_on.iter().any(|n| n == &node.name);
|
||||
let max_free = node
|
||||
.device_health
|
||||
.iter()
|
||||
.map(|d| d.vram_free_mb)
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
let need = profile.vram_mb.unwrap_or(0);
|
||||
let fits_free = max_free >= need;
|
||||
// Evictable estimate: free + the VRAM of loaded models this
|
||||
// profile is permitted to displace. The predicate must be the
|
||||
// same one the evictor applies, or a node ranks as able to make
|
||||
// room and then declines to make any.
|
||||
let evictable: u64 = node
|
||||
.models
|
||||
.values()
|
||||
.filter(|m| {
|
||||
matches!(m.status, cortex_core::node::ModelStatus::Loaded)
|
||||
&& fleet.catalogue.may_displace(&profile.id, &m.id)
|
||||
})
|
||||
// neuron reports vram_used_mb: null today; fall back to the
|
||||
// catalogue's declared footprint so the evictable estimate
|
||||
// isn't silently zero (which made every node rank equal and
|
||||
// sent the first live image cold-load to beast).
|
||||
.filter_map(|m| {
|
||||
m.vram_estimate_mb
|
||||
.or_else(|| fleet.catalogue.get(&m.id).and_then(|p| p.vram_mb))
|
||||
})
|
||||
.sum();
|
||||
let fits_after_evict = max_free.saturating_add(evictable) >= need;
|
||||
candidates.push((
|
||||
node.name.clone(),
|
||||
node.endpoint.clone(),
|
||||
pinned,
|
||||
fits_free,
|
||||
fits_after_evict,
|
||||
max_free,
|
||||
));
|
||||
}
|
||||
candidates.sort_by(|a, b| {
|
||||
b.2.cmp(&a.2) // pinned first
|
||||
.then(b.3.cmp(&a.3)) // then free-fit
|
||||
.then(b.4.cmp(&a.4)) // then evictable-fit
|
||||
.then(b.5.cmp(&a.5)) // then most free VRAM
|
||||
.then(a.0.cmp(&b.0)) // then stable by name
|
||||
});
|
||||
if let Some((n, e, _, fits_free, _, _)) = candidates.into_iter().next() {
|
||||
return Ok((n, e, fits_free));
|
||||
}
|
||||
|
||||
// No *healthy* feasible neuron. Distinguish a transient outage from a
|
||||
// permanent misconfiguration: if some neuron is topologically feasible
|
||||
// but currently unhealthy (e.g. it briefly missed polls while busy),
|
||||
// this is retryable — return 503 + Retry-After so the client backs off
|
||||
// and retries instead of treating a 404 as a hard failure. Only when no
|
||||
// neuron could *ever* satisfy the topology is it a permanent 404.
|
||||
let feasible_but_unhealthy = nodes.values().any(|node| {
|
||||
!node.healthy
|
||||
&& node
|
||||
.discovery
|
||||
.as_ref()
|
||||
.is_some_and(|disc| profile.is_feasible_on(&node.name, &disc.devices))
|
||||
});
|
||||
if feasible_but_unhealthy {
|
||||
Err(RouteError::FeasibleNodeUnhealthy {
|
||||
model_id: profile.id.clone(),
|
||||
})
|
||||
} else {
|
||||
Err(RouteError::NoFeasibleNeuron {
|
||||
model_id: profile.id.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Issue `POST {endpoint}/models/load` for this profile on this neuron,
|
||||
/// blocking until the load completes (neuron's load endpoint is
|
||||
/// synchronous — it returns 200 once VRAM is materialised). On success
|
||||
/// also inserts a `Loaded` entry into the local NodeState cache so the
|
||||
/// caller's subsequent endpoint lookup sees the new model without
|
||||
/// waiting for the next poll cycle.
|
||||
async fn cold_load(
|
||||
fleet: &Arc<CortexState>,
|
||||
node_name: &str,
|
||||
neuron_endpoint: &str,
|
||||
profile: &ModelProfile,
|
||||
) -> Result<(), RouteError> {
|
||||
let spec = profile_to_spec(fleet, node_name, profile).await;
|
||||
let url = format!("{neuron_endpoint}/models/load");
|
||||
tracing::info!(model = %profile.id, node = node_name, "cold-loading via /models/load");
|
||||
|
||||
// Generous timeout: a fresh download + safetensors mmap + device
|
||||
// copy for a 30B-class dense model can comfortably exceed 5 min on
|
||||
// a slow link. The HTTP client's own default already covers most
|
||||
// of this; pin a longer per-request bound just here.
|
||||
let resp = match fleet
|
||||
.http_client
|
||||
.post(&url)
|
||||
.timeout(Duration::from_secs(1800))
|
||||
.json(&spec)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
return Err(RouteError::ColdLoadFailed {
|
||||
model_id: profile.id.clone(),
|
||||
node: node_name.to_string(),
|
||||
message: format!("HTTP request failed: {e}"),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
let status = resp.status();
|
||||
if !status.is_success() {
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
// Neuron returns 400 "already loaded" when two concurrent
|
||||
// requests race the same model. Treat that as success — both
|
||||
// requests effectively achieved the same end state.
|
||||
if body.contains("already loaded") {
|
||||
tracing::info!(
|
||||
model = %profile.id,
|
||||
node = node_name,
|
||||
"cold-load saw 'already loaded' — treating as success"
|
||||
);
|
||||
} else {
|
||||
return Err(RouteError::ColdLoadFailed {
|
||||
model_id: profile.id.clone(),
|
||||
node: node_name.to_string(),
|
||||
message: format!("HTTP {status}: {body}"),
|
||||
});
|
||||
}
|
||||
} else {
|
||||
tracing::info!(model = %profile.id, node = node_name, "cold-load returned 200");
|
||||
}
|
||||
|
||||
// Warm the cache: insert a Loaded ModelEntry so the next
|
||||
// resolve() finds the model without waiting for the poll loop.
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
if let Some(node) = nodes.get_mut(node_name) {
|
||||
node.models.insert(
|
||||
profile.id.clone(),
|
||||
cortex_core::node::ModelEntry {
|
||||
id: profile.id.clone(),
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: Some(chrono::Utc::now()),
|
||||
vram_estimate_mb: profile.vram_mb,
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
// A model we just loaded is presumed servable until
|
||||
// the next poll says otherwise. Seeding `false` here
|
||||
// would make the cold-load path immediately
|
||||
// un-route what it had just placed.
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Translate a `ModelProfile` to a `ModelSpec` neuron's /models/load
|
||||
/// accepts. Devices are picked from the neuron's discovered topology —
|
||||
/// the first `min_devices` indices that meet `min_device_vram_mb`.
|
||||
async fn profile_to_spec(
|
||||
fleet: &Arc<CortexState>,
|
||||
node_name: &str,
|
||||
profile: &ModelProfile,
|
||||
) -> ModelSpec {
|
||||
let devices = {
|
||||
let nodes = fleet.nodes.read().await;
|
||||
let mut picked: Vec<u32> = Vec::new();
|
||||
if let Some(node) = nodes.get(node_name)
|
||||
&& let Some(disc) = &node.discovery
|
||||
{
|
||||
let min_vram = profile.min_device_vram_mb.unwrap_or(0);
|
||||
for d in &disc.devices {
|
||||
if d.vram_total_mb >= min_vram {
|
||||
picked.push(d.index);
|
||||
if picked.len() as u32 >= profile.min_devices {
|
||||
break;
|
||||
}
|
||||
}
|
||||
loaded_candidate.or(unloaded_candidate).ok_or_else(|| {
|
||||
if nodes.values().any(|n| n.healthy) {
|
||||
RouteError::ModelNotFound(model_id.to_string())
|
||||
} else {
|
||||
RouteError::NoHealthyNodes
|
||||
}
|
||||
}
|
||||
if picked.is_empty() {
|
||||
// Fall back to a 0..min_devices default; pick_feasible_neuron
|
||||
// already verified the topology satisfies the constraints,
|
||||
// so this only fires if discovery raced or was lost.
|
||||
(0..profile.min_devices).collect()
|
||||
} else {
|
||||
picked
|
||||
}
|
||||
})?
|
||||
};
|
||||
|
||||
let tensor_parallel = if profile.min_devices > 1 {
|
||||
Some(profile.min_devices)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
ModelSpec {
|
||||
model_id: qualified_model_id(profile),
|
||||
harness: profile.harness.clone(),
|
||||
quant: profile.quant.clone(),
|
||||
tensor_parallel,
|
||||
devices: Some(devices),
|
||||
}
|
||||
}
|
||||
|
||||
/// Prefix the catalogue id with the scheme when one is declared, so
|
||||
/// neuron resolves the load against the right registry. Without this,
|
||||
/// a profile pointing at the helexa registry would resolve via
|
||||
/// neuron's `default_source` (typically `huggingface`) and fetch
|
||||
/// bytes from the wrong place. Profiles that omit `source` continue
|
||||
/// to pass the bare id through, preserving the pre-Phase-3 contract.
|
||||
///
|
||||
/// Stays at module scope (not nested in `profile_to_spec`) so the unit
|
||||
/// tests can exercise it without spinning up CortexState topology.
|
||||
fn qualified_model_id(profile: &ModelProfile) -> String {
|
||||
match profile.source.as_deref() {
|
||||
Some(scheme) if !scheme.is_empty() => format!("{scheme}:{}", profile.id),
|
||||
_ => profile.id.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve neuron's `/models/{id}/endpoint` to its inference URL and
|
||||
/// build the final `RouteDecision`. Shared by all three priority
|
||||
/// branches above.
|
||||
async fn finish(
|
||||
fleet: &Arc<CortexState>,
|
||||
node_name: &str,
|
||||
neuron_endpoint: &str,
|
||||
model_id: &str,
|
||||
cold_start: bool,
|
||||
) -> Result<RouteDecision, RouteError> {
|
||||
// Ask the neuron for the inference endpoint for this model.
|
||||
let endpoint_url = format!(
|
||||
"{}/models/{}/endpoint",
|
||||
neuron_endpoint,
|
||||
@@ -581,123 +89,13 @@ async fn finish(
|
||||
_ => None,
|
||||
};
|
||||
|
||||
let raw = inference_endpoint.ok_or_else(|| {
|
||||
RouteError::EndpointResolveFailed(model_id.to_string(), node_name.to_string())
|
||||
let endpoint = inference_endpoint.ok_or_else(|| {
|
||||
RouteError::EndpointResolveFailed(model_id.to_string(), node_name.clone())
|
||||
})?;
|
||||
|
||||
// Rewrite loopback inference URLs to use the configured neuron host.
|
||||
// Neuron's default bind_url is `http://localhost:13131` (it can't
|
||||
// reliably know its own externally-resolvable name). Cortex sees a
|
||||
// URL that's only meaningful from the neuron host's own perspective;
|
||||
// proxying directly to localhost from a different cortex host would
|
||||
// hit nothing. Keep neuron's port and path (a future harness could
|
||||
// serve inference on a different port than the management API), but
|
||||
// swap the host for the one in cortex.toml.
|
||||
let endpoint = rewrite_loopback_host(&raw, neuron_endpoint).unwrap_or(raw);
|
||||
|
||||
Ok(RouteDecision {
|
||||
node_name: node_name.to_string(),
|
||||
node_name,
|
||||
endpoint,
|
||||
cold_start,
|
||||
resolved_model_id: model_id.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
/// If `inference_url`'s host is a loopback name (localhost / 127.0.0.1 /
|
||||
/// 0.0.0.0 / ::1), return a copy with the host replaced by
|
||||
/// `neuron_endpoint`'s host. Otherwise return None and the caller falls
|
||||
/// back to the inference URL as-is.
|
||||
fn rewrite_loopback_host(inference_url: &str, neuron_endpoint: &str) -> Option<String> {
|
||||
let inf = url::Url::parse(inference_url).ok()?;
|
||||
let inf_host = inf.host_str()?;
|
||||
let is_loopback = matches!(inf_host, "localhost" | "127.0.0.1" | "0.0.0.0" | "::1");
|
||||
if !is_loopback {
|
||||
return None;
|
||||
}
|
||||
let neuron = url::Url::parse(neuron_endpoint).ok()?;
|
||||
let new_host = neuron.host_str()?;
|
||||
let mut out = inf.clone();
|
||||
out.set_host(Some(new_host)).ok()?;
|
||||
// url::Url::to_string normalises an empty path to "/", which then
|
||||
// breaks downstream callers that do format!("{endpoint}/v1/...")
|
||||
// and produce a double slash. The proxy URL is treated as a base
|
||||
// string that the caller appends paths to, so strip the trailing
|
||||
// slash here.
|
||||
let s = out.to_string();
|
||||
Some(s.trim_end_matches('/').to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{ModelProfile, qualified_model_id, rewrite_loopback_host};
|
||||
|
||||
fn bare_profile(id: &str, source: Option<&str>) -> ModelProfile {
|
||||
ModelProfile {
|
||||
id: id.into(),
|
||||
harness: "candle".into(),
|
||||
quant: None,
|
||||
vram_mb: None,
|
||||
min_devices: 1,
|
||||
min_device_vram_mb: None,
|
||||
pinned_on: vec![],
|
||||
residency_priority: None,
|
||||
source: source.map(String::from),
|
||||
limit: None,
|
||||
cost: None,
|
||||
capabilities: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qualified_id_passes_through_when_source_absent() {
|
||||
let p = bare_profile("Qwen/Qwen3-30B", None);
|
||||
assert_eq!(qualified_model_id(&p), "Qwen/Qwen3-30B");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qualified_id_prefixes_when_source_set() {
|
||||
let p = bare_profile("Helexa/Qwen3.6-27B-Uncensored", Some("helexa"));
|
||||
assert_eq!(
|
||||
qualified_model_id(&p),
|
||||
"helexa:Helexa/Qwen3.6-27B-Uncensored"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qualified_id_passes_through_when_source_is_empty_string() {
|
||||
// An empty scheme is treated as absent — neuron's default_source
|
||||
// substitution kicks in.
|
||||
let p = bare_profile("Qwen/Qwen3-30B", Some(""));
|
||||
assert_eq!(qualified_model_id(&p), "Qwen/Qwen3-30B");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rewrites_localhost_keeps_port_and_path() {
|
||||
let out = rewrite_loopback_host(
|
||||
"http://localhost:13131",
|
||||
"http://beast.hanzalova.internal:13131",
|
||||
);
|
||||
assert_eq!(
|
||||
out.as_deref(),
|
||||
Some("http://beast.hanzalova.internal:13131")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rewrites_loopback_with_distinct_inference_port() {
|
||||
let out = rewrite_loopback_host("http://127.0.0.1:8080", "http://beast.lan:13131");
|
||||
assert_eq!(out.as_deref(), Some("http://beast.lan:8080"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn leaves_non_loopback_alone() {
|
||||
let out = rewrite_loopback_host("http://other.host:1234", "http://beast.lan:13131");
|
||||
assert_eq!(out, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_inference_url_returns_none() {
|
||||
let out = rewrite_loopback_host("not a url", "http://beast.lan:13131");
|
||||
assert_eq!(out, None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,105 +0,0 @@
|
||||
//! Served-usage ledger (#58): cortex meters, per principal and per UTC day,
|
||||
//! the tokens it has served on behalf of mesh accounts, and periodically
|
||||
//! reports **absolute** cumulative counters to helexa-upstream for
|
||||
//! reconciliation (operators are compensated for served tokens).
|
||||
//!
|
||||
//! Counters are cumulative-since-process-start for the current period;
|
||||
//! upstream upserts them monotonically (GREATEST), so re-sending the same
|
||||
//! value is idempotent and a flush that races another is harmless. (A
|
||||
//! process restart resets the in-memory counter; the monotonic upsert keeps
|
||||
//! upstream from regressing — at most it under-counts the restarted window,
|
||||
//! acceptable for beta. One cortex per operator token is assumed.)
|
||||
|
||||
use serde::Serialize;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Mutex;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
|
||||
pub struct ServedRow {
|
||||
pub account_id: String,
|
||||
pub key_id: String,
|
||||
pub period: String, // YYYY-MM-DD (UTC)
|
||||
pub served_tokens: u64,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct ServedUsage {
|
||||
inner: Mutex<HashMap<(String, String, String), u64>>,
|
||||
}
|
||||
|
||||
impl ServedUsage {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Add served tokens for a principal in today's (UTC) period.
|
||||
pub fn add(&self, account_id: &str, key_id: &str, tokens: u64) {
|
||||
if tokens == 0 {
|
||||
return;
|
||||
}
|
||||
let period = chrono::Utc::now().format("%Y-%m-%d").to_string();
|
||||
let mut m = self.inner.lock().expect("served-usage lock");
|
||||
*m.entry((account_id.to_string(), key_id.to_string(), period))
|
||||
.or_insert(0) += tokens;
|
||||
}
|
||||
|
||||
/// Absolute cumulative counters, for a flush to upstream.
|
||||
pub fn snapshot(&self) -> Vec<ServedRow> {
|
||||
let m = self.inner.lock().expect("served-usage lock");
|
||||
m.iter()
|
||||
.map(|((account_id, key_id, period), &served_tokens)| ServedRow {
|
||||
account_id: account_id.clone(),
|
||||
key_id: key_id.clone(),
|
||||
period: period.clone(),
|
||||
served_tokens,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// POST the absolute counters to upstream's `/authz/v1/served-usage`.
|
||||
pub async fn report(
|
||||
client: &reqwest::Client,
|
||||
base_url: &str,
|
||||
bearer: &str,
|
||||
rows: &[ServedRow],
|
||||
) -> Result<(), reqwest::Error> {
|
||||
if rows.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let url = format!("{}/authz/v1/served-usage", base_url.trim_end_matches('/'));
|
||||
client
|
||||
.post(url)
|
||||
.bearer_auth(bearer)
|
||||
.json(&serde_json::json!({ "rows": rows }))
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn accumulates_per_principal_and_period() {
|
||||
let su = ServedUsage::new();
|
||||
su.add("acct", "key", 10);
|
||||
su.add("acct", "key", 5);
|
||||
su.add("acct", "other", 7);
|
||||
su.add("acct", "key", 0); // no-op
|
||||
let mut rows = su.snapshot();
|
||||
rows.sort_by(|a, b| a.key_id.cmp(&b.key_id));
|
||||
assert_eq!(rows.len(), 2);
|
||||
let key_row = rows.iter().find(|r| r.key_id == "key").unwrap();
|
||||
assert_eq!(key_row.served_tokens, 15);
|
||||
assert_eq!(
|
||||
rows.iter()
|
||||
.find(|r| r.key_id == "other")
|
||||
.unwrap()
|
||||
.served_tokens,
|
||||
7
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,7 @@
|
||||
use crate::entitlements_chain::ChainedEntitlementProvider;
|
||||
use crate::entitlements_local::LocalEntitlementProvider;
|
||||
use crate::entitlements_upstream::UpstreamEntitlementProvider;
|
||||
use cortex_core::catalogue::ModelCatalogue;
|
||||
use cortex_core::config::{EvictionSettings, GatewayConfig, NeuronEndpoint};
|
||||
use cortex_core::entitlements::EntitlementProvider;
|
||||
use cortex_core::node::NodeState;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
/// Shared fleet state, protected by a RwLock for concurrent reader access.
|
||||
@@ -16,42 +11,6 @@ pub struct CortexState {
|
||||
pub eviction: EvictionSettings,
|
||||
pub catalogue: ModelCatalogue,
|
||||
pub http_client: reqwest::Client,
|
||||
/// Resolves bearer keys to principals and enforces token budgets (#47).
|
||||
/// A local/static provider today (#50); the upstream client later (#57).
|
||||
pub entitlements: Arc<dyn EntitlementProvider>,
|
||||
/// Whether to reject unauthenticated requests (#49). Read by the auth
|
||||
/// middleware once it lands.
|
||||
pub require_auth: bool,
|
||||
/// Per-principal served-token tally (#58), reported to upstream for
|
||||
/// operator reconciliation by the flush task when upstream is enabled.
|
||||
pub served_usage: Arc<crate::served_usage::ServedUsage>,
|
||||
/// Modalities discovered for models that are not currently loaded
|
||||
/// (#241), keyed by model id.
|
||||
///
|
||||
/// A loaded model reports its own capabilities every poll, so this
|
||||
/// only carries the cold ones — which is the case that mattered,
|
||||
/// since the image model spends nearly all its time evicted and was
|
||||
/// therefore advertising nothing. Neurons derive these from their
|
||||
/// local model cache; cortex asks once and keeps the answer, because
|
||||
/// what a given model id can do does not change.
|
||||
pub discovered_capabilities: RwLock<HashMap<String, Vec<String>>>,
|
||||
/// When each `(neuron, model id)` pair was last probed for
|
||||
/// capabilities.
|
||||
///
|
||||
/// A model no node has cached can never be answered, and the poll
|
||||
/// loop runs every few seconds — without this it would ask every
|
||||
/// node about every such model forever, turning a one-off lookup
|
||||
/// into steady background traffic and a stream of 404s in the logs.
|
||||
/// Resolved ids stop being probed entirely.
|
||||
///
|
||||
/// Keyed per node, not per model: nodes are polled in config order,
|
||||
/// so a model-only key let whichever node came first record the
|
||||
/// attempt and then, if it answered 404, suppress the question to
|
||||
/// every other node for the whole back-off window. Weights are not
|
||||
/// distributed evenly across the fleet, so that is the common case
|
||||
/// rather than an edge — it hid a GGUF that two of three nodes could
|
||||
/// describe perfectly well.
|
||||
pub capability_probe_attempts: RwLock<HashMap<(String, String), std::time::Instant>>,
|
||||
}
|
||||
|
||||
impl CortexState {
|
||||
@@ -61,73 +20,27 @@ impl CortexState {
|
||||
nodes.insert(
|
||||
nc.name.clone(),
|
||||
NodeState {
|
||||
device_health: Vec::new(),
|
||||
name: nc.name.clone(),
|
||||
endpoint: nc.endpoint.clone(),
|
||||
healthy: false,
|
||||
models: HashMap::new(),
|
||||
lifecycle_cycles: 0,
|
||||
last_poll: None,
|
||||
discovery: None,
|
||||
activation: None,
|
||||
model_load: HashMap::new(),
|
||||
consecutive_poll_failures: 0,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let catalogue = ModelCatalogue::load(&config.models_config);
|
||||
|
||||
// Local provider always handles operator + infra keys. When the
|
||||
// upstream client is enabled (#57), wrap it in the chain so locally
|
||||
// unknown keys fall through to the mesh authority; otherwise stay
|
||||
// purely local.
|
||||
let local = LocalEntitlementProvider::from_config(&config.entitlements);
|
||||
let entitlements: Arc<dyn EntitlementProvider> = if config.upstream.enabled {
|
||||
tracing::info!(url = %config.upstream.url, "upstream entitlement client enabled");
|
||||
Arc::new(ChainedEntitlementProvider::new(
|
||||
local,
|
||||
UpstreamEntitlementProvider::new(&config.upstream),
|
||||
))
|
||||
} else {
|
||||
Arc::new(local)
|
||||
};
|
||||
|
||||
Self {
|
||||
nodes: RwLock::new(nodes),
|
||||
neuron_configs: config.neurons.clone(),
|
||||
eviction: config.eviction.clone(),
|
||||
catalogue,
|
||||
// READ timeout, not a total one. reqwest's `.timeout()` is a
|
||||
// total deadline — "from when the request starts connecting
|
||||
// until the response body has finished" — which for a streamed
|
||||
// completion is a hard cap on how long a model may generate,
|
||||
// not a liveness check. At 300s and ~13 tok/s that capped every
|
||||
// stream at roughly 3.9k tokens: cortex severed the connection
|
||||
// mid-answer, the client reported the stream as terminated, and
|
||||
// neuron carried on generating into a socket nobody was reading.
|
||||
// A long agentic turn could therefore never complete through the
|
||||
// gateway however healthy the fleet was, and the failure looked
|
||||
// like a model fault because the model was blameless.
|
||||
//
|
||||
// `.read_timeout()` resets on every successful read, so it times
|
||||
// out a stalled upstream while letting a producing one run as
|
||||
// long as it needs. The same 300s is kept deliberately: it is
|
||||
// the headroom a cold model load needs before its first byte,
|
||||
// which is what the original total timeout was reaching for.
|
||||
//
|
||||
// This pairs with neuron's 1s SSE keep-alive: a healthy stream
|
||||
// puts bytes on the wire every second, so idle here means idle,
|
||||
// and the deadline finally measures what it was meant to.
|
||||
http_client: reqwest::Client::builder()
|
||||
.read_timeout(std::time::Duration::from_secs(300))
|
||||
.timeout(std::time::Duration::from_secs(300))
|
||||
.build()
|
||||
.expect("failed to build HTTP client"),
|
||||
entitlements,
|
||||
require_auth: config.entitlements.require_auth,
|
||||
served_usage: Arc::new(crate::served_usage::ServedUsage::new()),
|
||||
discovered_capabilities: RwLock::new(HashMap::new()),
|
||||
capability_probe_attempts: RwLock::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,286 +0,0 @@
|
||||
//! Alias resolution: a client request with `model: "helexa/small"`
|
||||
//! routes to the concrete model id (e.g. `Qwen/Qwen3-1.7B`), with the
|
||||
//! proxied request body rewritten so the upstream neuron sees a model
|
||||
//! name that matches its loaded handle.
|
||||
|
||||
mod common;
|
||||
|
||||
use cortex_core::config::{
|
||||
EvictionSettings, EvictionStrategy, GatewayConfig, GatewaySettings, NeuronEndpoint,
|
||||
};
|
||||
use cortex_core::node::{ModelEntry, ModelStatus};
|
||||
use cortex_gateway::state::CortexState;
|
||||
use serde_json::json;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
/// Write a `models.toml` with one alias to a unique temp path. Returns
|
||||
/// the path; the file persists for the test process and gets reaped by
|
||||
/// the OS at exit. Using $XDG_RUNTIME_DIR fallback for the temp dir
|
||||
/// keeps the file off shared /tmp on CI without pulling in tempfile.
|
||||
fn write_models_toml(alias: &str, target: &str) -> PathBuf {
|
||||
let contents = format!(
|
||||
r#"
|
||||
[aliases]
|
||||
"{alias}" = "{target}"
|
||||
"#
|
||||
);
|
||||
let mut path = std::env::temp_dir();
|
||||
let pid = std::process::id();
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos();
|
||||
path.push(format!("cortex-test-models-{pid}-{now}.toml"));
|
||||
std::fs::write(&path, contents).expect("write temp models.toml");
|
||||
path
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_alias_resolves_in_chat_completions() {
|
||||
let mock_url = common::spawn_mock_neuron().await;
|
||||
let models_path = write_models_toml("helexa/small", "test-model");
|
||||
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![NeuronEndpoint {
|
||||
name: "mock-node".into(),
|
||||
endpoint: mock_url,
|
||||
}],
|
||||
models_config: models_path.to_string_lossy().to_string(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
|
||||
// Seed the node as healthy with the concrete model loaded under
|
||||
// the target id. The poller doesn't run in this test; we just
|
||||
// populate state manually.
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("mock-node").expect("node must exist");
|
||||
node.healthy = true;
|
||||
node.models.insert(
|
||||
"test-model".into(),
|
||||
ModelEntry {
|
||||
id: "test-model".into(),
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: None,
|
||||
vram_estimate_mb: None,
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
// Sanity: the catalogue actually picked up the alias.
|
||||
assert_eq!(
|
||||
fleet.catalogue.resolve_alias("helexa/small"),
|
||||
"test-model",
|
||||
"alias should resolve to target id"
|
||||
);
|
||||
|
||||
// Spawn the gateway against this fleet.
|
||||
let app = cortex_gateway::build_app(Arc::clone(&fleet));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let gateway_addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
let gateway_url = format!("http://{gateway_addr}");
|
||||
|
||||
// Send a chat completion against the alias. The mock backend
|
||||
// echoes back the `model` field it received — so a body whose
|
||||
// model wasn't rewritten would come back as "helexa/small", and a
|
||||
// properly-rewritten one as "test-model".
|
||||
let client = reqwest::Client::new();
|
||||
let resp = client
|
||||
.post(format!("{gateway_url}/v1/chat/completions"))
|
||||
.json(&json!({
|
||||
"model": "helexa/small",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.expect("gateway should respond");
|
||||
|
||||
assert!(resp.status().is_success(), "gateway returned non-2xx");
|
||||
let body: serde_json::Value = resp.json().await.expect("response is JSON");
|
||||
assert_eq!(
|
||||
body.get("model").and_then(|m| m.as_str()),
|
||||
Some("test-model"),
|
||||
"mock backend should have seen the resolved model id, not the alias"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_aliases_surface_in_v1_models() {
|
||||
let mock_url = common::spawn_mock_neuron().await;
|
||||
let models_path = write_models_toml("helexa/small", "test-model");
|
||||
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![NeuronEndpoint {
|
||||
name: "mock-node".into(),
|
||||
endpoint: mock_url,
|
||||
}],
|
||||
models_config: models_path.to_string_lossy().to_string(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
|
||||
// Seed the target as loaded so the alias's mirrored entry shows
|
||||
// loaded=true.
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("mock-node").expect("node must exist");
|
||||
node.healthy = true;
|
||||
node.models.insert(
|
||||
"test-model".into(),
|
||||
ModelEntry {
|
||||
id: "test-model".into(),
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: None,
|
||||
vram_estimate_mb: Some(2000),
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let app = cortex_gateway::build_app(Arc::clone(&fleet));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let gateway_addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
let gateway_url = format!("http://{gateway_addr}");
|
||||
|
||||
let resp = reqwest::get(format!("{gateway_url}/v1/models"))
|
||||
.await
|
||||
.expect("gateway should respond");
|
||||
let body: serde_json::Value = resp.json().await.unwrap();
|
||||
let entries = body
|
||||
.get("data")
|
||||
.and_then(|d| d.as_array())
|
||||
.expect("data array");
|
||||
|
||||
// Both the alias and the target should be present.
|
||||
let ids: Vec<&str> = entries
|
||||
.iter()
|
||||
.filter_map(|e| e.get("id").and_then(|v| v.as_str()))
|
||||
.collect();
|
||||
assert!(ids.contains(&"test-model"), "target should be listed");
|
||||
assert!(ids.contains(&"helexa/small"), "alias should be listed");
|
||||
|
||||
// The alias's `loaded` flag and locations should mirror the target.
|
||||
let alias_entry = entries
|
||||
.iter()
|
||||
.find(|e| e.get("id").and_then(|v| v.as_str()) == Some("helexa/small"))
|
||||
.expect("alias entry");
|
||||
assert_eq!(alias_entry.get("loaded"), Some(&json!(true)));
|
||||
let locations = alias_entry
|
||||
.get("locations")
|
||||
.and_then(|l| l.as_array())
|
||||
.expect("locations array");
|
||||
assert_eq!(locations.len(), 1);
|
||||
assert_eq!(
|
||||
locations[0].get("node").and_then(|n| n.as_str()),
|
||||
Some("mock-node")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_alias_falls_through_for_unmapped_model() {
|
||||
// Catalogue has an alias for some-other-thing but the request
|
||||
// model "test-model" isn't an alias; resolution should be a no-op.
|
||||
let mock_url = common::spawn_mock_neuron().await;
|
||||
let models_path = write_models_toml("helexa/large", "definitely-not-loaded");
|
||||
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![NeuronEndpoint {
|
||||
name: "mock-node".into(),
|
||||
endpoint: mock_url,
|
||||
}],
|
||||
models_config: models_path.to_string_lossy().to_string(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("mock-node").expect("node must exist");
|
||||
node.healthy = true;
|
||||
node.models.insert(
|
||||
"test-model".into(),
|
||||
ModelEntry {
|
||||
id: "test-model".into(),
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: None,
|
||||
vram_estimate_mb: None,
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let app = cortex_gateway::build_app(Arc::clone(&fleet));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let gateway_addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
let gateway_url = format!("http://{gateway_addr}");
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway_url}/v1/chat/completions"))
|
||||
.json(&json!({
|
||||
"model": "test-model",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(resp.status().is_success());
|
||||
let body: serde_json::Value = resp.json().await.unwrap();
|
||||
assert_eq!(
|
||||
body.get("model").and_then(|m| m.as_str()),
|
||||
Some("test-model")
|
||||
);
|
||||
}
|
||||
@@ -123,212 +123,3 @@ async fn test_anthropic_invalid_request() {
|
||||
|
||||
assert_eq!(resp.status(), 400);
|
||||
}
|
||||
|
||||
/// Tool round-trip: an Anthropic `/v1/messages` request carrying tools
|
||||
/// (the Claude Code shape: `{name, description, input_schema}`) must
|
||||
/// reach the upstream neuron reshaped into OpenAI function-tool form,
|
||||
/// and tool history (`tool_use` / `tool_result` blocks) must become
|
||||
/// `tool_calls` / `role:"tool"` messages. This is the fix for the
|
||||
/// failure where the model received malformed tool defs and improvised
|
||||
/// an unparseable `<tool_use_name>` format.
|
||||
#[tokio::test]
|
||||
async fn test_anthropic_tools_reshaped_for_upstream() {
|
||||
let (mock_url, captured) = common::spawn_capturing_mock_neuron().await;
|
||||
let gw_url = common::spawn_gateway(&mock_url).await;
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let resp = client
|
||||
.post(format!("{gw_url}/v1/messages"))
|
||||
.header("content-type", "application/json")
|
||||
.json(&json!({
|
||||
"model": "test-model",
|
||||
"max_tokens": 100,
|
||||
"tools": [{
|
||||
"name": "Read",
|
||||
"description": "Read a file from disk",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {"path": {"type": "string"}},
|
||||
"required": ["path"]
|
||||
}
|
||||
}],
|
||||
"tool_choice": {"type": "auto"},
|
||||
"messages": [
|
||||
{"role": "user", "content": "read /etc/hosts"},
|
||||
{"role": "assistant", "content": [
|
||||
{"type": "text", "text": "Reading it."},
|
||||
{"type": "tool_use", "id": "toolu_42", "name": "Read",
|
||||
"input": {"path": "/etc/hosts"}}
|
||||
]},
|
||||
{"role": "user", "content": [
|
||||
{"type": "tool_result", "tool_use_id": "toolu_42",
|
||||
"content": "127.0.0.1 localhost"}
|
||||
]}
|
||||
]
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.expect("request should succeed");
|
||||
assert_eq!(resp.status(), 200);
|
||||
|
||||
let forwarded = {
|
||||
let guard = captured.lock().unwrap();
|
||||
guard.last().cloned().expect("upstream received a request")
|
||||
};
|
||||
|
||||
// Tool definitions reshaped to OpenAI function form.
|
||||
let tools = forwarded["tools"].as_array().expect("tools array");
|
||||
assert_eq!(tools[0]["type"], "function");
|
||||
assert_eq!(tools[0]["function"]["name"], "Read");
|
||||
assert_eq!(
|
||||
tools[0]["function"]["parameters"]["properties"]["path"]["type"],
|
||||
"string"
|
||||
);
|
||||
assert!(tools[0]["function"].get("input_schema").is_none());
|
||||
|
||||
// tool_choice mapped.
|
||||
assert_eq!(forwarded["tool_choice"], "auto");
|
||||
|
||||
// Message history: user, assistant(+tool_calls), tool, user.
|
||||
let msgs = forwarded["messages"].as_array().expect("messages array");
|
||||
let assistant = msgs
|
||||
.iter()
|
||||
.find(|m| m["role"] == "assistant")
|
||||
.expect("assistant turn");
|
||||
assert_eq!(assistant["tool_calls"][0]["id"], "toolu_42");
|
||||
assert_eq!(assistant["tool_calls"][0]["function"]["name"], "Read");
|
||||
// arguments is the parsed object, not a JSON string — the Qwen3.6
|
||||
// chat template iterates `tool_call.arguments | items`.
|
||||
assert_eq!(
|
||||
assistant["tool_calls"][0]["function"]["arguments"],
|
||||
json!({"path": "/etc/hosts"})
|
||||
);
|
||||
|
||||
let tool_msg = msgs
|
||||
.iter()
|
||||
.find(|m| m["role"] == "tool")
|
||||
.expect("tool result turn");
|
||||
assert_eq!(tool_msg["tool_call_id"], "toolu_42");
|
||||
assert_eq!(tool_msg["content"], "127.0.0.1 localhost");
|
||||
}
|
||||
|
||||
/// #24: a streaming Anthropic request gets a translated Anthropic SSE
|
||||
/// stream — not raw OpenAI frames. Verifies the full event sequence,
|
||||
/// text reassembly, and the content type.
|
||||
#[tokio::test]
|
||||
async fn test_anthropic_streaming_sse_translation() {
|
||||
let mock_url =
|
||||
common::spawn_streaming_mock_neuron(4, std::time::Duration::from_millis(20)).await;
|
||||
let gw_url = common::spawn_gateway(&mock_url).await;
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let resp = client
|
||||
.post(format!("{gw_url}/v1/messages"))
|
||||
.header("content-type", "application/json")
|
||||
.json(&json!({
|
||||
"model": "test-model",
|
||||
"max_tokens": 64,
|
||||
"stream": true,
|
||||
"messages": [{"role": "user", "content": "Hi"}]
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.expect("request should succeed");
|
||||
|
||||
assert_eq!(resp.status(), 200);
|
||||
assert!(
|
||||
resp.headers()
|
||||
.get("content-type")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or("")
|
||||
.starts_with("text/event-stream"),
|
||||
"anthropic stream must be SSE"
|
||||
);
|
||||
|
||||
let body = resp.text().await.expect("stream should complete");
|
||||
assert!(
|
||||
!body.contains("chat.completion.chunk"),
|
||||
"raw OpenAI frames must not leak through:\n{body}"
|
||||
);
|
||||
|
||||
let event_names: Vec<&str> = body
|
||||
.lines()
|
||||
.filter_map(|l| l.strip_prefix("event: "))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
event_names,
|
||||
vec![
|
||||
"message_start",
|
||||
"content_block_start",
|
||||
"content_block_delta",
|
||||
"content_block_delta",
|
||||
"content_block_delta",
|
||||
"content_block_delta",
|
||||
"content_block_stop",
|
||||
"message_delta",
|
||||
"message_stop",
|
||||
],
|
||||
"unexpected event sequence:\n{body}"
|
||||
);
|
||||
|
||||
// Reassemble the text deltas: the mock emits token0..token3.
|
||||
let text: String = body
|
||||
.lines()
|
||||
.filter_map(|l| l.strip_prefix("data: "))
|
||||
.filter_map(|d| serde_json::from_str::<serde_json::Value>(d).ok())
|
||||
.filter(|v| v["type"] == "content_block_delta")
|
||||
.filter_map(|v| v["delta"]["text"].as_str().map(String::from))
|
||||
.collect();
|
||||
assert_eq!(text, "token0token1token2token3");
|
||||
|
||||
// The mock sends no finish_reason — stop_reason defaults to
|
||||
// end_turn, and output_tokens falls back to the delta count.
|
||||
let message_delta = body
|
||||
.lines()
|
||||
.filter_map(|l| l.strip_prefix("data: "))
|
||||
.filter_map(|d| serde_json::from_str::<serde_json::Value>(d).ok())
|
||||
.find(|v| v["type"] == "message_delta")
|
||||
.expect("message_delta event present");
|
||||
assert_eq!(message_delta["delta"]["stop_reason"], "end_turn");
|
||||
assert_eq!(message_delta["usage"]["output_tokens"], 4);
|
||||
}
|
||||
|
||||
/// #24: an upstream usage frame (stream_options include_usage shape)
|
||||
/// rides into message_delta as input/output token counts.
|
||||
#[tokio::test]
|
||||
async fn test_anthropic_streaming_usage_propagation() {
|
||||
let mock_url = common::spawn_streaming_mock_neuron_with_usage(
|
||||
3,
|
||||
std::time::Duration::from_millis(10),
|
||||
225,
|
||||
42,
|
||||
)
|
||||
.await;
|
||||
let gw_url = common::spawn_gateway(&mock_url).await;
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let body = client
|
||||
.post(format!("{gw_url}/v1/messages"))
|
||||
.header("content-type", "application/json")
|
||||
.json(&json!({
|
||||
"model": "test-model",
|
||||
"max_tokens": 64,
|
||||
"stream": true,
|
||||
"messages": [{"role": "user", "content": "Hi"}]
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.expect("request should succeed")
|
||||
.text()
|
||||
.await
|
||||
.expect("stream should complete");
|
||||
|
||||
let message_delta = body
|
||||
.lines()
|
||||
.filter_map(|l| l.strip_prefix("data: "))
|
||||
.filter_map(|d| serde_json::from_str::<serde_json::Value>(d).ok())
|
||||
.find(|v| v["type"] == "message_delta")
|
||||
.expect("message_delta event present");
|
||||
assert_eq!(message_delta["usage"]["output_tokens"], 42);
|
||||
assert_eq!(message_delta["usage"]["input_tokens"], 225);
|
||||
}
|
||||
|
||||
@@ -1,274 +0,0 @@
|
||||
//! Integration tests for API-key auth + principal resolution (#49).
|
||||
//!
|
||||
//! Verifies the #63 rejection contract (401 invalid_api_key via the #60
|
||||
//! envelope) and that an authenticated request reaches neuron carrying the
|
||||
//! internal principal headers — while a client-supplied principal header is
|
||||
//! stripped (anti-spoofing).
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::Path;
|
||||
use axum::http::HeaderMap;
|
||||
use axum::routing::{get, post};
|
||||
use cortex_core::config::{
|
||||
ApiKeyConfig, EntitlementsConfig, EvictionSettings, EvictionStrategy, GatewayConfig,
|
||||
GatewaySettings, NeuronEndpoint,
|
||||
};
|
||||
use cortex_core::entitlements::{CapWindow, HEADER_ACCOUNT_ID, HEADER_KEY_ID};
|
||||
use cortex_core::node::{ModelEntry, ModelStatus};
|
||||
use cortex_gateway::state::CortexState;
|
||||
use serde_json::{Value, json};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
/// What the mock neuron observed on the inbound `/v1/chat/completions`
|
||||
/// request: the principal headers cortex stamped (or didn't).
|
||||
#[derive(Default)]
|
||||
struct Seen {
|
||||
account_id: Option<String>,
|
||||
key_id: Option<String>,
|
||||
}
|
||||
|
||||
/// Spawn a mock neuron that records the principal headers it receives and
|
||||
/// returns a trivial chat completion. Returns (base_url, observed).
|
||||
async fn spawn_capturing_neuron() -> (String, Arc<Mutex<Seen>>) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let base_url = format!("http://{addr}");
|
||||
let inference_url = base_url.clone();
|
||||
let seen: Arc<Mutex<Seen>> = Arc::new(Mutex::new(Seen::default()));
|
||||
let sink = Arc::clone(&seen);
|
||||
|
||||
let app = axum::Router::new()
|
||||
.route(
|
||||
"/models/{model_id}/endpoint",
|
||||
get(move |Path(_): Path<String>| {
|
||||
let url = inference_url.clone();
|
||||
async move { Json(json!({ "url": url })) }
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/v1/chat/completions",
|
||||
post(move |headers: HeaderMap, Json(body): Json<Value>| {
|
||||
let sink = Arc::clone(&sink);
|
||||
async move {
|
||||
{
|
||||
let mut s = sink.lock().unwrap();
|
||||
s.account_id = headers
|
||||
.get(HEADER_ACCOUNT_ID)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(str::to_string);
|
||||
s.key_id = headers
|
||||
.get(HEADER_KEY_ID)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(str::to_string);
|
||||
}
|
||||
let model = body.get("model").and_then(Value::as_str).unwrap_or("m");
|
||||
Json(json!({
|
||||
"id": "chatcmpl-auth-001",
|
||||
"object": "chat.completion",
|
||||
"created": 1700000000_u64,
|
||||
"model": model,
|
||||
"choices": [{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "ok"},
|
||||
"finish_reason": "stop"
|
||||
}],
|
||||
"usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}
|
||||
}))
|
||||
}
|
||||
}),
|
||||
)
|
||||
.with_state(());
|
||||
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
|
||||
(base_url, seen)
|
||||
}
|
||||
|
||||
/// Spawn a gateway with the given entitlements config, a single neuron, and
|
||||
/// `test-model` seeded as loaded (build_app spawns no poller).
|
||||
async fn spawn_gateway(neuron_url: &str, entitlements: EntitlementsConfig) -> String {
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![NeuronEndpoint {
|
||||
name: "mock-node".into(),
|
||||
endpoint: neuron_url.to_string(),
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements,
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("mock-node").unwrap();
|
||||
node.healthy = true;
|
||||
node.models.insert(
|
||||
"test-model".into(),
|
||||
ModelEntry {
|
||||
id: "test-model".into(),
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: None,
|
||||
vram_estimate_mb: Some(8000),
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let app = cortex_gateway::build_app(Arc::clone(&fleet));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
format!("http://{addr}")
|
||||
}
|
||||
|
||||
fn one_key_config(require_auth: bool) -> EntitlementsConfig {
|
||||
EntitlementsConfig {
|
||||
require_auth,
|
||||
keys: vec![ApiKeyConfig {
|
||||
key: "sk-good".into(),
|
||||
account_id: "acct-1".into(),
|
||||
key_id: Some("key-1".into()),
|
||||
hard_cap: None,
|
||||
window: CapWindow::Balance,
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
fn chat_body() -> Value {
|
||||
json!({
|
||||
"model": "test-model",
|
||||
"messages": [{"role": "user", "content": "hi"}]
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn missing_key_when_required_is_401_invalid_api_key() {
|
||||
let (neuron, _seen) = spawn_capturing_neuron().await;
|
||||
let gateway = spawn_gateway(&neuron, one_key_config(true)).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.json(&chat_body())
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::UNAUTHORIZED);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["error"]["code"], "invalid_api_key");
|
||||
assert_eq!(body["error"]["type"], "invalid_request_error");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unrecognized_key_is_ignored_when_auth_not_required() {
|
||||
let (neuron, seen) = spawn_capturing_neuron().await;
|
||||
// allow-anonymous mode: a placeholder/unknown bearer (as opencode,
|
||||
// Open WebUI, Agent Zero, litellm all send by default) must NOT be
|
||||
// rejected — it's ignored and the request is served anonymously.
|
||||
let gateway = spawn_gateway(&neuron, one_key_config(false)).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.bearer_auth("sk-dummy-placeholder")
|
||||
.json(&chat_body())
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::OK);
|
||||
let _ = resp.bytes().await.unwrap();
|
||||
// Served, but anonymous — no principal stamped from the bogus key.
|
||||
assert!(seen.lock().unwrap().account_id.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_key_is_401_when_auth_required() {
|
||||
let (neuron, seen) = spawn_capturing_neuron().await;
|
||||
// With auth required, a present-but-wrong credential is rejected.
|
||||
let gateway = spawn_gateway(&neuron, one_key_config(true)).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.bearer_auth("sk-wrong")
|
||||
.json(&chat_body())
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::UNAUTHORIZED);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["error"]["code"], "invalid_api_key");
|
||||
// Rejected before dispatch — neuron never saw the request.
|
||||
assert!(seen.lock().unwrap().account_id.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn valid_key_reaches_neuron_with_principal_headers() {
|
||||
let (neuron, seen) = spawn_capturing_neuron().await;
|
||||
let gateway = spawn_gateway(&neuron, one_key_config(true)).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.bearer_auth("sk-good")
|
||||
// A spoofed principal header must be stripped, not forwarded.
|
||||
.header(HEADER_ACCOUNT_ID, "attacker")
|
||||
.json(&chat_body())
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::OK);
|
||||
let s = seen.lock().unwrap();
|
||||
assert_eq!(s.account_id.as_deref(), Some("acct-1"));
|
||||
assert_eq!(s.key_id.as_deref(), Some("key-1"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn anonymous_allowed_when_auth_not_required() {
|
||||
let (neuron, seen) = spawn_capturing_neuron().await;
|
||||
let gateway = spawn_gateway(&neuron, EntitlementsConfig::default()).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.json(&chat_body())
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::OK);
|
||||
// No principal resolved → no principal headers stamped.
|
||||
let s = seen.lock().unwrap();
|
||||
assert!(s.account_id.is_none());
|
||||
assert!(s.key_id.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn health_is_public_even_when_auth_required() {
|
||||
let (neuron, _seen) = spawn_capturing_neuron().await;
|
||||
let gateway = spawn_gateway(&neuron, one_key_config(true)).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.get(format!("{gateway}/health"))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::OK);
|
||||
}
|
||||
@@ -1,255 +0,0 @@
|
||||
//! Integration tests for budget enforcement (#52) — the A0 seatbelt.
|
||||
//!
|
||||
//! A reservation over the key's hard cap is refused *before* neuron is hit,
|
||||
//! with the #63 code matching the cap-window semantics (rate_limit_exceeded
|
||||
//! + Retry-After for a resetting window, insufficient_quota for a hard
|
||||
//! balance). Spend never exceeds the cap. No 402, ever.
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::Path;
|
||||
use axum::routing::{get, post};
|
||||
use cortex_core::config::{
|
||||
ApiKeyConfig, EntitlementsConfig, EvictionSettings, EvictionStrategy, GatewayConfig,
|
||||
GatewaySettings, NeuronEndpoint,
|
||||
};
|
||||
use cortex_core::entitlements::{CapWindow, Principal};
|
||||
use cortex_core::node::{ModelEntry, ModelStatus};
|
||||
use cortex_gateway::state::CortexState;
|
||||
use serde_json::{Value, json};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
/// Mock neuron with a hit counter on the inference path, so a test can prove
|
||||
/// a request was (or wasn't) dispatched.
|
||||
async fn spawn_counting_neuron() -> (String, Arc<AtomicU64>) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let base_url = format!("http://{addr}");
|
||||
let inference_url = base_url.clone();
|
||||
let hits = Arc::new(AtomicU64::new(0));
|
||||
let sink = Arc::clone(&hits);
|
||||
|
||||
let app = axum::Router::new()
|
||||
.route(
|
||||
"/models/{model_id}/endpoint",
|
||||
get(move |Path(_): Path<String>| {
|
||||
let url = inference_url.clone();
|
||||
async move { Json(json!({ "url": url })) }
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/v1/chat/completions",
|
||||
post(move |Json(body): Json<Value>| {
|
||||
let sink = Arc::clone(&sink);
|
||||
async move {
|
||||
sink.fetch_add(1, Ordering::SeqCst);
|
||||
let model = body.get("model").and_then(Value::as_str).unwrap_or("m");
|
||||
Json(json!({
|
||||
"id": "chatcmpl-budget",
|
||||
"object": "chat.completion",
|
||||
"created": 1700000000_u64,
|
||||
"model": model,
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}
|
||||
}))
|
||||
}
|
||||
}),
|
||||
);
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
(base_url, hits)
|
||||
}
|
||||
|
||||
async fn spawn_gateway(neuron_url: &str, key: ApiKeyConfig) -> (Arc<CortexState>, String) {
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![NeuronEndpoint {
|
||||
name: "mock-node".into(),
|
||||
endpoint: neuron_url.to_string(),
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: EntitlementsConfig {
|
||||
require_auth: true,
|
||||
keys: vec![key],
|
||||
},
|
||||
upstream: Default::default(),
|
||||
};
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("mock-node").unwrap();
|
||||
node.healthy = true;
|
||||
node.models.insert(
|
||||
"test-model".into(),
|
||||
ModelEntry {
|
||||
id: "test-model".into(),
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: None,
|
||||
vram_estimate_mb: Some(8000),
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
let app = cortex_gateway::build_app(Arc::clone(&fleet));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
(fleet, format!("http://{addr}"))
|
||||
}
|
||||
|
||||
fn key(window: CapWindow, hard_cap: u64) -> ApiKeyConfig {
|
||||
ApiKeyConfig {
|
||||
key: "sk-cap".into(),
|
||||
account_id: "acct-cap".into(),
|
||||
key_id: Some("key-cap".into()),
|
||||
hard_cap: Some(hard_cap),
|
||||
window,
|
||||
}
|
||||
}
|
||||
|
||||
fn chat(max_tokens: u64) -> Value {
|
||||
json!({
|
||||
"model": "test-model",
|
||||
"max_tokens": max_tokens,
|
||||
"messages": [{"role": "user", "content": "hi"}]
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn balance_over_cap_is_429_insufficient_quota_before_dispatch() {
|
||||
let (neuron, hits) = spawn_counting_neuron().await;
|
||||
// Cap far below a single request's reservation (max_tokens 1000).
|
||||
let (_fleet, gateway) = spawn_gateway(&neuron, key(CapWindow::Balance, 10)).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.bearer_auth("sk-cap")
|
||||
.json(&chat(1000))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::TOO_MANY_REQUESTS);
|
||||
// Hard balance → no Retry-After.
|
||||
assert!(resp.headers().get(reqwest::header::RETRY_AFTER).is_none());
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["error"]["code"], "insufficient_quota");
|
||||
// Refused before dispatch — neuron never saw it.
|
||||
assert_eq!(hits.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rolling_over_cap_is_429_rate_limited_with_retry_after() {
|
||||
let (neuron, hits) = spawn_counting_neuron().await;
|
||||
let (_fleet, gateway) =
|
||||
spawn_gateway(&neuron, key(CapWindow::Rolling { seconds: 3600 }, 10)).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.bearer_auth("sk-cap")
|
||||
.json(&chat(1000))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::TOO_MANY_REQUESTS);
|
||||
let retry = resp
|
||||
.headers()
|
||||
.get(reqwest::header::RETRY_AFTER)
|
||||
.expect("rolling-window rejection must carry Retry-After");
|
||||
assert!(retry.to_str().unwrap().parse::<u64>().unwrap() >= 1);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["error"]["code"], "rate_limit_exceeded");
|
||||
assert_eq!(hits.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn within_cap_is_served() {
|
||||
let (neuron, hits) = spawn_counting_neuron().await;
|
||||
let (_fleet, gateway) = spawn_gateway(&neuron, key(CapWindow::Balance, 1_000_000)).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.bearer_auth("sk-cap")
|
||||
.json(&chat(50))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::OK);
|
||||
let _ = resp.bytes().await.unwrap();
|
||||
assert_eq!(hits.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a0_seatbelt_caps_a_runaway_fan_out() {
|
||||
// An Agent-Zero-style key with a modest cap: a burst of requests drains
|
||||
// it, then further requests are refused — the account stops draining and
|
||||
// spend never exceeds the cap.
|
||||
let (neuron, hits) = spawn_counting_neuron().await;
|
||||
let (fleet, gateway) = spawn_gateway(&neuron, key(CapWindow::Balance, 100)).await;
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let mut ok = 0;
|
||||
let mut refused = 0;
|
||||
for _ in 0..20 {
|
||||
let resp = client
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.bearer_auth("sk-cap")
|
||||
.json(&chat(20))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
match resp.status() {
|
||||
reqwest::StatusCode::OK => {
|
||||
ok += 1;
|
||||
let _ = resp.bytes().await.unwrap();
|
||||
}
|
||||
reqwest::StatusCode::TOO_MANY_REQUESTS => {
|
||||
refused += 1;
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["error"]["code"], "insufficient_quota");
|
||||
}
|
||||
other => panic!("unexpected status {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
assert!(ok >= 1, "some requests should be served");
|
||||
assert!(refused >= 1, "the cap must eventually refuse the fan-out");
|
||||
assert_eq!(
|
||||
hits.load(Ordering::SeqCst),
|
||||
ok,
|
||||
"refused requests never dispatched"
|
||||
);
|
||||
|
||||
// Spend never exceeded the hard cap (reservation prevents overshoot).
|
||||
// Poll briefly for in-flight settles to land.
|
||||
let principal = Principal {
|
||||
account_id: "acct-cap".into(),
|
||||
key_id: "key-cap".into(),
|
||||
};
|
||||
for _ in 0..50 {
|
||||
let snap = fleet.entitlements.snapshot(&principal).await.unwrap();
|
||||
if snap.reserved == 0 {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
|
||||
}
|
||||
let snap = fleet.entitlements.snapshot(&principal).await.unwrap();
|
||||
assert!(snap.spent <= 100, "spent {} exceeded cap", snap.spent);
|
||||
}
|
||||
@@ -1,208 +0,0 @@
|
||||
//! Cold models advertise what they can do (#241).
|
||||
//!
|
||||
//! Before this, `/v1/models` reported `capabilities: []` for anything not
|
||||
//! currently loaded, because the only source of capabilities was a loaded
|
||||
//! neuron reporting its own handle. That made image generation
|
||||
//! undiscoverable in practice: the image model is evicted almost all of
|
||||
//! the time, so a client listing our models saw an entry that looked like
|
||||
//! a text model with nothing to say about itself.
|
||||
//!
|
||||
//! The fix is discovery, not configuration — neurons derive modalities
|
||||
//! from their local model cache and cortex asks. These tests pin the two
|
||||
//! properties that matter: a cold model's capabilities reach `/v1/models`,
|
||||
//! and a node with no local evidence cannot erase them.
|
||||
|
||||
mod common;
|
||||
|
||||
use cortex_core::config::{
|
||||
EvictionSettings, EvictionStrategy, GatewayConfig, GatewaySettings, NeuronEndpoint,
|
||||
};
|
||||
use cortex_gateway::state::CortexState;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
/// A catalogue holding one image model that nothing has loaded.
|
||||
fn write_catalogue() -> std::path::PathBuf {
|
||||
static SEQ: AtomicU64 = AtomicU64::new(0);
|
||||
let toml = r#"
|
||||
[[models]]
|
||||
id = "Tongyi-MAI/Z-Image-Turbo"
|
||||
harness = "candle"
|
||||
min_devices = 1
|
||||
|
||||
[aliases]
|
||||
"helexa/image" = "Tongyi-MAI/Z-Image-Turbo"
|
||||
"#;
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"cortex_test_capability_models.{}.{}.toml",
|
||||
std::process::id(),
|
||||
SEQ.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
std::fs::write(&path, toml).unwrap();
|
||||
path
|
||||
}
|
||||
|
||||
fn fleet_for(endpoint: String) -> Arc<CortexState> {
|
||||
fleet_of(vec![("gpu", endpoint)])
|
||||
}
|
||||
|
||||
fn fleet_of(neurons: Vec<(&str, String)>) -> Arc<CortexState> {
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: neurons
|
||||
.into_iter()
|
||||
.map(|(name, endpoint)| NeuronEndpoint {
|
||||
name: name.into(),
|
||||
endpoint,
|
||||
})
|
||||
.collect(),
|
||||
models_config: write_catalogue().to_string_lossy().into_owned(),
|
||||
..Default::default()
|
||||
};
|
||||
Arc::new(CortexState::from_config(&config))
|
||||
}
|
||||
|
||||
/// Serve `fleet` and return its base URL, so a test can assert the
|
||||
/// actual HTTP surface rather than the state behind it.
|
||||
async fn serve(fleet: Arc<CortexState>) -> String {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let app = cortex_gateway::build_app(fleet);
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
format!("http://{addr}")
|
||||
}
|
||||
|
||||
/// The headline case: nothing loaded anywhere, and `/v1/models` still
|
||||
/// says the image model generates images. This is the acceptance
|
||||
/// criterion from #241 — asserted on the response a client actually
|
||||
/// receives, because that is where the bug was visible.
|
||||
#[tokio::test]
|
||||
async fn cold_image_model_advertises_image_capability() {
|
||||
let neuron = common::spawn_mock_neuron_with_capabilities(
|
||||
serde_json::json!([]),
|
||||
&[("Tongyi-MAI/Z-Image-Turbo", vec!["image"])],
|
||||
)
|
||||
.await;
|
||||
let fleet = fleet_for(neuron);
|
||||
|
||||
cortex_gateway::poller::poll_once(&fleet).await;
|
||||
|
||||
let gateway = serve(fleet).await;
|
||||
let body: serde_json::Value = reqwest::get(format!("{gateway}/v1/models"))
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let entry = body["data"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|m| m["id"] == "Tongyi-MAI/Z-Image-Turbo")
|
||||
.expect("the catalogued image model should be listed");
|
||||
assert_eq!(
|
||||
entry["loaded"], false,
|
||||
"precondition: this asserts the *cold* path"
|
||||
);
|
||||
assert_eq!(
|
||||
entry["capabilities"],
|
||||
serde_json::json!(["image"]),
|
||||
"a cold image model must still advertise that it generates images"
|
||||
);
|
||||
|
||||
// The alias is the id most clients will actually send, so it has to
|
||||
// carry the capability too — advertising it only on the concrete id
|
||||
// would leave the public-facing name looking like a text model.
|
||||
let alias = body["data"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|m| m["id"] == "helexa/image")
|
||||
.expect("the tier alias should be listed");
|
||||
assert_eq!(
|
||||
alias["capabilities"],
|
||||
serde_json::json!(["image"]),
|
||||
"the alias must advertise the same capabilities as its target"
|
||||
);
|
||||
}
|
||||
|
||||
/// A node that has never cached the weights answers 404. That must leave
|
||||
/// the id unknown rather than recording an empty list, or the first node
|
||||
/// polled would permanently pin a model to no capabilities — the exact
|
||||
/// failure this work removes, reintroduced from a different direction.
|
||||
#[tokio::test]
|
||||
async fn node_without_local_evidence_records_nothing() {
|
||||
let neuron = common::spawn_mock_neuron_with_capabilities(serde_json::json!([]), &[]).await;
|
||||
let fleet = fleet_for(neuron);
|
||||
|
||||
cortex_gateway::poller::poll_once(&fleet).await;
|
||||
|
||||
assert!(
|
||||
fleet.discovered_capabilities.read().await.is_empty(),
|
||||
"a 404 means 'no local evidence', which must not be cached as an answer"
|
||||
);
|
||||
}
|
||||
|
||||
/// Weights are not spread evenly across a fleet, so the node polled
|
||||
/// first routinely lacks a model that a later one has. The back-off that
|
||||
/// stops unanswerable ids being re-probed every cycle must not let that
|
||||
/// first node's 404 suppress the question to everyone else.
|
||||
///
|
||||
/// Found live: a GGUF that two of three neurons could describe reported
|
||||
/// no capabilities at all, because the one node without it was polled
|
||||
/// first and claimed the attempt for the whole window.
|
||||
#[tokio::test]
|
||||
async fn a_node_that_cannot_answer_does_not_starve_one_that_can() {
|
||||
let empty_handed =
|
||||
common::spawn_mock_neuron_with_capabilities(serde_json::json!([]), &[]).await;
|
||||
let has_weights = common::spawn_mock_neuron_with_capabilities(
|
||||
serde_json::json!([]),
|
||||
&[("Tongyi-MAI/Z-Image-Turbo", vec!["image"])],
|
||||
)
|
||||
.await;
|
||||
// Config order is poll order, so the node that 404s goes first.
|
||||
let fleet = fleet_of(vec![("first", empty_handed), ("second", has_weights)]);
|
||||
|
||||
cortex_gateway::poller::poll_once(&fleet).await;
|
||||
|
||||
assert_eq!(
|
||||
fleet
|
||||
.discovered_capabilities
|
||||
.read()
|
||||
.await
|
||||
.get("Tongyi-MAI/Z-Image-Turbo")
|
||||
.cloned(),
|
||||
Some(vec!["image".to_string()]),
|
||||
"the second node's answer must survive the first node's 404, \
|
||||
in the same poll cycle"
|
||||
);
|
||||
}
|
||||
|
||||
/// An empty list is likewise not an answer worth keeping: a repo cached
|
||||
/// without weights says nothing about what other nodes can serve.
|
||||
#[tokio::test]
|
||||
async fn empty_capability_list_is_not_recorded() {
|
||||
let neuron = common::spawn_mock_neuron_with_capabilities(
|
||||
serde_json::json!([]),
|
||||
&[("Tongyi-MAI/Z-Image-Turbo", vec![])],
|
||||
)
|
||||
.await;
|
||||
let fleet = fleet_for(neuron);
|
||||
|
||||
cortex_gateway::poller::poll_once(&fleet).await;
|
||||
|
||||
assert!(
|
||||
fleet.discovered_capabilities.read().await.is_empty(),
|
||||
"an empty list must not be cached as this model's capabilities"
|
||||
);
|
||||
}
|
||||
@@ -3,7 +3,7 @@
|
||||
use axum::body::Body;
|
||||
use axum::extract::Path;
|
||||
use axum::http::header;
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum::response::Response;
|
||||
use axum::routing::{get, post};
|
||||
use axum::{Json, Router};
|
||||
use cortex_core::config::{
|
||||
@@ -22,7 +22,6 @@ use tokio::net::TcpListener;
|
||||
/// - GET /models/:id/endpoint (returns the inference URL)
|
||||
/// - POST /models/unload (accepts unload requests)
|
||||
/// - GET /v1/chat/completions + POST /v1/chat/completions (inference)
|
||||
///
|
||||
/// Returns the neuron base URL.
|
||||
pub async fn spawn_mock_neuron() -> String {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
@@ -44,7 +43,6 @@ pub async fn spawn_mock_neuron() -> String {
|
||||
post(|Json(_body): Json<Value>| async { Json(json!({"status": "unloaded"})) }),
|
||||
)
|
||||
.route("/v1/chat/completions", post(mock_chat_completions))
|
||||
.route("/v1/responses", post(mock_responses))
|
||||
.route("/v1/models", get(mock_v1_models));
|
||||
|
||||
tokio::spawn(async move {
|
||||
@@ -54,64 +52,9 @@ pub async fn spawn_mock_neuron() -> String {
|
||||
base_url
|
||||
}
|
||||
|
||||
/// Like [`spawn_mock_neuron`] but captures the JSON body of every
|
||||
/// `POST /v1/chat/completions` it receives into the returned handle, so
|
||||
/// a test can assert what the gateway *actually forwarded upstream*
|
||||
/// (e.g. that Anthropic-shaped tools were reshaped to OpenAI form).
|
||||
pub async fn spawn_capturing_mock_neuron() -> (String, Arc<std::sync::Mutex<Vec<Value>>>) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let base_url = format!("http://{addr}");
|
||||
let inference_url = base_url.clone();
|
||||
let captured: Arc<std::sync::Mutex<Vec<Value>>> = Arc::new(std::sync::Mutex::new(Vec::new()));
|
||||
let sink = captured.clone();
|
||||
|
||||
let app = Router::new()
|
||||
.route("/models", get(mock_neuron_list_models))
|
||||
.route(
|
||||
"/models/{model_id}/endpoint",
|
||||
get(move |Path(_): Path<String>| {
|
||||
let url = inference_url.clone();
|
||||
async move { Json(json!({"url": url})) }
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/v1/chat/completions",
|
||||
post(move |Json(body): Json<Value>| {
|
||||
let sink = sink.clone();
|
||||
async move {
|
||||
let model = body
|
||||
.get("model")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("unknown");
|
||||
let resp = json!({
|
||||
"id": "chatcmpl-capture-001",
|
||||
"object": "chat.completion",
|
||||
"created": 1700000000_u64,
|
||||
"model": model,
|
||||
"choices": [{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "Hello from mock backend"},
|
||||
"finish_reason": "stop"
|
||||
}],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}
|
||||
});
|
||||
sink.lock().unwrap().push(body);
|
||||
Json(resp)
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
|
||||
(base_url, captured)
|
||||
}
|
||||
|
||||
async fn mock_neuron_list_models() -> Json<Value> {
|
||||
Json(json!([
|
||||
{"id": "test-model", "harness": "candle", "status": "loaded", "devices": [0], "vram_used_mb": 8000, "capabilities": ["text"], "tool_call": false, "reasoning": false}
|
||||
{"id": "test-model", "harness": "mistralrs", "status": "loaded", "devices": [0], "vram_used_mb": 8000}
|
||||
]))
|
||||
}
|
||||
|
||||
@@ -149,39 +92,6 @@ async fn mock_chat_completions(Json(body): Json<Value>) -> Json<Value> {
|
||||
}))
|
||||
}
|
||||
|
||||
async fn mock_responses(Json(body): Json<Value>) -> Json<Value> {
|
||||
let model = body
|
||||
.get("model")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("unknown");
|
||||
// Echo the model field back and synthesise a tiny ResponsesResponse.
|
||||
// Mirrors the shape neuron's /v1/responses handler emits so the
|
||||
// gateway test only needs to assert the proxy round-tripped it.
|
||||
Json(json!({
|
||||
"id": "resp-test-001",
|
||||
"object": "response",
|
||||
"created_at": 1700000000_u64,
|
||||
"status": "completed",
|
||||
"model": model,
|
||||
"output": [{
|
||||
"type": "message",
|
||||
"id": "msg-test-001",
|
||||
"role": "assistant",
|
||||
"content": [{
|
||||
"type": "output_text",
|
||||
"text": "Hello from mock backend",
|
||||
"annotations": []
|
||||
}],
|
||||
"status": "completed"
|
||||
}],
|
||||
"usage": {
|
||||
"input_tokens": 5,
|
||||
"output_tokens": 5,
|
||||
"total_tokens": 10
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
/// Spawns a mock neuron that returns SSE streaming responses for chat completions.
|
||||
pub async fn spawn_streaming_mock_neuron(chunk_count: usize, chunk_delay: Duration) -> String {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
@@ -251,120 +161,8 @@ pub async fn spawn_streaming_mock_neuron(chunk_count: usize, chunk_delay: Durati
|
||||
base_url
|
||||
}
|
||||
|
||||
/// Like `spawn_streaming_mock_neuron`, but the stream ends with an
|
||||
/// OpenAI `stream_options.include_usage`-style final chunk (empty
|
||||
/// choices + usage object) before `[DONE]` — the shape the gateway's
|
||||
/// token metrics (#21) extract counts from.
|
||||
pub async fn spawn_streaming_mock_neuron_with_usage(
|
||||
chunk_count: usize,
|
||||
chunk_delay: Duration,
|
||||
prompt_tokens: u64,
|
||||
completion_tokens: u64,
|
||||
) -> String {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let base_url = format!("http://{addr}");
|
||||
let inference_url = base_url.clone();
|
||||
|
||||
let app = Router::new()
|
||||
.route("/models", get(mock_neuron_list_models))
|
||||
.route(
|
||||
"/models/{model_id}/endpoint",
|
||||
get(move |Path(_model_id): Path<String>| {
|
||||
let url = inference_url.clone();
|
||||
async move { Json(json!({"url": url})) }
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/v1/chat/completions",
|
||||
post(move |Json(body): Json<Value>| async move {
|
||||
let model = body
|
||||
.get("model")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("unknown")
|
||||
.to_string();
|
||||
|
||||
let mut chunks: Vec<String> = (0..chunk_count)
|
||||
.map(|i| {
|
||||
let chunk = json!({
|
||||
"id": "chatcmpl-stream-002",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1700000000_u64,
|
||||
"model": model,
|
||||
"choices": [{
|
||||
"index": 0,
|
||||
"delta": { "content": format!("token{i}") },
|
||||
"finish_reason": null
|
||||
}]
|
||||
});
|
||||
format!("data: {chunk}\n\n")
|
||||
})
|
||||
.collect();
|
||||
let usage_chunk = json!({
|
||||
"id": "chatcmpl-stream-002",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1700000000_u64,
|
||||
"model": model,
|
||||
"choices": [],
|
||||
"usage": {
|
||||
"prompt_tokens": prompt_tokens,
|
||||
"completion_tokens": completion_tokens,
|
||||
"total_tokens": prompt_tokens + completion_tokens
|
||||
}
|
||||
});
|
||||
chunks.push(format!("data: {usage_chunk}\n\n"));
|
||||
chunks.push("data: [DONE]\n\n".to_string());
|
||||
|
||||
let delay = chunk_delay;
|
||||
let stream = stream::iter(chunks).then(move |chunk| async move {
|
||||
tokio::time::sleep(delay).await;
|
||||
Ok::<_, std::convert::Infallible>(chunk)
|
||||
});
|
||||
|
||||
Response::builder()
|
||||
.header(header::CONTENT_TYPE, "text/event-stream")
|
||||
.header(header::CACHE_CONTROL, "no-cache")
|
||||
.body(Body::from_stream(stream))
|
||||
.unwrap()
|
||||
}),
|
||||
);
|
||||
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
|
||||
base_url
|
||||
}
|
||||
|
||||
/// Spawns a mock neuron with a custom models list.
|
||||
pub async fn spawn_mock_neuron_with_models(models_response: Value) -> String {
|
||||
spawn_mock_neuron_with_models_and_health(models_response, default_health_response()).await
|
||||
}
|
||||
|
||||
/// Default `/health` response used by mocks that don't care about the
|
||||
/// activation field — empty devices, no in-flight pre-warm, state=ready.
|
||||
pub fn default_health_response() -> Value {
|
||||
json!({
|
||||
"uptime_secs": 0,
|
||||
"devices": [],
|
||||
"activation": {
|
||||
"state": "ready",
|
||||
"pending": [],
|
||||
"in_progress": null,
|
||||
"completed": [],
|
||||
"failed": []
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Variant of `spawn_mock_neuron_with_models` that also serves a
|
||||
/// `/health` body. Used by tests that drive the gateway's activation
|
||||
/// surface (poller reading /health, /v1/models synthesising Loading
|
||||
/// locations from in_progress / pending).
|
||||
pub async fn spawn_mock_neuron_with_models_and_health(
|
||||
models_response: Value,
|
||||
health_response: Value,
|
||||
) -> String {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let base_url = format!("http://{addr}");
|
||||
@@ -378,13 +176,6 @@ pub async fn spawn_mock_neuron_with_models_and_health(
|
||||
async move { Json(resp) }
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/health",
|
||||
get(move || {
|
||||
let resp = health_response.clone();
|
||||
async move { Json(resp) }
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/models/{model_id}/endpoint",
|
||||
get(move |Path(_model_id): Path<String>| {
|
||||
@@ -405,73 +196,6 @@ pub async fn spawn_mock_neuron_with_models_and_health(
|
||||
base_url
|
||||
}
|
||||
|
||||
/// Variant that also serves `GET /models/{id}/capabilities` (#241), the
|
||||
/// route a neuron answers from its local model cache for models nothing
|
||||
/// has loaded.
|
||||
///
|
||||
/// `capabilities` maps model id → modalities. Any id not listed gets a
|
||||
/// 404, which is how a real neuron says "these weights were never cached
|
||||
/// here" — distinct from an empty list, and the distinction is load-
|
||||
/// bearing, so the mock has to be able to express both.
|
||||
#[allow(dead_code)]
|
||||
pub async fn spawn_mock_neuron_with_capabilities(
|
||||
models_response: Value,
|
||||
capabilities: &[(&str, Vec<&str>)],
|
||||
) -> String {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let base_url = format!("http://{addr}");
|
||||
let health_response = default_health_response();
|
||||
let table: std::collections::HashMap<String, Vec<String>> = capabilities
|
||||
.iter()
|
||||
.map(|(id, caps)| {
|
||||
(
|
||||
(*id).to_string(),
|
||||
caps.iter().map(|c| (*c).to_string()).collect(),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
let table = Arc::new(table);
|
||||
|
||||
let app = Router::new()
|
||||
.route(
|
||||
"/models",
|
||||
get(move || {
|
||||
let resp = models_response.clone();
|
||||
async move { Json(resp) }
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/health",
|
||||
get(move || {
|
||||
let resp = health_response.clone();
|
||||
async move { Json(resp) }
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/models/{model_id}/capabilities",
|
||||
get(move |Path(model_id): Path<String>| {
|
||||
let table = table.clone();
|
||||
async move {
|
||||
match table.get(&model_id) {
|
||||
Some(caps) => Json(json!({ "capabilities": caps })).into_response(),
|
||||
None => (
|
||||
axum::http::StatusCode::NOT_FOUND,
|
||||
Json(json!({"error": "no local evidence"})),
|
||||
)
|
||||
.into_response(),
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
|
||||
base_url
|
||||
}
|
||||
|
||||
/// Spawns the cortex gateway with a single neuron pointing at `mock_url`.
|
||||
/// The node is pre-seeded as healthy with one loaded model ("test-model").
|
||||
/// Returns the gateway's base URL.
|
||||
@@ -496,8 +220,6 @@ pub async fn spawn_gateway_with_state(mock_url: &str) -> (Arc<CortexState>, Stri
|
||||
endpoint: mock_url.to_string(),
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
@@ -514,11 +236,6 @@ pub async fn spawn_gateway_with_state(mock_url: &str) -> (Arc<CortexState>, Stri
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: None,
|
||||
vram_estimate_mb: Some(8000),
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -533,31 +250,3 @@ pub async fn spawn_gateway_with_state(mock_url: &str) -> (Arc<CortexState>, Stri
|
||||
|
||||
(fleet, format!("http://{addr}"))
|
||||
}
|
||||
|
||||
/// Build a gateway state over two mock neurons (no poller; we seed state).
|
||||
pub async fn two_neuron_fleet(endpoint_a: &str, endpoint_b: &str) -> Arc<CortexState> {
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![
|
||||
NeuronEndpoint {
|
||||
name: "node-a".into(),
|
||||
endpoint: endpoint_a.to_string(),
|
||||
},
|
||||
NeuronEndpoint {
|
||||
name: "node-b".into(),
|
||||
endpoint: endpoint_b.to_string(),
|
||||
},
|
||||
],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
Arc::new(CortexState::from_config(&config))
|
||||
}
|
||||
|
||||
@@ -1,141 +0,0 @@
|
||||
mod common;
|
||||
|
||||
use serde_json::json;
|
||||
|
||||
#[tokio::test]
|
||||
async fn error_response_model_not_found() {
|
||||
let neuron_url = common::spawn_mock_neuron().await;
|
||||
let gateway_url = common::spawn_gateway(&neuron_url).await;
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
// Request a model that isn't loaded on the mock neuron.
|
||||
let resp = client
|
||||
.post(format!("{gateway_url}/v1/chat/completions"))
|
||||
.header("Content-Type", "application/json")
|
||||
.json(&json!({
|
||||
"model": "nonexistent-model",
|
||||
"messages": [{"role": "user", "content": "hi"}]
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.expect("request should succeed");
|
||||
|
||||
assert_eq!(resp.status(), axum::http::StatusCode::NOT_FOUND);
|
||||
|
||||
let body: serde_json::Value = resp.json().await.expect("valid json");
|
||||
let err = body.get("error").expect("response has error object");
|
||||
|
||||
// Broad type categorization
|
||||
assert_eq!(err.get("type").unwrap(), "invalid_request_error");
|
||||
// Specific machine-readable code
|
||||
assert_eq!(
|
||||
err.get("code").unwrap().as_str().unwrap(),
|
||||
"model_not_found"
|
||||
);
|
||||
// param is always null
|
||||
assert!(err.get("param").unwrap().is_null());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn error_response_missing_model_field() {
|
||||
let neuron_url = common::spawn_mock_neuron().await;
|
||||
let gateway_url = common::spawn_gateway(&neuron_url).await;
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
// Request without the required `model` field.
|
||||
let resp = client
|
||||
.post(format!("{gateway_url}/v1/chat/completions"))
|
||||
.header("Content-Type", "application/json")
|
||||
.json(&json!({
|
||||
"messages": [{"role": "user", "content": "hi"}]
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.expect("request should succeed");
|
||||
|
||||
assert_eq!(resp.status(), axum::http::StatusCode::BAD_REQUEST);
|
||||
|
||||
let body: serde_json::Value = resp.json().await.expect("valid json");
|
||||
let err = body.get("error").expect("response has error object");
|
||||
|
||||
assert_eq!(err.get("type").unwrap(), "invalid_request_error");
|
||||
assert_eq!(
|
||||
err.get("code").unwrap().as_str().unwrap(),
|
||||
"missing_model_field"
|
||||
);
|
||||
assert!(err.get("param").unwrap().is_null());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn error_response_no_healthy_nodes() {
|
||||
use cortex_core::config::{EvictionSettings, GatewayConfig, GatewaySettings, NeuronEndpoint};
|
||||
use std::sync::Arc;
|
||||
|
||||
// Create a gateway config with a neuron pointing at an unreachable port so no node is ever healthy.
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: cortex_core::config::EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![NeuronEndpoint {
|
||||
name: "dead-node".into(),
|
||||
endpoint: "http://127.0.0.1:1".into(),
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(cortex_gateway::state::CortexState::from_config(&config));
|
||||
|
||||
let app = cortex_gateway::build_app(fleet);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
|
||||
// Allow the poller a moment to mark the node unhealthy.
|
||||
tokio::time::sleep(std::time::Duration::from_millis(200)).await;
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let resp = client
|
||||
.post(format!("http://{addr}/v1/chat/completions"))
|
||||
.header("Content-Type", "application/json")
|
||||
.json(&json!({
|
||||
"model": "any-model",
|
||||
"messages": [{"role": "user", "content": "hi"}]
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.expect("request should succeed");
|
||||
|
||||
assert_eq!(resp.status(), axum::http::StatusCode::SERVICE_UNAVAILABLE);
|
||||
|
||||
// Transient 503 — the gateway advertises Retry-After so OpenAI-compatible
|
||||
// clients back off and retry rather than surfacing an opaque error (#63).
|
||||
let retry_after = resp
|
||||
.headers()
|
||||
.get(reqwest::header::RETRY_AFTER)
|
||||
.expect("transient 503 must carry Retry-After")
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_string();
|
||||
assert_eq!(retry_after, "5");
|
||||
|
||||
let body: serde_json::Value = resp.json().await.expect("valid json");
|
||||
let err = body.get("error").expect("response has error object");
|
||||
|
||||
assert_eq!(err.get("type").unwrap(), "api_error");
|
||||
assert_eq!(
|
||||
err.get("code").unwrap().as_str().unwrap(),
|
||||
"service_unavailable"
|
||||
);
|
||||
assert!(err.get("param").unwrap().is_null());
|
||||
}
|
||||
@@ -56,76 +56,6 @@ async fn spawn_eviction_mock() -> (String, Arc<tokio::sync::Mutex<Vec<String>>>)
|
||||
(base_url, unloaded)
|
||||
}
|
||||
|
||||
/// A fleet whose catalogue ranks models, so the displacement rules are
|
||||
/// exercised rather than defaulted. Writes the catalogue to a temp file
|
||||
/// because `CortexState` loads it from a path.
|
||||
fn make_fleet_with_catalogue(
|
||||
endpoint: &str,
|
||||
catalogue_toml: &str,
|
||||
tag: &str,
|
||||
) -> (Arc<CortexState>, std::path::PathBuf) {
|
||||
let path = std::env::temp_dir().join(format!("cortex-evict-catalogue-{tag}.toml"));
|
||||
std::fs::write(&path, catalogue_toml).expect("write test catalogue");
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![NeuronEndpoint {
|
||||
name: "gpu-node".into(),
|
||||
endpoint: endpoint.to_string(),
|
||||
}],
|
||||
models_config: path.to_string_lossy().into_owned(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
(Arc::new(CortexState::from_config(&config)), path)
|
||||
}
|
||||
|
||||
fn loaded(id: &str, age_secs: i64) -> ModelEntry {
|
||||
ModelEntry {
|
||||
id: id.into(),
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: Some(Utc::now() - chrono::Duration::seconds(age_secs)),
|
||||
vram_estimate_mb: Some(8000),
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The fleet policy under test. Two residency classes: image generation
|
||||
/// and the mid tier share a node and take turns on it; the flagship and
|
||||
/// the frontier model share a bigger one and take turns on that. Nothing
|
||||
/// in the everyday class may touch the big-node class.
|
||||
const TIERED: &str = r#"
|
||||
[[models]]
|
||||
id = "flagship"
|
||||
harness = "candle"
|
||||
residency_priority = 300
|
||||
|
||||
[[models]]
|
||||
id = "frontier"
|
||||
harness = "candle"
|
||||
residency_priority = 300
|
||||
|
||||
[[models]]
|
||||
id = "image"
|
||||
harness = "candle"
|
||||
residency_priority = 200
|
||||
|
||||
[[models]]
|
||||
id = "mid"
|
||||
harness = "candle"
|
||||
residency_priority = 200
|
||||
"#;
|
||||
|
||||
fn make_fleet(endpoint: &str, defrag_after: u32) -> Arc<CortexState> {
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
@@ -141,8 +71,6 @@ fn make_fleet(endpoint: &str, defrag_after: u32) -> Arc<CortexState> {
|
||||
endpoint: endpoint.to_string(),
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
Arc::new(CortexState::from_config(&config))
|
||||
}
|
||||
@@ -163,11 +91,6 @@ async fn test_evict_lru_model() {
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: Some(Utc::now() - chrono::Duration::hours(2)),
|
||||
vram_estimate_mb: Some(8000),
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
node.models.insert(
|
||||
@@ -177,16 +100,11 @@ async fn test_evict_lru_model() {
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: Some(Utc::now()),
|
||||
vram_estimate_mb: Some(8000),
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let evicted = cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node", None)
|
||||
let evicted = cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node")
|
||||
.await
|
||||
.expect("eviction should succeed");
|
||||
|
||||
@@ -219,7 +137,7 @@ async fn test_eviction_nothing_to_evict() {
|
||||
nodes.get_mut("gpu-node").unwrap().healthy = true;
|
||||
}
|
||||
|
||||
let evicted = cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node", None)
|
||||
let evicted = cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node")
|
||||
.await
|
||||
.expect("eviction should succeed");
|
||||
|
||||
@@ -245,16 +163,11 @@ async fn test_eviction_increments_lifecycle_cycles() {
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: None,
|
||||
vram_estimate_mb: None,
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node", None)
|
||||
cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node")
|
||||
.await
|
||||
.expect("eviction should succeed");
|
||||
|
||||
@@ -301,129 +214,3 @@ async fn test_last_accessed_updated_on_request() {
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
|
||||
/// Image generation must take the mid tier's node when it needs it —
|
||||
/// this is existing fleet behaviour and the change must preserve it.
|
||||
#[tokio::test]
|
||||
async fn image_generation_evicts_the_mid_tier() {
|
||||
let (mock_url, unloaded) = spawn_eviction_mock().await;
|
||||
let (fleet, path) = make_fleet_with_catalogue(&mock_url, TIERED, "image-takes-mid");
|
||||
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("gpu-node").unwrap();
|
||||
node.healthy = true;
|
||||
node.models.insert("mid".into(), loaded("mid", 60));
|
||||
}
|
||||
|
||||
let evicted = cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node", Some("image"))
|
||||
.await
|
||||
.expect("eviction should succeed");
|
||||
|
||||
assert_eq!(evicted, Some("mid".to_string()));
|
||||
assert_eq!(unloaded.lock().await.as_slice(), ["mid"]);
|
||||
std::fs::remove_file(path).ok();
|
||||
}
|
||||
|
||||
/// Image generation must never take the flagship's node. Its device
|
||||
/// constraints alone would let it land there, so nothing but priority
|
||||
/// stops this.
|
||||
#[tokio::test]
|
||||
async fn image_generation_cannot_evict_the_flagship() {
|
||||
let (mock_url, unloaded) = spawn_eviction_mock().await;
|
||||
let (fleet, path) = make_fleet_with_catalogue(&mock_url, TIERED, "image-spares-flagship");
|
||||
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("gpu-node").unwrap();
|
||||
node.healthy = true;
|
||||
node.models
|
||||
.insert("flagship".into(), loaded("flagship", 9999));
|
||||
}
|
||||
|
||||
let evicted = cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node", Some("image"))
|
||||
.await
|
||||
.expect("eviction should succeed");
|
||||
|
||||
assert_eq!(
|
||||
evicted, None,
|
||||
"the flagship outranks image generation, however stale it is"
|
||||
);
|
||||
assert!(unloaded.lock().await.is_empty());
|
||||
std::fs::remove_file(path).ok();
|
||||
}
|
||||
|
||||
/// The frontier tier may cold-swap the flagship off its node.
|
||||
#[tokio::test]
|
||||
async fn the_frontier_tier_evicts_the_flagship() {
|
||||
let (mock_url, unloaded) = spawn_eviction_mock().await;
|
||||
let (fleet, path) = make_fleet_with_catalogue(&mock_url, TIERED, "frontier-takes-flagship");
|
||||
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("gpu-node").unwrap();
|
||||
node.healthy = true;
|
||||
node.models
|
||||
.insert("flagship".into(), loaded("flagship", 10));
|
||||
}
|
||||
|
||||
let evicted = cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node", Some("frontier"))
|
||||
.await
|
||||
.expect("eviction should succeed");
|
||||
|
||||
assert_eq!(evicted, Some("flagship".to_string()));
|
||||
assert_eq!(unloaded.lock().await.as_slice(), ["flagship"]);
|
||||
std::fs::remove_file(path).ok();
|
||||
}
|
||||
|
||||
/// LRU still decides *which* victim, but only among the models the
|
||||
/// incoming one outranks. Here the flagship is by far the stalest, so a
|
||||
/// purely age-ordered evictor would take it.
|
||||
#[tokio::test]
|
||||
async fn lru_picks_the_oldest_displaceable_model_not_the_oldest_model() {
|
||||
let (mock_url, unloaded) = spawn_eviction_mock().await;
|
||||
let (fleet, path) = make_fleet_with_catalogue(&mock_url, TIERED, "lru-within-rank");
|
||||
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("gpu-node").unwrap();
|
||||
node.healthy = true;
|
||||
node.models
|
||||
.insert("flagship".into(), loaded("flagship", 9999));
|
||||
node.models.insert("mid".into(), loaded("mid", 60));
|
||||
}
|
||||
|
||||
let evicted = cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node", Some("image"))
|
||||
.await
|
||||
.expect("eviction should succeed");
|
||||
|
||||
assert_eq!(evicted, Some("mid".to_string()));
|
||||
assert_eq!(unloaded.lock().await.as_slice(), ["mid"]);
|
||||
std::fs::remove_file(path).ok();
|
||||
}
|
||||
|
||||
/// The other half of the cold-swap: after an image generation has taken
|
||||
/// the node, the next text request must be able to take it back. A
|
||||
/// strict priority order would let whichever model arrived first hold
|
||||
/// the node forever, which looks to a user like the text tier vanishing
|
||||
/// once somebody generated an image.
|
||||
#[tokio::test]
|
||||
async fn the_mid_tier_takes_its_node_back_from_image_generation() {
|
||||
let (mock_url, unloaded) = spawn_eviction_mock().await;
|
||||
let (fleet, path) = make_fleet_with_catalogue(&mock_url, TIERED, "mid-swaps-back");
|
||||
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("gpu-node").unwrap();
|
||||
node.healthy = true;
|
||||
node.models.insert("image".into(), loaded("image", 30));
|
||||
}
|
||||
|
||||
let evicted = cortex_gateway::evictor::evict_lru_on_node(&fleet, "gpu-node", Some("mid"))
|
||||
.await
|
||||
.expect("eviction should succeed");
|
||||
|
||||
assert_eq!(evicted, Some("image".to_string()));
|
||||
assert_eq!(unloaded.lock().await.as_slice(), ["image"]);
|
||||
std::fs::remove_file(path).ok();
|
||||
}
|
||||
|
||||
@@ -1,138 +0,0 @@
|
||||
//! Router: a catalogued model whose only topologically-feasible neuron is
|
||||
//! currently unhealthy is a *transient* condition (retryable 503), not a
|
||||
//! permanent 404. This is the exact shape of the beast incident: benjy/
|
||||
//! quadbrat (1 GPU, healthy) can't host the 27B, and beast (2 GPU) — the
|
||||
//! sole feasible node — briefly drops out → clients must back off and retry,
|
||||
//! not hard-fail.
|
||||
|
||||
use cortex_core::config::{
|
||||
EvictionSettings, EvictionStrategy, GatewayConfig, GatewaySettings, NeuronEndpoint,
|
||||
};
|
||||
use cortex_core::discovery::{DeviceInfo, DiscoveryResponse};
|
||||
use cortex_gateway::router::{self, RouteError};
|
||||
use cortex_gateway::state::CortexState;
|
||||
use std::sync::Arc;
|
||||
|
||||
fn devices(n: usize) -> Vec<DeviceInfo> {
|
||||
(0..n)
|
||||
.map(|i| DeviceInfo {
|
||||
index: i as u32,
|
||||
name: "RTX 5090".into(),
|
||||
vram_total_mb: 32_768,
|
||||
compute_capability: "9.0".into(),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn discovery(host: &str, n_devices: usize) -> DiscoveryResponse {
|
||||
DiscoveryResponse {
|
||||
hostname: host.into(),
|
||||
os: "Linux".into(),
|
||||
kernel: "7.0".into(),
|
||||
cuda_version: Some("13.0".into()),
|
||||
driver_version: Some("999".into()),
|
||||
devices: devices(n_devices),
|
||||
harnesses: vec!["candle".into()],
|
||||
cuda_unavailable_reason: None,
|
||||
max_prompt_tokens: 49_152,
|
||||
}
|
||||
}
|
||||
|
||||
/// Catalogue with one model needing 2 devices. Returns a temp path.
|
||||
///
|
||||
/// The filename is unique per call: tests in this binary run concurrently,
|
||||
/// and a shared path let one test truncate the file (`fs::write` truncates
|
||||
/// before writing) while another was loading it — yielding an empty
|
||||
/// catalogue and a spurious `ModelNotFound` instead of the routing error
|
||||
/// under test.
|
||||
fn write_catalogue() -> std::path::PathBuf {
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
static SEQ: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
let toml = r#"
|
||||
[[models]]
|
||||
id = "big-model"
|
||||
harness = "candle"
|
||||
min_devices = 2
|
||||
"#;
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"cortex_test_feasibility_models.{}.{}.toml",
|
||||
std::process::id(),
|
||||
SEQ.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
std::fs::write(&path, toml).unwrap();
|
||||
path
|
||||
}
|
||||
|
||||
async fn fleet_with(big_healthy: bool, big_devices: usize) -> Arc<CortexState> {
|
||||
let cat = write_catalogue();
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![
|
||||
NeuronEndpoint {
|
||||
name: "small".into(),
|
||||
endpoint: "http://127.0.0.1:1".into(),
|
||||
},
|
||||
NeuronEndpoint {
|
||||
name: "big".into(),
|
||||
endpoint: "http://127.0.0.1:2".into(),
|
||||
},
|
||||
],
|
||||
models_config: cat.to_string_lossy().into_owned(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
// "small" is healthy but only has 1 GPU → not feasible for the model.
|
||||
let small = nodes.get_mut("small").unwrap();
|
||||
small.healthy = true;
|
||||
small.discovery = Some(discovery("small", 1));
|
||||
// "big" has enough GPUs but its health is the variable under test.
|
||||
let big = nodes.get_mut("big").unwrap();
|
||||
big.healthy = big_healthy;
|
||||
big.discovery = Some(discovery("big", big_devices));
|
||||
}
|
||||
fleet
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn feasible_node_unhealthy_is_transient_503() {
|
||||
// big (2 GPU, the only feasible node) is unhealthy; small (1 GPU) is
|
||||
// healthy but can't host the model → retryable, not a permanent 404.
|
||||
let fleet = fleet_with(false, 2).await;
|
||||
let err = router::resolve(&fleet, "big-model")
|
||||
.await
|
||||
.expect_err("model can't be served right now");
|
||||
assert!(
|
||||
matches!(err, RouteError::FeasibleNodeUnhealthy { .. }),
|
||||
"expected FeasibleNodeUnhealthy, got {err:?}"
|
||||
);
|
||||
assert_eq!(err.http_status(), 503);
|
||||
assert_eq!(err.retry_after_secs(), Some(3));
|
||||
assert_eq!(err.code(), "service_unavailable");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn no_node_can_ever_satisfy_is_permanent_404() {
|
||||
// big is healthy but only has 1 GPU now (e.g. topology genuinely can't
|
||||
// satisfy min_devices=2 anywhere) → permanent, non-retryable 404.
|
||||
let fleet = fleet_with(true, 1).await;
|
||||
let err = router::resolve(&fleet, "big-model")
|
||||
.await
|
||||
.expect_err("no feasible topology");
|
||||
assert!(
|
||||
matches!(err, RouteError::NoFeasibleNeuron { .. }),
|
||||
"expected NoFeasibleNeuron, got {err:?}"
|
||||
);
|
||||
assert_eq!(err.http_status(), 404);
|
||||
assert_eq!(err.retry_after_secs(), None);
|
||||
}
|
||||
@@ -1,266 +0,0 @@
|
||||
//! `/v1/images/generations` proxy tests (#201).
|
||||
//!
|
||||
//! The mock neuron answers the images endpoint with a tiny valid
|
||||
//! envelope; the gateway must route by model, forward verbatim, and
|
||||
//! pass the response (including `usage.helexa_image_units`) through
|
||||
//! untouched.
|
||||
|
||||
mod common;
|
||||
|
||||
use axum::Router;
|
||||
use axum::extract::Path;
|
||||
use axum::response::Json;
|
||||
use axum::routing::{get, post};
|
||||
use common::spawn_gateway;
|
||||
use serde_json::{Value, json};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
/// Mock neuron that serves the images endpoint for "test-model".
|
||||
async fn spawn_images_mock_neuron() -> String {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let base_url = format!("http://{addr}");
|
||||
let inference_url = base_url.clone();
|
||||
|
||||
let app = Router::new()
|
||||
.route(
|
||||
"/models",
|
||||
get(|| async {
|
||||
Json(json!([
|
||||
{"id": "test-model", "harness": "candle", "status": "loaded",
|
||||
"devices": [0], "vram_used_mb": 14000,
|
||||
"capabilities": ["image"], "tool_call": false, "reasoning": false}
|
||||
]))
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/models/{model_id}/endpoint",
|
||||
get(move |Path(_): Path<String>| {
|
||||
let url = inference_url.clone();
|
||||
async move { Json(json!({"url": url})) }
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/v1/images/generations",
|
||||
post(|Json(body): Json<Value>| async move {
|
||||
// Echo enough of the request to prove verbatim forwarding.
|
||||
let seed = body.get("seed").cloned().unwrap_or(Value::Null);
|
||||
Json(json!({
|
||||
"created": 1700000000_u64,
|
||||
"data": [{"b64_json": "aGVsZXhh"}],
|
||||
"usage": {
|
||||
"helexa_image_units": 9.437184,
|
||||
"helexa_timing": {
|
||||
"encode_ms": 1673, "denoise_ms": 7894,
|
||||
"decode_ms": 1835, "steps": 9, "cfg": false
|
||||
}
|
||||
},
|
||||
"echo_seed": seed
|
||||
}))
|
||||
}),
|
||||
);
|
||||
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
base_url
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_images_proxy_round_trip() {
|
||||
let mock_url = spawn_images_mock_neuron().await;
|
||||
let gateway_url = spawn_gateway(&mock_url).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway_url}/v1/images/generations"))
|
||||
.json(&json!({
|
||||
"model": "test-model",
|
||||
"prompt": "a neon sign reading HELEXA",
|
||||
"size": "1024x1024",
|
||||
"seed": 42
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.expect("request should succeed");
|
||||
|
||||
assert_eq!(resp.status(), 200);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["data"][0]["b64_json"], "aGVsZXhh");
|
||||
// Metering unit passthrough (#202 reads this at the gateway).
|
||||
assert!(
|
||||
(body["usage"]["helexa_image_units"].as_f64().unwrap() - 9.437184).abs() < 1e-9,
|
||||
"helexa_image_units must pass through untouched"
|
||||
);
|
||||
assert_eq!(body["usage"]["helexa_timing"]["steps"], 9);
|
||||
// The request body reached the neuron verbatim.
|
||||
assert_eq!(body["echo_seed"], 42);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_images_missing_model_field() {
|
||||
let mock_url = spawn_images_mock_neuron().await;
|
||||
let gateway_url = spawn_gateway(&mock_url).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway_url}/v1/images/generations"))
|
||||
.json(&json!({"prompt": "no model here"}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), 400);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["error"]["code"], "missing_model_field");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_images_unknown_model_404() {
|
||||
let mock_url = spawn_images_mock_neuron().await;
|
||||
let gateway_url = spawn_gateway(&mock_url).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway_url}/v1/images/generations"))
|
||||
.json(&json!({"model": "no-such-model", "prompt": "x"}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), 404);
|
||||
}
|
||||
|
||||
// ── image budget enforcement (#202) ──────────────────────────────
|
||||
|
||||
use cortex_core::config::{
|
||||
ApiKeyConfig, EntitlementsConfig, EvictionSettings, EvictionStrategy, GatewayConfig,
|
||||
GatewaySettings, NeuronEndpoint,
|
||||
};
|
||||
use cortex_core::entitlements::CapWindow;
|
||||
use cortex_core::node::{ModelEntry, ModelStatus};
|
||||
use cortex_gateway::state::CortexState;
|
||||
use std::sync::Arc;
|
||||
|
||||
async fn spawn_keyed_gateway(neuron_url: &str, hard_cap: u64) -> String {
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![NeuronEndpoint {
|
||||
name: "mock-node".into(),
|
||||
endpoint: neuron_url.to_string(),
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: EntitlementsConfig {
|
||||
require_auth: true,
|
||||
keys: vec![ApiKeyConfig {
|
||||
key: "sk-img".into(),
|
||||
account_id: "acct-img".into(),
|
||||
key_id: Some("key-img".into()),
|
||||
hard_cap: Some(hard_cap),
|
||||
window: CapWindow::Balance,
|
||||
}],
|
||||
},
|
||||
upstream: Default::default(),
|
||||
};
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("mock-node").unwrap();
|
||||
node.healthy = true;
|
||||
node.models.insert(
|
||||
"test-model".into(),
|
||||
ModelEntry {
|
||||
id: "test-model".into(),
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: None,
|
||||
vram_estimate_mb: Some(14000),
|
||||
capabilities: vec!["image".into()],
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
let app = cortex_gateway::build_app(Arc::clone(&fleet));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
format!("http://{addr}")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_images_budget_rejected_before_dispatch() {
|
||||
let mock_url = spawn_images_mock_neuron().await;
|
||||
// A 1024²/9-step image reserves ~9.44 units ≈ 9438 tokens; a cap of
|
||||
// 100 tokens must fail-close before the neuron is touched.
|
||||
let gateway_url = spawn_keyed_gateway(&mock_url, 100).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway_url}/v1/images/generations"))
|
||||
.bearer_auth("sk-img")
|
||||
.json(&json!({"model": "test-model", "prompt": "a cat"}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::TOO_MANY_REQUESTS);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["error"]["code"], "insufficient_quota");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_images_within_budget_succeeds_and_settles() {
|
||||
let mock_url = spawn_images_mock_neuron().await;
|
||||
// Plenty of budget: reservation ~9438 tokens, settle at actual
|
||||
// 9.437184 units ≈ 9438 tokens.
|
||||
let gateway_url = spawn_keyed_gateway(&mock_url, 1_000_000).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway_url}/v1/images/generations"))
|
||||
.bearer_auth("sk-img")
|
||||
.json(&json!({"model": "test-model", "prompt": "a cat", "seed": 42}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), 200);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["data"][0]["b64_json"], "aGVsZXhh");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_images_cfg_doubles_reservation() {
|
||||
let mock_url = spawn_images_mock_neuron().await;
|
||||
// Cap sits between the plain (9.44u ≈ 9438t) and CFG (18.88u ≈
|
||||
// 18875t) reservations: plain passes, CFG fail-closes.
|
||||
let gateway_url = spawn_keyed_gateway(&mock_url, 12_000).await;
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let cfg_resp = client
|
||||
.post(format!("{gateway_url}/v1/images/generations"))
|
||||
.bearer_auth("sk-img")
|
||||
.json(&json!({
|
||||
"model": "test-model", "prompt": "a cat",
|
||||
"negative_prompt": "blurry"
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(cfg_resp.status(), reqwest::StatusCode::TOO_MANY_REQUESTS);
|
||||
|
||||
let plain_resp = client
|
||||
.post(format!("{gateway_url}/v1/images/generations"))
|
||||
.bearer_auth("sk-img")
|
||||
.json(&json!({"model": "test-model", "prompt": "a cat"}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(plain_resp.status(), 200);
|
||||
}
|
||||
@@ -1,205 +0,0 @@
|
||||
//! Load-aware routing across replicas (#55).
|
||||
//!
|
||||
//! When a model is loaded on more than one healthy neuron, the router picks
|
||||
//! the least-busy replica using the per-model admission load each neuron
|
||||
//! reports on `GET /health` (#53), rather than always taking the first.
|
||||
|
||||
mod common;
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::Path;
|
||||
use axum::http::{StatusCode, header};
|
||||
use axum::response::IntoResponse;
|
||||
use axum::routing::{get, post};
|
||||
use cortex_core::config::{
|
||||
EvictionSettings, EvictionStrategy, GatewayConfig, GatewaySettings, NeuronEndpoint,
|
||||
};
|
||||
use cortex_core::discovery::ModelLoad;
|
||||
use cortex_core::node::{ModelEntry, ModelStatus};
|
||||
use cortex_gateway::state::CortexState;
|
||||
use serde_json::{Value, json};
|
||||
use std::sync::Arc;
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
/// Seed a node as healthy with `test-model` loaded and a given admission load.
|
||||
async fn seed_loaded(fleet: &CortexState, node: &str, in_flight: usize, queue_depth: usize) {
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let n = nodes.get_mut(node).expect("node exists");
|
||||
n.healthy = true;
|
||||
n.models.insert(
|
||||
"test-model".into(),
|
||||
ModelEntry {
|
||||
id: "test-model".into(),
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: None,
|
||||
vram_estimate_mb: Some(8000),
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
servable: None,
|
||||
},
|
||||
);
|
||||
n.model_load.insert(
|
||||
"test-model".into(),
|
||||
ModelLoad {
|
||||
id: "test-model".into(),
|
||||
in_flight,
|
||||
queue_depth,
|
||||
max_in_flight: 8,
|
||||
max_queue_depth: 8,
|
||||
rejected_queue_full: 0,
|
||||
rejected_kv_timeout: 0,
|
||||
rejected_kv_unservable: 0,
|
||||
kv_budget_mb: 0,
|
||||
kv_available_mb: 0,
|
||||
rejected_timeout: 0,
|
||||
rejected_per_principal: 0,
|
||||
anon_in_flight: 0,
|
||||
anon_max_in_flight: 7,
|
||||
rejected_anon_yield: 0,
|
||||
tok_s_prefill: 0.0,
|
||||
tok_s_decode: 0.0,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/// Build a gateway state over two mock neurons (no poller; we seed state).
|
||||
async fn two_neuron_fleet(endpoint_a: &str, endpoint_b: &str) -> Arc<CortexState> {
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![
|
||||
NeuronEndpoint {
|
||||
name: "node-a".into(),
|
||||
endpoint: endpoint_a.to_string(),
|
||||
},
|
||||
NeuronEndpoint {
|
||||
name: "node-b".into(),
|
||||
endpoint: endpoint_b.to_string(),
|
||||
},
|
||||
],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
Arc::new(CortexState::from_config(&config))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn routes_to_least_busy_replica() {
|
||||
let neuron_a = common::spawn_mock_neuron().await;
|
||||
let neuron_b = common::spawn_mock_neuron().await;
|
||||
let fleet = two_neuron_fleet(&neuron_a, &neuron_b).await;
|
||||
|
||||
// A is busy (1 running + 3 queued), B is idle.
|
||||
seed_loaded(&fleet, "node-a", 1, 3).await;
|
||||
seed_loaded(&fleet, "node-b", 0, 0).await;
|
||||
|
||||
let route = cortex_gateway::router::resolve(&fleet, "test-model")
|
||||
.await
|
||||
.expect("model is loaded on both nodes");
|
||||
assert_eq!(route.node_name, "node-b", "should pick the idle replica");
|
||||
|
||||
// Flip the load: now B is the busy one.
|
||||
seed_loaded(&fleet, "node-a", 0, 0).await;
|
||||
seed_loaded(&fleet, "node-b", 1, 5).await;
|
||||
let route = cortex_gateway::router::resolve(&fleet, "test-model")
|
||||
.await
|
||||
.expect("still loaded");
|
||||
assert_eq!(route.node_name, "node-a", "should follow the lighter load");
|
||||
}
|
||||
|
||||
/// Mock neuron whose inference endpoint always returns a #63 backpressure
|
||||
/// envelope (503 + Retry-After) — simulating a saturated neuron.
|
||||
async fn spawn_busy_neuron() -> String {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let base_url = format!("http://{addr}");
|
||||
let inference_url = base_url.clone();
|
||||
let app = axum::Router::new()
|
||||
.route(
|
||||
"/models/{model_id}/endpoint",
|
||||
get(move |Path(_): Path<String>| {
|
||||
let url = inference_url.clone();
|
||||
async move { Json(json!({ "url": url })) }
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/v1/chat/completions",
|
||||
post(|| async {
|
||||
let body = json!({"error": {
|
||||
"message": "model is busy (admission queue full); retry shortly",
|
||||
"type": "rate_limit_error",
|
||||
"code": "rate_limit_exceeded",
|
||||
"param": null
|
||||
}});
|
||||
(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
[(header::RETRY_AFTER, "6")],
|
||||
Json(body),
|
||||
)
|
||||
.into_response()
|
||||
}),
|
||||
);
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
base_url
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn neuron_backpressure_is_propagated_intact() {
|
||||
// A saturated neuron's 503 + Retry-After + envelope must reach the client
|
||||
// verbatim — not unwrapped, remapped, or stripped (#55 / #63).
|
||||
let neuron = spawn_busy_neuron().await;
|
||||
let fleet = two_neuron_fleet(&neuron, &neuron).await;
|
||||
seed_loaded(&fleet, "node-a", 1, 8).await;
|
||||
|
||||
let app = cortex_gateway::build_app(Arc::clone(&fleet));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("http://{addr}/v1/chat/completions"))
|
||||
.json(&json!({"model": "test-model", "messages": [{"role": "user", "content": "hi"}]}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(
|
||||
resp.headers()
|
||||
.get(reqwest::header::RETRY_AFTER)
|
||||
.and_then(|v| v.to_str().ok()),
|
||||
Some("6"),
|
||||
"Retry-After must survive the proxy"
|
||||
);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["error"]["code"], "rate_limit_exceeded");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ties_break_deterministically_by_name() {
|
||||
let neuron_a = common::spawn_mock_neuron().await;
|
||||
let neuron_b = common::spawn_mock_neuron().await;
|
||||
let fleet = two_neuron_fleet(&neuron_a, &neuron_b).await;
|
||||
|
||||
// Equal load on both → stable pick (lowest node name).
|
||||
seed_loaded(&fleet, "node-a", 0, 0).await;
|
||||
seed_loaded(&fleet, "node-b", 0, 0).await;
|
||||
|
||||
let route = cortex_gateway::router::resolve(&fleet, "test-model")
|
||||
.await
|
||||
.expect("loaded");
|
||||
assert_eq!(route.node_name, "node-a", "ties break by name");
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user