Compare commits
47 Commits
feat/47-ph
...
feat/F4-ac
| Author | SHA1 | Date | |
|---|---|---|---|
|
1bf3348c8c
|
|||
|
c596519dbd
|
|||
|
8dd82776f1
|
|||
|
7a6f252fe0
|
|||
|
bb0d1e51b8
|
|||
|
f2ba12bbc5
|
|||
|
a9d7382be8
|
|||
|
d94c62c143
|
|||
|
cb9e7c7c2e
|
|||
|
2604b9f134
|
|||
|
178e3092d5
|
|||
|
46befde4cd
|
|||
|
cf87e156c5
|
|||
|
79073170ec
|
|||
|
71106afaf1
|
|||
|
d2dcdd6ebb
|
|||
|
222c2a6116
|
|||
|
1115bb0942
|
|||
|
63f578cb15
|
|||
|
76c90fa993
|
|||
|
7984d27553
|
|||
|
43ffffdccb
|
|||
|
5fd7736abd
|
|||
|
03fd4960c3
|
|||
|
5ed6bc3390
|
|||
|
cabec1d08a
|
|||
|
881fc85a4c
|
|||
|
b2ed20b55a
|
|||
|
bee27e9b9c
|
|||
|
87d9c291ce
|
|||
|
d4742467e0
|
|||
|
e7f7e376fc
|
|||
|
3b9a6e37f6
|
|||
|
526b662c5e
|
|||
|
db7e373b90
|
|||
|
5c1623a817
|
|||
|
3b60dd7a31
|
|||
|
4feaaf1cfb
|
|||
|
057bc71e80
|
|||
|
dd31c3cd49
|
|||
|
c83f1eb98c
|
|||
|
a60c9f1075
|
|||
|
b2bd86bfa5
|
|||
|
cdf87284af
|
|||
|
4f16b8c541
|
|||
|
486d7e9a8f
|
|||
|
bc74e0e95f
|
3
.gitignore
vendored
@@ -1,6 +1,9 @@
|
||||
/target
|
||||
/bench/node_modules
|
||||
/bench/dist
|
||||
/helexa.ai/node_modules
|
||||
/helexa.ai/dist
|
||||
helexa.ai/.env.local
|
||||
*.swp
|
||||
*.swo
|
||||
.idea/
|
||||
|
||||
26
CLAUDE.md
@@ -185,6 +185,32 @@ Run these locally before pushing. `cargo fmt --all` fixes formatting
|
||||
automatically. Clippy warnings must be resolved, not suppressed with
|
||||
`#[allow(...)]` unless there is a clear rationale.
|
||||
|
||||
## Development workflow
|
||||
|
||||
Work each change on its own branch; `main` stays releasable.
|
||||
|
||||
1. Implement on a feature branch (`fix/<issue>-…`, `feat/<issue>-…`).
|
||||
2. Run the CI triad locally (`cargo fmt --check --all`,
|
||||
`cargo clippy --workspace -- -D warnings`, `cargo test --workspace`).
|
||||
Local builds are **CPU-only** — the `#[cfg(feature = "cuda")]` neuron/TP
|
||||
paths do NOT compile locally. The branch CI's **CUDA type-check** job is
|
||||
the only thing that validates them, so for any neuron change the push to
|
||||
Gitea is the real gate, not a rubber stamp.
|
||||
3. Push the branch on local-green (no need to ask first), and background-watch
|
||||
its CI run via the gitea-mcp `actions_run_read` tools. Start the next piece
|
||||
of work meanwhile.
|
||||
4. Merge to `main` when the four **validation** jobs are green — Format,
|
||||
Clippy, Test, CUDA type-check. The SRPM / COPR / version-bump jobs are the
|
||||
deploy pipeline (they run on `main`), not validation — don't wait on them.
|
||||
5. Merging/pushing to `main` triggers the auto-deploy pipeline.
|
||||
|
||||
Docs-only changes (no `#[cfg(feature = "cuda")]` impact) can go straight to
|
||||
`main` — there's nothing for the CUDA type-check to prove.
|
||||
|
||||
SSH note: the gitea remote host offers multiple agent keys and cuts the
|
||||
connection before reaching the right one. This repo pins the working key via
|
||||
`git config core.sshCommand "ssh -i ~/.ssh/id_grenade -o IdentitiesOnly=yes"`.
|
||||
|
||||
## Environment
|
||||
|
||||
- Targets Fedora 43 (systemd, SELinux enforcing)
|
||||
|
||||
874
Cargo.lock
generated
@@ -7,6 +7,9 @@ members = [
|
||||
"crates/neuron",
|
||||
"crates/helexa-acp",
|
||||
"crates/helexa-bench",
|
||||
"crates/helexa-router",
|
||||
"crates/helexa-stream",
|
||||
"crates/helexa-upstream",
|
||||
]
|
||||
|
||||
[workspace.package]
|
||||
|
||||
@@ -48,3 +48,60 @@ vram_mb = 12288 # e.g. RTX 3060 (12 GB)
|
||||
pinned = [
|
||||
"your-org/embedding-model",
|
||||
]
|
||||
|
||||
# -- Entitlements (multi-tenant governance, #47) -------------------------
|
||||
# Identity + per-key token budgets. Omit this section entirely for the
|
||||
# legacy single-operator behaviour: requests are anonymous and uncapped.
|
||||
#
|
||||
# The local/static provider below is the source of truth for accounts,
|
||||
# keys, and hard caps until the upstream clearing house exists. Identity
|
||||
# rides standard bearer auth only — clients send
|
||||
# Authorization: Bearer <key>
|
||||
# no custom headers or body fields.
|
||||
|
||||
[entitlements]
|
||||
# Reject unauthenticated requests with 401 invalid_api_key. Leave false
|
||||
# (allow-anonymous) during rollout; flip to true once keys are issued.
|
||||
require_auth = false
|
||||
|
||||
# One entry per API key.
|
||||
[[entitlements.keys]]
|
||||
key = "sk-example-rolling" # the bearer token the client sends
|
||||
account_id = "team-research" # billable account (keys may share one)
|
||||
key_id = "research-ci" # stable label for ledger/metrics (optional)
|
||||
hard_cap = 5_000_000 # hard token cap over the window
|
||||
# Rolling window that resets — over-cap requests get 429 rate_limit_exceeded
|
||||
# + Retry-After, so well-behaved clients (opencode/AI SDK) back off and retry.
|
||||
window = { kind = "rolling", seconds = 3600 }
|
||||
|
||||
[[entitlements.keys]]
|
||||
key = "sk-example-balance"
|
||||
account_id = "team-research"
|
||||
key_id = "research-prepaid"
|
||||
hard_cap = 20_000_000
|
||||
# Hard balance, no reset — exhaustion returns 429 insufficient_quota
|
||||
# (the client surfaces and stops). This is the default when `window` is
|
||||
# omitted. Never 402.
|
||||
window = { kind = "balance" }
|
||||
|
||||
[[entitlements.keys]]
|
||||
key = "sk-example-infra"
|
||||
account_id = "operator"
|
||||
key_id = "infra"
|
||||
# No hard_cap → uncapped operator infra key (own fleet, own use). Still
|
||||
# metered for visibility.
|
||||
|
||||
# -- Upstream (helexa mesh) entitlements client (#57) --------------------
|
||||
# When enabled, a bearer key NOT found in [[entitlements.keys]] above is
|
||||
# validated against the helexa-upstream authority (mesh accounts), and its
|
||||
# budget is reserved/settled there. Operator-local keys (incl. the infra
|
||||
# key) never leave this process. Fail-closed: if upstream is unreachable a
|
||||
# request is refused (503 + Retry-After), never served un-authorized.
|
||||
# Disabled by default — a standalone operator runs purely local.
|
||||
[upstream]
|
||||
enabled = false
|
||||
# url = "https://upstream.helexa.ai"
|
||||
# Shared client bearer this cortex presents (maps to an operator_id
|
||||
# upstream). Override via CORTEX_UPSTREAM__BEARER in prod.
|
||||
# bearer = "replace-with-operator-client-secret"
|
||||
# timeout_secs = 5
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use crate::entitlements::CapWindow;
|
||||
use figment::{
|
||||
Figment,
|
||||
providers::{Env, Format, Toml},
|
||||
@@ -16,6 +17,76 @@ pub struct GatewayConfig {
|
||||
/// non-packaged / local runs.
|
||||
#[serde(default = "default_models_path")]
|
||||
pub models_config: String,
|
||||
/// Multi-tenant governance: auth + per-key token budgets (#47). Empty
|
||||
/// by default — anonymous, uncapped — so existing single-operator
|
||||
/// setups keep working until keys are configured.
|
||||
#[serde(default)]
|
||||
pub entitlements: EntitlementsConfig,
|
||||
/// helexa-upstream client (#57). When enabled, keys not found in the
|
||||
/// local `[entitlements]` config are validated against the mesh
|
||||
/// authority, and budget is reserved/settled there. Disabled by default
|
||||
/// — a single operator runs purely local.
|
||||
#[serde(default)]
|
||||
pub upstream: UpstreamClientConfig,
|
||||
}
|
||||
|
||||
/// `[upstream]` — the helexa-upstream authority client (#57). Locally
|
||||
/// unrecognised bearer keys are resolved against `url`'s `/authz/v1` surface
|
||||
/// (mesh accounts); local keys (operator + infra) never leave the process.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct UpstreamClientConfig {
|
||||
/// Enable the upstream fallthrough. Off → purely local entitlements.
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
/// Base URL of helexa-upstream (e.g. "https://upstream.helexa.ai").
|
||||
#[serde(default)]
|
||||
pub url: String,
|
||||
/// Shared client bearer this cortex presents to `/authz/v1` (maps to an
|
||||
/// operator_id upstream). Sent as `Authorization: Bearer <bearer>`.
|
||||
#[serde(default)]
|
||||
pub bearer: String,
|
||||
/// Per-call timeout (seconds) to upstream.
|
||||
#[serde(default = "default_upstream_timeout")]
|
||||
pub timeout_secs: u64,
|
||||
}
|
||||
|
||||
fn default_upstream_timeout() -> u64 {
|
||||
5
|
||||
}
|
||||
|
||||
/// `[entitlements]` — the local/static [`crate::entitlements::EntitlementProvider`]
|
||||
/// source of truth (#50). Accounts, keys, and hard caps live here; the
|
||||
/// future upstream client (#57) ignores this section.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct EntitlementsConfig {
|
||||
/// Reject unauthenticated requests with `401 invalid_api_key` when
|
||||
/// true. Default `false` (allow-anonymous) for dev / single-operator
|
||||
/// continuity.
|
||||
#[serde(default)]
|
||||
pub require_auth: bool,
|
||||
/// Static API keys and their budgets, consumed by the local provider.
|
||||
#[serde(default)]
|
||||
pub keys: Vec<ApiKeyConfig>,
|
||||
}
|
||||
|
||||
/// One configured API key: the bearer token, the account it bills to, and
|
||||
/// its hard cap. `[[entitlements.keys]]` in TOML.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ApiKeyConfig {
|
||||
/// The bearer token clients send in `Authorization: Bearer <key>`.
|
||||
pub key: String,
|
||||
/// Billable account. Multiple keys may share one account.
|
||||
pub account_id: String,
|
||||
/// Stable per-key identifier for ledger/metrics labels. Defaults to
|
||||
/// `account_id` when omitted, so the secret is never used as a label.
|
||||
#[serde(default)]
|
||||
pub key_id: Option<String>,
|
||||
/// Hard token cap. `None`/omitted = uncapped (e.g. operator infra key).
|
||||
#[serde(default)]
|
||||
pub hard_cap: Option<u64>,
|
||||
/// Cap-window semantics. Default: a non-resetting [`CapWindow::Balance`].
|
||||
#[serde(default)]
|
||||
pub window: CapWindow,
|
||||
}
|
||||
|
||||
fn default_models_path() -> String {
|
||||
@@ -87,6 +158,8 @@ impl Default for GatewayConfig {
|
||||
},
|
||||
neurons: vec![],
|
||||
models_config: default_models_path(),
|
||||
entitlements: EntitlementsConfig::default(),
|
||||
upstream: UpstreamClientConfig::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -68,6 +68,57 @@ pub struct HealthResponse {
|
||||
pub devices: Vec<DeviceHealth>,
|
||||
#[serde(default)]
|
||||
pub activation: ActivationStatus,
|
||||
/// Per-model admission load (#53): how many requests are running vs.
|
||||
/// queued on each loaded model right now. Cortex's load-aware router
|
||||
/// (#55) reads this to spread traffic across replicas and to propagate
|
||||
/// honest backpressure. `#[serde(default)]` keeps older gateways/neurons
|
||||
/// interoperable (absent → empty → treated as no load info).
|
||||
#[serde(default)]
|
||||
pub models: Vec<ModelLoad>,
|
||||
}
|
||||
|
||||
/// Live admission load for one loaded model (#53).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ModelLoad {
|
||||
pub id: String,
|
||||
/// Requests currently running (batch-1 → 0 or 1).
|
||||
pub in_flight: usize,
|
||||
/// Requests waiting in the bounded admission queue.
|
||||
pub queue_depth: usize,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod health_load_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn health_response_without_models_field_still_deserializes() {
|
||||
// A pre-#53 neuron's /health payload omits `models`; the gateway
|
||||
// must still parse it (serde default → empty).
|
||||
let json = r#"{"uptime_secs":42,"devices":[]}"#;
|
||||
let resp: HealthResponse = serde_json::from_str(json).expect("back-compat parse");
|
||||
assert_eq!(resp.uptime_secs, 42);
|
||||
assert!(resp.models.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn health_response_round_trips_model_load() {
|
||||
let resp = HealthResponse {
|
||||
uptime_secs: 1,
|
||||
devices: vec![],
|
||||
activation: ActivationStatus::default(),
|
||||
models: vec![ModelLoad {
|
||||
id: "Qwen/Qwen3.6-27B".into(),
|
||||
in_flight: 1,
|
||||
queue_depth: 3,
|
||||
}],
|
||||
};
|
||||
let s = serde_json::to_string(&resp).unwrap();
|
||||
let back: HealthResponse = serde_json::from_str(&s).unwrap();
|
||||
assert_eq!(back.models.len(), 1);
|
||||
assert_eq!(back.models[0].in_flight, 1);
|
||||
assert_eq!(back.models[0].queue_depth, 3);
|
||||
}
|
||||
}
|
||||
|
||||
/// High-level activation state of the neuron daemon. The HTTP listener
|
||||
|
||||
152
crates/cortex-core/src/entitlements.rs
Normal file
@@ -0,0 +1,152 @@
|
||||
//! Identity and entitlement primitives for multi-tenant governance (#47).
|
||||
//!
|
||||
//! Identity is the shared substrate the whole epic hangs off:
|
||||
//! `identity (principal) → accounting (spend) → policy → enforcement`. This
|
||||
//! module defines the seam — the [`EntitlementProvider`] trait and its data
|
||||
//! types — so the local/static provider (operator-config caps, in
|
||||
//! cortex-gateway) can land the auth + per-key-cap + amplification fix
|
||||
//! *before* any upstream clearing house exists. The future helexa-upstream
|
||||
//! client (#57) is just another impl of this trait.
|
||||
//!
|
||||
//! The provider owns three jobs:
|
||||
//! 1. **resolve** a bearer key to a [`Principal`] (drives auth, #49);
|
||||
//! 2. **reserve → settle/release** token budget around a request so spend
|
||||
//! can never overshoot a hard cap under concurrency (drives budget
|
||||
//! enforcement, #52);
|
||||
//! 3. expose a [`BudgetSnapshot`] for metering/metrics (#51).
|
||||
//!
|
||||
//! [`BudgetError`] carries the cap-window semantics so the caller can pick
|
||||
//! the correct #63 rejection (`rate_limit_exceeded` + `Retry-After` for a
|
||||
//! resetting window vs `insufficient_quota` for a hard balance) without the
|
||||
//! provider knowing anything about HTTP.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Internal header carrying the resolved account id from cortex to neuron.
|
||||
/// neuron trusts these over the WireGuard link (#54); cortex **strips** any
|
||||
/// client-supplied copy before stamping the authoritative value, so a client
|
||||
/// can never assert a principal directly.
|
||||
pub const HEADER_ACCOUNT_ID: &str = "x-helexa-account-id";
|
||||
/// Internal header carrying the resolved key id from cortex to neuron.
|
||||
pub const HEADER_KEY_ID: &str = "x-helexa-key-id";
|
||||
|
||||
/// Who a request is for. Resolved once at the edge from the bearer key and
|
||||
/// carried through the request context. `account_id` is the billable owner
|
||||
/// (spendable at any operator, by decision); `key_id` identifies the
|
||||
/// specific API key for per-key hard caps and ledger/metrics labels.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct Principal {
|
||||
pub account_id: String,
|
||||
pub key_id: String,
|
||||
}
|
||||
|
||||
/// Cap-window semantics for a key's hard cap. Determines which #63 code an
|
||||
/// over-cap reservation maps to.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub enum CapWindow {
|
||||
/// Hard balance — the cap never resets. Exhaustion is permanent
|
||||
/// (`429 insufficient_quota`, no `Retry-After`).
|
||||
#[default]
|
||||
Balance,
|
||||
/// Rolling window of `seconds` that resets. Exhaustion is transient
|
||||
/// (`429 rate_limit_exceeded` + `Retry-After` until reset).
|
||||
Rolling { seconds: u64 },
|
||||
}
|
||||
|
||||
/// An outstanding budget reservation. The caller holds this opaque handle
|
||||
/// between [`EntitlementProvider::reserve`] and exactly one of
|
||||
/// [`EntitlementProvider::settle`] / [`EntitlementProvider::release`]. Not
|
||||
/// `Clone` — a reservation is consumed once.
|
||||
#[derive(Debug)]
|
||||
pub struct Reservation {
|
||||
/// Provider-local handle; opaque to the caller.
|
||||
pub id: u64,
|
||||
/// The principal this reservation belongs to.
|
||||
pub principal: Principal,
|
||||
/// Tokens reserved against the cap.
|
||||
pub reserved: u64,
|
||||
}
|
||||
|
||||
/// A point-in-time view of a key's budget, for metering and metrics (#51).
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct BudgetSnapshot {
|
||||
/// Hard cap in tokens. `None` means uncapped (e.g. an operator infra
|
||||
/// key, #58).
|
||||
pub hard_cap: Option<u64>,
|
||||
/// Settled spend in the current window.
|
||||
pub spent: u64,
|
||||
/// Sum of outstanding (un-settled) reservations.
|
||||
pub reserved: u64,
|
||||
}
|
||||
|
||||
/// Authentication failure — the bearer key could not be resolved.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum AuthError {
|
||||
/// The key is genuinely unknown → `401 invalid_api_key` (#49/#63).
|
||||
#[error("invalid or unknown API key")]
|
||||
InvalidKey,
|
||||
/// The authority that could resolve the key is unreachable (e.g. the
|
||||
/// helexa-upstream client failed, #57). Fail **closed** but distinctly:
|
||||
/// a transient outage must surface as `503 service_unavailable` +
|
||||
/// `Retry-After`, never `401` — a real key must not be rejected as
|
||||
/// invalid during an upstream blip.
|
||||
#[error("entitlement authority unavailable; retry in {retry_after_secs}s")]
|
||||
Unavailable { retry_after_secs: u64 },
|
||||
}
|
||||
|
||||
/// Why a reservation was refused. Carries enough for the caller to build the
|
||||
/// correct #63 envelope without the provider touching HTTP.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum BudgetError {
|
||||
/// A resetting window is exhausted → `429 rate_limit_exceeded` +
|
||||
/// `Retry-After: retry_after_secs`.
|
||||
#[error(
|
||||
"rolling-window budget exhausted ({requested} requested, {available} available); \
|
||||
resets in {retry_after_secs}s"
|
||||
)]
|
||||
RateLimited {
|
||||
requested: u64,
|
||||
available: u64,
|
||||
retry_after_secs: u64,
|
||||
},
|
||||
/// A hard balance is exhausted → `429 insufficient_quota` (no
|
||||
/// `Retry-After`; the client surfaces and stops). Never `402`.
|
||||
#[error("hard balance exhausted ({requested} requested, {available} available)")]
|
||||
InsufficientQuota { requested: u64, available: u64 },
|
||||
}
|
||||
|
||||
/// The seam between cortex's enforcement and whatever decides entitlement —
|
||||
/// a local/static config provider today (#50), the helexa-upstream client
|
||||
/// later (#57). All methods are async so the upstream impl can do network
|
||||
/// I/O; the local impl resolves in-process.
|
||||
#[async_trait]
|
||||
pub trait EntitlementProvider: Send + Sync {
|
||||
/// Resolve a bearer API key to its principal. `Err(InvalidKey)` for an
|
||||
/// unknown/empty key.
|
||||
async fn resolve(&self, api_key: &str) -> Result<Principal, AuthError>;
|
||||
|
||||
/// Reserve up to `max_tokens` against the principal's cap. Returns a
|
||||
/// handle on success, or a [`BudgetError`] (which the caller maps to a
|
||||
/// #63 `429`) if the reservation would exceed the cap. Reserving the
|
||||
/// *maximum* a request could consume before dispatch is what prevents
|
||||
/// overshoot under concurrency.
|
||||
async fn reserve(
|
||||
&self,
|
||||
principal: &Principal,
|
||||
max_tokens: u64,
|
||||
) -> Result<Reservation, BudgetError>;
|
||||
|
||||
/// Settle a reservation with the tokens actually consumed, releasing the
|
||||
/// unused remainder back to the cap.
|
||||
async fn settle(&self, reservation: Reservation, actual_tokens: u64);
|
||||
|
||||
/// Release a reservation in full — e.g. dispatch failed before any
|
||||
/// tokens were consumed.
|
||||
async fn release(&self, reservation: Reservation);
|
||||
|
||||
/// Current budget snapshot for a principal, for metering/metrics.
|
||||
/// `None` if the provider doesn't track this principal.
|
||||
async fn snapshot(&self, principal: &Principal) -> Option<BudgetSnapshot>;
|
||||
}
|
||||
@@ -54,10 +54,26 @@ pub struct ModelLimit {
|
||||
pub output: usize,
|
||||
}
|
||||
|
||||
/// Operator-set pricing in USD per 1M tokens.
|
||||
/// Operator-set pricing, **USD per 1,000,000 tokens, as JSON numbers**
|
||||
/// (`float`) — the models.dev/opencode `cost` convention, which is what
|
||||
/// helexa's primary client reads. NOT per-token, NOT decimal strings (that
|
||||
/// is OpenRouter's `pricing` shape, which helexa deliberately does not emit
|
||||
/// — see #68). A client must not rescale by 10⁶.
|
||||
///
|
||||
/// Self-hosted deployments typically leave both at `0.0`. Cache fields are
|
||||
/// optional — set when the backend supports a prefix-cache discount tier.
|
||||
/// `cost` is sourced from the operator's `models.toml` catalogue profile and
|
||||
/// surfaced verbatim on `/v1/models`. The *absent* vs *zero* distinction is
|
||||
/// intentional and load-bearing (#68):
|
||||
/// - **`cost` absent** (the whole object omitted) — the model is **not
|
||||
/// priced**: the operator has not declared a rate. Clients should treat
|
||||
/// spend as unknown, not free.
|
||||
/// - **`cost` present with `input`/`output` = `0.0`** — the model is
|
||||
/// **intentionally free** (self-hosted, no charge). opencode renders `$0`.
|
||||
///
|
||||
/// Cache fields are optional — set them only when the backend supports a
|
||||
/// prefix-cache discount tier (relevant once cache-token reporting, #64,
|
||||
/// lands). The advertised rate here must equal the rate metering (#51) and
|
||||
/// reconciliation (#58/#59) bill against; today both read this catalogue
|
||||
/// value.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ModelCost {
|
||||
/// USD per 1M input (prompt) tokens.
|
||||
@@ -98,7 +114,8 @@ pub struct ModelInfo {
|
||||
/// `None` when neither the catalogue nor the loaded model can provide it.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub limit: Option<ModelLimit>,
|
||||
/// Operator-set pricing in USD per 1M tokens (0.0 = free/self-hosted).
|
||||
/// Operator-set pricing — see [`ModelCost`] for units and the
|
||||
/// absent (not priced) vs `0.0` (intentionally free) distinction.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cost: Option<ModelCost>,
|
||||
/// `true` when the model's tokenizer contains recognised tool-call
|
||||
|
||||
@@ -3,6 +3,7 @@ pub mod build_info;
|
||||
pub mod catalogue;
|
||||
pub mod config;
|
||||
pub mod discovery;
|
||||
pub mod entitlements;
|
||||
pub mod error_envelope;
|
||||
pub mod harness;
|
||||
pub mod metrics;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use crate::discovery::{ActivationStatus, DiscoveryResponse};
|
||||
use crate::discovery::{ActivationStatus, DiscoveryResponse, ModelLoad};
|
||||
use crate::harness::{ModelCost, ModelLimit};
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -27,6 +27,17 @@ pub struct NodeState {
|
||||
/// to synthesize `Loading` locations so clients see a catalogued
|
||||
/// model that's mid-prewarm as "loading", not "missing".
|
||||
pub activation: Option<ActivationStatus>,
|
||||
/// Last-seen per-model admission load from this neuron's `/health`
|
||||
/// (#53), keyed by model id. The router (#55) reads it to pick the
|
||||
/// least-busy replica when a model is loaded on more than one neuron.
|
||||
/// Empty until the first /health poll reports load.
|
||||
pub model_load: HashMap<String, ModelLoad>,
|
||||
/// Consecutive failed `/models` polls. The poller marks a node
|
||||
/// unhealthy only once this crosses a threshold, so a single transient
|
||||
/// miss (e.g. a neuron momentarily slow to answer while busy) doesn't
|
||||
/// yank the node — and all its models — out of routing. Reset to 0 on
|
||||
/// any successful poll.
|
||||
pub consecutive_poll_failures: u32,
|
||||
}
|
||||
|
||||
/// A model registered on a node, with its runtime status.
|
||||
@@ -125,7 +136,9 @@ pub struct CortexModelEntry {
|
||||
/// at load time. `None` when neither source provides it.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub limit: Option<ModelLimit>,
|
||||
/// Operator-set pricing in USD per 1M tokens (0.0 = free/self-hosted).
|
||||
/// Operator-set pricing from the catalogue profile — see
|
||||
/// [`cortex_core::harness::ModelCost`] for units (USD per 1M tokens) and
|
||||
/// the absent (not priced) vs `0.0` (intentionally free) distinction.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cost: Option<ModelCost>,
|
||||
/// `true` when any neuron reports this model supports tool calls.
|
||||
|
||||
@@ -6,6 +6,8 @@ license.workspace = true
|
||||
|
||||
[dependencies]
|
||||
cortex-core.workspace = true
|
||||
helexa-stream = { path = "../helexa-stream" }
|
||||
async-trait.workspace = true
|
||||
tokio.workspace = true
|
||||
axum.workspace = true
|
||||
tower.workspace = true
|
||||
|
||||
@@ -32,6 +32,8 @@ pub async fn stream_translated(
|
||||
openai_body: axum::body::Bytes,
|
||||
model_id: &str,
|
||||
node_name: &str,
|
||||
inbound_headers: &axum::http::HeaderMap,
|
||||
usage_sink: Option<crate::metering::UsageSink>,
|
||||
) -> Response {
|
||||
let url = format!("{endpoint}/v1/chat/completions");
|
||||
tracing::info!(
|
||||
@@ -42,13 +44,14 @@ pub async fn stream_translated(
|
||||
"proxying streaming request (anthropic SSE translation)"
|
||||
);
|
||||
|
||||
let upstream = match client
|
||||
.post(&url)
|
||||
.header("content-type", "application/json")
|
||||
.body(openai_body)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
let request = crate::auth::forward_principal_headers(
|
||||
client
|
||||
.post(&url)
|
||||
.header("content-type", "application/json")
|
||||
.body(openai_body),
|
||||
inbound_headers,
|
||||
);
|
||||
let upstream = match request.send().await {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
@@ -94,6 +97,10 @@ pub async fn stream_translated(
|
||||
let mut saw_tool_call = false;
|
||||
let mut last_finish: Option<String> = None;
|
||||
let mut frames = 0u64;
|
||||
// Engine-truth usage for metering (#51), scanned from the upstream
|
||||
// frames (neuron emits a final `usage` object on the stream, #48).
|
||||
let mut usage_prompt = 0u64;
|
||||
let mut usage_completion = 0u64;
|
||||
|
||||
'outer: while let Some(block) = upstream.next().await {
|
||||
let block = match block {
|
||||
@@ -121,6 +128,15 @@ pub async fn stream_translated(
|
||||
continue;
|
||||
}
|
||||
tracing::trace!(node = %node, frame = %data, "anthropic stream: upstream frame");
|
||||
// Capture usage for metering before translation — the
|
||||
// usage object rides on a late frame (often after the
|
||||
// last content delta).
|
||||
if let Some(p) = crate::proxy::last_count_for(data, "prompt_tokens") {
|
||||
usage_prompt = p;
|
||||
}
|
||||
if let Some(c) = crate::proxy::last_count_for(data, "completion_tokens") {
|
||||
usage_completion = c;
|
||||
}
|
||||
let Ok(chunk) = serde_json::from_str::<ChatCompletionChunk>(data) else {
|
||||
tracing::debug!(node = %node, "anthropic stream: unparsable upstream frame skipped");
|
||||
continue;
|
||||
@@ -162,6 +178,14 @@ pub async fn stream_translated(
|
||||
terminated = done,
|
||||
"anthropic stream complete"
|
||||
);
|
||||
|
||||
// Settle metering with the observed usage (#51). Runs on every exit
|
||||
// path of the pump — clean end, early break, or upstream error — so
|
||||
// the reservation is always resolved. `(0, 0)` when no usage frame
|
||||
// was seen, which releases without recording spend.
|
||||
if let Some(sink) = usage_sink {
|
||||
sink(usage_prompt, usage_completion);
|
||||
}
|
||||
});
|
||||
|
||||
Response::builder()
|
||||
|
||||
144
crates/cortex-gateway/src/auth.rs
Normal file
@@ -0,0 +1,144 @@
|
||||
//! API-key authentication + principal resolution (#49).
|
||||
//!
|
||||
//! Identity rides standard bearer auth only — `Authorization: Bearer <key>`
|
||||
//! — which is what keeps every tier OpenAI-compatible by construction (no
|
||||
//! custom required headers or body fields, per #47). The middleware resolves
|
||||
//! the key to a [`Principal`] via the [`EntitlementProvider`], carries it in
|
||||
//! the request extensions for cortex-side metering/enforcement (#51/#52), and
|
||||
//! stamps it as internal headers on the request so it reaches neuron, which
|
||||
//! trusts cortex's assertion over WireGuard (#54).
|
||||
//!
|
||||
//! Anti-spoofing: any client-supplied principal header is **stripped** before
|
||||
//! the authoritative value is stamped, so a client can never assert a
|
||||
//! principal it didn't authenticate as.
|
||||
//!
|
||||
//! Rejection contract (#63): missing key under `require_auth`, or any present
|
||||
//! but unresolvable key, yields `401 invalid_api_key` in the #60 envelope.
|
||||
|
||||
use crate::error::envelope_response;
|
||||
use crate::state::CortexState;
|
||||
use axum::extract::{Request, State};
|
||||
use axum::http::header::AUTHORIZATION;
|
||||
use axum::http::{HeaderMap, HeaderValue};
|
||||
use axum::middleware::Next;
|
||||
use axum::response::Response;
|
||||
use cortex_core::entitlements::{AuthError, HEADER_ACCOUNT_ID, HEADER_KEY_ID};
|
||||
use cortex_core::error_envelope::OpenAiError;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Endpoints that never require auth: liveness/readiness probes. Everything
|
||||
/// else flows through resolution.
|
||||
fn is_public(path: &str) -> bool {
|
||||
path == "/health" || path == "/"
|
||||
}
|
||||
|
||||
/// Extract the bearer token from an `Authorization` header value, if present
|
||||
/// and well-formed. Scheme match is case-insensitive per RFC 7235.
|
||||
fn parse_bearer(headers: &HeaderMap) -> Option<String> {
|
||||
let raw = headers.get(AUTHORIZATION)?.to_str().ok()?;
|
||||
let (scheme, token) = raw.split_once(' ')?;
|
||||
if scheme.eq_ignore_ascii_case("bearer") {
|
||||
let token = token.trim();
|
||||
(!token.is_empty()).then(|| token.to_string())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Axum middleware: resolve the bearer key, attach the principal, stamp the
|
||||
/// internal headers. Wired in `build_app` via `from_fn_with_state`.
|
||||
pub async fn require_principal(
|
||||
State(fleet): State<Arc<CortexState>>,
|
||||
mut req: Request,
|
||||
next: Next,
|
||||
) -> Response {
|
||||
if is_public(req.uri().path()) {
|
||||
return next.run(req).await;
|
||||
}
|
||||
|
||||
// Anti-spoof: drop any client-supplied principal headers up front.
|
||||
{
|
||||
let headers = req.headers_mut();
|
||||
headers.remove(HEADER_ACCOUNT_ID);
|
||||
headers.remove(HEADER_KEY_ID);
|
||||
}
|
||||
|
||||
match parse_bearer(req.headers()) {
|
||||
Some(key) => match fleet.entitlements.resolve(&key).await {
|
||||
Ok(principal) => {
|
||||
// Stamp the authoritative principal for neuron. Account/key
|
||||
// ids come from operator config, so they're valid header
|
||||
// values; guard anyway and skip a malformed one rather than
|
||||
// panic.
|
||||
if let (Ok(account), Ok(key_id)) = (
|
||||
HeaderValue::from_str(&principal.account_id),
|
||||
HeaderValue::from_str(&principal.key_id),
|
||||
) {
|
||||
let headers = req.headers_mut();
|
||||
headers.insert(HEADER_ACCOUNT_ID, account);
|
||||
headers.insert(HEADER_KEY_ID, key_id);
|
||||
}
|
||||
// Carry the typed principal for cortex-side metering (#51)
|
||||
// and budget enforcement (#52).
|
||||
req.extensions_mut().insert(principal);
|
||||
next.run(req).await
|
||||
}
|
||||
// The entitlement authority is unreachable (upstream client
|
||||
// blip, #57). Fail **closed but distinct**: a transient outage
|
||||
// must not reject a real key as `401 invalid_api_key` — it's a
|
||||
// retryable `503`. This holds regardless of require_auth: we
|
||||
// can't safely serve a key we couldn't authorize.
|
||||
Err(AuthError::Unavailable { retry_after_secs }) => {
|
||||
envelope_response(OpenAiError::service_unavailable(
|
||||
"entitlement authority temporarily unavailable",
|
||||
Some(retry_after_secs),
|
||||
))
|
||||
}
|
||||
// A genuinely unrecognized key only hard-fails when auth is
|
||||
// *required*. In allow-anonymous mode (the default) we IGNORE it
|
||||
// and serve unauthenticated — otherwise the placeholder keys that
|
||||
// OpenAI-compatible clients send by default (opencode, Open WebUI,
|
||||
// Agent Zero, litellm) would all break though the operator never
|
||||
// opted into auth. Pre-#49 the bearer was never inspected; this
|
||||
// preserves that for require_auth=false.
|
||||
Err(AuthError::InvalidKey) => {
|
||||
if fleet.require_auth {
|
||||
unauthorized("invalid API key")
|
||||
} else {
|
||||
tracing::debug!(
|
||||
"ignoring unrecognized bearer token (require_auth=false): serving anonymously"
|
||||
);
|
||||
next.run(req).await
|
||||
}
|
||||
}
|
||||
},
|
||||
None => {
|
||||
if fleet.require_auth {
|
||||
unauthorized("missing API key; supply 'Authorization: Bearer <key>'")
|
||||
} else {
|
||||
next.run(req).await
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `401 invalid_api_key` in the standard envelope (#63).
|
||||
fn unauthorized(message: &str) -> Response {
|
||||
envelope_response(OpenAiError::invalid_api_key(message))
|
||||
}
|
||||
|
||||
/// Copy the cortex-stamped principal headers from an inbound [`HeaderMap`]
|
||||
/// onto an outbound reqwest builder. Used by the Anthropic proxy paths,
|
||||
/// which construct their own upstream requests instead of going through
|
||||
/// [`crate::proxy::forward_request`] (which forwards all headers verbatim).
|
||||
pub fn forward_principal_headers(
|
||||
mut builder: reqwest::RequestBuilder,
|
||||
headers: &HeaderMap,
|
||||
) -> reqwest::RequestBuilder {
|
||||
for name in [HEADER_ACCOUNT_ID, HEADER_KEY_ID] {
|
||||
if let Some(value) = headers.get(name) {
|
||||
builder = builder.header(name, value);
|
||||
}
|
||||
}
|
||||
builder
|
||||
}
|
||||
112
crates/cortex-gateway/src/entitlements_chain.rs
Normal file
@@ -0,0 +1,112 @@
|
||||
//! Chained entitlement provider (#57): operator-local keys first, mesh
|
||||
//! upstream for everything else.
|
||||
//!
|
||||
//! `resolve` tries the [`LocalEntitlementProvider`] (operator + infra keys —
|
||||
//! never a network hop); only a locally-unknown key falls through to
|
||||
//! [`UpstreamEntitlementProvider`]. Because the local provider treats an
|
||||
//! unconfigured principal as uncapped, reserve/settle/release/snapshot must
|
||||
//! **not** blindly hit local — they dispatch to whichever backend resolved
|
||||
//! that account, remembered in a map keyed by `account_id` (populated at
|
||||
//! resolve time).
|
||||
|
||||
use crate::entitlements_local::LocalEntitlementProvider;
|
||||
use crate::entitlements_upstream::UpstreamEntitlementProvider;
|
||||
use async_trait::async_trait;
|
||||
use cortex_core::entitlements::{
|
||||
AuthError, BudgetError, BudgetSnapshot, EntitlementProvider, Principal, Reservation,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum Backend {
|
||||
Local,
|
||||
Upstream,
|
||||
}
|
||||
|
||||
pub struct ChainedEntitlementProvider {
|
||||
local: LocalEntitlementProvider,
|
||||
upstream: UpstreamEntitlementProvider,
|
||||
/// account_id → which backend owns it, learned at resolve time.
|
||||
backends: RwLock<HashMap<String, Backend>>,
|
||||
}
|
||||
|
||||
impl ChainedEntitlementProvider {
|
||||
pub fn new(local: LocalEntitlementProvider, upstream: UpstreamEntitlementProvider) -> Self {
|
||||
Self {
|
||||
local,
|
||||
upstream,
|
||||
backends: RwLock::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn record(&self, account_id: &str, backend: Backend) {
|
||||
self.backends
|
||||
.write()
|
||||
.await
|
||||
.insert(account_id.to_string(), backend);
|
||||
}
|
||||
|
||||
/// The backend that owns `account_id`. Defaults to `Upstream` for an
|
||||
/// account never resolved this process-lifetime (a resolve always
|
||||
/// precedes reserve in a request, so this is just a safe fallback —
|
||||
/// upstream fails closed if the account is bogus).
|
||||
async fn backend_for(&self, account_id: &str) -> Backend {
|
||||
self.backends
|
||||
.read()
|
||||
.await
|
||||
.get(account_id)
|
||||
.copied()
|
||||
.unwrap_or(Backend::Upstream)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl EntitlementProvider for ChainedEntitlementProvider {
|
||||
async fn resolve(&self, api_key: &str) -> Result<Principal, AuthError> {
|
||||
match self.local.resolve(api_key).await {
|
||||
Ok(p) => {
|
||||
self.record(&p.account_id, Backend::Local).await;
|
||||
Ok(p)
|
||||
}
|
||||
Err(AuthError::InvalidKey) => {
|
||||
let p = self.upstream.resolve(api_key).await?;
|
||||
self.record(&p.account_id, Backend::Upstream).await;
|
||||
Ok(p)
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
async fn reserve(
|
||||
&self,
|
||||
principal: &Principal,
|
||||
max_tokens: u64,
|
||||
) -> Result<Reservation, BudgetError> {
|
||||
match self.backend_for(&principal.account_id).await {
|
||||
Backend::Local => self.local.reserve(principal, max_tokens).await,
|
||||
Backend::Upstream => self.upstream.reserve(principal, max_tokens).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn settle(&self, reservation: Reservation, actual_tokens: u64) {
|
||||
match self.backend_for(&reservation.principal.account_id).await {
|
||||
Backend::Local => self.local.settle(reservation, actual_tokens).await,
|
||||
Backend::Upstream => self.upstream.settle(reservation, actual_tokens).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn release(&self, reservation: Reservation) {
|
||||
match self.backend_for(&reservation.principal.account_id).await {
|
||||
Backend::Local => self.local.release(reservation).await,
|
||||
Backend::Upstream => self.upstream.release(reservation).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn snapshot(&self, principal: &Principal) -> Option<BudgetSnapshot> {
|
||||
match self.backend_for(&principal.account_id).await {
|
||||
Backend::Local => self.local.snapshot(principal).await,
|
||||
Backend::Upstream => self.upstream.snapshot(principal).await,
|
||||
}
|
||||
}
|
||||
}
|
||||
317
crates/cortex-gateway/src/entitlements_local.rs
Normal file
@@ -0,0 +1,317 @@
|
||||
//! The local/static [`EntitlementProvider`] (#50).
|
||||
//!
|
||||
//! Accounts, keys, and hard caps come from operator config
|
||||
//! ([`cortex_core::config::EntitlementsConfig`]); reservations and settled
|
||||
//! spend are tracked in-process. This lands auth + per-key caps + the
|
||||
//! amplification fix before any upstream clearing house exists; the future
|
||||
//! helexa-upstream client (#57) implements the same trait.
|
||||
//!
|
||||
//! Budget math is serialized under a single [`std::sync::Mutex`] so
|
||||
//! reserve/settle/release are atomic — a key's `spent + reserved` can never
|
||||
//! exceed its hard cap even under concurrent requests (the #52 guarantee).
|
||||
//! The lock is held only for the in-memory arithmetic, never across an
|
||||
//! await.
|
||||
|
||||
use cortex_core::config::{ApiKeyConfig, EntitlementsConfig};
|
||||
use cortex_core::entitlements::{
|
||||
AuthError, BudgetError, BudgetSnapshot, CapWindow, EntitlementProvider, Principal, Reservation,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::Instant;
|
||||
|
||||
/// Per-key budget configuration (resolved from [`ApiKeyConfig`]).
|
||||
struct Budget {
|
||||
hard_cap: Option<u64>,
|
||||
window: CapWindow,
|
||||
}
|
||||
|
||||
/// Live, mutable accounting for one key over its current window.
|
||||
#[derive(Default)]
|
||||
struct Ledger {
|
||||
/// Settled spend in the current window.
|
||||
spent: u64,
|
||||
/// Sum of outstanding (un-settled) reservations.
|
||||
reserved: u64,
|
||||
/// Start of the current rolling window; `None` until the first reserve.
|
||||
/// Unused for [`CapWindow::Balance`].
|
||||
window_start: Option<Instant>,
|
||||
}
|
||||
|
||||
pub struct LocalEntitlementProvider {
|
||||
/// Bearer token → principal.
|
||||
keys: HashMap<String, Principal>,
|
||||
/// `key_id` → budget config.
|
||||
budgets: HashMap<String, Budget>,
|
||||
/// `key_id` → live ledger.
|
||||
ledgers: Mutex<HashMap<String, Ledger>>,
|
||||
/// Monotonic source of opaque reservation handles.
|
||||
next_id: AtomicU64,
|
||||
}
|
||||
|
||||
impl LocalEntitlementProvider {
|
||||
/// Build from the `[entitlements]` config. A key without an explicit
|
||||
/// `key_id` is tracked at `account_id` granularity (its secret is never
|
||||
/// used as a label).
|
||||
pub fn from_config(config: &EntitlementsConfig) -> Self {
|
||||
let mut keys = HashMap::new();
|
||||
let mut budgets = HashMap::new();
|
||||
for ApiKeyConfig {
|
||||
key,
|
||||
account_id,
|
||||
key_id,
|
||||
hard_cap,
|
||||
window,
|
||||
} in &config.keys
|
||||
{
|
||||
let key_id = key_id.clone().unwrap_or_else(|| account_id.clone());
|
||||
keys.insert(
|
||||
key.clone(),
|
||||
Principal {
|
||||
account_id: account_id.clone(),
|
||||
key_id: key_id.clone(),
|
||||
},
|
||||
);
|
||||
budgets.insert(
|
||||
key_id,
|
||||
Budget {
|
||||
hard_cap: *hard_cap,
|
||||
window: window.clone(),
|
||||
},
|
||||
);
|
||||
}
|
||||
Self {
|
||||
keys,
|
||||
budgets,
|
||||
ledgers: Mutex::new(HashMap::new()),
|
||||
next_id: AtomicU64::new(1),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Tokens still available under `cap` given current `spent`/`reserved`.
|
||||
/// `None` cap = unlimited.
|
||||
fn available(cap: Option<u64>, spent: u64, reserved: u64) -> Option<u64> {
|
||||
cap.map(|c| c.saturating_sub(spent).saturating_sub(reserved))
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl EntitlementProvider for LocalEntitlementProvider {
|
||||
async fn resolve(&self, api_key: &str) -> Result<Principal, AuthError> {
|
||||
self.keys.get(api_key).cloned().ok_or(AuthError::InvalidKey)
|
||||
}
|
||||
|
||||
async fn reserve(
|
||||
&self,
|
||||
principal: &Principal,
|
||||
max_tokens: u64,
|
||||
) -> Result<Reservation, BudgetError> {
|
||||
// A principal with no configured budget (or an uncapped one) always
|
||||
// reserves; we still track spend for metrics.
|
||||
let budget = self.budgets.get(&principal.key_id);
|
||||
let (cap, window) = match budget {
|
||||
Some(b) => (b.hard_cap, b.window.clone()),
|
||||
None => (None, CapWindow::Balance),
|
||||
};
|
||||
|
||||
let mut ledgers = self.ledgers.lock().expect("ledger mutex poisoned");
|
||||
let ledger = ledgers.entry(principal.key_id.clone()).or_default();
|
||||
|
||||
// Lazily reset a rolling window that has elapsed before checking.
|
||||
let mut retry_after_secs = 0;
|
||||
if let CapWindow::Rolling { seconds } = window {
|
||||
let now = Instant::now();
|
||||
match ledger.window_start {
|
||||
Some(start) if now.duration_since(start).as_secs() < seconds => {
|
||||
retry_after_secs = seconds - now.duration_since(start).as_secs();
|
||||
}
|
||||
_ => {
|
||||
// First reserve, or the window has fully elapsed: reset.
|
||||
ledger.spent = 0;
|
||||
ledger.window_start = Some(now);
|
||||
retry_after_secs = seconds;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(avail) = available(cap, ledger.spent, ledger.reserved)
|
||||
&& max_tokens > avail
|
||||
{
|
||||
return Err(match window {
|
||||
CapWindow::Rolling { .. } => BudgetError::RateLimited {
|
||||
requested: max_tokens,
|
||||
available: avail,
|
||||
// At least 1s so clients don't hot-loop on a sub-second
|
||||
// remainder.
|
||||
retry_after_secs: retry_after_secs.max(1),
|
||||
},
|
||||
CapWindow::Balance => BudgetError::InsufficientQuota {
|
||||
requested: max_tokens,
|
||||
available: avail,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
ledger.reserved += max_tokens;
|
||||
Ok(Reservation {
|
||||
id: self.next_id.fetch_add(1, Ordering::Relaxed),
|
||||
principal: principal.clone(),
|
||||
reserved: max_tokens,
|
||||
})
|
||||
}
|
||||
|
||||
async fn settle(&self, reservation: Reservation, actual_tokens: u64) {
|
||||
let mut ledgers = self.ledgers.lock().expect("ledger mutex poisoned");
|
||||
if let Some(ledger) = ledgers.get_mut(&reservation.principal.key_id) {
|
||||
ledger.reserved = ledger.reserved.saturating_sub(reservation.reserved);
|
||||
ledger.spent += actual_tokens;
|
||||
}
|
||||
}
|
||||
|
||||
async fn release(&self, reservation: Reservation) {
|
||||
let mut ledgers = self.ledgers.lock().expect("ledger mutex poisoned");
|
||||
if let Some(ledger) = ledgers.get_mut(&reservation.principal.key_id) {
|
||||
ledger.reserved = ledger.reserved.saturating_sub(reservation.reserved);
|
||||
}
|
||||
}
|
||||
|
||||
async fn snapshot(&self, principal: &Principal) -> Option<BudgetSnapshot> {
|
||||
let ledgers = self.ledgers.lock().expect("ledger mutex poisoned");
|
||||
let (spent, reserved) = ledgers
|
||||
.get(&principal.key_id)
|
||||
.map(|l| (l.spent, l.reserved))
|
||||
.unwrap_or((0, 0));
|
||||
let hard_cap = self.budgets.get(&principal.key_id).and_then(|b| b.hard_cap);
|
||||
Some(BudgetSnapshot {
|
||||
hard_cap,
|
||||
spent,
|
||||
reserved,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn provider() -> LocalEntitlementProvider {
|
||||
let config = EntitlementsConfig {
|
||||
require_auth: true,
|
||||
keys: vec![
|
||||
ApiKeyConfig {
|
||||
key: "sk-balance".into(),
|
||||
account_id: "acct-a".into(),
|
||||
key_id: Some("key-balance".into()),
|
||||
hard_cap: Some(1_000),
|
||||
window: CapWindow::Balance,
|
||||
},
|
||||
ApiKeyConfig {
|
||||
key: "sk-rolling".into(),
|
||||
account_id: "acct-b".into(),
|
||||
key_id: Some("key-rolling".into()),
|
||||
hard_cap: Some(500),
|
||||
window: CapWindow::Rolling { seconds: 3_600 },
|
||||
},
|
||||
ApiKeyConfig {
|
||||
key: "sk-infra".into(),
|
||||
account_id: "operator".into(),
|
||||
key_id: Some("key-infra".into()),
|
||||
hard_cap: None,
|
||||
window: CapWindow::Balance,
|
||||
},
|
||||
],
|
||||
};
|
||||
LocalEntitlementProvider::from_config(&config)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resolves_configured_key_to_principal() {
|
||||
let p = provider();
|
||||
let principal = p.resolve("sk-balance").await.expect("known key resolves");
|
||||
assert_eq!(principal.account_id, "acct-a");
|
||||
assert_eq!(principal.key_id, "key-balance");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_key_is_invalid() {
|
||||
let p = provider();
|
||||
assert!(matches!(
|
||||
p.resolve("sk-nope").await,
|
||||
Err(AuthError::InvalidKey)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reserve_settle_release_round_trip() {
|
||||
let p = provider();
|
||||
let principal = p.resolve("sk-balance").await.unwrap();
|
||||
|
||||
let r = p.reserve(&principal, 400).await.expect("within cap");
|
||||
// Reserved, not yet spent.
|
||||
let snap = p.snapshot(&principal).await.unwrap();
|
||||
assert_eq!(snap.hard_cap, Some(1_000));
|
||||
assert_eq!(snap.reserved, 400);
|
||||
assert_eq!(snap.spent, 0);
|
||||
|
||||
// Used fewer tokens than reserved → remainder released, spend exact.
|
||||
p.settle(r, 250).await;
|
||||
let snap = p.snapshot(&principal).await.unwrap();
|
||||
assert_eq!(snap.reserved, 0);
|
||||
assert_eq!(snap.spent, 250);
|
||||
|
||||
// A reservation that is released contributes no spend.
|
||||
let r2 = p.reserve(&principal, 100).await.unwrap();
|
||||
p.release(r2).await;
|
||||
let snap = p.snapshot(&principal).await.unwrap();
|
||||
assert_eq!(snap.reserved, 0);
|
||||
assert_eq!(snap.spent, 250);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn balance_over_cap_is_insufficient_quota() {
|
||||
let p = provider();
|
||||
let principal = p.resolve("sk-balance").await.unwrap();
|
||||
// Reserve most of the cap, then ask for more than remains.
|
||||
let _r = p.reserve(&principal, 900).await.unwrap();
|
||||
let err = p.reserve(&principal, 200).await.expect_err("over cap");
|
||||
match err {
|
||||
BudgetError::InsufficientQuota {
|
||||
requested,
|
||||
available,
|
||||
} => {
|
||||
assert_eq!(requested, 200);
|
||||
assert_eq!(available, 100);
|
||||
}
|
||||
other => panic!("expected InsufficientQuota, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rolling_over_cap_is_rate_limited_with_retry_after() {
|
||||
let p = provider();
|
||||
let principal = p.resolve("sk-rolling").await.unwrap();
|
||||
let _r = p.reserve(&principal, 500).await.unwrap();
|
||||
let err = p.reserve(&principal, 1).await.expect_err("over cap");
|
||||
match err {
|
||||
BudgetError::RateLimited {
|
||||
retry_after_secs, ..
|
||||
} => {
|
||||
assert!(retry_after_secs >= 1, "must advertise a retry hint");
|
||||
assert!(retry_after_secs <= 3_600);
|
||||
}
|
||||
other => panic!("expected RateLimited, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn uncapped_infra_key_never_refuses() {
|
||||
let p = provider();
|
||||
let principal = p.resolve("sk-infra").await.unwrap();
|
||||
let r = p.reserve(&principal, 10_000_000).await.expect("uncapped");
|
||||
p.settle(r, 10_000_000).await;
|
||||
let snap = p.snapshot(&principal).await.unwrap();
|
||||
assert_eq!(snap.hard_cap, None);
|
||||
assert_eq!(snap.spent, 10_000_000);
|
||||
}
|
||||
}
|
||||
246
crates/cortex-gateway/src/entitlements_upstream.rs
Normal file
@@ -0,0 +1,246 @@
|
||||
//! helexa-upstream client (#57): an [`EntitlementProvider`] that resolves
|
||||
//! keys and reserves/settles budget against the mesh authority's
|
||||
//! `/authz/v1` surface (B2). It is "just another impl of the trait" — cortex
|
||||
//! enforcement (`auth.rs`, `metering.rs`) is unchanged.
|
||||
//!
|
||||
//! **Fail closed.** When upstream is unreachable, `resolve` returns
|
||||
//! [`AuthError::Unavailable`] (→ `503`, never `401`) and `reserve` refuses
|
||||
//! with a retryable [`BudgetError::RateLimited`] — a request is never served
|
||||
//! on an un-authorized key, and a real key is never rejected as invalid
|
||||
//! during a blip.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use cortex_core::config::UpstreamClientConfig;
|
||||
use cortex_core::entitlements::{
|
||||
AuthError, BudgetError, BudgetSnapshot, EntitlementProvider, Principal, Reservation,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use std::time::Duration;
|
||||
|
||||
/// Retry-After (seconds) advertised when we fail closed on an upstream
|
||||
/// outage.
|
||||
const FAIL_CLOSED_RETRY_SECS: u64 = 5;
|
||||
|
||||
pub struct UpstreamEntitlementProvider {
|
||||
client: reqwest::Client,
|
||||
base_url: String,
|
||||
bearer: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct PrincipalDto {
|
||||
account_id: String,
|
||||
key_id: String,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct SnapshotDto {
|
||||
hard_cap: Option<u64>,
|
||||
spent: u64,
|
||||
reserved: u64,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct ResolveResp {
|
||||
principal: PrincipalDto,
|
||||
#[allow(dead_code)]
|
||||
snapshot: Option<SnapshotDto>,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct ReserveResp {
|
||||
reservation_id: Option<i64>,
|
||||
rejected: Option<Rejection>,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
enum Rejection {
|
||||
InsufficientQuota {
|
||||
requested: u64,
|
||||
available: u64,
|
||||
},
|
||||
RateLimited {
|
||||
requested: u64,
|
||||
available: u64,
|
||||
retry_after_secs: u64,
|
||||
},
|
||||
}
|
||||
|
||||
impl UpstreamEntitlementProvider {
|
||||
pub fn new(cfg: &UpstreamClientConfig) -> Self {
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(cfg.timeout_secs))
|
||||
.build()
|
||||
.expect("failed to build upstream HTTP client");
|
||||
Self {
|
||||
client,
|
||||
base_url: cfg.url.trim_end_matches('/').to_string(),
|
||||
bearer: cfg.bearer.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
fn url(&self, path: &str) -> String {
|
||||
format!("{}{}", self.base_url, path)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl EntitlementProvider for UpstreamEntitlementProvider {
|
||||
async fn resolve(&self, api_key: &str) -> Result<Principal, AuthError> {
|
||||
let resp = self
|
||||
.client
|
||||
.post(self.url("/authz/v1/resolve"))
|
||||
.bearer_auth(&self.bearer)
|
||||
.json(&serde_json::json!({ "api_key": api_key }))
|
||||
.send()
|
||||
.await;
|
||||
let resp = match resp {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "upstream resolve unreachable; failing closed");
|
||||
return Err(AuthError::Unavailable {
|
||||
retry_after_secs: FAIL_CLOSED_RETRY_SECS,
|
||||
});
|
||||
}
|
||||
};
|
||||
if resp.status().as_u16() == 401 {
|
||||
return Err(AuthError::InvalidKey);
|
||||
}
|
||||
if !resp.status().is_success() {
|
||||
return Err(AuthError::Unavailable {
|
||||
retry_after_secs: FAIL_CLOSED_RETRY_SECS,
|
||||
});
|
||||
}
|
||||
match resp.json::<ResolveResp>().await {
|
||||
Ok(r) => Ok(Principal {
|
||||
account_id: r.principal.account_id,
|
||||
key_id: r.principal.key_id,
|
||||
}),
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "upstream resolve: bad body; failing closed");
|
||||
Err(AuthError::Unavailable {
|
||||
retry_after_secs: FAIL_CLOSED_RETRY_SECS,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn reserve(
|
||||
&self,
|
||||
principal: &Principal,
|
||||
max_tokens: u64,
|
||||
) -> Result<Reservation, BudgetError> {
|
||||
let fail_closed = || BudgetError::RateLimited {
|
||||
requested: max_tokens,
|
||||
available: 0,
|
||||
retry_after_secs: FAIL_CLOSED_RETRY_SECS,
|
||||
};
|
||||
let resp = self
|
||||
.client
|
||||
.post(self.url("/authz/v1/reserve"))
|
||||
.bearer_auth(&self.bearer)
|
||||
.json(&serde_json::json!({
|
||||
"account_id": principal.account_id,
|
||||
"key_id": principal.key_id,
|
||||
"max_tokens": max_tokens,
|
||||
}))
|
||||
.send()
|
||||
.await;
|
||||
let resp = match resp {
|
||||
Ok(r) if r.status().is_success() => r,
|
||||
Ok(r) => {
|
||||
tracing::warn!(status = %r.status(), "upstream reserve non-2xx; failing closed");
|
||||
return Err(fail_closed());
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "upstream reserve unreachable; failing closed");
|
||||
return Err(fail_closed());
|
||||
}
|
||||
};
|
||||
match resp.json::<ReserveResp>().await {
|
||||
Ok(ReserveResp {
|
||||
reservation_id: Some(id),
|
||||
..
|
||||
}) => Ok(Reservation {
|
||||
id: id as u64,
|
||||
principal: principal.clone(),
|
||||
reserved: max_tokens,
|
||||
}),
|
||||
Ok(ReserveResp {
|
||||
rejected:
|
||||
Some(Rejection::InsufficientQuota {
|
||||
requested,
|
||||
available,
|
||||
}),
|
||||
..
|
||||
}) => Err(BudgetError::InsufficientQuota {
|
||||
requested,
|
||||
available,
|
||||
}),
|
||||
Ok(ReserveResp {
|
||||
rejected:
|
||||
Some(Rejection::RateLimited {
|
||||
requested,
|
||||
available,
|
||||
retry_after_secs,
|
||||
}),
|
||||
..
|
||||
}) => Err(BudgetError::RateLimited {
|
||||
requested,
|
||||
available,
|
||||
retry_after_secs,
|
||||
}),
|
||||
_ => Err(fail_closed()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn settle(&self, reservation: Reservation, actual_tokens: u64) {
|
||||
// Best-effort; a lost settle is reaped by the upstream sweeper (B2).
|
||||
let _ = self
|
||||
.client
|
||||
.post(self.url("/authz/v1/settle"))
|
||||
.bearer_auth(&self.bearer)
|
||||
.json(&serde_json::json!({
|
||||
"reservation_id": reservation.id as i64,
|
||||
"actual_tokens": actual_tokens,
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.inspect_err(
|
||||
|e| tracing::warn!(error = %e, "upstream settle failed (sweeper will reap)"),
|
||||
);
|
||||
}
|
||||
|
||||
async fn release(&self, reservation: Reservation) {
|
||||
let _ = self
|
||||
.client
|
||||
.post(self.url("/authz/v1/release"))
|
||||
.bearer_auth(&self.bearer)
|
||||
.json(&serde_json::json!({ "reservation_id": reservation.id as i64 }))
|
||||
.send()
|
||||
.await
|
||||
.inspect_err(
|
||||
|e| tracing::warn!(error = %e, "upstream release failed (sweeper will reap)"),
|
||||
);
|
||||
}
|
||||
|
||||
async fn snapshot(&self, principal: &Principal) -> Option<BudgetSnapshot> {
|
||||
let resp = self
|
||||
.client
|
||||
.post(self.url("/authz/v1/snapshot"))
|
||||
.bearer_auth(&self.bearer)
|
||||
.json(&serde_json::json!({
|
||||
"account_id": principal.account_id,
|
||||
"key_id": principal.key_id,
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.ok()?;
|
||||
if !resp.status().is_success() {
|
||||
return None;
|
||||
}
|
||||
let dto = resp.json::<SnapshotDto>().await.ok()?;
|
||||
Some(BudgetSnapshot {
|
||||
hard_cap: dto.hard_cap,
|
||||
spent: dto.spent,
|
||||
reserved: dto.reserved,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -190,7 +190,7 @@ async fn completions(
|
||||
/// `POST /v1/messages` — accept Anthropic format, translate, proxy, translate back.
|
||||
async fn anthropic_messages(
|
||||
State(fleet): State<Arc<CortexState>>,
|
||||
_headers: HeaderMap,
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
) -> Response {
|
||||
// Parse as Anthropic request.
|
||||
@@ -306,6 +306,29 @@ async fn anthropic_messages(
|
||||
}
|
||||
let start = Instant::now();
|
||||
|
||||
// Per-request metering + budget enforcement (#51/#52), same lifecycle as
|
||||
// the OpenAI paths. Estimate from the translated OpenAI body (what neuron
|
||||
// sees). Refuse over-cap before dispatch via the #63 envelope; otherwise
|
||||
// build the sink consumed by whichever branch runs below.
|
||||
let usage_sink = match crate::metering::principal_from_headers(&headers) {
|
||||
Some(principal) => {
|
||||
let advertised =
|
||||
advertised_output_limit(&fleet, &route.node_name, &route.resolved_model_id).await;
|
||||
let max_tokens = crate::metering::reservation_estimate(&openai_body, advertised);
|
||||
match crate::metering::reserve_or_reject(
|
||||
Arc::clone(&fleet.entitlements),
|
||||
&principal,
|
||||
max_tokens,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(guard) => Some(crate::metering::usage_sink(principal, guard)),
|
||||
Err(env) => return crate::error::envelope_response(env),
|
||||
}
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
|
||||
if is_streaming {
|
||||
// Anthropic SSE translation (#24): upstream speaks OpenAI SSE;
|
||||
// re-frame it event-by-event into Anthropic's message_start /
|
||||
@@ -316,6 +339,8 @@ async fn anthropic_messages(
|
||||
openai_body,
|
||||
&model_id,
|
||||
&route.node_name,
|
||||
&headers,
|
||||
usage_sink,
|
||||
)
|
||||
.await;
|
||||
metrics::histogram!("cortex_request_duration_seconds", &labels)
|
||||
@@ -335,13 +360,16 @@ async fn anthropic_messages(
|
||||
cold_start = route.cold_start,
|
||||
"proxying request"
|
||||
);
|
||||
let upstream_resp = fleet
|
||||
.http_client
|
||||
.post(&target_url)
|
||||
.body(openai_body)
|
||||
.header("content-type", "application/json")
|
||||
.send()
|
||||
.await;
|
||||
let upstream_resp = crate::auth::forward_principal_headers(
|
||||
fleet
|
||||
.http_client
|
||||
.post(&target_url)
|
||||
.body(openai_body)
|
||||
.header("content-type", "application/json"),
|
||||
&headers,
|
||||
)
|
||||
.send()
|
||||
.await;
|
||||
|
||||
let upstream_resp = match upstream_resp {
|
||||
Ok(r) => r,
|
||||
@@ -437,6 +465,15 @@ async fn anthropic_messages(
|
||||
|
||||
metrics::histogram!("cortex_request_duration_seconds", &labels)
|
||||
.record(start.elapsed().as_secs_f64());
|
||||
// Settle metering with the upstream usage (#51). Scanned from the
|
||||
// raw body — same engine-truth source as the streaming path — so we
|
||||
// don't depend on the typed usage struct's optionality.
|
||||
if let Some(sink) = usage_sink {
|
||||
let tail = String::from_utf8_lossy(&body_bytes);
|
||||
let prompt = proxy::last_count_for(&tail, "prompt_tokens").unwrap_or(0);
|
||||
let completion = proxy::last_count_for(&tail, "completion_tokens").unwrap_or(0);
|
||||
sink(prompt, completion);
|
||||
}
|
||||
// Did the model actually produce a structured tool call, or just
|
||||
// text? This is the single most useful signal for "is tool
|
||||
// calling working end-to-end" — a `false` here alongside a
|
||||
@@ -724,6 +761,19 @@ async fn proxy_with_metrics(
|
||||
body: Bytes,
|
||||
model_id: &str,
|
||||
) -> Response {
|
||||
// Fail-fast prompt pre-validation (#56): refuse a prompt that already
|
||||
// exceeds the model's advertised context window *before* dispatching to
|
||||
// neuron — the same `400 context_length_exceeded` neuron would emit on
|
||||
// overflow, just earlier and without burning a cold-load/queue slot.
|
||||
// cortex has no tokenizer, so the estimate under-counts and neuron stays
|
||||
// the exact wall; we only catch gross overages (the A0 failure mode).
|
||||
if let Some(context) = advertised_context(fleet, &route.node_name, model_id).await {
|
||||
let est = estimate_prompt_tokens(&body);
|
||||
if est > context {
|
||||
return context_length_exceeded_response(context, est, &headers);
|
||||
}
|
||||
}
|
||||
|
||||
let labels = [
|
||||
("model", model_id.to_string()),
|
||||
("node", route.node_name.clone()),
|
||||
@@ -734,9 +784,42 @@ async fn proxy_with_metrics(
|
||||
metrics::counter!("cortex_cold_starts_total", &labels).increment(1);
|
||||
}
|
||||
|
||||
// Per-request metering + budget enforcement (#51/#52): reconstruct the
|
||||
// principal from the middleware-stamped headers, reserve the request's
|
||||
// upper-bound cost (prompt estimate + max output), and build the
|
||||
// completion sink that settles actual spend when the response finishes.
|
||||
// A reservation over the hard cap is refused *before* dispatch with the
|
||||
// #63 envelope. Anonymous requests skip all of this. Must happen before
|
||||
// `headers`/`body` are moved into the proxy.
|
||||
let usage_sink = match crate::metering::principal_from_headers(&headers) {
|
||||
Some(principal) => {
|
||||
let advertised = advertised_output_limit(fleet, &route.node_name, model_id).await;
|
||||
let max_tokens = crate::metering::reservation_estimate(&body, advertised);
|
||||
match crate::metering::reserve_or_reject(
|
||||
Arc::clone(&fleet.entitlements),
|
||||
&principal,
|
||||
max_tokens,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(guard) => Some(crate::metering::usage_sink(principal, guard)),
|
||||
Err(env) => return crate::error::envelope_response(env),
|
||||
}
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
|
||||
let start = Instant::now();
|
||||
let result =
|
||||
proxy::forward_request(&fleet.http_client, route, path, headers, body, model_id).await;
|
||||
let result = proxy::forward_request(
|
||||
&fleet.http_client,
|
||||
route,
|
||||
path,
|
||||
headers,
|
||||
body,
|
||||
model_id,
|
||||
usage_sink,
|
||||
)
|
||||
.await;
|
||||
let duration = start.elapsed();
|
||||
|
||||
match result {
|
||||
@@ -755,6 +838,117 @@ async fn proxy_with_metrics(
|
||||
}
|
||||
}
|
||||
|
||||
/// The model's advertised `limit.output` (#62) on a given node, used as the
|
||||
/// default output budget for budget reservations (#52) when the request
|
||||
/// omits `max_(completion_)tokens`. `None` when the node/model/limit is
|
||||
/// unknown — callers fall back to [`crate::metering::FALLBACK_MAX_OUTPUT`].
|
||||
async fn advertised_output_limit(
|
||||
fleet: &CortexState,
|
||||
node_name: &str,
|
||||
model_id: &str,
|
||||
) -> Option<u64> {
|
||||
let nodes = fleet.nodes.read().await;
|
||||
nodes
|
||||
.get(node_name)?
|
||||
.models
|
||||
.get(model_id)?
|
||||
.limit
|
||||
.as_ref()
|
||||
.map(|l| l.output as u64)
|
||||
}
|
||||
|
||||
/// The model's advertised hard context window (`limit.context`, #62/#67) on a
|
||||
/// node, used for fail-fast prompt pre-validation (#56). `None` when no limit
|
||||
/// is known — pre-validation is then skipped and neuron remains the wall.
|
||||
async fn advertised_context(fleet: &CortexState, node_name: &str, model_id: &str) -> Option<u64> {
|
||||
let nodes = fleet.nodes.read().await;
|
||||
nodes
|
||||
.get(node_name)?
|
||||
.models
|
||||
.get(model_id)?
|
||||
.limit
|
||||
.as_ref()
|
||||
.map(|l| l.context as u64)
|
||||
}
|
||||
|
||||
/// Conservative prompt-token estimate (~4 chars/token over message text).
|
||||
/// cortex has no tokenizer; under-counting is the safe direction — we only
|
||||
/// pre-reject gross overages (#56), and neuron enforces the exact wall.
|
||||
fn estimate_prompt_tokens(body: &[u8]) -> u64 {
|
||||
let Ok(v) = serde_json::from_slice::<Value>(body) else {
|
||||
return (body.len() as u64 / 4).max(1);
|
||||
};
|
||||
let mut chars = 0usize;
|
||||
if let Some(messages) = v.get("messages").and_then(Value::as_array) {
|
||||
for m in messages {
|
||||
match m.get("content") {
|
||||
Some(Value::String(s)) => chars += s.len(),
|
||||
Some(Value::Array(parts)) => {
|
||||
for p in parts {
|
||||
if let Some(t) = p.get("text").and_then(Value::as_str) {
|
||||
chars += t.len();
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
chars += 8; // rough per-message role/formatting overhead
|
||||
}
|
||||
} else if let Some(prompt) = v.get("prompt").and_then(Value::as_str) {
|
||||
chars += prompt.len(); // legacy /v1/completions
|
||||
} else {
|
||||
return (body.len() as u64 / 4).max(1);
|
||||
}
|
||||
(chars as u64 / 4).max(1)
|
||||
}
|
||||
|
||||
/// Client-specific, advisory guidance for an over-long prompt (#56),
|
||||
/// fingerprinted from `User-Agent`. Strictly advisory: it rides the
|
||||
/// `X-Helexa-Advice` header only, never the error envelope, and behaviour
|
||||
/// never depends on it. Unknown clients get nothing.
|
||||
fn client_advice(headers: &HeaderMap) -> Option<&'static str> {
|
||||
let ua = headers
|
||||
.get(axum::http::header::USER_AGENT)?
|
||||
.to_str()
|
||||
.ok()?
|
||||
.to_ascii_lowercase();
|
||||
if ua.contains("litellm") {
|
||||
Some(
|
||||
"litellm forwards the full context; lower the configured context window or enable client-side compaction",
|
||||
)
|
||||
} else if ua.contains("agent-zero") || ua.contains("agent zero") {
|
||||
Some("reduce the conversation/context size or summarize earlier turns before resending")
|
||||
} else if ua.contains("zed") {
|
||||
Some("reduce the assistant context window in Zed's settings")
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// `400 context_length_exceeded` for an over-long prompt caught at the edge
|
||||
/// (#56), in the #60 envelope — the same shape neuron emits on overflow, so
|
||||
/// clients (opencode auto-compacts) handle it identically. Attaches the
|
||||
/// advisory `X-Helexa-Advice` header for fingerprinted clients.
|
||||
fn context_length_exceeded_response(
|
||||
context: u64,
|
||||
prompt_est: u64,
|
||||
headers: &HeaderMap,
|
||||
) -> Response {
|
||||
let env = OpenAiError::context_length_exceeded(format!(
|
||||
"This model's maximum context length is {context} tokens. Your request is \
|
||||
estimated at ~{prompt_est} tokens. Please reduce the length of the messages."
|
||||
))
|
||||
.with_extra("max", json!(context))
|
||||
.with_extra("estimated_prompt_tokens", json!(prompt_est));
|
||||
let mut response = crate::error::envelope_response(env);
|
||||
if let Some(advice) = client_advice(headers)
|
||||
&& let Ok(value) = axum::http::HeaderValue::from_str(advice)
|
||||
{
|
||||
response.headers_mut().insert("x-helexa-advice", value);
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
/// Update `last_accessed` timestamp for a model on a node (drives LRU eviction).
|
||||
async fn touch_model(fleet: &CortexState, node_name: &str, model_id: &str) {
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
pub mod anthropic_sse;
|
||||
pub mod auth;
|
||||
pub mod entitlements_chain;
|
||||
pub mod entitlements_local;
|
||||
pub mod entitlements_upstream;
|
||||
pub mod error;
|
||||
pub mod evictor;
|
||||
pub mod handlers;
|
||||
pub mod metering;
|
||||
pub mod metrics;
|
||||
pub mod poller;
|
||||
pub mod proxy;
|
||||
@@ -10,15 +15,26 @@ pub mod state;
|
||||
|
||||
use anyhow::Result;
|
||||
use axum::Router;
|
||||
use axum::middleware::from_fn_with_state;
|
||||
use cortex_core::config::GatewayConfig;
|
||||
use std::sync::Arc;
|
||||
use tower_http::cors::CorsLayer;
|
||||
use tower_http::trace::TraceLayer;
|
||||
|
||||
/// Build the Axum application router with all routes wired up.
|
||||
///
|
||||
/// Layer order (outermost first): trace → CORS → auth → handlers. CORS is
|
||||
/// outer to auth so preflight `OPTIONS` short-circuits before resolution;
|
||||
/// auth (`require_principal`) resolves the bearer key, attaches the
|
||||
/// principal, and stamps the internal principal headers before any handler
|
||||
/// runs.
|
||||
pub fn build_app(fleet: Arc<state::CortexState>) -> Router {
|
||||
Router::new()
|
||||
.merge(handlers::api_routes())
|
||||
.layer(from_fn_with_state(
|
||||
Arc::clone(&fleet),
|
||||
auth::require_principal,
|
||||
))
|
||||
.layer(CorsLayer::permissive())
|
||||
.layer(TraceLayer::new_for_http())
|
||||
.with_state(fleet)
|
||||
|
||||
219
crates/cortex-gateway/src/metering.rs
Normal file
@@ -0,0 +1,219 @@
|
||||
//! Per-request token metering (#51).
|
||||
//!
|
||||
//! Captures the real `(prompt, completion)` usage of every request and feeds
|
||||
//! it to two places: the [`EntitlementProvider`] spend ledger (via
|
||||
//! reserve→settle) and per-principal Prometheus counters. The principal is
|
||||
//! reconstructed from the internal headers the auth middleware stamped (#49),
|
||||
//! so this works uniformly across every proxy path without threading the
|
||||
//! typed principal through each handler.
|
||||
//!
|
||||
//! The reserve→settle lifecycle is established here but, in this phase,
|
||||
//! reserves **zero** tokens — metering only, no enforcement. Budget
|
||||
//! enforcement (#52) flips the reserved amount to the real
|
||||
//! `prompt + max_output` and handles the [`BudgetError`] rejection; the
|
||||
//! settle/release plumbing is identical, so that change is localized.
|
||||
//!
|
||||
//! [`ReservationGuard`] makes leaks impossible: settling records actual
|
||||
//! spend and releases the unused remainder; dropping a guard that was never
|
||||
//! settled releases the whole reservation. So an early return, error path,
|
||||
//! or dropped stream can't strand a reservation.
|
||||
|
||||
use axum::http::HeaderMap;
|
||||
use cortex_core::entitlements::{
|
||||
BudgetError, EntitlementProvider, HEADER_ACCOUNT_ID, HEADER_KEY_ID, Principal,
|
||||
};
|
||||
use cortex_core::error_envelope::OpenAiError;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Fallback output-token budget when neither the request nor the model's
|
||||
/// advertised limit gives one. Bounds the reservation so a capped key is
|
||||
/// still gated even on under-specified requests (#52).
|
||||
pub const FALLBACK_MAX_OUTPUT: u64 = 4096;
|
||||
|
||||
/// Invoked exactly once at request completion with best-effort
|
||||
/// `(prompt_tokens, completion_tokens)`. When no usage could be observed
|
||||
/// (e.g. a pre-dispatch failure or a dropped stream) it is dropped unused —
|
||||
/// which releases the held reservation via [`ReservationGuard`]'s `Drop`.
|
||||
pub type UsageSink = Box<dyn FnOnce(u64, u64) + Send>;
|
||||
|
||||
/// Reconstruct the principal from the cortex-stamped internal headers. The
|
||||
/// auth middleware strips any client copy and stamps the authoritative value,
|
||||
/// so these headers are trustworthy within cortex. `None` for anonymous
|
||||
/// (unauthenticated) requests.
|
||||
pub fn principal_from_headers(headers: &HeaderMap) -> Option<Principal> {
|
||||
let account_id = headers.get(HEADER_ACCOUNT_ID)?.to_str().ok()?.to_string();
|
||||
let key_id = headers.get(HEADER_KEY_ID)?.to_str().ok()?.to_string();
|
||||
Some(Principal { account_id, key_id })
|
||||
}
|
||||
|
||||
/// Emit per-principal spend counters (#51). Labelled by account/key only —
|
||||
/// both are operator-bounded, so cardinality is controlled.
|
||||
pub fn record_spend(principal: &Principal, prompt: u64, completion: u64) {
|
||||
let labels = [
|
||||
("account", principal.account_id.clone()),
|
||||
("key", principal.key_id.clone()),
|
||||
];
|
||||
metrics::counter!("cortex_spend_tokens_total", &labels).increment(prompt + completion);
|
||||
metrics::counter!("cortex_spend_prompt_tokens_total", &labels).increment(prompt);
|
||||
metrics::counter!("cortex_spend_completion_tokens_total", &labels).increment(completion);
|
||||
}
|
||||
|
||||
/// Holds a budget reservation for the life of a request. [`settle`] records
|
||||
/// actual spend and releases the remainder; an un-settled guard releases the
|
||||
/// whole reservation when dropped. Anonymous requests carry an empty guard,
|
||||
/// where every operation is a no-op.
|
||||
///
|
||||
/// [`settle`]: ReservationGuard::settle
|
||||
pub struct ReservationGuard {
|
||||
provider: Arc<dyn EntitlementProvider>,
|
||||
reservation: Option<cortex_core::entitlements::Reservation>,
|
||||
}
|
||||
|
||||
impl ReservationGuard {
|
||||
/// An empty guard for an anonymous request — no reservation to resolve.
|
||||
pub fn anonymous(provider: Arc<dyn EntitlementProvider>) -> Self {
|
||||
Self {
|
||||
provider,
|
||||
reservation: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Wrap an already-acquired reservation.
|
||||
fn held(
|
||||
provider: Arc<dyn EntitlementProvider>,
|
||||
reservation: cortex_core::entitlements::Reservation,
|
||||
) -> Self {
|
||||
Self {
|
||||
provider,
|
||||
reservation: Some(reservation),
|
||||
}
|
||||
}
|
||||
|
||||
/// Settle with the tokens actually consumed, disarming the drop-release.
|
||||
/// Spawns the (fast, in-process for the local provider) settle so the
|
||||
/// caller — which may be a sync stream-completion callback — needn't
|
||||
/// await.
|
||||
pub fn settle(mut self, actual_tokens: u64) {
|
||||
if let Some(reservation) = self.reservation.take() {
|
||||
let provider = Arc::clone(&self.provider);
|
||||
tokio::spawn(async move {
|
||||
provider.settle(reservation, actual_tokens).await;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ReservationGuard {
|
||||
fn drop(&mut self) {
|
||||
if let Some(reservation) = self.reservation.take() {
|
||||
let provider = Arc::clone(&self.provider);
|
||||
tokio::spawn(async move {
|
||||
provider.release(reservation).await;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the completion sink for an authenticated request: record spend and
|
||||
/// settle the reservation with the observed total. Dropping it unused (no
|
||||
/// usage observed) releases the reservation via the guard.
|
||||
pub fn usage_sink(principal: Principal, guard: ReservationGuard) -> UsageSink {
|
||||
Box::new(move |prompt, completion| {
|
||||
record_spend(&principal, prompt, completion);
|
||||
guard.settle(prompt + completion);
|
||||
})
|
||||
}
|
||||
|
||||
/// Reserve the request's upper-bound token cost for the principal, refusing
|
||||
/// *before* dispatch if it would exceed the hard cap (#52). On success
|
||||
/// returns a guard the caller settles with actual usage; on refusal returns
|
||||
/// the #63 envelope (`rate_limit_exceeded` + `Retry-After` for a resetting
|
||||
/// window, `insufficient_quota` for a hard balance — never `402`).
|
||||
pub async fn reserve_or_reject(
|
||||
provider: Arc<dyn EntitlementProvider>,
|
||||
principal: &Principal,
|
||||
max_tokens: u64,
|
||||
) -> Result<ReservationGuard, OpenAiError> {
|
||||
match provider.reserve(principal, max_tokens).await {
|
||||
Ok(reservation) => Ok(ReservationGuard::held(provider, reservation)),
|
||||
Err(err) => Err(budget_error_to_envelope(err)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Map a [`BudgetError`] to the #63 envelope. The provider chose the window
|
||||
/// semantics; this only translates them to HTTP.
|
||||
fn budget_error_to_envelope(err: BudgetError) -> OpenAiError {
|
||||
match err {
|
||||
BudgetError::RateLimited {
|
||||
retry_after_secs, ..
|
||||
} => OpenAiError::rate_limit_exceeded(err.to_string(), retry_after_secs),
|
||||
BudgetError::InsufficientQuota { .. } => OpenAiError::insufficient_quota(err.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Upper-bound tokens to reserve for a request (#52): an over-estimate of
|
||||
/// the prompt plus the maximum output. `advertised_output` is the model's
|
||||
/// `limit.output` (#62), used when the request omits `max_(completion_)tokens`.
|
||||
/// Over-reserving is safe — settle corrects spend to the actual usage.
|
||||
pub fn reservation_estimate(body: &[u8], advertised_output: Option<u64>) -> u64 {
|
||||
let max_output = requested_max_output(body)
|
||||
.or(advertised_output)
|
||||
.unwrap_or(FALLBACK_MAX_OUTPUT);
|
||||
estimate_prompt_tokens(body).saturating_add(max_output)
|
||||
}
|
||||
|
||||
/// The client's requested output cap, from `max_completion_tokens` (or the
|
||||
/// legacy `max_tokens`). `None` when unspecified.
|
||||
fn requested_max_output(body: &[u8]) -> Option<u64> {
|
||||
let v: serde_json::Value = serde_json::from_slice(body).ok()?;
|
||||
v.get("max_completion_tokens")
|
||||
.or_else(|| v.get("max_tokens"))
|
||||
.and_then(serde_json::Value::as_u64)
|
||||
}
|
||||
|
||||
/// Rough prompt-token estimate at ~4 chars/token over the whole body. cortex
|
||||
/// has no tokenizer; JSON overhead makes this a conservative over-estimate,
|
||||
/// and neuron remains the exact context wall (#56/#60). Settle reconciles to
|
||||
/// the real usage afterward.
|
||||
fn estimate_prompt_tokens(body: &[u8]) -> u64 {
|
||||
(body.len() as u64 / 4).max(1)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn requested_max_output_prefers_max_completion_tokens() {
|
||||
let body = br#"{"model":"m","max_completion_tokens":256,"max_tokens":99}"#;
|
||||
assert_eq!(requested_max_output(body), Some(256));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn requested_max_output_falls_back_to_legacy_max_tokens() {
|
||||
let body = br#"{"model":"m","max_tokens":128}"#;
|
||||
assert_eq!(requested_max_output(body), Some(128));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn estimate_uses_requested_output_when_present() {
|
||||
// Requested output dominates; prompt estimate is small for a tiny body.
|
||||
let body = br#"{"model":"m","max_tokens":1000}"#;
|
||||
let est = reservation_estimate(body, Some(8192));
|
||||
assert!(est >= 1000 && est < 1100, "est was {est}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn estimate_uses_advertised_output_when_request_omits_it() {
|
||||
let body = br#"{"model":"m","messages":[]}"#;
|
||||
let est = reservation_estimate(body, Some(8192));
|
||||
assert!(est >= 8192, "est was {est}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn estimate_falls_back_when_nothing_advertised() {
|
||||
let body = br#"{"model":"m"}"#;
|
||||
let est = reservation_estimate(body, None);
|
||||
assert!(est >= FALLBACK_MAX_OUTPUT, "est was {est}");
|
||||
}
|
||||
}
|
||||
@@ -63,4 +63,16 @@ fn describe_metrics() {
|
||||
"cortex_cold_starts_total",
|
||||
"Total number of cold-start model loads"
|
||||
);
|
||||
metrics::describe_counter!(
|
||||
"cortex_spend_tokens_total",
|
||||
"Total metered tokens (prompt + completion) per principal, labelled by account/key (#51)"
|
||||
);
|
||||
metrics::describe_counter!(
|
||||
"cortex_spend_prompt_tokens_total",
|
||||
"Metered prompt tokens per principal, labelled by account/key (#51)"
|
||||
);
|
||||
metrics::describe_counter!(
|
||||
"cortex_spend_completion_tokens_total",
|
||||
"Metered completion tokens per principal, labelled by account/key (#51)"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -5,12 +5,29 @@ use crate::state::CortexState;
|
||||
use chrono::Utc;
|
||||
use cortex_core::discovery::{DiscoveryResponse, HealthResponse};
|
||||
use cortex_core::harness::ModelInfo;
|
||||
use cortex_core::node::{ModelEntry, ModelStatus};
|
||||
use cortex_core::node::{ModelEntry, ModelStatus, NodeState};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
const POLL_INTERVAL: Duration = Duration::from_secs(10);
|
||||
|
||||
/// Consecutive failed `/models` polls before a node is marked unhealthy.
|
||||
/// Debounces transient misses (a busy neuron briefly slow to answer) so a
|
||||
/// single blip can't yank a node — and its models — out of routing. At the
|
||||
/// 10s poll interval this tolerates ~20s of flapping before evicting.
|
||||
const POLL_FAILURE_THRESHOLD: u32 = 3;
|
||||
|
||||
/// Record a failed poll for `node`, marking it unhealthy only once failures
|
||||
/// reach [`POLL_FAILURE_THRESHOLD`]. Below the threshold the node keeps its
|
||||
/// last-known health, riding over transient misses. A successful poll resets
|
||||
/// the counter (see the success arm in `poll_once`).
|
||||
fn record_poll_failure(node: &mut NodeState) {
|
||||
node.consecutive_poll_failures = node.consecutive_poll_failures.saturating_add(1);
|
||||
if node.consecutive_poll_failures >= POLL_FAILURE_THRESHOLD {
|
||||
node.healthy = false;
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs forever, polling all neurons on a fixed interval.
|
||||
pub async fn poll_loop(fleet: Arc<CortexState>) {
|
||||
loop {
|
||||
@@ -138,13 +155,14 @@ async fn poll_neuron(fleet: &CortexState, name: &str, endpoint: &str) {
|
||||
// Remove models no longer reported by the neuron.
|
||||
node.models.retain(|id, _| seen.contains(id));
|
||||
|
||||
node.consecutive_poll_failures = 0;
|
||||
node.healthy = true;
|
||||
node.last_poll = Some(Utc::now());
|
||||
tracing::debug!(node = name, models = models.len(), "poll ok");
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(node = name, error = %e, "failed to parse /models response");
|
||||
node.healthy = false;
|
||||
record_poll_failure(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -154,11 +172,11 @@ async fn poll_neuron(fleet: &CortexState, name: &str, endpoint: &str) {
|
||||
status = %resp.status(),
|
||||
"neuron returned non-success status"
|
||||
);
|
||||
node.healthy = false;
|
||||
record_poll_failure(node);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(node = name, error = %e, "failed to reach neuron");
|
||||
node.healthy = false;
|
||||
record_poll_failure(node);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -200,6 +218,9 @@ async fn poll_health(fleet: &CortexState, name: &str, endpoint: &str) {
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
if let Some(node) = nodes.get_mut(name) {
|
||||
node.activation = Some(h.activation);
|
||||
// Per-model admission load (#53) → keyed by id for the
|
||||
// load-aware router (#55).
|
||||
node.model_load = h.models.into_iter().map(|m| (m.id.clone(), m)).collect();
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
|
||||
@@ -1,21 +1,27 @@
|
||||
//! Streaming HTTP reverse proxy to neuron backends.
|
||||
//!
|
||||
//! For streaming requests, SSE chunks are forwarded as they arrive.
|
||||
//! The proxy captures timing information for metrics but does not
|
||||
//! buffer the full response.
|
||||
//! The streaming *mechanism* — forward an SSE body chunk-for-chunk without
|
||||
//! buffering, observing the bytes for metrics — lives in the shared
|
||||
//! [`helexa_stream`] crate (#71), so cortex and helexa-router use one
|
||||
//! implementation. This module supplies cortex's *policy*: the
|
||||
//! [`CortexMetrics`] observer (per-request token metrics + per-principal
|
||||
//! reservation settle), cortex's logging contract, and the cortex error
|
||||
//! envelope. The usage-extraction helper is re-exported from the shared
|
||||
//! crate so existing call sites keep working.
|
||||
|
||||
use crate::router::RouteDecision;
|
||||
use anyhow::Result;
|
||||
use axum::body::Body;
|
||||
use axum::http::{HeaderMap, StatusCode};
|
||||
use axum::http::HeaderMap;
|
||||
use axum::http::StatusCode;
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use futures::Stream;
|
||||
use futures::stream::BoxStream;
|
||||
use helexa_stream::{BodyTail, ChunkObserver, StreamError};
|
||||
use reqwest::Client;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::Instant;
|
||||
|
||||
/// Re-export the shared usage-extraction helper. Several cortex modules
|
||||
/// (`handlers`, `anthropic_sse`) pull token counts out of a buffered body
|
||||
/// tail via this function; it lives in `helexa-stream` now.
|
||||
pub use helexa_stream::last_count_for;
|
||||
|
||||
/// Proxy a request body to the resolved backend node and stream the response.
|
||||
///
|
||||
/// Logging contract: every call emits exactly one structured event at
|
||||
@@ -31,6 +37,7 @@ pub async fn forward_request(
|
||||
headers: HeaderMap,
|
||||
body: bytes::Bytes,
|
||||
model_id: &str,
|
||||
usage_sink: Option<crate::metering::UsageSink>,
|
||||
) -> Result<Response, ProxyError> {
|
||||
let request_start = Instant::now();
|
||||
let url = format!("{}{}", route.endpoint, path);
|
||||
@@ -41,66 +48,41 @@ pub async fn forward_request(
|
||||
"proxying request"
|
||||
);
|
||||
|
||||
let mut req_builder = client.post(&url).body(body);
|
||||
let observer = CortexMetrics::new(model_id, &route.node_name, request_start, usage_sink);
|
||||
|
||||
// Forward relevant headers.
|
||||
for (key, value) in headers.iter() {
|
||||
if key == "host" || key == "content-length" {
|
||||
continue; // reqwest sets these
|
||||
}
|
||||
req_builder = req_builder.header(key, value);
|
||||
}
|
||||
let response = helexa_stream::forward_streaming(client, &url, headers, body, observer)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
match &e {
|
||||
StreamError::Upstream(err) => tracing::warn!(
|
||||
node = %route.node_name,
|
||||
url = %url,
|
||||
error = %err,
|
||||
"proxy: upstream request failed (network)"
|
||||
),
|
||||
StreamError::ResponseBuild(err) => tracing::warn!(
|
||||
node = %route.node_name,
|
||||
url = %url,
|
||||
error = %err,
|
||||
"proxy: failed to build response"
|
||||
),
|
||||
}
|
||||
ProxyError::from(e)
|
||||
})?;
|
||||
|
||||
let upstream_resp = match req_builder.send().await {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
node = %route.node_name,
|
||||
url = %url,
|
||||
error = %e,
|
||||
"proxy: upstream request failed (network)"
|
||||
);
|
||||
return Err(ProxyError::Upstream(e));
|
||||
}
|
||||
};
|
||||
|
||||
let upstream_status = upstream_resp.status();
|
||||
if !upstream_status.is_success() {
|
||||
if !response.status().is_success() {
|
||||
// Streaming body — can't snippet without breaking the stream
|
||||
// pass-through. Log status + URL; the client still gets the
|
||||
// upstream status, just without the leaked body.
|
||||
tracing::warn!(
|
||||
node = %route.node_name,
|
||||
url = %url,
|
||||
status = upstream_status.as_u16(),
|
||||
status = response.status().as_u16(),
|
||||
"proxy: upstream returned non-2xx"
|
||||
);
|
||||
}
|
||||
|
||||
let status = StatusCode::from_u16(upstream_status.as_u16()).unwrap_or(StatusCode::BAD_GATEWAY);
|
||||
|
||||
let resp_headers = upstream_resp.headers().clone();
|
||||
let stream = TokenMetricsStream::new(
|
||||
Box::pin(upstream_resp.bytes_stream()),
|
||||
TokenMetrics::new(model_id, &route.node_name, request_start),
|
||||
);
|
||||
|
||||
let body = Body::from_stream(stream);
|
||||
|
||||
let mut response = Response::builder().status(status);
|
||||
for (key, value) in resp_headers.iter() {
|
||||
response = response.header(key, value);
|
||||
}
|
||||
|
||||
response.body(body).map_err(|e| {
|
||||
tracing::warn!(
|
||||
node = %route.node_name,
|
||||
url = %url,
|
||||
error = %e,
|
||||
"proxy: failed to build response"
|
||||
);
|
||||
ProxyError::ResponseBuild(e.to_string())
|
||||
})
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
@@ -111,6 +93,15 @@ pub enum ProxyError {
|
||||
ResponseBuild(String),
|
||||
}
|
||||
|
||||
impl From<StreamError> for ProxyError {
|
||||
fn from(e: StreamError) -> Self {
|
||||
match e {
|
||||
StreamError::Upstream(err) => ProxyError::Upstream(err),
|
||||
StreamError::ResponseBuild(msg) => ProxyError::ResponseBuild(msg),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl IntoResponse for ProxyError {
|
||||
fn into_response(self) -> Response {
|
||||
let (status, code, message) = match &self {
|
||||
@@ -138,9 +129,10 @@ impl IntoResponse for ProxyError {
|
||||
//
|
||||
// The proxy never buffers or re-serialises the upstream body — chunks
|
||||
// are forwarded verbatim. For metrics it observes each chunk's arrival
|
||||
// time and keeps a bounded tail of the body text, from which the final
|
||||
// OpenAI `usage` object (present on the last SSE chunk and on
|
||||
// non-streaming JSON bodies alike) yields engine-truth token counts.
|
||||
// time and keeps a bounded tail of the body text (via the shared
|
||||
// `helexa_stream::BodyTail`), from which the final OpenAI `usage` object
|
||||
// (present on the last SSE chunk and on non-streaming JSON bodies alike)
|
||||
// yields engine-truth token counts.
|
||||
//
|
||||
// Emitted per request, labelled {model, node}:
|
||||
// cortex_time_to_first_token_seconds (histogram) — first body chunk
|
||||
@@ -154,42 +146,29 @@ impl IntoResponse for ProxyError {
|
||||
/// non-streaming bodies.
|
||||
const TAIL_CAP_BYTES: usize = 64 * 1024;
|
||||
|
||||
/// Find the value of the LAST `"key": <integer>` occurrence in `tail`.
|
||||
/// Pure and chunk-boundary-safe (the tail is contiguous appended text).
|
||||
/// The quoted-needle form means `completion_tokens` never matches
|
||||
/// `completion_tokens_details`.
|
||||
pub(crate) fn last_count_for(tail: &str, key: &str) -> Option<u64> {
|
||||
let needle = format!("\"{key}\"");
|
||||
let mut result = None;
|
||||
for (idx, _) in tail.match_indices(&needle) {
|
||||
let rest = tail[idx + needle.len()..].trim_start();
|
||||
let Some(rest) = rest.strip_prefix(':') else {
|
||||
continue;
|
||||
};
|
||||
let rest = rest.trim_start();
|
||||
let digits: &str = &rest[..rest
|
||||
.char_indices()
|
||||
.find(|(_, c)| !c.is_ascii_digit())
|
||||
.map(|(i, _)| i)
|
||||
.unwrap_or(rest.len())];
|
||||
if let Ok(v) = digits.parse::<u64>() {
|
||||
result = Some(v);
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
struct TokenMetrics {
|
||||
/// cortex's [`ChunkObserver`]: per-request token metrics plus the
|
||||
/// per-principal reservation settle. Drives cortex policy over the shared
|
||||
/// streaming mechanism.
|
||||
struct CortexMetrics {
|
||||
labels: [(&'static str, String); 2],
|
||||
request_start: Instant,
|
||||
first_chunk: Option<Instant>,
|
||||
last_chunk: Option<Instant>,
|
||||
tail: String,
|
||||
tail: BodyTail,
|
||||
finished: bool,
|
||||
/// Per-principal metering hook (#51). Invoked exactly once in `finish`
|
||||
/// with the observed `(prompt, completion)` so the reservation can be
|
||||
/// settled and spend recorded. `None` for anonymous requests.
|
||||
usage_sink: Option<crate::metering::UsageSink>,
|
||||
}
|
||||
|
||||
impl TokenMetrics {
|
||||
fn new(model_id: &str, node_name: &str, request_start: Instant) -> Self {
|
||||
impl CortexMetrics {
|
||||
fn new(
|
||||
model_id: &str,
|
||||
node_name: &str,
|
||||
request_start: Instant,
|
||||
usage_sink: Option<crate::metering::UsageSink>,
|
||||
) -> Self {
|
||||
Self {
|
||||
labels: [
|
||||
("model", model_id.to_string()),
|
||||
@@ -198,25 +177,19 @@ impl TokenMetrics {
|
||||
request_start,
|
||||
first_chunk: None,
|
||||
last_chunk: None,
|
||||
tail: String::new(),
|
||||
tail: BodyTail::new(TAIL_CAP_BYTES),
|
||||
finished: false,
|
||||
usage_sink,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ChunkObserver for CortexMetrics {
|
||||
fn observe(&mut self, chunk: &[u8]) {
|
||||
let now = Instant::now();
|
||||
self.first_chunk.get_or_insert(now);
|
||||
self.last_chunk = Some(now);
|
||||
self.tail.push_str(&String::from_utf8_lossy(chunk));
|
||||
if self.tail.len() > TAIL_CAP_BYTES {
|
||||
// Keep the newest half; the usage object is always at the
|
||||
// very end of the body. Split at a char boundary.
|
||||
let mut cut = self.tail.len() - TAIL_CAP_BYTES / 2;
|
||||
while !self.tail.is_char_boundary(cut) {
|
||||
cut += 1;
|
||||
}
|
||||
self.tail.drain(..cut);
|
||||
}
|
||||
self.tail.push(chunk);
|
||||
}
|
||||
|
||||
/// Emit the metrics exactly once — called on clean stream end and
|
||||
@@ -227,130 +200,45 @@ impl TokenMetrics {
|
||||
return;
|
||||
}
|
||||
self.finished = true;
|
||||
let Some(first) = self.first_chunk else {
|
||||
return; // no body ever arrived — nothing to record
|
||||
};
|
||||
let ttft = first.duration_since(self.request_start).as_secs_f64();
|
||||
metrics::histogram!("cortex_time_to_first_token_seconds", &self.labels).record(ttft);
|
||||
|
||||
if let Some(prompt) = last_count_for(&self.tail, "prompt_tokens") {
|
||||
metrics::counter!("cortex_prompt_tokens_total", &self.labels).increment(prompt);
|
||||
}
|
||||
let Some(completion) = last_count_for(&self.tail, "completion_tokens") else {
|
||||
return;
|
||||
};
|
||||
if completion == 0 {
|
||||
return;
|
||||
}
|
||||
metrics::counter!("cortex_completion_tokens_total", &self.labels).increment(completion);
|
||||
let prompt = last_count_for(self.tail.as_str(), "prompt_tokens");
|
||||
let completion = last_count_for(self.tail.as_str(), "completion_tokens");
|
||||
|
||||
let last = self.last_chunk.unwrap_or(first);
|
||||
let decode_window = last.duration_since(first).as_secs_f64();
|
||||
// Streaming: rate over the decode window (first→last chunk).
|
||||
// Non-streaming bodies arrive as ~one chunk (window ≈ 0), where
|
||||
// the only honest denominator is the full request duration.
|
||||
let secs = if decode_window >= 0.1 {
|
||||
decode_window
|
||||
} else {
|
||||
last.duration_since(self.request_start).as_secs_f64()
|
||||
};
|
||||
if secs > 0.0 {
|
||||
metrics::histogram!("cortex_tokens_per_second", &self.labels)
|
||||
.record(completion as f64 / secs);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Per-model metrics — only when body chunks actually arrived.
|
||||
if let Some(first) = self.first_chunk {
|
||||
let ttft = first.duration_since(self.request_start).as_secs_f64();
|
||||
metrics::histogram!("cortex_time_to_first_token_seconds", &self.labels).record(ttft);
|
||||
|
||||
/// Pass-through stream wrapper that feeds [`TokenMetrics`]. Emits on
|
||||
/// clean end-of-stream; the Drop impl covers client disconnects.
|
||||
struct TokenMetricsStream {
|
||||
inner: BoxStream<'static, Result<bytes::Bytes, reqwest::Error>>,
|
||||
metrics: TokenMetrics,
|
||||
}
|
||||
|
||||
impl TokenMetricsStream {
|
||||
fn new(
|
||||
inner: BoxStream<'static, Result<bytes::Bytes, reqwest::Error>>,
|
||||
metrics: TokenMetrics,
|
||||
) -> Self {
|
||||
Self { inner, metrics }
|
||||
}
|
||||
}
|
||||
|
||||
impl Stream for TokenMetricsStream {
|
||||
type Item = Result<bytes::Bytes, reqwest::Error>;
|
||||
|
||||
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
let this = self.get_mut();
|
||||
match this.inner.as_mut().poll_next(cx) {
|
||||
Poll::Ready(Some(Ok(chunk))) => {
|
||||
this.metrics.observe(&chunk);
|
||||
Poll::Ready(Some(Ok(chunk)))
|
||||
if let Some(prompt) = prompt {
|
||||
metrics::counter!("cortex_prompt_tokens_total", &self.labels).increment(prompt);
|
||||
}
|
||||
Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))),
|
||||
Poll::Ready(None) => {
|
||||
this.metrics.finish();
|
||||
Poll::Ready(None)
|
||||
if let Some(completion) = completion.filter(|c| *c > 0) {
|
||||
metrics::counter!("cortex_completion_tokens_total", &self.labels)
|
||||
.increment(completion);
|
||||
|
||||
let last = self.last_chunk.unwrap_or(first);
|
||||
let decode_window = last.duration_since(first).as_secs_f64();
|
||||
// Streaming: rate over the decode window (first→last chunk).
|
||||
// Non-streaming bodies arrive as ~one chunk (window ≈ 0),
|
||||
// where the only honest denominator is the full request
|
||||
// duration.
|
||||
let secs = if decode_window >= 0.1 {
|
||||
decode_window
|
||||
} else {
|
||||
last.duration_since(self.request_start).as_secs_f64()
|
||||
};
|
||||
if secs > 0.0 {
|
||||
metrics::histogram!("cortex_tokens_per_second", &self.labels)
|
||||
.record(completion as f64 / secs);
|
||||
}
|
||||
}
|
||||
Poll::Pending => Poll::Pending,
|
||||
}
|
||||
|
||||
// Per-principal metering + reservation settle (#51). Always runs so
|
||||
// the reservation is resolved even when no usage/body was observed
|
||||
// (sink with (0, 0) → settle 0 → release).
|
||||
if let Some(sink) = self.usage_sink.take() {
|
||||
sink(prompt.unwrap_or(0), completion.unwrap_or(0));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TokenMetricsStream {
|
||||
fn drop(&mut self) {
|
||||
self.metrics.finish();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::last_count_for;
|
||||
|
||||
#[test]
|
||||
fn extracts_counts_from_final_sse_usage_chunk() {
|
||||
let tail = concat!(
|
||||
"data: {\"choices\":[{\"delta\":{\"content\":\"hi\"}}]}\n\n",
|
||||
"data: {\"choices\":[],\"usage\":{\"prompt_tokens\":225,",
|
||||
"\"completion_tokens\":42,\"total_tokens\":267}}\n\n",
|
||||
"data: [DONE]\n\n"
|
||||
);
|
||||
assert_eq!(last_count_for(tail, "prompt_tokens"), Some(225));
|
||||
assert_eq!(last_count_for(tail, "completion_tokens"), Some(42));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_counts_from_non_streaming_body() {
|
||||
let tail = "{\"choices\":[{\"message\":{\"content\":\"hi\"}}],\
|
||||
\"usage\":{\"prompt_tokens\": 12, \"completion_tokens\": 7}}";
|
||||
assert_eq!(last_count_for(tail, "prompt_tokens"), Some(12));
|
||||
assert_eq!(last_count_for(tail, "completion_tokens"), Some(7));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ignores_details_variants_and_takes_last_occurrence() {
|
||||
// completion_tokens_details must not shadow completion_tokens,
|
||||
// and the LAST usage object wins (matters when content echoes
|
||||
// a usage-shaped string earlier in the stream).
|
||||
let tail = concat!(
|
||||
"data: {\"usage\":{\"completion_tokens\":1}}\n\n",
|
||||
"data: {\"usage\":{\"completion_tokens\":99,",
|
||||
"\"completion_tokens_details\":{\"reasoning_tokens\":3}}}\n\n"
|
||||
);
|
||||
assert_eq!(last_count_for(tail, "completion_tokens"), Some(99));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_keys_yield_none() {
|
||||
assert_eq!(
|
||||
last_count_for("data: [DONE]\n\n", "completion_tokens"),
|
||||
None
|
||||
);
|
||||
assert_eq!(last_count_for("", "prompt_tokens"), None);
|
||||
// key present but non-numeric value
|
||||
assert_eq!(
|
||||
last_count_for("\"completion_tokens\": null", "completion_tokens"),
|
||||
None
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,6 +50,10 @@ pub enum RouteError {
|
||||
"model '{model_id}' is in the catalogue but no healthy neuron's topology satisfies its constraints"
|
||||
)]
|
||||
NoFeasibleNeuron { model_id: String },
|
||||
#[error(
|
||||
"model '{model_id}' is feasible on a neuron that is currently unhealthy — retry shortly"
|
||||
)]
|
||||
FeasibleNodeUnhealthy { model_id: String },
|
||||
#[error("cold-load of '{model_id}' on '{node}' failed: {message}")]
|
||||
ColdLoadFailed {
|
||||
model_id: String,
|
||||
@@ -68,7 +72,9 @@ impl RouteError {
|
||||
/// safe to retry the same request); everything else is 404.
|
||||
pub fn http_status(&self) -> u16 {
|
||||
match self {
|
||||
RouteError::NoHealthyNodes | RouteError::ModelRecovering { .. } => 503,
|
||||
RouteError::NoHealthyNodes
|
||||
| RouteError::ModelRecovering { .. }
|
||||
| RouteError::FeasibleNodeUnhealthy { .. } => 503,
|
||||
_ => 404,
|
||||
}
|
||||
}
|
||||
@@ -81,7 +87,8 @@ impl RouteError {
|
||||
| RouteError::EndpointResolveFailed(_, _)
|
||||
| RouteError::NoFeasibleNeuron { .. }
|
||||
| RouteError::ColdLoadFailed { .. }
|
||||
| RouteError::ModelRecovering { .. } => "api_error",
|
||||
| RouteError::ModelRecovering { .. }
|
||||
| RouteError::FeasibleNodeUnhealthy { .. } => "api_error",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -94,6 +101,7 @@ impl RouteError {
|
||||
RouteError::NoFeasibleNeuron { .. } => "service_unavailable",
|
||||
RouteError::ColdLoadFailed { .. } => "service_unavailable",
|
||||
RouteError::ModelRecovering { .. } => "service_unavailable",
|
||||
RouteError::FeasibleNodeUnhealthy { .. } => "service_unavailable",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -105,6 +113,7 @@ impl RouteError {
|
||||
pub fn retry_after_secs(&self) -> Option<u64> {
|
||||
match self {
|
||||
RouteError::ModelRecovering { .. } => Some(2),
|
||||
RouteError::FeasibleNodeUnhealthy { .. } => Some(3),
|
||||
RouteError::NoHealthyNodes => Some(5),
|
||||
_ => None,
|
||||
}
|
||||
@@ -132,7 +141,9 @@ pub async fn resolve(
|
||||
// Snapshot loaded / unloaded / recovering state from the poller cache.
|
||||
let (loaded_route, unloaded_route, recovering_node, any_healthy) = {
|
||||
let nodes = fleet.nodes.read().await;
|
||||
let mut loaded_route = None;
|
||||
// All healthy nodes with the model loaded, each with its current
|
||||
// admission load (#53) so we can pick the least-busy replica (#55).
|
||||
let mut loaded_candidates: Vec<(String, String, usize)> = Vec::new();
|
||||
let mut unloaded_route = None;
|
||||
let mut recovering_node = None;
|
||||
let mut any_healthy = false;
|
||||
@@ -144,8 +155,15 @@ pub async fn resolve(
|
||||
if let Some(entry) = node.models.get(model_id) {
|
||||
match entry.status {
|
||||
ModelStatus::Loaded | ModelStatus::Reloading => {
|
||||
loaded_route = Some((node.name.clone(), node.endpoint.clone(), false));
|
||||
break;
|
||||
// Least-busy score: in-flight + queued from the
|
||||
// neuron's last /health (#53). Unknown load (no poll
|
||||
// yet) scores 0 so the replica stays eligible.
|
||||
let score = node
|
||||
.model_load
|
||||
.get(model_id)
|
||||
.map(|l| l.in_flight + l.queue_depth)
|
||||
.unwrap_or(0);
|
||||
loaded_candidates.push((node.name.clone(), node.endpoint.clone(), score));
|
||||
}
|
||||
ModelStatus::Unloaded => {
|
||||
if unloaded_route.is_none() {
|
||||
@@ -175,6 +193,12 @@ pub async fn resolve(
|
||||
}
|
||||
}
|
||||
}
|
||||
// Pick the least-busy loaded replica; ties break by node name for
|
||||
// deterministic routing. `false` = not a cold start.
|
||||
let loaded_route = loaded_candidates
|
||||
.into_iter()
|
||||
.min_by(|a, b| a.2.cmp(&b.2).then_with(|| a.0.cmp(&b.0)))
|
||||
.map(|(name, endpoint, _score)| (name, endpoint, false));
|
||||
(loaded_route, unloaded_route, recovering_node, any_healthy)
|
||||
};
|
||||
|
||||
@@ -237,11 +261,32 @@ async fn pick_feasible_neuron(
|
||||
b.2.cmp(&a.2) // pinned first (true > false)
|
||||
.then(a.0.cmp(&b.0))
|
||||
});
|
||||
let pick = candidates.into_iter().next();
|
||||
pick.map(|(n, e, _)| (n, e))
|
||||
.ok_or_else(|| RouteError::NoFeasibleNeuron {
|
||||
if let Some((n, e, _)) = candidates.into_iter().next() {
|
||||
return Ok((n, e));
|
||||
}
|
||||
|
||||
// No *healthy* feasible neuron. Distinguish a transient outage from a
|
||||
// permanent misconfiguration: if some neuron is topologically feasible
|
||||
// but currently unhealthy (e.g. it briefly missed polls while busy),
|
||||
// this is retryable — return 503 + Retry-After so the client backs off
|
||||
// and retries instead of treating a 404 as a hard failure. Only when no
|
||||
// neuron could *ever* satisfy the topology is it a permanent 404.
|
||||
let feasible_but_unhealthy = nodes.values().any(|node| {
|
||||
!node.healthy
|
||||
&& node
|
||||
.discovery
|
||||
.as_ref()
|
||||
.is_some_and(|disc| profile.is_feasible_on(&node.name, &disc.devices))
|
||||
});
|
||||
if feasible_but_unhealthy {
|
||||
Err(RouteError::FeasibleNodeUnhealthy {
|
||||
model_id: profile.id.clone(),
|
||||
})
|
||||
} else {
|
||||
Err(RouteError::NoFeasibleNeuron {
|
||||
model_id: profile.id.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Issue `POST {endpoint}/models/load` for this profile on this neuron,
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
use crate::entitlements_chain::ChainedEntitlementProvider;
|
||||
use crate::entitlements_local::LocalEntitlementProvider;
|
||||
use crate::entitlements_upstream::UpstreamEntitlementProvider;
|
||||
use cortex_core::catalogue::ModelCatalogue;
|
||||
use cortex_core::config::{EvictionSettings, GatewayConfig, NeuronEndpoint};
|
||||
use cortex_core::entitlements::EntitlementProvider;
|
||||
use cortex_core::node::NodeState;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
/// Shared fleet state, protected by a RwLock for concurrent reader access.
|
||||
@@ -11,6 +16,12 @@ pub struct CortexState {
|
||||
pub eviction: EvictionSettings,
|
||||
pub catalogue: ModelCatalogue,
|
||||
pub http_client: reqwest::Client,
|
||||
/// Resolves bearer keys to principals and enforces token budgets (#47).
|
||||
/// A local/static provider today (#50); the upstream client later (#57).
|
||||
pub entitlements: Arc<dyn EntitlementProvider>,
|
||||
/// Whether to reject unauthenticated requests (#49). Read by the auth
|
||||
/// middleware once it lands.
|
||||
pub require_auth: bool,
|
||||
}
|
||||
|
||||
impl CortexState {
|
||||
@@ -28,12 +39,29 @@ impl CortexState {
|
||||
last_poll: None,
|
||||
discovery: None,
|
||||
activation: None,
|
||||
model_load: HashMap::new(),
|
||||
consecutive_poll_failures: 0,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let catalogue = ModelCatalogue::load(&config.models_config);
|
||||
|
||||
// Local provider always handles operator + infra keys. When the
|
||||
// upstream client is enabled (#57), wrap it in the chain so locally
|
||||
// unknown keys fall through to the mesh authority; otherwise stay
|
||||
// purely local.
|
||||
let local = LocalEntitlementProvider::from_config(&config.entitlements);
|
||||
let entitlements: Arc<dyn EntitlementProvider> = if config.upstream.enabled {
|
||||
tracing::info!(url = %config.upstream.url, "upstream entitlement client enabled");
|
||||
Arc::new(ChainedEntitlementProvider::new(
|
||||
local,
|
||||
UpstreamEntitlementProvider::new(&config.upstream),
|
||||
))
|
||||
} else {
|
||||
Arc::new(local)
|
||||
};
|
||||
|
||||
Self {
|
||||
nodes: RwLock::new(nodes),
|
||||
neuron_configs: config.neurons.clone(),
|
||||
@@ -43,6 +71,8 @@ impl CortexState {
|
||||
.timeout(std::time::Duration::from_secs(300))
|
||||
.build()
|
||||
.expect("failed to build HTTP client"),
|
||||
entitlements,
|
||||
require_auth: config.entitlements.require_auth,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,6 +56,8 @@ async fn test_alias_resolves_in_chat_completions() {
|
||||
endpoint: mock_url,
|
||||
}],
|
||||
models_config: models_path.to_string_lossy().to_string(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
@@ -141,6 +143,8 @@ async fn test_aliases_surface_in_v1_models() {
|
||||
endpoint: mock_url,
|
||||
}],
|
||||
models_config: models_path.to_string_lossy().to_string(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
@@ -229,6 +233,8 @@ async fn test_alias_falls_through_for_unmapped_model() {
|
||||
endpoint: mock_url,
|
||||
}],
|
||||
models_config: models_path.to_string_lossy().to_string(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
|
||||
273
crates/cortex-gateway/tests/auth.rs
Normal file
@@ -0,0 +1,273 @@
|
||||
//! Integration tests for API-key auth + principal resolution (#49).
|
||||
//!
|
||||
//! Verifies the #63 rejection contract (401 invalid_api_key via the #60
|
||||
//! envelope) and that an authenticated request reaches neuron carrying the
|
||||
//! internal principal headers — while a client-supplied principal header is
|
||||
//! stripped (anti-spoofing).
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::Path;
|
||||
use axum::http::HeaderMap;
|
||||
use axum::routing::{get, post};
|
||||
use cortex_core::config::{
|
||||
ApiKeyConfig, EntitlementsConfig, EvictionSettings, EvictionStrategy, GatewayConfig,
|
||||
GatewaySettings, NeuronEndpoint,
|
||||
};
|
||||
use cortex_core::entitlements::{CapWindow, HEADER_ACCOUNT_ID, HEADER_KEY_ID};
|
||||
use cortex_core::node::{ModelEntry, ModelStatus};
|
||||
use cortex_gateway::state::CortexState;
|
||||
use serde_json::{Value, json};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
/// What the mock neuron observed on the inbound `/v1/chat/completions`
|
||||
/// request: the principal headers cortex stamped (or didn't).
|
||||
#[derive(Default)]
|
||||
struct Seen {
|
||||
account_id: Option<String>,
|
||||
key_id: Option<String>,
|
||||
}
|
||||
|
||||
/// Spawn a mock neuron that records the principal headers it receives and
|
||||
/// returns a trivial chat completion. Returns (base_url, observed).
|
||||
async fn spawn_capturing_neuron() -> (String, Arc<Mutex<Seen>>) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let base_url = format!("http://{addr}");
|
||||
let inference_url = base_url.clone();
|
||||
let seen: Arc<Mutex<Seen>> = Arc::new(Mutex::new(Seen::default()));
|
||||
let sink = Arc::clone(&seen);
|
||||
|
||||
let app = axum::Router::new()
|
||||
.route(
|
||||
"/models/{model_id}/endpoint",
|
||||
get(move |Path(_): Path<String>| {
|
||||
let url = inference_url.clone();
|
||||
async move { Json(json!({ "url": url })) }
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/v1/chat/completions",
|
||||
post(move |headers: HeaderMap, Json(body): Json<Value>| {
|
||||
let sink = Arc::clone(&sink);
|
||||
async move {
|
||||
{
|
||||
let mut s = sink.lock().unwrap();
|
||||
s.account_id = headers
|
||||
.get(HEADER_ACCOUNT_ID)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(str::to_string);
|
||||
s.key_id = headers
|
||||
.get(HEADER_KEY_ID)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(str::to_string);
|
||||
}
|
||||
let model = body.get("model").and_then(Value::as_str).unwrap_or("m");
|
||||
Json(json!({
|
||||
"id": "chatcmpl-auth-001",
|
||||
"object": "chat.completion",
|
||||
"created": 1700000000_u64,
|
||||
"model": model,
|
||||
"choices": [{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "ok"},
|
||||
"finish_reason": "stop"
|
||||
}],
|
||||
"usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}
|
||||
}))
|
||||
}
|
||||
}),
|
||||
)
|
||||
.with_state(());
|
||||
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
|
||||
(base_url, seen)
|
||||
}
|
||||
|
||||
/// Spawn a gateway with the given entitlements config, a single neuron, and
|
||||
/// `test-model` seeded as loaded (build_app spawns no poller).
|
||||
async fn spawn_gateway(neuron_url: &str, entitlements: EntitlementsConfig) -> String {
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![NeuronEndpoint {
|
||||
name: "mock-node".into(),
|
||||
endpoint: neuron_url.to_string(),
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements,
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("mock-node").unwrap();
|
||||
node.healthy = true;
|
||||
node.models.insert(
|
||||
"test-model".into(),
|
||||
ModelEntry {
|
||||
id: "test-model".into(),
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: None,
|
||||
vram_estimate_mb: Some(8000),
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let app = cortex_gateway::build_app(Arc::clone(&fleet));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
format!("http://{addr}")
|
||||
}
|
||||
|
||||
fn one_key_config(require_auth: bool) -> EntitlementsConfig {
|
||||
EntitlementsConfig {
|
||||
require_auth,
|
||||
keys: vec![ApiKeyConfig {
|
||||
key: "sk-good".into(),
|
||||
account_id: "acct-1".into(),
|
||||
key_id: Some("key-1".into()),
|
||||
hard_cap: None,
|
||||
window: CapWindow::Balance,
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
fn chat_body() -> Value {
|
||||
json!({
|
||||
"model": "test-model",
|
||||
"messages": [{"role": "user", "content": "hi"}]
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn missing_key_when_required_is_401_invalid_api_key() {
|
||||
let (neuron, _seen) = spawn_capturing_neuron().await;
|
||||
let gateway = spawn_gateway(&neuron, one_key_config(true)).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.json(&chat_body())
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::UNAUTHORIZED);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["error"]["code"], "invalid_api_key");
|
||||
assert_eq!(body["error"]["type"], "invalid_request_error");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unrecognized_key_is_ignored_when_auth_not_required() {
|
||||
let (neuron, seen) = spawn_capturing_neuron().await;
|
||||
// allow-anonymous mode: a placeholder/unknown bearer (as opencode,
|
||||
// Open WebUI, Agent Zero, litellm all send by default) must NOT be
|
||||
// rejected — it's ignored and the request is served anonymously.
|
||||
let gateway = spawn_gateway(&neuron, one_key_config(false)).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.bearer_auth("sk-dummy-placeholder")
|
||||
.json(&chat_body())
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::OK);
|
||||
let _ = resp.bytes().await.unwrap();
|
||||
// Served, but anonymous — no principal stamped from the bogus key.
|
||||
assert!(seen.lock().unwrap().account_id.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_key_is_401_when_auth_required() {
|
||||
let (neuron, seen) = spawn_capturing_neuron().await;
|
||||
// With auth required, a present-but-wrong credential is rejected.
|
||||
let gateway = spawn_gateway(&neuron, one_key_config(true)).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.bearer_auth("sk-wrong")
|
||||
.json(&chat_body())
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::UNAUTHORIZED);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["error"]["code"], "invalid_api_key");
|
||||
// Rejected before dispatch — neuron never saw the request.
|
||||
assert!(seen.lock().unwrap().account_id.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn valid_key_reaches_neuron_with_principal_headers() {
|
||||
let (neuron, seen) = spawn_capturing_neuron().await;
|
||||
let gateway = spawn_gateway(&neuron, one_key_config(true)).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.bearer_auth("sk-good")
|
||||
// A spoofed principal header must be stripped, not forwarded.
|
||||
.header(HEADER_ACCOUNT_ID, "attacker")
|
||||
.json(&chat_body())
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::OK);
|
||||
let s = seen.lock().unwrap();
|
||||
assert_eq!(s.account_id.as_deref(), Some("acct-1"));
|
||||
assert_eq!(s.key_id.as_deref(), Some("key-1"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn anonymous_allowed_when_auth_not_required() {
|
||||
let (neuron, seen) = spawn_capturing_neuron().await;
|
||||
let gateway = spawn_gateway(&neuron, EntitlementsConfig::default()).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.json(&chat_body())
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::OK);
|
||||
// No principal resolved → no principal headers stamped.
|
||||
let s = seen.lock().unwrap();
|
||||
assert!(s.account_id.is_none());
|
||||
assert!(s.key_id.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn health_is_public_even_when_auth_required() {
|
||||
let (neuron, _seen) = spawn_capturing_neuron().await;
|
||||
let gateway = spawn_gateway(&neuron, one_key_config(true)).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.get(format!("{gateway}/health"))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::OK);
|
||||
}
|
||||
254
crates/cortex-gateway/tests/budget_enforcement.rs
Normal file
@@ -0,0 +1,254 @@
|
||||
//! Integration tests for budget enforcement (#52) — the A0 seatbelt.
|
||||
//!
|
||||
//! A reservation over the key's hard cap is refused *before* neuron is hit,
|
||||
//! with the #63 code matching the cap-window semantics (rate_limit_exceeded
|
||||
//! + Retry-After for a resetting window, insufficient_quota for a hard
|
||||
//! balance). Spend never exceeds the cap. No 402, ever.
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::Path;
|
||||
use axum::routing::{get, post};
|
||||
use cortex_core::config::{
|
||||
ApiKeyConfig, EntitlementsConfig, EvictionSettings, EvictionStrategy, GatewayConfig,
|
||||
GatewaySettings, NeuronEndpoint,
|
||||
};
|
||||
use cortex_core::entitlements::{CapWindow, Principal};
|
||||
use cortex_core::node::{ModelEntry, ModelStatus};
|
||||
use cortex_gateway::state::CortexState;
|
||||
use serde_json::{Value, json};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
/// Mock neuron with a hit counter on the inference path, so a test can prove
|
||||
/// a request was (or wasn't) dispatched.
|
||||
async fn spawn_counting_neuron() -> (String, Arc<AtomicU64>) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let base_url = format!("http://{addr}");
|
||||
let inference_url = base_url.clone();
|
||||
let hits = Arc::new(AtomicU64::new(0));
|
||||
let sink = Arc::clone(&hits);
|
||||
|
||||
let app = axum::Router::new()
|
||||
.route(
|
||||
"/models/{model_id}/endpoint",
|
||||
get(move |Path(_): Path<String>| {
|
||||
let url = inference_url.clone();
|
||||
async move { Json(json!({ "url": url })) }
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/v1/chat/completions",
|
||||
post(move |Json(body): Json<Value>| {
|
||||
let sink = Arc::clone(&sink);
|
||||
async move {
|
||||
sink.fetch_add(1, Ordering::SeqCst);
|
||||
let model = body.get("model").and_then(Value::as_str).unwrap_or("m");
|
||||
Json(json!({
|
||||
"id": "chatcmpl-budget",
|
||||
"object": "chat.completion",
|
||||
"created": 1700000000_u64,
|
||||
"model": model,
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}
|
||||
}))
|
||||
}
|
||||
}),
|
||||
);
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
(base_url, hits)
|
||||
}
|
||||
|
||||
async fn spawn_gateway(neuron_url: &str, key: ApiKeyConfig) -> (Arc<CortexState>, String) {
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![NeuronEndpoint {
|
||||
name: "mock-node".into(),
|
||||
endpoint: neuron_url.to_string(),
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: EntitlementsConfig {
|
||||
require_auth: true,
|
||||
keys: vec![key],
|
||||
},
|
||||
upstream: Default::default(),
|
||||
};
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("mock-node").unwrap();
|
||||
node.healthy = true;
|
||||
node.models.insert(
|
||||
"test-model".into(),
|
||||
ModelEntry {
|
||||
id: "test-model".into(),
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: None,
|
||||
vram_estimate_mb: Some(8000),
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
let app = cortex_gateway::build_app(Arc::clone(&fleet));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
(fleet, format!("http://{addr}"))
|
||||
}
|
||||
|
||||
fn key(window: CapWindow, hard_cap: u64) -> ApiKeyConfig {
|
||||
ApiKeyConfig {
|
||||
key: "sk-cap".into(),
|
||||
account_id: "acct-cap".into(),
|
||||
key_id: Some("key-cap".into()),
|
||||
hard_cap: Some(hard_cap),
|
||||
window,
|
||||
}
|
||||
}
|
||||
|
||||
fn chat(max_tokens: u64) -> Value {
|
||||
json!({
|
||||
"model": "test-model",
|
||||
"max_tokens": max_tokens,
|
||||
"messages": [{"role": "user", "content": "hi"}]
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn balance_over_cap_is_429_insufficient_quota_before_dispatch() {
|
||||
let (neuron, hits) = spawn_counting_neuron().await;
|
||||
// Cap far below a single request's reservation (max_tokens 1000).
|
||||
let (_fleet, gateway) = spawn_gateway(&neuron, key(CapWindow::Balance, 10)).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.bearer_auth("sk-cap")
|
||||
.json(&chat(1000))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::TOO_MANY_REQUESTS);
|
||||
// Hard balance → no Retry-After.
|
||||
assert!(resp.headers().get(reqwest::header::RETRY_AFTER).is_none());
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["error"]["code"], "insufficient_quota");
|
||||
// Refused before dispatch — neuron never saw it.
|
||||
assert_eq!(hits.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rolling_over_cap_is_429_rate_limited_with_retry_after() {
|
||||
let (neuron, hits) = spawn_counting_neuron().await;
|
||||
let (_fleet, gateway) =
|
||||
spawn_gateway(&neuron, key(CapWindow::Rolling { seconds: 3600 }, 10)).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.bearer_auth("sk-cap")
|
||||
.json(&chat(1000))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::TOO_MANY_REQUESTS);
|
||||
let retry = resp
|
||||
.headers()
|
||||
.get(reqwest::header::RETRY_AFTER)
|
||||
.expect("rolling-window rejection must carry Retry-After");
|
||||
assert!(retry.to_str().unwrap().parse::<u64>().unwrap() >= 1);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["error"]["code"], "rate_limit_exceeded");
|
||||
assert_eq!(hits.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn within_cap_is_served() {
|
||||
let (neuron, hits) = spawn_counting_neuron().await;
|
||||
let (_fleet, gateway) = spawn_gateway(&neuron, key(CapWindow::Balance, 1_000_000)).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.bearer_auth("sk-cap")
|
||||
.json(&chat(50))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::OK);
|
||||
let _ = resp.bytes().await.unwrap();
|
||||
assert_eq!(hits.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a0_seatbelt_caps_a_runaway_fan_out() {
|
||||
// An Agent-Zero-style key with a modest cap: a burst of requests drains
|
||||
// it, then further requests are refused — the account stops draining and
|
||||
// spend never exceeds the cap.
|
||||
let (neuron, hits) = spawn_counting_neuron().await;
|
||||
let (fleet, gateway) = spawn_gateway(&neuron, key(CapWindow::Balance, 100)).await;
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let mut ok = 0;
|
||||
let mut refused = 0;
|
||||
for _ in 0..20 {
|
||||
let resp = client
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.bearer_auth("sk-cap")
|
||||
.json(&chat(20))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
match resp.status() {
|
||||
reqwest::StatusCode::OK => {
|
||||
ok += 1;
|
||||
let _ = resp.bytes().await.unwrap();
|
||||
}
|
||||
reqwest::StatusCode::TOO_MANY_REQUESTS => {
|
||||
refused += 1;
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["error"]["code"], "insufficient_quota");
|
||||
}
|
||||
other => panic!("unexpected status {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
assert!(ok >= 1, "some requests should be served");
|
||||
assert!(refused >= 1, "the cap must eventually refuse the fan-out");
|
||||
assert_eq!(
|
||||
hits.load(Ordering::SeqCst),
|
||||
ok,
|
||||
"refused requests never dispatched"
|
||||
);
|
||||
|
||||
// Spend never exceeded the hard cap (reservation prevents overshoot).
|
||||
// Poll briefly for in-flight settles to land.
|
||||
let principal = Principal {
|
||||
account_id: "acct-cap".into(),
|
||||
key_id: "key-cap".into(),
|
||||
};
|
||||
for _ in 0..50 {
|
||||
let snap = fleet.entitlements.snapshot(&principal).await.unwrap();
|
||||
if snap.reserved == 0 {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
|
||||
}
|
||||
let snap = fleet.entitlements.snapshot(&principal).await.unwrap();
|
||||
assert!(snap.spent <= 100, "spent {} exceeded cap", snap.spent);
|
||||
}
|
||||
@@ -429,6 +429,8 @@ pub async fn spawn_gateway_with_state(mock_url: &str) -> (Arc<CortexState>, Stri
|
||||
endpoint: mock_url.to_string(),
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
|
||||
@@ -88,6 +88,8 @@ async fn error_response_no_healthy_nodes() {
|
||||
endpoint: "http://127.0.0.1:1".into(),
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(cortex_gateway::state::CortexState::from_config(&config));
|
||||
|
||||
@@ -71,6 +71,8 @@ fn make_fleet(endpoint: &str, defrag_after: u32) -> Arc<CortexState> {
|
||||
endpoint: endpoint.to_string(),
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
Arc::new(CortexState::from_config(&config))
|
||||
}
|
||||
|
||||
125
crates/cortex-gateway/tests/feasibility_routing.rs
Normal file
@@ -0,0 +1,125 @@
|
||||
//! Router: a catalogued model whose only topologically-feasible neuron is
|
||||
//! currently unhealthy is a *transient* condition (retryable 503), not a
|
||||
//! permanent 404. This is the exact shape of the beast incident: benjy/
|
||||
//! quadbrat (1 GPU, healthy) can't host the 27B, and beast (2 GPU) — the
|
||||
//! sole feasible node — briefly drops out → clients must back off and retry,
|
||||
//! not hard-fail.
|
||||
|
||||
use cortex_core::config::{
|
||||
EvictionSettings, EvictionStrategy, GatewayConfig, GatewaySettings, NeuronEndpoint,
|
||||
};
|
||||
use cortex_core::discovery::{DeviceInfo, DiscoveryResponse};
|
||||
use cortex_gateway::router::{self, RouteError};
|
||||
use cortex_gateway::state::CortexState;
|
||||
use std::sync::Arc;
|
||||
|
||||
fn devices(n: usize) -> Vec<DeviceInfo> {
|
||||
(0..n)
|
||||
.map(|i| DeviceInfo {
|
||||
index: i as u32,
|
||||
name: "RTX 5090".into(),
|
||||
vram_total_mb: 32_768,
|
||||
compute_capability: "9.0".into(),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn discovery(host: &str, n_devices: usize) -> DiscoveryResponse {
|
||||
DiscoveryResponse {
|
||||
hostname: host.into(),
|
||||
os: "Linux".into(),
|
||||
kernel: "7.0".into(),
|
||||
cuda_version: Some("13.0".into()),
|
||||
driver_version: Some("999".into()),
|
||||
devices: devices(n_devices),
|
||||
harnesses: vec!["candle".into()],
|
||||
cuda_unavailable_reason: None,
|
||||
max_prompt_tokens: 49_152,
|
||||
}
|
||||
}
|
||||
|
||||
/// Catalogue with one model needing 2 devices. Returns a temp path.
|
||||
fn write_catalogue() -> std::path::PathBuf {
|
||||
let toml = r#"
|
||||
[[models]]
|
||||
id = "big-model"
|
||||
harness = "candle"
|
||||
min_devices = 2
|
||||
"#;
|
||||
let path = std::env::temp_dir().join("cortex_test_feasibility_models.toml");
|
||||
std::fs::write(&path, toml).unwrap();
|
||||
path
|
||||
}
|
||||
|
||||
async fn fleet_with(big_healthy: bool, big_devices: usize) -> Arc<CortexState> {
|
||||
let cat = write_catalogue();
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![
|
||||
NeuronEndpoint {
|
||||
name: "small".into(),
|
||||
endpoint: "http://127.0.0.1:1".into(),
|
||||
},
|
||||
NeuronEndpoint {
|
||||
name: "big".into(),
|
||||
endpoint: "http://127.0.0.1:2".into(),
|
||||
},
|
||||
],
|
||||
models_config: cat.to_string_lossy().into_owned(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
// "small" is healthy but only has 1 GPU → not feasible for the model.
|
||||
let small = nodes.get_mut("small").unwrap();
|
||||
small.healthy = true;
|
||||
small.discovery = Some(discovery("small", 1));
|
||||
// "big" has enough GPUs but its health is the variable under test.
|
||||
let big = nodes.get_mut("big").unwrap();
|
||||
big.healthy = big_healthy;
|
||||
big.discovery = Some(discovery("big", big_devices));
|
||||
}
|
||||
fleet
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn feasible_node_unhealthy_is_transient_503() {
|
||||
// big (2 GPU, the only feasible node) is unhealthy; small (1 GPU) is
|
||||
// healthy but can't host the model → retryable, not a permanent 404.
|
||||
let fleet = fleet_with(false, 2).await;
|
||||
let err = router::resolve(&fleet, "big-model")
|
||||
.await
|
||||
.expect_err("model can't be served right now");
|
||||
assert!(
|
||||
matches!(err, RouteError::FeasibleNodeUnhealthy { .. }),
|
||||
"expected FeasibleNodeUnhealthy, got {err:?}"
|
||||
);
|
||||
assert_eq!(err.http_status(), 503);
|
||||
assert_eq!(err.retry_after_secs(), Some(3));
|
||||
assert_eq!(err.code(), "service_unavailable");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn no_node_can_ever_satisfy_is_permanent_404() {
|
||||
// big is healthy but only has 1 GPU now (e.g. topology genuinely can't
|
||||
// satisfy min_devices=2 anywhere) → permanent, non-retryable 404.
|
||||
let fleet = fleet_with(true, 1).await;
|
||||
let err = router::resolve(&fleet, "big-model")
|
||||
.await
|
||||
.expect_err("no feasible topology");
|
||||
assert!(
|
||||
matches!(err, RouteError::NoFeasibleNeuron { .. }),
|
||||
"expected NoFeasibleNeuron, got {err:?}"
|
||||
);
|
||||
assert_eq!(err.http_status(), 404);
|
||||
assert_eq!(err.retry_after_secs(), None);
|
||||
}
|
||||
190
crates/cortex-gateway/tests/load_routing.rs
Normal file
@@ -0,0 +1,190 @@
|
||||
//! Load-aware routing across replicas (#55).
|
||||
//!
|
||||
//! When a model is loaded on more than one healthy neuron, the router picks
|
||||
//! the least-busy replica using the per-model admission load each neuron
|
||||
//! reports on `GET /health` (#53), rather than always taking the first.
|
||||
|
||||
mod common;
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::Path;
|
||||
use axum::http::{StatusCode, header};
|
||||
use axum::response::IntoResponse;
|
||||
use axum::routing::{get, post};
|
||||
use cortex_core::config::{
|
||||
EvictionSettings, EvictionStrategy, GatewayConfig, GatewaySettings, NeuronEndpoint,
|
||||
};
|
||||
use cortex_core::discovery::ModelLoad;
|
||||
use cortex_core::node::{ModelEntry, ModelStatus};
|
||||
use cortex_gateway::state::CortexState;
|
||||
use serde_json::{Value, json};
|
||||
use std::sync::Arc;
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
/// Seed a node as healthy with `test-model` loaded and a given admission load.
|
||||
async fn seed_loaded(fleet: &CortexState, node: &str, in_flight: usize, queue_depth: usize) {
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let n = nodes.get_mut(node).expect("node exists");
|
||||
n.healthy = true;
|
||||
n.models.insert(
|
||||
"test-model".into(),
|
||||
ModelEntry {
|
||||
id: "test-model".into(),
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: None,
|
||||
vram_estimate_mb: Some(8000),
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
},
|
||||
);
|
||||
n.model_load.insert(
|
||||
"test-model".into(),
|
||||
ModelLoad {
|
||||
id: "test-model".into(),
|
||||
in_flight,
|
||||
queue_depth,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/// Build a gateway state over two mock neurons (no poller; we seed state).
|
||||
async fn two_neuron_fleet(endpoint_a: &str, endpoint_b: &str) -> Arc<CortexState> {
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![
|
||||
NeuronEndpoint {
|
||||
name: "node-a".into(),
|
||||
endpoint: endpoint_a.to_string(),
|
||||
},
|
||||
NeuronEndpoint {
|
||||
name: "node-b".into(),
|
||||
endpoint: endpoint_b.to_string(),
|
||||
},
|
||||
],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
Arc::new(CortexState::from_config(&config))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn routes_to_least_busy_replica() {
|
||||
let neuron_a = common::spawn_mock_neuron().await;
|
||||
let neuron_b = common::spawn_mock_neuron().await;
|
||||
let fleet = two_neuron_fleet(&neuron_a, &neuron_b).await;
|
||||
|
||||
// A is busy (1 running + 3 queued), B is idle.
|
||||
seed_loaded(&fleet, "node-a", 1, 3).await;
|
||||
seed_loaded(&fleet, "node-b", 0, 0).await;
|
||||
|
||||
let route = cortex_gateway::router::resolve(&fleet, "test-model")
|
||||
.await
|
||||
.expect("model is loaded on both nodes");
|
||||
assert_eq!(route.node_name, "node-b", "should pick the idle replica");
|
||||
|
||||
// Flip the load: now B is the busy one.
|
||||
seed_loaded(&fleet, "node-a", 0, 0).await;
|
||||
seed_loaded(&fleet, "node-b", 1, 5).await;
|
||||
let route = cortex_gateway::router::resolve(&fleet, "test-model")
|
||||
.await
|
||||
.expect("still loaded");
|
||||
assert_eq!(route.node_name, "node-a", "should follow the lighter load");
|
||||
}
|
||||
|
||||
/// Mock neuron whose inference endpoint always returns a #63 backpressure
|
||||
/// envelope (503 + Retry-After) — simulating a saturated neuron.
|
||||
async fn spawn_busy_neuron() -> String {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let base_url = format!("http://{addr}");
|
||||
let inference_url = base_url.clone();
|
||||
let app = axum::Router::new()
|
||||
.route(
|
||||
"/models/{model_id}/endpoint",
|
||||
get(move |Path(_): Path<String>| {
|
||||
let url = inference_url.clone();
|
||||
async move { Json(json!({ "url": url })) }
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/v1/chat/completions",
|
||||
post(|| async {
|
||||
let body = json!({"error": {
|
||||
"message": "model is busy (admission queue full); retry shortly",
|
||||
"type": "rate_limit_error",
|
||||
"code": "rate_limit_exceeded",
|
||||
"param": null
|
||||
}});
|
||||
(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
[(header::RETRY_AFTER, "6")],
|
||||
Json(body),
|
||||
)
|
||||
.into_response()
|
||||
}),
|
||||
);
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
base_url
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn neuron_backpressure_is_propagated_intact() {
|
||||
// A saturated neuron's 503 + Retry-After + envelope must reach the client
|
||||
// verbatim — not unwrapped, remapped, or stripped (#55 / #63).
|
||||
let neuron = spawn_busy_neuron().await;
|
||||
let fleet = two_neuron_fleet(&neuron, &neuron).await;
|
||||
seed_loaded(&fleet, "node-a", 1, 8).await;
|
||||
|
||||
let app = cortex_gateway::build_app(Arc::clone(&fleet));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("http://{addr}/v1/chat/completions"))
|
||||
.json(&json!({"model": "test-model", "messages": [{"role": "user", "content": "hi"}]}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(
|
||||
resp.headers()
|
||||
.get(reqwest::header::RETRY_AFTER)
|
||||
.and_then(|v| v.to_str().ok()),
|
||||
Some("6"),
|
||||
"Retry-After must survive the proxy"
|
||||
);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["error"]["code"], "rate_limit_exceeded");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ties_break_deterministically_by_name() {
|
||||
let neuron_a = common::spawn_mock_neuron().await;
|
||||
let neuron_b = common::spawn_mock_neuron().await;
|
||||
let fleet = two_neuron_fleet(&neuron_a, &neuron_b).await;
|
||||
|
||||
// Equal load on both → stable pick (lowest node name).
|
||||
seed_loaded(&fleet, "node-a", 0, 0).await;
|
||||
seed_loaded(&fleet, "node-b", 0, 0).await;
|
||||
|
||||
let route = cortex_gateway::router::resolve(&fleet, "test-model")
|
||||
.await
|
||||
.expect("loaded");
|
||||
assert_eq!(route.node_name, "node-a", "ties break by name");
|
||||
}
|
||||
209
crates/cortex-gateway/tests/metering.rs
Normal file
@@ -0,0 +1,209 @@
|
||||
//! Integration tests for per-request token metering (#51).
|
||||
//!
|
||||
//! Drives authenticated requests through the gateway to a mock neuron that
|
||||
//! reports a fixed `usage` object, then asserts the EntitlementProvider's
|
||||
//! spend ledger reflects cumulative per-key spend and that reservations
|
||||
//! settle to actual (no outstanding reserved tokens once requests complete).
|
||||
|
||||
mod common;
|
||||
|
||||
use cortex_core::config::{
|
||||
ApiKeyConfig, EntitlementsConfig, EvictionSettings, EvictionStrategy, GatewayConfig,
|
||||
GatewaySettings, NeuronEndpoint,
|
||||
};
|
||||
use cortex_core::entitlements::{CapWindow, Principal};
|
||||
use cortex_core::node::{ModelEntry, ModelStatus};
|
||||
use cortex_gateway::state::CortexState;
|
||||
use serde_json::json;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
const ACCOUNT: &str = "acct-meter";
|
||||
const KEY_ID: &str = "key-meter";
|
||||
const BEARER: &str = "sk-meter";
|
||||
|
||||
/// The mock neuron (common::spawn_mock_neuron) reports this fixed usage on
|
||||
/// every chat completion.
|
||||
const PROMPT_PER_REQ: u64 = 10;
|
||||
const COMPLETION_PER_REQ: u64 = 5;
|
||||
|
||||
async fn spawn_metered_gateway(neuron_url: &str) -> (Arc<CortexState>, String) {
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![NeuronEndpoint {
|
||||
name: "mock-node".into(),
|
||||
endpoint: neuron_url.to_string(),
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: EntitlementsConfig {
|
||||
require_auth: true,
|
||||
keys: vec![ApiKeyConfig {
|
||||
key: BEARER.into(),
|
||||
account_id: ACCOUNT.into(),
|
||||
key_id: Some(KEY_ID.into()),
|
||||
hard_cap: Some(1_000_000),
|
||||
window: CapWindow::Balance,
|
||||
}],
|
||||
},
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("mock-node").unwrap();
|
||||
node.healthy = true;
|
||||
node.models.insert(
|
||||
"test-model".into(),
|
||||
ModelEntry {
|
||||
id: "test-model".into(),
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: None,
|
||||
vram_estimate_mb: Some(8000),
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let app = cortex_gateway::build_app(Arc::clone(&fleet));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
(fleet, format!("http://{addr}"))
|
||||
}
|
||||
|
||||
fn principal() -> Principal {
|
||||
Principal {
|
||||
account_id: ACCOUNT.into(),
|
||||
key_id: KEY_ID.into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Poll the provider ledger until settled spend reaches `expected` (settle
|
||||
/// runs in a spawned task after the response stream finishes) or time out.
|
||||
async fn await_spent(fleet: &CortexState, expected: u64) -> u64 {
|
||||
let principal = principal();
|
||||
for _ in 0..100 {
|
||||
let snap = fleet.entitlements.snapshot(&principal).await.unwrap();
|
||||
if snap.spent >= expected {
|
||||
return snap.spent;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
fleet.entitlements.snapshot(&principal).await.unwrap().spent
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cumulative_spend_is_metered_per_key() {
|
||||
let neuron = common::spawn_mock_neuron().await;
|
||||
let (fleet, gateway) = spawn_metered_gateway(&neuron).await;
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
const N: u64 = 3;
|
||||
for _ in 0..N {
|
||||
let resp = client
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.bearer_auth(BEARER)
|
||||
.json(&json!({"model": "test-model", "messages": [{"role": "user", "content": "hi"}]}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::OK);
|
||||
// Drain the body so the response stream finishes and metering settles.
|
||||
let _ = resp.bytes().await.unwrap();
|
||||
}
|
||||
|
||||
let expected = N * (PROMPT_PER_REQ + COMPLETION_PER_REQ);
|
||||
let spent = await_spent(&fleet, expected).await;
|
||||
assert_eq!(
|
||||
spent, expected,
|
||||
"ledger must reflect cumulative per-key spend"
|
||||
);
|
||||
|
||||
// Reservations settled to actual — nothing left outstanding.
|
||||
let snap = fleet.entitlements.snapshot(&principal()).await.unwrap();
|
||||
assert_eq!(snap.reserved, 0, "all reservations must settle/release");
|
||||
assert_eq!(snap.hard_cap, Some(1_000_000));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn anonymous_request_records_no_spend() {
|
||||
// require_auth=false so the unauthenticated request is served, but with
|
||||
// no principal it must not touch any ledger.
|
||||
let neuron = common::spawn_mock_neuron().await;
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![NeuronEndpoint {
|
||||
name: "mock-node".into(),
|
||||
endpoint: neuron.clone(),
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: EntitlementsConfig::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let node = nodes.get_mut("mock-node").unwrap();
|
||||
node.healthy = true;
|
||||
node.models.insert(
|
||||
"test-model".into(),
|
||||
ModelEntry {
|
||||
id: "test-model".into(),
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: None,
|
||||
vram_estimate_mb: Some(8000),
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
let app = cortex_gateway::build_app(Arc::clone(&fleet));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("http://{addr}/v1/chat/completions"))
|
||||
.json(&json!({"model": "test-model", "messages": [{"role": "user", "content": "hi"}]}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::OK);
|
||||
let _ = resp.bytes().await.unwrap();
|
||||
|
||||
// An unconfigured principal has a zeroed snapshot — nothing was metered.
|
||||
let snap = fleet
|
||||
.entitlements
|
||||
.snapshot(&Principal {
|
||||
account_id: "nobody".into(),
|
||||
key_id: "nobody".into(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(snap.spent, 0);
|
||||
}
|
||||
132
crates/cortex-gateway/tests/model_cost.rs
Normal file
@@ -0,0 +1,132 @@
|
||||
//! Issue #68: the `cost` wire contract on `GET /v1/models`.
|
||||
//!
|
||||
//! `cost` is operator-set pricing sourced from the `models.toml` catalogue
|
||||
//! profile (the source of truth today; the marketplace clearing house #59
|
||||
//! later — both must read the same value metering/#51 bills against). The
|
||||
//! shape is the models.dev/opencode convention: **USD per 1,000,000 tokens,
|
||||
//! as JSON numbers**, with optional `cache_read`/`cache_write` tiers. This
|
||||
//! test pins:
|
||||
//! - the units/shape (per-million floats, not per-token, not strings);
|
||||
//! - that cache fields flow through when present and are omitted otherwise;
|
||||
//! - the load-bearing **absent vs `0.0`** distinction (#68): a model with
|
||||
//! no catalogue `cost` omits the key entirely (price unknown), distinct
|
||||
//! from an explicit `0.0` (intentionally free).
|
||||
//!
|
||||
//! Catalogue-only models surface via Pass 1 of `list_models` even with no
|
||||
//! feasible neuron, so this is hermetic — no nodes or poller needed.
|
||||
|
||||
use cortex_core::config::{
|
||||
EvictionSettings, EvictionStrategy, GatewayConfig, GatewaySettings, NeuronEndpoint,
|
||||
};
|
||||
use cortex_gateway::state::CortexState;
|
||||
use std::sync::Arc;
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
#[tokio::test]
|
||||
async fn v1_models_cost_units_shape_and_absent_vs_zero() {
|
||||
// Three catalogue models exercise the whole contract: a priced model
|
||||
// with cache tiers, an intentionally-free model (explicit 0.0), and an
|
||||
// unpriced model (no `cost` block at all).
|
||||
let models_toml = r#"
|
||||
[[models]]
|
||||
id = "priced-model"
|
||||
harness = "candle"
|
||||
cost.input = 0.5
|
||||
cost.output = 1.5
|
||||
cost.cache_read = 0.05
|
||||
cost.cache_write = 0.6
|
||||
|
||||
[[models]]
|
||||
id = "free-model"
|
||||
harness = "candle"
|
||||
cost.input = 0.0
|
||||
cost.output = 0.0
|
||||
|
||||
[[models]]
|
||||
id = "unpriced-model"
|
||||
harness = "candle"
|
||||
"#;
|
||||
let cat_path = std::env::temp_dir().join("cortex_test_issue68_models.toml");
|
||||
std::fs::write(&cat_path, models_toml).unwrap();
|
||||
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
// Never contacted: build_app does not spawn the poller, so the
|
||||
// catalogue alone drives /v1/models.
|
||||
neurons: vec![NeuronEndpoint {
|
||||
name: "mock-node".into(),
|
||||
endpoint: "http://127.0.0.1:1".into(),
|
||||
}],
|
||||
models_config: cat_path.to_string_lossy().into_owned(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
let app = cortex_gateway::build_app(Arc::clone(&fleet));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
|
||||
let body: serde_json::Value = reqwest::Client::new()
|
||||
.get(format!("http://{addr}/v1/models"))
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let data = body["data"].as_array().expect("data is an array");
|
||||
let entry = |id: &str| {
|
||||
data.iter()
|
||||
.find(|m| m["id"] == id)
|
||||
.unwrap_or_else(|| panic!("{id} present in /v1/models"))
|
||||
.clone()
|
||||
};
|
||||
|
||||
// Priced model: exact values flow through as JSON numbers (USD per 1M
|
||||
// tokens). If anything rescaled by 10⁶ or stringified, these fail.
|
||||
let priced = entry("priced-model");
|
||||
assert_eq!(priced["cost"]["input"], 0.5);
|
||||
assert_eq!(priced["cost"]["output"], 1.5);
|
||||
assert_eq!(priced["cost"]["cache_read"], 0.05);
|
||||
assert_eq!(priced["cost"]["cache_write"], 0.6);
|
||||
assert!(
|
||||
priced["cost"]["input"].is_number(),
|
||||
"cost.input must be a JSON number, not a string"
|
||||
);
|
||||
|
||||
// Intentionally free: cost present, rates explicitly 0.0. Unset cache
|
||||
// tiers are omitted (skip_serializing_if), not emitted as null/0.
|
||||
let free = entry("free-model");
|
||||
assert_eq!(free["cost"]["input"], 0.0);
|
||||
assert_eq!(free["cost"]["output"], 0.0);
|
||||
assert!(
|
||||
free["cost"].get("cache_read").is_none(),
|
||||
"absent cache tiers must be omitted, not null"
|
||||
);
|
||||
assert!(free["cost"].get("cache_write").is_none());
|
||||
|
||||
// Unpriced: the whole `cost` object is omitted — "price unknown",
|
||||
// distinct from the free model's explicit 0.0. This is the #68
|
||||
// distinction opencode needs to avoid showing $0 for a model whose
|
||||
// price simply hasn't been declared.
|
||||
let unpriced = entry("unpriced-model");
|
||||
assert!(
|
||||
unpriced.get("cost").is_none(),
|
||||
"a model with no catalogue cost must omit `cost` entirely, got {:?}",
|
||||
unpriced.get("cost")
|
||||
);
|
||||
|
||||
let _ = std::fs::remove_file(&cat_path);
|
||||
}
|
||||
@@ -54,6 +54,8 @@ capabilities = ["text"]
|
||||
endpoint: "http://127.0.0.1:1".into(),
|
||||
}],
|
||||
models_config: cat_path.to_string_lossy().into_owned(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
|
||||
@@ -31,6 +31,8 @@ async fn test_poller_discovers_models() {
|
||||
endpoint: mock_url,
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
@@ -82,6 +84,8 @@ async fn test_poller_updates_gateway_models_endpoint() {
|
||||
endpoint: mock_url,
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
@@ -153,6 +157,8 @@ async fn test_models_endpoint_unions_capabilities_across_nodes() {
|
||||
},
|
||||
],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
@@ -215,6 +221,8 @@ async fn test_poller_marks_unreachable_node_unhealthy() {
|
||||
endpoint: "http://127.0.0.1:1".into(),
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
@@ -224,10 +232,26 @@ async fn test_poller_marks_unreachable_node_unhealthy() {
|
||||
nodes.get_mut("dead-node").unwrap().healthy = true;
|
||||
}
|
||||
|
||||
// Debounce (#53 follow-up): a single missed poll must NOT evict a
|
||||
// previously-healthy node — a busy neuron briefly slow to answer
|
||||
// shouldn't yank its models out of routing.
|
||||
cortex_gateway::poller::poll_once(&fleet).await;
|
||||
assert!(
|
||||
fleet.nodes.read().await.get("dead-node").unwrap().healthy,
|
||||
"one failed poll should not mark a healthy node unhealthy"
|
||||
);
|
||||
|
||||
let nodes = fleet.nodes.read().await;
|
||||
assert!(!nodes.get("dead-node").unwrap().healthy);
|
||||
// It flips unhealthy only after POLL_FAILURE_THRESHOLD (3) consecutive
|
||||
// failures.
|
||||
cortex_gateway::poller::poll_once(&fleet).await;
|
||||
cortex_gateway::poller::poll_once(&fleet).await;
|
||||
assert!(
|
||||
!fleet.nodes.read().await.get("dead-node").unwrap().healthy,
|
||||
"three consecutive failed polls should mark the node unhealthy"
|
||||
);
|
||||
|
||||
// A subsequent successful poll would reset the counter and restore
|
||||
// health; covered implicitly by the discovery tests above.
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -252,6 +276,8 @@ async fn test_poller_removes_stale_models() {
|
||||
endpoint: mock_url,
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
@@ -282,6 +308,8 @@ async fn test_poller_removes_stale_models() {
|
||||
endpoint: new_mock_url,
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet2 = Arc::new(CortexState::from_config(&config2));
|
||||
@@ -363,6 +391,8 @@ async fn test_poller_captures_activation_from_health() {
|
||||
endpoint: mock_url,
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
@@ -407,6 +437,8 @@ async fn test_poller_parses_recovering_status() {
|
||||
endpoint: mock_url,
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
|
||||
175
crates/cortex-gateway/tests/prompt_prevalidation.rs
Normal file
@@ -0,0 +1,175 @@
|
||||
//! Fail-fast prompt pre-validation + advisory client hints (#56).
|
||||
//!
|
||||
//! cortex refuses a prompt that already exceeds the model's advertised
|
||||
//! context window before dispatching to neuron — the same #60
|
||||
//! `context_length_exceeded` envelope neuron would emit, just earlier — and
|
||||
//! attaches an advisory `X-Helexa-Advice` header for fingerprinted clients.
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::Path;
|
||||
use axum::routing::{get, post};
|
||||
use cortex_core::config::{
|
||||
EvictionSettings, EvictionStrategy, GatewayConfig, GatewaySettings, NeuronEndpoint,
|
||||
};
|
||||
use cortex_core::harness::ModelLimit;
|
||||
use cortex_core::node::{ModelEntry, ModelStatus};
|
||||
use cortex_gateway::state::CortexState;
|
||||
use serde_json::{Value, json};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
/// Mock neuron with a hit counter, so a test can prove a request was (or
|
||||
/// wasn't) dispatched past the gateway's pre-validation.
|
||||
async fn spawn_counting_neuron() -> (String, Arc<AtomicU64>) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let base_url = format!("http://{addr}");
|
||||
let inference_url = base_url.clone();
|
||||
let hits = Arc::new(AtomicU64::new(0));
|
||||
let sink = Arc::clone(&hits);
|
||||
let app = axum::Router::new()
|
||||
.route(
|
||||
"/models/{model_id}/endpoint",
|
||||
get(move |Path(_): Path<String>| {
|
||||
let url = inference_url.clone();
|
||||
async move { Json(json!({ "url": url })) }
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/v1/chat/completions",
|
||||
post(move || {
|
||||
let sink = Arc::clone(&sink);
|
||||
async move {
|
||||
sink.fetch_add(1, Ordering::SeqCst);
|
||||
Json(json!({
|
||||
"id": "c", "object": "chat.completion", "created": 1_700_000_000_u64,
|
||||
"model": "test-model",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}
|
||||
}))
|
||||
}
|
||||
}),
|
||||
);
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
(base_url, hits)
|
||||
}
|
||||
|
||||
/// Gateway over one neuron with `test-model` loaded and a tiny advertised
|
||||
/// context window (so a modest prompt overflows it).
|
||||
async fn spawn_gateway(neuron: &str, context: usize) -> String {
|
||||
let config = GatewayConfig {
|
||||
gateway: GatewaySettings {
|
||||
listen: "127.0.0.1:0".into(),
|
||||
metrics_listen: "127.0.0.1:0".into(),
|
||||
},
|
||||
eviction: EvictionSettings {
|
||||
strategy: EvictionStrategy::Lru,
|
||||
defrag_after_cycles: 0,
|
||||
},
|
||||
neurons: vec![NeuronEndpoint {
|
||||
name: "mock-node".into(),
|
||||
endpoint: neuron.to_string(),
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
let fleet = Arc::new(CortexState::from_config(&config));
|
||||
{
|
||||
let mut nodes = fleet.nodes.write().await;
|
||||
let n = nodes.get_mut("mock-node").unwrap();
|
||||
n.healthy = true;
|
||||
n.models.insert(
|
||||
"test-model".into(),
|
||||
ModelEntry {
|
||||
id: "test-model".into(),
|
||||
status: ModelStatus::Loaded,
|
||||
last_accessed: None,
|
||||
vram_estimate_mb: Some(8000),
|
||||
capabilities: Vec::new(),
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
limit: Some(ModelLimit {
|
||||
context,
|
||||
input: None,
|
||||
output: 16,
|
||||
}),
|
||||
},
|
||||
);
|
||||
}
|
||||
let app = cortex_gateway::build_app(Arc::clone(&fleet));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
format!("http://{addr}")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn over_long_prompt_is_rejected_before_dispatch() {
|
||||
let (neuron, hits) = spawn_counting_neuron().await;
|
||||
let gateway = spawn_gateway(&neuron, 50).await; // tiny 50-token window
|
||||
|
||||
// ~1200 chars → ~300 est tokens, well over 50.
|
||||
let big = "word ".repeat(240);
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.header("user-agent", "litellm/1.0")
|
||||
.json(&json!({"model": "test-model", "messages": [{"role": "user", "content": big}]}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::BAD_REQUEST);
|
||||
// Advisory hint for the fingerprinted client (header only, never body).
|
||||
assert!(
|
||||
resp.headers().get("x-helexa-advice").is_some(),
|
||||
"litellm should get advice"
|
||||
);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["error"]["code"], "context_length_exceeded");
|
||||
assert_eq!(body["error"]["max"], 50);
|
||||
// Refused at the edge — neuron never saw it.
|
||||
assert_eq!(hits.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn within_context_passes_through() {
|
||||
let (neuron, hits) = spawn_counting_neuron().await;
|
||||
let gateway = spawn_gateway(&neuron, 4096).await;
|
||||
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
.json(&json!({"model": "test-model", "messages": [{"role": "user", "content": "hi"}]}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::OK);
|
||||
let _ = resp.bytes().await.unwrap();
|
||||
assert_eq!(hits.load(Ordering::SeqCst), 1, "served by neuron");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_client_gets_no_advice_header() {
|
||||
let (neuron, _hits) = spawn_counting_neuron().await;
|
||||
let gateway = spawn_gateway(&neuron, 50).await;
|
||||
|
||||
let big = "word ".repeat(240);
|
||||
let resp = reqwest::Client::new()
|
||||
.post(format!("{gateway}/v1/chat/completions"))
|
||||
// no/unknown User-Agent → no advice, but still a clean 400
|
||||
.json(&json!({"model": "test-model", "messages": [{"role": "user", "content": big}]}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::BAD_REQUEST);
|
||||
assert!(resp.headers().get("x-helexa-advice").is_none());
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["error"]["code"], "context_length_exceeded");
|
||||
}
|
||||
@@ -117,6 +117,8 @@ async fn test_no_healthy_nodes() {
|
||||
endpoint: "http://127.0.0.1:1".into(),
|
||||
}],
|
||||
models_config: "/dev/null".into(),
|
||||
entitlements: Default::default(),
|
||||
upstream: Default::default(),
|
||||
};
|
||||
let fleet = std::sync::Arc::new(cortex_gateway::state::CortexState::from_config(&config));
|
||||
|
||||
|
||||
105
crates/cortex-gateway/tests/upstream_chain.rs
Normal file
@@ -0,0 +1,105 @@
|
||||
//! B3: the chained entitlement provider (local → upstream) and fail-closed
|
||||
//! semantics, exercised against a mock helexa-upstream `/authz/v1`.
|
||||
|
||||
use axum::{Json, Router, routing::post};
|
||||
use cortex_core::config::{ApiKeyConfig, EntitlementsConfig, UpstreamClientConfig};
|
||||
use cortex_core::entitlements::{AuthError, EntitlementProvider};
|
||||
use cortex_gateway::entitlements_chain::ChainedEntitlementProvider;
|
||||
use cortex_gateway::entitlements_local::LocalEntitlementProvider;
|
||||
use cortex_gateway::entitlements_upstream::UpstreamEntitlementProvider;
|
||||
use serde_json::{Value, json};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
/// Mock upstream: `mesh-key` resolves to a mesh account; anything else 401.
|
||||
/// reserve always grants reservation 1.
|
||||
async fn spawn_mock_upstream() -> String {
|
||||
async fn resolve(Json(body): Json<Value>) -> axum::response::Response {
|
||||
use axum::response::IntoResponse;
|
||||
if body["api_key"] == "mesh-key" {
|
||||
Json(json!({"principal": {"account_id": "mesh-acct", "key_id": "mesh-key-1"}}))
|
||||
.into_response()
|
||||
} else {
|
||||
(
|
||||
axum::http::StatusCode::UNAUTHORIZED,
|
||||
Json(json!({"error": {"code": "invalid_api_key"}})),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
}
|
||||
async fn reserve() -> Json<Value> {
|
||||
Json(json!({ "reservation_id": 1 }))
|
||||
}
|
||||
let app = Router::new()
|
||||
.route("/authz/v1/resolve", post(resolve))
|
||||
.route("/authz/v1/reserve", post(reserve));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
format!("http://{addr}")
|
||||
}
|
||||
|
||||
fn local_with_key() -> LocalEntitlementProvider {
|
||||
let cfg = EntitlementsConfig {
|
||||
require_auth: false,
|
||||
keys: vec![ApiKeyConfig {
|
||||
key: "local-key".into(),
|
||||
account_id: "op".into(),
|
||||
key_id: None,
|
||||
hard_cap: None,
|
||||
window: Default::default(),
|
||||
}],
|
||||
};
|
||||
LocalEntitlementProvider::from_config(&cfg)
|
||||
}
|
||||
|
||||
fn chain(local: LocalEntitlementProvider, url: &str) -> ChainedEntitlementProvider {
|
||||
let upstream = UpstreamEntitlementProvider::new(&UpstreamClientConfig {
|
||||
enabled: true,
|
||||
url: url.to_string(),
|
||||
bearer: "client-secret".into(),
|
||||
timeout_secs: 5,
|
||||
});
|
||||
ChainedEntitlementProvider::new(local, upstream)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_key_resolves_locally() {
|
||||
let url = spawn_mock_upstream().await;
|
||||
let c = chain(local_with_key(), &url);
|
||||
let p = c.resolve("local-key").await.expect("local resolves");
|
||||
assert_eq!(p.account_id, "op");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_key_falls_through_to_upstream() {
|
||||
let url = spawn_mock_upstream().await;
|
||||
let c = chain(local_with_key(), &url);
|
||||
let p = c.resolve("mesh-key").await.expect("upstream resolves");
|
||||
assert_eq!(p.account_id, "mesh-acct");
|
||||
assert_eq!(p.key_id, "mesh-key-1");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_everywhere_is_invalid_key() {
|
||||
let url = spawn_mock_upstream().await;
|
||||
let c = chain(local_with_key(), &url);
|
||||
match c.resolve("nope").await {
|
||||
Err(AuthError::InvalidKey) => {}
|
||||
other => panic!("expected InvalidKey, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn upstream_unreachable_fails_closed_as_unavailable() {
|
||||
// No mock — point at a dead port. A locally-unknown key must surface
|
||||
// Unavailable (→ 503), never InvalidKey (→ 401).
|
||||
let c = chain(local_with_key(), "http://127.0.0.1:1");
|
||||
match c.resolve("some-mesh-key").await {
|
||||
Err(AuthError::Unavailable { retry_after_secs }) => assert!(retry_after_secs > 0),
|
||||
other => panic!("expected Unavailable, got {other:?}"),
|
||||
}
|
||||
// A local key still resolves without touching upstream.
|
||||
assert_eq!(c.resolve("local-key").await.unwrap().account_id, "op");
|
||||
}
|
||||
41
crates/helexa-router/Cargo.toml
Normal file
@@ -0,0 +1,41 @@
|
||||
[package]
|
||||
name = "helexa-router"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[[bin]]
|
||||
name = "helexa-router"
|
||||
path = "src/main.rs"
|
||||
|
||||
[lib]
|
||||
name = "helexa_router"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[dependencies]
|
||||
cortex-core = { workspace = true }
|
||||
helexa-stream = { path = "../helexa-stream" }
|
||||
|
||||
tokio = { workspace = true }
|
||||
axum = { workspace = true }
|
||||
tower-http = { workspace = true }
|
||||
reqwest = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
figment = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
clap = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
# Jail (isolated cwd + env) for config tests.
|
||||
figment = { workspace = true, features = ["test"] }
|
||||
# Self-signed cert generation + a minimal HTTPS server for the outbound
|
||||
# TLS-pinning tests (#74).
|
||||
rcgen = "0.13"
|
||||
rustls = "0.23"
|
||||
tokio-rustls = "0.26"
|
||||
243
crates/helexa-router/src/catalogue.rs
Normal file
@@ -0,0 +1,243 @@
|
||||
//! Federation catalogue (#75) — the router's aggregate `/v1/models`.
|
||||
//!
|
||||
//! Presents the **deduped union** of every reachable cortex's `/v1/models`
|
||||
//! as the router's own catalogue, so an opencode client doing discovery
|
||||
//! against the router resolves the whole federation without knowing about
|
||||
//! operators or cortexes (resolves #61's "Router/discovery contract").
|
||||
//!
|
||||
//! Re-tiering: the fractal design is neuron ← cortex ← router. At the
|
||||
//! router tier the "nodes" are **cortexes**, so the merged entry's
|
||||
//! `feasible_on` / `locations` are rewritten to **operator names**, not the
|
||||
//! neuron names a cortex reports. That keeps the federation view honest
|
||||
//! ("served by these operators") without leaking each operator's internal
|
||||
//! topology (neuron names, per-device VRAM) to end users.
|
||||
//!
|
||||
//! Conflict resolution when operators advertise the same model with
|
||||
//! different enrichment:
|
||||
//! - **`limit`** → the *tightest* (smallest `context`), so a client never
|
||||
//! overflows the most-constrained operator that might serve it (same rule
|
||||
//! cortex uses across its neurons).
|
||||
//! - **`cost`** → the *cheapest* (lowest input, then output), the
|
||||
//! federation "from" price. Richer policy (a range, region/price-aware
|
||||
//! selection) couples to #68 and is left as a follow-up.
|
||||
|
||||
use crate::state::{CortexTopology, entry_feasible};
|
||||
use cortex_core::harness::{ModelCost, ModelLimit};
|
||||
use cortex_core::node::{CortexModelEntry, ModelLocation, ModelStatus};
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Build the federation catalogue: the deduped union of every reachable
|
||||
/// cortex's serveable models, merged across operators and sorted by id.
|
||||
pub fn aggregate_models(topology: &HashMap<String, CortexTopology>) -> Vec<CortexModelEntry> {
|
||||
// Iterate cortexes in name order so `feasible_on` / `locations` and the
|
||||
// limit/cost tie-breaks are deterministic regardless of map ordering.
|
||||
let mut cortexes: Vec<(&String, &CortexTopology)> = topology.iter().collect();
|
||||
cortexes.sort_by(|a, b| a.0.cmp(b.0));
|
||||
|
||||
let mut merged: HashMap<String, CortexModelEntry> = HashMap::new();
|
||||
for (cortex_name, t) in cortexes {
|
||||
if !t.reachable {
|
||||
continue;
|
||||
}
|
||||
for entry in t.models.values() {
|
||||
// Only surface models the cortex can actually serve — a
|
||||
// catalogue-only entry no neuron can host shouldn't appear in
|
||||
// the federation view.
|
||||
if !entry_feasible(entry) {
|
||||
continue;
|
||||
}
|
||||
merged
|
||||
.entry(entry.id.clone())
|
||||
.and_modify(|acc| merge_into(acc, cortex_name, entry))
|
||||
.or_insert_with(|| router_entry(cortex_name, entry));
|
||||
}
|
||||
}
|
||||
|
||||
let mut out: Vec<CortexModelEntry> = merged.into_values().collect();
|
||||
out.sort_by(|a, b| a.id.cmp(&b.id));
|
||||
out
|
||||
}
|
||||
|
||||
/// Seed a federation entry from the first cortex that serves the model,
|
||||
/// re-tiering `feasible_on` / `locations` to the operator name.
|
||||
fn router_entry(cortex: &str, e: &CortexModelEntry) -> CortexModelEntry {
|
||||
CortexModelEntry {
|
||||
id: e.id.clone(),
|
||||
object: "model".into(),
|
||||
created: e.created,
|
||||
owned_by: e.owned_by.clone(),
|
||||
loaded: e.loaded,
|
||||
feasible_on: vec![cortex.to_string()],
|
||||
locations: loaded_location(cortex, e),
|
||||
capabilities: e.capabilities.clone(),
|
||||
limit: e.limit.clone(),
|
||||
cost: e.cost.clone(),
|
||||
tool_call: e.tool_call,
|
||||
reasoning: e.reasoning,
|
||||
}
|
||||
}
|
||||
|
||||
/// Fold another cortex's view of the same model into the merged entry.
|
||||
fn merge_into(acc: &mut CortexModelEntry, cortex: &str, e: &CortexModelEntry) {
|
||||
acc.loaded |= e.loaded;
|
||||
acc.feasible_on.push(cortex.to_string());
|
||||
acc.locations.extend(loaded_location(cortex, e));
|
||||
for cap in &e.capabilities {
|
||||
if !acc.capabilities.contains(cap) {
|
||||
acc.capabilities.push(cap.clone());
|
||||
}
|
||||
}
|
||||
acc.tool_call |= e.tool_call;
|
||||
acc.reasoning |= e.reasoning;
|
||||
acc.limit = tightest_limit(acc.limit.take(), e.limit.clone());
|
||||
acc.cost = cheapest_cost(acc.cost.take(), e.cost.clone());
|
||||
}
|
||||
|
||||
/// A single cortex-tier location when the model is loaded at that operator;
|
||||
/// empty when only cold-loadable. Neuron-level VRAM is deliberately dropped.
|
||||
fn loaded_location(cortex: &str, e: &CortexModelEntry) -> Vec<ModelLocation> {
|
||||
if e.loaded {
|
||||
vec![ModelLocation {
|
||||
node: cortex.to_string(),
|
||||
status: ModelStatus::Loaded,
|
||||
vram_estimate_mb: None,
|
||||
}]
|
||||
} else {
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Smaller `context` wins — never advertise more headroom than the
|
||||
/// most-constrained operator can honour.
|
||||
fn tightest_limit(a: Option<ModelLimit>, b: Option<ModelLimit>) -> Option<ModelLimit> {
|
||||
match (a, b) {
|
||||
(None, x) | (x, None) => x,
|
||||
(Some(a), Some(b)) => Some(if b.context < a.context { b } else { a }),
|
||||
}
|
||||
}
|
||||
|
||||
/// Cheapest by (input, output) price — the federation "from" price.
|
||||
fn cheapest_cost(a: Option<ModelCost>, b: Option<ModelCost>) -> Option<ModelCost> {
|
||||
match (a, b) {
|
||||
(None, x) | (x, None) => x,
|
||||
(Some(a), Some(b)) => Some(if (b.input, b.output) < (a.input, a.output) {
|
||||
b
|
||||
} else {
|
||||
a
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::state::CortexTopology;
|
||||
|
||||
fn entry(id: &str, loaded: bool, feasible: bool) -> CortexModelEntry {
|
||||
CortexModelEntry {
|
||||
id: id.into(),
|
||||
object: "model".into(),
|
||||
created: 0,
|
||||
owned_by: "helexa".into(),
|
||||
loaded,
|
||||
feasible_on: if feasible || loaded {
|
||||
vec!["some-neuron".into()]
|
||||
} else {
|
||||
vec![]
|
||||
},
|
||||
locations: vec![],
|
||||
capabilities: vec![],
|
||||
limit: None,
|
||||
cost: None,
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn cortex(reachable: bool, entries: Vec<CortexModelEntry>) -> CortexTopology {
|
||||
CortexTopology {
|
||||
reachable,
|
||||
consecutive_failures: 0,
|
||||
last_poll: None,
|
||||
healthy_nodes: 1,
|
||||
total_nodes: 1,
|
||||
models: entries.into_iter().map(|e| (e.id.clone(), e)).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dedupes_and_merges_availability_across_cortexes() {
|
||||
let mut topo = HashMap::new();
|
||||
// c-a: model loaded. c-b: same model only cold-loadable.
|
||||
topo.insert("c-a".into(), cortex(true, vec![entry("m", true, true)]));
|
||||
topo.insert("c-b".into(), cortex(true, vec![entry("m", false, true)]));
|
||||
|
||||
let out = aggregate_models(&topo);
|
||||
assert_eq!(out.len(), 1, "duplicate model id collapses to one");
|
||||
let m = &out[0];
|
||||
assert!(m.loaded, "loaded somewhere → loaded");
|
||||
// feasible_on re-tiered to operator names, both present, sorted.
|
||||
assert_eq!(m.feasible_on, vec!["c-a".to_string(), "c-b".to_string()]);
|
||||
// Only the loaded operator contributes a location, named by operator.
|
||||
assert_eq!(m.locations.len(), 1);
|
||||
assert_eq!(m.locations[0].node, "c-a");
|
||||
assert_eq!(m.locations[0].vram_estimate_mb, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unreachable_cortex_is_excluded() {
|
||||
let mut topo = HashMap::new();
|
||||
topo.insert("up".into(), cortex(true, vec![entry("m", true, true)]));
|
||||
topo.insert(
|
||||
"down".into(),
|
||||
cortex(false, vec![entry("other", true, true)]),
|
||||
);
|
||||
let out = aggregate_models(&topo);
|
||||
assert_eq!(out.len(), 1);
|
||||
assert_eq!(out[0].id, "m");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn catalogue_only_infeasible_entries_are_hidden() {
|
||||
let mut topo = HashMap::new();
|
||||
topo.insert("c".into(), cortex(true, vec![entry("ghost", false, false)]));
|
||||
assert!(aggregate_models(&topo).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_tightest_limit_and_cheapest_cost() {
|
||||
let mut a = entry("m", true, true);
|
||||
a.limit = Some(ModelLimit {
|
||||
context: 32_768,
|
||||
input: None,
|
||||
output: 4096,
|
||||
});
|
||||
a.cost = Some(ModelCost {
|
||||
input: 0.50,
|
||||
output: 1.50,
|
||||
cache_read: None,
|
||||
cache_write: None,
|
||||
});
|
||||
let mut b = entry("m", true, true);
|
||||
b.limit = Some(ModelLimit {
|
||||
context: 16_384, // tighter
|
||||
input: None,
|
||||
output: 4096,
|
||||
});
|
||||
b.cost = Some(ModelCost {
|
||||
input: 0.20, // cheaper
|
||||
output: 0.80,
|
||||
cache_read: None,
|
||||
cache_write: None,
|
||||
});
|
||||
|
||||
let mut topo = HashMap::new();
|
||||
topo.insert("c-a".into(), cortex(true, vec![a]));
|
||||
topo.insert("c-b".into(), cortex(true, vec![b]));
|
||||
|
||||
let out = aggregate_models(&topo);
|
||||
assert_eq!(out.len(), 1);
|
||||
assert_eq!(out[0].limit.as_ref().unwrap().context, 16_384);
|
||||
assert_eq!(out[0].cost.as_ref().unwrap().input, 0.20);
|
||||
}
|
||||
}
|
||||
100
crates/helexa-router/src/config.rs
Normal file
@@ -0,0 +1,100 @@
|
||||
use figment::{
|
||||
Figment,
|
||||
providers::{Env, Format, Toml},
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::Path;
|
||||
|
||||
/// Top-level `helexa-router` configuration.
|
||||
///
|
||||
/// Loaded from TOML with `HELEXA_ROUTER_`-prefixed env overrides (using
|
||||
/// `__` as the nesting separator), matching the cortex/neuron convention.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RouterConfig {
|
||||
pub router: RouterSettings,
|
||||
/// Downstream cortex endpoints the router can dispatch to. The skeleton
|
||||
/// (#70) only loads these; capacity/catalogue polling (#72) and
|
||||
/// capacity-aware dispatch (#73) consume them later.
|
||||
#[serde(default)]
|
||||
pub cortexes: Vec<CortexEndpoint>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RouterSettings {
|
||||
/// Address to listen on for the inbound API (e.g. "0.0.0.0:8088").
|
||||
///
|
||||
/// Plaintext only — operator/edge nginx terminates client TLS in front
|
||||
/// of the router (see #69's TLS posture). The router never owns an
|
||||
/// inbound TLS listener.
|
||||
pub listen: String,
|
||||
/// How often (seconds) the background poller refreshes each cortex's
|
||||
/// health + `/v1/models` topology (#72). Defaults to 10s, matching the
|
||||
/// cortex↔neuron poll cadence one tier down.
|
||||
#[serde(default = "default_poll_interval_secs")]
|
||||
pub poll_interval_secs: u64,
|
||||
/// This router instance's region (e.g. "eu-west"). When set, dispatch
|
||||
/// (#73) prefers cortexes whose `region` matches, before falling back to
|
||||
/// any feasible cortex. `None` → no geo affinity.
|
||||
#[serde(default)]
|
||||
pub region: Option<String>,
|
||||
}
|
||||
|
||||
fn default_poll_interval_secs() -> u64 {
|
||||
10
|
||||
}
|
||||
|
||||
/// One downstream cortex the router may proxy to. The router verifies the
|
||||
/// cortex's outbound TLS cert (#74) and routes on capacity (#73); it holds
|
||||
/// no entitlement logic of its own and forwards the client bearer verbatim.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct CortexEndpoint {
|
||||
/// Human-readable label (e.g. "lair-cafe").
|
||||
pub name: String,
|
||||
/// Base URL of the cortex gateway (e.g. "https://cortex.example.com").
|
||||
pub endpoint: String,
|
||||
/// Optional region tag (e.g. "eu-west") for geo affinity in dispatch
|
||||
/// (#73). `None` → no region preference applies to this cortex.
|
||||
#[serde(default)]
|
||||
pub region: Option<String>,
|
||||
/// Path to a PEM trust anchor that **enrols** this cortex (#74): the
|
||||
/// expected CA (or self-signed cert) the cortex's TLS cert must chain
|
||||
/// to. When set on an `https://` endpoint, the router builds a client
|
||||
/// that trusts **only** this anchor (platform roots disabled), so the
|
||||
/// outbound router→cortex hop — which carries the client's bearer —
|
||||
/// reaches a cert the router was told to expect, and a rogue endpoint
|
||||
/// presenting any other (even publicly-valid) cert is rejected at the
|
||||
/// TLS handshake. A rejected handshake surfaces as a connection error,
|
||||
/// which the poller (#72) already treats as unreachable → excluded.
|
||||
///
|
||||
/// `None` → standard platform-root validation (use for cortexes behind
|
||||
/// a publicly-trusted cert, or plaintext `http://` on a private network
|
||||
/// where the WireGuard mesh is the trust boundary).
|
||||
#[serde(default)]
|
||||
pub tls_ca: Option<String>,
|
||||
}
|
||||
|
||||
impl RouterConfig {
|
||||
/// Load configuration from a TOML file, with environment variable
|
||||
/// overrides prefixed with `HELEXA_ROUTER_` and `__` as the separator
|
||||
/// (e.g. `HELEXA_ROUTER_ROUTER__LISTEN=0.0.0.0:8088`).
|
||||
pub fn load(path: impl AsRef<Path>) -> Result<Self, Box<figment::Error>> {
|
||||
Figment::new()
|
||||
.merge(Toml::file(path))
|
||||
.merge(Env::prefixed("HELEXA_ROUTER_").split("__"))
|
||||
.extract()
|
||||
.map_err(Box::new)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for RouterConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
router: RouterSettings {
|
||||
listen: "0.0.0.0:8088".into(),
|
||||
poll_interval_secs: default_poll_interval_secs(),
|
||||
region: None,
|
||||
},
|
||||
cortexes: vec![],
|
||||
}
|
||||
}
|
||||
}
|
||||
221
crates/helexa-router/src/dispatch.rs
Normal file
@@ -0,0 +1,221 @@
|
||||
//! Capacity-aware dispatch (#73) — the router's data path.
|
||||
//!
|
||||
//! Given an inbound request's `model`, pick a reachable cortex that can
|
||||
//! serve it (preferring warm/loaded, region-affine, higher-headroom),
|
||||
//! forward the client's bearer **unchanged** (auth stays at cortex), and
|
||||
//! stream the response back verbatim via the shared [`helexa_stream`]
|
||||
//! module. Cortex's #63-shaped rejections (`429 rate_limit_exceeded`,
|
||||
//! `400 context_length_exceeded`, …) pass through untouched. Transport
|
||||
//! failures fail over to the next feasible cortex; a genuine HTTP response —
|
||||
//! any status — is returned as-is and never retried away.
|
||||
//!
|
||||
//! The router holds **no entitlement logic**: it routes on capacity, not
|
||||
//! budget.
|
||||
|
||||
use crate::config::CortexEndpoint;
|
||||
use crate::error::envelope_response;
|
||||
use crate::state::RouterState;
|
||||
use axum::body::Bytes;
|
||||
use axum::http::HeaderMap;
|
||||
use axum::response::Response;
|
||||
use cortex_core::error_envelope::OpenAiError;
|
||||
use helexa_stream::{ChunkObserver, StreamError};
|
||||
use std::cmp::Reverse;
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Retry-After hint (seconds) on the router's own transient rejections.
|
||||
const RETRY_AFTER_SECS: u64 = 5;
|
||||
|
||||
/// Outcome of choosing where to send a request.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum Selection {
|
||||
/// Feasible reachable cortexes, best-first (failover order).
|
||||
Candidates(Vec<CortexEndpoint>),
|
||||
/// Some cortex knows the model but none are reachable right now → 503.
|
||||
NoReachableCapacity,
|
||||
/// No configured cortex serves the model at all → 404.
|
||||
UnknownModel,
|
||||
}
|
||||
|
||||
/// Rank the reachable cortexes that can serve `model`, best-first.
|
||||
///
|
||||
/// Ordering (each a tie-break for the next): loaded/warm before cold-loadable
|
||||
/// · region match before not · more healthy nodes before fewer · name for
|
||||
/// determinism.
|
||||
pub async fn select_cortexes(state: &RouterState, model: &str) -> Selection {
|
||||
let topo = state.topology.read().await;
|
||||
let by_name: HashMap<&str, &CortexEndpoint> = state
|
||||
.cortexes
|
||||
.iter()
|
||||
.map(|c| (c.name.as_str(), c))
|
||||
.collect();
|
||||
|
||||
let mut ranked: Vec<Ranked> = Vec::new();
|
||||
let mut known_anywhere = false;
|
||||
|
||||
for (name, t) in topo.iter() {
|
||||
let Some(entry) = t.models.get(model) else {
|
||||
continue;
|
||||
};
|
||||
if !crate::state::entry_feasible(entry) {
|
||||
continue;
|
||||
}
|
||||
// Known even via an unreachable cortex's last-good poll — lets us
|
||||
// tell "temporarily down" (503) from "nobody serves it" (404).
|
||||
known_anywhere = true;
|
||||
if !t.reachable {
|
||||
continue;
|
||||
}
|
||||
let Some(ep) = by_name.get(name.as_str()) else {
|
||||
continue;
|
||||
};
|
||||
let region_match = match (&state.region, &ep.region) {
|
||||
(Some(r), Some(cr)) => r == cr,
|
||||
_ => false,
|
||||
};
|
||||
ranked.push(Ranked {
|
||||
loaded: entry.loaded,
|
||||
region_match,
|
||||
healthy_nodes: t.healthy_nodes,
|
||||
endpoint: (*ep).clone(),
|
||||
});
|
||||
}
|
||||
|
||||
if ranked.is_empty() {
|
||||
return if known_anywhere {
|
||||
Selection::NoReachableCapacity
|
||||
} else {
|
||||
Selection::UnknownModel
|
||||
};
|
||||
}
|
||||
|
||||
ranked.sort_by(|a, b| {
|
||||
// false < true, so negate the "good" booleans to sort good first.
|
||||
(
|
||||
!a.loaded,
|
||||
!a.region_match,
|
||||
Reverse(a.healthy_nodes),
|
||||
&a.endpoint.name,
|
||||
)
|
||||
.cmp(&(
|
||||
!b.loaded,
|
||||
!b.region_match,
|
||||
Reverse(b.healthy_nodes),
|
||||
&b.endpoint.name,
|
||||
))
|
||||
});
|
||||
|
||||
Selection::Candidates(ranked.into_iter().map(|r| r.endpoint).collect())
|
||||
}
|
||||
|
||||
struct Ranked {
|
||||
loaded: bool,
|
||||
region_match: bool,
|
||||
healthy_nodes: u32,
|
||||
endpoint: CortexEndpoint,
|
||||
}
|
||||
|
||||
/// Proxy an inbound inference request to a capacity-bearing cortex.
|
||||
///
|
||||
/// `path` is the inference path to forward to (same on the cortex, e.g.
|
||||
/// `/v1/chat/completions`). The body is parsed only to extract `model`.
|
||||
pub async fn dispatch(
|
||||
state: &RouterState,
|
||||
path: &str,
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
) -> Response {
|
||||
let Some(model) = extract_model(&body) else {
|
||||
return envelope_response(OpenAiError::new(
|
||||
400,
|
||||
"invalid_request_error",
|
||||
"missing_model_field",
|
||||
"missing 'model' field in request body",
|
||||
));
|
||||
};
|
||||
|
||||
let candidates = match select_cortexes(state, &model).await {
|
||||
Selection::Candidates(c) => c,
|
||||
Selection::UnknownModel => {
|
||||
return envelope_response(
|
||||
OpenAiError::new(
|
||||
404,
|
||||
"invalid_request_error",
|
||||
"model_not_found",
|
||||
format!("no operator serves model '{model}'"),
|
||||
)
|
||||
.with_param("model"),
|
||||
);
|
||||
}
|
||||
Selection::NoReachableCapacity => {
|
||||
return envelope_response(OpenAiError::service_unavailable(
|
||||
format!("model '{model}' is temporarily unavailable on all operators"),
|
||||
Some(RETRY_AFTER_SECS),
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
// Try candidates in order, failing over only on transport errors. A
|
||||
// genuine HTTP response (any status — including cortex's #63 429/400)
|
||||
// is returned verbatim and never retried away.
|
||||
for ep in &candidates {
|
||||
// A candidate whose pinned TLS client failed to build (#74) is
|
||||
// disabled — skip it and fail over, same as an unreachable cortex.
|
||||
let Some(client) = state.client_for(&ep.name) else {
|
||||
tracing::warn!(cortex = %ep.name, "no TLS client (disabled); skipping candidate");
|
||||
continue;
|
||||
};
|
||||
let url = format!("{}{}", ep.endpoint, path);
|
||||
tracing::info!(cortex = %ep.name, url = %url, model = %model, "dispatching");
|
||||
match helexa_stream::forward_streaming(
|
||||
client,
|
||||
&url,
|
||||
headers.clone(),
|
||||
body.clone(),
|
||||
NoopObserver,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(resp) => return resp,
|
||||
Err(StreamError::Upstream(e)) => {
|
||||
tracing::warn!(
|
||||
cortex = %ep.name,
|
||||
url = %url,
|
||||
error = %e,
|
||||
"cortex unreachable; failing over"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
Err(StreamError::ResponseBuild(msg)) => {
|
||||
tracing::error!(cortex = %ep.name, error = %msg, "failed to build proxied response");
|
||||
return envelope_response(OpenAiError::without_code(
|
||||
500,
|
||||
"api_error",
|
||||
"failed to build proxied response",
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Every feasible cortex failed to connect.
|
||||
tracing::warn!(model = %model, tried = candidates.len(), "all feasible operators unreachable");
|
||||
envelope_response(OpenAiError::service_unavailable(
|
||||
format!("all operators able to serve '{model}' are unreachable"),
|
||||
Some(RETRY_AFTER_SECS),
|
||||
))
|
||||
}
|
||||
|
||||
/// Pull the `model` field out of a request body without re-serialising it.
|
||||
fn extract_model(body: &Bytes) -> Option<String> {
|
||||
let v: serde_json::Value = serde_json::from_slice(body).ok()?;
|
||||
v.get("model")?.as_str().map(str::to_string)
|
||||
}
|
||||
|
||||
/// The router proxies bytes verbatim and keeps no per-request policy, so it
|
||||
/// needs no observation hooks. (Token metrics/metering stay at cortex.)
|
||||
struct NoopObserver;
|
||||
|
||||
impl ChunkObserver for NoopObserver {
|
||||
fn observe(&mut self, _chunk: &[u8]) {}
|
||||
fn finish(&mut self) {}
|
||||
}
|
||||
27
crates/helexa-router/src/error.rs
Normal file
@@ -0,0 +1,27 @@
|
||||
//! Router adapter from the shared, axum-agnostic
|
||||
//! [`cortex_core::error_envelope::OpenAiError`] (#60/#63) to an axum
|
||||
//! [`Response`], setting `Retry-After` when the envelope carries one.
|
||||
//!
|
||||
//! cortex-core owns the envelope shape; this is the only place the router
|
||||
//! crosses from that data into axum. Mirrors cortex-gateway's adapter so
|
||||
//! the router's own rejections (no feasible operator, all unreachable) are
|
||||
//! the same #63-shaped envelopes clients already understand — distinct from
|
||||
//! cortex's rejections, which the router proxies through verbatim.
|
||||
|
||||
use axum::http::{HeaderValue, StatusCode, header};
|
||||
use axum::response::{IntoResponse, Json, Response};
|
||||
use cortex_core::error_envelope::OpenAiError;
|
||||
|
||||
/// Render an [`OpenAiError`] as an axum response (status + JSON envelope +
|
||||
/// optional `Retry-After`).
|
||||
pub fn envelope_response(err: OpenAiError) -> Response {
|
||||
let status = StatusCode::from_u16(err.status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
|
||||
let retry_after = err.retry_after_secs;
|
||||
let mut response = (status, Json(err.body())).into_response();
|
||||
if let Some(secs) = retry_after
|
||||
&& let Ok(value) = HeaderValue::from_str(&secs.to_string())
|
||||
{
|
||||
response.headers_mut().insert(header::RETRY_AFTER, value);
|
||||
}
|
||||
response
|
||||
}
|
||||
89
crates/helexa-router/src/handlers.rs
Normal file
@@ -0,0 +1,89 @@
|
||||
use crate::state::RouterState;
|
||||
use crate::{catalogue, dispatch};
|
||||
use axum::body::Bytes;
|
||||
use axum::http::HeaderMap;
|
||||
use axum::response::Response;
|
||||
use axum::{Json, Router, extract::State, routing::get, routing::post};
|
||||
use serde_json::{Value, json};
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Routes served by the router. Inference paths are capacity-aware-dispatched
|
||||
/// (#73) to a downstream cortex; `/health` and a stub `/v1/models` are local.
|
||||
pub fn api_routes() -> Router<Arc<RouterState>> {
|
||||
Router::new()
|
||||
.route("/v1/chat/completions", post(chat_completions))
|
||||
.route("/v1/completions", post(completions))
|
||||
.route("/v1/responses", post(responses))
|
||||
.route("/v1/messages", post(messages))
|
||||
.route("/v1/models", get(list_models))
|
||||
.route("/health", get(health))
|
||||
.route("/", get(health))
|
||||
}
|
||||
|
||||
// ── Inference paths — forwarded verbatim to a chosen cortex ──────────
|
||||
//
|
||||
// Each handler dispatches to the same path on a capacity-bearing cortex.
|
||||
// The body is parsed only to read `model`; the bearer and bytes are
|
||||
// forwarded unchanged, and the SSE response streams back verbatim.
|
||||
|
||||
async fn chat_completions(
|
||||
State(state): State<Arc<RouterState>>,
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
) -> Response {
|
||||
dispatch::dispatch(&state, "/v1/chat/completions", headers, body).await
|
||||
}
|
||||
|
||||
async fn completions(
|
||||
State(state): State<Arc<RouterState>>,
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
) -> Response {
|
||||
dispatch::dispatch(&state, "/v1/completions", headers, body).await
|
||||
}
|
||||
|
||||
async fn responses(
|
||||
State(state): State<Arc<RouterState>>,
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
) -> Response {
|
||||
dispatch::dispatch(&state, "/v1/responses", headers, body).await
|
||||
}
|
||||
|
||||
async fn messages(
|
||||
State(state): State<Arc<RouterState>>,
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
) -> Response {
|
||||
dispatch::dispatch(&state, "/v1/messages", headers, body).await
|
||||
}
|
||||
|
||||
/// `GET /health` — router liveness plus a summary of downstream cortex
|
||||
/// reachability from the topology poller (#72). `status` reflects the
|
||||
/// router process itself (always `ok` if it answers); downstream health is
|
||||
/// the informational `cortexes` block, so a fully-degraded fleet doesn't
|
||||
/// make the router look dead to its own liveness probe.
|
||||
async fn health(State(state): State<Arc<RouterState>>) -> Json<Value> {
|
||||
let topo = state.topology.read().await;
|
||||
let reachable = topo.values().filter(|t| t.reachable).count();
|
||||
Json(json!({
|
||||
"status": "ok",
|
||||
"cortexes": {
|
||||
"configured": state.cortexes.len(),
|
||||
"reachable": reachable,
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
/// `GET /v1/models` — the federation catalogue (#75): the deduped union of
|
||||
/// every reachable cortex's `/v1/models`, so a client doing discovery
|
||||
/// against the router resolves the whole federation without knowing about
|
||||
/// operators or cortexes.
|
||||
async fn list_models(State(state): State<Arc<RouterState>>) -> Json<Value> {
|
||||
let topo = state.topology.read().await;
|
||||
let data: Vec<Value> = catalogue::aggregate_models(&topo)
|
||||
.iter()
|
||||
.map(|e| json!(e))
|
||||
.collect();
|
||||
Json(json!({ "object": "list", "data": data }))
|
||||
}
|
||||
60
crates/helexa-router/src/lib.rs
Normal file
@@ -0,0 +1,60 @@
|
||||
//! helexa-router — public multi-operator ingress proxy (router.helexa.ai).
|
||||
//!
|
||||
//! The router is the data-plane *ingress* tier: a geo-distributed,
|
||||
//! capacity-aware, OpenAI/Anthropic-compatible reverse proxy in front of
|
||||
//! many operator-run cortexes ("cortex-of-cortexes"). End users configure
|
||||
//! one `baseURL` and the router forwards their request to a cortex with
|
||||
//! capacity, proxying #63-shaped rejections back verbatim.
|
||||
//!
|
||||
//! It holds **zero entitlement logic** — auth/budget stays at cortex
|
||||
//! (epic #47); the router forwards the client bearer unchanged and routes
|
||||
//! on capacity (epic #69). A background [`poller`] keeps a live
|
||||
//! per-cortex topology (#72) that the dispatcher (#73) will route on.
|
||||
|
||||
pub mod catalogue;
|
||||
pub mod config;
|
||||
pub mod dispatch;
|
||||
pub mod error;
|
||||
pub mod handlers;
|
||||
pub mod poller;
|
||||
pub mod state;
|
||||
|
||||
use anyhow::Result;
|
||||
use config::RouterConfig;
|
||||
use std::sync::Arc;
|
||||
use tower_http::cors::CorsLayer;
|
||||
use tower_http::trace::TraceLayer;
|
||||
|
||||
/// Build the axum application: handlers + CORS + tracing. No auth layer —
|
||||
/// the router asserts no identity of its own and forwards the client bearer
|
||||
/// to the downstream cortex, which authenticates it (#69).
|
||||
pub fn build_app(state: Arc<state::RouterState>) -> axum::Router {
|
||||
axum::Router::new()
|
||||
.merge(handlers::api_routes())
|
||||
.layer(CorsLayer::permissive())
|
||||
.layer(TraceLayer::new_for_http())
|
||||
.with_state(state)
|
||||
}
|
||||
|
||||
/// Start the router: build state from config and bind the plaintext HTTP
|
||||
/// listener. TLS is terminated by edge nginx ahead of this process.
|
||||
pub async fn run(config: RouterConfig) -> Result<()> {
|
||||
let state = Arc::new(state::RouterState::from_config(&config));
|
||||
|
||||
// Background topology poller (#72): refresh each cortex's health +
|
||||
// catalogue so routing decisions see live capacity.
|
||||
let poller_state = Arc::clone(&state);
|
||||
tokio::spawn(async move {
|
||||
poller::poll_loop(poller_state).await;
|
||||
});
|
||||
|
||||
let app = build_app(Arc::clone(&state));
|
||||
|
||||
let listen_addr = config.router.listen.parse::<std::net::SocketAddr>()?;
|
||||
tracing::info!("helexa-router listening on {listen_addr}");
|
||||
|
||||
let listener = tokio::net::TcpListener::bind(listen_addr).await?;
|
||||
axum::serve(listener, app).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
52
crates/helexa-router/src/main.rs
Normal file
@@ -0,0 +1,52 @@
|
||||
use anyhow::Result;
|
||||
use clap::{Parser, Subcommand};
|
||||
use helexa_router::config::RouterConfig;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "helexa-router")]
|
||||
#[command(about = "Public multi-operator ingress proxy for helexa")]
|
||||
#[command(version)]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
command: Commands,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Commands {
|
||||
/// Start the router server.
|
||||
Serve {
|
||||
/// Path to the router config file.
|
||||
#[arg(short, long, default_value = "helexa-router.toml")]
|
||||
config: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new("info,helexa_router=debug")),
|
||||
)
|
||||
.init();
|
||||
|
||||
let cli = Cli::parse();
|
||||
|
||||
match cli.command {
|
||||
Commands::Serve { config } => {
|
||||
let cfg = RouterConfig::load(&config)
|
||||
.map_err(|e| anyhow::anyhow!("failed to load config from '{config}': {e}"))?;
|
||||
|
||||
tracing::info!(
|
||||
cortexes = cfg.cortexes.len(),
|
||||
listen = %cfg.router.listen,
|
||||
"starting helexa-router"
|
||||
);
|
||||
|
||||
helexa_router::run(cfg).await?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
150
crates/helexa-router/src/poller.rs
Normal file
@@ -0,0 +1,150 @@
|
||||
//! Background poller that refreshes the multi-operator topology (#72).
|
||||
//!
|
||||
//! The same pattern as cortex↔neuron, one tier up: periodically poll each
|
||||
//! configured cortex's `GET /v1/models` (catalogue × topology feasibility +
|
||||
//! loaded state) and `GET /health` (coarse node-health/load), building the
|
||||
//! live map the dispatcher (#73) routes on. An unreachable or erroring
|
||||
//! cortex is debounced over [`POLL_FAILURE_THRESHOLD`] consecutive misses,
|
||||
//! then flipped unhealthy and excluded from routing; it recovers on the
|
||||
//! next successful poll.
|
||||
|
||||
use crate::state::RouterState;
|
||||
use chrono::Utc;
|
||||
use cortex_core::node::CortexModelEntry;
|
||||
use serde::Deserialize;
|
||||
use std::time::Duration;
|
||||
|
||||
/// Per-cortex HTTP timeout for each poll request.
|
||||
const POLL_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
/// Consecutive failed polls before a cortex is marked unreachable. Mirrors
|
||||
/// cortex's neuron-poll debounce: a single blip (a busy cortex briefly slow
|
||||
/// to answer) can't yank it — and all its models — out of routing.
|
||||
pub const POLL_FAILURE_THRESHOLD: u32 = 3;
|
||||
|
||||
/// cortex's `/v1/models` envelope — `{ "object": "list", "data": [...] }`.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ModelsEnvelope {
|
||||
#[serde(default)]
|
||||
data: Vec<CortexModelEntry>,
|
||||
}
|
||||
|
||||
/// The subset of cortex's `/health` the router reads.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct CortexHealth {
|
||||
nodes: CortexHealthNodes,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct CortexHealthNodes {
|
||||
healthy: u32,
|
||||
total: u32,
|
||||
}
|
||||
|
||||
/// Run forever, polling all cortexes on the configured interval.
|
||||
pub async fn poll_loop(state: std::sync::Arc<RouterState>) {
|
||||
loop {
|
||||
poll_once(&state).await;
|
||||
tokio::time::sleep(state.poll_interval).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Poll every configured cortex once. Public for testing.
|
||||
pub async fn poll_once(state: &RouterState) {
|
||||
for cortex in &state.cortexes {
|
||||
poll_cortex(state, &cortex.name, &cortex.endpoint).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Poll one cortex: refresh its model map from `/v1/models`, then its node
|
||||
/// health from `/health`. A `/v1/models` failure debounces toward
|
||||
/// unreachable; the `/health` poll is best-effort and never flips
|
||||
/// reachability on its own (a cortex serving `/v1/models` is routable even
|
||||
/// if `/health` momentarily isn't).
|
||||
async fn poll_cortex(state: &RouterState, name: &str, endpoint: &str) {
|
||||
// A cortex whose pinned TLS client failed to build (#74) is disabled:
|
||||
// there is no client to poll with, so it stays unreachable.
|
||||
let Some(client) = state.client_for(name) else {
|
||||
let mut topo = state.topology.write().await;
|
||||
if let Some(entry) = topo.get_mut(name) {
|
||||
entry.consecutive_failures = entry.consecutive_failures.saturating_add(1);
|
||||
entry.reachable = false;
|
||||
}
|
||||
tracing::warn!(cortex = name, "no TLS client (disabled); skipping poll");
|
||||
return;
|
||||
};
|
||||
|
||||
let models = fetch_models(client, endpoint).await;
|
||||
|
||||
let mut topo = state.topology.write().await;
|
||||
let Some(entry) = topo.get_mut(name) else {
|
||||
return; // not a configured cortex (shouldn't happen)
|
||||
};
|
||||
|
||||
match models {
|
||||
Ok(models) => {
|
||||
entry.models = models.into_iter().map(|m| (m.id.clone(), m)).collect();
|
||||
entry.reachable = true;
|
||||
entry.consecutive_failures = 0;
|
||||
entry.last_poll = Some(Utc::now());
|
||||
tracing::debug!(cortex = name, models = entry.models.len(), "poll ok");
|
||||
}
|
||||
Err(reason) => {
|
||||
entry.consecutive_failures = entry.consecutive_failures.saturating_add(1);
|
||||
if entry.consecutive_failures >= POLL_FAILURE_THRESHOLD {
|
||||
entry.reachable = false;
|
||||
}
|
||||
tracing::warn!(
|
||||
cortex = name,
|
||||
failures = entry.consecutive_failures,
|
||||
reachable = entry.reachable,
|
||||
reason,
|
||||
"cortex poll failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
drop(topo);
|
||||
|
||||
// Best-effort health (node counts). Never flips reachability.
|
||||
if let Some((healthy, total)) = fetch_health(client, endpoint).await {
|
||||
let mut topo = state.topology.write().await;
|
||||
if let Some(entry) = topo.get_mut(name) {
|
||||
entry.healthy_nodes = healthy;
|
||||
entry.total_nodes = total;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// GET `/v1/models`, returning the parsed entries or a short failure reason.
|
||||
async fn fetch_models(
|
||||
client: &reqwest::Client,
|
||||
endpoint: &str,
|
||||
) -> Result<Vec<CortexModelEntry>, &'static str> {
|
||||
let url = format!("{endpoint}/v1/models");
|
||||
let resp = client
|
||||
.get(&url)
|
||||
.timeout(POLL_TIMEOUT)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|_| "unreachable")?;
|
||||
if !resp.status().is_success() {
|
||||
return Err("non-success status");
|
||||
}
|
||||
let envelope = resp
|
||||
.json::<ModelsEnvelope>()
|
||||
.await
|
||||
.map_err(|_| "bad json")?;
|
||||
Ok(envelope.data)
|
||||
}
|
||||
|
||||
/// GET `/health`, returning `(healthy, total)` node counts. `None` on any
|
||||
/// failure — the caller leaves the previous counts in place.
|
||||
async fn fetch_health(client: &reqwest::Client, endpoint: &str) -> Option<(u32, u32)> {
|
||||
let url = format!("{endpoint}/health");
|
||||
let resp = client.get(&url).timeout(POLL_TIMEOUT).send().await.ok()?;
|
||||
if !resp.status().is_success() {
|
||||
return None;
|
||||
}
|
||||
let health = resp.json::<CortexHealth>().await.ok()?;
|
||||
Some((health.nodes.healthy, health.nodes.total))
|
||||
}
|
||||
144
crates/helexa-router/src/state.rs
Normal file
@@ -0,0 +1,144 @@
|
||||
use crate::config::{CortexEndpoint, RouterConfig};
|
||||
use chrono::{DateTime, Utc};
|
||||
use cortex_core::node::CortexModelEntry;
|
||||
use std::collections::HashMap;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
/// Shared router state: the configured cortex list plus the live topology
|
||||
/// map the poller (#72) maintains and the dispatcher (#73) will route on.
|
||||
///
|
||||
/// This is the router tier of the fractal neuron ← cortex ← router design:
|
||||
/// just as cortex polls each neuron for capacity/catalogue, the router
|
||||
/// polls each cortex's `/health` + `/v1/models`.
|
||||
#[derive(Debug)]
|
||||
pub struct RouterState {
|
||||
/// Downstream cortex endpoints, as configured.
|
||||
pub cortexes: Vec<CortexEndpoint>,
|
||||
/// Per-cortex HTTP client, keyed by cortex name (#74). A cortex enrolled
|
||||
/// with a `tls_ca` gets a client that trusts only that anchor; others
|
||||
/// get a default client. A cortex whose `tls_ca` failed to load is
|
||||
/// **absent** here — `client_for` returns `None` and it is never
|
||||
/// polled or routed to (fail closed: a misconfigured pin must not
|
||||
/// silently fall back to unpinned TLS).
|
||||
clients: HashMap<String, reqwest::Client>,
|
||||
/// This router instance's region, for dispatch geo affinity (#73).
|
||||
pub region: Option<String>,
|
||||
/// How often the poller refreshes the topology.
|
||||
pub poll_interval: Duration,
|
||||
/// Live per-cortex topology, keyed by cortex name. Pre-populated from
|
||||
/// config (every configured cortex present, `reachable = false`) so the
|
||||
/// poller and handlers always find an entry; the poller flips
|
||||
/// reachability and fills the model map.
|
||||
pub topology: RwLock<HashMap<String, CortexTopology>>,
|
||||
}
|
||||
|
||||
/// Live view of one downstream cortex, refreshed each poll.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct CortexTopology {
|
||||
/// Whether the cortex is currently routable. Flipped `false` only after
|
||||
/// [`crate::poller::POLL_FAILURE_THRESHOLD`] consecutive failed polls
|
||||
/// (debounces transient blips); restored on the next successful poll.
|
||||
pub reachable: bool,
|
||||
/// Consecutive failed polls; reset to 0 on success.
|
||||
pub consecutive_failures: u32,
|
||||
/// Timestamp of the last successful poll.
|
||||
pub last_poll: Option<DateTime<Utc>>,
|
||||
/// Healthy / total neuron counts from the cortex's `/health` (coarse
|
||||
/// load signal; #73 refines headroom). 0/0 until first health poll.
|
||||
pub healthy_nodes: u32,
|
||||
pub total_nodes: u32,
|
||||
/// The cortex's full `/v1/models` entries, keyed by model id. Stored
|
||||
/// whole (not distilled to a loaded/feasible bool) so the federation
|
||||
/// catalogue (#75) can preserve per-model `limit`/`cost`/capabilities.
|
||||
pub models: HashMap<String, CortexModelEntry>,
|
||||
}
|
||||
|
||||
/// Whether a cortex can serve this model — loaded now, or feasible to
|
||||
/// cold-load (its catalogue × topology says some neuron can host it).
|
||||
pub fn entry_feasible(entry: &CortexModelEntry) -> bool {
|
||||
entry.loaded || !entry.feasible_on.is_empty()
|
||||
}
|
||||
|
||||
impl RouterState {
|
||||
pub fn from_config(config: &RouterConfig) -> Self {
|
||||
let topology = config
|
||||
.cortexes
|
||||
.iter()
|
||||
.map(|c| (c.name.clone(), CortexTopology::default()))
|
||||
.collect();
|
||||
|
||||
// One client per cortex. A `tls_ca` that fails to load omits the
|
||||
// cortex from the map (fail closed) rather than degrading to an
|
||||
// unpinned client.
|
||||
let mut clients = HashMap::new();
|
||||
for c in &config.cortexes {
|
||||
match build_client(c.tls_ca.as_deref()) {
|
||||
Ok(client) => {
|
||||
clients.insert(c.name.clone(), client);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(
|
||||
cortex = %c.name,
|
||||
tls_ca = c.tls_ca.as_deref().unwrap_or(""),
|
||||
error = %e,
|
||||
"failed to build pinned TLS client; cortex disabled (fail closed)"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Self {
|
||||
cortexes: config.cortexes.clone(),
|
||||
clients,
|
||||
region: config.router.region.clone(),
|
||||
poll_interval: Duration::from_secs(config.router.poll_interval_secs),
|
||||
topology: RwLock::new(topology),
|
||||
}
|
||||
}
|
||||
|
||||
/// The HTTP client to use for `name`, or `None` if the cortex is
|
||||
/// disabled (its `tls_ca` failed to load). Callers must treat `None` as
|
||||
/// "not routable / not pollable".
|
||||
pub fn client_for(&self, name: &str) -> Option<&reqwest::Client> {
|
||||
self.clients.get(name)
|
||||
}
|
||||
|
||||
/// Names of reachable cortexes that can serve `model_id` (loaded or
|
||||
/// feasible to cold-load). Groundwork for capacity-aware dispatch (#73);
|
||||
/// unreachable cortexes are excluded by construction.
|
||||
pub async fn cortexes_serving(&self, model_id: &str) -> Vec<String> {
|
||||
let topo = self.topology.read().await;
|
||||
topo.iter()
|
||||
.filter(|(_, t)| t.reachable)
|
||||
.filter(|(_, t)| t.models.get(model_id).is_some_and(entry_feasible))
|
||||
.map(|(name, _)| name.clone())
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a cortex HTTP client. With `tls_ca` set, the client trusts **only**
|
||||
/// that PEM anchor (platform roots disabled) — pinning the router→cortex hop
|
||||
/// to an enrolled cert (#74). Without it, standard platform-root validation.
|
||||
pub fn build_client(tls_ca: Option<&str>) -> Result<reqwest::Client, BuildClientError> {
|
||||
let mut builder = reqwest::Client::builder();
|
||||
if let Some(path) = tls_ca {
|
||||
let pem = std::fs::read(path).map_err(|e| BuildClientError::Read(path.to_string(), e))?;
|
||||
let cert = reqwest::Certificate::from_pem(&pem).map_err(BuildClientError::Parse)?;
|
||||
builder = builder
|
||||
.tls_built_in_root_certs(false)
|
||||
.add_root_certificate(cert);
|
||||
}
|
||||
builder.build().map_err(BuildClientError::Build)
|
||||
}
|
||||
|
||||
/// Why a cortex's pinned client could not be built (→ cortex disabled).
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum BuildClientError {
|
||||
#[error("reading TLS anchor '{0}'")]
|
||||
Read(String, #[source] std::io::Error),
|
||||
#[error("parsing TLS anchor PEM")]
|
||||
Parse(#[source] reqwest::Error),
|
||||
#[error("building HTTP client")]
|
||||
Build(#[source] reqwest::Error),
|
||||
}
|
||||
132
crates/helexa-router/tests/catalogue.rs
Normal file
@@ -0,0 +1,132 @@
|
||||
//! End-to-end federation-catalogue test for #75: poll two mock cortexes
|
||||
//! that overlap on a model, then `GET /v1/models` on the router and verify
|
||||
//! the deduped union with merged availability and preserved limit/cost.
|
||||
|
||||
use axum::Router;
|
||||
use axum::routing::get;
|
||||
use helexa_router::config::{CortexEndpoint, RouterConfig};
|
||||
use helexa_router::poller::poll_once;
|
||||
use helexa_router::state::RouterState;
|
||||
use serde_json::{Value, json};
|
||||
use std::sync::Arc;
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
/// Spawn a mock cortex serving the given `/v1/models` `data` array.
|
||||
async fn spawn_cortex(models: Value) -> String {
|
||||
let models = Arc::new(models);
|
||||
let app = Router::new()
|
||||
.route(
|
||||
"/v1/models",
|
||||
get({
|
||||
let models = Arc::clone(&models);
|
||||
move || {
|
||||
let models = Arc::clone(&models);
|
||||
async move { axum::Json(json!({ "object": "list", "data": &*models })) }
|
||||
}
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/health",
|
||||
get(|| async { axum::Json(json!({"status":"ok","nodes":{"healthy":1,"total":1}})) }),
|
||||
);
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
format!("http://{addr}")
|
||||
}
|
||||
|
||||
/// Spawn the router (with poller) wired to the given cortex endpoints, and
|
||||
/// poll once synchronously so the topology is populated before we query.
|
||||
async fn spawn_router(cortexes: Vec<CortexEndpoint>) -> String {
|
||||
let cfg = RouterConfig {
|
||||
cortexes,
|
||||
..Default::default()
|
||||
};
|
||||
let state = Arc::new(RouterState::from_config(&cfg));
|
||||
poll_once(&state).await; // deterministic: fill topology now
|
||||
|
||||
let app = helexa_router::build_app(Arc::clone(&state));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
format!("http://{addr}")
|
||||
}
|
||||
|
||||
fn model(id: &str, loaded: bool, feasible_on: &[&str], ctx: u64, input_cost: f64) -> Value {
|
||||
json!({
|
||||
"id": id,
|
||||
"object": "model",
|
||||
"created": 0,
|
||||
"owned_by": "helexa",
|
||||
"loaded": loaded,
|
||||
"feasible_on": feasible_on,
|
||||
"locations": [],
|
||||
"limit": { "context": ctx, "output": 4096 },
|
||||
"cost": { "input": input_cost, "output": input_cost * 3.0 }
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn federation_catalogue_dedupes_and_preserves_limit_cost() {
|
||||
// cortex A: "shared" loaded (ctx 32768, $0.50) + "only-a" loaded.
|
||||
let a = spawn_cortex(json!([
|
||||
model("shared", true, &["beast"], 32_768, 0.50),
|
||||
model("only-a", true, &["beast"], 8_192, 1.00),
|
||||
]))
|
||||
.await;
|
||||
// cortex B: "shared" cold-loadable, tighter ctx (16384), cheaper ($0.20).
|
||||
let b = spawn_cortex(json!([model("shared", false, &["benjy"], 16_384, 0.20)])).await;
|
||||
|
||||
let router = spawn_router(vec![
|
||||
CortexEndpoint {
|
||||
name: "op-a".into(),
|
||||
endpoint: a,
|
||||
region: None,
|
||||
tls_ca: None,
|
||||
},
|
||||
CortexEndpoint {
|
||||
name: "op-b".into(),
|
||||
endpoint: b,
|
||||
region: None,
|
||||
tls_ca: None,
|
||||
},
|
||||
])
|
||||
.await;
|
||||
|
||||
let body: Value = reqwest::get(format!("{router}/v1/models"))
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(body["object"], "list");
|
||||
let data = body["data"].as_array().unwrap();
|
||||
// Deduped union: "shared" once + "only-a".
|
||||
assert_eq!(data.len(), 2);
|
||||
|
||||
let shared = data.iter().find(|m| m["id"] == "shared").unwrap();
|
||||
// Loaded somewhere (op-a) → loaded.
|
||||
assert_eq!(shared["loaded"], true);
|
||||
// feasible_on re-tiered to operator names, both present, sorted.
|
||||
let feasible: Vec<&str> = shared["feasible_on"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|v| v.as_str().unwrap())
|
||||
.collect();
|
||||
assert_eq!(feasible, vec!["op-a", "op-b"]);
|
||||
// Tightest limit (16384) and cheapest cost ($0.20) win.
|
||||
assert_eq!(shared["limit"]["context"], 16_384);
|
||||
assert_eq!(shared["cost"]["input"], 0.20);
|
||||
// Loaded location named by operator, no neuron VRAM leaked.
|
||||
let locs = shared["locations"].as_array().unwrap();
|
||||
assert_eq!(locs.len(), 1);
|
||||
assert_eq!(locs[0]["node"], "op-a");
|
||||
|
||||
assert!(data.iter().any(|m| m["id"] == "only-a"));
|
||||
}
|
||||
301
crates/helexa-router/tests/dispatch.rs
Normal file
@@ -0,0 +1,301 @@
|
||||
//! Capacity-aware dispatch acceptance tests for #73.
|
||||
//!
|
||||
//! Covers: a request routes to a cortex serving the model; the client's
|
||||
//! bearer reaches the cortex; cortex's #63 rejections pass through verbatim
|
||||
//! and are NOT retried away; transport failure fails over to another
|
||||
//! feasible cortex; unknown model → 404, no reachable capacity → 503; and
|
||||
//! the selection ranking (warm/region/headroom).
|
||||
|
||||
use axum::body::Bytes;
|
||||
use axum::extract::State;
|
||||
use axum::http::{HeaderMap, StatusCode};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum::routing::post;
|
||||
use axum::{Json, Router};
|
||||
use cortex_core::node::CortexModelEntry;
|
||||
use helexa_router::config::{CortexEndpoint, RouterConfig};
|
||||
use helexa_router::dispatch::{Selection, dispatch, select_cortexes};
|
||||
use helexa_router::state::{CortexTopology, RouterState};
|
||||
use serde_json::{Value, json};
|
||||
use std::collections::HashMap;
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
/// A minimal `CortexModelEntry` for MODEL with the given serveability.
|
||||
fn model_entry(loaded: bool, feasible: bool) -> CortexModelEntry {
|
||||
CortexModelEntry {
|
||||
id: MODEL.into(),
|
||||
object: "model".into(),
|
||||
created: 0,
|
||||
owned_by: "helexa".into(),
|
||||
loaded,
|
||||
feasible_on: if feasible || loaded {
|
||||
vec!["n".into()]
|
||||
} else {
|
||||
vec![]
|
||||
},
|
||||
locations: vec![],
|
||||
capabilities: vec![],
|
||||
limit: None,
|
||||
cost: None,
|
||||
tool_call: false,
|
||||
reasoning: false,
|
||||
}
|
||||
}
|
||||
|
||||
const MODEL: &str = "Qwen/Qwen3-Coder-30B";
|
||||
|
||||
// ── Mock cortex backend ──────────────────────────────────────────────
|
||||
|
||||
/// Behaviour of a mock cortex, carried in axum State.
|
||||
#[derive(Clone)]
|
||||
struct MockCortex {
|
||||
/// Identifies which cortex answered, echoed in the 200 body.
|
||||
name: &'static str,
|
||||
/// When true, return a genuine #63-shaped `429 rate_limit_exceeded`.
|
||||
rate_limited: bool,
|
||||
}
|
||||
|
||||
async fn mock_handler(State(m): State<MockCortex>, headers: HeaderMap) -> Response {
|
||||
if m.rate_limited {
|
||||
return (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
Json(json!({"error":{"type":"rate_limit_error","code":"rate_limit_exceeded","message":"slow down","param":null}})),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
let auth = headers
|
||||
.get("authorization")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
Json(json!({ "served_by": m.name, "auth_seen": auth })).into_response()
|
||||
}
|
||||
|
||||
async fn spawn_cortex(mock: MockCortex) -> String {
|
||||
let app = Router::new()
|
||||
.route("/v1/chat/completions", post(mock_handler))
|
||||
.with_state(mock);
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
format!("http://{addr}")
|
||||
}
|
||||
|
||||
fn ok_cortex(name: &'static str) -> MockCortex {
|
||||
MockCortex {
|
||||
name,
|
||||
rate_limited: false,
|
||||
}
|
||||
}
|
||||
|
||||
// ── Helpers to build state with a hand-set topology ──────────────────
|
||||
|
||||
fn state_with(cortexes: Vec<CortexEndpoint>, region: Option<String>) -> RouterState {
|
||||
let cfg = RouterConfig {
|
||||
cortexes,
|
||||
..Default::default()
|
||||
};
|
||||
let mut state = RouterState::from_config(&cfg);
|
||||
state.region = region;
|
||||
state
|
||||
}
|
||||
|
||||
/// Overwrite the topology entry for `name` so tests control reachability and
|
||||
/// model serveability directly (no live poll).
|
||||
async fn set_topology(
|
||||
state: &RouterState,
|
||||
name: &str,
|
||||
reachable: bool,
|
||||
loaded: bool,
|
||||
feasible: bool,
|
||||
healthy_nodes: u32,
|
||||
) {
|
||||
let mut topo = state.topology.write().await;
|
||||
let mut models = HashMap::new();
|
||||
models.insert(MODEL.to_string(), model_entry(loaded, feasible));
|
||||
topo.insert(
|
||||
name.to_string(),
|
||||
CortexTopology {
|
||||
reachable,
|
||||
consecutive_failures: 0,
|
||||
last_poll: None,
|
||||
healthy_nodes,
|
||||
total_nodes: healthy_nodes,
|
||||
models,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
fn ep(name: &str, endpoint: &str, region: Option<&str>) -> CortexEndpoint {
|
||||
CortexEndpoint {
|
||||
name: name.into(),
|
||||
endpoint: endpoint.into(),
|
||||
region: region.map(str::to_string),
|
||||
tls_ca: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn chat_body() -> Bytes {
|
||||
Bytes::from(format!("{{\"model\":\"{MODEL}\",\"stream\":false}}"))
|
||||
}
|
||||
|
||||
async fn body_json(resp: Response) -> (StatusCode, Value) {
|
||||
let status = resp.status();
|
||||
let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX)
|
||||
.await
|
||||
.unwrap();
|
||||
let v = serde_json::from_slice(&bytes).unwrap_or(Value::Null);
|
||||
(status, v)
|
||||
}
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn routes_to_serving_cortex_and_forwards_bearer() {
|
||||
let url = spawn_cortex(ok_cortex("c1")).await;
|
||||
let state = state_with(vec![ep("c1", &url, None)], None);
|
||||
set_topology(&state, "c1", true, true, true, 2).await;
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("authorization", "Bearer sk-test-123".parse().unwrap());
|
||||
|
||||
let resp = dispatch(&state, "/v1/chat/completions", headers, chat_body()).await;
|
||||
let (status, body) = body_json(resp).await;
|
||||
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert_eq!(body["served_by"], "c1");
|
||||
// Bearer reached the cortex unchanged.
|
||||
assert_eq!(body["auth_seen"], "Bearer sk-test-123");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cortex_429_passes_through_and_is_not_retried() {
|
||||
// c1 (ranked first: loaded) returns a genuine 429; c2 would return 200.
|
||||
let c1 = spawn_cortex(MockCortex {
|
||||
name: "c1",
|
||||
rate_limited: true,
|
||||
})
|
||||
.await;
|
||||
let c2 = spawn_cortex(ok_cortex("c2")).await;
|
||||
let state = state_with(vec![ep("c1", &c1, None), ep("c2", &c2, None)], None);
|
||||
// Both reachable + loaded; c1 has more headroom so it ranks first.
|
||||
set_topology(&state, "c1", true, true, true, 5).await;
|
||||
set_topology(&state, "c2", true, true, true, 1).await;
|
||||
|
||||
let resp = dispatch(
|
||||
&state,
|
||||
"/v1/chat/completions",
|
||||
HeaderMap::new(),
|
||||
chat_body(),
|
||||
)
|
||||
.await;
|
||||
let (status, body) = body_json(resp).await;
|
||||
|
||||
// The genuine 4xx is returned verbatim — NOT retried to c2.
|
||||
assert_eq!(status, StatusCode::TOO_MANY_REQUESTS);
|
||||
assert_eq!(body["error"]["code"], "rate_limit_exceeded");
|
||||
assert!(body.get("served_by").is_none(), "must not have hit c2");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fails_over_to_next_cortex_on_transport_error() {
|
||||
// c_dead ranks first (more headroom) but its endpoint is a closed port;
|
||||
// c_live is the fallback. The router must fail over and c_live serves.
|
||||
let live = spawn_cortex(ok_cortex("c_live")).await;
|
||||
let state = state_with(
|
||||
vec![
|
||||
ep("c_dead", "http://127.0.0.1:1", None),
|
||||
ep("c_live", &live, None),
|
||||
],
|
||||
None,
|
||||
);
|
||||
set_topology(&state, "c_dead", true, true, true, 9).await;
|
||||
set_topology(&state, "c_live", true, true, true, 1).await;
|
||||
|
||||
let resp = dispatch(
|
||||
&state,
|
||||
"/v1/chat/completions",
|
||||
HeaderMap::new(),
|
||||
chat_body(),
|
||||
)
|
||||
.await;
|
||||
let (status, body) = body_json(resp).await;
|
||||
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert_eq!(body["served_by"], "c_live");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_model_is_404() {
|
||||
let state = state_with(vec![ep("c1", "http://127.0.0.1:1", None)], None);
|
||||
// Topology has no entry for MODEL at all.
|
||||
let resp = dispatch(
|
||||
&state,
|
||||
"/v1/chat/completions",
|
||||
HeaderMap::new(),
|
||||
chat_body(),
|
||||
)
|
||||
.await;
|
||||
let (status, body) = body_json(resp).await;
|
||||
assert_eq!(status, StatusCode::NOT_FOUND);
|
||||
assert_eq!(body["error"]["code"], "model_not_found");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn known_but_all_unreachable_is_503() {
|
||||
let state = state_with(vec![ep("c1", "http://127.0.0.1:1", None)], None);
|
||||
// Cortex knows the model (from a prior good poll) but is now unreachable.
|
||||
set_topology(&state, "c1", false, true, true, 2).await;
|
||||
let resp = dispatch(
|
||||
&state,
|
||||
"/v1/chat/completions",
|
||||
HeaderMap::new(),
|
||||
chat_body(),
|
||||
)
|
||||
.await;
|
||||
let (status, body) = body_json(resp).await;
|
||||
assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(body["error"]["code"], "service_unavailable");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn missing_model_field_is_400() {
|
||||
let state = state_with(vec![ep("c1", "http://127.0.0.1:1", None)], None);
|
||||
let resp = dispatch(
|
||||
&state,
|
||||
"/v1/chat/completions",
|
||||
HeaderMap::new(),
|
||||
Bytes::from_static(b"{\"messages\":[]}"),
|
||||
)
|
||||
.await;
|
||||
let (status, body) = body_json(resp).await;
|
||||
assert_eq!(status, StatusCode::BAD_REQUEST);
|
||||
assert_eq!(body["error"]["code"], "missing_model_field");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ranking_prefers_loaded_then_region_then_headroom() {
|
||||
// Router is in eu-west. Candidates:
|
||||
// warm-eu : loaded, region match, 1 node → best
|
||||
// warm-us : loaded, no region, 9 nodes
|
||||
// cold-eu : feasible only, region match → worst (cold)
|
||||
let state = state_with(
|
||||
vec![
|
||||
ep("warm-eu", "http://127.0.0.1:1", Some("eu-west")),
|
||||
ep("warm-us", "http://127.0.0.1:1", Some("us-east")),
|
||||
ep("cold-eu", "http://127.0.0.1:1", Some("eu-west")),
|
||||
],
|
||||
Some("eu-west".into()),
|
||||
);
|
||||
set_topology(&state, "warm-eu", true, true, true, 1).await;
|
||||
set_topology(&state, "warm-us", true, true, true, 9).await;
|
||||
set_topology(&state, "cold-eu", true, false, true, 5).await;
|
||||
|
||||
let Selection::Candidates(order) = select_cortexes(&state, MODEL).await else {
|
||||
panic!("expected candidates");
|
||||
};
|
||||
let names: Vec<&str> = order.iter().map(|e| e.name.as_str()).collect();
|
||||
assert_eq!(names, vec!["warm-eu", "warm-us", "cold-eu"]);
|
||||
}
|
||||
97
crates/helexa-router/tests/skeleton.rs
Normal file
@@ -0,0 +1,97 @@
|
||||
//! Skeleton acceptance tests for #70: the router builds, serves `/health`
|
||||
//! and `/v1/models` on a plaintext port, and loads its cortex-endpoint list
|
||||
//! from TOML with env overrides.
|
||||
|
||||
use helexa_router::config::{CortexEndpoint, RouterConfig};
|
||||
use helexa_router::state::RouterState;
|
||||
use std::sync::Arc;
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
/// Bind the router app on an ephemeral port and return its base URL.
|
||||
async fn spawn_router(cortexes: Vec<CortexEndpoint>) -> String {
|
||||
let cfg = RouterConfig {
|
||||
cortexes,
|
||||
..Default::default()
|
||||
};
|
||||
let state = Arc::new(RouterState::from_config(&cfg));
|
||||
let app = helexa_router::build_app(state);
|
||||
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
|
||||
format!("http://{addr}")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn health_reports_configured_cortex_count() {
|
||||
let base = spawn_router(vec![
|
||||
CortexEndpoint {
|
||||
name: "a".into(),
|
||||
endpoint: "https://a.example.com".into(),
|
||||
region: None,
|
||||
tls_ca: None,
|
||||
},
|
||||
CortexEndpoint {
|
||||
name: "b".into(),
|
||||
endpoint: "https://b.example.com".into(),
|
||||
region: None,
|
||||
tls_ca: None,
|
||||
},
|
||||
])
|
||||
.await;
|
||||
|
||||
let body: serde_json::Value = reqwest::get(format!("{base}/health"))
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(body["status"], "ok");
|
||||
assert_eq!(body["cortexes"]["configured"], 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn models_returns_empty_openai_list() {
|
||||
let base = spawn_router(vec![]).await;
|
||||
|
||||
let resp = reqwest::get(format!("{base}/v1/models")).await.unwrap();
|
||||
assert!(resp.status().is_success());
|
||||
|
||||
let body: serde_json::Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["object"], "list");
|
||||
assert_eq!(body["data"].as_array().unwrap().len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(clippy::result_large_err)]
|
||||
fn config_loads_from_toml_with_env_override() {
|
||||
figment::Jail::expect_with(|jail| {
|
||||
jail.create_file(
|
||||
"helexa-router.toml",
|
||||
r#"
|
||||
[router]
|
||||
listen = "127.0.0.1:8088"
|
||||
|
||||
[[cortexes]]
|
||||
name = "lair-cafe"
|
||||
endpoint = "https://cortex.lair.cafe"
|
||||
"#,
|
||||
)?;
|
||||
|
||||
// Env override wins over the TOML value.
|
||||
jail.set_env("HELEXA_ROUTER_ROUTER__LISTEN", "0.0.0.0:9099");
|
||||
|
||||
let cfg = RouterConfig::load("helexa-router.toml").expect("load config");
|
||||
|
||||
assert_eq!(cfg.router.listen, "0.0.0.0:9099");
|
||||
assert_eq!(cfg.cortexes.len(), 1);
|
||||
assert_eq!(cfg.cortexes[0].name, "lair-cafe");
|
||||
assert_eq!(cfg.cortexes[0].endpoint, "https://cortex.lair.cafe");
|
||||
|
||||
Ok(())
|
||||
});
|
||||
}
|
||||
210
crates/helexa-router/tests/tls.rs
Normal file
@@ -0,0 +1,210 @@
|
||||
//! Outbound TLS pinning tests for #74.
|
||||
//!
|
||||
//! Proves the router, as a TLS client to cortexes, reaches a cortex
|
||||
//! presenting its **enrolled** cert and rejects one presenting an
|
||||
//! unexpected (or untrusted) cert — and that a rejected handshake flows
|
||||
//! through the existing reachability path (#72) to exclude the cortex.
|
||||
//!
|
||||
//! A minimal `tokio-rustls` HTTPS server presents a self-signed cert; the
|
||||
//! router's `reqwest` client (native-tls) validates against the PEM anchor
|
||||
//! enrolled in config. Server (rustls) and client (native-tls) interoperate
|
||||
//! at the protocol level — what matters is the trust decision.
|
||||
|
||||
use helexa_router::config::{CortexEndpoint, RouterConfig};
|
||||
use helexa_router::poller::poll_once;
|
||||
use helexa_router::state::{RouterState, build_client};
|
||||
use std::io::Write;
|
||||
use std::sync::Arc;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpListener;
|
||||
use tokio_rustls::TlsAcceptor;
|
||||
|
||||
/// A self-signed cert: PEM (for the reqwest pin file) + DER cert/key (for
|
||||
/// the rustls server).
|
||||
struct TestCert {
|
||||
cert_pem: String,
|
||||
cert_der: rustls::pki_types::CertificateDer<'static>,
|
||||
key_der: Vec<u8>,
|
||||
}
|
||||
|
||||
fn make_cert() -> TestCert {
|
||||
let key = rcgen::generate_simple_self_signed(vec!["127.0.0.1".to_string()]).unwrap();
|
||||
TestCert {
|
||||
cert_pem: key.cert.pem(),
|
||||
cert_der: key.cert.der().clone(),
|
||||
key_der: key.key_pair.serialize_der(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Write a cert PEM to a unique temp file (named by `tag`) and return the
|
||||
/// path. `tag` is caller-unique (we use the bound port), so no randomness.
|
||||
fn write_pem(tag: &str, pem: &str) -> String {
|
||||
let path = std::env::temp_dir().join(format!("helexa-router-tls-{tag}.pem"));
|
||||
let mut f = std::fs::File::create(&path).unwrap();
|
||||
f.write_all(pem.as_bytes()).unwrap();
|
||||
path.to_string_lossy().into_owned()
|
||||
}
|
||||
|
||||
/// Spawn a minimal HTTPS server presenting `cert`, answering every request
|
||||
/// with a canned `/v1/models`-shaped 200. Returns its `https://` base URL.
|
||||
async fn spawn_https(cert: &TestCert) -> String {
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
|
||||
let key = rustls::pki_types::PrivateKeyDer::Pkcs8(rustls::pki_types::PrivatePkcs8KeyDer::from(
|
||||
cert.key_der.clone(),
|
||||
));
|
||||
let config = rustls::ServerConfig::builder()
|
||||
.with_no_client_auth()
|
||||
.with_single_cert(vec![cert.cert_der.clone()], key)
|
||||
.unwrap();
|
||||
let acceptor = TlsAcceptor::from(Arc::new(config));
|
||||
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let Ok((stream, _)) = listener.accept().await else {
|
||||
continue;
|
||||
};
|
||||
let acceptor = acceptor.clone();
|
||||
tokio::spawn(async move {
|
||||
if let Ok(mut tls) = acceptor.accept(stream).await {
|
||||
let mut buf = [0u8; 2048];
|
||||
let _ = tls.read(&mut buf).await; // consume request line/headers
|
||||
let body = "{\"object\":\"list\",\"data\":[]}";
|
||||
let resp = format!(
|
||||
"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
|
||||
body.len(),
|
||||
body
|
||||
);
|
||||
let _ = tls.write_all(resp.as_bytes()).await;
|
||||
let _ = tls.shutdown().await;
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
format!("https://{addr}")
|
||||
}
|
||||
|
||||
fn tag_for(url: &str) -> String {
|
||||
url.rsplit(':').next().unwrap_or("0").to_string()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pinned_client_accepts_enrolled_cert_and_rejects_others() {
|
||||
let server_cert = make_cert();
|
||||
let other_cert = make_cert();
|
||||
let url = spawn_https(&server_cert).await;
|
||||
let tag = tag_for(&url);
|
||||
|
||||
let good_pin = write_pem(&format!("{tag}-good"), &server_cert.cert_pem);
|
||||
let bad_pin = write_pem(&format!("{tag}-bad"), &other_cert.cert_pem);
|
||||
|
||||
// Enrolled with the server's own cert → handshake trusted → 200.
|
||||
let good = build_client(Some(&good_pin)).unwrap();
|
||||
let resp = good.get(format!("{url}/v1/models")).send().await;
|
||||
assert!(resp.is_ok(), "enrolled cert must be accepted: {resp:?}");
|
||||
assert_eq!(resp.unwrap().status(), 200);
|
||||
|
||||
// Enrolled with a different cert → server's cert is unexpected → reject.
|
||||
let bad = build_client(Some(&bad_pin)).unwrap();
|
||||
assert!(
|
||||
bad.get(format!("{url}/v1/models")).send().await.is_err(),
|
||||
"unexpected cert must be rejected"
|
||||
);
|
||||
|
||||
// No enrollment (default platform roots) → self-signed cert untrusted.
|
||||
let default = build_client(None).unwrap();
|
||||
assert!(
|
||||
default
|
||||
.get(format!("{url}/v1/models"))
|
||||
.send()
|
||||
.await
|
||||
.is_err(),
|
||||
"un-enrolled self-signed cert must be rejected by default roots"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn poller_excludes_cortex_with_unexpected_cert() {
|
||||
let server_cert = make_cert();
|
||||
let other_cert = make_cert();
|
||||
let url = spawn_https(&server_cert).await;
|
||||
let tag = tag_for(&url);
|
||||
|
||||
let good_pin = write_pem(&format!("{tag}-pgood"), &server_cert.cert_pem);
|
||||
let bad_pin = write_pem(&format!("{tag}-pbad"), &other_cert.cert_pem);
|
||||
|
||||
// Cortex A enrolled correctly → reachable. Cortex B enrolled with the
|
||||
// wrong cert → TLS handshake fails → excluded.
|
||||
let cfg = RouterConfig {
|
||||
cortexes: vec![
|
||||
CortexEndpoint {
|
||||
name: "good".into(),
|
||||
endpoint: url.clone(),
|
||||
region: None,
|
||||
tls_ca: Some(good_pin),
|
||||
},
|
||||
CortexEndpoint {
|
||||
name: "bad".into(),
|
||||
endpoint: url.clone(),
|
||||
region: None,
|
||||
tls_ca: Some(bad_pin),
|
||||
},
|
||||
],
|
||||
..Default::default()
|
||||
};
|
||||
let state = RouterState::from_config(&cfg);
|
||||
poll_once(&state).await;
|
||||
|
||||
let topo = state.topology.read().await;
|
||||
assert!(
|
||||
topo["good"].reachable,
|
||||
"correctly-enrolled cortex reachable"
|
||||
);
|
||||
assert!(
|
||||
!topo["bad"].reachable,
|
||||
"cortex presenting an unexpected cert is excluded"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn misconfigured_pin_disables_cortex_fail_closed() {
|
||||
// A `tls_ca` pointing at a nonexistent file must NOT fall back to an
|
||||
// unpinned client — the cortex is disabled entirely.
|
||||
let cfg = RouterConfig {
|
||||
cortexes: vec![
|
||||
CortexEndpoint {
|
||||
name: "broken".into(),
|
||||
endpoint: "https://127.0.0.1:1".into(),
|
||||
region: None,
|
||||
tls_ca: Some("/no/such/anchor.pem".into()),
|
||||
},
|
||||
CortexEndpoint {
|
||||
name: "plain".into(),
|
||||
endpoint: "http://127.0.0.1:1".into(),
|
||||
region: None,
|
||||
tls_ca: None,
|
||||
},
|
||||
],
|
||||
..Default::default()
|
||||
};
|
||||
let state = RouterState::from_config(&cfg);
|
||||
assert!(
|
||||
state.client_for("broken").is_none(),
|
||||
"a cortex with an unloadable pin is disabled (fail closed)"
|
||||
);
|
||||
assert!(
|
||||
state.client_for("plain").is_some(),
|
||||
"an un-pinned cortex still gets a client"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_client_rejects_garbage_pem() {
|
||||
let path = write_pem(
|
||||
"garbage",
|
||||
"-----BEGIN CERTIFICATE-----\nnope\n-----END CERTIFICATE-----",
|
||||
);
|
||||
assert!(build_client(Some(&path)).is_err());
|
||||
}
|
||||
172
crates/helexa-router/tests/topology.rs
Normal file
@@ -0,0 +1,172 @@
|
||||
//! Topology-poller acceptance tests for #72: the router maintains a live
|
||||
//! map of which cortexes serve which models, marks an unreachable/erroring
|
||||
//! cortex unhealthy and excludes it from routing, and recovers it once
|
||||
//! reachable again.
|
||||
|
||||
use axum::extract::State;
|
||||
use axum::http::StatusCode;
|
||||
use axum::routing::get;
|
||||
use axum::{Json, Router};
|
||||
use helexa_router::config::{CortexEndpoint, RouterConfig};
|
||||
use helexa_router::poller::{POLL_FAILURE_THRESHOLD, poll_once};
|
||||
use helexa_router::state::{RouterState, entry_feasible};
|
||||
use serde_json::{Value, json};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
/// Shared "is this mock cortex up?" flag, toggled by tests to simulate
|
||||
/// outage and recovery.
|
||||
#[derive(Clone)]
|
||||
struct MockState {
|
||||
up: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
async fn mock_models(State(s): State<MockState>) -> Result<Json<Value>, StatusCode> {
|
||||
if !s.up.load(Ordering::SeqCst) {
|
||||
return Err(StatusCode::SERVICE_UNAVAILABLE);
|
||||
}
|
||||
Ok(Json(json!({
|
||||
"object": "list",
|
||||
"data": [
|
||||
{
|
||||
"id": "Qwen/Qwen3-Coder-30B",
|
||||
"object": "model",
|
||||
"created": 0,
|
||||
"owned_by": "helexa",
|
||||
"loaded": true,
|
||||
"feasible_on": ["beast"],
|
||||
"locations": [{"node": "beast", "status": "loaded", "vram_estimate_mb": 19000}]
|
||||
},
|
||||
{
|
||||
"id": "Qwen/Qwen3-VL-8B",
|
||||
"object": "model",
|
||||
"created": 0,
|
||||
"owned_by": "helexa",
|
||||
"loaded": false,
|
||||
"feasible_on": ["beast"],
|
||||
"locations": []
|
||||
}
|
||||
]
|
||||
})))
|
||||
}
|
||||
|
||||
async fn mock_health(State(s): State<MockState>) -> Result<Json<Value>, StatusCode> {
|
||||
if !s.up.load(Ordering::SeqCst) {
|
||||
return Err(StatusCode::SERVICE_UNAVAILABLE);
|
||||
}
|
||||
Ok(Json(json!({
|
||||
"status": "ok",
|
||||
"nodes": { "healthy": 2, "total": 3 }
|
||||
})))
|
||||
}
|
||||
|
||||
/// Spawn a mock cortex; returns (base_url, up_flag).
|
||||
async fn spawn_mock_cortex() -> (String, Arc<AtomicBool>) {
|
||||
let up = Arc::new(AtomicBool::new(true));
|
||||
let state = MockState { up: up.clone() };
|
||||
let app = Router::new()
|
||||
.route("/v1/models", get(mock_models))
|
||||
.route("/health", get(mock_health))
|
||||
.with_state(state);
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
(format!("http://{addr}"), up)
|
||||
}
|
||||
|
||||
fn state_for(name: &str, endpoint: &str) -> RouterState {
|
||||
let cfg = RouterConfig {
|
||||
cortexes: vec![CortexEndpoint {
|
||||
name: name.into(),
|
||||
endpoint: endpoint.into(),
|
||||
region: None,
|
||||
tls_ca: None,
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
RouterState::from_config(&cfg)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn poll_builds_live_topology() {
|
||||
let (base, _up) = spawn_mock_cortex().await;
|
||||
let state = state_for("c1", &base);
|
||||
|
||||
poll_once(&state).await;
|
||||
|
||||
let topo = state.topology.read().await;
|
||||
let c1 = topo.get("c1").expect("cortex present");
|
||||
assert!(c1.reachable, "should be reachable after a good poll");
|
||||
assert_eq!(c1.consecutive_failures, 0);
|
||||
assert!(c1.last_poll.is_some());
|
||||
assert_eq!((c1.healthy_nodes, c1.total_nodes), (2, 3));
|
||||
|
||||
// Loaded model: loaded + feasible. Catalogue-only model: feasible only
|
||||
// (not loaded, but feasible_on non-empty).
|
||||
let coder = c1.models.get("Qwen/Qwen3-Coder-30B").unwrap();
|
||||
assert!(coder.loaded && entry_feasible(coder));
|
||||
let vl = c1.models.get("Qwen/Qwen3-VL-8B").unwrap();
|
||||
assert!(!vl.loaded && entry_feasible(vl));
|
||||
drop(topo);
|
||||
|
||||
// The routing helper sees both serveable models on the reachable cortex.
|
||||
assert_eq!(
|
||||
state.cortexes_serving("Qwen/Qwen3-VL-8B").await,
|
||||
vec!["c1".to_string()]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unreachable_cortex_excluded_then_recovers() {
|
||||
let (base, up) = spawn_mock_cortex().await;
|
||||
let state = state_for("c1", &base);
|
||||
|
||||
// Healthy first.
|
||||
poll_once(&state).await;
|
||||
assert!(state.topology.read().await["c1"].reachable);
|
||||
|
||||
// Take it down. The first failures debounce (stay reachable) until the
|
||||
// threshold; only then is it excluded.
|
||||
up.store(false, Ordering::SeqCst);
|
||||
for i in 1..POLL_FAILURE_THRESHOLD {
|
||||
poll_once(&state).await;
|
||||
assert!(
|
||||
state.topology.read().await["c1"].reachable,
|
||||
"still reachable after {i} failure(s) (below threshold)"
|
||||
);
|
||||
}
|
||||
poll_once(&state).await; // crosses the threshold
|
||||
{
|
||||
let topo = state.topology.read().await;
|
||||
assert!(!topo["c1"].reachable, "excluded after threshold failures");
|
||||
assert!(topo["c1"].consecutive_failures >= POLL_FAILURE_THRESHOLD);
|
||||
}
|
||||
// Excluded from routing.
|
||||
assert!(
|
||||
state
|
||||
.cortexes_serving("Qwen/Qwen3-Coder-30B")
|
||||
.await
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
// Bring it back: the next successful poll restores it.
|
||||
up.store(true, Ordering::SeqCst);
|
||||
poll_once(&state).await;
|
||||
let topo = state.topology.read().await;
|
||||
assert!(topo["c1"].reachable, "recovered after a good poll");
|
||||
assert_eq!(topo["c1"].consecutive_failures, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unconfigured_endpoint_is_unreachable() {
|
||||
// Nothing listening on this port → polls fail; below threshold it stays
|
||||
// at its initial unreachable state, and never panics.
|
||||
let state = state_for("dead", "http://127.0.0.1:1");
|
||||
poll_once(&state).await;
|
||||
let topo = state.topology.read().await;
|
||||
assert!(!topo["dead"].reachable);
|
||||
assert_eq!(topo["dead"].consecutive_failures, 1);
|
||||
}
|
||||
21
crates/helexa-stream/Cargo.toml
Normal file
@@ -0,0 +1,21 @@
|
||||
[package]
|
||||
name = "helexa-stream"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lib]
|
||||
name = "helexa_stream"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[dependencies]
|
||||
axum = { workspace = true }
|
||||
reqwest = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { workspace = true }
|
||||
tokio-stream = { workspace = true }
|
||||
async-stream = "0.3"
|
||||
290
crates/helexa-stream/src/lib.rs
Normal file
@@ -0,0 +1,290 @@
|
||||
//! Shared streaming reverse-proxy mechanism (#71).
|
||||
//!
|
||||
//! cortex and helexa-router both need to proxy an OpenAI/Anthropic SSE
|
||||
//! response from a downstream backend **verbatim** — chunks forwarded as
|
||||
//! they arrive, never buffering the full body — while observing the bytes
|
||||
//! for metrics/metering. This crate owns that mechanism so there is one
|
||||
//! implementation, not one per tier.
|
||||
//!
|
||||
//! The split is mechanism vs policy:
|
||||
//!
|
||||
//! - **Mechanism (here):** [`forward_streaming`] POSTs to a backend and
|
||||
//! streams the response body back through an [`ObservedStream`], which
|
||||
//! feeds every chunk to a caller-supplied [`ChunkObserver`] and calls
|
||||
//! [`ChunkObserver::finish`] exactly once on clean end-of-stream or on
|
||||
//! drop (client disconnect mid-stream). [`BodyTail`] and
|
||||
//! [`last_count_for`] are the reusable pieces an observer uses to pull
|
||||
//! the trailing OpenAI `usage` object out of the streamed bytes.
|
||||
//! - **Policy (caller):** what to *do* with the observed bytes — which
|
||||
//! metric names to emit, which labels, whether to settle a per-principal
|
||||
//! reservation — lives in the consumer's `ChunkObserver` impl, not here.
|
||||
//!
|
||||
//! The proxy is status-agnostic: a non-2xx upstream response (e.g. a
|
||||
//! cortex `429 rate_limit_exceeded`) is streamed back with its status and
|
||||
//! headers intact, so honest backpressure reaches the client unchanged.
|
||||
//! Only a network failure or a malformed response build is an error.
|
||||
|
||||
use axum::body::{Body, Bytes};
|
||||
use axum::http::{HeaderMap, StatusCode};
|
||||
use axum::response::Response;
|
||||
use futures::Stream;
|
||||
use futures::stream::BoxStream;
|
||||
use reqwest::Client;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
/// Observes the bytes of a streamed proxy response without altering them.
|
||||
///
|
||||
/// `observe` is called for each forwarded chunk; `finish` is called
|
||||
/// exactly once — on clean end-of-stream or on drop — and implementations
|
||||
/// must be idempotent (the [`ObservedStream`] guards against a double call,
|
||||
/// but a `finish` that runs side effects should still self-guard).
|
||||
pub trait ChunkObserver: Send + Unpin + 'static {
|
||||
/// A body chunk has been forwarded downstream. The slice is the exact
|
||||
/// bytes the client receives.
|
||||
fn observe(&mut self, chunk: &[u8]);
|
||||
|
||||
/// The stream has ended (cleanly or via client disconnect). Called once.
|
||||
fn finish(&mut self);
|
||||
}
|
||||
|
||||
/// A bounded accumulator for the tail of a streamed body.
|
||||
///
|
||||
/// The OpenAI `usage` object rides on the final SSE chunk (and sits at the
|
||||
/// end of a non-streaming JSON body), so retaining a generous tail is
|
||||
/// enough to recover token counts via [`last_count_for`]; the cap bounds
|
||||
/// memory on huge bodies. Appends are char-boundary-safe.
|
||||
#[derive(Debug)]
|
||||
pub struct BodyTail {
|
||||
tail: String,
|
||||
cap: usize,
|
||||
}
|
||||
|
||||
impl BodyTail {
|
||||
/// Create a tail retaining at most `cap` bytes.
|
||||
pub fn new(cap: usize) -> Self {
|
||||
Self {
|
||||
tail: String::new(),
|
||||
cap,
|
||||
}
|
||||
}
|
||||
|
||||
/// Append a chunk, trimming from the front past the cap. When trimming,
|
||||
/// the newest half is kept (the usage object is always at the very end).
|
||||
pub fn push(&mut self, chunk: &[u8]) {
|
||||
self.tail.push_str(&String::from_utf8_lossy(chunk));
|
||||
if self.tail.len() > self.cap {
|
||||
let mut cut = self.tail.len() - self.cap / 2;
|
||||
while !self.tail.is_char_boundary(cut) {
|
||||
cut += 1;
|
||||
}
|
||||
self.tail.drain(..cut);
|
||||
}
|
||||
}
|
||||
|
||||
/// The retained tail text.
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.tail
|
||||
}
|
||||
}
|
||||
|
||||
/// Find the value of the LAST `"key": <integer>` occurrence in `tail`.
|
||||
///
|
||||
/// Pure and chunk-boundary-safe (the tail is contiguous appended text).
|
||||
/// The quoted-needle form means `completion_tokens` never matches
|
||||
/// `completion_tokens_details`, and taking the last occurrence means the
|
||||
/// final `usage` object wins even if content earlier in the stream echoed
|
||||
/// a usage-shaped string.
|
||||
pub fn last_count_for(tail: &str, key: &str) -> Option<u64> {
|
||||
let needle = format!("\"{key}\"");
|
||||
let mut result = None;
|
||||
for (idx, _) in tail.match_indices(&needle) {
|
||||
let rest = tail[idx + needle.len()..].trim_start();
|
||||
let Some(rest) = rest.strip_prefix(':') else {
|
||||
continue;
|
||||
};
|
||||
let rest = rest.trim_start();
|
||||
let digits: &str = &rest[..rest
|
||||
.char_indices()
|
||||
.find(|(_, c)| !c.is_ascii_digit())
|
||||
.map(|(i, _)| i)
|
||||
.unwrap_or(rest.len())];
|
||||
if let Ok(v) = digits.parse::<u64>() {
|
||||
result = Some(v);
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// Error from [`forward_streaming`]. Distinguishes a network/transport
|
||||
/// failure reaching the backend from a failure assembling the downstream
|
||||
/// response. A non-2xx upstream *status* is not an error — it is streamed
|
||||
/// through verbatim.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum StreamError {
|
||||
#[error("upstream request failed")]
|
||||
Upstream(reqwest::Error),
|
||||
#[error("failed to build response")]
|
||||
ResponseBuild(String),
|
||||
}
|
||||
|
||||
/// POST `body` to `url` and stream the response back verbatim through
|
||||
/// `observer`.
|
||||
///
|
||||
/// Request headers are forwarded except `host` / `content-length` (reqwest
|
||||
/// sets these). The returned [`Response`] carries the upstream status and
|
||||
/// headers unchanged — including non-2xx — with a body that streams the
|
||||
/// upstream bytes chunk-for-chunk, feeding each chunk to `observer`.
|
||||
pub async fn forward_streaming<O: ChunkObserver>(
|
||||
client: &Client,
|
||||
url: &str,
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
observer: O,
|
||||
) -> Result<Response, StreamError> {
|
||||
let mut req_builder = client.post(url).body(body);
|
||||
for (key, value) in headers.iter() {
|
||||
if key == "host" || key == "content-length" {
|
||||
continue; // reqwest sets these
|
||||
}
|
||||
req_builder = req_builder.header(key, value);
|
||||
}
|
||||
|
||||
let upstream = req_builder.send().await.map_err(StreamError::Upstream)?;
|
||||
|
||||
let status =
|
||||
StatusCode::from_u16(upstream.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY);
|
||||
let resp_headers = upstream.headers().clone();
|
||||
|
||||
let stream = ObservedStream::new(Box::pin(upstream.bytes_stream()), observer);
|
||||
let body = Body::from_stream(stream);
|
||||
|
||||
let mut response = Response::builder().status(status);
|
||||
for (key, value) in resp_headers.iter() {
|
||||
response = response.header(key, value);
|
||||
}
|
||||
response
|
||||
.body(body)
|
||||
.map_err(|e| StreamError::ResponseBuild(e.to_string()))
|
||||
}
|
||||
|
||||
/// Pass-through stream wrapper that feeds a [`ChunkObserver`]. Forwards
|
||||
/// each chunk verbatim, calls `observe` per chunk, and `finish` once on
|
||||
/// clean end-of-stream; the `Drop` impl covers client disconnects.
|
||||
pub struct ObservedStream<O: ChunkObserver> {
|
||||
inner: BoxStream<'static, Result<Bytes, reqwest::Error>>,
|
||||
observer: O,
|
||||
finished: bool,
|
||||
}
|
||||
|
||||
impl<O: ChunkObserver> ObservedStream<O> {
|
||||
/// Wrap a byte stream with an observer.
|
||||
pub fn new(inner: BoxStream<'static, Result<Bytes, reqwest::Error>>, observer: O) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
observer,
|
||||
finished: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn finish(&mut self) {
|
||||
if self.finished {
|
||||
return;
|
||||
}
|
||||
self.finished = true;
|
||||
self.observer.finish();
|
||||
}
|
||||
}
|
||||
|
||||
impl<O: ChunkObserver> Stream for ObservedStream<O> {
|
||||
type Item = Result<Bytes, reqwest::Error>;
|
||||
|
||||
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
let this = self.get_mut();
|
||||
match this.inner.as_mut().poll_next(cx) {
|
||||
Poll::Ready(Some(Ok(chunk))) => {
|
||||
this.observer.observe(&chunk);
|
||||
Poll::Ready(Some(Ok(chunk)))
|
||||
}
|
||||
Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))),
|
||||
Poll::Ready(None) => {
|
||||
this.finish();
|
||||
Poll::Ready(None)
|
||||
}
|
||||
Poll::Pending => Poll::Pending,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<O: ChunkObserver> Drop for ObservedStream<O> {
|
||||
fn drop(&mut self) {
|
||||
self.finish();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn extracts_counts_from_final_sse_usage_chunk() {
|
||||
let tail = concat!(
|
||||
"data: {\"choices\":[{\"delta\":{\"content\":\"hi\"}}]}\n\n",
|
||||
"data: {\"choices\":[],\"usage\":{\"prompt_tokens\":225,",
|
||||
"\"completion_tokens\":42,\"total_tokens\":267}}\n\n",
|
||||
"data: [DONE]\n\n"
|
||||
);
|
||||
assert_eq!(last_count_for(tail, "prompt_tokens"), Some(225));
|
||||
assert_eq!(last_count_for(tail, "completion_tokens"), Some(42));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_counts_from_non_streaming_body() {
|
||||
let tail = "{\"choices\":[{\"message\":{\"content\":\"hi\"}}],\
|
||||
\"usage\":{\"prompt_tokens\": 12, \"completion_tokens\": 7}}";
|
||||
assert_eq!(last_count_for(tail, "prompt_tokens"), Some(12));
|
||||
assert_eq!(last_count_for(tail, "completion_tokens"), Some(7));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ignores_details_variants_and_takes_last_occurrence() {
|
||||
// completion_tokens_details must not shadow completion_tokens,
|
||||
// and the LAST usage object wins (matters when content echoes
|
||||
// a usage-shaped string earlier in the stream).
|
||||
let tail = concat!(
|
||||
"data: {\"usage\":{\"completion_tokens\":1}}\n\n",
|
||||
"data: {\"usage\":{\"completion_tokens\":99,",
|
||||
"\"completion_tokens_details\":{\"reasoning_tokens\":3}}}\n\n"
|
||||
);
|
||||
assert_eq!(last_count_for(tail, "completion_tokens"), Some(99));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_keys_yield_none() {
|
||||
assert_eq!(
|
||||
last_count_for("data: [DONE]\n\n", "completion_tokens"),
|
||||
None
|
||||
);
|
||||
assert_eq!(last_count_for("", "prompt_tokens"), None);
|
||||
// key present but non-numeric value
|
||||
assert_eq!(
|
||||
last_count_for("\"completion_tokens\": null", "completion_tokens"),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn body_tail_retains_usage_after_cap_trim() {
|
||||
// Cap small enough that the filler forces several front-trims, but
|
||||
// (as in production, where cap ≫ the usage object) large enough that
|
||||
// the trailing usage object survives the newest-half retention.
|
||||
let mut tail = BodyTail::new(512);
|
||||
for _ in 0..100 {
|
||||
tail.push(b"data: {\"choices\":[{\"delta\":{\"content\":\"x\"}}]}\n\n");
|
||||
}
|
||||
assert!(tail.as_str().len() <= 512, "cap must bound the tail");
|
||||
tail.push(b"data: {\"usage\":{\"prompt_tokens\":5,\"completion_tokens\":9}}\n\n");
|
||||
assert_eq!(last_count_for(tail.as_str(), "prompt_tokens"), Some(5));
|
||||
assert_eq!(last_count_for(tail.as_str(), "completion_tokens"), Some(9));
|
||||
}
|
||||
}
|
||||
162
crates/helexa-stream/tests/streaming.rs
Normal file
@@ -0,0 +1,162 @@
|
||||
//! Integration tests for the shared streaming proxy (#71): proves a backend
|
||||
//! SSE response is forwarded chunk-for-chunk (no buffering), the observer
|
||||
//! sees every byte and finishes once, and non-2xx is streamed through with
|
||||
//! its status intact — the behaviours both cortex and helexa-router rely on.
|
||||
|
||||
use axum::Router;
|
||||
use axum::body::Body;
|
||||
use axum::http::{HeaderMap, StatusCode};
|
||||
use axum::response::Response;
|
||||
use axum::routing::post;
|
||||
use helexa_stream::{BodyTail, ChunkObserver, forward_streaming, last_count_for};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
/// Observer that records what it saw, for assertions.
|
||||
#[derive(Clone, Default)]
|
||||
struct RecordingObserver {
|
||||
inner: Arc<Mutex<Recorded>>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Recorded {
|
||||
chunks: usize,
|
||||
finished: usize,
|
||||
tail: String,
|
||||
}
|
||||
|
||||
impl ChunkObserver for RecordingObserver {
|
||||
fn observe(&mut self, chunk: &[u8]) {
|
||||
let mut r = self.inner.lock().unwrap();
|
||||
r.chunks += 1;
|
||||
r.tail.push_str(&String::from_utf8_lossy(chunk));
|
||||
}
|
||||
fn finish(&mut self) {
|
||||
self.inner.lock().unwrap().finished += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/// Mock backend that streams 5 SSE chunks with 30ms gaps, then a usage
|
||||
/// chunk and `[DONE]`.
|
||||
async fn sse_handler() -> Response {
|
||||
let chunks: Vec<&'static str> = vec![
|
||||
"data: {\"choices\":[{\"delta\":{\"content\":\"a\"}}]}\n\n",
|
||||
"data: {\"choices\":[{\"delta\":{\"content\":\"b\"}}]}\n\n",
|
||||
"data: {\"choices\":[{\"delta\":{\"content\":\"c\"}}]}\n\n",
|
||||
"data: {\"choices\":[{\"delta\":{\"content\":\"d\"}}]}\n\n",
|
||||
"data: {\"choices\":[{\"delta\":{\"content\":\"e\"}}]}\n\n",
|
||||
"data: {\"choices\":[],\"usage\":{\"prompt_tokens\":11,\"completion_tokens\":5}}\n\n",
|
||||
"data: [DONE]\n\n",
|
||||
];
|
||||
let stream = async_stream::stream! {
|
||||
for c in chunks {
|
||||
tokio::time::sleep(Duration::from_millis(30)).await;
|
||||
yield Ok::<_, std::io::Error>(axum::body::Bytes::from_static(c.as_bytes()));
|
||||
}
|
||||
};
|
||||
Response::new(Body::from_stream(stream))
|
||||
}
|
||||
|
||||
async fn rate_limited_handler() -> Response {
|
||||
Response::builder()
|
||||
.status(StatusCode::TOO_MANY_REQUESTS)
|
||||
.body(Body::from("{\"error\":{\"type\":\"rate_limit_exceeded\"}}"))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn spawn_backend(router: Router) -> String {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, router).await.unwrap();
|
||||
});
|
||||
format!("http://{addr}")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streams_chunks_incrementally_and_observes_usage() {
|
||||
let base = spawn_backend(Router::new().route("/v1/chat/completions", post(sse_handler))).await;
|
||||
let observer = RecordingObserver::default();
|
||||
let probe = observer.clone();
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let resp = forward_streaming(
|
||||
&client,
|
||||
&format!("{base}/v1/chat/completions"),
|
||||
HeaderMap::new(),
|
||||
axum::body::Bytes::from_static(b"{\"model\":\"x\",\"stream\":true}"),
|
||||
observer,
|
||||
)
|
||||
.await
|
||||
.expect("forward ok");
|
||||
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
|
||||
// Read the proxied body as a stream, timestamping arrivals.
|
||||
let mut body = resp.into_body().into_data_stream();
|
||||
let mut arrivals: Vec<Instant> = Vec::new();
|
||||
let mut collected = String::new();
|
||||
use futures::StreamExt;
|
||||
while let Some(item) = body.next().await {
|
||||
let bytes = item.unwrap();
|
||||
arrivals.push(Instant::now());
|
||||
collected.push_str(&String::from_utf8_lossy(&bytes));
|
||||
}
|
||||
|
||||
// Incremental delivery: first and last chunk are meaningfully apart
|
||||
// (5×30ms gaps), proving no full-response buffering.
|
||||
let spread = *arrivals.last().unwrap() - arrivals[0];
|
||||
assert!(
|
||||
spread >= Duration::from_millis(100),
|
||||
"expected incremental delivery, spread was {spread:?}"
|
||||
);
|
||||
|
||||
// The client received the terminator and the usage object verbatim.
|
||||
assert!(collected.contains("data: [DONE]"));
|
||||
|
||||
// The observer saw the bytes and finished exactly once.
|
||||
let r = probe.inner.lock().unwrap();
|
||||
assert!(r.chunks >= 5, "observer saw {} chunks", r.chunks);
|
||||
assert_eq!(r.finished, 1, "finish must run exactly once");
|
||||
assert_eq!(last_count_for(&r.tail, "prompt_tokens"), Some(11));
|
||||
assert_eq!(last_count_for(&r.tail, "completion_tokens"), Some(5));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn non_2xx_is_streamed_through_verbatim() {
|
||||
let base =
|
||||
spawn_backend(Router::new().route("/v1/chat/completions", post(rate_limited_handler)))
|
||||
.await;
|
||||
let observer = RecordingObserver::default();
|
||||
let probe = observer.clone();
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let resp = forward_streaming(
|
||||
&client,
|
||||
&format!("{base}/v1/chat/completions"),
|
||||
HeaderMap::new(),
|
||||
axum::body::Bytes::new(),
|
||||
observer,
|
||||
)
|
||||
.await
|
||||
.expect("forward ok");
|
||||
|
||||
// Backpressure status reaches the client unchanged.
|
||||
assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS);
|
||||
let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(String::from_utf8_lossy(&body).contains("rate_limit_exceeded"));
|
||||
|
||||
// finish still runs once even with a tiny non-streaming body.
|
||||
assert_eq!(probe.inner.lock().unwrap().finished, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn body_tail_smoke() {
|
||||
let mut tail = BodyTail::new(128);
|
||||
tail.push(b"hello ");
|
||||
tail.push(b"world");
|
||||
assert_eq!(tail.as_str(), "hello world");
|
||||
}
|
||||
63
crates/helexa-upstream/Cargo.toml
Normal file
@@ -0,0 +1,63 @@
|
||||
[package]
|
||||
name = "helexa-upstream"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[[bin]]
|
||||
name = "helexa-upstream"
|
||||
path = "src/main.rs"
|
||||
|
||||
[lib]
|
||||
name = "helexa_upstream"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[dependencies]
|
||||
tokio = { workspace = true }
|
||||
axum = { workspace = true }
|
||||
tower-http = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
figment = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
clap = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
|
||||
# PostgreSQL — the mesh authority's system of record. Runtime query API
|
||||
# (not the compile-time `query!` macros) so the crate builds in CI without a
|
||||
# live database or a committed `.sqlx` offline cache; correctness is covered
|
||||
# by the gated integration tests. (Macro adoption is a later refinement once
|
||||
# a dev DB + offline cache exist.)
|
||||
sqlx = { version = "0.8", default-features = false, features = [
|
||||
"runtime-tokio",
|
||||
"tls-rustls",
|
||||
"postgres",
|
||||
"macros",
|
||||
"migrate",
|
||||
"chrono",
|
||||
"uuid",
|
||||
] }
|
||||
uuid = { version = "1", features = ["v4", "serde"] }
|
||||
sha2 = "0.10"
|
||||
subtle = "2.6"
|
||||
# Web auth (B4): argon2id password hashing, JWT sessions, CSPRNG secrets,
|
||||
# transactional email.
|
||||
argon2 = "0.5"
|
||||
jsonwebtoken = "9"
|
||||
rand = "0.8"
|
||||
lettre = { version = "0.11", default-features = false, features = [
|
||||
"tokio1-rustls-tls",
|
||||
"smtp-transport",
|
||||
"builder",
|
||||
] }
|
||||
|
||||
# cortex-core for the shared #63 OpenAiError envelope on the authz surface.
|
||||
cortex-core = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
figment = { workspace = true, features = ["test"] }
|
||||
reqwest = { workspace = true }
|
||||
137
crates/helexa-upstream/migrations/0001_init.sql
Normal file
@@ -0,0 +1,137 @@
|
||||
-- helexa-upstream initial schema (#59): accounts, keys, ledger, top-up
|
||||
-- codes, served-usage. The mesh-level authority's system of record.
|
||||
--
|
||||
-- Token amounts are BIGINT (i64) throughout; the cortex EntitlementProvider
|
||||
-- carries u64 but mesh allocations sit comfortably inside i64 and Postgres
|
||||
-- has no unsigned type.
|
||||
|
||||
CREATE EXTENSION IF NOT EXISTS citext;
|
||||
CREATE EXTENSION IF NOT EXISTS pgcrypto;
|
||||
|
||||
-- ── Users (web auth: email + password) ──────────────────────────────
|
||||
CREATE TABLE users (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
email CITEXT NOT NULL UNIQUE,
|
||||
password_hash TEXT NOT NULL, -- argon2id PHC string
|
||||
email_verified BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
-- Browser fingerprint captured at registration (#abuse). Best-effort,
|
||||
-- client-supplied; the primary signal for silent multi-account
|
||||
-- detection. NULL when the client could not produce one.
|
||||
registration_fingerprint TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX users_registration_fingerprint_idx
|
||||
ON users (registration_fingerprint)
|
||||
WHERE registration_fingerprint IS NOT NULL;
|
||||
|
||||
-- Single-use email tokens for verification and password reset. Only the
|
||||
-- sha256 of the emailed secret is stored.
|
||||
CREATE TABLE email_tokens (
|
||||
token_hash BYTEA PRIMARY KEY,
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
kind TEXT NOT NULL CHECK (kind IN ('verify', 'reset')),
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
consumed_at TIMESTAMPTZ
|
||||
);
|
||||
CREATE INDEX email_tokens_user_idx ON email_tokens (user_id);
|
||||
|
||||
-- ── Accounts (the billable allocation ledger) ───────────────────────
|
||||
CREATE TABLE accounts (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
owner_user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
allocation_total BIGINT NOT NULL DEFAULT 0,
|
||||
allocation_spent BIGINT NOT NULL DEFAULT 0,
|
||||
allocation_reserved BIGINT NOT NULL DEFAULT 0,
|
||||
-- 'deactivated' is the SILENT abuse flag: keys stop authorizing but no
|
||||
-- surface ever tells the user why (see resolve → 401).
|
||||
status TEXT NOT NULL DEFAULT 'active'
|
||||
CHECK (status IN ('active', 'deactivated')),
|
||||
-- This account shares a registration fingerprint with >= 1 other.
|
||||
fingerprint_flagged BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
-- The no-overshoot backstop to the atomic reserve UPDATE.
|
||||
CONSTRAINT accounts_no_overshoot
|
||||
CHECK (allocation_spent + allocation_reserved <= allocation_total),
|
||||
CONSTRAINT accounts_nonneg
|
||||
CHECK (allocation_spent >= 0 AND allocation_reserved >= 0)
|
||||
);
|
||||
CREATE INDEX accounts_owner_idx ON accounts (owner_user_id);
|
||||
|
||||
-- ── API keys (Principal.key_id = api_keys.id) ───────────────────────
|
||||
CREATE TABLE api_keys (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
account_id UUID NOT NULL REFERENCES accounts(id) ON DELETE CASCADE,
|
||||
key_hash BYTEA NOT NULL, -- sha256(raw key)
|
||||
key_prefix TEXT NOT NULL, -- non-secret display prefix
|
||||
label TEXT NOT NULL DEFAULT '',
|
||||
status TEXT NOT NULL DEFAULT 'active'
|
||||
CHECK (status IN ('active', 'archived')),
|
||||
-- Per-key sub-cap: 'hardcap' = absolute tokens; 'percent' = % of the
|
||||
-- account's allocation_total (resolved to an absolute at reserve time).
|
||||
limit_kind TEXT NOT NULL DEFAULT 'percent'
|
||||
CHECK (limit_kind IN ('percent', 'hardcap')),
|
||||
limit_value BIGINT NOT NULL DEFAULT 100,
|
||||
-- serde of cortex_core::entitlements::CapWindow (Balance | Rolling).
|
||||
cap_window JSONB NOT NULL DEFAULT '{"kind":"balance"}'::jsonb,
|
||||
-- Per-key running ledger (mirrors the account ledger; Balance semantics
|
||||
-- in this migration — rolling-window reset lands with the authz API).
|
||||
key_spent BIGINT NOT NULL DEFAULT 0,
|
||||
key_reserved BIGINT NOT NULL DEFAULT 0,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT api_keys_key_nonneg
|
||||
CHECK (key_spent >= 0 AND key_reserved >= 0)
|
||||
);
|
||||
-- A raw key resolves only while active; the hash is unique among active keys.
|
||||
CREATE UNIQUE INDEX api_keys_active_hash_idx
|
||||
ON api_keys (key_hash) WHERE status = 'active';
|
||||
CREATE INDEX api_keys_account_idx ON api_keys (account_id);
|
||||
|
||||
-- ── Reservations (reserve → settle/release) ─────────────────────────
|
||||
-- id is BIGSERIAL so it maps to the cortex Reservation.id (u64) verbatim,
|
||||
-- with the Postgres sequence as the sole global authority.
|
||||
CREATE TABLE reservations (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
account_id UUID NOT NULL REFERENCES accounts(id) ON DELETE CASCADE,
|
||||
key_id UUID NOT NULL REFERENCES api_keys(id) ON DELETE CASCADE,
|
||||
reserved BIGINT NOT NULL,
|
||||
actual BIGINT,
|
||||
state TEXT NOT NULL DEFAULT 'open'
|
||||
CHECK (state IN ('open', 'settled', 'released')),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
settled_at TIMESTAMPTZ
|
||||
);
|
||||
-- The sweeper scans open reservations by age.
|
||||
CREATE INDEX reservations_open_idx
|
||||
ON reservations (created_at) WHERE state = 'open';
|
||||
|
||||
-- ── Top-up codes (hybrid allocation) ────────────────────────────────
|
||||
CREATE TABLE top_up_codes (
|
||||
code_hash BYTEA PRIMARY KEY, -- sha256(raw code)
|
||||
value BIGINT NOT NULL, -- tokens this code grants
|
||||
denomination TEXT, -- human label (e.g. "small")
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
redeemed_by UUID REFERENCES accounts(id) ON DELETE SET NULL,
|
||||
redeemed_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
-- ── Served-usage ledger (#58 reconciliation) ────────────────────────
|
||||
-- Absolute per-(operator, account, key, period) served tokens, upserted by
|
||||
-- each cortex; reconciliation rolls these up for operator compensation.
|
||||
CREATE TABLE served_usage (
|
||||
operator_id TEXT NOT NULL,
|
||||
account_id UUID NOT NULL,
|
||||
key_id UUID NOT NULL,
|
||||
period DATE NOT NULL,
|
||||
served_tokens BIGINT NOT NULL DEFAULT 0,
|
||||
reconciled_at TIMESTAMPTZ,
|
||||
PRIMARY KEY (operator_id, account_id, key_id, period)
|
||||
);
|
||||
|
||||
-- ── Web sessions (DB-backed; alt/complement to stateless JWT) ───────
|
||||
CREATE TABLE sessions (
|
||||
token_hash BYTEA PRIMARY KEY, -- sha256(session token)
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX sessions_user_idx ON sessions (user_id);
|
||||
284
crates/helexa-upstream/src/authz.rs
Normal file
@@ -0,0 +1,284 @@
|
||||
//! `/authz/v1` — the machine surface cortex's `UpstreamEntitlementProvider`
|
||||
//! (#57) consumes. It mirrors the `cortex_core::entitlements::EntitlementProvider`
|
||||
//! trait 1:1 (resolve / reserve / settle / release / snapshot) over the B1
|
||||
//! ledger.
|
||||
//!
|
||||
//! Contract notes for the cortex client:
|
||||
//! - A **non-2xx** response means the authority could not give an
|
||||
//! authoritative answer (bad caller auth, malformed request, server
|
||||
//! error) → the client should **fail closed**.
|
||||
//! - `reserve` returns **200** whether granted or budget-refused: the body
|
||||
//! carries either `reservation_id` or a `rejected` discriminant. A budget
|
||||
//! refusal is an authoritative answer, not a transport failure.
|
||||
//! - Rejections that are genuinely auth failures use the #63 `OpenAiError`
|
||||
//! envelope so they can be surfaced verbatim.
|
||||
|
||||
use crate::crypto::sha256;
|
||||
use crate::error::envelope_response;
|
||||
use crate::ledger::{self, LedgerError};
|
||||
use crate::state::AppState;
|
||||
use axum::extract::{Request, State};
|
||||
use axum::http::{StatusCode, header};
|
||||
use axum::middleware::Next;
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum::routing::post;
|
||||
use axum::{Json, Router};
|
||||
use cortex_core::error_envelope::OpenAiError;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use subtle::ConstantTimeEq;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// The operator a validated client bearer identifies (served-usage
|
||||
/// attribution, #58). Inserted into request extensions by [`client_auth`].
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct OperatorId(pub String);
|
||||
|
||||
/// Build the `/authz/v1` router with the client-auth layer applied.
|
||||
pub fn router(state: &AppState) -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/authz/v1/resolve", post(resolve))
|
||||
.route("/authz/v1/reserve", post(reserve))
|
||||
.route("/authz/v1/settle", post(settle))
|
||||
.route("/authz/v1/release", post(release))
|
||||
.route("/authz/v1/snapshot", post(snapshot))
|
||||
.layer(axum::middleware::from_fn_with_state(
|
||||
state.clone(),
|
||||
client_auth,
|
||||
))
|
||||
}
|
||||
|
||||
// ── client auth (shared bearer → operator_id) ───────────────────────
|
||||
|
||||
/// Validate the caller's `Authorization: Bearer` against the configured
|
||||
/// client tokens (constant-time) and stamp the `operator_id`. When no tokens
|
||||
/// are configured the surface is open (dev) and a synthetic operator is
|
||||
/// used.
|
||||
async fn client_auth(State(state): State<AppState>, mut req: Request, next: Next) -> Response {
|
||||
let tokens = &state.config.client_auth.tokens;
|
||||
if tokens.is_empty() {
|
||||
req.extensions_mut().insert(OperatorId("dev".into()));
|
||||
return next.run(req).await;
|
||||
}
|
||||
let presented = req
|
||||
.headers()
|
||||
.get(header::AUTHORIZATION)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.strip_prefix("Bearer "))
|
||||
.map(str::trim)
|
||||
.unwrap_or("");
|
||||
let matched = tokens
|
||||
.iter()
|
||||
.find(|t| t.token.as_bytes().ct_eq(presented.as_bytes()).into());
|
||||
match matched {
|
||||
Some(t) => {
|
||||
req.extensions_mut()
|
||||
.insert(OperatorId(t.operator_id.clone()));
|
||||
next.run(req).await
|
||||
}
|
||||
None => envelope_response(OpenAiError::invalid_api_key(
|
||||
"missing or invalid client credentials",
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
// ── DTOs ────────────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ResolveReq {
|
||||
api_key: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct PrincipalDto {
|
||||
account_id: String,
|
||||
key_id: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct SnapshotDto {
|
||||
hard_cap: Option<i64>,
|
||||
spent: i64,
|
||||
reserved: i64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ResolveResp {
|
||||
principal: PrincipalDto,
|
||||
snapshot: SnapshotDto,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ReserveReq {
|
||||
account_id: String,
|
||||
key_id: String,
|
||||
max_tokens: i64,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Default)]
|
||||
struct ReserveResp {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
reservation_id: Option<i64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
rejected: Option<Rejection>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
enum Rejection {
|
||||
InsufficientQuota {
|
||||
requested: i64,
|
||||
available: i64,
|
||||
},
|
||||
// Part of the frozen wire contract so the cortex client (#57) can map it
|
||||
// without a later breaking change. Not yet constructed: the B1 ledger
|
||||
// implements Balance caps only; rolling-window key sub-caps (which yield
|
||||
// this) land in a follow-up.
|
||||
#[allow(dead_code)]
|
||||
RateLimited {
|
||||
requested: i64,
|
||||
available: i64,
|
||||
retry_after_secs: u64,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct SettleReq {
|
||||
reservation_id: i64,
|
||||
actual_tokens: i64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ReservationRef {
|
||||
reservation_id: i64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct SnapshotReq {
|
||||
account_id: String,
|
||||
key_id: String,
|
||||
}
|
||||
|
||||
// ── handlers ────────────────────────────────────────────────────────
|
||||
|
||||
/// `POST /authz/v1/resolve` — bearer key → principal + snapshot, or
|
||||
/// `401 invalid_api_key` (also for a deactivated account: no clue).
|
||||
async fn resolve(State(state): State<AppState>, Json(req): Json<ResolveReq>) -> Response {
|
||||
match ledger::resolve_key(&state.pool, &sha256(&req.api_key)).await {
|
||||
Ok(Some(p)) => Json(ResolveResp {
|
||||
principal: PrincipalDto {
|
||||
account_id: p.account_id.to_string(),
|
||||
key_id: p.key_id.to_string(),
|
||||
},
|
||||
snapshot: SnapshotDto {
|
||||
hard_cap: Some(p.hard_cap),
|
||||
spent: p.key_spent,
|
||||
reserved: p.key_reserved,
|
||||
},
|
||||
})
|
||||
.into_response(),
|
||||
Ok(None) => envelope_response(OpenAiError::invalid_api_key("invalid or unknown API key")),
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e, "resolve query failed");
|
||||
envelope_response(OpenAiError::service_unavailable("authority error", Some(5)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `POST /authz/v1/reserve` — 200 with `reservation_id` (granted) or
|
||||
/// `rejected` (budget). Non-2xx only for bad input / server error.
|
||||
async fn reserve(State(state): State<AppState>, Json(req): Json<ReserveReq>) -> Response {
|
||||
let (Ok(account_id), Ok(key_id)) = (
|
||||
Uuid::parse_str(&req.account_id),
|
||||
Uuid::parse_str(&req.key_id),
|
||||
) else {
|
||||
return bad_request("account_id and key_id must be UUIDs");
|
||||
};
|
||||
match ledger::reserve(&state.pool, account_id, key_id, req.max_tokens).await {
|
||||
Ok(reservation_id) => Json(ReserveResp {
|
||||
reservation_id: Some(reservation_id),
|
||||
rejected: None,
|
||||
})
|
||||
.into_response(),
|
||||
Err(LedgerError::InsufficientQuota {
|
||||
requested,
|
||||
available,
|
||||
}) => Json(ReserveResp {
|
||||
reservation_id: None,
|
||||
rejected: Some(Rejection::InsufficientQuota {
|
||||
requested,
|
||||
available,
|
||||
}),
|
||||
})
|
||||
.into_response(),
|
||||
Err(LedgerError::AccountNotFound | LedgerError::KeyNotFound) => {
|
||||
// Resolve succeeded earlier; the principal vanished (archived /
|
||||
// deactivated). Treat as no budget — fail closed at the client.
|
||||
Json(ReserveResp {
|
||||
reservation_id: None,
|
||||
rejected: Some(Rejection::InsufficientQuota {
|
||||
requested: req.max_tokens,
|
||||
available: 0,
|
||||
}),
|
||||
})
|
||||
.into_response()
|
||||
}
|
||||
Err(LedgerError::Db(e)) => {
|
||||
tracing::error!(error = %e, "reserve failed");
|
||||
envelope_response(OpenAiError::service_unavailable("authority error", Some(5)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `POST /authz/v1/settle` — idempotent; `204`.
|
||||
async fn settle(State(state): State<AppState>, Json(req): Json<SettleReq>) -> Response {
|
||||
match ledger::settle(&state.pool, req.reservation_id, req.actual_tokens).await {
|
||||
Ok(()) => StatusCode::NO_CONTENT.into_response(),
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e, "settle failed");
|
||||
envelope_response(OpenAiError::service_unavailable("authority error", Some(5)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `POST /authz/v1/release` — idempotent; `204`.
|
||||
async fn release(State(state): State<AppState>, Json(req): Json<ReservationRef>) -> Response {
|
||||
match ledger::release(&state.pool, req.reservation_id).await {
|
||||
Ok(()) => StatusCode::NO_CONTENT.into_response(),
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e, "release failed");
|
||||
envelope_response(OpenAiError::service_unavailable("authority error", Some(5)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `POST /authz/v1/snapshot` — `{hard_cap, spent, reserved}` or `404`.
|
||||
async fn snapshot(State(state): State<AppState>, Json(req): Json<SnapshotReq>) -> Response {
|
||||
let (Ok(account_id), Ok(key_id)) = (
|
||||
Uuid::parse_str(&req.account_id),
|
||||
Uuid::parse_str(&req.key_id),
|
||||
) else {
|
||||
return bad_request("account_id and key_id must be UUIDs");
|
||||
};
|
||||
match ledger::snapshot(&state.pool, account_id, key_id).await {
|
||||
Ok(Some((hard_cap, spent, reserved))) => Json(SnapshotDto {
|
||||
hard_cap: Some(hard_cap),
|
||||
spent,
|
||||
reserved,
|
||||
})
|
||||
.into_response(),
|
||||
Ok(None) => StatusCode::NOT_FOUND.into_response(),
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e, "snapshot failed");
|
||||
envelope_response(OpenAiError::service_unavailable("authority error", Some(5)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn bad_request(msg: &str) -> Response {
|
||||
envelope_response(OpenAiError::new(
|
||||
400,
|
||||
"invalid_request_error",
|
||||
"invalid_request",
|
||||
msg,
|
||||
))
|
||||
}
|
||||
261
crates/helexa-upstream/src/config.rs
Normal file
@@ -0,0 +1,261 @@
|
||||
//! helexa-upstream configuration: loaded from `helexa-upstream.toml` with
|
||||
//! figment, `UPSTREAM_`-prefixed env overrides (mirrors the cortex/router
|
||||
//! convention, e.g. `UPSTREAM_SERVER__LISTEN`, `UPSTREAM_DB__URL`).
|
||||
|
||||
use figment::{
|
||||
Figment,
|
||||
providers::{Env, Format, Toml},
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::Path;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct UpstreamConfig {
|
||||
#[serde(default)]
|
||||
pub server: ServerSettings,
|
||||
pub db: DbSettings,
|
||||
#[serde(default)]
|
||||
pub grant: GrantSettings,
|
||||
#[serde(default)]
|
||||
pub abuse: AbuseSettings,
|
||||
#[serde(default)]
|
||||
pub client_auth: ClientAuthSettings,
|
||||
#[serde(default)]
|
||||
pub authz: AuthzSettings,
|
||||
#[serde(default)]
|
||||
pub auth: AuthSettings,
|
||||
#[serde(default)]
|
||||
pub email: EmailSettings,
|
||||
}
|
||||
|
||||
/// `[auth]` — web-session signing + token lifetimes (B4). Web sessions are
|
||||
/// JWTs, distinct from inference API keys.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct AuthSettings {
|
||||
/// HMAC secret for signing session JWTs. MUST be overridden in prod
|
||||
/// (env `UPSTREAM_AUTH__JWT_SECRET`); the default is dev-only.
|
||||
#[serde(default = "default_jwt_secret")]
|
||||
pub jwt_secret: String,
|
||||
/// Session token lifetime (seconds).
|
||||
#[serde(default = "default_session_ttl")]
|
||||
pub session_ttl_secs: u64,
|
||||
/// Email verification / password-reset token lifetime (seconds).
|
||||
#[serde(default = "default_email_token_ttl")]
|
||||
pub email_token_ttl_secs: u64,
|
||||
/// Public base URL of the frontend, used to build verify/reset links.
|
||||
#[serde(default = "default_app_base_url")]
|
||||
pub app_base_url: String,
|
||||
}
|
||||
|
||||
impl Default for AuthSettings {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
jwt_secret: default_jwt_secret(),
|
||||
session_ttl_secs: default_session_ttl(),
|
||||
email_token_ttl_secs: default_email_token_ttl(),
|
||||
app_base_url: default_app_base_url(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `[email]` — transactional email transport for verify/reset.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct EmailSettings {
|
||||
/// `"log"` (dev — logs the link) or `"smtp"`.
|
||||
#[serde(default = "default_email_provider")]
|
||||
pub provider: String,
|
||||
/// SMTP relay URL (e.g. "smtp://user:pass@host:587") when provider=smtp.
|
||||
#[serde(default)]
|
||||
pub smtp_url: Option<String>,
|
||||
/// `From:` address.
|
||||
#[serde(default = "default_from_addr")]
|
||||
pub from_addr: String,
|
||||
}
|
||||
|
||||
impl Default for EmailSettings {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
provider: default_email_provider(),
|
||||
smtp_url: None,
|
||||
from_addr: default_from_addr(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `[client_auth]` — credentials operators' cortexes present to `/authz/v1`.
|
||||
/// Each token maps to an `operator_id` (served-usage attribution, #58). This
|
||||
/// transport credential is distinct from end-user API keys (which ride in
|
||||
/// the `resolve` body). v2 adds mTLS.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct ClientAuthSettings {
|
||||
/// When empty the authz surface is **open** (dev only; logged at warn).
|
||||
#[serde(default)]
|
||||
pub tokens: Vec<ClientToken>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ClientToken {
|
||||
/// Shared bearer a cortex presents.
|
||||
pub token: String,
|
||||
/// Operator this token identifies.
|
||||
pub operator_id: String,
|
||||
}
|
||||
|
||||
/// `[authz]` — reservation lifecycle knobs.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct AuthzSettings {
|
||||
/// Open reservations older than this are swept (released), self-healing
|
||||
/// a reservation whose settle/release from cortex was lost.
|
||||
#[serde(default = "default_reservation_ttl")]
|
||||
pub reservation_ttl_secs: u64,
|
||||
/// How often the sweeper runs.
|
||||
#[serde(default = "default_sweep_interval")]
|
||||
pub sweep_interval_secs: u64,
|
||||
}
|
||||
|
||||
impl Default for AuthzSettings {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
reservation_ttl_secs: default_reservation_ttl(),
|
||||
sweep_interval_secs: default_sweep_interval(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ServerSettings {
|
||||
/// Address to listen on (e.g. "0.0.0.0:8090"). Plaintext — edge nginx
|
||||
/// terminates TLS, consistent with the rest of the stack.
|
||||
#[serde(default = "default_listen")]
|
||||
pub listen: String,
|
||||
}
|
||||
|
||||
impl Default for ServerSettings {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
listen: default_listen(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct DbSettings {
|
||||
/// PostgreSQL connection URL (e.g. "postgres://user:pass@host/helexa").
|
||||
pub url: String,
|
||||
/// Max pool connections.
|
||||
#[serde(default = "default_max_connections")]
|
||||
pub max_connections: u32,
|
||||
}
|
||||
|
||||
/// `[grant]` — the flat free token grant every email-verified account
|
||||
/// receives (the floor of the hybrid allocation model; top-up codes extend
|
||||
/// it).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct GrantSettings {
|
||||
#[serde(default = "default_free_grant")]
|
||||
pub free_token_grant: i64,
|
||||
}
|
||||
|
||||
impl Default for GrantSettings {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
free_token_grant: default_free_grant(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `[abuse]` — silent multi-account abuse detection. When at least
|
||||
/// `fingerprint_account_threshold` accounts share one registration
|
||||
/// fingerprint, all of them are silently deactivated (no notice to the
|
||||
/// user; deactivation only surfaces as ordinary inference rejections).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct AbuseSettings {
|
||||
#[serde(default = "default_fingerprint_threshold")]
|
||||
pub fingerprint_account_threshold: i64,
|
||||
}
|
||||
|
||||
impl Default for AbuseSettings {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
fingerprint_account_threshold: default_fingerprint_threshold(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn default_listen() -> String {
|
||||
"0.0.0.0:8090".into()
|
||||
}
|
||||
fn default_max_connections() -> u32 {
|
||||
16
|
||||
}
|
||||
fn default_free_grant() -> i64 {
|
||||
1_000_000
|
||||
}
|
||||
fn default_fingerprint_threshold() -> i64 {
|
||||
5
|
||||
}
|
||||
fn default_reservation_ttl() -> u64 {
|
||||
120
|
||||
}
|
||||
fn default_sweep_interval() -> u64 {
|
||||
60
|
||||
}
|
||||
fn default_jwt_secret() -> String {
|
||||
"dev-insecure-change-me".into()
|
||||
}
|
||||
fn default_session_ttl() -> u64 {
|
||||
7 * 24 * 3600
|
||||
}
|
||||
fn default_email_token_ttl() -> u64 {
|
||||
24 * 3600
|
||||
}
|
||||
fn default_app_base_url() -> String {
|
||||
"http://localhost:5173".into()
|
||||
}
|
||||
fn default_email_provider() -> String {
|
||||
"log".into()
|
||||
}
|
||||
fn default_from_addr() -> String {
|
||||
"helexa <no-reply@helexa.ai>".into()
|
||||
}
|
||||
|
||||
impl UpstreamConfig {
|
||||
/// Load from a TOML file with `UPSTREAM_`-prefixed env overrides
|
||||
/// (`__` nesting separator).
|
||||
pub fn load(path: impl AsRef<Path>) -> Result<Self, Box<figment::Error>> {
|
||||
Figment::new()
|
||||
.merge(Toml::file(path))
|
||||
.merge(Env::prefixed("UPSTREAM_").split("__"))
|
||||
.extract()
|
||||
.map_err(Box::new)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
#[allow(clippy::result_large_err)]
|
||||
fn loads_toml_with_env_override_and_defaults() {
|
||||
figment::Jail::expect_with(|jail| {
|
||||
jail.create_file(
|
||||
"helexa-upstream.toml",
|
||||
r#"
|
||||
[db]
|
||||
url = "postgres://localhost/helexa"
|
||||
"#,
|
||||
)?;
|
||||
jail.set_env("UPSTREAM_SERVER__LISTEN", "127.0.0.1:9099");
|
||||
|
||||
let cfg = UpstreamConfig::load("helexa-upstream.toml").expect("load");
|
||||
assert_eq!(cfg.server.listen, "127.0.0.1:9099");
|
||||
assert_eq!(cfg.db.url, "postgres://localhost/helexa");
|
||||
// Defaults applied when sections omitted.
|
||||
assert_eq!(cfg.grant.free_token_grant, 1_000_000);
|
||||
assert_eq!(cfg.abuse.fingerprint_account_threshold, 5);
|
||||
assert_eq!(cfg.db.max_connections, 16);
|
||||
Ok(())
|
||||
});
|
||||
}
|
||||
}
|
||||
113
crates/helexa-upstream/src/crypto.rs
Normal file
@@ -0,0 +1,113 @@
|
||||
//! Hashing + secret-generation helpers.
|
||||
//!
|
||||
//! - **Passwords** (low-entropy) → argon2id PHC strings.
|
||||
//! - **API keys / top-up codes / email + session tokens** (high-entropy
|
||||
//! secrets minted here) → stored only as their sha256; sha256 is the fast,
|
||||
//! sufficient choice for high-entropy material.
|
||||
|
||||
use argon2::Argon2;
|
||||
use argon2::password_hash::rand_core::OsRng as ArgonOsRng;
|
||||
use argon2::password_hash::{PasswordHash, PasswordHasher, PasswordVerifier, SaltString};
|
||||
use rand::RngCore;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
/// sha256 of `input`, as raw bytes (matches the `BYTEA` columns).
|
||||
pub fn sha256(input: &str) -> Vec<u8> {
|
||||
let mut h = Sha256::new();
|
||||
h.update(input.as_bytes());
|
||||
h.finalize().to_vec()
|
||||
}
|
||||
|
||||
/// Hash a password with argon2id, returning a PHC string for storage.
|
||||
pub fn hash_password(password: &str) -> Result<String, argon2::password_hash::Error> {
|
||||
let salt = SaltString::generate(&mut ArgonOsRng);
|
||||
Ok(Argon2::default()
|
||||
.hash_password(password.as_bytes(), &salt)?
|
||||
.to_string())
|
||||
}
|
||||
|
||||
/// Verify a password against a stored PHC hash. `false` on any mismatch or
|
||||
/// malformed hash (never panics).
|
||||
pub fn verify_password(password: &str, phc: &str) -> bool {
|
||||
match PasswordHash::new(phc) {
|
||||
Ok(parsed) => Argon2::default()
|
||||
.verify_password(password.as_bytes(), &parsed)
|
||||
.is_ok(),
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// A fresh URL-safe high-entropy secret (256 bits) for email/session/reset
|
||||
/// tokens. The caller stores only `sha256` of this and emails/returns the
|
||||
/// raw value.
|
||||
pub fn random_token() -> String {
|
||||
let mut bytes = [0u8; 32];
|
||||
rand::rngs::OsRng.fill_bytes(&mut bytes);
|
||||
base62(&bytes)
|
||||
}
|
||||
|
||||
/// Mint a new API key: `(raw, prefix)`. `raw` is shown to the user once;
|
||||
/// only `sha256(raw)` is stored. The prefix is a non-secret display tag.
|
||||
pub fn generate_api_key() -> (String, String) {
|
||||
let mut bytes = [0u8; 32];
|
||||
rand::rngs::OsRng.fill_bytes(&mut bytes);
|
||||
let raw = format!("sk-helexa-{}", base62(&bytes));
|
||||
// Non-secret prefix for the dashboard list (scheme + first few chars).
|
||||
let prefix: String = raw.chars().take(14).collect();
|
||||
(raw, prefix)
|
||||
}
|
||||
|
||||
/// base62 encode (0-9A-Za-z) — URL/clipboard friendly, no padding.
|
||||
fn base62(bytes: &[u8]) -> String {
|
||||
const ALPHABET: &[u8] = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
|
||||
// Treat the bytes as a big-endian integer and base62 it. 32 bytes → ~43
|
||||
// chars. Simple repeated-division over a big-uint built from the bytes.
|
||||
let mut digits: Vec<u8> = vec![0];
|
||||
for &byte in bytes {
|
||||
let mut carry = byte as u32;
|
||||
for d in digits.iter_mut() {
|
||||
let v = (*d as u32) * 256 + carry;
|
||||
*d = (v % 62) as u8;
|
||||
carry = v / 62;
|
||||
}
|
||||
while carry > 0 {
|
||||
digits.push((carry % 62) as u8);
|
||||
carry /= 62;
|
||||
}
|
||||
}
|
||||
digits
|
||||
.iter()
|
||||
.rev()
|
||||
.map(|&d| ALPHABET[d as usize] as char)
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn password_round_trips_and_rejects_wrong() {
|
||||
let phc = hash_password("correct horse").unwrap();
|
||||
assert!(verify_password("correct horse", &phc));
|
||||
assert!(!verify_password("wrong", &phc));
|
||||
assert!(!verify_password("correct horse", "not-a-phc-string"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn api_key_has_scheme_prefix_and_unique_body() {
|
||||
let (raw, prefix) = generate_api_key();
|
||||
assert!(raw.starts_with("sk-helexa-"));
|
||||
assert!(prefix.starts_with("sk-helexa-"));
|
||||
let (raw2, _) = generate_api_key();
|
||||
assert_ne!(raw, raw2, "keys are unique");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn random_tokens_are_unique_and_nonempty() {
|
||||
let a = random_token();
|
||||
let b = random_token();
|
||||
assert!(!a.is_empty());
|
||||
assert_ne!(a, b);
|
||||
}
|
||||
}
|
||||
20
crates/helexa-upstream/src/db.rs
Normal file
@@ -0,0 +1,20 @@
|
||||
//! PostgreSQL pool + embedded migrations.
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use sqlx::postgres::{PgPool, PgPoolOptions};
|
||||
|
||||
/// Connect to Postgres and run embedded migrations (`./migrations`).
|
||||
pub async fn connect_and_migrate(url: &str, max_connections: u32) -> Result<PgPool> {
|
||||
let pool = PgPoolOptions::new()
|
||||
.max_connections(max_connections)
|
||||
.connect(url)
|
||||
.await
|
||||
.with_context(|| "connecting to PostgreSQL")?;
|
||||
|
||||
sqlx::migrate!("./migrations")
|
||||
.run(&pool)
|
||||
.await
|
||||
.with_context(|| "running migrations")?;
|
||||
|
||||
Ok(pool)
|
||||
}
|
||||
64
crates/helexa-upstream/src/email.rs
Normal file
@@ -0,0 +1,64 @@
|
||||
//! Transactional email for verification + password-reset links.
|
||||
//!
|
||||
//! Two transports: `Log` (dev — writes the link to the log so flows are
|
||||
//! testable without a relay) and `Smtp` (lettre over rustls). Built from
|
||||
//! `[email]` config.
|
||||
|
||||
use crate::config::EmailSettings;
|
||||
use anyhow::{Context, Result};
|
||||
use lettre::message::Mailbox;
|
||||
use lettre::{AsyncSmtpTransport, AsyncTransport, Message, Tokio1Executor};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub enum EmailSender {
|
||||
/// Dev: log the message instead of sending.
|
||||
Log { from: String },
|
||||
Smtp {
|
||||
from: String,
|
||||
transport: AsyncSmtpTransport<Tokio1Executor>,
|
||||
},
|
||||
}
|
||||
|
||||
impl EmailSender {
|
||||
pub fn from_config(cfg: &EmailSettings) -> Result<Self> {
|
||||
match cfg.provider.as_str() {
|
||||
"smtp" => {
|
||||
let url = cfg
|
||||
.smtp_url
|
||||
.as_deref()
|
||||
.context("[email].smtp_url required when provider = \"smtp\"")?;
|
||||
let transport = AsyncSmtpTransport::<Tokio1Executor>::from_url(url)
|
||||
.context("parsing [email].smtp_url")?
|
||||
.build();
|
||||
Ok(EmailSender::Smtp {
|
||||
from: cfg.from_addr.clone(),
|
||||
transport,
|
||||
})
|
||||
}
|
||||
_ => Ok(EmailSender::Log {
|
||||
from: cfg.from_addr.clone(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Send a plaintext email. Errors are returned but the caller treats
|
||||
/// send failures as non-fatal to the request (the user can re-request).
|
||||
pub async fn send(&self, to: &str, subject: &str, body: &str) -> Result<()> {
|
||||
match self {
|
||||
EmailSender::Log { from } => {
|
||||
tracing::info!(%from, %to, %subject, body, "EMAIL (log transport)");
|
||||
Ok(())
|
||||
}
|
||||
EmailSender::Smtp { from, transport } => {
|
||||
let msg = Message::builder()
|
||||
.from(from.parse::<Mailbox>().context("parsing from_addr")?)
|
||||
.to(to.parse::<Mailbox>().context("parsing recipient")?)
|
||||
.subject(subject)
|
||||
.body(body.to_string())
|
||||
.context("building message")?;
|
||||
transport.send(msg).await.context("sending email")?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
21
crates/helexa-upstream/src/error.rs
Normal file
@@ -0,0 +1,21 @@
|
||||
//! Adapter from the shared, axum-agnostic
|
||||
//! [`cortex_core::error_envelope::OpenAiError`] (#60/#63) to an axum
|
||||
//! response, with `Retry-After`. The `/authz/v1` surface speaks the #63
|
||||
//! envelope so cortex (an OpenAI-compatible proxy) can forward rejections
|
||||
//! verbatim. (The future `/web/v1` surface uses a plain JSON error shape.)
|
||||
|
||||
use axum::http::{HeaderValue, StatusCode, header};
|
||||
use axum::response::{IntoResponse, Json, Response};
|
||||
use cortex_core::error_envelope::OpenAiError;
|
||||
|
||||
pub fn envelope_response(err: OpenAiError) -> Response {
|
||||
let status = StatusCode::from_u16(err.status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
|
||||
let retry_after = err.retry_after_secs;
|
||||
let mut response = (status, Json(err.body())).into_response();
|
||||
if let Some(secs) = retry_after
|
||||
&& let Ok(value) = HeaderValue::from_str(&secs.to_string())
|
||||
{
|
||||
response.headers_mut().insert(header::RETRY_AFTER, value);
|
||||
}
|
||||
response
|
||||
}
|
||||
21
crates/helexa-upstream/src/handlers.rs
Normal file
@@ -0,0 +1,21 @@
|
||||
//! HTTP handlers. B1 ships `/health`; the authz (`/authz/v1`) and web
|
||||
//! (`/web/v1`) surfaces land in later phases.
|
||||
|
||||
use crate::state::AppState;
|
||||
use axum::{Json, Router, extract::State, routing::get};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
pub fn routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/health", get(health))
|
||||
.route("/", get(health))
|
||||
}
|
||||
|
||||
/// `GET /health` — liveness + a database round-trip (`SELECT 1`).
|
||||
async fn health(State(state): State<AppState>) -> Json<Value> {
|
||||
let db_ok = sqlx::query("SELECT 1").execute(&state.pool).await.is_ok();
|
||||
Json(json!({
|
||||
"status": if db_ok { "ok" } else { "degraded" },
|
||||
"db": if db_ok { "ok" } else { "unreachable" },
|
||||
}))
|
||||
}
|
||||
338
crates/helexa-upstream/src/ledger.rs
Normal file
@@ -0,0 +1,338 @@
|
||||
//! The allocation ledger: reserve → settle/release with the no-overshoot
|
||||
//! guarantee enforced by a row-locked transaction.
|
||||
//!
|
||||
//! Each reserve takes `SELECT … FOR UPDATE` on the account (and key) row, so
|
||||
//! concurrent reserves from many cortexes serialize and `spent + reserved`
|
||||
//! can never exceed the effective cap. The `accounts_no_overshoot` CHECK is
|
||||
//! the DB-level backstop. Settle/release are idempotent (they only act on a
|
||||
//! reservation still in `open`).
|
||||
//!
|
||||
//! Per-key effective cap = `min(resolved key cap, remaining account
|
||||
//! allocation)`. The key cap is resolved from its `limit_kind`:
|
||||
//! `hardcap` → the value verbatim; `percent` → that % of the account's
|
||||
//! `allocation_total`.
|
||||
//!
|
||||
//! Cap-window semantics: this module implements **Balance** (non-resetting)
|
||||
//! caps. Rolling-window key sub-caps (and the `RateLimited` rejection that
|
||||
//! rides them) land with the authz API (B2); today an over-cap is always
|
||||
//! `InsufficientQuota`.
|
||||
|
||||
use sqlx::postgres::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// A bearer key resolved to its principal + a budget snapshot.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ResolvedPrincipal {
|
||||
pub account_id: Uuid,
|
||||
pub key_id: Uuid,
|
||||
/// Effective per-key absolute cap (the key sub-cap; the account cap
|
||||
/// still binds at reserve time).
|
||||
pub hard_cap: i64,
|
||||
pub key_spent: i64,
|
||||
pub key_reserved: i64,
|
||||
}
|
||||
|
||||
/// Resolve a key by its `sha256` hash to its principal, or `None` when the
|
||||
/// key is unknown/archived **or its account is deactivated** (the silent
|
||||
/// abuse flag — indistinguishable from an unknown key, by design: no clue).
|
||||
pub async fn resolve_key(
|
||||
pool: &PgPool,
|
||||
key_hash: &[u8],
|
||||
) -> Result<Option<ResolvedPrincipal>, sqlx::Error> {
|
||||
let row = sqlx::query(
|
||||
"SELECT k.id AS key_id, k.account_id, k.limit_kind, k.limit_value, \
|
||||
k.key_spent, k.key_reserved, a.allocation_total \
|
||||
FROM api_keys k JOIN accounts a ON a.id = k.account_id \
|
||||
WHERE k.key_hash = $1 AND k.status = 'active' AND a.status = 'active'",
|
||||
)
|
||||
.bind(key_hash)
|
||||
.fetch_optional(pool)
|
||||
.await?;
|
||||
Ok(row.map(|r| {
|
||||
let total: i64 = sqlx::Row::get(&r, "allocation_total");
|
||||
let limit_kind: String = sqlx::Row::get(&r, "limit_kind");
|
||||
let limit_value: i64 = sqlx::Row::get(&r, "limit_value");
|
||||
ResolvedPrincipal {
|
||||
account_id: sqlx::Row::get(&r, "account_id"),
|
||||
key_id: sqlx::Row::get(&r, "key_id"),
|
||||
hard_cap: resolve_abs_cap(&limit_kind, limit_value, total),
|
||||
key_spent: sqlx::Row::get(&r, "key_spent"),
|
||||
key_reserved: sqlx::Row::get(&r, "key_reserved"),
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
/// Per-key budget snapshot `(hard_cap, spent, reserved)`, or `None` if the
|
||||
/// key/account isn't an active pair.
|
||||
pub async fn snapshot(
|
||||
pool: &PgPool,
|
||||
account_id: Uuid,
|
||||
key_id: Uuid,
|
||||
) -> Result<Option<(i64, i64, i64)>, sqlx::Error> {
|
||||
let row = sqlx::query(
|
||||
"SELECT k.limit_kind, k.limit_value, k.key_spent, k.key_reserved, a.allocation_total \
|
||||
FROM api_keys k JOIN accounts a ON a.id = k.account_id \
|
||||
WHERE k.id = $1 AND k.account_id = $2 AND k.status = 'active' AND a.status = 'active'",
|
||||
)
|
||||
.bind(key_id)
|
||||
.bind(account_id)
|
||||
.fetch_optional(pool)
|
||||
.await?;
|
||||
Ok(row.map(|r| {
|
||||
let total: i64 = sqlx::Row::get(&r, "allocation_total");
|
||||
let limit_kind: String = sqlx::Row::get(&r, "limit_kind");
|
||||
let limit_value: i64 = sqlx::Row::get(&r, "limit_value");
|
||||
let cap = resolve_abs_cap(&limit_kind, limit_value, total);
|
||||
(
|
||||
cap,
|
||||
sqlx::Row::get::<i64, _>(&r, "key_spent"),
|
||||
sqlx::Row::get::<i64, _>(&r, "key_reserved"),
|
||||
)
|
||||
}))
|
||||
}
|
||||
|
||||
/// Release every `open` reservation older than `max_age_secs`, returning
|
||||
/// each one's reserved tokens to its account and key in a single statement.
|
||||
/// The lost-settle self-heal. Returns the number swept.
|
||||
pub async fn sweep_stale(pool: &PgPool, max_age_secs: i64) -> Result<u64, sqlx::Error> {
|
||||
// Data-modifying CTEs: release stale rows, then fold their reserved sums
|
||||
// back into accounts and api_keys. All in one atomic statement.
|
||||
let result = sqlx::query(
|
||||
"WITH stale AS ( \
|
||||
UPDATE reservations SET state = 'released', settled_at = now() \
|
||||
WHERE state = 'open' AND created_at < now() - make_interval(secs => $1) \
|
||||
RETURNING account_id, key_id, reserved \
|
||||
), acct AS ( \
|
||||
UPDATE accounts a SET allocation_reserved = allocation_reserved - s.total \
|
||||
FROM (SELECT account_id, SUM(reserved) AS total FROM stale GROUP BY account_id) s \
|
||||
WHERE a.id = s.account_id \
|
||||
) \
|
||||
UPDATE api_keys k SET key_reserved = key_reserved - s.total \
|
||||
FROM (SELECT key_id, SUM(reserved) AS total FROM stale GROUP BY key_id) s \
|
||||
WHERE k.id = s.key_id",
|
||||
)
|
||||
.bind(max_age_secs as f64)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(result.rows_affected())
|
||||
}
|
||||
|
||||
/// Resolve a key's per-key cap to an absolute token count.
|
||||
///
|
||||
/// `percent` is `floor(allocation_total * limit_value / 100)`; `hardcap` is
|
||||
/// `limit_value` verbatim. Computed in i128 to avoid overflow, floored at 0.
|
||||
pub fn resolve_abs_cap(limit_kind: &str, limit_value: i64, allocation_total: i64) -> i64 {
|
||||
let cap = match limit_kind {
|
||||
"percent" => (allocation_total as i128 * limit_value as i128) / 100,
|
||||
_ => limit_value as i128, // "hardcap" (and any unknown → treat as absolute)
|
||||
};
|
||||
cap.clamp(0, i64::MAX as i128) as i64
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum LedgerError {
|
||||
#[error("account not found")]
|
||||
AccountNotFound,
|
||||
#[error("api key not found or not active")]
|
||||
KeyNotFound,
|
||||
/// Account balance or a Balance-window key sub-cap is exhausted.
|
||||
#[error("insufficient quota: requested {requested}, available {available}")]
|
||||
InsufficientQuota { requested: i64, available: i64 },
|
||||
#[error(transparent)]
|
||||
Db(#[from] sqlx::Error),
|
||||
}
|
||||
|
||||
/// Reserve `max_tokens` against `account_id`/`key_id`. Returns the
|
||||
/// reservation id (the `BIGSERIAL`, mapped to the cortex `Reservation.id`).
|
||||
pub async fn reserve(
|
||||
pool: &PgPool,
|
||||
account_id: Uuid,
|
||||
key_id: Uuid,
|
||||
max_tokens: i64,
|
||||
) -> Result<i64, LedgerError> {
|
||||
let mut tx = pool.begin().await?;
|
||||
|
||||
// Lock the account row — serializes concurrent reserves on this account.
|
||||
let acct = sqlx::query(
|
||||
"SELECT allocation_total, allocation_spent, allocation_reserved \
|
||||
FROM accounts WHERE id = $1 AND status = 'active' FOR UPDATE",
|
||||
)
|
||||
.bind(account_id)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
let Some(acct) = acct else {
|
||||
return Err(LedgerError::AccountNotFound);
|
||||
};
|
||||
let total: i64 = sqlx::Row::get(&acct, "allocation_total");
|
||||
let spent: i64 = sqlx::Row::get(&acct, "allocation_spent");
|
||||
let reserved: i64 = sqlx::Row::get(&acct, "allocation_reserved");
|
||||
let account_avail = total - spent - reserved;
|
||||
|
||||
// Lock the key row and resolve its absolute sub-cap.
|
||||
let key = sqlx::query(
|
||||
"SELECT limit_kind, limit_value, key_spent, key_reserved \
|
||||
FROM api_keys WHERE id = $1 AND account_id = $2 AND status = 'active' FOR UPDATE",
|
||||
)
|
||||
.bind(key_id)
|
||||
.bind(account_id)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
let Some(key) = key else {
|
||||
return Err(LedgerError::KeyNotFound);
|
||||
};
|
||||
let limit_kind: String = sqlx::Row::get(&key, "limit_kind");
|
||||
let limit_value: i64 = sqlx::Row::get(&key, "limit_value");
|
||||
let key_spent: i64 = sqlx::Row::get(&key, "key_spent");
|
||||
let key_reserved: i64 = sqlx::Row::get(&key, "key_reserved");
|
||||
let key_cap = resolve_abs_cap(&limit_kind, limit_value, total);
|
||||
let key_avail = key_cap - key_spent - key_reserved;
|
||||
|
||||
let available = account_avail.min(key_avail).max(0);
|
||||
if max_tokens > available {
|
||||
// tx rolls back on drop
|
||||
return Err(LedgerError::InsufficientQuota {
|
||||
requested: max_tokens,
|
||||
available,
|
||||
});
|
||||
}
|
||||
|
||||
let id: i64 = sqlx::Row::get(
|
||||
&sqlx::query(
|
||||
"INSERT INTO reservations (account_id, key_id, reserved, state) \
|
||||
VALUES ($1, $2, $3, 'open') RETURNING id",
|
||||
)
|
||||
.bind(account_id)
|
||||
.bind(key_id)
|
||||
.bind(max_tokens)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?,
|
||||
"id",
|
||||
);
|
||||
sqlx::query("UPDATE accounts SET allocation_reserved = allocation_reserved + $1 WHERE id = $2")
|
||||
.bind(max_tokens)
|
||||
.bind(account_id)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
sqlx::query("UPDATE api_keys SET key_reserved = key_reserved + $1 WHERE id = $2")
|
||||
.bind(max_tokens)
|
||||
.bind(key_id)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
tx.commit().await?;
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// Settle a reservation with the actual tokens used (clamped to
|
||||
/// `[0, reserved]`). Idempotent: a second settle (or settle after release)
|
||||
/// is a no-op.
|
||||
pub async fn settle(
|
||||
pool: &PgPool,
|
||||
reservation_id: i64,
|
||||
actual_tokens: i64,
|
||||
) -> Result<(), LedgerError> {
|
||||
let mut tx = pool.begin().await?;
|
||||
let row = sqlx::query(
|
||||
"UPDATE reservations SET state = 'settled', settled_at = now(), \
|
||||
actual = LEAST(GREATEST($2, 0), reserved) \
|
||||
WHERE id = $1 AND state = 'open' \
|
||||
RETURNING reserved, account_id, key_id, actual",
|
||||
)
|
||||
.bind(reservation_id)
|
||||
.bind(actual_tokens)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
let Some(row) = row else {
|
||||
return Ok(()); // already settled/released, or unknown → idempotent no-op
|
||||
};
|
||||
let reserved: i64 = sqlx::Row::get(&row, "reserved");
|
||||
let actual: i64 = sqlx::Row::get(&row, "actual");
|
||||
let account_id: Uuid = sqlx::Row::get(&row, "account_id");
|
||||
let key_id: Uuid = sqlx::Row::get(&row, "key_id");
|
||||
|
||||
sqlx::query(
|
||||
"UPDATE accounts SET allocation_reserved = allocation_reserved - $1, \
|
||||
allocation_spent = allocation_spent + $2 WHERE id = $3",
|
||||
)
|
||||
.bind(reserved)
|
||||
.bind(actual)
|
||||
.bind(account_id)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
sqlx::query(
|
||||
"UPDATE api_keys SET key_reserved = key_reserved - $1, key_spent = key_spent + $2 WHERE id = $3",
|
||||
)
|
||||
.bind(reserved)
|
||||
.bind(actual)
|
||||
.bind(key_id)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
tx.commit().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Release a reservation, returning its full reserved amount to the
|
||||
/// allocation. Idempotent.
|
||||
pub async fn release(pool: &PgPool, reservation_id: i64) -> Result<(), LedgerError> {
|
||||
let mut tx = pool.begin().await?;
|
||||
let row = sqlx::query(
|
||||
"UPDATE reservations SET state = 'released', settled_at = now() \
|
||||
WHERE id = $1 AND state = 'open' \
|
||||
RETURNING reserved, account_id, key_id",
|
||||
)
|
||||
.bind(reservation_id)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
let Some(row) = row else {
|
||||
return Ok(());
|
||||
};
|
||||
let reserved: i64 = sqlx::Row::get(&row, "reserved");
|
||||
let account_id: Uuid = sqlx::Row::get(&row, "account_id");
|
||||
let key_id: Uuid = sqlx::Row::get(&row, "key_id");
|
||||
|
||||
sqlx::query("UPDATE accounts SET allocation_reserved = allocation_reserved - $1 WHERE id = $2")
|
||||
.bind(reserved)
|
||||
.bind(account_id)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
sqlx::query("UPDATE api_keys SET key_reserved = key_reserved - $1 WHERE id = $2")
|
||||
.bind(reserved)
|
||||
.bind(key_id)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
tx.commit().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::resolve_abs_cap;
|
||||
|
||||
#[test]
|
||||
fn hardcap_is_verbatim() {
|
||||
assert_eq!(resolve_abs_cap("hardcap", 50_000, 1_000_000), 50_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn percent_is_fraction_of_allocation() {
|
||||
assert_eq!(resolve_abs_cap("percent", 25, 1_000_000), 250_000);
|
||||
assert_eq!(resolve_abs_cap("percent", 100, 1_000_000), 1_000_000);
|
||||
// floor
|
||||
assert_eq!(resolve_abs_cap("percent", 33, 10), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn percent_does_not_overflow_on_large_allocation() {
|
||||
// total * value would overflow i64 if not widened to i128.
|
||||
let cap = resolve_abs_cap("percent", 100, i64::MAX);
|
||||
assert_eq!(cap, i64::MAX);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn negative_or_zero_clamps_to_zero() {
|
||||
assert_eq!(resolve_abs_cap("hardcap", -5, 100), 0);
|
||||
assert_eq!(resolve_abs_cap("percent", 0, 1_000_000), 0);
|
||||
}
|
||||
}
|
||||
88
crates/helexa-upstream/src/lib.rs
Normal file
@@ -0,0 +1,88 @@
|
||||
//! helexa-upstream — the mesh-level account/authorization authority (#59).
|
||||
//!
|
||||
//! The clearing house above cortex: it issues accounts and API keys, holds
|
||||
//! the real token-allocation ledger, authorizes inference in real time
|
||||
//! (reserve → settle, fail-closed), and tracks served usage for operator
|
||||
//! reconciliation. cortex's `UpstreamEntitlementProvider` (#57) is a client
|
||||
//! of the `/authz/v1` surface; the helexa.ai frontend is a client of the
|
||||
//! `/web/v1` surface.
|
||||
//!
|
||||
//! Landed so far: B1 — schema + reserve→settle [`ledger`] (no-overshoot) +
|
||||
//! `/health`. B2 — the `/authz/v1` [`authz`] surface (resolve/reserve/
|
||||
//! settle/release/snapshot) with shared-bearer client auth and a
|
||||
//! stale-reservation sweeper.
|
||||
|
||||
pub mod authz;
|
||||
pub mod config;
|
||||
pub mod crypto;
|
||||
pub mod db;
|
||||
pub mod email;
|
||||
pub mod error;
|
||||
pub mod handlers;
|
||||
pub mod ledger;
|
||||
pub mod state;
|
||||
pub mod web;
|
||||
|
||||
use anyhow::Result;
|
||||
use config::UpstreamConfig;
|
||||
use email::EmailSender;
|
||||
use state::AppState;
|
||||
use std::time::Duration;
|
||||
use tower_http::cors::CorsLayer;
|
||||
use tower_http::trace::TraceLayer;
|
||||
|
||||
/// Build the axum application.
|
||||
pub fn build_app(state: AppState) -> axum::Router {
|
||||
axum::Router::new()
|
||||
.merge(handlers::routes())
|
||||
.merge(authz::router(&state))
|
||||
.merge(web::router(&state))
|
||||
// The /web/v1 surface is called cross-origin by the browser SPA in
|
||||
// dev; same-origin behind nginx in prod. Permissive is fine — these
|
||||
// endpoints authenticate via bearer/JWT, not cookies.
|
||||
.layer(CorsLayer::permissive())
|
||||
.layer(TraceLayer::new_for_http())
|
||||
.with_state(state)
|
||||
}
|
||||
|
||||
/// Start the service: connect Postgres, run migrations, spawn the
|
||||
/// reservation sweeper, bind the listener.
|
||||
pub async fn run(config: UpstreamConfig) -> Result<()> {
|
||||
let pool = db::connect_and_migrate(&config.db.url, config.db.max_connections).await?;
|
||||
let email = EmailSender::from_config(&config.email)?;
|
||||
let listen = config.server.listen.clone();
|
||||
let state = AppState::new(pool, config, email);
|
||||
|
||||
if state.config.client_auth.tokens.is_empty() {
|
||||
tracing::warn!(
|
||||
"no [client_auth] tokens configured — the /authz/v1 surface is OPEN (dev only)"
|
||||
);
|
||||
}
|
||||
|
||||
// Stale-reservation sweeper: releases open reservations whose
|
||||
// settle/release from cortex was lost, self-healing allocation_reserved.
|
||||
spawn_sweeper(&state);
|
||||
|
||||
let addr = listen.parse::<std::net::SocketAddr>()?;
|
||||
tracing::info!("helexa-upstream listening on {addr}");
|
||||
|
||||
let listener = tokio::net::TcpListener::bind(addr).await?;
|
||||
axum::serve(listener, build_app(state)).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn spawn_sweeper(state: &AppState) {
|
||||
let pool = state.pool.clone();
|
||||
let ttl = state.config.authz.reservation_ttl_secs as i64;
|
||||
let interval = Duration::from_secs(state.config.authz.sweep_interval_secs);
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::time::sleep(interval).await;
|
||||
match ledger::sweep_stale(&pool, ttl).await {
|
||||
Ok(n) if n > 0 => tracing::info!(swept = n, "released stale reservations"),
|
||||
Ok(_) => {}
|
||||
Err(e) => tracing::warn!(error = %e, "reservation sweep failed"),
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
46
crates/helexa-upstream/src/main.rs
Normal file
@@ -0,0 +1,46 @@
|
||||
use anyhow::Result;
|
||||
use clap::{Parser, Subcommand};
|
||||
use helexa_upstream::config::UpstreamConfig;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "helexa-upstream")]
|
||||
#[command(about = "Mesh-level account & authorization authority for helexa")]
|
||||
#[command(version)]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
command: Commands,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Commands {
|
||||
/// Start the upstream server.
|
||||
Serve {
|
||||
/// Path to the config file.
|
||||
#[arg(short, long, default_value = "helexa-upstream.toml")]
|
||||
config: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new("info,helexa_upstream=debug")),
|
||||
)
|
||||
.init();
|
||||
|
||||
let cli = Cli::parse();
|
||||
|
||||
match cli.command {
|
||||
Commands::Serve { config } => {
|
||||
let cfg = UpstreamConfig::load(&config)
|
||||
.map_err(|e| anyhow::anyhow!("failed to load config from '{config}': {e}"))?;
|
||||
tracing::info!(listen = %cfg.server.listen, "starting helexa-upstream");
|
||||
helexa_upstream::run(cfg).await?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
23
crates/helexa-upstream/src/state.rs
Normal file
@@ -0,0 +1,23 @@
|
||||
//! Shared application state.
|
||||
|
||||
use crate::config::UpstreamConfig;
|
||||
use crate::email::EmailSender;
|
||||
use sqlx::postgres::PgPool;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct AppState {
|
||||
pub pool: PgPool,
|
||||
pub config: Arc<UpstreamConfig>,
|
||||
pub email: EmailSender,
|
||||
}
|
||||
|
||||
impl AppState {
|
||||
pub fn new(pool: PgPool, config: UpstreamConfig, email: EmailSender) -> Self {
|
||||
Self {
|
||||
pool,
|
||||
config: Arc::new(config),
|
||||
email,
|
||||
}
|
||||
}
|
||||
}
|
||||
567
crates/helexa-upstream/src/web.rs
Normal file
@@ -0,0 +1,567 @@
|
||||
//! `/web/v1` — the human-facing account API the helexa.ai frontend (#F4)
|
||||
//! consumes: email+password auth (register / verify / login / reset),
|
||||
//! API-key CRUD with per-key limits, and the account balance. Web sessions
|
||||
//! are JWTs, **distinct** from inference API keys.
|
||||
//!
|
||||
//! Errors use a plain JSON shape `{ "error": { "message", "code" } }` (web
|
||||
//! clients, not OpenAI clients — the #63 envelope is the authz surface).
|
||||
//!
|
||||
//! Silent fingerprint abuse (no clue to the abuser): registration captures
|
||||
//! the browser fingerprint and always succeeds; when ≥ threshold accounts
|
||||
//! share one fingerprint, all are silently `deactivated` (keys then resolve
|
||||
//! as ordinary `401`s at the authz surface — never a "banned" signal).
|
||||
|
||||
use crate::crypto::{generate_api_key, hash_password, random_token, sha256, verify_password};
|
||||
use crate::state::AppState;
|
||||
use axum::extract::{Path, Request, State};
|
||||
use axum::http::{StatusCode, header};
|
||||
use axum::middleware::Next;
|
||||
use axum::response::{IntoResponse, Json, Response};
|
||||
use axum::routing::{get, post};
|
||||
use axum::{Extension, Router};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use jsonwebtoken::{DecodingKey, EncodingKey, Header, Validation, decode, encode};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::json;
|
||||
use sqlx::Row;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub fn router(state: &AppState) -> Router<AppState> {
|
||||
let protected = Router::new()
|
||||
.route("/web/v1/account", get(account))
|
||||
.route("/web/v1/keys", get(list_keys).post(create_key))
|
||||
.route("/web/v1/keys/{id}/archive", post(archive_key))
|
||||
.route(
|
||||
"/web/v1/keys/{id}/limit",
|
||||
axum::routing::patch(update_key_limit),
|
||||
)
|
||||
.layer(axum::middleware::from_fn_with_state(
|
||||
state.clone(),
|
||||
require_session,
|
||||
));
|
||||
|
||||
Router::new()
|
||||
.route("/web/v1/register", post(register))
|
||||
.route("/web/v1/verify", post(verify))
|
||||
.route("/web/v1/login", post(login))
|
||||
.route("/web/v1/password-reset/request", post(reset_request))
|
||||
.route("/web/v1/password-reset/confirm", post(reset_confirm))
|
||||
.merge(protected)
|
||||
}
|
||||
|
||||
// ── errors ──────────────────────────────────────────────────────────
|
||||
|
||||
enum WebError {
|
||||
BadRequest(&'static str),
|
||||
Unauthorized,
|
||||
Internal,
|
||||
}
|
||||
|
||||
impl IntoResponse for WebError {
|
||||
fn into_response(self) -> Response {
|
||||
let (status, code, message) = match self {
|
||||
WebError::BadRequest(m) => (StatusCode::BAD_REQUEST, "bad_request", m),
|
||||
WebError::Unauthorized => (StatusCode::UNAUTHORIZED, "unauthorized", "unauthorized"),
|
||||
WebError::Internal => (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"internal_error",
|
||||
"internal error",
|
||||
),
|
||||
};
|
||||
(
|
||||
status,
|
||||
Json(json!({"error": {"message": message, "code": code}})),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
}
|
||||
|
||||
impl From<sqlx::Error> for WebError {
|
||||
fn from(e: sqlx::Error) -> Self {
|
||||
tracing::error!(error = %e, "web db error");
|
||||
WebError::Internal
|
||||
}
|
||||
}
|
||||
|
||||
type WebResult<T> = Result<T, WebError>;
|
||||
|
||||
// ── sessions (JWT) ──────────────────────────────────────────────────
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
struct Claims {
|
||||
sub: String, // user id
|
||||
exp: usize,
|
||||
}
|
||||
|
||||
fn mint_session(state: &AppState, user_id: Uuid) -> WebResult<String> {
|
||||
let exp = (Utc::now() + Duration::seconds(state.config.auth.session_ttl_secs as i64))
|
||||
.timestamp() as usize;
|
||||
let claims = Claims {
|
||||
sub: user_id.to_string(),
|
||||
exp,
|
||||
};
|
||||
encode(
|
||||
&Header::default(),
|
||||
&claims,
|
||||
&EncodingKey::from_secret(state.config.auth.jwt_secret.as_bytes()),
|
||||
)
|
||||
.map_err(|_| WebError::Internal)
|
||||
}
|
||||
|
||||
/// Authenticated user id, injected by [`require_session`].
|
||||
#[derive(Clone)]
|
||||
struct AuthUser(Uuid);
|
||||
|
||||
async fn require_session(State(state): State<AppState>, mut req: Request, next: Next) -> Response {
|
||||
let token = req
|
||||
.headers()
|
||||
.get(header::AUTHORIZATION)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.strip_prefix("Bearer "))
|
||||
.map(str::trim);
|
||||
let Some(token) = token else {
|
||||
return WebError::Unauthorized.into_response();
|
||||
};
|
||||
let decoded = decode::<Claims>(
|
||||
token,
|
||||
&DecodingKey::from_secret(state.config.auth.jwt_secret.as_bytes()),
|
||||
&Validation::default(),
|
||||
);
|
||||
match decoded
|
||||
.ok()
|
||||
.and_then(|d| Uuid::parse_str(&d.claims.sub).ok())
|
||||
{
|
||||
Some(uid) => {
|
||||
req.extensions_mut().insert(AuthUser(uid));
|
||||
next.run(req).await
|
||||
}
|
||||
None => WebError::Unauthorized.into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The caller's single account id.
|
||||
async fn account_id_for(state: &AppState, user_id: Uuid) -> WebResult<Uuid> {
|
||||
let row = sqlx::query("SELECT id FROM accounts WHERE owner_user_id = $1")
|
||||
.bind(user_id)
|
||||
.fetch_optional(&state.pool)
|
||||
.await?;
|
||||
row.map(|r| r.get::<Uuid, _>("id"))
|
||||
.ok_or(WebError::Internal)
|
||||
}
|
||||
|
||||
// ── auth lifecycle ──────────────────────────────────────────────────
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct RegisterReq {
|
||||
email: String,
|
||||
password: String,
|
||||
#[serde(default)]
|
||||
fingerprint: Option<String>,
|
||||
}
|
||||
|
||||
/// `POST /web/v1/register` — always returns `202`, regardless of whether the
|
||||
/// email was new, already taken, or fingerprint-flagged (no enumeration, no
|
||||
/// abuse clue).
|
||||
async fn register(State(state): State<AppState>, Json(req): Json<RegisterReq>) -> Response {
|
||||
match register_inner(&state, req).await {
|
||||
Ok(()) | Err(WebError::BadRequest(_)) => {}
|
||||
Err(e) => return e.into_response(),
|
||||
}
|
||||
// Generic 202 whatever happened above (except hard server errors).
|
||||
StatusCode::ACCEPTED.into_response()
|
||||
}
|
||||
|
||||
async fn register_inner(state: &AppState, req: RegisterReq) -> WebResult<()> {
|
||||
if !req.email.contains('@') {
|
||||
return Err(WebError::BadRequest("invalid email"));
|
||||
}
|
||||
if req.password.len() < 8 {
|
||||
return Err(WebError::BadRequest("password too short (min 8)"));
|
||||
}
|
||||
let phc = hash_password(&req.password).map_err(|_| WebError::Internal)?;
|
||||
|
||||
// Insert the user; a duplicate email silently no-ops (no enumeration).
|
||||
let user_id: Option<Uuid> = sqlx::query(
|
||||
"INSERT INTO users (email, password_hash, registration_fingerprint) \
|
||||
VALUES ($1, $2, $3) ON CONFLICT (email) DO NOTHING RETURNING id",
|
||||
)
|
||||
.bind(&req.email)
|
||||
.bind(&phc)
|
||||
.bind(&req.fingerprint)
|
||||
.fetch_optional(&state.pool)
|
||||
.await?
|
||||
.map(|r| r.get("id"));
|
||||
|
||||
let Some(user_id) = user_id else {
|
||||
return Ok(()); // email already registered — say nothing
|
||||
};
|
||||
|
||||
// Account with the flat free grant.
|
||||
sqlx::query("INSERT INTO accounts (owner_user_id, allocation_total) VALUES ($1, $2)")
|
||||
.bind(user_id)
|
||||
.bind(state.config.grant.free_token_grant)
|
||||
.execute(&state.pool)
|
||||
.await?;
|
||||
|
||||
// Silent fingerprint abuse handling.
|
||||
if let Some(fp) = req.fingerprint.as_deref().filter(|f| !f.is_empty()) {
|
||||
apply_fingerprint_policy(state, fp).await?;
|
||||
}
|
||||
|
||||
// Email verification link.
|
||||
let token = random_token();
|
||||
let expires: DateTime<Utc> =
|
||||
Utc::now() + Duration::seconds(state.config.auth.email_token_ttl_secs as i64);
|
||||
sqlx::query(
|
||||
"INSERT INTO email_tokens (token_hash, user_id, kind, expires_at) \
|
||||
VALUES ($1, $2, 'verify', $3)",
|
||||
)
|
||||
.bind(sha256(&token))
|
||||
.bind(user_id)
|
||||
.bind(expires)
|
||||
.execute(&state.pool)
|
||||
.await?;
|
||||
|
||||
let link = format!("{}/verify?token={token}", state.config.auth.app_base_url);
|
||||
let _ = state
|
||||
.email
|
||||
.send(
|
||||
&req.email,
|
||||
"Verify your helexa account",
|
||||
&format!("Welcome to helexa. Verify your email:\n\n{link}\n"),
|
||||
)
|
||||
.await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Count accounts sharing `fp`; flag them, and silently deactivate all once
|
||||
/// the count reaches the configured threshold. No response difference — the
|
||||
/// abuser gets no signal.
|
||||
async fn apply_fingerprint_policy(state: &AppState, fp: &str) -> WebResult<()> {
|
||||
let count: i64 =
|
||||
sqlx::query_scalar("SELECT count(*) FROM users WHERE registration_fingerprint = $1")
|
||||
.bind(fp)
|
||||
.fetch_one(&state.pool)
|
||||
.await?;
|
||||
if count > 1 {
|
||||
sqlx::query(
|
||||
"UPDATE accounts SET fingerprint_flagged = true \
|
||||
WHERE owner_user_id IN (SELECT id FROM users WHERE registration_fingerprint = $1)",
|
||||
)
|
||||
.bind(fp)
|
||||
.execute(&state.pool)
|
||||
.await?;
|
||||
}
|
||||
if count >= state.config.abuse.fingerprint_account_threshold {
|
||||
let res = sqlx::query(
|
||||
"UPDATE accounts SET status = 'deactivated' \
|
||||
WHERE owner_user_id IN (SELECT id FROM users WHERE registration_fingerprint = $1)",
|
||||
)
|
||||
.bind(fp)
|
||||
.execute(&state.pool)
|
||||
.await?;
|
||||
tracing::warn!(
|
||||
fingerprint = fp,
|
||||
accounts = res.rows_affected(),
|
||||
"silently deactivated fingerprint-abusing accounts"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct TokenReq {
|
||||
token: String,
|
||||
}
|
||||
|
||||
/// `POST /web/v1/verify` — consume a verification token, mark verified.
|
||||
async fn verify(State(state): State<AppState>, Json(req): Json<TokenReq>) -> WebResult<Response> {
|
||||
let row = sqlx::query(
|
||||
"UPDATE email_tokens SET consumed_at = now() \
|
||||
WHERE token_hash = $1 AND kind = 'verify' AND consumed_at IS NULL AND expires_at > now() \
|
||||
RETURNING user_id",
|
||||
)
|
||||
.bind(sha256(&req.token))
|
||||
.fetch_optional(&state.pool)
|
||||
.await?;
|
||||
let Some(row) = row else {
|
||||
return Err(WebError::BadRequest("invalid or expired token"));
|
||||
};
|
||||
let user_id: Uuid = row.get("user_id");
|
||||
sqlx::query("UPDATE users SET email_verified = true WHERE id = $1")
|
||||
.bind(user_id)
|
||||
.execute(&state.pool)
|
||||
.await?;
|
||||
Ok(StatusCode::OK.into_response())
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct LoginReq {
|
||||
email: String,
|
||||
password: String,
|
||||
}
|
||||
|
||||
/// `POST /web/v1/login` — verify password + email-verified → session JWT.
|
||||
async fn login(State(state): State<AppState>, Json(req): Json<LoginReq>) -> WebResult<Response> {
|
||||
let row = sqlx::query("SELECT id, password_hash, email_verified FROM users WHERE email = $1")
|
||||
.bind(&req.email)
|
||||
.fetch_optional(&state.pool)
|
||||
.await?;
|
||||
// Generic 401 for every failure mode (no enumeration).
|
||||
let Some(row) = row else {
|
||||
return Err(WebError::Unauthorized);
|
||||
};
|
||||
let phc: String = row.get("password_hash");
|
||||
let verified: bool = row.get("email_verified");
|
||||
if !verify_password(&req.password, &phc) || !verified {
|
||||
return Err(WebError::Unauthorized);
|
||||
}
|
||||
let user_id: Uuid = row.get("id");
|
||||
let token = mint_session(&state, user_id)?;
|
||||
Ok(Json(json!({
|
||||
"token": token,
|
||||
"expires_in": state.config.auth.session_ttl_secs,
|
||||
}))
|
||||
.into_response())
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct EmailReq {
|
||||
email: String,
|
||||
}
|
||||
|
||||
/// `POST /web/v1/password-reset/request` — always `202` (no enumeration);
|
||||
/// mints + emails a reset token only if the account exists.
|
||||
async fn reset_request(State(state): State<AppState>, Json(req): Json<EmailReq>) -> Response {
|
||||
// The inner only ever yields `Internal` (DB failure); a missing email is
|
||||
// Ok(()) so there's no enumeration. Surface 500 on a real error, else 202.
|
||||
match reset_request_inner(&state, &req.email).await {
|
||||
Ok(()) => StatusCode::ACCEPTED.into_response(),
|
||||
Err(e) => e.into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn reset_request_inner(state: &AppState, email: &str) -> WebResult<()> {
|
||||
let row = sqlx::query("SELECT id FROM users WHERE email = $1")
|
||||
.bind(email)
|
||||
.fetch_optional(&state.pool)
|
||||
.await?;
|
||||
let Some(row) = row else { return Ok(()) };
|
||||
let user_id: Uuid = row.get("id");
|
||||
let token = random_token();
|
||||
let expires: DateTime<Utc> =
|
||||
Utc::now() + Duration::seconds(state.config.auth.email_token_ttl_secs as i64);
|
||||
sqlx::query(
|
||||
"INSERT INTO email_tokens (token_hash, user_id, kind, expires_at) \
|
||||
VALUES ($1, $2, 'reset', $3)",
|
||||
)
|
||||
.bind(sha256(&token))
|
||||
.bind(user_id)
|
||||
.bind(expires)
|
||||
.execute(&state.pool)
|
||||
.await?;
|
||||
let link = format!("{}/reset?token={token}", state.config.auth.app_base_url);
|
||||
let _ = state
|
||||
.email
|
||||
.send(
|
||||
email,
|
||||
"Reset your helexa password",
|
||||
&format!("Reset your password:\n\n{link}\n"),
|
||||
)
|
||||
.await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ResetConfirmReq {
|
||||
token: String,
|
||||
new_password: String,
|
||||
}
|
||||
|
||||
/// `POST /web/v1/password-reset/confirm` — consume reset token, rotate hash.
|
||||
async fn reset_confirm(
|
||||
State(state): State<AppState>,
|
||||
Json(req): Json<ResetConfirmReq>,
|
||||
) -> WebResult<Response> {
|
||||
if req.new_password.len() < 8 {
|
||||
return Err(WebError::BadRequest("password too short (min 8)"));
|
||||
}
|
||||
let row = sqlx::query(
|
||||
"UPDATE email_tokens SET consumed_at = now() \
|
||||
WHERE token_hash = $1 AND kind = 'reset' AND consumed_at IS NULL AND expires_at > now() \
|
||||
RETURNING user_id",
|
||||
)
|
||||
.bind(sha256(&req.token))
|
||||
.fetch_optional(&state.pool)
|
||||
.await?;
|
||||
let Some(row) = row else {
|
||||
return Err(WebError::BadRequest("invalid or expired token"));
|
||||
};
|
||||
let user_id: Uuid = row.get("user_id");
|
||||
let phc = hash_password(&req.new_password).map_err(|_| WebError::Internal)?;
|
||||
sqlx::query("UPDATE users SET password_hash = $1 WHERE id = $2")
|
||||
.bind(phc)
|
||||
.bind(user_id)
|
||||
.execute(&state.pool)
|
||||
.await?;
|
||||
Ok(StatusCode::OK.into_response())
|
||||
}
|
||||
|
||||
// ── account + keys (protected) ──────────────────────────────────────
|
||||
|
||||
async fn account(
|
||||
State(state): State<AppState>,
|
||||
Extension(user): Extension<AuthUser>,
|
||||
) -> WebResult<Response> {
|
||||
let acct = account_id_for(&state, user.0).await?;
|
||||
let row = sqlx::query(
|
||||
"SELECT allocation_total, allocation_spent, allocation_reserved FROM accounts WHERE id = $1",
|
||||
)
|
||||
.bind(acct)
|
||||
.fetch_one(&state.pool)
|
||||
.await?;
|
||||
Ok(Json(json!({
|
||||
"account_id": acct.to_string(),
|
||||
"allocation_total": row.get::<i64, _>("allocation_total"),
|
||||
"allocation_spent": row.get::<i64, _>("allocation_spent"),
|
||||
"allocation_reserved": row.get::<i64, _>("allocation_reserved"),
|
||||
}))
|
||||
.into_response())
|
||||
}
|
||||
|
||||
async fn list_keys(
|
||||
State(state): State<AppState>,
|
||||
Extension(user): Extension<AuthUser>,
|
||||
) -> WebResult<Response> {
|
||||
let acct = account_id_for(&state, user.0).await?;
|
||||
let rows = sqlx::query(
|
||||
"SELECT id, key_prefix, label, status, limit_kind, limit_value, key_spent, key_reserved, \
|
||||
created_at \
|
||||
FROM api_keys WHERE account_id = $1 ORDER BY created_at DESC",
|
||||
)
|
||||
.bind(acct)
|
||||
.fetch_all(&state.pool)
|
||||
.await?;
|
||||
let keys: Vec<_> = rows
|
||||
.iter()
|
||||
.map(|r| {
|
||||
json!({
|
||||
"id": r.get::<Uuid, _>("id").to_string(),
|
||||
"prefix": r.get::<String, _>("key_prefix"),
|
||||
"label": r.get::<String, _>("label"),
|
||||
"status": r.get::<String, _>("status"),
|
||||
"limit_kind": r.get::<String, _>("limit_kind"),
|
||||
"limit_value": r.get::<i64, _>("limit_value"),
|
||||
"spent": r.get::<i64, _>("key_spent"),
|
||||
"reserved": r.get::<i64, _>("key_reserved"),
|
||||
"created_at": r.get::<DateTime<Utc>, _>("created_at").to_rfc3339(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
Ok(Json(json!({ "keys": keys })).into_response())
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct CreateKeyReq {
|
||||
#[serde(default)]
|
||||
label: String,
|
||||
/// "percent" | "hardcap" (default percent=100 → full allocation).
|
||||
#[serde(default)]
|
||||
limit_kind: Option<String>,
|
||||
#[serde(default)]
|
||||
limit_value: Option<i64>,
|
||||
}
|
||||
|
||||
async fn create_key(
|
||||
State(state): State<AppState>,
|
||||
Extension(user): Extension<AuthUser>,
|
||||
Json(req): Json<CreateKeyReq>,
|
||||
) -> WebResult<Response> {
|
||||
let acct = account_id_for(&state, user.0).await?;
|
||||
let limit_kind = match req.limit_kind.as_deref() {
|
||||
Some("hardcap") => "hardcap",
|
||||
_ => "percent",
|
||||
};
|
||||
let limit_value = req.limit_value.unwrap_or(100).max(0);
|
||||
let (raw, prefix) = generate_api_key();
|
||||
let id: Uuid = sqlx::query(
|
||||
"INSERT INTO api_keys (account_id, key_hash, key_prefix, label, limit_kind, limit_value) \
|
||||
VALUES ($1, $2, $3, $4, $5, $6) RETURNING id",
|
||||
)
|
||||
.bind(acct)
|
||||
.bind(sha256(&raw))
|
||||
.bind(&prefix)
|
||||
.bind(&req.label)
|
||||
.bind(limit_kind)
|
||||
.bind(limit_value)
|
||||
.fetch_one(&state.pool)
|
||||
.await?
|
||||
.get("id");
|
||||
// The raw key is shown exactly once.
|
||||
Ok((
|
||||
StatusCode::CREATED,
|
||||
Json(json!({
|
||||
"id": id.to_string(),
|
||||
"key": raw,
|
||||
"prefix": prefix,
|
||||
"limit_kind": limit_kind,
|
||||
"limit_value": limit_value,
|
||||
})),
|
||||
)
|
||||
.into_response())
|
||||
}
|
||||
|
||||
async fn archive_key(
|
||||
State(state): State<AppState>,
|
||||
Extension(user): Extension<AuthUser>,
|
||||
Path(id): Path<Uuid>,
|
||||
) -> WebResult<Response> {
|
||||
let acct = account_id_for(&state, user.0).await?;
|
||||
let res = sqlx::query(
|
||||
"UPDATE api_keys SET status = 'archived' WHERE id = $1 AND account_id = $2 AND status = 'active'",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(acct)
|
||||
.execute(&state.pool)
|
||||
.await?;
|
||||
if res.rows_affected() == 0 {
|
||||
return Err(WebError::BadRequest("no such active key"));
|
||||
}
|
||||
Ok(StatusCode::NO_CONTENT.into_response())
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct UpdateLimitReq {
|
||||
limit_kind: String,
|
||||
limit_value: i64,
|
||||
}
|
||||
|
||||
async fn update_key_limit(
|
||||
State(state): State<AppState>,
|
||||
Extension(user): Extension<AuthUser>,
|
||||
Path(id): Path<Uuid>,
|
||||
Json(req): Json<UpdateLimitReq>,
|
||||
) -> WebResult<Response> {
|
||||
if req.limit_kind != "percent" && req.limit_kind != "hardcap" {
|
||||
return Err(WebError::BadRequest(
|
||||
"limit_kind must be percent or hardcap",
|
||||
));
|
||||
}
|
||||
if req.limit_value < 0 {
|
||||
return Err(WebError::BadRequest("limit_value must be >= 0"));
|
||||
}
|
||||
let acct = account_id_for(&state, user.0).await?;
|
||||
let res = sqlx::query(
|
||||
"UPDATE api_keys SET limit_kind = $1, limit_value = $2 WHERE id = $3 AND account_id = $4",
|
||||
)
|
||||
.bind(&req.limit_kind)
|
||||
.bind(req.limit_value)
|
||||
.bind(id)
|
||||
.bind(acct)
|
||||
.execute(&state.pool)
|
||||
.await?;
|
||||
if res.rows_affected() == 0 {
|
||||
return Err(WebError::BadRequest("no such key"));
|
||||
}
|
||||
Ok(StatusCode::NO_CONTENT.into_response())
|
||||
}
|
||||
243
crates/helexa-upstream/tests/authz_pg.rs
Normal file
@@ -0,0 +1,243 @@
|
||||
//! Integration tests for the `/authz/v1` surface against a real Postgres,
|
||||
//! driving the built axum app over HTTP. Gated on `UPSTREAM_TEST_DATABASE_URL`
|
||||
//! (skips cleanly when unset, so CI stays green without a DB):
|
||||
//!
|
||||
//! UPSTREAM_TEST_DATABASE_URL=postgres://helexa:helexa@localhost/helexa_test \
|
||||
//! cargo test -p helexa-upstream --test authz_pg
|
||||
|
||||
use helexa_upstream::config::{ClientToken, UpstreamConfig};
|
||||
use helexa_upstream::crypto::sha256;
|
||||
use helexa_upstream::db::connect_and_migrate;
|
||||
use helexa_upstream::state::AppState;
|
||||
use serde_json::{Value, json};
|
||||
use sqlx::Executor;
|
||||
use sqlx::Row;
|
||||
use sqlx::postgres::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
const CLIENT_TOKEN: &str = "test-operator-token";
|
||||
|
||||
async fn spawn_or_skip(test: &str) -> Option<(String, PgPool)> {
|
||||
let Ok(url) = std::env::var("UPSTREAM_TEST_DATABASE_URL") else {
|
||||
eprintln!("skipping {test}: UPSTREAM_TEST_DATABASE_URL not set");
|
||||
return None;
|
||||
};
|
||||
let pool = connect_and_migrate(&url, 16).await.expect("migrate");
|
||||
|
||||
let mut config = UpstreamConfig {
|
||||
server: Default::default(),
|
||||
db: helexa_upstream::config::DbSettings {
|
||||
url,
|
||||
max_connections: 16,
|
||||
},
|
||||
grant: Default::default(),
|
||||
abuse: Default::default(),
|
||||
client_auth: Default::default(),
|
||||
authz: Default::default(),
|
||||
auth: Default::default(),
|
||||
email: Default::default(),
|
||||
};
|
||||
config.client_auth.tokens.push(ClientToken {
|
||||
token: CLIENT_TOKEN.into(),
|
||||
operator_id: "op-test".into(),
|
||||
});
|
||||
|
||||
let email = helexa_upstream::email::EmailSender::from_config(&config.email).unwrap();
|
||||
let state = AppState::new(pool.clone(), config, email);
|
||||
let app = helexa_upstream::build_app(state);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
Some((format!("http://{addr}"), pool))
|
||||
}
|
||||
|
||||
/// Seed an account with `total` allocation and an active key with raw value
|
||||
/// `raw` (percent=100). Optionally deactivate the account. Returns
|
||||
/// (account_id, key_id).
|
||||
async fn seed_key(pool: &PgPool, total: i64, raw: &str, deactivated: bool) -> (Uuid, Uuid) {
|
||||
let user_id: Uuid = pool
|
||||
.fetch_one(
|
||||
sqlx::query(
|
||||
"INSERT INTO users (email, password_hash, email_verified) VALUES ($1,'x',true) RETURNING id",
|
||||
)
|
||||
.bind(format!("u-{}@t.local", Uuid::new_v4())),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.get("id");
|
||||
let status = if deactivated { "deactivated" } else { "active" };
|
||||
let account_id: Uuid = pool
|
||||
.fetch_one(
|
||||
sqlx::query(
|
||||
"INSERT INTO accounts (owner_user_id, allocation_total, status) VALUES ($1,$2,$3) RETURNING id",
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(total)
|
||||
.bind(status),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.get("id");
|
||||
let key_id: Uuid = pool
|
||||
.fetch_one(
|
||||
sqlx::query(
|
||||
"INSERT INTO api_keys (account_id, key_hash, key_prefix, limit_kind, limit_value) \
|
||||
VALUES ($1,$2,'sk-test','percent',100) RETURNING id",
|
||||
)
|
||||
.bind(account_id)
|
||||
.bind(sha256(raw)),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.get("id");
|
||||
(account_id, key_id)
|
||||
}
|
||||
|
||||
fn client() -> reqwest::Client {
|
||||
reqwest::Client::new()
|
||||
}
|
||||
|
||||
async fn post(
|
||||
c: &reqwest::Client,
|
||||
url: String,
|
||||
body: Value,
|
||||
bearer: Option<&str>,
|
||||
) -> reqwest::Response {
|
||||
let mut req = c.post(url).json(&body);
|
||||
if let Some(b) = bearer {
|
||||
req = req.bearer_auth(b);
|
||||
}
|
||||
req.send().await.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resolve_reserve_settle_round_trip() {
|
||||
let Some((base, pool)) = spawn_or_skip("resolve_reserve_settle_round_trip").await else {
|
||||
return;
|
||||
};
|
||||
let raw = format!("sk-{}", Uuid::new_v4());
|
||||
let (account_id, key_id) = seed_key(&pool, 1000, &raw, false).await;
|
||||
let c = client();
|
||||
|
||||
// resolve
|
||||
let r = post(
|
||||
&c,
|
||||
format!("{base}/authz/v1/resolve"),
|
||||
json!({"api_key": raw}),
|
||||
Some(CLIENT_TOKEN),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(r.status(), 200);
|
||||
let body: Value = r.json().await.unwrap();
|
||||
assert_eq!(body["principal"]["account_id"], account_id.to_string());
|
||||
assert_eq!(body["principal"]["key_id"], key_id.to_string());
|
||||
assert_eq!(body["snapshot"]["hard_cap"], 1000);
|
||||
|
||||
// reserve 400
|
||||
let r = post(
|
||||
&c,
|
||||
format!("{base}/authz/v1/reserve"),
|
||||
json!({"account_id": account_id, "key_id": key_id, "max_tokens": 400}),
|
||||
Some(CLIENT_TOKEN),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(r.status(), 200);
|
||||
let body: Value = r.json().await.unwrap();
|
||||
let rid = body["reservation_id"].as_i64().expect("granted");
|
||||
|
||||
// settle 150
|
||||
let r = post(
|
||||
&c,
|
||||
format!("{base}/authz/v1/settle"),
|
||||
json!({"reservation_id": rid, "actual_tokens": 150}),
|
||||
Some(CLIENT_TOKEN),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(r.status(), 204);
|
||||
|
||||
// snapshot reflects spend
|
||||
let r = post(
|
||||
&c,
|
||||
format!("{base}/authz/v1/snapshot"),
|
||||
json!({"account_id": account_id, "key_id": key_id}),
|
||||
Some(CLIENT_TOKEN),
|
||||
)
|
||||
.await;
|
||||
let body: Value = r.json().await.unwrap();
|
||||
assert_eq!(body["spent"], 150);
|
||||
assert_eq!(body["reserved"], 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn over_cap_reserve_is_rejected_not_errored() {
|
||||
let Some((base, pool)) = spawn_or_skip("over_cap_reserve_is_rejected_not_errored").await else {
|
||||
return;
|
||||
};
|
||||
let raw = format!("sk-{}", Uuid::new_v4());
|
||||
let (account_id, key_id) = seed_key(&pool, 100, &raw, false).await;
|
||||
let c = client();
|
||||
let r = post(
|
||||
&c,
|
||||
format!("{base}/authz/v1/reserve"),
|
||||
json!({"account_id": account_id, "key_id": key_id, "max_tokens": 999}),
|
||||
Some(CLIENT_TOKEN),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(r.status(), 200, "budget refusal is an authoritative 200");
|
||||
let body: Value = r.json().await.unwrap();
|
||||
assert!(body["reservation_id"].is_null());
|
||||
assert_eq!(body["rejected"]["kind"], "insufficient_quota");
|
||||
assert_eq!(body["rejected"]["available"], 100);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn deactivated_account_resolves_as_invalid_no_clue() {
|
||||
let Some((base, pool)) = spawn_or_skip("deactivated_account_resolves_as_invalid_no_clue").await
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let raw = format!("sk-{}", Uuid::new_v4());
|
||||
seed_key(&pool, 1000, &raw, true).await; // deactivated
|
||||
let c = client();
|
||||
let r = post(
|
||||
&c,
|
||||
format!("{base}/authz/v1/resolve"),
|
||||
json!({"api_key": raw}),
|
||||
Some(CLIENT_TOKEN),
|
||||
)
|
||||
.await;
|
||||
// Indistinguishable from an unknown key.
|
||||
assert_eq!(r.status(), 401);
|
||||
let body: Value = r.json().await.unwrap();
|
||||
assert_eq!(body["error"]["code"], "invalid_api_key");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn missing_client_auth_is_401_before_db() {
|
||||
let Some((base, pool)) = spawn_or_skip("missing_client_auth_is_401_before_db").await else {
|
||||
return;
|
||||
};
|
||||
let raw = format!("sk-{}", Uuid::new_v4());
|
||||
seed_key(&pool, 1000, &raw, false).await;
|
||||
let c = client();
|
||||
// No bearer → rejected by client_auth.
|
||||
let r = post(
|
||||
&c,
|
||||
format!("{base}/authz/v1/resolve"),
|
||||
json!({"api_key": raw}),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(r.status(), 401);
|
||||
// Wrong bearer → also rejected.
|
||||
let r = post(
|
||||
&c,
|
||||
format!("{base}/authz/v1/resolve"),
|
||||
json!({"api_key": raw}),
|
||||
Some("wrong"),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(r.status(), 401);
|
||||
}
|
||||
204
crates/helexa-upstream/tests/ledger_pg.rs
Normal file
@@ -0,0 +1,204 @@
|
||||
//! Integration tests for the allocation ledger against a real PostgreSQL.
|
||||
//!
|
||||
//! Gated on `UPSTREAM_TEST_DATABASE_URL` — when unset (CI's generic runner,
|
||||
//! local builds without a DB), every test logs a skip and returns, so
|
||||
//! `cargo test --workspace` stays green without Postgres. Point the env var
|
||||
//! at a throwaway database to exercise the no-overshoot guarantee and
|
||||
//! settle/release idempotency:
|
||||
//!
|
||||
//! UPSTREAM_TEST_DATABASE_URL=postgres://helexa:helexa@localhost/helexa_test \
|
||||
//! cargo test -p helexa-upstream --test ledger_pg
|
||||
|
||||
use helexa_upstream::db::connect_and_migrate;
|
||||
use helexa_upstream::ledger::{self, LedgerError};
|
||||
use sqlx::Executor;
|
||||
use sqlx::Row;
|
||||
use sqlx::postgres::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Returns a migrated pool, or `None` (with a skip log) when the env var is
|
||||
/// unset.
|
||||
async fn pool_or_skip(test: &str) -> Option<PgPool> {
|
||||
let Ok(url) = std::env::var("UPSTREAM_TEST_DATABASE_URL") else {
|
||||
eprintln!("skipping {test}: UPSTREAM_TEST_DATABASE_URL not set");
|
||||
return None;
|
||||
};
|
||||
Some(
|
||||
connect_and_migrate(&url, 16)
|
||||
.await
|
||||
.expect("connect + migrate"),
|
||||
)
|
||||
}
|
||||
|
||||
/// Seed a verified user + account (with `total` allocation) + an active key
|
||||
/// (percent=100 so the account cap binds). Returns (account_id, key_id).
|
||||
async fn seed(pool: &PgPool, total: i64) -> (Uuid, Uuid) {
|
||||
let user_id: Uuid = pool
|
||||
.fetch_one(
|
||||
sqlx::query(
|
||||
"INSERT INTO users (email, password_hash, email_verified) \
|
||||
VALUES ($1, 'x', true) RETURNING id",
|
||||
)
|
||||
.bind(format!("u-{}@test.local", Uuid::new_v4())),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.get("id");
|
||||
let account_id: Uuid = pool
|
||||
.fetch_one(
|
||||
sqlx::query(
|
||||
"INSERT INTO accounts (owner_user_id, allocation_total) \
|
||||
VALUES ($1, $2) RETURNING id",
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(total),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.get("id");
|
||||
let key_id: Uuid = pool
|
||||
.fetch_one(
|
||||
sqlx::query(
|
||||
"INSERT INTO api_keys (account_id, key_hash, key_prefix, limit_kind, limit_value) \
|
||||
VALUES ($1, $2, 'sk-test', 'percent', 100) RETURNING id",
|
||||
)
|
||||
.bind(account_id)
|
||||
.bind(Uuid::new_v4().as_bytes().to_vec()),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.get("id");
|
||||
(account_id, key_id)
|
||||
}
|
||||
|
||||
async fn account_cols(pool: &PgPool, account_id: Uuid) -> (i64, i64) {
|
||||
let row = pool
|
||||
.fetch_one(
|
||||
sqlx::query("SELECT allocation_spent, allocation_reserved FROM accounts WHERE id = $1")
|
||||
.bind(account_id),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
(row.get("allocation_spent"), row.get("allocation_reserved"))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_reserves_never_overshoot() {
|
||||
let Some(pool) = pool_or_skip("concurrent_reserves_never_overshoot").await else {
|
||||
return;
|
||||
};
|
||||
// Allocation admits exactly 5 reservations of 100 (cap 500).
|
||||
let (account_id, key_id) = seed(&pool, 500).await;
|
||||
|
||||
let mut handles = Vec::new();
|
||||
for _ in 0..20 {
|
||||
let pool = pool.clone();
|
||||
handles.push(tokio::spawn(async move {
|
||||
ledger::reserve(&pool, account_id, key_id, 100).await
|
||||
}));
|
||||
}
|
||||
let mut ok = 0;
|
||||
let mut quota = 0;
|
||||
for h in handles {
|
||||
match h.await.unwrap() {
|
||||
Ok(_) => ok += 1,
|
||||
Err(LedgerError::InsufficientQuota { .. }) => quota += 1,
|
||||
Err(e) => panic!("unexpected error: {e}"),
|
||||
}
|
||||
}
|
||||
assert_eq!(ok, 5, "exactly 5 reserves of 100 fit in a 500 allocation");
|
||||
assert_eq!(quota, 15);
|
||||
|
||||
let (spent, reserved) = account_cols(&pool, account_id).await;
|
||||
assert_eq!(spent, 0);
|
||||
assert_eq!(reserved, 500, "reserved exactly the cap, never over");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn settle_is_idempotent_and_reconciles_spend() {
|
||||
let Some(pool) = pool_or_skip("settle_is_idempotent_and_reconciles_spend").await else {
|
||||
return;
|
||||
};
|
||||
let (account_id, key_id) = seed(&pool, 1000).await;
|
||||
let rid = ledger::reserve(&pool, account_id, key_id, 400)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Settle actual=150 (< reserved 400): spent=150, reserved back to 0.
|
||||
ledger::settle(&pool, rid, 150).await.unwrap();
|
||||
let (spent, reserved) = account_cols(&pool, account_id).await;
|
||||
assert_eq!((spent, reserved), (150, 0));
|
||||
|
||||
// Second settle is a no-op.
|
||||
ledger::settle(&pool, rid, 999).await.unwrap();
|
||||
let (spent2, reserved2) = account_cols(&pool, account_id).await;
|
||||
assert_eq!((spent2, reserved2), (150, 0), "settle is idempotent");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn release_returns_reservation_and_is_idempotent() {
|
||||
let Some(pool) = pool_or_skip("release_returns_reservation_and_is_idempotent").await else {
|
||||
return;
|
||||
};
|
||||
let (account_id, key_id) = seed(&pool, 1000).await;
|
||||
let rid = ledger::reserve(&pool, account_id, key_id, 300)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(account_cols(&pool, account_id).await, (0, 300));
|
||||
|
||||
ledger::release(&pool, rid).await.unwrap();
|
||||
assert_eq!(account_cols(&pool, account_id).await, (0, 0));
|
||||
// Idempotent; settle-after-release also a no-op.
|
||||
ledger::release(&pool, rid).await.unwrap();
|
||||
ledger::settle(&pool, rid, 100).await.unwrap();
|
||||
assert_eq!(account_cols(&pool, account_id).await, (0, 0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn hardcap_key_subcap_binds_below_account() {
|
||||
let Some(pool) = pool_or_skip("hardcap_key_subcap_binds_below_account").await else {
|
||||
return;
|
||||
};
|
||||
// Account has 1000 but the key is hard-capped at 200.
|
||||
let user_id: Uuid = pool
|
||||
.fetch_one(
|
||||
sqlx::query(
|
||||
"INSERT INTO users (email, password_hash, email_verified) \
|
||||
VALUES ($1, 'x', true) RETURNING id",
|
||||
)
|
||||
.bind(format!("u-{}@test.local", Uuid::new_v4())),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.get("id");
|
||||
let account_id: Uuid = pool
|
||||
.fetch_one(
|
||||
sqlx::query(
|
||||
"INSERT INTO accounts (owner_user_id, allocation_total) VALUES ($1, 1000) RETURNING id",
|
||||
)
|
||||
.bind(user_id),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.get("id");
|
||||
let key_id: Uuid = pool
|
||||
.fetch_one(
|
||||
sqlx::query(
|
||||
"INSERT INTO api_keys (account_id, key_hash, key_prefix, limit_kind, limit_value) \
|
||||
VALUES ($1, $2, 'sk-test', 'hardcap', 200) RETURNING id",
|
||||
)
|
||||
.bind(account_id)
|
||||
.bind(Uuid::new_v4().as_bytes().to_vec()),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.get("id");
|
||||
|
||||
ledger::reserve(&pool, account_id, key_id, 200)
|
||||
.await
|
||||
.unwrap();
|
||||
match ledger::reserve(&pool, account_id, key_id, 1).await {
|
||||
Err(LedgerError::InsufficientQuota { available, .. }) => assert_eq!(available, 0),
|
||||
other => panic!("expected InsufficientQuota, got {other:?}"),
|
||||
}
|
||||
}
|
||||
296
crates/helexa-upstream/tests/web_pg.rs
Normal file
@@ -0,0 +1,296 @@
|
||||
//! Integration tests for the `/web/v1` account API + the silent fingerprint
|
||||
//! abuse policy, driving the built app over HTTP against a real Postgres.
|
||||
//! Gated on `UPSTREAM_TEST_DATABASE_URL` (skips cleanly when unset).
|
||||
|
||||
use helexa_upstream::config::{ClientToken, UpstreamConfig};
|
||||
use helexa_upstream::crypto::sha256;
|
||||
use helexa_upstream::db::connect_and_migrate;
|
||||
use helexa_upstream::email::EmailSender;
|
||||
use helexa_upstream::state::AppState;
|
||||
use serde_json::{Value, json};
|
||||
use sqlx::Executor;
|
||||
use sqlx::Row;
|
||||
use sqlx::postgres::PgPool;
|
||||
|
||||
const CLIENT_TOKEN: &str = "web-test-operator-token";
|
||||
|
||||
async fn spawn_or_skip(test: &str) -> Option<(String, PgPool)> {
|
||||
let Ok(url) = std::env::var("UPSTREAM_TEST_DATABASE_URL") else {
|
||||
eprintln!("skipping {test}: UPSTREAM_TEST_DATABASE_URL not set");
|
||||
return None;
|
||||
};
|
||||
let pool = connect_and_migrate(&url, 16).await.expect("migrate");
|
||||
let mut config = UpstreamConfig {
|
||||
server: Default::default(),
|
||||
db: helexa_upstream::config::DbSettings {
|
||||
url,
|
||||
max_connections: 16,
|
||||
},
|
||||
grant: Default::default(),
|
||||
abuse: Default::default(),
|
||||
client_auth: Default::default(),
|
||||
authz: Default::default(),
|
||||
auth: Default::default(),
|
||||
email: Default::default(), // Log transport
|
||||
};
|
||||
config.client_auth.tokens.push(ClientToken {
|
||||
token: CLIENT_TOKEN.into(),
|
||||
operator_id: "op-web".into(),
|
||||
});
|
||||
let email = EmailSender::from_config(&config.email).unwrap();
|
||||
let state = AppState::new(pool.clone(), config, email);
|
||||
let app = helexa_upstream::build_app(state);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
Some((format!("http://{addr}"), pool))
|
||||
}
|
||||
|
||||
fn unique_email() -> String {
|
||||
format!("u-{}@test.local", uuid::Uuid::new_v4())
|
||||
}
|
||||
|
||||
async fn post(url: String, body: Value, bearer: Option<&str>) -> reqwest::Response {
|
||||
let c = reqwest::Client::new();
|
||||
let mut req = c.post(url).json(&body);
|
||||
if let Some(b) = bearer {
|
||||
req = req.bearer_auth(b);
|
||||
}
|
||||
req.send().await.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn verify_endpoint_consumes_token_once() {
|
||||
let Some((base, pool)) = spawn_or_skip("verify_endpoint_consumes_token_once").await else {
|
||||
return;
|
||||
};
|
||||
let email = unique_email();
|
||||
// Register, then mint a verify token directly (the raw token is only in
|
||||
// the email; here we insert a known one to drive the endpoint).
|
||||
assert_eq!(
|
||||
post(
|
||||
format!("{base}/web/v1/register"),
|
||||
json!({"email": email, "password": "password123"}),
|
||||
None
|
||||
)
|
||||
.await
|
||||
.status(),
|
||||
202
|
||||
);
|
||||
let user_id: uuid::Uuid = pool
|
||||
.fetch_one(sqlx::query("SELECT id FROM users WHERE email = $1").bind(&email))
|
||||
.await
|
||||
.unwrap()
|
||||
.get("id");
|
||||
let raw = "verify-raw-token-xyz";
|
||||
pool.execute(
|
||||
sqlx::query(
|
||||
"INSERT INTO email_tokens (token_hash, user_id, kind, expires_at) \
|
||||
VALUES ($1, $2, 'verify', now() + interval '1 hour')",
|
||||
)
|
||||
.bind(sha256(raw))
|
||||
.bind(user_id),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
post(format!("{base}/web/v1/verify"), json!({"token": raw}), None)
|
||||
.await
|
||||
.status(),
|
||||
200
|
||||
);
|
||||
// Consumed → second attempt fails.
|
||||
assert_eq!(
|
||||
post(format!("{base}/web/v1/verify"), json!({"token": raw}), None)
|
||||
.await
|
||||
.status(),
|
||||
400
|
||||
);
|
||||
|
||||
let verified: bool = pool
|
||||
.fetch_one(sqlx::query("SELECT email_verified FROM users WHERE id = $1").bind(user_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.get("email_verified");
|
||||
assert!(verified);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn account_lifecycle_and_key_resolves_then_archives() {
|
||||
let Some((base, pool)) =
|
||||
spawn_or_skip("account_lifecycle_and_key_resolves_then_archives").await
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let email = unique_email();
|
||||
post(
|
||||
format!("{base}/web/v1/register"),
|
||||
json!({"email": email, "password": "password123"}),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
// Bypass the email step for the login/key portion.
|
||||
pool.execute(
|
||||
sqlx::query("UPDATE users SET email_verified = true WHERE email = $1").bind(&email),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// login → session JWT
|
||||
let r = post(
|
||||
format!("{base}/web/v1/login"),
|
||||
json!({"email": email, "password": "password123"}),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(r.status(), 200);
|
||||
let token = r.json::<Value>().await.unwrap()["token"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.to_string();
|
||||
|
||||
// create key (raw shown once)
|
||||
let r = post(
|
||||
format!("{base}/web/v1/keys"),
|
||||
json!({"label": "laptop"}),
|
||||
Some(&token),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(r.status(), 201);
|
||||
let body: Value = r.json().await.unwrap();
|
||||
let raw_key = body["key"].as_str().unwrap().to_string();
|
||||
let key_id = body["id"].as_str().unwrap().to_string();
|
||||
assert!(raw_key.starts_with("sk-helexa-"));
|
||||
|
||||
// account balance reflects the free grant
|
||||
let r = reqwest::Client::new()
|
||||
.get(format!("{base}/web/v1/account"))
|
||||
.bearer_auth(&token)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
r.json::<Value>().await.unwrap()["allocation_total"],
|
||||
1_000_000
|
||||
);
|
||||
|
||||
// list keys shows the prefix, never the raw secret
|
||||
let r = reqwest::Client::new()
|
||||
.get(format!("{base}/web/v1/keys"))
|
||||
.bearer_auth(&token)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let listed = r.json::<Value>().await.unwrap();
|
||||
let k = &listed["keys"][0];
|
||||
assert_eq!(k["id"], key_id);
|
||||
assert!(k.get("key").is_none(), "raw secret never listed");
|
||||
assert!(k["prefix"].as_str().unwrap().starts_with("sk-helexa-"));
|
||||
|
||||
// the key authorizes at the authz surface
|
||||
let r = post(
|
||||
format!("{base}/authz/v1/resolve"),
|
||||
json!({"api_key": raw_key}),
|
||||
Some(CLIENT_TOKEN),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(r.status(), 200);
|
||||
|
||||
// archive → the key no longer resolves
|
||||
let r = post(
|
||||
format!("{base}/web/v1/keys/{key_id}/archive"),
|
||||
json!({}),
|
||||
Some(&token),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(r.status(), 204);
|
||||
let r = post(
|
||||
format!("{base}/authz/v1/resolve"),
|
||||
json!({"api_key": raw_key}),
|
||||
Some(CLIENT_TOKEN),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(r.status(), 401);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fingerprint_abuse_silently_deactivates_all_no_clue() {
|
||||
let Some((base, pool)) =
|
||||
spawn_or_skip("fingerprint_abuse_silently_deactivates_all_no_clue").await
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let fp = format!("fp-{}", uuid::Uuid::new_v4());
|
||||
|
||||
// 5 registrations sharing one fingerprint — every one returns a normal 202.
|
||||
let mut emails = Vec::new();
|
||||
for _ in 0..5 {
|
||||
let email = unique_email();
|
||||
let r = post(
|
||||
format!("{base}/web/v1/register"),
|
||||
json!({"email": email, "password": "password123", "fingerprint": fp}),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(r.status(), 202, "registration always looks successful");
|
||||
emails.push(email);
|
||||
}
|
||||
|
||||
// Silent effect: all 5 accounts are deactivated + flagged.
|
||||
let (deactivated, flagged): (i64, i64) = {
|
||||
let row = pool
|
||||
.fetch_one(
|
||||
sqlx::query(
|
||||
"SELECT \
|
||||
count(*) FILTER (WHERE a.status = 'deactivated') AS d, \
|
||||
count(*) FILTER (WHERE a.fingerprint_flagged) AS f \
|
||||
FROM accounts a JOIN users u ON u.id = a.owner_user_id \
|
||||
WHERE u.registration_fingerprint = $1",
|
||||
)
|
||||
.bind(&fp),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
(row.get("d"), row.get("f"))
|
||||
};
|
||||
assert_eq!(deactivated, 5, "all sharing accounts silently deactivated");
|
||||
assert_eq!(flagged, 5);
|
||||
|
||||
// No clue at the authz surface: a key on a deactivated account resolves
|
||||
// as an ordinary 401, indistinguishable from an unknown key.
|
||||
let acct: uuid::Uuid = pool
|
||||
.fetch_one(
|
||||
sqlx::query(
|
||||
"SELECT a.id FROM accounts a JOIN users u ON u.id = a.owner_user_id \
|
||||
WHERE u.registration_fingerprint = $1 LIMIT 1",
|
||||
)
|
||||
.bind(&fp),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.get("id");
|
||||
let raw = "sk-helexa-deactivated-probe";
|
||||
pool.execute(
|
||||
sqlx::query(
|
||||
"INSERT INTO api_keys (account_id, key_hash, key_prefix) VALUES ($1, $2, 'sk-helexa-')",
|
||||
)
|
||||
.bind(acct)
|
||||
.bind(sha256(raw)),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let r = post(
|
||||
format!("{base}/authz/v1/resolve"),
|
||||
json!({"api_key": raw}),
|
||||
Some(CLIENT_TOKEN),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
r.status(),
|
||||
401,
|
||||
"deactivated account's key looks like any invalid key"
|
||||
);
|
||||
}
|
||||
@@ -13,6 +13,7 @@ use axum::response::sse::{Event, KeepAlive, Sse};
|
||||
use axum::response::{IntoResponse, Json};
|
||||
use axum::routing::{get, post};
|
||||
use cortex_core::discovery::{DiscoveryResponse, HealthResponse};
|
||||
use cortex_core::entitlements::{HEADER_ACCOUNT_ID, HEADER_KEY_ID};
|
||||
use cortex_core::harness::ModelSpec;
|
||||
use cortex_core::openai::{ChatCompletionRequest, MessageContent};
|
||||
use cortex_core::responses::{ResponsesRequest, ResponsesUsage};
|
||||
@@ -71,6 +72,12 @@ async fn health_handler(State(state): State<Arc<NeuronState>>) -> Json<HealthRes
|
||||
// know about activation lifecycle.
|
||||
let mut snapshot = state.health_cache.snapshot().await;
|
||||
snapshot.activation = state.activation.snapshot().await;
|
||||
// Per-model admission load (#53) — read live from the candle harness so
|
||||
// cortex's load-aware router (#55) can spread traffic and propagate
|
||||
// backpressure. Absent when no candle harness is present.
|
||||
if let Some(candle) = &state.candle {
|
||||
snapshot.models = candle.load_snapshot().await;
|
||||
}
|
||||
Json(snapshot)
|
||||
}
|
||||
|
||||
@@ -228,6 +235,17 @@ fn default_enable_thinking(req: &mut ChatCompletionRequest, include_thinking: bo
|
||||
}
|
||||
}
|
||||
|
||||
/// The request's principal for fair-share admission (#54), reconstructed
|
||||
/// from the internal headers cortex stamps (#49). cortex strips any
|
||||
/// client-supplied copy and asserts the authoritative value, so over the
|
||||
/// trusted WireGuard link these are safe to key fair-share on. `None` for an
|
||||
/// unauthenticated/direct request — exempt from the per-principal cap.
|
||||
fn principal_key(headers: &axum::http::HeaderMap) -> Option<String> {
|
||||
let account = headers.get(HEADER_ACCOUNT_ID)?.to_str().ok()?;
|
||||
let key = headers.get(HEADER_KEY_ID)?.to_str().ok()?;
|
||||
Some(format!("{account}/{key}"))
|
||||
}
|
||||
|
||||
/// OpenAI-compatible chat completions. Dispatches to streaming SSE when
|
||||
/// `stream: true` is set on the request; otherwise returns a single
|
||||
/// `ChatCompletionResponse`.
|
||||
@@ -271,8 +289,14 @@ async fn chat_completions(
|
||||
// true`) keep reasoning on.
|
||||
default_enable_thinking(&mut req, include_thinking);
|
||||
|
||||
// Fair-share admission principal (#54), from cortex's stamped headers.
|
||||
let principal = principal_key(&headers);
|
||||
|
||||
if req.stream.unwrap_or(false) {
|
||||
match candle.chat_completion_stream_with(req, chat_config).await {
|
||||
match candle
|
||||
.chat_completion_stream_with(req, chat_config, principal)
|
||||
.await
|
||||
{
|
||||
Ok(rx) => {
|
||||
// Each chunk → one SSE `data: {json}` line. After the
|
||||
// channel closes, append the OpenAI [DONE] terminator.
|
||||
@@ -289,7 +313,7 @@ async fn chat_completions(
|
||||
Err(e) => inference_error_response(e),
|
||||
}
|
||||
} else {
|
||||
match candle.chat_completion(req).await {
|
||||
match candle.chat_completion(req, principal).await {
|
||||
Ok(resp) => Json(resp).into_response(),
|
||||
Err(e) => inference_error_response(e),
|
||||
}
|
||||
@@ -302,6 +326,7 @@ async fn chat_completions(
|
||||
/// event stream into the Responses event family.
|
||||
async fn responses(
|
||||
State(state): State<Arc<NeuronState>>,
|
||||
headers: axum::http::HeaderMap,
|
||||
Json(req): Json<ResponsesRequest>,
|
||||
) -> impl IntoResponse {
|
||||
let Some(candle) = state.candle.as_ref().map(Arc::clone) else {
|
||||
@@ -336,9 +361,12 @@ async fn responses(
|
||||
};
|
||||
chat_req.stream = Some(stream_requested);
|
||||
|
||||
// Fair-share admission principal (#54), from cortex's stamped headers.
|
||||
let principal = principal_key(&headers);
|
||||
|
||||
if stream_requested {
|
||||
match candle
|
||||
.responses_stream(chat_req, response_id, message_item_id)
|
||||
.responses_stream(chat_req, response_id, message_item_id, principal)
|
||||
.await
|
||||
{
|
||||
Ok(rx) => {
|
||||
@@ -362,7 +390,7 @@ async fn responses(
|
||||
// and translate the result. We don't currently re-tokenise
|
||||
// to compute usage; the harness returns it via the chat
|
||||
// response and we pass it through.
|
||||
match candle.chat_completion(chat_req).await {
|
||||
match candle.chat_completion(chat_req, principal).await {
|
||||
Ok(chat_resp) => {
|
||||
// Extract the assistant text (chat completions
|
||||
// always emits one choice on the candle path).
|
||||
@@ -486,6 +514,24 @@ fn inference_error_response(err: InferenceError) -> axum::response::Response {
|
||||
"template_render_failed",
|
||||
format!("chat template could not render this request: {detail}"),
|
||||
),
|
||||
// Admission control refused on load (#53): a fast, retryable "busy"
|
||||
// signal. 503 (service busy) + Retry-After; opencode/AI SDK back off.
|
||||
InferenceError::Overloaded { retry_after_secs } => OpenAiError::new(
|
||||
503,
|
||||
"rate_limit_error",
|
||||
"rate_limit_exceeded",
|
||||
"model is busy (admission queue full); retry shortly",
|
||||
)
|
||||
.with_retry_after(retry_after_secs),
|
||||
// Per-principal fair-share cap (#54): 429 rate_limit_exceeded +
|
||||
// Retry-After — the caller is sending too many concurrent requests.
|
||||
InferenceError::PerPrincipalLimit { retry_after_secs } => OpenAiError::new(
|
||||
429,
|
||||
"rate_limit_error",
|
||||
"rate_limit_exceeded",
|
||||
"too many concurrent requests for this key; retry shortly",
|
||||
)
|
||||
.with_retry_after(retry_after_secs),
|
||||
InferenceError::Other(e) => OpenAiError::without_code(500, "api_error", format!("{e:#}")),
|
||||
};
|
||||
envelope_response(env)
|
||||
@@ -660,6 +706,26 @@ mod error_envelope_tests {
|
||||
assert_eq!(error["required_mb"], 8_192);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn overloaded_is_503_rate_limited_with_retry_after() {
|
||||
// Admission rejection (#53) → fast, retryable backpressure.
|
||||
let resp = inference_error_response(InferenceError::Overloaded {
|
||||
retry_after_secs: 7,
|
||||
});
|
||||
assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
let retry = resp
|
||||
.headers()
|
||||
.get(axum::http::header::RETRY_AFTER)
|
||||
.expect("admission rejection must advertise Retry-After");
|
||||
assert_eq!(retry.to_str().unwrap(), "7");
|
||||
|
||||
let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX)
|
||||
.await
|
||||
.unwrap();
|
||||
let body: Value = serde_json::from_slice(&bytes).unwrap();
|
||||
assert_eq!(body["error"]["code"], "rate_limit_exceeded");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn insufficient_vram_carries_retry_after() {
|
||||
// Transient 503 — VRAM frees as in-flight requests finish, so the
|
||||
|
||||
@@ -85,6 +85,68 @@ pub struct CandleHarnessConfig {
|
||||
/// `/models`, and enforces it. These knobs tune that derivation.
|
||||
#[serde(default)]
|
||||
pub context_limit: ContextLimitConfig,
|
||||
|
||||
/// Admission control (#53): bounds the per-model wait queue so a busy
|
||||
/// model returns a fast, retryable `429`/`503` instead of stalling new
|
||||
/// requests until their client times out.
|
||||
#[serde(default)]
|
||||
pub admission: AdmissionConfig,
|
||||
}
|
||||
|
||||
/// `[harness.candle.admission]` settings (#53).
|
||||
///
|
||||
/// Inference is batch-1, so `max_in_flight` is 1 in practice; the queue
|
||||
/// (`max_queue_depth`) absorbs short bursts, and `max_wait_secs` caps how
|
||||
/// long a queued request waits before it's refused with backpressure.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct AdmissionConfig {
|
||||
/// Concurrent running requests per model. Batch-1 inference → 1.
|
||||
#[serde(default = "default_admission_max_in_flight")]
|
||||
pub max_in_flight: usize,
|
||||
/// Queued (waiting) requests allowed beyond the in-flight one. The
|
||||
/// `(max_in_flight + max_queue_depth + 1)`-th request is refused
|
||||
/// immediately with `429`/`503` + `Retry-After`.
|
||||
#[serde(default = "default_admission_max_queue_depth")]
|
||||
pub max_queue_depth: usize,
|
||||
/// Maximum seconds a queued request waits for the in-flight slot before
|
||||
/// it is refused (turns the old ~300s client-side hang into a fast,
|
||||
/// honest signal).
|
||||
#[serde(default = "default_admission_max_wait_secs")]
|
||||
pub max_wait_secs: u64,
|
||||
/// Per-principal fair-share cap (#54): max in-flight + queued requests
|
||||
/// for any single principal (resolved from the `x-helexa-*` headers
|
||||
/// cortex stamps), so one client can't monopolize the queue while others
|
||||
/// wait. Over-cap → `429 rate_limit_exceeded` + `Retry-After`. `0`
|
||||
/// disables the cap; anonymous requests are always exempt.
|
||||
#[serde(default = "default_admission_max_per_principal")]
|
||||
pub max_per_principal: usize,
|
||||
}
|
||||
|
||||
impl Default for AdmissionConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_in_flight: default_admission_max_in_flight(),
|
||||
max_queue_depth: default_admission_max_queue_depth(),
|
||||
max_wait_secs: default_admission_max_wait_secs(),
|
||||
max_per_principal: default_admission_max_per_principal(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn default_admission_max_in_flight() -> usize {
|
||||
1
|
||||
}
|
||||
|
||||
fn default_admission_max_queue_depth() -> usize {
|
||||
8
|
||||
}
|
||||
|
||||
fn default_admission_max_wait_secs() -> u64 {
|
||||
30
|
||||
}
|
||||
|
||||
fn default_admission_max_per_principal() -> usize {
|
||||
2
|
||||
}
|
||||
|
||||
/// `[harness.candle.prefix_cache]` settings.
|
||||
|
||||
298
crates/neuron/src/harness/admission.rs
Normal file
@@ -0,0 +1,298 @@
|
||||
//! Per-model admission control (#53).
|
||||
//!
|
||||
//! Inference against a loaded model is batch-1: one request runs at a time,
|
||||
//! serialized by the model's `inference_lock` (single-GPU) / `pool` mutex
|
||||
//! (TP). Before this, the wait for that lock was an **unbounded FIFO of
|
||||
//! mutex waiters with no timeout** — a busy model made every new request
|
||||
//! hang until its client gave up (~300s) with an opaque error.
|
||||
//!
|
||||
//! [`AdmissionController`] replaces that implicit unbounded wait with an
|
||||
//! explicit bounded scheduler: at most `max_in_flight` running (1, batch-1)
|
||||
//! plus a bounded queue of `max_queue_depth` waiters, each waiting at most
|
||||
//! `max_wait`. When the queue is full or the wait elapses, the request is
|
||||
//! rejected *immediately* — an honest, fast, retryable "busy" signal
|
||||
//! (`429`/`503` + `Retry-After` per #63) instead of a silent stall.
|
||||
//!
|
||||
//! The controller is pure async (no CUDA), so the inference paths just call
|
||||
//! [`AdmissionController::enter`] before taking the inference lock and hold
|
||||
//! the returned [`AdmissionPermit`] for the request's lifetime. Its counters
|
||||
//! ([`in_flight`](AdmissionController::in_flight) /
|
||||
//! [`queue_depth`](AdmissionController::queue_depth)) are lock-free, so
|
||||
//! `/health` can read live load without contending with inference.
|
||||
|
||||
use crate::config::AdmissionConfig;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
|
||||
|
||||
/// Why admission was refused. All map to the #63 backpressure envelope
|
||||
/// (`rate_limit_exceeded` + `Retry-After`); they differ in cause (and HTTP
|
||||
/// status — load → `503`, per-principal → `429`).
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum AdmissionRejection {
|
||||
/// The bounded wait queue was already full (server-side load).
|
||||
QueueFull { retry_after_secs: u64 },
|
||||
/// A queue slot was taken but the in-flight slot didn't free within
|
||||
/// `max_wait` (server-side load).
|
||||
Timeout { retry_after_secs: u64 },
|
||||
/// This principal already has `max_per_principal` requests in flight or
|
||||
/// queued (#54 fair-share) — one principal can't monopolize the model.
|
||||
PrincipalCap { retry_after_secs: u64 },
|
||||
}
|
||||
|
||||
impl AdmissionRejection {
|
||||
pub fn retry_after_secs(&self) -> u64 {
|
||||
match self {
|
||||
AdmissionRejection::QueueFull { retry_after_secs }
|
||||
| AdmissionRejection::Timeout { retry_after_secs }
|
||||
| AdmissionRejection::PrincipalCap { retry_after_secs } => *retry_after_secs,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Admission accounting, mutated under a brief lock (never held across an
|
||||
/// await). `pending` is queued + in-flight overall; `per_principal` is the
|
||||
/// same count keyed by principal for fair-share (#54).
|
||||
#[derive(Default, Debug)]
|
||||
struct AdmissionState {
|
||||
pending: usize,
|
||||
per_principal: HashMap<String, usize>,
|
||||
}
|
||||
|
||||
/// Bounded batch-1 scheduler for one loaded model, with per-principal
|
||||
/// fair-share.
|
||||
pub struct AdmissionController {
|
||||
/// In-flight slots — `max_in_flight` permits (1 for batch-1).
|
||||
slots: Arc<Semaphore>,
|
||||
/// Queued + in-flight accounting (overall + per principal).
|
||||
state: Arc<Mutex<AdmissionState>>,
|
||||
/// `max_in_flight + max_queue_depth` — the overall rejection threshold.
|
||||
max_pending: usize,
|
||||
/// Max in-flight + queued for any single principal (#54). `0` disables.
|
||||
max_per_principal: usize,
|
||||
max_in_flight: usize,
|
||||
max_wait: Duration,
|
||||
}
|
||||
|
||||
impl AdmissionController {
|
||||
pub fn new(cfg: &AdmissionConfig) -> Self {
|
||||
// A controller with zero in-flight slots would deadlock; clamp.
|
||||
let max_in_flight = cfg.max_in_flight.max(1);
|
||||
Self {
|
||||
slots: Arc::new(Semaphore::new(max_in_flight)),
|
||||
state: Arc::new(Mutex::new(AdmissionState::default())),
|
||||
max_pending: max_in_flight + cfg.max_queue_depth,
|
||||
max_per_principal: cfg.max_per_principal,
|
||||
max_in_flight,
|
||||
max_wait: Duration::from_secs(cfg.max_wait_secs),
|
||||
}
|
||||
}
|
||||
|
||||
/// Admit a request for `principal` (`None` = anonymous, exempt from the
|
||||
/// per-principal cap). Reserves a queue slot — fast-rejecting if the
|
||||
/// overall queue is full or the principal is over its fair-share cap —
|
||||
/// then waits up to `max_wait` for an in-flight slot. The returned permit
|
||||
/// must be held for the request's lifetime; dropping it frees the slots.
|
||||
pub async fn enter(
|
||||
&self,
|
||||
principal: Option<&str>,
|
||||
) -> Result<AdmissionPermit, AdmissionRejection> {
|
||||
// Decision + reservation under one brief lock so concurrent callers
|
||||
// can't both slip past the thresholds. No await is held here.
|
||||
{
|
||||
let mut st = self.state.lock().expect("admission state poisoned");
|
||||
if st.pending >= self.max_pending {
|
||||
return Err(AdmissionRejection::QueueFull {
|
||||
retry_after_secs: self.retry_hint(st.pending),
|
||||
});
|
||||
}
|
||||
if let Some(p) = principal
|
||||
&& self.max_per_principal > 0
|
||||
&& st.per_principal.get(p).copied().unwrap_or(0) >= self.max_per_principal
|
||||
{
|
||||
return Err(AdmissionRejection::PrincipalCap {
|
||||
retry_after_secs: self.retry_hint(st.pending),
|
||||
});
|
||||
}
|
||||
st.pending += 1;
|
||||
if let Some(p) = principal {
|
||||
*st.per_principal.entry(p.to_string()).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
|
||||
match tokio::time::timeout(self.max_wait, Arc::clone(&self.slots).acquire_owned()).await {
|
||||
Ok(Ok(permit)) => Ok(AdmissionPermit {
|
||||
_permit: permit,
|
||||
state: Arc::clone(&self.state),
|
||||
principal: principal.map(str::to_string),
|
||||
}),
|
||||
// Semaphore is never closed; treat a closed/elapsed wait the same.
|
||||
Ok(Err(_)) | Err(_) => {
|
||||
self.release(principal);
|
||||
Err(AdmissionRejection::Timeout {
|
||||
retry_after_secs: self.retry_hint(self.max_pending),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Roll back a reserved-but-not-admitted slot (wait timed out).
|
||||
fn release(&self, principal: Option<&str>) {
|
||||
let mut st = self.state.lock().expect("admission state poisoned");
|
||||
st.pending = st.pending.saturating_sub(1);
|
||||
decrement_principal(&mut st.per_principal, principal);
|
||||
}
|
||||
|
||||
/// Requests currently running (holding an in-flight slot).
|
||||
pub fn in_flight(&self) -> usize {
|
||||
self.max_in_flight
|
||||
.saturating_sub(self.slots.available_permits())
|
||||
}
|
||||
|
||||
/// Requests waiting for an in-flight slot.
|
||||
pub fn queue_depth(&self) -> usize {
|
||||
let pending = self.state.lock().expect("admission state poisoned").pending;
|
||||
pending.saturating_sub(self.in_flight())
|
||||
}
|
||||
|
||||
/// Rough `Retry-After`: scale with how backed-up the model is, clamped to
|
||||
/// a sane band. Without per-request timing this is a heuristic, but it
|
||||
/// gives well-behaved clients (opencode/AI SDK) a sensible backoff.
|
||||
fn retry_hint(&self, pending: usize) -> u64 {
|
||||
let queued = pending.saturating_sub(self.max_in_flight) as u64;
|
||||
((queued + 1) * 2).clamp(1, 120)
|
||||
}
|
||||
}
|
||||
|
||||
/// Decrement (and prune at zero) a principal's outstanding count.
|
||||
fn decrement_principal(map: &mut HashMap<String, usize>, principal: Option<&str>) {
|
||||
if let Some(p) = principal
|
||||
&& let Some(count) = map.get_mut(p)
|
||||
{
|
||||
*count -= 1;
|
||||
if *count == 0 {
|
||||
map.remove(p);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Held for a request's lifetime; frees the in-flight + queue slot (and the
|
||||
/// principal's fair-share slot) on drop.
|
||||
#[derive(Debug)]
|
||||
pub struct AdmissionPermit {
|
||||
_permit: OwnedSemaphorePermit,
|
||||
state: Arc<Mutex<AdmissionState>>,
|
||||
principal: Option<String>,
|
||||
}
|
||||
|
||||
impl Drop for AdmissionPermit {
|
||||
fn drop(&mut self) {
|
||||
let mut st = self.state.lock().expect("admission state poisoned");
|
||||
st.pending = st.pending.saturating_sub(1);
|
||||
decrement_principal(&mut st.per_principal, self.principal.as_deref());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Config with the per-principal cap disabled (0) — most tests exercise
|
||||
/// the overall queue with anonymous (`None`) callers.
|
||||
fn cfg(max_in_flight: usize, max_queue_depth: usize, max_wait_secs: u64) -> AdmissionConfig {
|
||||
AdmissionConfig {
|
||||
max_in_flight,
|
||||
max_queue_depth,
|
||||
max_wait_secs,
|
||||
max_per_principal: 0,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admits_up_to_in_flight_and_reports_load() {
|
||||
let ctrl = AdmissionController::new(&cfg(1, 4, 30));
|
||||
assert_eq!(ctrl.in_flight(), 0);
|
||||
let p = ctrl.enter(None).await.expect("first admits");
|
||||
assert_eq!(ctrl.in_flight(), 1);
|
||||
assert_eq!(ctrl.queue_depth(), 0);
|
||||
drop(p);
|
||||
assert_eq!(ctrl.in_flight(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejects_when_queue_full() {
|
||||
// 1 in-flight + 1 queue slot = capacity 2; the 3rd is refused fast.
|
||||
let ctrl = Arc::new(AdmissionController::new(&cfg(1, 1, 30)));
|
||||
let _running = ctrl.enter(None).await.expect("admit running");
|
||||
|
||||
// Fill the single queue slot with a waiter that parks on the semaphore.
|
||||
let ctrl2 = Arc::clone(&ctrl);
|
||||
let waiter = tokio::spawn(async move { ctrl2.enter(None).await.map(|p| drop(p)) });
|
||||
// Give the waiter a moment to occupy the queue slot.
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
assert_eq!(ctrl.queue_depth(), 1);
|
||||
|
||||
// Queue full → immediate QueueFull with a Retry-After hint.
|
||||
match ctrl.enter(None).await {
|
||||
Err(AdmissionRejection::QueueFull { retry_after_secs }) => {
|
||||
assert!(retry_after_secs >= 1)
|
||||
}
|
||||
other => panic!("expected QueueFull, got {other:?}"),
|
||||
}
|
||||
|
||||
// Release the runner so the parked waiter can proceed and finish.
|
||||
drop(_running);
|
||||
waiter.await.unwrap().unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejects_on_wait_timeout() {
|
||||
// Zero queue depth + a runner holding the only slot → a second
|
||||
// request can't even queue, so it's QueueFull, not Timeout. Use a
|
||||
// queue of 1 and a tiny max_wait to exercise the timeout path.
|
||||
let ctrl = Arc::new(AdmissionController::new(&cfg(1, 1, 0)));
|
||||
let _running = ctrl.enter(None).await.expect("admit running");
|
||||
// max_wait 0 → the queued request times out almost immediately.
|
||||
match ctrl.enter(None).await {
|
||||
Err(AdmissionRejection::Timeout { .. }) => {}
|
||||
other => panic!("expected Timeout, got {other:?}"),
|
||||
}
|
||||
// The timed-out request released its queue slot.
|
||||
assert_eq!(ctrl.queue_depth(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn per_principal_cap_protects_other_principals() {
|
||||
// Generous overall queue, but each principal capped at 1 in-flight+
|
||||
// queued. Principal A holds the running slot; A's second request is
|
||||
// refused (PrincipalCap) rather than occupying the queue, so B's
|
||||
// single request still gets a queue slot and proceeds.
|
||||
let cfg = AdmissionConfig {
|
||||
max_in_flight: 1,
|
||||
max_queue_depth: 8,
|
||||
max_wait_secs: 30,
|
||||
max_per_principal: 1,
|
||||
};
|
||||
let ctrl = Arc::new(AdmissionController::new(&cfg));
|
||||
|
||||
let _a1 = ctrl.enter(Some("acct-a/key-a")).await.expect("A admits");
|
||||
|
||||
// A is over its fair-share cap → fast PrincipalCap, no queue slot taken.
|
||||
match ctrl.enter(Some("acct-a/key-a")).await {
|
||||
Err(AdmissionRejection::PrincipalCap { retry_after_secs }) => {
|
||||
assert!(retry_after_secs >= 1)
|
||||
}
|
||||
other => panic!("expected PrincipalCap, got {other:?}"),
|
||||
}
|
||||
|
||||
// B (a different principal) is admitted to the queue and proceeds
|
||||
// once A releases — it was never stuck behind A's backlog.
|
||||
let ctrl2 = Arc::clone(&ctrl);
|
||||
let b = tokio::spawn(async move { ctrl2.enter(Some("acct-b/key-b")).await.map(drop) });
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
assert_eq!(ctrl.queue_depth(), 1, "B is queued, not rejected");
|
||||
drop(_a1);
|
||||
b.await.unwrap().expect("B is served after A releases");
|
||||
}
|
||||
}
|
||||
@@ -33,7 +33,7 @@ use crate::wire::{
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
|
||||
#[cfg(feature = "cuda")]
|
||||
use std::time::Duration;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
@@ -81,6 +81,9 @@ pub struct CandleHarness {
|
||||
/// Context-limit derivation settings (#67), read in `list_models`
|
||||
/// to compute each model's advertised `limit{context,input,output}`.
|
||||
context_limit_cfg: crate::config::ContextLimitConfig,
|
||||
/// Admission-control settings (#53), used to build each loaded model's
|
||||
/// [`super::admission::AdmissionController`] at load time.
|
||||
admission_cfg: crate::config::AdmissionConfig,
|
||||
}
|
||||
|
||||
/// Devices/capabilities snapshot of a model entering auto-recovery
|
||||
@@ -146,6 +149,16 @@ impl LoadedHandle {
|
||||
}
|
||||
}
|
||||
|
||||
/// Current admission load (#53): `(in_flight, queue_depth)`. Lock-free,
|
||||
/// so `/health` can read it without contending with inference.
|
||||
pub fn load(&self) -> (usize, usize) {
|
||||
match self {
|
||||
LoadedHandle::Single(m) => (m.admission.in_flight(), m.admission.queue_depth()),
|
||||
#[cfg(feature = "cuda")]
|
||||
LoadedHandle::Tp(m) => (m.admission.in_flight(), m.admission.queue_depth()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Modalities the loaded model supports. Stage B7 (single-GPU) +
|
||||
/// TP-vision (#12) — both single-GPU and TP loads advertise
|
||||
/// `"vision"` when a replicated vision tower materialised.
|
||||
@@ -192,23 +205,50 @@ impl LoadedHandle {
|
||||
/// `NEURON_MAX_PROMPT_TOKENS`, when explicitly set, is applied as a
|
||||
/// clamp-only upper bound on the derived `context` — a backstop, not
|
||||
/// the authority. Unset → no clamp; the derivation stands alone.
|
||||
pub async fn derived_limit(
|
||||
/// Refresh the cached free-VRAM reading used by [`Self::derived_limit`]
|
||||
/// (#53). Queries the device worker — so it MUST run off the request
|
||||
/// path (background refresher / load-time seed), never from a control
|
||||
/// endpoint, since the query queues behind inference on the worker.
|
||||
/// Single-GPU caches the device's free VRAM; TP caches the tightest
|
||||
/// free across ranks (the same value `derived_limit` used pre-cache).
|
||||
pub async fn refresh_free_mb(&self) {
|
||||
let free = match self {
|
||||
LoadedHandle::Single(m) => m.query_vram().await.0,
|
||||
#[cfg(feature = "cuda")]
|
||||
LoadedHandle::Tp(m) => m.query_vram_tightest_free_mb().await,
|
||||
};
|
||||
// Don't clobber a good cached value with a transient `0`
|
||||
// (worker gone/poisoned sentinel).
|
||||
if free > 0 {
|
||||
match self {
|
||||
LoadedHandle::Single(m) => m.last_free_mb.store(free, Ordering::Release),
|
||||
#[cfg(feature = "cuda")]
|
||||
LoadedHandle::Tp(m) => m.last_free_mb.store(free, Ordering::Release),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn derived_limit(
|
||||
&self,
|
||||
cfg: &crate::config::ContextLimitConfig,
|
||||
) -> Option<cortex_core::harness::ModelLimit> {
|
||||
if !cfg.enabled {
|
||||
return None;
|
||||
}
|
||||
// Read the *cached* free VRAM — never query the device worker here.
|
||||
// This runs on `GET /models`; a live query would queue behind
|
||||
// inference on the worker thread and stall the control plane (#53).
|
||||
// The cache is refreshed off the request path (load + background task).
|
||||
let (profile, free_mb, rate) = match self {
|
||||
LoadedHandle::Single(m) => (
|
||||
m.context_profile?,
|
||||
m.query_vram().await.0,
|
||||
m.last_free_mb.load(Ordering::Acquire),
|
||||
m.prefill_rate.get(),
|
||||
),
|
||||
#[cfg(feature = "cuda")]
|
||||
LoadedHandle::Tp(m) => (
|
||||
m.context_profile?,
|
||||
m.query_vram_tightest_free_mb().await,
|
||||
m.last_free_mb.load(Ordering::Acquire),
|
||||
m.prefill_rate.get(),
|
||||
),
|
||||
};
|
||||
@@ -305,6 +345,10 @@ pub struct LoadedModel {
|
||||
/// for the TP path (which already had this invariant by accident
|
||||
/// because the pool lock covered the same window).
|
||||
pub inference_lock: tokio::sync::Mutex<()>,
|
||||
/// Bounded admission scheduler (#53). Gated *before* `inference_lock`
|
||||
/// so a busy model refuses overflow fast instead of growing an
|
||||
/// unbounded, untimed queue of lock waiters.
|
||||
pub admission: super::admission::AdmissionController,
|
||||
/// Open/close token IDs for the reasoning marker this model
|
||||
/// emits, populated once at load time by probing the tokenizer's
|
||||
/// added-tokens table. `None` for non-reasoning models or
|
||||
@@ -374,6 +418,13 @@ pub struct LoadedModel {
|
||||
/// request-path enforcement reads this — `0` means "not derived yet"
|
||||
/// → fall back to the static `NEURON_MAX_PROMPT_TOKENS`.
|
||||
pub derived_input_cap: AtomicUsize,
|
||||
/// Cached free VRAM (MiB) for the control plane (#53). `derived_limit`
|
||||
/// (served by `GET /models`) reads this instead of querying the device
|
||||
/// worker, which during inference is saturated processing forward jobs —
|
||||
/// a live query would queue behind them and stall `/models`, tripping
|
||||
/// cortex's health poller into marking the node unhealthy. Refreshed off
|
||||
/// the request path: seeded at load, then by a background task.
|
||||
pub last_free_mb: AtomicU64,
|
||||
}
|
||||
|
||||
impl LoadedModel {
|
||||
@@ -422,6 +473,10 @@ pub struct TpLoadedModel {
|
||||
/// serialises subprocess RPC traffic on the pool's
|
||||
/// `Vec<Worker>` channels.
|
||||
pub pool: tokio::sync::Mutex<super::tp::WorkerPool>,
|
||||
/// Bounded admission scheduler (#53), mirroring the single-GPU path.
|
||||
/// Gated before the pool lock so an overloaded TP model returns fast
|
||||
/// backpressure instead of an unbounded, untimed wait.
|
||||
pub admission: super::admission::AdmissionController,
|
||||
/// Handle into the leader device worker's TP slab. The boxed
|
||||
/// `TpLeaderModel` (with its embedded `Arc<Comm>` clones and
|
||||
/// per-rank CUDA tensors) lives on the worker thread; we hold an
|
||||
@@ -482,6 +537,10 @@ pub struct TpLoadedModel {
|
||||
/// Mint for pool-wide snapshot ids. Plain counter; uniqueness only
|
||||
/// needs to hold per model lifetime (snapshots die with the model).
|
||||
pub next_snapshot_id: std::sync::atomic::AtomicU64,
|
||||
/// Cached tightest free VRAM (MiB) for the control plane (#53) — see
|
||||
/// [`LoadedModel::last_free_mb`]. Read by `derived_limit` so `GET /models`
|
||||
/// never fans a VRAM query out to the (inference-saturated) TP workers.
|
||||
pub last_free_mb: AtomicU64,
|
||||
}
|
||||
|
||||
#[cfg(feature = "cuda")]
|
||||
@@ -1088,6 +1147,32 @@ fn debug_poison_armed(model_id: &str) -> bool {
|
||||
armed && !FIRED.swap(true, Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Background control-plane VRAM cache refresher (#53). Every few seconds,
|
||||
/// refreshes each loaded model's `last_free_mb` so `derived_limit` (served
|
||||
/// by `GET /models`) reads a cached value and never queries the device
|
||||
/// worker on the request path — a live query would queue behind inference
|
||||
/// forward jobs on the worker thread, stalling `/models` for seconds and
|
||||
/// tripping cortex's health poller into evicting the node from routing.
|
||||
/// Holds a `Weak` so a shutting-down harness lets the task exit. The query
|
||||
/// itself may queue behind inference, but that only delays this background
|
||||
/// refresh — no request-path caller is ever blocked.
|
||||
async fn vram_cache_refresh_loop(weak: std::sync::Weak<CandleHarness>) {
|
||||
const REFRESH_INTERVAL: std::time::Duration = std::time::Duration::from_secs(5);
|
||||
loop {
|
||||
tokio::time::sleep(REFRESH_INTERVAL).await;
|
||||
let Some(this) = weak.upgrade() else {
|
||||
return; // harness dropped — exit
|
||||
};
|
||||
// Snapshot handles, then release the read lock before awaiting the
|
||||
// (possibly slow) worker queries so we never hold it across an await.
|
||||
let handles: Vec<LoadedHandle> = this.models.read().await.values().cloned().collect();
|
||||
drop(this);
|
||||
for handle in handles {
|
||||
handle.refresh_free_mb().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Background auto-recovery task (#17). Drains poisoned model ids and
|
||||
/// rebuilds each via [`CandleHarness::recover_one`]. Holds a `Weak` so a
|
||||
/// shutting-down harness lets the task exit; processes one id at a time,
|
||||
@@ -1261,25 +1346,67 @@ fn validate_vision_prefill(prompt_len: usize, vram_free_mb: u64) -> Result<(), I
|
||||
/// the caller as `max`), or if free VRAM is below the floor. Enforcing
|
||||
/// the *derived* cap means a VRAM-tight host rejects a prompt that
|
||||
/// wouldn't fit, instead of accepting it and OOMing mid-prefill.
|
||||
///
|
||||
/// The third VRAM check — the length-aware backstop (#65) — closes the
|
||||
/// poll-vs-request snapshot gap #67 leaves open. `max` is
|
||||
/// `effective_prompt_cap()`, the input budget derived at **/models poll
|
||||
/// time** from the tightest card's free VRAM *then*. If free VRAM has
|
||||
/// since dropped (a co-resident model loaded, a concurrent prefill grew
|
||||
/// its KV), a prompt at-or-below that now-stale cap still clears the
|
||||
/// static floor yet no longer fits — and OOMs mid-prefill, poisoning the
|
||||
/// device context (the 2026-05-26 beast incident the #47 work exists to
|
||||
/// eliminate). So we re-run the same length×KV-vs-VRAM physics #67 uses
|
||||
/// for the cap, but against **request-time** free VRAM, reusing the
|
||||
/// model's [`ContextProfile`] rather than re-deriving the KV cost. This
|
||||
/// gives the text path the live-VRAM guard the vision path already has
|
||||
/// (`validate_vision_prefill`). `profile`/`kv_bytes_per_token_per_card`
|
||||
/// are per-card and `vram_free_mb` is the tightest card's free VRAM, so
|
||||
/// the two are commensurable on both single-GPU and TP loads.
|
||||
fn validate_request(
|
||||
prompt_len: usize,
|
||||
vram_free_mb: u64,
|
||||
max: usize,
|
||||
profile: Option<&super::context_limit::ContextProfile>,
|
||||
cfg: &crate::config::ContextLimitConfig,
|
||||
) -> Result<(), InferenceError> {
|
||||
if prompt_len > max {
|
||||
return Err(InferenceError::PromptTooLong { prompt_len, max });
|
||||
}
|
||||
// VRAM check is skipped on CPU loads (vram_free_mb == 0 sentinel)
|
||||
// VRAM checks are skipped on CPU loads (vram_free_mb == 0 sentinel)
|
||||
// because the (0, 0) reply from `query_vram` is also what a missing
|
||||
// worker returns. The CPU path has no per-GPU memory limit anyway —
|
||||
// host RAM is bounded by the OOM killer, not this check.
|
||||
if vram_free_mb == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
let min = min_free_vram_mb();
|
||||
if vram_free_mb != 0 && vram_free_mb < min {
|
||||
if vram_free_mb < min {
|
||||
return Err(InferenceError::InsufficientVram {
|
||||
free_mb: vram_free_mb,
|
||||
required_mb: min,
|
||||
});
|
||||
}
|
||||
// Length-aware backstop (#65): KV the whole sequence (prompt +
|
||||
// generation reserve) will occupy, plus the prefill activation
|
||||
// headroom, plus the static floor as an additive cushion — all per
|
||||
// card. A degenerate zero-KV profile (no full-attention layers) or a
|
||||
// model with no captured profile skips this and rides the floor
|
||||
// check above, mirroring `derive_limit`'s VRAM-ceiling fallback.
|
||||
if let Some(profile) = profile
|
||||
&& profile.kv_bytes_per_token_per_card > 0
|
||||
{
|
||||
let tokens = (prompt_len as u64).saturating_add(cfg.output_reserve_tokens as u64);
|
||||
let kv_mb = profile.kv_bytes_per_token_per_card.saturating_mul(tokens) / (1024 * 1024);
|
||||
let required_mb = kv_mb
|
||||
.saturating_add(cfg.activation_headroom_mb)
|
||||
.saturating_add(min);
|
||||
if required_mb > vram_free_mb {
|
||||
return Err(InferenceError::InsufficientVram {
|
||||
free_mb: vram_free_mb,
|
||||
required_mb,
|
||||
});
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1565,6 +1692,7 @@ impl CandleHarness {
|
||||
recovery_tx,
|
||||
prefix_cache_cfg: config.prefix_cache.clone(),
|
||||
context_limit_cfg: config.context_limit.clone(),
|
||||
admission_cfg: config.admission.clone(),
|
||||
});
|
||||
// Background auto-recovery task (#17). Holds a `Weak` so it can't
|
||||
// keep the harness alive. Spawned only when a tokio runtime is
|
||||
@@ -1573,6 +1701,11 @@ impl CandleHarness {
|
||||
if tokio::runtime::Handle::try_current().is_ok() {
|
||||
let weak = Arc::downgrade(&this);
|
||||
tokio::spawn(recovery_loop(weak, recovery_rx));
|
||||
// Control-plane VRAM cache refresher (#53): keeps each loaded
|
||||
// model's `last_free_mb` current off the request path, so
|
||||
// `derived_limit` / `GET /models` never query the device worker
|
||||
// (which is saturated during inference) and never stall.
|
||||
tokio::spawn(vram_cache_refresh_loop(Arc::downgrade(&this)));
|
||||
}
|
||||
this
|
||||
}
|
||||
@@ -2006,6 +2139,7 @@ impl CandleHarness {
|
||||
pub async fn chat_completion(
|
||||
&self,
|
||||
request: ChatCompletionRequest,
|
||||
principal: Option<String>,
|
||||
) -> Result<ChatCompletionResponse, InferenceError> {
|
||||
let handle = {
|
||||
let models = self.models.read().await;
|
||||
@@ -2030,7 +2164,7 @@ impl CandleHarness {
|
||||
LoadedHandle::Single(m) => m,
|
||||
#[cfg(feature = "cuda")]
|
||||
LoadedHandle::Tp(m) => {
|
||||
return self.chat_completion_tp(m, request).await;
|
||||
return self.chat_completion_tp(m, request, principal).await;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -2059,6 +2193,15 @@ impl CandleHarness {
|
||||
return Err(self.trigger_recovery(&model_id).await);
|
||||
}
|
||||
|
||||
// Admission control (#53): refuse fast if the bounded queue is full
|
||||
// or the wait elapses, rather than joining an unbounded lock-wait.
|
||||
// The permit is held for the whole request (released on drop).
|
||||
let _admit = loaded
|
||||
.admission
|
||||
.enter(principal.as_deref())
|
||||
.await
|
||||
.map_err(InferenceError::from)?;
|
||||
|
||||
// Serialise concurrent requests against this model. Holds for
|
||||
// the duration of clear_kv_cache → prefill → decode so two
|
||||
// requests' chunked-prefill sequences can't interleave on the
|
||||
@@ -2149,7 +2292,13 @@ impl CandleHarness {
|
||||
"chat_completion: starting"
|
||||
);
|
||||
|
||||
validate_request(prompt_len, vram_free_mb, loaded.effective_prompt_cap())?;
|
||||
validate_request(
|
||||
prompt_len,
|
||||
vram_free_mb,
|
||||
loaded.effective_prompt_cap(),
|
||||
loaded.context_profile.as_ref(),
|
||||
&self.context_limit_cfg,
|
||||
)?;
|
||||
if vision_route.is_some() {
|
||||
validate_vision_prefill(prompt_len, vram_free_mb)?;
|
||||
}
|
||||
@@ -2378,9 +2527,14 @@ impl CandleHarness {
|
||||
pub async fn chat_completion_stream(
|
||||
&self,
|
||||
request: ChatCompletionRequest,
|
||||
principal: Option<String>,
|
||||
) -> Result<mpsc::Receiver<ChatCompletionChunk>, InferenceError> {
|
||||
self.chat_completion_stream_with(request, wire_chat::ChatProjectionConfig::default())
|
||||
.await
|
||||
self.chat_completion_stream_with(
|
||||
request,
|
||||
wire_chat::ChatProjectionConfig::default(),
|
||||
principal,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Same as [`Self::chat_completion_stream`] but lets the caller
|
||||
@@ -2391,8 +2545,9 @@ impl CandleHarness {
|
||||
&self,
|
||||
request: ChatCompletionRequest,
|
||||
mut config: wire_chat::ChatProjectionConfig,
|
||||
principal: Option<String>,
|
||||
) -> Result<mpsc::Receiver<ChatCompletionChunk>, InferenceError> {
|
||||
let stream = self.inference_stream(request).await?;
|
||||
let stream = self.inference_stream(request, principal).await?;
|
||||
// Fill in the model's reasoning markers if the caller
|
||||
// didn't pre-populate them — they're a property of the
|
||||
// loaded model (which the HTTP handler doesn't reach into
|
||||
@@ -2419,9 +2574,10 @@ impl CandleHarness {
|
||||
request: ChatCompletionRequest,
|
||||
response_id: String,
|
||||
message_item_id: String,
|
||||
principal: Option<String>,
|
||||
) -> Result<mpsc::Receiver<crate::wire::openai_responses::ResponseStreamFrame>, InferenceError>
|
||||
{
|
||||
let stream = self.inference_stream(request).await?;
|
||||
let stream = self.inference_stream(request, principal).await?;
|
||||
let meta = crate::wire::openai_responses::ResponseMeta {
|
||||
response_id,
|
||||
created_at: stream.created,
|
||||
@@ -2442,6 +2598,7 @@ impl CandleHarness {
|
||||
async fn inference_stream(
|
||||
&self,
|
||||
request: ChatCompletionRequest,
|
||||
principal: Option<String>,
|
||||
) -> Result<InferenceStream, InferenceError> {
|
||||
let handle = {
|
||||
let models = self.models.read().await;
|
||||
@@ -2466,7 +2623,7 @@ impl CandleHarness {
|
||||
LoadedHandle::Single(m) => m,
|
||||
#[cfg(feature = "cuda")]
|
||||
LoadedHandle::Tp(m) => {
|
||||
return self.inference_tp_stream(m, request).await;
|
||||
return self.inference_tp_stream(m, request, principal).await;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -2595,7 +2752,13 @@ impl CandleHarness {
|
||||
);
|
||||
}
|
||||
|
||||
validate_request(prompt_len, vram_free_mb, loaded.effective_prompt_cap())?;
|
||||
validate_request(
|
||||
prompt_len,
|
||||
vram_free_mb,
|
||||
loaded.effective_prompt_cap(),
|
||||
loaded.context_profile.as_ref(),
|
||||
&self.context_limit_cfg,
|
||||
)?;
|
||||
if vision_route.is_some() {
|
||||
validate_vision_prefill(prompt_len, vram_free_mb)?;
|
||||
}
|
||||
@@ -2610,6 +2773,15 @@ impl CandleHarness {
|
||||
// role chunk was already sent above, so the client sees
|
||||
// immediate "stream open" feedback even when this request
|
||||
// queues behind another for the lock.
|
||||
// Admission control (#53): refuse before opening the stream if the
|
||||
// model's bounded queue is full / the wait elapses. The permit moves
|
||||
// into the inference task and is held until it completes.
|
||||
let admit = loaded
|
||||
.admission
|
||||
.enter(principal.as_deref())
|
||||
.await
|
||||
.map_err(InferenceError::from)?;
|
||||
|
||||
let tool_schemas = build_tool_schemas(&request);
|
||||
if let (Some(worker), Some(handle)) = (loaded.worker.clone(), loaded.arch_handle) {
|
||||
#[cfg(feature = "cuda")]
|
||||
@@ -2620,6 +2792,7 @@ impl CandleHarness {
|
||||
let tool_schemas_inner = tool_schemas.clone();
|
||||
tokio::spawn(
|
||||
async move {
|
||||
let _admit = admit;
|
||||
let _inference_guard = loaded_for_task.inference_lock.lock().await;
|
||||
match stream_inference_via_worker(
|
||||
worker,
|
||||
@@ -2680,6 +2853,7 @@ impl CandleHarness {
|
||||
let tool_call_tokens_inner = loaded.tool_call_tokens.clone();
|
||||
let tool_schemas_inner = tool_schemas.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _admit = admit;
|
||||
let _g = span_for_task.enter();
|
||||
// `blocking_lock` is safe here: spawn_blocking runs on
|
||||
// a dedicated thread, not on the async runtime, so
|
||||
@@ -2779,6 +2953,24 @@ pub struct InferenceStream {
|
||||
/// Auto-recovery (#17) — rebuild a poisoned model's device context
|
||||
/// automatically instead of leaving it bricked until a human reloads.
|
||||
impl CandleHarness {
|
||||
/// Per-model admission load for `GET /health` (#53): in-flight + queued
|
||||
/// counts for every resident model. Lock-free per-model reads, so this
|
||||
/// only briefly holds the registry read lock to enumerate handles.
|
||||
pub async fn load_snapshot(&self) -> Vec<cortex_core::discovery::ModelLoad> {
|
||||
let models = self.models.read().await;
|
||||
models
|
||||
.values()
|
||||
.map(|handle| {
|
||||
let (in_flight, queue_depth) = handle.load();
|
||||
cortex_core::discovery::ModelLoad {
|
||||
id: handle.model_id().to_string(),
|
||||
in_flight,
|
||||
queue_depth,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// True while `model_id` is being auto-recovered (its slot is briefly
|
||||
/// absent from the registry during the reload).
|
||||
pub async fn is_recovering(&self, model_id: &str) -> bool {
|
||||
@@ -2890,7 +3082,7 @@ impl Harness for CandleHarness {
|
||||
// physics + live free VRAM + measured prefill rate. `None`
|
||||
// for arches without a context profile. `cost` stays
|
||||
// operator-set in the catalogue, filled by the gateway.
|
||||
let limit = h.derived_limit(&self.context_limit_cfg).await;
|
||||
let limit = h.derived_limit(&self.context_limit_cfg);
|
||||
out.push(ModelInfo {
|
||||
id: h.model_id().into(),
|
||||
harness: "candle".into(),
|
||||
@@ -3128,6 +3320,7 @@ impl Harness for CandleHarness {
|
||||
worker,
|
||||
arch_handle,
|
||||
inference_lock: tokio::sync::Mutex::new(()),
|
||||
admission: super::admission::AdmissionController::new(&self.admission_cfg),
|
||||
reasoning_tokens,
|
||||
tool_call_tokens,
|
||||
chat_template,
|
||||
@@ -3139,6 +3332,7 @@ impl Harness for CandleHarness {
|
||||
context_profile,
|
||||
prefill_rate: super::context_limit::PrefillRateEma::new(),
|
||||
derived_input_cap: AtomicUsize::new(0),
|
||||
last_free_mb: AtomicU64::new(0),
|
||||
});
|
||||
if loaded.prefix_cache.is_some() {
|
||||
tracing::info!(
|
||||
@@ -3149,6 +3343,14 @@ impl Harness for CandleHarness {
|
||||
);
|
||||
}
|
||||
|
||||
// Seed the control-plane VRAM cache (#53) while the worker is idle
|
||||
// (load just finished), so `/models` has a value before the
|
||||
// background refresher's first tick and never queries the worker.
|
||||
let (free_mb, _) = loaded.query_vram().await;
|
||||
if free_mb > 0 {
|
||||
loaded.last_free_mb.store(free_mb, Ordering::Release);
|
||||
}
|
||||
|
||||
let mut models = self.models.write().await;
|
||||
models.insert(spec.model_id.clone(), LoadedHandle::Single(loaded));
|
||||
tracing::info!(model = %spec.model_id, "model loaded");
|
||||
@@ -3372,6 +3574,7 @@ impl CandleHarness {
|
||||
tokenizer,
|
||||
devices: devices.clone(),
|
||||
pool: TMutex::new(pool),
|
||||
admission: super::admission::AdmissionController::new(&self.admission_cfg),
|
||||
leader_handle,
|
||||
leader_device: leader_device.clone(),
|
||||
poisoned: AtomicBool::new(false),
|
||||
@@ -3398,6 +3601,7 @@ impl CandleHarness {
|
||||
),
|
||||
prefill_rate: super::context_limit::PrefillRateEma::new(),
|
||||
derived_input_cap: AtomicUsize::new(0),
|
||||
last_free_mb: AtomicU64::new(0),
|
||||
next_snapshot_id: std::sync::atomic::AtomicU64::new(1),
|
||||
});
|
||||
if tp_loaded.prefix_cache.is_some() {
|
||||
@@ -3409,6 +3613,14 @@ impl CandleHarness {
|
||||
);
|
||||
}
|
||||
|
||||
// Seed the control-plane VRAM cache (#53) — tightest free across
|
||||
// ranks, while the workers are idle post-load — so `/models` never
|
||||
// fans a query out to the inference-busy TP workers.
|
||||
let free_mb = tp_loaded.query_vram_tightest_free_mb().await;
|
||||
if free_mb > 0 {
|
||||
tp_loaded.last_free_mb.store(free_mb, Ordering::Release);
|
||||
}
|
||||
|
||||
let mut models = self.models.write().await;
|
||||
models.insert(spec.model_id.clone(), LoadedHandle::Tp(tp_loaded));
|
||||
tracing::info!(
|
||||
@@ -3438,6 +3650,7 @@ impl CandleHarness {
|
||||
&self,
|
||||
tp: Arc<TpLoadedModel>,
|
||||
request: ChatCompletionRequest,
|
||||
principal: Option<String>,
|
||||
) -> Result<ChatCompletionResponse, InferenceError> {
|
||||
// Tag every line of this request with a short req_id so a
|
||||
// grep over journalctl reconstructs one request even when
|
||||
@@ -3474,7 +3687,11 @@ impl CandleHarness {
|
||||
}
|
||||
|
||||
let tp_for_marker = Arc::clone(&tp);
|
||||
let handle = tokio::spawn(chat_completion_tp_inner(tp, request).instrument(span.clone()));
|
||||
let context_limit_cfg = self.context_limit_cfg.clone();
|
||||
let handle = tokio::spawn(
|
||||
chat_completion_tp_inner(tp, request, principal, context_limit_cfg)
|
||||
.instrument(span.clone()),
|
||||
);
|
||||
match handle.await {
|
||||
Ok(Ok(resp)) => Ok(resp),
|
||||
Ok(Err(e)) => {
|
||||
@@ -3545,6 +3762,7 @@ impl CandleHarness {
|
||||
&self,
|
||||
tp: Arc<TpLoadedModel>,
|
||||
request: ChatCompletionRequest,
|
||||
principal: Option<String>,
|
||||
) -> Result<InferenceStream, InferenceError> {
|
||||
if tp.poisoned.load(Ordering::Acquire) {
|
||||
return Err(self.trigger_recovery(&request.model).await);
|
||||
@@ -3685,15 +3903,30 @@ impl CandleHarness {
|
||||
"TP chat_completion (stream): starting"
|
||||
);
|
||||
|
||||
validate_request(prompt_len, vram_free_mb, tp.effective_prompt_cap())?;
|
||||
validate_request(
|
||||
prompt_len,
|
||||
vram_free_mb,
|
||||
tp.effective_prompt_cap(),
|
||||
tp.context_profile.as_ref(),
|
||||
&self.context_limit_cfg,
|
||||
)?;
|
||||
if vision_route.is_some() {
|
||||
validate_vision_prefill(prompt_len, vram_free_mb)?;
|
||||
}
|
||||
|
||||
// Admission control (#53): refuse before opening the stream; the
|
||||
// permit moves into the orchestration task and is held for its life.
|
||||
let admit = tp
|
||||
.admission
|
||||
.enter(principal.as_deref())
|
||||
.await
|
||||
.map_err(InferenceError::from)?;
|
||||
|
||||
let tool_schemas = build_tool_schemas(&request);
|
||||
let tp_for_task = Arc::clone(&tp);
|
||||
tokio::spawn(
|
||||
async move {
|
||||
let _admit = admit;
|
||||
let mut failure: Option<String> = None;
|
||||
let mut pool = acquire_pool_lock(&tp_for_task.pool, &model_id).await;
|
||||
let leader_handle = tp_for_task.leader_handle;
|
||||
@@ -4196,6 +4429,8 @@ impl CandleHarness {
|
||||
async fn chat_completion_tp_inner(
|
||||
tp: Arc<TpLoadedModel>,
|
||||
request: ChatCompletionRequest,
|
||||
principal: Option<String>,
|
||||
context_limit_cfg: crate::config::ContextLimitConfig,
|
||||
) -> Result<ChatCompletionResponse, InferenceError> {
|
||||
let req_start = std::time::Instant::now();
|
||||
let model_id = request.model.clone();
|
||||
@@ -4279,11 +4514,25 @@ async fn chat_completion_tp_inner(
|
||||
"TP chat_completion: starting"
|
||||
);
|
||||
|
||||
validate_request(prompt_len, vram_free_mb, tp.effective_prompt_cap())?;
|
||||
validate_request(
|
||||
prompt_len,
|
||||
vram_free_mb,
|
||||
tp.effective_prompt_cap(),
|
||||
tp.context_profile.as_ref(),
|
||||
&context_limit_cfg,
|
||||
)?;
|
||||
if vision_route.is_some() {
|
||||
validate_vision_prefill(prompt_len, vram_free_mb)?;
|
||||
}
|
||||
|
||||
// Admission control (#53): bounded queue + fast reject before joining
|
||||
// the pool-lock wait. Held for the whole request (released on drop).
|
||||
let _admit = tp
|
||||
.admission
|
||||
.enter(principal.as_deref())
|
||||
.await
|
||||
.map_err(InferenceError::from)?;
|
||||
|
||||
// Acquire the pool lock for the duration of the request. After
|
||||
// Phase 3 the leader's TpLeaderModel lives in the device worker
|
||||
// thread, so the pool lock now serialises only subprocess RPC
|
||||
@@ -4826,10 +5075,35 @@ pub enum InferenceError {
|
||||
/// failure mode that hid several client-compat bugs. Maps to 422.
|
||||
#[error("chat template could not render this request: {detail}")]
|
||||
TemplateRenderFailed { detail: String },
|
||||
/// Admission control (#53) refused on load: the model's bounded queue is
|
||||
/// full or the wait elapsed. Maps to `503 rate_limit_exceeded` +
|
||||
/// `Retry-After` — a fast, retryable "busy" signal, not a stall.
|
||||
#[error("model is busy; retry after {retry_after_secs}s")]
|
||||
Overloaded { retry_after_secs: u64 },
|
||||
/// Per-principal fair-share cap (#54) exceeded: this principal already
|
||||
/// has its max requests in flight/queued. Maps to `429
|
||||
/// rate_limit_exceeded` + `Retry-After`; a well-behaved client self-paces.
|
||||
#[error("per-principal in-flight limit reached; retry after {retry_after_secs}s")]
|
||||
PerPrincipalLimit { retry_after_secs: u64 },
|
||||
#[error(transparent)]
|
||||
Other(#[from] anyhow::Error),
|
||||
}
|
||||
|
||||
impl From<super::admission::AdmissionRejection> for InferenceError {
|
||||
fn from(rejection: super::admission::AdmissionRejection) -> Self {
|
||||
use super::admission::AdmissionRejection;
|
||||
match rejection {
|
||||
AdmissionRejection::QueueFull { retry_after_secs }
|
||||
| AdmissionRejection::Timeout { retry_after_secs } => {
|
||||
InferenceError::Overloaded { retry_after_secs }
|
||||
}
|
||||
AdmissionRejection::PrincipalCap { retry_after_secs } => {
|
||||
InferenceError::PerPrincipalLimit { retry_after_secs }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the model's prompt from a [`ChatCompletionRequest`].
|
||||
///
|
||||
/// Prefers the model's own `chat_template` when one was loaded
|
||||
@@ -6563,6 +6837,110 @@ mod tests {
|
||||
assert!(validate_vision_prefill(12_960, 12_445).is_ok());
|
||||
}
|
||||
|
||||
// ── #65: request-time length-aware VRAM backstop (text prefill) ──
|
||||
|
||||
/// A beast-like profile: 16 full-attn layers, 4 kv heads, head_dim
|
||||
/// 256, f16, TP=2 → 32 KiB/token/card (same numbers as the
|
||||
/// `context_limit` unit tests). At defaults this makes the
|
||||
/// length-aware footprint `(prompt_len + 8192)/32 + 2048 + 1500` MiB
|
||||
/// per card.
|
||||
fn backstop_profile() -> super::super::context_limit::ContextProfile {
|
||||
super::super::context_limit::ContextProfile {
|
||||
max_position_embeddings: 262_144,
|
||||
kv_bytes_per_token_per_card: super::super::context_limit::kv_bytes_per_token(
|
||||
16, 4, 256, 2, 2,
|
||||
),
|
||||
world_size: 2,
|
||||
}
|
||||
}
|
||||
|
||||
/// A prompt under the cap with ample free VRAM passes; the same
|
||||
/// prompt over the cap is `PromptTooLong` before any VRAM math.
|
||||
#[test]
|
||||
fn validate_request_cap_and_fit() {
|
||||
let cfg = crate::config::ContextLimitConfig::default();
|
||||
let profile = backstop_profile();
|
||||
// Under cap, 40 GB free → fits.
|
||||
assert!(validate_request(8_000, 40_000, 100_000, Some(&profile), &cfg).is_ok());
|
||||
// Over the cap → PromptTooLong, independent of VRAM.
|
||||
assert!(matches!(
|
||||
validate_request(100_001, 40_000, 100_000, Some(&profile), &cfg),
|
||||
Err(InferenceError::PromptTooLong { .. })
|
||||
));
|
||||
}
|
||||
|
||||
/// The CPU sentinel (`vram_free_mb == 0`) skips every VRAM check,
|
||||
/// including the new length-aware one — host RAM is the OOM killer's
|
||||
/// problem, not this guard's.
|
||||
#[test]
|
||||
fn validate_request_cpu_sentinel_skips_vram() {
|
||||
let cfg = crate::config::ContextLimitConfig::default();
|
||||
let profile = backstop_profile();
|
||||
assert!(validate_request(1_000_000, 0, 2_000_000, Some(&profile), &cfg).is_ok());
|
||||
}
|
||||
|
||||
/// The static floor remains a backstop: free VRAM below
|
||||
/// `min_free_vram_mb()` is rejected before the length-aware estimate
|
||||
/// even runs (so `required_mb` is the floor, not the KV footprint).
|
||||
#[test]
|
||||
fn validate_request_static_floor_still_binds() {
|
||||
let cfg = crate::config::ContextLimitConfig::default();
|
||||
let profile = backstop_profile();
|
||||
assert!(matches!(
|
||||
validate_request(10, 800, 100_000, Some(&profile), &cfg),
|
||||
Err(InferenceError::InsufficientVram {
|
||||
free_mb: 800,
|
||||
required_mb: 1500
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
/// A model with no captured profile (non-qwen3_5 arch) has no
|
||||
/// length-aware physics to apply, so it rides only the static floor —
|
||||
/// a fitting prompt with VRAM above the floor passes.
|
||||
#[test]
|
||||
fn validate_request_no_profile_rides_floor() {
|
||||
let cfg = crate::config::ContextLimitConfig::default();
|
||||
assert!(validate_request(500_000, 5_000, 1_000_000, None, &cfg).is_ok());
|
||||
}
|
||||
|
||||
/// The acceptance test (#65): a cap derived against *ample* free VRAM
|
||||
/// is later applied at request time against *tightened* free VRAM. A
|
||||
/// prompt sized exactly at the now-stale `effective_prompt_cap()`
|
||||
/// clears the cap and the static floor, yet no longer fits — the
|
||||
/// length-aware backstop catches it with a clean `InsufficientVram`
|
||||
/// instead of an OOM-poisoned context. Same prompt with the original
|
||||
/// ample VRAM still passes, proving the guard only bites on staleness.
|
||||
#[test]
|
||||
fn validate_request_catches_poll_vs_request_staleness() {
|
||||
let cfg = crate::config::ContextLimitConfig::default();
|
||||
let profile = backstop_profile();
|
||||
|
||||
// Cap derived at /models poll time with 40 GB free on the tightest
|
||||
// card — throughput binds, giving input = 87040 (the issue's
|
||||
// worked beast figure).
|
||||
let limit = super::super::context_limit::derive_limit(&profile, 40_000, 800.0, None, &cfg);
|
||||
let cap = limit.input.expect("input budget derived");
|
||||
assert_eq!(cap, 87_040);
|
||||
|
||||
// With that same ample VRAM, a prompt at the cap still fits.
|
||||
assert!(validate_request(cap, 40_000, cap, Some(&profile), &cfg).is_ok());
|
||||
|
||||
// Now free VRAM has dropped to 5 GB between the poll and the
|
||||
// request (a co-resident model loaded). The prompt is still ≤ cap
|
||||
// and clears the 1500 MiB floor, but its footprint —
|
||||
// (87040 + 8192)/32 + 2048 + 1500 = 6524 MiB — exceeds 5000 MiB.
|
||||
let err = validate_request(cap, 5_000, cap, Some(&profile), &cfg)
|
||||
.expect_err("stale cap must not let an over-VRAM prompt through");
|
||||
assert!(matches!(
|
||||
err,
|
||||
InferenceError::InsufficientVram {
|
||||
free_mb: 5_000,
|
||||
required_mb: 6_524
|
||||
}
|
||||
));
|
||||
}
|
||||
|
||||
// ── Tool-call body parsing ───────────────────────────────────────
|
||||
|
||||
fn weather_schemas() -> ToolSchemas {
|
||||
|
||||
@@ -100,9 +100,9 @@ pub const KV_CACHE_DTYPE_BYTES: usize = 2;
|
||||
/// state, not a growing cache). Sharded across the TP world: per-rank
|
||||
/// KV-head count is `n_kv_heads / world_size`.
|
||||
///
|
||||
/// `2 ×` accounts for K and V. Shared by the limit derivation here and
|
||||
/// the per-rank load-time logging in the TP paths (and, in future, by
|
||||
/// #65's length-aware pre-flight guard).
|
||||
/// `2 ×` accounts for K and V. Shared by the limit derivation here, the
|
||||
/// per-rank load-time logging in the TP paths, and #65's request-time
|
||||
/// length-aware pre-flight guard (`candle::validate_request`).
|
||||
pub fn kv_bytes_per_token(
|
||||
n_full_attn_layers: usize,
|
||||
n_kv_heads: usize,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
//! Harness registry — maps harness names to trait implementations.
|
||||
|
||||
pub mod admission;
|
||||
pub mod arch;
|
||||
pub mod candle;
|
||||
pub mod chat_template;
|
||||
|
||||
@@ -30,6 +30,9 @@ impl HealthCache {
|
||||
// direct read from the cache stays a well-typed
|
||||
// HealthResponse on the wire.
|
||||
activation: Default::default(),
|
||||
// Per-model admission load is overlaid by the api handler
|
||||
// from the candle harness (#53); the cache doesn't own it.
|
||||
models: Vec::new(),
|
||||
}),
|
||||
has_gpus: RwLock::new(false),
|
||||
}
|
||||
|
||||
@@ -114,6 +114,12 @@ async fn test_health_endpoint() {
|
||||
|
||||
let body: serde_json::Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["uptime_secs"], 0);
|
||||
// Per-model admission load (#53) is always present, even with no models
|
||||
// loaded (empty array) — cortex's load-aware router (#55) relies on it.
|
||||
assert!(
|
||||
body["models"].is_array(),
|
||||
"/health must expose a models load array"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
39
helexa-router.example.toml
Normal file
@@ -0,0 +1,39 @@
|
||||
# helexa-router.example.toml — example configuration
|
||||
#
|
||||
# Copy to helexa-router.toml and adjust for your environment.
|
||||
#
|
||||
# Environment variable overrides use the HELEXA_ROUTER_ prefix with __
|
||||
# separators:
|
||||
# HELEXA_ROUTER_ROUTER__LISTEN=0.0.0.0:8088
|
||||
|
||||
[router]
|
||||
# Plaintext listener. Operator/edge nginx terminates client TLS in front of
|
||||
# the router — the router never owns an inbound TLS listener.
|
||||
listen = "0.0.0.0:8088"
|
||||
|
||||
# How often (seconds) to refresh each cortex's health + /v1/models topology.
|
||||
# poll_interval_secs = 10
|
||||
|
||||
# -- Downstream cortexes -------------------------------------------------
|
||||
# Each [[cortexes]] entry is an operator-run cortex the router may dispatch
|
||||
# to. The router forwards the client's bearer verbatim (auth stays at
|
||||
# cortex) and routes on capacity (preferring matching `region`).
|
||||
#
|
||||
# Outbound TLS pinning (optional): set `tls_ca` to a PEM trust anchor that
|
||||
# enrols this cortex — the CA (or self-signed cert) its TLS cert must chain
|
||||
# to. The router then trusts ONLY that anchor for this cortex (platform
|
||||
# roots disabled), so the router->cortex hop (which carries the client's
|
||||
# bearer) reaches the cert you expect and a rogue endpoint presenting any
|
||||
# other cert is rejected at the handshake. A cortex whose `tls_ca` fails to
|
||||
# load is disabled (fail closed). Omit `tls_ca` for a publicly-trusted cert
|
||||
# or plaintext http:// on a private (e.g. WireGuard) network.
|
||||
|
||||
# [[cortexes]]
|
||||
# name = "lair-cafe"
|
||||
# endpoint = "https://cortex.lair.cafe"
|
||||
# region = "eu-west"
|
||||
# tls_ca = "/etc/helexa-router/pins/lair-cafe.pem"
|
||||
|
||||
# [[cortexes]]
|
||||
# name = "example-operator"
|
||||
# endpoint = "https://cortex.example.com"
|
||||
52
helexa-upstream.example.toml
Normal file
@@ -0,0 +1,52 @@
|
||||
# helexa-upstream.example.toml — mesh-level account/authorization authority
|
||||
#
|
||||
# Copy to helexa-upstream.toml and adjust. Env overrides use the UPSTREAM_
|
||||
# prefix with __ separators, e.g. UPSTREAM_DB__URL=postgres://...
|
||||
|
||||
[server]
|
||||
# Plaintext listener; edge nginx terminates TLS (consistent with the stack).
|
||||
listen = "0.0.0.0:8090"
|
||||
|
||||
[db]
|
||||
# PostgreSQL connection URL. Required.
|
||||
url = "postgres://helexa:helexa@localhost/helexa_upstream"
|
||||
# max_connections = 16
|
||||
|
||||
[grant]
|
||||
# Flat free token grant every email-verified account receives (the floor of
|
||||
# the hybrid allocation; single-use top-up codes extend it).
|
||||
# free_token_grant = 1000000
|
||||
|
||||
[abuse]
|
||||
# When this many accounts share one registration fingerprint, all are
|
||||
# silently deactivated (no notice to the user).
|
||||
# fingerprint_account_threshold = 5
|
||||
|
||||
# -- Client auth: credentials operators' cortexes present to /authz/v1.
|
||||
# Each token maps to an operator_id (served-usage attribution). When no
|
||||
# tokens are configured the authz surface is OPEN (dev only). Distinct from
|
||||
# end-user API keys, which ride inside the resolve request body.
|
||||
# [[client_auth.tokens]]
|
||||
# token = "replace-with-a-strong-shared-secret"
|
||||
# operator_id = "lair-cafe"
|
||||
|
||||
[authz]
|
||||
# Open reservations older than this are swept (released), self-healing a
|
||||
# reservation whose settle/release from a cortex was lost.
|
||||
# reservation_ttl_secs = 120
|
||||
# sweep_interval_secs = 60
|
||||
|
||||
[auth]
|
||||
# HMAC secret for signing web-session JWTs. MUST be overridden in prod via
|
||||
# UPSTREAM_AUTH__JWT_SECRET; the built-in default is dev-only.
|
||||
# jwt_secret = "change-me"
|
||||
# session_ttl_secs = 604800 # 7 days
|
||||
# email_token_ttl_secs = 86400 # 24 hours
|
||||
# Frontend base URL used to build verify/reset links in emails.
|
||||
app_base_url = "https://helexa.ai"
|
||||
|
||||
[email]
|
||||
# "log" (dev: logs the link) or "smtp".
|
||||
provider = "log"
|
||||
# smtp_url = "smtp://user:pass@smtp.example.com:587"
|
||||
from_addr = "helexa <no-reply@helexa.ai>"
|
||||
20
helexa.ai/.env.example
Normal file
@@ -0,0 +1,20 @@
|
||||
# helexa.ai frontend env. Copy to .env.local for local dev (gitignored).
|
||||
|
||||
# Mesh data-plane (helexa-router, OpenAI-compatible inference). In dev,
|
||||
# vite proxies /v1 and /health here.
|
||||
VITE_ROUTER_BASE_URL=http://localhost:8088
|
||||
|
||||
# Account control-plane (helexa-upstream). In dev, vite proxies /api here
|
||||
# (rewritten to /web/v1).
|
||||
VITE_ACCOUNT_BASE_URL=http://localhost:8090
|
||||
|
||||
# Public-beta banner.
|
||||
VITE_PUBLIC_BETA=true
|
||||
|
||||
# Models for the chat workspace (F3+).
|
||||
# VITE_ANON_MODEL=...
|
||||
# VITE_DEFAULT_MODEL=...
|
||||
|
||||
# Develop the account dashboard (F4) against an in-browser mock before the
|
||||
# upstream account API ships.
|
||||
# VITE_USE_MOCK_ACCOUNT_API=true
|
||||
6
helexa.ai/.gitignore
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
node_modules
|
||||
dist
|
||||
*.local
|
||||
.env.local
|
||||
.env.*.local
|
||||
*.tsbuildinfo
|
||||
34
helexa.ai/README.md
Normal file
@@ -0,0 +1,34 @@
|
||||
# helexa.ai
|
||||
|
||||
The public-beta frontend for the helexa mesh: a chat-first landing experience
|
||||
(anonymous + authenticated, with all chat history kept client-side in
|
||||
IndexedDB — no server-side history), a `/mission` page on European digital
|
||||
sovereignty, and full account self-service (register, recover, manage API
|
||||
keys, set per-key limits, redeem top-up codes) against `helexa-upstream`.
|
||||
|
||||
Vite + React (SWC) + TypeScript + react-bootstrap + react-router + react-i18next.
|
||||
Lives as a top-level folder in the cortex monorepo; it is **not** a Cargo crate.
|
||||
|
||||
## Develop
|
||||
|
||||
```sh
|
||||
cd helexa.ai
|
||||
npm install
|
||||
cp .env.example .env.local # adjust backend URLs
|
||||
npm run dev # vite dev server, proxies /v1+/health → router, /api → upstream
|
||||
```
|
||||
|
||||
Other scripts: `npm run build` (`tsc -b && vite build` → `dist/`), `npm run
|
||||
preview`, `npm run lint`, `npm run typecheck`.
|
||||
|
||||
In dev, `vite.config.ts` proxies the mesh data-plane (helexa-router) and the
|
||||
account control-plane (helexa-upstream) same-origin. Run a local router
|
||||
(`cargo run -p helexa-router`) for the chat path and a local helexa-upstream
|
||||
for the account path.
|
||||
|
||||
## Status
|
||||
|
||||
F0 scaffold. Theming + i18n (33 languages, usage-ordered selector), the
|
||||
`/mission` page, the chat workspace (Dexie + streaming), and the account
|
||||
dashboard land in subsequent phases — see
|
||||
`~/.claude/plans/we-need-to-plan-modular-graham.md`.
|
||||
23
helexa.ai/eslint.config.js
Normal file
@@ -0,0 +1,23 @@
|
||||
import js from "@eslint/js";
|
||||
import globals from "globals";
|
||||
import reactHooks from "eslint-plugin-react-hooks";
|
||||
import reactRefresh from "eslint-plugin-react-refresh";
|
||||
import tseslint from "typescript-eslint";
|
||||
import { defineConfig, globalIgnores } from "eslint/config";
|
||||
|
||||
export default defineConfig([
|
||||
globalIgnores(["dist"]),
|
||||
{
|
||||
files: ["**/*.{ts,tsx}"],
|
||||
extends: [
|
||||
js.configs.recommended,
|
||||
tseslint.configs.recommended,
|
||||
reactHooks.configs.flat.recommended,
|
||||
reactRefresh.configs.vite,
|
||||
],
|
||||
languageOptions: {
|
||||
ecmaVersion: 2020,
|
||||
globals: globals.browser,
|
||||
},
|
||||
},
|
||||
]);
|
||||
24
helexa.ai/index.html
Normal file
@@ -0,0 +1,24 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>helexa.ai</title>
|
||||
<meta name="title" content="helexa.ai" />
|
||||
<meta
|
||||
name="description"
|
||||
content="helexa — near-frontier AI on a sovereign, operator-run mesh. Chat now; bring your own key."
|
||||
/>
|
||||
<meta property="og:type" content="website" />
|
||||
<meta property="og:url" content="https://helexa.ai/" />
|
||||
<meta property="og:title" content="helexa.ai" />
|
||||
<meta
|
||||
property="og:description"
|
||||
content="helexa — near-frontier AI on a sovereign, operator-run mesh."
|
||||
/>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.tsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
4115
helexa.ai/package-lock.json
generated
Normal file
43
helexa.ai/package.json
Normal file
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"name": "helexa.ai",
|
||||
"private": true,
|
||||
"version": "0.0.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "tsc -b && vite build",
|
||||
"preview": "vite preview",
|
||||
"lint": "eslint .",
|
||||
"typecheck": "tsc -b",
|
||||
"i18n:check": "node ./scripts/check-i18n-keys.mjs",
|
||||
"i18n:meta": "node ./scripts/check-i18n-metadata.mjs",
|
||||
"i18n:lang-labels": "node ./scripts/check-i18n-lang-labels.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"@fingerprintjs/fingerprintjs": "^4.6.2",
|
||||
"bootstrap": "^5.3.8",
|
||||
"dexie": "^4.2.0",
|
||||
"dexie-react-hooks": "^4.2.0",
|
||||
"i18next": "^25.7.1",
|
||||
"react": "^19.2.0",
|
||||
"react-bootstrap": "^2.10.10",
|
||||
"react-dom": "^19.2.0",
|
||||
"react-i18next": "^16.4.0",
|
||||
"react-icons": "^5.5.0",
|
||||
"react-router-dom": "^7.10.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.39.1",
|
||||
"@types/node": "^24.10.1",
|
||||
"@types/react": "^19.2.5",
|
||||
"@types/react-dom": "^19.2.3",
|
||||
"@vitejs/plugin-react-swc": "^4.2.0",
|
||||
"eslint": "^9.39.1",
|
||||
"eslint-plugin-react-hooks": "^7.0.1",
|
||||
"eslint-plugin-react-refresh": "^0.4.24",
|
||||
"globals": "^16.5.0",
|
||||
"typescript": "~5.9.3",
|
||||
"typescript-eslint": "^8.46.4",
|
||||
"vite": "^7.2.0"
|
||||
}
|
||||
}
|
||||
BIN
helexa.ai/public/banner.png
Normal file
|
After Width: | Height: | Size: 304 KiB |
BIN
helexa.ai/public/bg-logo-right.png
Normal file
|
After Width: | Height: | Size: 1.2 MiB |
BIN
helexa.ai/public/logo.png
Normal file
|
After Width: | Height: | Size: 111 KiB |
BIN
helexa.ai/public/people-mesh-1200x630.png
Normal file
|
After Width: | Height: | Size: 1.2 MiB |
BIN
helexa.ai/public/people-mesh-1536x1024.png
Normal file
|
After Width: | Height: | Size: 2.7 MiB |
BIN
helexa.ai/public/person-helix-1200x630.png
Normal file
|
After Width: | Height: | Size: 1.3 MiB |
BIN
helexa.ai/public/person-helix-1536x1024.png
Normal file
|
After Width: | Height: | Size: 2.2 MiB |
332
helexa.ai/scripts/check-i18n-keys.mjs
Normal file
@@ -0,0 +1,332 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Simple i18n key consistency checker.
|
||||
*
|
||||
* Compares translation JSONs for all configured languages against the English
|
||||
* baseline, for each namespace (common, home, chat).
|
||||
*
|
||||
* Exit codes:
|
||||
* - 0: all good
|
||||
* - 1: inconsistencies found or unexpected error
|
||||
*/
|
||||
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import url from "url";
|
||||
|
||||
// Adjust these if your i18n structure changes.
|
||||
const ROOT = path.resolve(
|
||||
path.dirname(url.fileURLToPath(import.meta.url)),
|
||||
"..",
|
||||
);
|
||||
const RESOURCES_DIR = path.join(ROOT, "src", "i18n", "resources");
|
||||
|
||||
// Namespaces to validate.
|
||||
const NAMESPACES = ["common", "mission", "chat", "account"];
|
||||
|
||||
// Languages to validate should track SUPPORTED_LANGUAGES in src/i18n/languages.ts.
|
||||
// NOTE: This list is intentionally narrower than SUPPORTED_LANGUAGES and does not
|
||||
// enforce that every supported language has wired resources. That enforcement is
|
||||
// implemented further below by checking the i18n index.
|
||||
const LANGUAGES = [
|
||||
"bg",
|
||||
"de",
|
||||
"el",
|
||||
"en",
|
||||
"es",
|
||||
"et",
|
||||
"fr",
|
||||
"it",
|
||||
"pt",
|
||||
"ro",
|
||||
"ru",
|
||||
];
|
||||
|
||||
function readJson(filePath) {
|
||||
try {
|
||||
const raw = fs.readFileSync(filePath, "utf8");
|
||||
return JSON.parse(raw);
|
||||
} catch (err) {
|
||||
throw new Error(`Failed to read/parse JSON at ${filePath}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively walk an object and collect all key paths using dot-notation.
|
||||
* Arrays are traversed structurally but their indexes are not part of the key path
|
||||
* (we only care that the shape exists, not array lengths).
|
||||
*/
|
||||
function collectKeyPaths(obj, prefix = "") {
|
||||
const keys = new Set();
|
||||
|
||||
if (obj === null || obj === undefined) {
|
||||
return keys;
|
||||
}
|
||||
|
||||
if (typeof obj !== "object") {
|
||||
if (prefix) keys.add(prefix);
|
||||
return keys;
|
||||
}
|
||||
|
||||
// If this is an array, walk its elements but don't add indices to the path.
|
||||
if (Array.isArray(obj)) {
|
||||
obj.forEach((item) => {
|
||||
for (const childKey of collectKeyPaths(item, prefix)) {
|
||||
keys.add(childKey);
|
||||
}
|
||||
});
|
||||
return keys;
|
||||
}
|
||||
|
||||
// Plain object
|
||||
for (const [k, v] of Object.entries(obj)) {
|
||||
const nextPrefix = prefix ? `${prefix}.${k}` : k;
|
||||
if (v !== null && typeof v === "object") {
|
||||
for (const childKey of collectKeyPaths(v, nextPrefix)) {
|
||||
keys.add(childKey);
|
||||
}
|
||||
} else {
|
||||
keys.add(nextPrefix);
|
||||
}
|
||||
}
|
||||
|
||||
return keys;
|
||||
}
|
||||
|
||||
function diffKeys(baseSet, targetSet) {
|
||||
const missing = [];
|
||||
const extra = [];
|
||||
|
||||
for (const k of baseSet) {
|
||||
if (!targetSet.has(k)) missing.push(k);
|
||||
}
|
||||
|
||||
for (const k of targetSet) {
|
||||
if (!baseSet.has(k)) extra.push(k);
|
||||
}
|
||||
|
||||
missing.sort();
|
||||
extra.sort();
|
||||
|
||||
return { missing, extra };
|
||||
}
|
||||
|
||||
function logHeader(title) {
|
||||
// Simple console formatting without external deps.
|
||||
console.log("\n" + "=".repeat(title.length));
|
||||
console.log(title);
|
||||
console.log("=".repeat(title.length));
|
||||
}
|
||||
|
||||
function main() {
|
||||
let hadIssues = false;
|
||||
|
||||
console.log("helexa.ai i18n key consistency check");
|
||||
console.log(`Root: ${ROOT}`);
|
||||
console.log(`Resources dir: ${RESOURCES_DIR}`);
|
||||
console.log(`Languages: ${LANGUAGES.join(", ")}`);
|
||||
console.log(`Namespaces: ${NAMESPACES.join(", ")}`);
|
||||
|
||||
// --- Wiring check: ensure each SUPPORTED_LANGUAGES entry has resources registered ---
|
||||
//
|
||||
// This prevents cases where a language is:
|
||||
// - present in LanguageCode and SUPPORTED_LANGUAGES
|
||||
// - has translation JSONs under src/i18n/resources/<code>/
|
||||
// but is *not* wired into the i18n `resources` object (and thus silently
|
||||
// falls back to English at runtime).
|
||||
try {
|
||||
const languagesTs = fs.readFileSync(
|
||||
path.join(ROOT, "src", "i18n", "languages.ts"),
|
||||
"utf8",
|
||||
);
|
||||
const i18nIndexTs = fs.readFileSync(
|
||||
path.join(ROOT, "src", "i18n", "index.ts"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
// Parse SUPPORTED_LANGUAGES from languages.ts
|
||||
const marker = "export const SUPPORTED_LANGUAGES";
|
||||
const start = languagesTs.indexOf(marker);
|
||||
if (start === -1) {
|
||||
throw new Error("Could not find `SUPPORTED_LANGUAGES` in languages.ts");
|
||||
}
|
||||
const after = languagesTs.slice(start);
|
||||
const bracketIndex = after.indexOf("[");
|
||||
const closingIndex = after.indexOf("];");
|
||||
if (bracketIndex === -1 || closingIndex === -1) {
|
||||
throw new Error("Malformed SUPPORTED_LANGUAGES array");
|
||||
}
|
||||
const arraySlice = after.slice(bracketIndex + 1, closingIndex);
|
||||
const supportedFromTs = new Set();
|
||||
const codeRegex = /"([^"]+)"/g;
|
||||
let m;
|
||||
while ((m = codeRegex.exec(arraySlice)) !== null) {
|
||||
supportedFromTs.add(m[1]);
|
||||
}
|
||||
|
||||
console.log(
|
||||
`SUPPORTED_LANGUAGES from TS for wiring check: ${Array.from(
|
||||
supportedFromTs,
|
||||
)
|
||||
.sort()
|
||||
.join(", ")}`,
|
||||
);
|
||||
|
||||
// Languages that are intentionally "future" and may not yet have
|
||||
// resources wired in. Keep them out of the hard failure path so
|
||||
// CI does not break while their translations are still pending.
|
||||
const FUTURE_LANGUAGES = new Set(["ig", "om", "so", "ti", "wo"]);
|
||||
|
||||
// Parse i18n resources object keys from index.ts.
|
||||
// We look for a block like:
|
||||
// const resources: Resource = {
|
||||
// en: { ... },
|
||||
// fr: { ... },
|
||||
// };
|
||||
const resourcesMarker = "const resources: Resource = {";
|
||||
const resStart = i18nIndexTs.indexOf(resourcesMarker);
|
||||
if (resStart === -1) {
|
||||
throw new Error("Could not find `const resources: Resource` in index.ts");
|
||||
}
|
||||
const resAfter = i18nIndexTs.slice(resStart + resourcesMarker.length);
|
||||
const resEndIndex = resAfter.indexOf("};");
|
||||
if (resEndIndex === -1) {
|
||||
throw new Error("Malformed resources object in index.ts");
|
||||
}
|
||||
const resourcesBlock = resAfter.slice(0, resEndIndex);
|
||||
|
||||
// Extract top-level language keys: lines starting with two spaces then <code>:
|
||||
// Example: " en: {" or " fr: {"
|
||||
const wiredLangs = new Set();
|
||||
const lineRegex = /^\s*([a-z]{2}):\s*{\s*$/gm;
|
||||
let lm;
|
||||
while ((lm = lineRegex.exec(resourcesBlock)) !== null) {
|
||||
wiredLangs.add(lm[1]);
|
||||
}
|
||||
|
||||
console.log(
|
||||
`Languages wired in i18n resources: ${Array.from(wiredLangs)
|
||||
.sort()
|
||||
.join(", ")}`,
|
||||
);
|
||||
|
||||
// Now compare SUPPORTED_LANGUAGES vs wired resources.
|
||||
const missingWiring = [];
|
||||
for (const code of supportedFromTs) {
|
||||
if (FUTURE_LANGUAGES.has(code)) {
|
||||
// These are explicitly allowed to be missing until their
|
||||
// translations and wiring land.
|
||||
continue;
|
||||
}
|
||||
if (!wiredLangs.has(code)) {
|
||||
missingWiring.push(code);
|
||||
}
|
||||
}
|
||||
|
||||
if (missingWiring.length > 0) {
|
||||
hadIssues = true;
|
||||
console.error(
|
||||
"\nERROR: Some SUPPORTED_LANGUAGES codes are not wired into the i18n `resources` object in src/i18n/index.ts:",
|
||||
);
|
||||
for (const code of missingWiring.sort()) {
|
||||
console.error(` - ${code}`);
|
||||
}
|
||||
console.error(
|
||||
"These languages will silently fall back to English at runtime. Ensure that:",
|
||||
);
|
||||
console.error(
|
||||
" 1) ./resources/<code>/{common,home,chat}.json exist, and",
|
||||
);
|
||||
console.error(
|
||||
" 2) They are imported and registered in the `resources` object.",
|
||||
);
|
||||
console.error("");
|
||||
} else {
|
||||
console.log(
|
||||
"OK: Every SUPPORTED_LANGUAGES entry (excluding future languages) is wired into the i18n resources object.",
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
hadIssues = true;
|
||||
console.error(
|
||||
"ERROR: Failed while checking SUPPORTED_LANGUAGES wiring in i18n index:",
|
||||
err.message,
|
||||
);
|
||||
}
|
||||
|
||||
for (const ns of NAMESPACES) {
|
||||
logHeader(`Namespace: ${ns}`);
|
||||
|
||||
const basePath = path.join(RESOURCES_DIR, "en", `${ns}.json`);
|
||||
let baseJson;
|
||||
try {
|
||||
baseJson = readJson(basePath);
|
||||
} catch (err) {
|
||||
console.error(err.message);
|
||||
hadIssues = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
const baseKeys = collectKeyPaths(baseJson);
|
||||
console.log(`Baseline (en) keys: ${baseKeys.size}`);
|
||||
|
||||
for (const lang of LANGUAGES) {
|
||||
if (lang === "en") continue;
|
||||
|
||||
const langPath = path.join(RESOURCES_DIR, lang, `${ns}.json`);
|
||||
if (!fs.existsSync(langPath)) {
|
||||
console.error(` [${lang}] MISSING file: ${langPath}`);
|
||||
hadIssues = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
let langJson;
|
||||
try {
|
||||
langJson = readJson(langPath);
|
||||
} catch (err) {
|
||||
console.error(` [${lang}] ${err.message}`);
|
||||
hadIssues = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
const langKeys = collectKeyPaths(langJson);
|
||||
const { missing, extra } = diffKeys(baseKeys, langKeys);
|
||||
|
||||
if (missing.length === 0 && extra.length === 0) {
|
||||
console.log(` [${lang}] OK (keys: ${langKeys.size})`);
|
||||
} else {
|
||||
hadIssues = true;
|
||||
console.log(` [${lang}] Issues found:`);
|
||||
if (missing.length > 0) {
|
||||
console.log(" Missing keys (present in en, absent here):");
|
||||
for (const k of missing) {
|
||||
console.log(` - ${k}`);
|
||||
}
|
||||
}
|
||||
if (extra.length > 0) {
|
||||
console.log(" Extra keys (present here, absent in en):");
|
||||
for (const k of extra) {
|
||||
console.log(` + ${k}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log("");
|
||||
if (hadIssues) {
|
||||
console.error("i18n check completed with inconsistencies.");
|
||||
process.exitCode = 1;
|
||||
} else {
|
||||
console.log("i18n check completed successfully. All keys are consistent.");
|
||||
process.exitCode = 0;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
main();
|
||||
} catch (err) {
|
||||
console.error("Unexpected error while running i18n check:", err);
|
||||
process.exitCode = 1;
|
||||
}
|
||||