rob thijssen
bc74e0e95f
Some checks failed
CI / Format (push) Successful in 38s
CI / CUDA type-check (push) Successful in 1m39s
CI / Clippy (push) Successful in 2m26s
CI / Test (push) Successful in 4m49s
CI / Build cortex SRPM (push) Has been skipped
CI / Build neuron SRPM (push) Has been skipped
CI / Publish cortex to COPR (push) Has been skipped
CI / Publish neuron to COPR (push) Has been skipped
CI / Bump version in source (push) Has been skipped
build-prerelease / Package helexa-bench RPM (push) Blocked by required conditions
build-prerelease / Resolve version stamps + change detection (push) Successful in 32s
build-prerelease / Build neuron-blackwell (push) Successful in 1m40s
build-prerelease / Build neuron-ada (push) Successful in 2m19s
build-prerelease / Build neuron-ampere (push) Successful in 2m22s
build-prerelease / Lint (fmt + clippy) (push) Successful in 2m49s
build-prerelease / Build cortex binary (push) Successful in 3m0s
build-prerelease / Test (push) Successful in 4m25s
build-prerelease / Package cortex RPM (push) Successful in 1m32s
build-prerelease / Package helexa-neuron-ada RPM (push) Successful in 1m50s
build-prerelease / Package helexa-neuron-ampere RPM (push) Successful in 1m49s
build-prerelease / Package helexa-neuron-blackwell RPM (push) Successful in 1m54s
build-prerelease / Build helexa-bench binary (push) Successful in 2m12s
build-prerelease / Publish to rpm.lair.cafe (unstable) (push) Has been cancelled
Stage 1's build seam (#50): the interface auth, metering, and budget enforcement all hang off, with a local/static provider so the A0 amplification fix can land before any upstream clearing house exists. The future helexa-upstream client (#57) is just another impl. - cortex-core::entitlements: Principal {account_id, key_id}, CapWindow (Balance | Rolling{seconds}), Reservation handle, BudgetSnapshot, AuthError/BudgetError, and the async EntitlementProvider trait (resolve / reserve / settle / release / snapshot). BudgetError carries the window semantics so callers pick the #63 code (rate_limit_exceeded + Retry-After vs insufficient_quota) without the provider touching HTTP. - cortex-core::config: [entitlements] section on GatewayConfig (require_auth + [[entitlements.keys]] with account_id, optional key_id, hard_cap, window). Additive + serde(default) — anonymous/uncapped when omitted, so existing setups are unaffected. - cortex-gateway::entitlements_local: LocalEntitlementProvider. Budget math serialized under one Mutex so spent+reserved can never exceed a hard cap under concurrency (the #52 guarantee); rolling windows reset lazily; uncapped keys (no hard_cap) always reserve but still meter. - CortexState gains Arc<dyn EntitlementProvider> + require_auth, built in from_config. Not yet consumed by the request path — auth middleware is 1b (#49), enforcement is 1d (#52). - cortex.example.toml documents the section; test GatewayConfig literals updated for the new field. 6 provider unit tests (resolve, unknown-key, round-trip, balance/rolling over-cap codes, uncapped infra key). Local fmt/clippy/test all green. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
93 lines
3.1 KiB
TOML
93 lines
3.1 KiB
TOML
# cortex.example.toml — example configuration
|
|
#
|
|
# Copy to cortex.toml and adjust for your environment.
|
|
#
|
|
# Environment variable overrides use CORTEX_ prefix with __ separators:
|
|
# CORTEX_GATEWAY__LISTEN=0.0.0.0:31313
|
|
|
|
# Path to the model catalogue (limits, cost, pinning, aliases, feasibility).
|
|
# Defaults to the packaged location below; uncomment to override for a
|
|
# non-packaged / local run.
|
|
# models_config = "/etc/cortex/models.toml"
|
|
|
|
[gateway]
|
|
listen = "0.0.0.0:31313"
|
|
metrics_listen = "0.0.0.0:31314"
|
|
|
|
[eviction]
|
|
strategy = "lru"
|
|
# Restart neurons after this many load/unload cycles to defragment VRAM.
|
|
# Set to 0 to disable.
|
|
defrag_after_cycles = 50
|
|
|
|
# -- Nodes ---------------------------------------------------------------
|
|
# Each [[nodes]] entry declares a neuron daemon in the fleet.
|
|
# Models are discovered by polling the neuron's /models endpoint.
|
|
# Pinned models (see models.toml) are never evicted.
|
|
|
|
[[nodes]]
|
|
name = "gpu-large"
|
|
endpoint = "http://gpu-large.internal:8080"
|
|
vram_mb = 49152 # e.g. 2x RTX 4090 (48 GB combined)
|
|
pinned = [
|
|
"your-org/large-model",
|
|
]
|
|
|
|
[[nodes]]
|
|
name = "gpu-medium"
|
|
endpoint = "http://gpu-medium.internal:8080"
|
|
vram_mb = 24576 # e.g. RTX 4090 (24 GB)
|
|
pinned = [
|
|
"your-org/medium-model",
|
|
]
|
|
|
|
[[nodes]]
|
|
name = "gpu-small"
|
|
endpoint = "http://gpu-small.internal:8080"
|
|
vram_mb = 12288 # e.g. RTX 3060 (12 GB)
|
|
pinned = [
|
|
"your-org/embedding-model",
|
|
]
|
|
|
|
# -- Entitlements (multi-tenant governance, #47) -------------------------
|
|
# Identity + per-key token budgets. Omit this section entirely for the
|
|
# legacy single-operator behaviour: requests are anonymous and uncapped.
|
|
#
|
|
# The local/static provider below is the source of truth for accounts,
|
|
# keys, and hard caps until the upstream clearing house exists. Identity
|
|
# rides standard bearer auth only — clients send
|
|
# Authorization: Bearer <key>
|
|
# no custom headers or body fields.
|
|
|
|
[entitlements]
|
|
# Reject unauthenticated requests with 401 invalid_api_key. Leave false
|
|
# (allow-anonymous) during rollout; flip to true once keys are issued.
|
|
require_auth = false
|
|
|
|
# One entry per API key.
|
|
[[entitlements.keys]]
|
|
key = "sk-example-rolling" # the bearer token the client sends
|
|
account_id = "team-research" # billable account (keys may share one)
|
|
key_id = "research-ci" # stable label for ledger/metrics (optional)
|
|
hard_cap = 5_000_000 # hard token cap over the window
|
|
# Rolling window that resets — over-cap requests get 429 rate_limit_exceeded
|
|
# + Retry-After, so well-behaved clients (opencode/AI SDK) back off and retry.
|
|
window = { kind = "rolling", seconds = 3600 }
|
|
|
|
[[entitlements.keys]]
|
|
key = "sk-example-balance"
|
|
account_id = "team-research"
|
|
key_id = "research-prepaid"
|
|
hard_cap = 20_000_000
|
|
# Hard balance, no reset — exhaustion returns 429 insufficient_quota
|
|
# (the client surfaces and stops). This is the default when `window` is
|
|
# omitted. Never 402.
|
|
window = { kind = "balance" }
|
|
|
|
[[entitlements.keys]]
|
|
key = "sk-example-infra"
|
|
account_id = "operator"
|
|
key_id = "infra"
|
|
# No hard_cap → uncapped operator infra key (own fleet, own use). Still
|
|
# metered for visibility.
|