rob thijssen
f4117224fc
All checks were successful
CI / Format (push) Successful in 35s
CI / CUDA type-check (push) Successful in 1m37s
CI / Clippy (push) Successful in 2m53s
CI / Test (push) Successful in 6m41s
CI / Build cortex SRPM (push) Has been skipped
CI / Build neuron SRPM (push) Has been skipped
CI / Publish cortex to COPR (push) Has been skipped
CI / Publish neuron to COPR (push) Has been skipped
CI / Bump version in source (push) Has been skipped
Operators are now metered for the tokens they serve on behalf of mesh accounts, and the upstream rolls that up for compensation. cortex-gateway: - served_usage.rs: an in-process per-(account,key,UTC-day) served-token counter, incremented in metering::usage_sink alongside spend/settle for every authenticated request. A flush task (spawned in run() when [upstream].enabled, mirroring poller/evictor) POSTs ABSOLUTE cumulative counters to upstream on [upstream].served_usage_report_interval_secs. - The no-limit infra key is the operator's local key with hard_cap=None (already supported) — it's metered for served-usage but never budget- refused and never hits upstream. helexa-upstream: - POST /authz/v1/served-usage: upserts rows keyed by (operator_id from the client bearer, account, key, period) with GREATEST(existing, incoming) — monotonic + idempotent, so re-sends, races, and a restarted cortex's lower counter never regress the total. - reconcile.rs + `helexa-upstream reconcile` CLI: rolls up unreconciled served_usage per operator/period (SUM::bigint), stamps reconciled_at, prints the totals. Payout mechanism out of scope. Validated against a throwaway Postgres: monotonic upsert (100→250, a 50 re-send stays 250, same-value idempotent) and reconcile rollup + stamp-once; cortex counter unit test for per-principal accumulation. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01F6o3ddqmYNh9kzdwq6eowh
110 lines
3.9 KiB
TOML
110 lines
3.9 KiB
TOML
# cortex.example.toml — example configuration
|
|
#
|
|
# Copy to cortex.toml and adjust for your environment.
|
|
#
|
|
# Environment variable overrides use CORTEX_ prefix with __ separators:
|
|
# CORTEX_GATEWAY__LISTEN=0.0.0.0:31313
|
|
|
|
# Path to the model catalogue (limits, cost, pinning, aliases, feasibility).
|
|
# Defaults to the packaged location below; uncomment to override for a
|
|
# non-packaged / local run.
|
|
# models_config = "/etc/cortex/models.toml"
|
|
|
|
[gateway]
|
|
listen = "0.0.0.0:31313"
|
|
metrics_listen = "0.0.0.0:31314"
|
|
|
|
[eviction]
|
|
strategy = "lru"
|
|
# Restart neurons after this many load/unload cycles to defragment VRAM.
|
|
# Set to 0 to disable.
|
|
defrag_after_cycles = 50
|
|
|
|
# -- Nodes ---------------------------------------------------------------
|
|
# Each [[nodes]] entry declares a neuron daemon in the fleet.
|
|
# Models are discovered by polling the neuron's /models endpoint.
|
|
# Pinned models (see models.toml) are never evicted.
|
|
|
|
[[nodes]]
|
|
name = "gpu-large"
|
|
endpoint = "http://gpu-large.internal:8080"
|
|
vram_mb = 49152 # e.g. 2x RTX 4090 (48 GB combined)
|
|
pinned = [
|
|
"your-org/large-model",
|
|
]
|
|
|
|
[[nodes]]
|
|
name = "gpu-medium"
|
|
endpoint = "http://gpu-medium.internal:8080"
|
|
vram_mb = 24576 # e.g. RTX 4090 (24 GB)
|
|
pinned = [
|
|
"your-org/medium-model",
|
|
]
|
|
|
|
[[nodes]]
|
|
name = "gpu-small"
|
|
endpoint = "http://gpu-small.internal:8080"
|
|
vram_mb = 12288 # e.g. RTX 3060 (12 GB)
|
|
pinned = [
|
|
"your-org/embedding-model",
|
|
]
|
|
|
|
# -- Entitlements (multi-tenant governance, #47) -------------------------
|
|
# Identity + per-key token budgets. Omit this section entirely for the
|
|
# legacy single-operator behaviour: requests are anonymous and uncapped.
|
|
#
|
|
# The local/static provider below is the source of truth for accounts,
|
|
# keys, and hard caps until the upstream clearing house exists. Identity
|
|
# rides standard bearer auth only — clients send
|
|
# Authorization: Bearer <key>
|
|
# no custom headers or body fields.
|
|
|
|
[entitlements]
|
|
# Reject unauthenticated requests with 401 invalid_api_key. Leave false
|
|
# (allow-anonymous) during rollout; flip to true once keys are issued.
|
|
require_auth = false
|
|
|
|
# One entry per API key.
|
|
[[entitlements.keys]]
|
|
key = "sk-example-rolling" # the bearer token the client sends
|
|
account_id = "team-research" # billable account (keys may share one)
|
|
key_id = "research-ci" # stable label for ledger/metrics (optional)
|
|
hard_cap = 5_000_000 # hard token cap over the window
|
|
# Rolling window that resets — over-cap requests get 429 rate_limit_exceeded
|
|
# + Retry-After, so well-behaved clients (opencode/AI SDK) back off and retry.
|
|
window = { kind = "rolling", seconds = 3600 }
|
|
|
|
[[entitlements.keys]]
|
|
key = "sk-example-balance"
|
|
account_id = "team-research"
|
|
key_id = "research-prepaid"
|
|
hard_cap = 20_000_000
|
|
# Hard balance, no reset — exhaustion returns 429 insufficient_quota
|
|
# (the client surfaces and stops). This is the default when `window` is
|
|
# omitted. Never 402.
|
|
window = { kind = "balance" }
|
|
|
|
[[entitlements.keys]]
|
|
key = "sk-example-infra"
|
|
account_id = "operator"
|
|
key_id = "infra"
|
|
# No hard_cap → uncapped operator infra key (own fleet, own use). Still
|
|
# metered for visibility.
|
|
|
|
# -- Upstream (helexa mesh) entitlements client (#57) --------------------
|
|
# When enabled, a bearer key NOT found in [[entitlements.keys]] above is
|
|
# validated against the helexa-upstream authority (mesh accounts), and its
|
|
# budget is reserved/settled there. Operator-local keys (incl. the infra
|
|
# key) never leave this process. Fail-closed: if upstream is unreachable a
|
|
# request is refused (503 + Retry-After), never served un-authorized.
|
|
# Disabled by default — a standalone operator runs purely local.
|
|
[upstream]
|
|
enabled = false
|
|
# url = "https://upstream.helexa.ai"
|
|
# Shared client bearer this cortex presents (maps to an operator_id
|
|
# upstream). Override via CORTEX_UPSTREAM__BEARER in prod.
|
|
# bearer = "replace-with-operator-client-secret"
|
|
# timeout_secs = 5
|
|
# How often to flush served-usage counters to upstream for reconciliation (#58).
|
|
# served_usage_report_interval_secs = 60
|