rob thijssen
3cccc2c56b
Stage 1 of the candle-native pivot. Replaces the external-process harness model (mistralrs over HTTP, llamacpp placeholder) with an in-process Harness trait whose sole implementation is candle. The trait keeps its shape so future engines slot in additively, but start/stop default to no-ops and HarnessConfig drops endpoint and systemd_unit since no harness needs external supervision. Behaviour is unchanged on the wire: load_model returns a "not implemented yet (Stage 2)" error and list_models is empty. The gateway-side proxy, poller, and router are untouched. CLAUDE.md Phase 11 (llama.cpp) and Phase 12 (mistral.rs COPR) are marked superseded; the staged plan lives in ~/.claude/plans/create-a-more-aggressive-calm-naur.md. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
46 lines
1.2 KiB
TOML
46 lines
1.2 KiB
TOML
# cortex.example.toml — example configuration
|
|
#
|
|
# Copy to cortex.toml and adjust for your environment.
|
|
#
|
|
# Environment variable overrides use CORTEX_ prefix with __ separators:
|
|
# CORTEX_GATEWAY__LISTEN=0.0.0.0:31313
|
|
|
|
[gateway]
|
|
listen = "0.0.0.0:31313"
|
|
metrics_listen = "0.0.0.0:31314"
|
|
|
|
[eviction]
|
|
strategy = "lru"
|
|
# Restart neurons after this many load/unload cycles to defragment VRAM.
|
|
# Set to 0 to disable.
|
|
defrag_after_cycles = 50
|
|
|
|
# -- Nodes ---------------------------------------------------------------
|
|
# Each [[nodes]] entry declares a neuron daemon in the fleet.
|
|
# Models are discovered by polling the neuron's /models endpoint.
|
|
# Pinned models (see models.toml) are never evicted.
|
|
|
|
[[nodes]]
|
|
name = "gpu-large"
|
|
endpoint = "http://gpu-large.internal:8080"
|
|
vram_mb = 49152 # e.g. 2x RTX 4090 (48 GB combined)
|
|
pinned = [
|
|
"your-org/large-model",
|
|
]
|
|
|
|
[[nodes]]
|
|
name = "gpu-medium"
|
|
endpoint = "http://gpu-medium.internal:8080"
|
|
vram_mb = 24576 # e.g. RTX 4090 (24 GB)
|
|
pinned = [
|
|
"your-org/medium-model",
|
|
]
|
|
|
|
[[nodes]]
|
|
name = "gpu-small"
|
|
endpoint = "http://gpu-small.internal:8080"
|
|
vram_mb = 12288 # e.g. RTX 3060 (12 GB)
|
|
pinned = [
|
|
"your-org/embedding-model",
|
|
]
|