rob thijssen
3f94c50817
Previous defaults collided with well-trodden infra services and with the Linux ephemeral port range: - cortex API 8000 — common dev-server default (Django, minio UI) - cortex metrics 9100 — Prometheus node_exporter default - neuron API 9090 — Cockpit default on Fedora, Prometheus self Move to helexa-themed palindromic ports, all below Linux's 32768-60999 ephemeral range and not registered to any well-known service: - cortex API 31313 - cortex metrics 31314 - neuron API 13131 Updated places: - cortex.example.toml, neuron.example.toml defaults - default impls in cortex-core and neuron config - cortex-cli --endpoint default for the status subcommand - doc comments citing example URLs - README.md and CLAUDE.md snippets Consumers already on the old ports need a one-line edit in their /etc/cortex/cortex.toml or /etc/neuron/neuron.toml to match; firewall rules and prometheus scrape configs will also need updating. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
46 lines
1.2 KiB
TOML
46 lines
1.2 KiB
TOML
# cortex.example.toml — example configuration
|
|
#
|
|
# Copy to cortex.toml and adjust for your environment.
|
|
#
|
|
# Environment variable overrides use CORTEX_ prefix with __ separators:
|
|
# CORTEX_GATEWAY__LISTEN=0.0.0.0:31313
|
|
|
|
[gateway]
|
|
listen = "0.0.0.0:31313"
|
|
metrics_listen = "0.0.0.0:31314"
|
|
|
|
[eviction]
|
|
strategy = "lru"
|
|
# Restart mistralrs after this many load/unload cycles to defragment VRAM.
|
|
# Set to 0 to disable.
|
|
defrag_after_cycles = 50
|
|
|
|
# -- Nodes ---------------------------------------------------------------
|
|
# Each [[nodes]] entry declares a mistral.rs instance in the fleet.
|
|
# Models are discovered by polling the node's /v1/models endpoint.
|
|
# Pinned models are never evicted.
|
|
|
|
[[nodes]]
|
|
name = "gpu-large"
|
|
endpoint = "http://gpu-large.internal:8080"
|
|
vram_mb = 49152 # e.g. 2x RTX 4090 (48 GB combined)
|
|
pinned = [
|
|
"your-org/large-model",
|
|
]
|
|
|
|
[[nodes]]
|
|
name = "gpu-medium"
|
|
endpoint = "http://gpu-medium.internal:8080"
|
|
vram_mb = 24576 # e.g. RTX 4090 (24 GB)
|
|
pinned = [
|
|
"your-org/medium-model",
|
|
]
|
|
|
|
[[nodes]]
|
|
name = "gpu-small"
|
|
endpoint = "http://gpu-small.internal:8080"
|
|
vram_mb = 12288 # e.g. RTX 3060 (12 GB)
|
|
pinned = [
|
|
"your-org/embedding-model",
|
|
]
|