Files
helexa/helexa-bench.example.toml
T
grenadeandClaude Opus 4.8 c4f3a21c34
CI / Format (push) Successful in 9s
CI / Clippy (push) Successful in 2m20s
CI / Format (pull_request) Failing after 10m57s
CI / Test (push) Failing after 11m7s
CI / CUDA type-check (push) Successful in 14m2s
CI / Build cortex SRPM (push) Has been skipped
CI / CUDA type-check (pull_request) Successful in 14m12s
CI / Build neuron SRPM (push) Has been skipped
CI / Publish cortex to COPR (push) Has been skipped
CI / Publish neuron to COPR (push) Has been skipped
CI / Bump version in source (push) Has been skipped
CI / Clippy (pull_request) Successful in 2m25s
CI / Test (pull_request) Successful in 5m57s
CI / Build cortex SRPM (pull_request) Has been skipped
CI / Build neuron SRPM (pull_request) Has been skipped
CI / Publish cortex to COPR (pull_request) Has been skipped
CI / Publish neuron to COPR (pull_request) Has been skipped
CI / Bump version in source (pull_request) Has been skipped
feat(bench): concurrency-sweep knee report + /api/concurrency (#137 T3)
Turns the #89 concurrency-burst data into a capacity verdict. Adds
Store::concurrency() — per (target, model) at the latest build, the
throughput / p95-TTFT / queue-wait / reject-rate curve across burst
widths, plus the derived knee: the highest concurrency the model
sustains before it sheds load or its p95 TTFT tail stretches past 2× the
lightest-load baseline. This is the data-backed justification for a
max_in_flight ceiling.

Surfaced three ways:
- `helexa-bench report --concurrency` (md/json)
- GET /api/concurrency — the bench UI's live-capacity view
- example config enables the sweep [1,2,4,8,16] to bracket beast's
  max_in_flight=8 so the knee is visible

Tests: the knee lands on the last sustainable level; None when even the
lightest level sheds.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012GaW6F2j3zsQ6yABKPsXXe
2026-07-09 17:57:03 +03:00

84 lines
3.4 KiB
TOML

# helexa-bench — continuous, version-aware fleet benchmark harness.
#
# Hits each neuron directly, exercises warm models, and records every run
# with full build/version provenance into SQLite. Once a neuron build has
# `samples_per_version` results for a (model, scenario), later sweeps skip
# it until a new build SHA ships — so a steady fleet costs only cheap
# version polls.
#
# Env overrides: BENCH_-prefixed, `__` for nesting
# (e.g. BENCH_BENCH__SAMPLES_PER_VERSION=10).
[bench]
# Pause between full sweeps of all targets (seconds).
sweep_interval_secs = 1800
# Target measured samples per (target, build SHA, model, scenario).
samples_per_version = 5
# Pause between successive measured iterations against one model.
iteration_pause_secs = 2
# Per-request timeout (seconds); generous for cold lazy-loads.
request_timeout_secs = 600
# SQLite system-of-record.
db_path = "/var/lib/helexa-bench/bench.sqlite"
[scenarios]
# One chat-latency scenario is generated per size (chat:128, chat:4096).
# For a context-length scaling curve (#88), add a ladder up to the model's
# limit, e.g. [128, 1024, 4096, 16384, 65536, 131072]; then
# `helexa-bench report --scaling` (or GET /api/scaling) shows prefill &
# decode tok/s vs context with the decode-flatness verdict. Larger contexts
# cost more per sample, so widen the ladder deliberately.
prompt_sizes = [128, 4096]
max_tokens = 256
# Concurrency / agentic-load scenarios (#89): one concurrency:<n> scenario
# per level, each firing N simultaneous streams to characterize the real
# a0/hermes/opencode fan-out. Enable deliberately — a burst puts genuine
# simultaneous load on the serving fleet. Bracket the model's max_in_flight
# (beast serves 8) so `helexa-bench report --concurrency` (or GET
# /api/concurrency) can locate the sustainable-concurrency knee (#137): the
# highest N before the model sheds load or its p95 TTFT tail breaks.
concurrency_levels = [1, 2, 4, 8, 16]
concurrency_prompt_tokens = 512
# Capability probes (#91): each runs a fixed prompt and stores the full
# output for quality scoring — the reasoning/planning axis the speed
# scenarios miss. Opt-in (empty by default). After a sweep records them,
# `helexa-bench report --capability` lists run ids + artifacts; score with
# `helexa-bench score --id <n> --score <x>` (manual now; LLM-judge later).
# [[scenarios.capability_probes]]
# name = "rust-plan"
# max_tokens = 2048
# prompt = """
# Write an implementation plan for adding rate limiting to an Axum service.
# Honor existing conventions, call out trade-offs, and sequence the work.
# """
# Read-only JSON API (consumed by the bench UI + programmatic access),
# served alongside the sweep loop by `run` (or standalone via `serve`).
[api]
enabled = true
listen = "0.0.0.0:13132"
# One [[targets]] block per neuron on the fleet. `kind = "neuron"` (the
# default) gets build metadata via GET /version and warm-model discovery
# via GET /models.
[[targets]]
name = "beast"
endpoint = "http://beast.hanzalova.internal:13131"
[[targets]]
name = "benjy"
endpoint = "http://benjy.hanzalova.internal:13131"
[[targets]]
name = "quadbrat"
endpoint = "http://quadbrat.hanzalova.internal:13131"
# Future: compare against a non-neuron OpenAI-compatible engine. `kind =
# "openai"` skips neuron-only metadata; point `endpoint` at the /v1 base.
# [[targets]]
# name = "llamacpp-ref"
# kind = "openai"
# endpoint = "http://benjy.hanzalova.internal:8080/v1"
# label = "llama.cpp"