rob thijssen
140f9124ee
All checks were successful
CI / Format (push) Successful in 43s
CI / CUDA type-check (push) Successful in 2m35s
CI / Clippy (push) Successful in 2m46s
CI / Test (push) Successful in 5m54s
CI / Build cortex SRPM (push) Has been skipped
CI / Publish cortex to COPR (push) Has been skipped
CI / Build neuron SRPM (push) Has been skipped
CI / Publish neuron to COPR (push) Has been skipped
CI / Bump version in source (push) Has been skipped
CI / Format (pull_request) Successful in 43s
CI / CUDA type-check (pull_request) Successful in 1m53s
CI / Clippy (pull_request) Successful in 2m57s
CI / Test (pull_request) Successful in 7m42s
CI / Build cortex SRPM (pull_request) Has been skipped
CI / Build neuron SRPM (pull_request) Has been skipped
CI / Publish cortex to COPR (pull_request) Has been skipped
CI / Publish neuron to COPR (pull_request) Has been skipped
CI / Bump version in source (pull_request) Has been skipped
The six perf scenarios measure speed/resources; this measures the axis they miss — reasoning/planning quality — so the frontier A/B (F3) can pick on capability, not just throughput. Per the chosen approach: store the artifact always, with schema for BOTH a manual score and a future LLM-judge; start manual. - scenario: CapabilityScenario (capability:<name>) runs a fixed prompt and captures the full output text (stream_and_measure gains a capture_text path); opt-in via config.capability_probes (empty default — long outputs, deliberate). - store: three additive columns (artifact, quality_score, scorer); capability_runs(unscored_only) worklist + set_score(id, score, scorer). Drill-down RunRow omits the large artifact column. - cli: `helexa-bench score --id <n> --score <x> [--scorer ...]` (manual); `report --capability` (per-model median score + per-run artifact snippets); GET /api/capability. LLM-judge deferred (schema ready). - example config documents an implementation-planning probe. Tests: artifact storage + scoring lifecycle, capability scenario built from config, capability markdown (median + snippet). Part of the Performance observability epic (#83), O7 — completes the milestone. Feeds the F3 frontier A/B decision gate (#94). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VrJ4i3pfLRSTM76o3ofnVq
81 lines
3.2 KiB
TOML
81 lines
3.2 KiB
TOML
# helexa-bench — continuous, version-aware fleet benchmark harness.
|
|
#
|
|
# Hits each neuron directly, exercises warm models, and records every run
|
|
# with full build/version provenance into SQLite. Once a neuron build has
|
|
# `samples_per_version` results for a (model, scenario), later sweeps skip
|
|
# it until a new build SHA ships — so a steady fleet costs only cheap
|
|
# version polls.
|
|
#
|
|
# Env overrides: BENCH_-prefixed, `__` for nesting
|
|
# (e.g. BENCH_BENCH__SAMPLES_PER_VERSION=10).
|
|
|
|
[bench]
|
|
# Pause between full sweeps of all targets (seconds).
|
|
sweep_interval_secs = 1800
|
|
# Target measured samples per (target, build SHA, model, scenario).
|
|
samples_per_version = 5
|
|
# Pause between successive measured iterations against one model.
|
|
iteration_pause_secs = 2
|
|
# Per-request timeout (seconds); generous for cold lazy-loads.
|
|
request_timeout_secs = 600
|
|
# SQLite system-of-record.
|
|
db_path = "/var/lib/helexa-bench/bench.sqlite"
|
|
|
|
[scenarios]
|
|
# One chat-latency scenario is generated per size (chat:128, chat:4096).
|
|
# For a context-length scaling curve (#88), add a ladder up to the model's
|
|
# limit, e.g. [128, 1024, 4096, 16384, 65536, 131072]; then
|
|
# `helexa-bench report --scaling` (or GET /api/scaling) shows prefill &
|
|
# decode tok/s vs context with the decode-flatness verdict. Larger contexts
|
|
# cost more per sample, so widen the ladder deliberately.
|
|
prompt_sizes = [128, 4096]
|
|
max_tokens = 256
|
|
# Concurrency / agentic-load scenarios (#89): one concurrency:<n> scenario
|
|
# per level, each firing N simultaneous streams to characterize the real
|
|
# a0/hermes/opencode fan-out. Empty by default — enable deliberately, since
|
|
# a burst puts genuine simultaneous load on the serving fleet.
|
|
# concurrency_levels = [2, 4, 8]
|
|
# concurrency_prompt_tokens = 512
|
|
|
|
# Capability probes (#91): each runs a fixed prompt and stores the full
|
|
# output for quality scoring — the reasoning/planning axis the speed
|
|
# scenarios miss. Opt-in (empty by default). After a sweep records them,
|
|
# `helexa-bench report --capability` lists run ids + artifacts; score with
|
|
# `helexa-bench score --id <n> --score <x>` (manual now; LLM-judge later).
|
|
# [[scenarios.capability_probes]]
|
|
# name = "rust-plan"
|
|
# max_tokens = 2048
|
|
# prompt = """
|
|
# Write an implementation plan for adding rate limiting to an Axum service.
|
|
# Honor existing conventions, call out trade-offs, and sequence the work.
|
|
# """
|
|
|
|
# Read-only JSON API (consumed by the bench UI + programmatic access),
|
|
# served alongside the sweep loop by `run` (or standalone via `serve`).
|
|
[api]
|
|
enabled = true
|
|
listen = "0.0.0.0:13132"
|
|
|
|
# One [[targets]] block per neuron on the fleet. `kind = "neuron"` (the
|
|
# default) gets build metadata via GET /version and warm-model discovery
|
|
# via GET /models.
|
|
[[targets]]
|
|
name = "beast"
|
|
endpoint = "http://beast.hanzalova.internal:13131"
|
|
|
|
[[targets]]
|
|
name = "benjy"
|
|
endpoint = "http://benjy.hanzalova.internal:13131"
|
|
|
|
[[targets]]
|
|
name = "quadbrat"
|
|
endpoint = "http://quadbrat.hanzalova.internal:13131"
|
|
|
|
# Future: compare against a non-neuron OpenAI-compatible engine. `kind =
|
|
# "openai"` skips neuron-only metadata; point `endpoint` at the /v1 base.
|
|
# [[targets]]
|
|
# name = "llamacpp-ref"
|
|
# kind = "openai"
|
|
# endpoint = "http://benjy.hanzalova.internal:8080/v1"
|
|
# label = "llama.cpp"
|