From 21c35ff2c304d6782ad1080fee26fe10a961e3fd Mon Sep 17 00:00:00 2001 From: rob thijssen Date: Sun, 2 Aug 2026 15:50:57 +0300 Subject: [PATCH] docs(prompt): record helexa#179 as settled; shift the oc risk to precedence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Passthrough is verified and pinned by regression tests that assert on what cortex forwarded upstream, so tireless can rely on it. The design no longer carries it as an open dependency. Two residual behaviours replace it, neither a passthrough defect: Multiple system messages are forwarded unmerged and in order, last one wins. OpenCode sends its own preamble alongside an agent's configured prompt, so the stage 5 question becomes "did implement.oc.md arrive last", not "did it arrive". That failure is silent -- an agent that reads as a generic coding assistant and ignores Out of scope -- and would present as a prompt-quality problem rather than a plumbing one, so stage 5 now asserts on the upstream request. helexa#223: /no_think is ignored on /v1/responses, where a thinking model can spend its whole output budget on reasoning and return "" with status incomplete. Added RunOutcome::OutputBudgetExhausted so that is retryable but blameless -- a mis-sized ceiling must not trip a circuit breaker on a lane with nothing wrong with it. Config pins the chat/completions surface. The qwen3_next gap is recorded as a gap, not a hole: the system slot is not arch-branched, so there is no family-specific path to fail. Fleet now offers Qwen3-Coder-Next, a better fit for executing a written spec, but cold and feasible only on beast where the pinned 27B lives. Recorded as an operator decision per generic.md §14 rather than taken here. Config also notes why tireless pins a model name and never a helexa/* capability alias. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DHhHtohxcdk1PL3tfnYJdH --- asset/config/config.toml.tmpl | 27 ++++-- crates/tireless-entities/src/run.rs | 79 +++++++++++++++++ dashboard/src/api/generated/RunOutcome.ts | 2 +- doc/plan/design.md | 100 +++++++++++++++++----- prompt/readme.md | 61 ++++++++++--- 5 files changed, 229 insertions(+), 40 deletions(-) diff --git a/asset/config/config.toml.tmpl b/asset/config/config.toml.tmpl index 548c977..95206fe 100644 --- a/asset/config/config.toml.tmpl +++ b/asset/config/config.toml.tmpl @@ -88,12 +88,29 @@ failure_threshold = 3 # Anthropic blocks. tireless refuses to start if `provider` or `base_url` looks # Anthropic-shaped. See doc/plan/design.md §3. provider = "lair-helexa" -model = "Qwen/Qwen3.6-27B" base_url = "http://hanzalova.internal:31313/v1" -# The implementation system prompt reaches the model through -# OpenCode -> cortex -> neuron, relying on helexa's faithful-passthrough -# guarantee (helexa/helexa#179, still open). If a run behaves as though it never -# saw its prompt, suspect the passthrough before the prompt. + +# Pin a model name, never a capability alias (`helexa/large`, `helexa/balanced`, +# `helexa/small`). An alias that starts resolving elsewhere would change how +# tireless implements plans between one job and the next, with no deploy and no +# signal -- the same reason the agent packages are pinned. +# +# Current fleet options for this lane: +# Qwen/Qwen3.6-27B warm, pinned on beast (= helexa/large today), +# system prompt live-verified (helexa#179) +# Qwen/Qwen3-Coder-Next coder-specialised, better suited to executing a +# written spec -- but cold, feasible only on +# beast, so adopting it displaces the pinned 27B. +# Operator decision; see doc/plan/design.md §2.4. +# Qwen/Qwen3-Next-80B-A3B-Thinking as above, and see the surface note below. +model = "Qwen/Qwen3.6-27B" + +# Prefer the chat/completions surface. On /v1/responses, `/no_think` is ignored +# and a small output budget can be spent entirely on the reasoning block, +# returning "" with status "incomplete" (helexa#223, open). Treat that as its own +# outcome -- it is a token-budget artifact, not a failed run, and must not +# consume a retry. +surface = "chat_completions" max_concurrent = 2 max_runs_per_window = 240 window_hours = 5 diff --git a/crates/tireless-entities/src/run.rs b/crates/tireless-entities/src/run.rs index 9015c26..defb95e 100644 --- a/crates/tireless-entities/src/run.rs +++ b/crates/tireless-entities/src/run.rs @@ -59,12 +59,91 @@ pub enum RunOutcome { RateLimited, /// Stopped because a tireless budget was exhausted. Retryable next window. BudgetExhausted, + /// The model spent its whole output budget on reasoning and returned nothing. + /// + /// Distinct from [`Self::Failed`] on purpose. On helexa's `/v1/responses` + /// surface `/no_think` is ignored, so a thinking model with a small + /// `max_output_tokens` returns `""` with `status: "incomplete"` + /// ([helexa#223](https://git.lair.cafe/helexa/helexa/issues/223)). That is a + /// budget artifact, not a broken run: it is retryable with a larger budget + /// and must not count toward the circuit breaker, or a mis-sized ceiling + /// would trip a lane that has nothing wrong with it. + OutputBudgetExhausted, /// Exceeded the wall-clock ceiling for its lane. TimedOut, /// Cancelled by an operator. Cancelled, } +impl RunOutcome { + /// Whether this outcome counts toward a lane's consecutive-failure tally. + /// + /// Only outcomes that indicate something actually wrong do. Being throttled, + /// running out of an allowance, or being cancelled are all expected + /// operating conditions, not evidence of a fault. + pub fn counts_as_failure(self) -> bool { + match self { + Self::Failed | Self::TimedOut => true, + Self::Succeeded + | Self::RateLimited + | Self::BudgetExhausted + | Self::OutputBudgetExhausted + | Self::Cancelled => false, + } + } + + /// Whether the job should be retried rather than marked terminal. + pub fn is_retryable(self) -> bool { + matches!( + self, + Self::RateLimited + | Self::BudgetExhausted + | Self::OutputBudgetExhausted + | Self::TimedOut + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn throttling_and_budgets_are_not_faults() { + for outcome in [ + RunOutcome::RateLimited, + RunOutcome::BudgetExhausted, + RunOutcome::OutputBudgetExhausted, + RunOutcome::Cancelled, + RunOutcome::Succeeded, + ] { + assert!( + !outcome.counts_as_failure(), + "{outcome:?} must not trip the circuit breaker" + ); + } + } + + #[test] + fn real_faults_count() { + assert!(RunOutcome::Failed.counts_as_failure()); + assert!(RunOutcome::TimedOut.counts_as_failure()); + } + + #[test] + fn an_empty_thinking_response_is_retryable_but_blameless() { + // The helexa#223 case: retry it with a bigger budget, do not blame the lane. + let o = RunOutcome::OutputBudgetExhausted; + assert!(o.is_retryable()); + assert!(!o.counts_as_failure()); + } + + #[test] + fn a_hard_failure_is_not_retried() { + assert!(!RunOutcome::Failed.is_retryable()); + } +} + /// One invocation of one agent against one job. /// /// A job may have several runs: a retry after a rate limit, or a follow-up turn diff --git a/dashboard/src/api/generated/RunOutcome.ts b/dashboard/src/api/generated/RunOutcome.ts index 7f39e8b..6ebf6fd 100644 --- a/dashboard/src/api/generated/RunOutcome.ts +++ b/dashboard/src/api/generated/RunOutcome.ts @@ -3,4 +3,4 @@ /** * How a run ended. */ -export type RunOutcome = "succeeded" | "failed" | "rate_limited" | "budget_exhausted" | "timed_out" | "cancelled"; +export type RunOutcome = "succeeded" | "failed" | "rate_limited" | "budget_exhausted" | "output_budget_exhausted" | "timed_out" | "cancelled"; diff --git a/doc/plan/design.md b/doc/plan/design.md index 765a597..2d7955f 100644 --- a/doc/plan/design.md +++ b/doc/plan/design.md @@ -140,19 +140,50 @@ comment and a `tireless/blocked` label, seconds after the planning run. The same plan unvalidated costs an OpenCode run, a branch, and an operator's review attention before anyone notices the spec was unusable. -**Dependency: helexa faithful passthrough.** The OpenCode system prompt reaches -the model via OpenCode → cortex → neuron, which relies on helexa's guarantee of -no injection, no rewriting, no defaults -([helexa/helexa#179](https://git.lair.cafe/helexa/helexa/issues/179)). +**Dependency: helexa faithful passthrough — settled.** The OpenCode system prompt +reaches the model via OpenCode → cortex → neuron, relying on helexa's guarantee +of no injection, no rewriting, no defaults +([helexa/helexa#179](https://git.lair.cafe/helexa/helexa/issues/179), **closed +2026-08-02**). -That issue is **open** — the principle was decided 2026-07-17, but verification -and documentation are outstanding, and two unchecked items bear directly here: -neuron chat templating applying the system role correctly per arch family -*including `/no_think` interaction* (Qwen3.6-27B is the target model), and -behaviour with multiple system messages. Stage 5 is effectively the second -consumer of that guarantee after the chat SPA, and is well placed to surface -exactly those bugs. Debugging order follows from this: if an OpenCode run behaves -as though it never saw its prompt, suspect the passthrough before the prompt. +Verified live on all three surfaces, streaming and non-streaming, with a negative +control proving nothing is injected when no prompt is sent, and pinned by +regression tests that assert on what cortex *forwarded upstream* rather than on +the reply. tireless can rely on it. + +Two residual behaviours shape stage 5, and neither is a passthrough defect: + +- **Several system messages are forwarded unmerged, in order, and the last one + wins.** OpenCode sends its own preamble alongside an agent's configured + `prompt`, so ordering decides whether `implement.oc.md` governs. The stage 5 + question is therefore not *did the prompt arrive* but **did it arrive last**. + The failure is silent: an agent that behaves like a generic coding assistant, + ignoring `Out of scope`, with no error anywhere. +- **Thinking models on `/v1/responses` can return an empty string** + ([helexa#223](https://git.lair.cafe/helexa/helexa/issues/223), open). + `/no_think` is honoured on chat/completions but not on Responses, where a small + `max_output_tokens` may be spent entirely on the reasoning block, yielding `""` + with `status: "incomplete"`. tireless must treat that as a distinct outcome + rather than an empty success, or it will burn a retry on a budget artifact. + +**Model choice for this lane is an operator decision** (`generic.md` §14 — +placement of load on shared infrastructure). The fleet now offers, via cortex: + +| Model | Alias | State | System prompt | +| --- | --- | --- | --- | +| `Qwen/Qwen3.6-27B` | `helexa/large` | warm, pinned on beast | verified | +| `Qwen/Qwen3-Coder-Next` | — | cold; feasible only on beast | shared code path, not live-tested | +| `Qwen/Qwen3-Next-80B-A3B-Thinking` | — | cold; feasible only on beast | shared code path, not live-tested | + +A coder-specialised model is the obvious fit for executing a written spec, but +adopting one means displacing the pinned 27B that currently serves +`helexa/large`. That trade is not tireless's to make; it is recorded here and +revisited with stage 5 evidence. + +tireless pins a **model name, not an alias**, for the same reason it pins agent +package versions: an alias that silently starts resolving to a different model +would change implementation behaviour between one job and the next with no +deploy and no signal. --- @@ -474,15 +505,25 @@ target helexa cortex. Enforce `assert_not_anthropic` from config. Register a custom OpenCode agent carrying `prompt/implement.oc.md` as its system prompt. Route plan-descended implementation jobs here. -**Start with a passthrough probe.** Before wiring anything real, confirm a system -prompt reaches Qwen3.6-27B intact through OpenCode → cortex → neuron — the -"reply only with the word PONG" test named in helexa#179. That guarantee is not -yet verified (§2.4), and discovering it fails here is far cheaper than debugging -it through a failed implementation run. +**Start with a precedence probe.** cortex passthrough is settled (§2.4), so the +open question is ordering: OpenCode sends its own preamble alongside the agent's +configured `prompt`, and the last system message wins. Capture what OpenCode +actually sends upstream and assert `implement.oc.md` arrives last — asserting on +model behaviour instead would not distinguish "prompt ignored" from "prompt +honoured but model disagreed". + +If it does not arrive last, that is a stage 5 work item, not a blocker: options +are an OpenCode agent that suppresses the built-in preamble, or folding the +instructions into the user turn where ordering is under tireless's control. + +**Handle `status: "incomplete"` explicitly** (helexa#223): an empty output from a +thinking model on the Responses surface is a token-budget artifact, not a failed +run, and must not consume a retry or trip the circuit breaker. *Done when:* a child issue created by a stage-3 planning run is implemented -end-to-end by OpenCode on the GPU fleet, with zero subscription usage, and the -run demonstrably respected its `Out of scope` section. +end-to-end by OpenCode on the GPU fleet, with zero subscription usage; the run +demonstrably respected its `Out of scope` section; and a captured upstream +request shows `implement.oc.md` in the winning position. ### Stage 6 — Scheduling and dashboard control @@ -558,10 +599,23 @@ catches a well-formed plan that is simply wrong about the codebase, and there is no unit test for whether a prompt produces good plans. That is measured on real issues in stage 3, before stage 5 spends anything on acting on them. -**The OpenCode prompt path is unverified end to end.** It depends on helexa#179, -which is open (§2.4). Worth confirming with a trivial probe — a system prompt -that measurably changes output — at the start of stage 5 rather than debugging it -through a failed implementation run. +**System prompt precedence in the OpenCode lane is unverified.** Passthrough is +settled (helexa#179 closed), but OpenCode's own preamble and +`implement.oc.md` both arrive as system messages and the last wins (§2.4). If +tireless loses that ordering, the symptom is not an error but a generic-feeling +agent that ignores `Out of scope` — the exact failure the prompt exists to +prevent, presenting as a prompt-quality problem rather than a plumbing one. +Checked first in stage 5, by asserting on the upstream request rather than on +behaviour. + +**Adding a `qwen3_next` model to the lane would re-open a verified assumption.** +The system slot is not arch-branched — rendering goes through helexa's shared +`chat_template.rs` using each model's own `tokenizer_config` — so there is no +family-specific code to fail, and template tests cover the shared path. But +`Qwen3-Coder-Next` and `Qwen3-Next-80B-A3B-Thinking` have not been live-tested, +because both are feasible only on beast where the pinned 27B is resident. If the +operator decides the coder model is worth the displacement, probe it while warm +rather than forcing an eviction to find out. **Not yet decided:** whether a failed implementation should automatically open a `tireless/blocked` issue describing what it could not do, or simply comment on diff --git a/prompt/readme.md b/prompt/readme.md index 90a2762..075bd76 100644 --- a/prompt/readme.md +++ b/prompt/readme.md @@ -50,19 +50,58 @@ OpenCode → helexa cortex → neuron, and depends on helexa's passthrough guara no injection, no rewriting, no defaults ([helexa/helexa#179](https://git.lair.cafe/helexa/helexa/issues/179)). -That issue is **open**. The principle was decided 2026-07-17; verification and -documentation are outstanding. Two of its unchecked items bear directly on this -prompt: +**That issue is closed (2026-08-02) and the guarantee holds.** It was verified +live through cortex on all three surfaces — `/v1/chat/completions`, +`/v1/responses`, `/v1/messages` — streaming and non-streaming, with a negative +control confirming nothing is injected when the caller sends no system prompt. +Regression tests assert on what cortex *forwarded upstream*, captured from a mock +neuron, rather than on the reply: a gateway that silently dropped the prompt +would still return a convincing answer, so asserting on the response would not +catch the regression. -- neuron chat templating applying the system role correctly per arch family, - *including interaction with `/no_think` handling* — and Qwen3.6-27B is the - model this prompt targets; -- behaviour when multiple system messages are present. +Two consequences for this prompt, both worth knowing before debugging one. -tireless stage 5 is effectively the second consumer of that guarantee after the -chat SPA, and a good way to surface exactly those bugs. If an OpenCode run -behaves as though it never saw this prompt, suspect the passthrough before -suspecting the prompt. +### Multiple system messages: the last one wins + +cortex forwards several system messages **unmerged and in order**, and does not +editorialise; the model resolves precedence, and observably the last one wins. + +This is the failure mode to check first, because OpenCode sends its own preamble +alongside an agent's configured `prompt`. If `implement.oc.md` arrives *before* +OpenCode's own system content, its instructions lose. The symptom is not an error +— it is an agent that behaves like a generic coding assistant: helpful, +plausible, and ignoring `Out of scope`. + +So the stage 5 check is not "did the prompt arrive" (settled) but **"did it +arrive last"**. Verify by asserting on what OpenCode sends upstream, not on how +the model behaves — the same reasoning helexa used for its own tests. + +### Thinking models on `/v1/responses` can return nothing + +[helexa#223](https://git.lair.cafe/helexa/helexa/issues/223), open: `/no_think` +is honoured on `/v1/chat/completions` but ignored on `/v1/responses`. On the +Responses surface a small `max_output_tokens` can be spent entirely on the +reasoning block, and the caller gets `""` with `status: "incomplete"`. + +That matters here because the natural upgrade path for this lane is a *thinking* +model. An unattended run receiving an empty string looks like agent failure +rather than a token-budget artifact, and would burn a retry. Prefer +chat/completions, budget output tokens generously, and treat +`status: "incomplete"` as a distinct outcome rather than an empty success. + +### The `qwen3_next` family is unverified, but not unhandled + +`Qwen3-Coder-Next` and `Qwen3-Next-80B-A3B-Thinking` were not live-tested: both +are catalogue-feasible only on beast, where the pinned 27B is resident, so +testing them means displacing a production model for a multi-minute cold load. + +What makes that a gap rather than a hole: **the system slot is not +arch-branched.** Rendering goes through the shared `chat_template.rs` path, +applying the template the model ships in its own `tokenizer_config` — there is no +`qwen3_next`-specific code for a system prompt to fall down, and template tests +exercise the shared path directly. If tireless ever points this lane at a Next +model, probe it while it is warm for other reasons rather than forcing an +eviction. ## Editing guidance