feat: client-side validation, cycling detection, quantity prompt fix

- validate_strategy(): hard error if quantity is not a parseable decimal
  (catches "ATR_SIZED" etc. before sending to swym API); soft warning if
  a sell rule has no entry_price stop-loss or no bars_since_entry time exit
- Hard validation errors skip the backtest and feed errors back to the LLM
  via IterationRecord.validation_notes included in summary()
- json_contains_kind(): recursive helper to search strategy JSON tree
- diagnose_history(): add cycling detection — triggers is_converged when
  any avg_sharpe value appears 3+ times in history (not just last 3 streak),
  catching the alternating RSI<30 / RSI<25 pattern seen in the latest run
- prompts: clarify that quantity must parse as a float; list invalid
  placeholder strings ("ATR_SIZED", "FULL_BALANCE", "dynamic", etc.)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-09 17:56:59 +02:00
parent e27aabae34
commit b947f48b01
2 changed files with 162 additions and 37 deletions

View File

@@ -16,6 +16,8 @@ struct IterationRecord {
iteration: u32,
strategy: Value,
results: Vec<BacktestResult>,
/// Client-side validation issues found before submitting to swym.
validation_notes: Vec<String>,
}
impl IterationRecord {
@@ -42,18 +44,81 @@ impl IterationRecord {
" strategy: {}",
serde_json::to_string(&self.strategy).unwrap_or_default()
));
for note in &self.validation_notes {
lines.push(format!(" ⚠ VALIDATION: {note}"));
}
for r in &self.results {
lines.push(r.summary_line());
}
if self.results.is_empty() && !self.validation_notes.is_empty() {
lines.push(" backtest skipped due to validation errors".to_string());
} else {
lines.push(format!(
" avg_sharpe={:.3} best_sharpe={:.3}",
self.avg_sharpe(),
self.best_sharpe()
));
}
lines.join("\n")
}
}
/// Validate a strategy JSON before submitting to swym.
///
/// Returns `(hard_errors, warnings)`.
/// Hard errors block submission; warnings are noted but allow the backtest to proceed.
fn validate_strategy(strategy: &Value) -> (Vec<String>, Vec<String>) {
let mut hard = Vec::new();
let mut warn = Vec::new();
let empty = vec![];
let rules = strategy["rules"].as_array().unwrap_or(&empty);
for (i, rule) in rules.iter().enumerate() {
// Quantity must be a parseable decimal number string
if let Some(qty) = rule["then"]["quantity"].as_str() {
if qty.parse::<f64>().is_err() {
hard.push(format!(
"rule[{i}] quantity {:?} is not a valid decimal number — \
use a plain string like \"0.001\", never a placeholder like {:?}",
qty, qty
));
}
}
// Exit rules must have a stop-loss and a time exit
let side = rule["then"]["side"].as_str().unwrap_or("");
if side == "sell" {
if !json_contains_kind(&rule["when"], "entry_price") {
warn.push(format!(
"rule[{i}] (sell) has no stop-loss — add an entry_price-based compare condition"
));
}
if !json_contains_kind(&rule["when"], "bars_since_entry") {
warn.push(format!(
"rule[{i}] (sell) has no time exit — add a bars_since_entry >= N compare condition"
));
}
}
}
(hard, warn)
}
/// Recursively search a JSON value for any object with the given `"kind"` string.
fn json_contains_kind(v: &Value, kind: &str) -> bool {
match v {
Value::Object(map) => {
if map.get("kind").and_then(|k| k.as_str()) == Some(kind) {
return true;
}
map.values().any(|child| json_contains_kind(child, kind))
}
Value::Array(arr) => arr.iter().any(|child| json_contains_kind(child, kind)),
_ => false,
}
}
pub async fn run(cli: &Cli) -> Result<()> {
// Parse instruments
let instruments: Vec<Instrument> = cli
@@ -231,10 +296,39 @@ pub async fn run(cli: &Cli) -> Result<()> {
);
debug!("strategy JSON:\n{}", serde_json::to_string_pretty(&strategy).unwrap_or_default());
// Client-side validation
let (hard_errors, soft_warnings) = validate_strategy(&strategy);
if !hard_errors.is_empty() {
warn!("strategy validation failed:");
for e in &hard_errors {
warn!(" - {e}");
}
}
if !soft_warnings.is_empty() {
for w in &soft_warnings {
warn!(" ⚠ {w}");
}
}
let mut validation_notes = hard_errors.clone();
validation_notes.extend(soft_warnings);
// Save the strategy JSON
let strat_path = cli.output_dir.join(format!("strategy_{iteration:03}.json"));
std::fs::write(&strat_path, serde_json::to_string_pretty(&strategy)?)?;
// Hard validation errors: skip the expensive backtest and give immediate feedback.
if !hard_errors.is_empty() {
let record = IterationRecord {
iteration,
strategy: strategy.clone(),
results: vec![],
validation_notes,
};
info!("{}", record.summary());
history.push(record);
continue;
}
// Run backtests against all instruments (in-sample)
let mut results: Vec<BacktestResult> = Vec::new();
@@ -289,6 +383,7 @@ pub async fn run(cli: &Cli) -> Result<()> {
iteration,
strategy: strategy.clone(),
results,
validation_notes,
};
info!("{}", record.summary());
@@ -499,21 +594,50 @@ pub fn diagnose_history(history: &[IterationRecord]) -> (String, bool) {
.into_iter()
.collect();
// --- Convergence detection ---
// If the last 3 iterations all have avg_sharpe within 0.03 of each other,
// the model is stuck in a local optimum and needs a hard reset.
// --- Convergence / cycling detection ---
if history.len() >= 3 {
// Collect finite avg_sharpe values from all iterations with actual results.
let all_sharpes: Vec<f64> = history
.iter()
.filter(|rec| !rec.results.is_empty())
.map(|rec| rec.avg_sharpe())
.filter(|s| s.is_finite())
.collect();
// Check 1: Any avg_sharpe value appears 3+ times across full history (cycling).
let cycling_sharpe = all_sharpes.iter().find(|&&s| {
all_sharpes.iter().filter(|&&s2| (s2 - s).abs() < 0.005).count() >= 3
}).copied();
// Check 2: Last 3 iterations within 0.03 avg_sharpe spread (local convergence).
let recent = &history[history.len().saturating_sub(3)..];
let recent_sharpes: Vec<f64> = recent
.iter()
.map(|r| r.avg_sharpe())
.filter(|rec| !rec.results.is_empty())
.map(|rec| rec.avg_sharpe())
.filter(|s| s.is_finite())
.collect();
if recent_sharpes.len() == 3 {
let (last3_spread, last3_converged) = if recent_sharpes.len() >= 3 {
let max_s = recent_sharpes.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
let min_s = recent_sharpes.iter().cloned().fold(f64::INFINITY, f64::min);
if max_s - min_s < 0.03 {
(max_s - min_s, max_s - min_s < 0.03)
} else {
(0.0, false)
};
if cycling_sharpe.is_some() || last3_converged {
is_converged = true;
let reason = if let Some(s) = cycling_sharpe {
format!(
"CYCLING DETECTED: avg_sharpe {s:.3} has appeared 3+ times in history — \
the model is oscillating between the same two approaches"
)
} else {
format!(
"last 3 iterations produced nearly identical results \
(avg Sharpe spread {last3_spread:.3})"
)
};
let untried: Vec<&str> = ["1h", "4h", "15m", "1d"]
.iter()
.copied()
@@ -529,17 +653,14 @@ pub fn diagnose_history(history: &[IterationRecord]) -> (String, bool) {
)
};
notes.push(format!(
"⚠ CONVERGENCE DETECTED: The last 3 iterations produced nearly identical \
results (avg Sharpe spread {:.3}). Showing the best strategy is \
"⚠ CONVERGENCE DETECTED: {reason}. Showing the best strategy is \
suppressed to prevent anchoring.{interval_hint} \
You MUST try a fundamentally different approach — different indicator \
family, different candle interval, or radically simplified conditions. \
Do NOT refine the previous strategy.",
max_s - min_s,
));
}
}
}
// --- Zero-trade check ---
let zero_trade_iters = history

View File

@@ -51,8 +51,11 @@ bars_since_entry — complete bars elapsed since position was opened
balance — free balance of a named asset (e.g. "usdt", "usdc")
### Quantity
Action quantity MUST be a fixed decimal string, e.g. `"quantity": "0.001"`.
Action quantity MUST be a fixed decimal string that parses as a floating-point number,
e.g. `"quantity": "0.001"`.
NEVER use an expression object for quantity — only plain decimal strings are accepted.
NEVER use placeholder strings like `"ATR_SIZED"`, `"FULL_BALANCE"`, `"percent_of_balance"`,
`"dynamic"`, or any non-numeric string — these will be rejected immediately.
### Multi-timeframe
Any expression can reference a different timeframe via "timeframe" field.
@@ -314,6 +317,7 @@ Common mistakes to NEVER make:
- Always gate buy rules with position state "flat" and sell rules with "long"
- Never add a short-entry (sell when flat) rule — spot markets are long-only
- Never use an expression object for `quantity` — it must always be a plain decimal string like `"0.001"`
- Never use a placeholder string for `quantity` — `"ATR_SIZED"`, `"FULL_BALANCE"`, `"dynamic"`, etc. are all invalid and will be rejected. Use `"0.001"` or similar.
"##
)
}