feat: client-side validation, cycling detection, quantity prompt fix

- validate_strategy(): hard error if quantity is not a parseable decimal
  (catches "ATR_SIZED" etc. before sending to swym API); soft warning if
  a sell rule has no entry_price stop-loss or no bars_since_entry time exit
- Hard validation errors skip the backtest and feed errors back to the LLM
  via IterationRecord.validation_notes included in summary()
- json_contains_kind(): recursive helper to search strategy JSON tree
- diagnose_history(): add cycling detection — triggers is_converged when
  any avg_sharpe value appears 3+ times in history (not just last 3 streak),
  catching the alternating RSI<30 / RSI<25 pattern seen in the latest run
- prompts: clarify that quantity must parse as a float; list invalid
  placeholder strings ("ATR_SIZED", "FULL_BALANCE", "dynamic", etc.)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-09 17:56:59 +02:00
parent e27aabae34
commit b947f48b01
2 changed files with 162 additions and 37 deletions

View File

@@ -16,6 +16,8 @@ struct IterationRecord {
iteration: u32, iteration: u32,
strategy: Value, strategy: Value,
results: Vec<BacktestResult>, results: Vec<BacktestResult>,
/// Client-side validation issues found before submitting to swym.
validation_notes: Vec<String>,
} }
impl IterationRecord { impl IterationRecord {
@@ -42,18 +44,81 @@ impl IterationRecord {
" strategy: {}", " strategy: {}",
serde_json::to_string(&self.strategy).unwrap_or_default() serde_json::to_string(&self.strategy).unwrap_or_default()
)); ));
for note in &self.validation_notes {
lines.push(format!(" ⚠ VALIDATION: {note}"));
}
for r in &self.results { for r in &self.results {
lines.push(r.summary_line()); lines.push(r.summary_line());
} }
lines.push(format!( if self.results.is_empty() && !self.validation_notes.is_empty() {
" avg_sharpe={:.3} best_sharpe={:.3}", lines.push(" backtest skipped due to validation errors".to_string());
self.avg_sharpe(), } else {
self.best_sharpe() lines.push(format!(
)); " avg_sharpe={:.3} best_sharpe={:.3}",
self.avg_sharpe(),
self.best_sharpe()
));
}
lines.join("\n") lines.join("\n")
} }
} }
/// Validate a strategy JSON before submitting to swym.
///
/// Returns `(hard_errors, warnings)`.
/// Hard errors block submission; warnings are noted but allow the backtest to proceed.
fn validate_strategy(strategy: &Value) -> (Vec<String>, Vec<String>) {
let mut hard = Vec::new();
let mut warn = Vec::new();
let empty = vec![];
let rules = strategy["rules"].as_array().unwrap_or(&empty);
for (i, rule) in rules.iter().enumerate() {
// Quantity must be a parseable decimal number string
if let Some(qty) = rule["then"]["quantity"].as_str() {
if qty.parse::<f64>().is_err() {
hard.push(format!(
"rule[{i}] quantity {:?} is not a valid decimal number — \
use a plain string like \"0.001\", never a placeholder like {:?}",
qty, qty
));
}
}
// Exit rules must have a stop-loss and a time exit
let side = rule["then"]["side"].as_str().unwrap_or("");
if side == "sell" {
if !json_contains_kind(&rule["when"], "entry_price") {
warn.push(format!(
"rule[{i}] (sell) has no stop-loss — add an entry_price-based compare condition"
));
}
if !json_contains_kind(&rule["when"], "bars_since_entry") {
warn.push(format!(
"rule[{i}] (sell) has no time exit — add a bars_since_entry >= N compare condition"
));
}
}
}
(hard, warn)
}
/// Recursively search a JSON value for any object with the given `"kind"` string.
fn json_contains_kind(v: &Value, kind: &str) -> bool {
match v {
Value::Object(map) => {
if map.get("kind").and_then(|k| k.as_str()) == Some(kind) {
return true;
}
map.values().any(|child| json_contains_kind(child, kind))
}
Value::Array(arr) => arr.iter().any(|child| json_contains_kind(child, kind)),
_ => false,
}
}
pub async fn run(cli: &Cli) -> Result<()> { pub async fn run(cli: &Cli) -> Result<()> {
// Parse instruments // Parse instruments
let instruments: Vec<Instrument> = cli let instruments: Vec<Instrument> = cli
@@ -231,10 +296,39 @@ pub async fn run(cli: &Cli) -> Result<()> {
); );
debug!("strategy JSON:\n{}", serde_json::to_string_pretty(&strategy).unwrap_or_default()); debug!("strategy JSON:\n{}", serde_json::to_string_pretty(&strategy).unwrap_or_default());
// Client-side validation
let (hard_errors, soft_warnings) = validate_strategy(&strategy);
if !hard_errors.is_empty() {
warn!("strategy validation failed:");
for e in &hard_errors {
warn!(" - {e}");
}
}
if !soft_warnings.is_empty() {
for w in &soft_warnings {
warn!(" ⚠ {w}");
}
}
let mut validation_notes = hard_errors.clone();
validation_notes.extend(soft_warnings);
// Save the strategy JSON // Save the strategy JSON
let strat_path = cli.output_dir.join(format!("strategy_{iteration:03}.json")); let strat_path = cli.output_dir.join(format!("strategy_{iteration:03}.json"));
std::fs::write(&strat_path, serde_json::to_string_pretty(&strategy)?)?; std::fs::write(&strat_path, serde_json::to_string_pretty(&strategy)?)?;
// Hard validation errors: skip the expensive backtest and give immediate feedback.
if !hard_errors.is_empty() {
let record = IterationRecord {
iteration,
strategy: strategy.clone(),
results: vec![],
validation_notes,
};
info!("{}", record.summary());
history.push(record);
continue;
}
// Run backtests against all instruments (in-sample) // Run backtests against all instruments (in-sample)
let mut results: Vec<BacktestResult> = Vec::new(); let mut results: Vec<BacktestResult> = Vec::new();
@@ -289,6 +383,7 @@ pub async fn run(cli: &Cli) -> Result<()> {
iteration, iteration,
strategy: strategy.clone(), strategy: strategy.clone(),
results, results,
validation_notes,
}; };
info!("{}", record.summary()); info!("{}", record.summary());
@@ -499,45 +594,71 @@ pub fn diagnose_history(history: &[IterationRecord]) -> (String, bool) {
.into_iter() .into_iter()
.collect(); .collect();
// --- Convergence detection --- // --- Convergence / cycling detection ---
// If the last 3 iterations all have avg_sharpe within 0.03 of each other,
// the model is stuck in a local optimum and needs a hard reset.
if history.len() >= 3 { if history.len() >= 3 {
// Collect finite avg_sharpe values from all iterations with actual results.
let all_sharpes: Vec<f64> = history
.iter()
.filter(|rec| !rec.results.is_empty())
.map(|rec| rec.avg_sharpe())
.filter(|s| s.is_finite())
.collect();
// Check 1: Any avg_sharpe value appears 3+ times across full history (cycling).
let cycling_sharpe = all_sharpes.iter().find(|&&s| {
all_sharpes.iter().filter(|&&s2| (s2 - s).abs() < 0.005).count() >= 3
}).copied();
// Check 2: Last 3 iterations within 0.03 avg_sharpe spread (local convergence).
let recent = &history[history.len().saturating_sub(3)..]; let recent = &history[history.len().saturating_sub(3)..];
let recent_sharpes: Vec<f64> = recent let recent_sharpes: Vec<f64> = recent
.iter() .iter()
.map(|r| r.avg_sharpe()) .filter(|rec| !rec.results.is_empty())
.map(|rec| rec.avg_sharpe())
.filter(|s| s.is_finite()) .filter(|s| s.is_finite())
.collect(); .collect();
if recent_sharpes.len() == 3 { let (last3_spread, last3_converged) = if recent_sharpes.len() >= 3 {
let max_s = recent_sharpes.iter().cloned().fold(f64::NEG_INFINITY, f64::max); let max_s = recent_sharpes.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
let min_s = recent_sharpes.iter().cloned().fold(f64::INFINITY, f64::min); let min_s = recent_sharpes.iter().cloned().fold(f64::INFINITY, f64::min);
if max_s - min_s < 0.03 { (max_s - min_s, max_s - min_s < 0.03)
is_converged = true; } else {
let untried: Vec<&str> = ["1h", "4h", "15m", "1d"] (0.0, false)
.iter() };
.copied()
.filter(|iv| !intervals_tried.iter().any(|t| t == iv)) if cycling_sharpe.is_some() || last3_converged {
.collect(); is_converged = true;
let interval_hint = if untried.is_empty() { let reason = if let Some(s) = cycling_sharpe {
String::new() format!(
} else { "CYCLING DETECTED: avg_sharpe {s:.3} has appeared 3+ times in history — \
format!( the model is oscillating between the same two approaches"
" You have only tried intervals: {}. Switch to {}.", )
intervals_tried.join(", "), } else {
untried.join(" or ") format!(
) "last 3 iterations produced nearly identical results \
}; (avg Sharpe spread {last3_spread:.3})"
notes.push(format!( )
"⚠ CONVERGENCE DETECTED: The last 3 iterations produced nearly identical \ };
results (avg Sharpe spread {:.3}). Showing the best strategy is \ let untried: Vec<&str> = ["1h", "4h", "15m", "1d"]
suppressed to prevent anchoring.{interval_hint} \ .iter()
You MUST try a fundamentally different approach — different indicator \ .copied()
family, different candle interval, or radically simplified conditions. \ .filter(|iv| !intervals_tried.iter().any(|t| t == iv))
Do NOT refine the previous strategy.", .collect();
max_s - min_s, let interval_hint = if untried.is_empty() {
)); String::new()
} } else {
format!(
" You have only tried intervals: {}. Switch to {}.",
intervals_tried.join(", "),
untried.join(" or ")
)
};
notes.push(format!(
"⚠ CONVERGENCE DETECTED: {reason}. Showing the best strategy is \
suppressed to prevent anchoring.{interval_hint} \
You MUST try a fundamentally different approach — different indicator \
family, different candle interval, or radically simplified conditions. \
Do NOT refine the previous strategy.",
));
} }
} }

View File

@@ -51,8 +51,11 @@ bars_since_entry — complete bars elapsed since position was opened
balance — free balance of a named asset (e.g. "usdt", "usdc") balance — free balance of a named asset (e.g. "usdt", "usdc")
### Quantity ### Quantity
Action quantity MUST be a fixed decimal string, e.g. `"quantity": "0.001"`. Action quantity MUST be a fixed decimal string that parses as a floating-point number,
e.g. `"quantity": "0.001"`.
NEVER use an expression object for quantity — only plain decimal strings are accepted. NEVER use an expression object for quantity — only plain decimal strings are accepted.
NEVER use placeholder strings like `"ATR_SIZED"`, `"FULL_BALANCE"`, `"percent_of_balance"`,
`"dynamic"`, or any non-numeric string — these will be rejected immediately.
### Multi-timeframe ### Multi-timeframe
Any expression can reference a different timeframe via "timeframe" field. Any expression can reference a different timeframe via "timeframe" field.
@@ -314,6 +317,7 @@ Common mistakes to NEVER make:
- Always gate buy rules with position state "flat" and sell rules with "long" - Always gate buy rules with position state "flat" and sell rules with "long"
- Never add a short-entry (sell when flat) rule — spot markets are long-only - Never add a short-entry (sell when flat) rule — spot markets are long-only
- Never use an expression object for `quantity` — it must always be a plain decimal string like `"0.001"` - Never use an expression object for `quantity` — it must always be a plain decimal string like `"0.001"`
- Never use a placeholder string for `quantity` — `"ATR_SIZED"`, `"FULL_BALANCE"`, `"dynamic"`, etc. are all invalid and will be rejected. Use `"0.001"` or similar.
"## "##
) )
} }