diff --git a/src/agent.rs b/src/agent.rs index bb6def6..bcb8988 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -16,6 +16,8 @@ struct IterationRecord { iteration: u32, strategy: Value, results: Vec, + /// Client-side validation issues found before submitting to swym. + validation_notes: Vec, } impl IterationRecord { @@ -42,18 +44,81 @@ impl IterationRecord { " strategy: {}", serde_json::to_string(&self.strategy).unwrap_or_default() )); + for note in &self.validation_notes { + lines.push(format!(" ⚠ VALIDATION: {note}")); + } for r in &self.results { lines.push(r.summary_line()); } - lines.push(format!( - " avg_sharpe={:.3} best_sharpe={:.3}", - self.avg_sharpe(), - self.best_sharpe() - )); + if self.results.is_empty() && !self.validation_notes.is_empty() { + lines.push(" backtest skipped due to validation errors".to_string()); + } else { + lines.push(format!( + " avg_sharpe={:.3} best_sharpe={:.3}", + self.avg_sharpe(), + self.best_sharpe() + )); + } lines.join("\n") } } +/// Validate a strategy JSON before submitting to swym. +/// +/// Returns `(hard_errors, warnings)`. +/// Hard errors block submission; warnings are noted but allow the backtest to proceed. +fn validate_strategy(strategy: &Value) -> (Vec, Vec) { + let mut hard = Vec::new(); + let mut warn = Vec::new(); + + let empty = vec![]; + let rules = strategy["rules"].as_array().unwrap_or(&empty); + + for (i, rule) in rules.iter().enumerate() { + // Quantity must be a parseable decimal number string + if let Some(qty) = rule["then"]["quantity"].as_str() { + if qty.parse::().is_err() { + hard.push(format!( + "rule[{i}] quantity {:?} is not a valid decimal number — \ + use a plain string like \"0.001\", never a placeholder like {:?}", + qty, qty + )); + } + } + + // Exit rules must have a stop-loss and a time exit + let side = rule["then"]["side"].as_str().unwrap_or(""); + if side == "sell" { + if !json_contains_kind(&rule["when"], "entry_price") { + warn.push(format!( + "rule[{i}] (sell) has no stop-loss — add an entry_price-based compare condition" + )); + } + if !json_contains_kind(&rule["when"], "bars_since_entry") { + warn.push(format!( + "rule[{i}] (sell) has no time exit — add a bars_since_entry >= N compare condition" + )); + } + } + } + + (hard, warn) +} + +/// Recursively search a JSON value for any object with the given `"kind"` string. +fn json_contains_kind(v: &Value, kind: &str) -> bool { + match v { + Value::Object(map) => { + if map.get("kind").and_then(|k| k.as_str()) == Some(kind) { + return true; + } + map.values().any(|child| json_contains_kind(child, kind)) + } + Value::Array(arr) => arr.iter().any(|child| json_contains_kind(child, kind)), + _ => false, + } +} + pub async fn run(cli: &Cli) -> Result<()> { // Parse instruments let instruments: Vec = cli @@ -231,10 +296,39 @@ pub async fn run(cli: &Cli) -> Result<()> { ); debug!("strategy JSON:\n{}", serde_json::to_string_pretty(&strategy).unwrap_or_default()); + // Client-side validation + let (hard_errors, soft_warnings) = validate_strategy(&strategy); + if !hard_errors.is_empty() { + warn!("strategy validation failed:"); + for e in &hard_errors { + warn!(" - {e}"); + } + } + if !soft_warnings.is_empty() { + for w in &soft_warnings { + warn!(" ⚠ {w}"); + } + } + let mut validation_notes = hard_errors.clone(); + validation_notes.extend(soft_warnings); + // Save the strategy JSON let strat_path = cli.output_dir.join(format!("strategy_{iteration:03}.json")); std::fs::write(&strat_path, serde_json::to_string_pretty(&strategy)?)?; + // Hard validation errors: skip the expensive backtest and give immediate feedback. + if !hard_errors.is_empty() { + let record = IterationRecord { + iteration, + strategy: strategy.clone(), + results: vec![], + validation_notes, + }; + info!("{}", record.summary()); + history.push(record); + continue; + } + // Run backtests against all instruments (in-sample) let mut results: Vec = Vec::new(); @@ -289,6 +383,7 @@ pub async fn run(cli: &Cli) -> Result<()> { iteration, strategy: strategy.clone(), results, + validation_notes, }; info!("{}", record.summary()); @@ -499,45 +594,71 @@ pub fn diagnose_history(history: &[IterationRecord]) -> (String, bool) { .into_iter() .collect(); - // --- Convergence detection --- - // If the last 3 iterations all have avg_sharpe within 0.03 of each other, - // the model is stuck in a local optimum and needs a hard reset. + // --- Convergence / cycling detection --- if history.len() >= 3 { + // Collect finite avg_sharpe values from all iterations with actual results. + let all_sharpes: Vec = history + .iter() + .filter(|rec| !rec.results.is_empty()) + .map(|rec| rec.avg_sharpe()) + .filter(|s| s.is_finite()) + .collect(); + + // Check 1: Any avg_sharpe value appears 3+ times across full history (cycling). + let cycling_sharpe = all_sharpes.iter().find(|&&s| { + all_sharpes.iter().filter(|&&s2| (s2 - s).abs() < 0.005).count() >= 3 + }).copied(); + + // Check 2: Last 3 iterations within 0.03 avg_sharpe spread (local convergence). let recent = &history[history.len().saturating_sub(3)..]; let recent_sharpes: Vec = recent .iter() - .map(|r| r.avg_sharpe()) + .filter(|rec| !rec.results.is_empty()) + .map(|rec| rec.avg_sharpe()) .filter(|s| s.is_finite()) .collect(); - if recent_sharpes.len() == 3 { + let (last3_spread, last3_converged) = if recent_sharpes.len() >= 3 { let max_s = recent_sharpes.iter().cloned().fold(f64::NEG_INFINITY, f64::max); let min_s = recent_sharpes.iter().cloned().fold(f64::INFINITY, f64::min); - if max_s - min_s < 0.03 { - is_converged = true; - let untried: Vec<&str> = ["1h", "4h", "15m", "1d"] - .iter() - .copied() - .filter(|iv| !intervals_tried.iter().any(|t| t == iv)) - .collect(); - let interval_hint = if untried.is_empty() { - String::new() - } else { - format!( - " You have only tried intervals: {}. Switch to {}.", - intervals_tried.join(", "), - untried.join(" or ") - ) - }; - notes.push(format!( - "⚠ CONVERGENCE DETECTED: The last 3 iterations produced nearly identical \ - results (avg Sharpe spread {:.3}). Showing the best strategy is \ - suppressed to prevent anchoring.{interval_hint} \ - You MUST try a fundamentally different approach — different indicator \ - family, different candle interval, or radically simplified conditions. \ - Do NOT refine the previous strategy.", - max_s - min_s, - )); - } + (max_s - min_s, max_s - min_s < 0.03) + } else { + (0.0, false) + }; + + if cycling_sharpe.is_some() || last3_converged { + is_converged = true; + let reason = if let Some(s) = cycling_sharpe { + format!( + "CYCLING DETECTED: avg_sharpe {s:.3} has appeared 3+ times in history — \ + the model is oscillating between the same two approaches" + ) + } else { + format!( + "last 3 iterations produced nearly identical results \ + (avg Sharpe spread {last3_spread:.3})" + ) + }; + let untried: Vec<&str> = ["1h", "4h", "15m", "1d"] + .iter() + .copied() + .filter(|iv| !intervals_tried.iter().any(|t| t == iv)) + .collect(); + let interval_hint = if untried.is_empty() { + String::new() + } else { + format!( + " You have only tried intervals: {}. Switch to {}.", + intervals_tried.join(", "), + untried.join(" or ") + ) + }; + notes.push(format!( + "⚠ CONVERGENCE DETECTED: {reason}. Showing the best strategy is \ + suppressed to prevent anchoring.{interval_hint} \ + You MUST try a fundamentally different approach — different indicator \ + family, different candle interval, or radically simplified conditions. \ + Do NOT refine the previous strategy.", + )); } } diff --git a/src/prompts.rs b/src/prompts.rs index 9ab69ef..7826b9c 100644 --- a/src/prompts.rs +++ b/src/prompts.rs @@ -51,8 +51,11 @@ bars_since_entry — complete bars elapsed since position was opened balance — free balance of a named asset (e.g. "usdt", "usdc") ### Quantity -Action quantity MUST be a fixed decimal string, e.g. `"quantity": "0.001"`. +Action quantity MUST be a fixed decimal string that parses as a floating-point number, +e.g. `"quantity": "0.001"`. NEVER use an expression object for quantity — only plain decimal strings are accepted. +NEVER use placeholder strings like `"ATR_SIZED"`, `"FULL_BALANCE"`, `"percent_of_balance"`, +`"dynamic"`, or any non-numeric string — these will be rejected immediately. ### Multi-timeframe Any expression can reference a different timeframe via "timeframe" field. @@ -314,6 +317,7 @@ Common mistakes to NEVER make: - Always gate buy rules with position state "flat" and sell rules with "long" - Never add a short-entry (sell when flat) rule — spot markets are long-only - Never use an expression object for `quantity` — it must always be a plain decimal string like `"0.001"` +- Never use a placeholder string for `quantity` — `"ATR_SIZED"`, `"FULL_BALANCE"`, `"dynamic"`, etc. are all invalid and will be rejected. Use `"0.001"` or similar. "## ) }