feat: client-side validation, cycling detection, quantity prompt fix
- validate_strategy(): hard error if quantity is not a parseable decimal
(catches "ATR_SIZED" etc. before sending to swym API); soft warning if
a sell rule has no entry_price stop-loss or no bars_since_entry time exit
- Hard validation errors skip the backtest and feed errors back to the LLM
via IterationRecord.validation_notes included in summary()
- json_contains_kind(): recursive helper to search strategy JSON tree
- diagnose_history(): add cycling detection — triggers is_converged when
any avg_sharpe value appears 3+ times in history (not just last 3 streak),
catching the alternating RSI<30 / RSI<25 pattern seen in the latest run
- prompts: clarify that quantity must parse as a float; list invalid
placeholder strings ("ATR_SIZED", "FULL_BALANCE", "dynamic", etc.)
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
141
src/agent.rs
141
src/agent.rs
@@ -16,6 +16,8 @@ struct IterationRecord {
|
|||||||
iteration: u32,
|
iteration: u32,
|
||||||
strategy: Value,
|
strategy: Value,
|
||||||
results: Vec<BacktestResult>,
|
results: Vec<BacktestResult>,
|
||||||
|
/// Client-side validation issues found before submitting to swym.
|
||||||
|
validation_notes: Vec<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl IterationRecord {
|
impl IterationRecord {
|
||||||
@@ -42,18 +44,81 @@ impl IterationRecord {
|
|||||||
" strategy: {}",
|
" strategy: {}",
|
||||||
serde_json::to_string(&self.strategy).unwrap_or_default()
|
serde_json::to_string(&self.strategy).unwrap_or_default()
|
||||||
));
|
));
|
||||||
|
for note in &self.validation_notes {
|
||||||
|
lines.push(format!(" ⚠ VALIDATION: {note}"));
|
||||||
|
}
|
||||||
for r in &self.results {
|
for r in &self.results {
|
||||||
lines.push(r.summary_line());
|
lines.push(r.summary_line());
|
||||||
}
|
}
|
||||||
|
if self.results.is_empty() && !self.validation_notes.is_empty() {
|
||||||
|
lines.push(" backtest skipped due to validation errors".to_string());
|
||||||
|
} else {
|
||||||
lines.push(format!(
|
lines.push(format!(
|
||||||
" avg_sharpe={:.3} best_sharpe={:.3}",
|
" avg_sharpe={:.3} best_sharpe={:.3}",
|
||||||
self.avg_sharpe(),
|
self.avg_sharpe(),
|
||||||
self.best_sharpe()
|
self.best_sharpe()
|
||||||
));
|
));
|
||||||
|
}
|
||||||
lines.join("\n")
|
lines.join("\n")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Validate a strategy JSON before submitting to swym.
|
||||||
|
///
|
||||||
|
/// Returns `(hard_errors, warnings)`.
|
||||||
|
/// Hard errors block submission; warnings are noted but allow the backtest to proceed.
|
||||||
|
fn validate_strategy(strategy: &Value) -> (Vec<String>, Vec<String>) {
|
||||||
|
let mut hard = Vec::new();
|
||||||
|
let mut warn = Vec::new();
|
||||||
|
|
||||||
|
let empty = vec![];
|
||||||
|
let rules = strategy["rules"].as_array().unwrap_or(&empty);
|
||||||
|
|
||||||
|
for (i, rule) in rules.iter().enumerate() {
|
||||||
|
// Quantity must be a parseable decimal number string
|
||||||
|
if let Some(qty) = rule["then"]["quantity"].as_str() {
|
||||||
|
if qty.parse::<f64>().is_err() {
|
||||||
|
hard.push(format!(
|
||||||
|
"rule[{i}] quantity {:?} is not a valid decimal number — \
|
||||||
|
use a plain string like \"0.001\", never a placeholder like {:?}",
|
||||||
|
qty, qty
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exit rules must have a stop-loss and a time exit
|
||||||
|
let side = rule["then"]["side"].as_str().unwrap_or("");
|
||||||
|
if side == "sell" {
|
||||||
|
if !json_contains_kind(&rule["when"], "entry_price") {
|
||||||
|
warn.push(format!(
|
||||||
|
"rule[{i}] (sell) has no stop-loss — add an entry_price-based compare condition"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if !json_contains_kind(&rule["when"], "bars_since_entry") {
|
||||||
|
warn.push(format!(
|
||||||
|
"rule[{i}] (sell) has no time exit — add a bars_since_entry >= N compare condition"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
(hard, warn)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Recursively search a JSON value for any object with the given `"kind"` string.
|
||||||
|
fn json_contains_kind(v: &Value, kind: &str) -> bool {
|
||||||
|
match v {
|
||||||
|
Value::Object(map) => {
|
||||||
|
if map.get("kind").and_then(|k| k.as_str()) == Some(kind) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
map.values().any(|child| json_contains_kind(child, kind))
|
||||||
|
}
|
||||||
|
Value::Array(arr) => arr.iter().any(|child| json_contains_kind(child, kind)),
|
||||||
|
_ => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn run(cli: &Cli) -> Result<()> {
|
pub async fn run(cli: &Cli) -> Result<()> {
|
||||||
// Parse instruments
|
// Parse instruments
|
||||||
let instruments: Vec<Instrument> = cli
|
let instruments: Vec<Instrument> = cli
|
||||||
@@ -231,10 +296,39 @@ pub async fn run(cli: &Cli) -> Result<()> {
|
|||||||
);
|
);
|
||||||
debug!("strategy JSON:\n{}", serde_json::to_string_pretty(&strategy).unwrap_or_default());
|
debug!("strategy JSON:\n{}", serde_json::to_string_pretty(&strategy).unwrap_or_default());
|
||||||
|
|
||||||
|
// Client-side validation
|
||||||
|
let (hard_errors, soft_warnings) = validate_strategy(&strategy);
|
||||||
|
if !hard_errors.is_empty() {
|
||||||
|
warn!("strategy validation failed:");
|
||||||
|
for e in &hard_errors {
|
||||||
|
warn!(" - {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !soft_warnings.is_empty() {
|
||||||
|
for w in &soft_warnings {
|
||||||
|
warn!(" ⚠ {w}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut validation_notes = hard_errors.clone();
|
||||||
|
validation_notes.extend(soft_warnings);
|
||||||
|
|
||||||
// Save the strategy JSON
|
// Save the strategy JSON
|
||||||
let strat_path = cli.output_dir.join(format!("strategy_{iteration:03}.json"));
|
let strat_path = cli.output_dir.join(format!("strategy_{iteration:03}.json"));
|
||||||
std::fs::write(&strat_path, serde_json::to_string_pretty(&strategy)?)?;
|
std::fs::write(&strat_path, serde_json::to_string_pretty(&strategy)?)?;
|
||||||
|
|
||||||
|
// Hard validation errors: skip the expensive backtest and give immediate feedback.
|
||||||
|
if !hard_errors.is_empty() {
|
||||||
|
let record = IterationRecord {
|
||||||
|
iteration,
|
||||||
|
strategy: strategy.clone(),
|
||||||
|
results: vec![],
|
||||||
|
validation_notes,
|
||||||
|
};
|
||||||
|
info!("{}", record.summary());
|
||||||
|
history.push(record);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
// Run backtests against all instruments (in-sample)
|
// Run backtests against all instruments (in-sample)
|
||||||
let mut results: Vec<BacktestResult> = Vec::new();
|
let mut results: Vec<BacktestResult> = Vec::new();
|
||||||
|
|
||||||
@@ -289,6 +383,7 @@ pub async fn run(cli: &Cli) -> Result<()> {
|
|||||||
iteration,
|
iteration,
|
||||||
strategy: strategy.clone(),
|
strategy: strategy.clone(),
|
||||||
results,
|
results,
|
||||||
|
validation_notes,
|
||||||
};
|
};
|
||||||
|
|
||||||
info!("{}", record.summary());
|
info!("{}", record.summary());
|
||||||
@@ -499,21 +594,50 @@ pub fn diagnose_history(history: &[IterationRecord]) -> (String, bool) {
|
|||||||
.into_iter()
|
.into_iter()
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
// --- Convergence detection ---
|
// --- Convergence / cycling detection ---
|
||||||
// If the last 3 iterations all have avg_sharpe within 0.03 of each other,
|
|
||||||
// the model is stuck in a local optimum and needs a hard reset.
|
|
||||||
if history.len() >= 3 {
|
if history.len() >= 3 {
|
||||||
|
// Collect finite avg_sharpe values from all iterations with actual results.
|
||||||
|
let all_sharpes: Vec<f64> = history
|
||||||
|
.iter()
|
||||||
|
.filter(|rec| !rec.results.is_empty())
|
||||||
|
.map(|rec| rec.avg_sharpe())
|
||||||
|
.filter(|s| s.is_finite())
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
// Check 1: Any avg_sharpe value appears 3+ times across full history (cycling).
|
||||||
|
let cycling_sharpe = all_sharpes.iter().find(|&&s| {
|
||||||
|
all_sharpes.iter().filter(|&&s2| (s2 - s).abs() < 0.005).count() >= 3
|
||||||
|
}).copied();
|
||||||
|
|
||||||
|
// Check 2: Last 3 iterations within 0.03 avg_sharpe spread (local convergence).
|
||||||
let recent = &history[history.len().saturating_sub(3)..];
|
let recent = &history[history.len().saturating_sub(3)..];
|
||||||
let recent_sharpes: Vec<f64> = recent
|
let recent_sharpes: Vec<f64> = recent
|
||||||
.iter()
|
.iter()
|
||||||
.map(|r| r.avg_sharpe())
|
.filter(|rec| !rec.results.is_empty())
|
||||||
|
.map(|rec| rec.avg_sharpe())
|
||||||
.filter(|s| s.is_finite())
|
.filter(|s| s.is_finite())
|
||||||
.collect();
|
.collect();
|
||||||
if recent_sharpes.len() == 3 {
|
let (last3_spread, last3_converged) = if recent_sharpes.len() >= 3 {
|
||||||
let max_s = recent_sharpes.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
|
let max_s = recent_sharpes.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
|
||||||
let min_s = recent_sharpes.iter().cloned().fold(f64::INFINITY, f64::min);
|
let min_s = recent_sharpes.iter().cloned().fold(f64::INFINITY, f64::min);
|
||||||
if max_s - min_s < 0.03 {
|
(max_s - min_s, max_s - min_s < 0.03)
|
||||||
|
} else {
|
||||||
|
(0.0, false)
|
||||||
|
};
|
||||||
|
|
||||||
|
if cycling_sharpe.is_some() || last3_converged {
|
||||||
is_converged = true;
|
is_converged = true;
|
||||||
|
let reason = if let Some(s) = cycling_sharpe {
|
||||||
|
format!(
|
||||||
|
"CYCLING DETECTED: avg_sharpe {s:.3} has appeared 3+ times in history — \
|
||||||
|
the model is oscillating between the same two approaches"
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
format!(
|
||||||
|
"last 3 iterations produced nearly identical results \
|
||||||
|
(avg Sharpe spread {last3_spread:.3})"
|
||||||
|
)
|
||||||
|
};
|
||||||
let untried: Vec<&str> = ["1h", "4h", "15m", "1d"]
|
let untried: Vec<&str> = ["1h", "4h", "15m", "1d"]
|
||||||
.iter()
|
.iter()
|
||||||
.copied()
|
.copied()
|
||||||
@@ -529,17 +653,14 @@ pub fn diagnose_history(history: &[IterationRecord]) -> (String, bool) {
|
|||||||
)
|
)
|
||||||
};
|
};
|
||||||
notes.push(format!(
|
notes.push(format!(
|
||||||
"⚠ CONVERGENCE DETECTED: The last 3 iterations produced nearly identical \
|
"⚠ CONVERGENCE DETECTED: {reason}. Showing the best strategy is \
|
||||||
results (avg Sharpe spread {:.3}). Showing the best strategy is \
|
|
||||||
suppressed to prevent anchoring.{interval_hint} \
|
suppressed to prevent anchoring.{interval_hint} \
|
||||||
You MUST try a fundamentally different approach — different indicator \
|
You MUST try a fundamentally different approach — different indicator \
|
||||||
family, different candle interval, or radically simplified conditions. \
|
family, different candle interval, or radically simplified conditions. \
|
||||||
Do NOT refine the previous strategy.",
|
Do NOT refine the previous strategy.",
|
||||||
max_s - min_s,
|
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// --- Zero-trade check ---
|
// --- Zero-trade check ---
|
||||||
let zero_trade_iters = history
|
let zero_trade_iters = history
|
||||||
|
|||||||
@@ -51,8 +51,11 @@ bars_since_entry — complete bars elapsed since position was opened
|
|||||||
balance — free balance of a named asset (e.g. "usdt", "usdc")
|
balance — free balance of a named asset (e.g. "usdt", "usdc")
|
||||||
|
|
||||||
### Quantity
|
### Quantity
|
||||||
Action quantity MUST be a fixed decimal string, e.g. `"quantity": "0.001"`.
|
Action quantity MUST be a fixed decimal string that parses as a floating-point number,
|
||||||
|
e.g. `"quantity": "0.001"`.
|
||||||
NEVER use an expression object for quantity — only plain decimal strings are accepted.
|
NEVER use an expression object for quantity — only plain decimal strings are accepted.
|
||||||
|
NEVER use placeholder strings like `"ATR_SIZED"`, `"FULL_BALANCE"`, `"percent_of_balance"`,
|
||||||
|
`"dynamic"`, or any non-numeric string — these will be rejected immediately.
|
||||||
|
|
||||||
### Multi-timeframe
|
### Multi-timeframe
|
||||||
Any expression can reference a different timeframe via "timeframe" field.
|
Any expression can reference a different timeframe via "timeframe" field.
|
||||||
@@ -314,6 +317,7 @@ Common mistakes to NEVER make:
|
|||||||
- Always gate buy rules with position state "flat" and sell rules with "long"
|
- Always gate buy rules with position state "flat" and sell rules with "long"
|
||||||
- Never add a short-entry (sell when flat) rule — spot markets are long-only
|
- Never add a short-entry (sell when flat) rule — spot markets are long-only
|
||||||
- Never use an expression object for `quantity` — it must always be a plain decimal string like `"0.001"`
|
- Never use an expression object for `quantity` — it must always be a plain decimal string like `"0.001"`
|
||||||
|
- Never use a placeholder string for `quantity` — `"ATR_SIZED"`, `"FULL_BALANCE"`, `"dynamic"`, etc. are all invalid and will be rejected. Use `"0.001"` or similar.
|
||||||
"##
|
"##
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user