fix(swym): parse result_summary from actual API response structure

The swym API response structure differs from what the code previously
assumed. Fix all field extraction to match the real shape:

- total_positions: backtest_metadata.position_count (not top-level)
- sharpe_ratio, win_rate, profit_factor: instruments.{key}.{field}.value
  wrapped decimal strings (not plain floats); treat Decimal::MAX sentinel
  (~7.9e28) as None
- net_pnl: instruments.{key}.pnl (plain decimal string)
- instrument key derived as "{exchange_no_underscores}-{base}_{quote}"

Also fix coverage-based backtest_from clamping: after the coverage
check, compute the effective backtest start as the max first_open across
all instruments × common intervals, so strategies never fail with
"requested range outside available data". Log per-interval date ranges
for each instrument at startup.

Additionally:
- Compact format_audit_summary to handle {"rules":[...],"total_bars":N}
  structure with per-condition true_count/evaluated breakdown
- Drop avg_bars from summary_line (field absent from API)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-09 14:22:29 +02:00
parent c7a2d65539
commit fb1145acae
2 changed files with 138 additions and 34 deletions

View File

@@ -69,17 +69,24 @@ pub async fn run(cli: &Cli) -> Result<()> {
instruments.len() instruments.len()
); );
let mut available_intervals: Vec<String> = Vec::new(); let mut available_intervals: Vec<String> = Vec::new();
// Store coverage so we can compute the effective backtest start after intersecting intervals.
let mut all_coverage: Vec<(&Instrument, Vec<crate::swym::CandleCoverage>)> = Vec::new();
for inst in &instruments { for inst in &instruments {
match swym.candle_coverage(&inst.exchange, &inst.symbol).await { match swym.candle_coverage(&inst.exchange, &inst.symbol).await {
Ok(coverage) => { Ok(coverage) => {
let intervals: Vec<&str> = coverage.iter().map(|c| c.interval.as_str()).collect(); let intervals: Vec<&str> = coverage.iter().map(|c| c.interval.as_str()).collect();
info!("{}: intervals {:?}", inst.symbol, intervals); let interval_ranges: Vec<String> = coverage
.iter()
.map(|c| format!("{}({} {})", c.interval, c.first_open, c.last_close))
.collect();
info!("{}: {}", inst.symbol, interval_ranges.join(", "));
if available_intervals.is_empty() { if available_intervals.is_empty() {
available_intervals = coverage.iter().map(|c| c.interval.clone()).collect(); available_intervals = coverage.iter().map(|c| c.interval.clone()).collect();
} else { } else {
// Intersect — only keep intervals available for ALL instruments // Intersect — only keep intervals available for ALL instruments
available_intervals.retain(|iv| intervals.contains(&iv.as_str())); available_intervals.retain(|iv| intervals.contains(&iv.as_str()));
} }
all_coverage.push((inst, coverage));
} }
Err(e) => { Err(e) => {
warn!("could not check coverage for {}: {e}", inst.symbol); warn!("could not check coverage for {}: {e}", inst.symbol);
@@ -91,6 +98,24 @@ pub async fn run(cli: &Cli) -> Result<()> {
} }
info!("common intervals: {:?}", available_intervals); info!("common intervals: {:?}", available_intervals);
// Derive the effective backtest start: take the latest first_open across all instruments
// for the common intervals so that strategies can always find their required candle data.
let mut effective_backtest_from = cli.backtest_from.clone();
for (inst, coverage) in &all_coverage {
for c in coverage {
if available_intervals.contains(&c.interval) && c.first_open > effective_backtest_from {
warn!(
"{} {}: data starts {}, clamping backtest_from from {} → {}",
inst.symbol, c.interval, c.first_open, effective_backtest_from, c.first_open
);
effective_backtest_from = c.first_open.clone();
}
}
}
if effective_backtest_from != cli.backtest_from {
info!("effective backtest_from: {effective_backtest_from}");
}
// Load DSL schema for the system prompt // Load DSL schema for the system prompt
let schema = include_str!("dsl-schema.json"); let schema = include_str!("dsl-schema.json");
let system = prompts::system_prompt(schema); let system = prompts::system_prompt(schema);
@@ -200,12 +225,12 @@ pub async fn run(cli: &Cli) -> Result<()> {
let mut results: Vec<BacktestResult> = Vec::new(); let mut results: Vec<BacktestResult> = Vec::new();
for inst in &instruments { for inst in &instruments {
info!("backtesting {} on {}...", inst.symbol, cli.backtest_from); info!("backtesting {} on {}...", inst.symbol, effective_backtest_from);
match run_single_backtest( match run_single_backtest(
&swym, &swym,
inst, inst,
&strategy, &strategy,
&cli.backtest_from, &effective_backtest_from,
&cli.backtest_to, &cli.backtest_to,
&cli.initial_balance, &cli.initial_balance,
&cli.fees_percent, &cli.fees_percent,
@@ -393,7 +418,13 @@ async fn run_single_backtest(
.await .await
.context("poll")?; .context("poll")?;
Ok(BacktestResult::from_response(&final_resp, &inst.symbol)) Ok(BacktestResult::from_response(
&final_resp,
&inst.symbol,
&inst.exchange,
&inst.base(),
&inst.quote(),
))
} }
fn save_validated_strategy( fn save_validated_strategy(

View File

@@ -52,22 +52,47 @@ pub struct BacktestResult {
} }
impl BacktestResult { impl BacktestResult {
pub fn from_response(resp: &PaperRunResponse, instrument: &str) -> Self { /// Parse a backtest response.
///
/// `exchange`, `base`, `quote` are needed to derive the instrument key used
/// in the `result_summary.instruments` map (e.g. `binancespot-eth_usdc`).
pub fn from_response(
resp: &PaperRunResponse,
instrument: &str,
exchange: &str,
base: &str,
quote: &str,
) -> Self {
let summary = resp.result_summary.as_ref(); let summary = resp.result_summary.as_ref();
if let Some(s) = summary {
tracing::debug!("[{instrument}] result_summary: {}", serde_json::to_string(s).unwrap_or_default());
} else {
tracing::debug!("[{instrument}] result_summary: null");
}
// The API key for per-instrument stats: "binance_spot" + "eth" + "usdc" → "binancespot-eth_usdc"
let inst_key = format!("{}-{}_{}", exchange.replace('_', ""), base, quote);
let total_positions = summary.and_then(|s| {
s["backtest_metadata"]["position_count"].as_u64().map(|v| v as u32)
});
let inst_stats = summary.and_then(|s| s["instruments"].get(&inst_key));
Self { Self {
run_id: resp.id, run_id: resp.id,
instrument: instrument.to_string(), instrument: instrument.to_string(),
status: resp.status.clone(), status: resp.status.clone(),
total_positions: summary.and_then(|s| s["total_positions"].as_u64().map(|v| v as u32)), total_positions,
winning_positions: summary.and_then(|s| s["winning_positions"].as_u64().map(|v| v as u32)), winning_positions: None,
losing_positions: summary.and_then(|s| s["losing_positions"].as_u64().map(|v| v as u32)), losing_positions: None,
win_rate: summary.and_then(|s| s["win_rate"].as_f64()), win_rate: inst_stats.and_then(|s| parse_ratio_value(&s["win_rate"])),
profit_factor: summary.and_then(|s| s["profit_factor"].as_f64()), profit_factor: inst_stats.and_then(|s| parse_ratio_value(&s["profit_factor"])),
total_pnl: summary.and_then(|s| s["total_pnl"].as_f64()), total_pnl: inst_stats.and_then(|s| parse_decimal_str(&s["pnl"])),
net_pnl: summary.and_then(|s| s["net_pnl"].as_f64()), net_pnl: inst_stats.and_then(|s| parse_decimal_str(&s["pnl"])),
sharpe_ratio: summary.and_then(|s| s["sharpe_ratio"].as_f64()), sharpe_ratio: inst_stats.and_then(|s| parse_ratio_value(&s["sharpe_ratio"])),
total_fees: summary.and_then(|s| s["total_fees"].as_f64()), total_fees: None,
avg_bars_in_trade: summary.and_then(|s| s["avg_bars_in_trade"].as_f64()), avg_bars_in_trade: None,
error_message: resp.error_message.clone(), error_message: resp.error_message.clone(),
condition_audit_summary: summary.and_then(|s| s.get("condition_audit_summary").cloned()), condition_audit_summary: summary.and_then(|s| s.get("condition_audit_summary").cloned()),
} }
@@ -83,14 +108,13 @@ impl BacktestResult {
); );
} }
let mut s = format!( let mut s = format!(
"[{}] trades={} win_rate={:.1}% pf={:.2} net_pnl={:.2} sharpe={:.2} avg_bars={:.1}", "[{}] trades={} win_rate={:.1}% pf={:.2} net_pnl={:.2} sharpe={:.2}",
self.instrument, self.instrument,
self.total_positions.unwrap_or(0), self.total_positions.unwrap_or(0),
self.win_rate.unwrap_or(0.0) * 100.0, self.win_rate.unwrap_or(0.0) * 100.0,
self.profit_factor.unwrap_or(0.0), self.profit_factor.unwrap_or(0.0),
self.net_pnl.unwrap_or(0.0), self.net_pnl.unwrap_or(0.0),
self.sharpe_ratio.unwrap_or(0.0), self.sharpe_ratio.unwrap_or(0.0),
self.avg_bars_in_trade.unwrap_or(0.0),
); );
if self.total_positions.unwrap_or(0) == 0 { if self.total_positions.unwrap_or(0) == 0 {
if let Some(audit) = &self.condition_audit_summary { if let Some(audit) = &self.condition_audit_summary {
@@ -113,33 +137,82 @@ impl BacktestResult {
} }
} }
/// Parse a `{"interval": null, "value": "123.45"}` ratio wrapper.
/// Returns `None` for null, missing, or sentinel values (Decimal::MAX ≈ 7.9e28).
fn parse_ratio_value(v: &Value) -> Option<f64> {
let s = v.get("value")?.as_str()?;
let f: f64 = s.parse().ok()?;
if f.abs() > 1e20 { None } else { Some(f) }
}
/// Parse a plain decimal string JSON value.
/// Returns `None` for null, missing, or sentinel values.
fn parse_decimal_str(v: &Value) -> Option<f64> {
let f: f64 = v.as_str()?.parse().ok()?;
if f.abs() > 1e20 { None } else { Some(f) }
}
/// Render a condition_audit_summary Value into a compact one-line string. /// Render a condition_audit_summary Value into a compact one-line string.
/// Handles both object and array shapes we might receive from the API. ///
/// Handles the primary shape from the swym API:
/// `{"rules": [{rule_index, times_fired, conditions: [{kind, path, true_count, insufficient_data_count}]}], "total_bars": N}`
fn format_audit_summary(audit: &Value) -> String { fn format_audit_summary(audit: &Value) -> String {
match audit { // Primary shape: {"rules": [...], "total_bars": N}
Value::Object(map) => map if let Some(rules) = audit.get("rules").and_then(|r| r.as_array()) {
let total_bars = audit["total_bars"].as_u64().unwrap_or(0);
let parts: Vec<String> = rules
.iter() .iter()
.map(|(k, v)| format!("{k}={v}")) .map(|rule| {
.collect::<Vec<_>>() let idx = rule["rule_index"].as_u64().unwrap_or(0);
.join(", "), let fired = rule["times_fired"].as_u64().unwrap_or(0);
Value::Array(arr) => arr let cond_summary = rule["conditions"]
.as_array()
.map(|conds| {
conds
.iter() .iter()
.filter_map(|item| { .map(|c| {
let name = item.get("name").or_else(|| item.get("condition"))?.as_str()?; let kind = c["kind"].as_str().unwrap_or("?");
// Try common field names for hit counts let true_count = c["true_count"].as_u64().unwrap_or(0);
if let (Some(true_count), Some(total)) = ( let nodata = c["insufficient_data_count"].as_u64().unwrap_or(0);
item.get("true_count").or_else(|| item.get("hit_count")).or_else(|| item.get("true_bars")).and_then(|v| v.as_u64()), let evaluated = total_bars.saturating_sub(nodata);
item.get("total").or_else(|| item.get("total_bars")).or_else(|| item.get("evaluated")).and_then(|v| v.as_u64()), if nodata > total_bars / 2 {
) { format!("{kind}:{true_count}/{evaluated}[{nodata}nd]")
Some(format!("{name}: {true_count}/{total}"))
} else { } else {
Some(format!("{name}: {item}")) format!("{kind}:{true_count}/{evaluated}")
} }
}) })
.collect::<Vec<_>>() .collect::<Vec<_>>()
.join(", "), .join(" ")
other => other.to_string(), })
.unwrap_or_default();
format!("R{idx}(f={fired})[{cond_summary}]")
})
.collect();
return parts.join(" | ");
} }
// Fallback: plain array of named conditions
if let Some(arr) = audit.as_array() {
return arr
.iter()
.filter_map(|item| {
let name = item.get("name").or_else(|| item.get("condition"))?.as_str()?;
let true_count = item
.get("true_count")
.or_else(|| item.get("hit_count"))
.and_then(|v| v.as_u64());
let total = item
.get("total")
.or_else(|| item.get("total_bars"))
.and_then(|v| v.as_u64());
match (true_count, total) {
(Some(t), Some(n)) => Some(format!("{name}: {t}/{n}")),
_ => Some(format!("{name}: {item}")),
}
})
.collect::<Vec<_>>()
.join(", ");
}
audit.to_string()
} }
impl SwymClient { impl SwymClient {