diff --git a/Cargo.lock b/Cargo.lock index 16d48e4..e509620 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1380,6 +1380,7 @@ dependencies = [ "clap", "figment", "newsfeed-entities", + "regex", "reqwest", "serde", "serde_json", diff --git a/crates/newsfeed-fetch/src/lib.rs b/crates/newsfeed-fetch/src/lib.rs index 37a381f..857d49f 100644 --- a/crates/newsfeed-fetch/src/lib.rs +++ b/crates/newsfeed-fetch/src/lib.rs @@ -31,6 +31,18 @@ const DEFAULT_UA: &str = concat!( " (feed discovery)" ); +/// Browser-like `Accept` advertising a feed preference. Sending no `Accept` (reqwest's +/// default) trips some anti-bot filters — notably Reddit, which hard-403s a header-less +/// client while only rate-limiting a browser-shaped one. +const FEED_ACCEPT: &str = + "application/rss+xml, application/atom+xml, application/xml;q=0.9, text/html;q=0.8, */*;q=0.7"; + +/// Attach the browser-shaped headers every feed request should carry. +fn feed_headers(rb: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + rb.header(reqwest::header::ACCEPT, FEED_ACCEPT) + .header(reqwest::header::ACCEPT_LANGUAGE, "en-US,en;q=0.9") +} + // --------------------------------------------------------------------------- // Worker path: fetch a known feed URL and parse it into backend-agnostic entries. // --------------------------------------------------------------------------- @@ -43,8 +55,7 @@ pub async fn fetch_entries(client: &Client, url: &str) -> anyhow::Result anyhow::Result> { - let bytes = client - .get(url) + let bytes = feed_headers(client.get(url)) .send() .await .with_context(|| format!("fetching {url}"))? @@ -170,9 +181,7 @@ impl HttpFeedProber { /// Fetch a URL, returning the final URL (after redirects) and the body bytes. async fn fetch(&self, url: &str) -> anyhow::Result<(Url, Vec)> { - let resp = self - .client - .get(url) + let resp = feed_headers(self.client.get(url)) .send() .await .with_context(|| format!("fetching {url}"))? diff --git a/crates/newsfeed-modelwatch/Cargo.toml b/crates/newsfeed-modelwatch/Cargo.toml index 94f1e11..a6bc41b 100644 --- a/crates/newsfeed-modelwatch/Cargo.toml +++ b/crates/newsfeed-modelwatch/Cargo.toml @@ -24,6 +24,7 @@ reqwest.workspace = true serde.workspace = true serde_json.workspace = true chrono.workspace = true +regex.workspace = true anyhow.workspace = true tracing.workspace = true diff --git a/crates/newsfeed-modelwatch/src/lib.rs b/crates/newsfeed-modelwatch/src/lib.rs index 3346bc2..76e4766 100644 --- a/crates/newsfeed-modelwatch/src/lib.rs +++ b/crates/newsfeed-modelwatch/src/lib.rs @@ -9,12 +9,18 @@ //! weights bite. See the newsfeed CLAUDE.md house rule. use std::collections::HashMap; +use std::sync::OnceLock; use chrono::{DateTime, Utc}; +use regex::Regex; use serde::{Deserialize, Serialize}; use newsfeed_entities::item::CandidateSubmission; +/// Below this, a `safetensors.total` is treated as bogus (some repos report a tiny/partial +/// total) and we fall back to the parameter count in the repo name. +const IMPLAUSIBLE_PARAMS_FLOOR: u64 = 100_000_000; + /// A model record from the HF Hub API. The list and detail endpoints share this shape; /// detail-only fields (`safetensors`, `card_data`) are optional so one struct serves both. /// Unknown fields are ignored, so the API can grow without breaking us. @@ -95,6 +101,34 @@ pub fn license(m: &HfModel) -> Option { .find_map(|t| t.strip_prefix("license:").map(|s| s.to_lowercase())) } +/// Parse a parameter count from a repo name, e.g. `Qwen3-235B-A22B` → 235e9, +/// `granite-3b-a600m` → 3e9, `SmolLM-360M` → 360e6. Takes the first `B|M` token, which +/// by convention is the model's total size (the second, for MoE, is the active count). +pub fn params_from_name(id: &str) -> Option { + static RE: OnceLock = OnceLock::new(); + let re = RE.get_or_init(|| Regex::new(r"(?i)(\d+(?:\.\d+)?)\s*([bm])\b").unwrap()); + let name = id.rsplit('/').next().unwrap_or(id); + let caps = re.captures(name)?; + let n: f64 = caps.get(1)?.as_str().parse().ok()?; + let mult = if caps.get(2)?.as_str().eq_ignore_ascii_case("b") { + 1e9 + } else { + 1e6 + }; + Some((n * mult) as u64) +} + +/// Best available parameter count: trust `safetensors.total` when it's plausible, else fall +/// back to the size in the repo name (some repos report a bogus tiny total, which would +/// both misrender and let an oversized model slip past the size filter). +pub fn effective_params(m: &HfModel) -> Option { + let total = m.safetensors.as_ref().map(|s| s.total).filter(|&t| t > 0); + match total { + Some(t) if t >= IMPLAUSIBLE_PARAMS_FLOOR => Some(t), + _ => params_from_name(&m.id).or(total), + } +} + /// The admission predicate: which releases are worth submitting. Configurable; an empty /// allowlist means "don't filter on this axis". #[derive(Debug, Clone, Serialize, Deserialize)] @@ -158,12 +192,15 @@ impl Predicate { { return Err("pipeline_tag not allowed"); } - match &m.safetensors { - None if self.require_safetensors => return Err("no safetensors"), - Some(st) if self.max_params > 0 && st.total > self.max_params => { - return Err("too large"); + if self.require_safetensors && m.safetensors.is_none() { + return Err("no safetensors"); + } + if self.max_params > 0 { + if let Some(p) = effective_params(m) { + if p > self.max_params { + return Err("too large"); + } } - _ => {} } if !self.licenses.is_empty() && !license(m).is_some_and(|l| self.licenses.iter().any(|a| a == &l)) @@ -186,7 +223,7 @@ pub fn to_submission(m: &HfModel, source: &str) -> CandidateSubmission { .or_else(|| m.id.split('/').next().map(str::to_string)); let mut parts = Vec::new(); - if let Some(total) = m.safetensors.as_ref().map(|s| s.total).filter(|&t| t > 0) { + if let Some(total) = effective_params(m) { parts.push(human_params(total)); } if let Some(l) = license(m) { @@ -348,4 +385,49 @@ mod tests { assert_eq!(human_params(14_000_000_000), "14.0B params"); assert_eq!(human_params(770_000_000), "770M params"); } + + #[test] + fn params_from_name_takes_first_size_token() { + assert_eq!( + params_from_name("Qwen/Qwen3-235B-A22B"), + Some(235_000_000_000) + ); + assert_eq!( + params_from_name("ibm-granite/granite-3b-a600m"), + Some(3_000_000_000) + ); + assert_eq!(params_from_name("x/Ornith-1.0-35B"), Some(35_000_000_000)); + assert_eq!(params_from_name("HF/SmolLM-360M"), Some(360_000_000)); + assert_eq!( + params_from_name("nvidia/Qwen3.6-27B-NVFP4"), + Some(27_000_000_000) + ); + assert_eq!(params_from_name("openai/gpt-oss"), None); // no size token + } + + #[test] + fn effective_params_falls_back_when_total_is_bogus() { + // Bogus tiny total (the real Ornith case) → use the name's size. + let m = + model(r#"{"id":"deepreinforce-ai/Ornith-1.0-35B","safetensors":{"total":1000000}}"#); + assert_eq!(effective_params(&m), Some(35_000_000_000)); + // Plausible total → trust it. + let m = model(r#"{"id":"a/b-7B","safetensors":{"total":7000000000}}"#); + assert_eq!(effective_params(&m), Some(7_000_000_000)); + // No safetensors block → still recover a size from the name. + assert_eq!( + effective_params(&model(r#"{"id":"a/Foo-9B"}"#)), + Some(9_000_000_000) + ); + } + + #[test] + fn bogus_total_does_not_let_an_oversized_model_through() { + // 70B model reporting a 1M total must still be rejected by the size cap. + let m = model( + r#"{"id":"x/Whale-70B","pipeline_tag":"text-generation", + "cardData":{"license":"mit"},"safetensors":{"total":1000000}}"#, + ); + assert_eq!(Predicate::default().admit(&m), Err("too large")); + } }