rob thijssen
1193c9aefc
media:content-only mapping dropped media:thumbnail elements, but YouTube (and podcast) feeds carry their only image there — media:content is the video/audio itself. Map thumbnails as image/thumbnail media so the SPA's existing image pick renders them. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HuVbgSbyfRBkd4XkEDHV4B
424 lines
16 KiB
Rust
424 lines
16 KiB
Rust
//! Outbound feed I/O — the one place that talks HTTP to the wider web.
|
|
//!
|
|
//! Two consumers:
|
|
//! - the **worker** polls known feed URLs via [`fetch_entries`];
|
|
//! - the **API** resolves an arbitrary page URL to a concrete feed via [`HttpFeedProber`]
|
|
//! (the [`newsfeed_core::ports::FeedProbe`] adapter) when a user adds a source, and
|
|
//! bulk-imports feeds from OPML via [`parse_opml`].
|
|
//!
|
|
//! Kept out of `newsfeed-core` (which is I/O-free) and `newsfeed-data` (SQLite only);
|
|
//! this is the feed adapter crate.
|
|
|
|
use std::sync::OnceLock;
|
|
use std::time::Duration;
|
|
|
|
use anyhow::Context;
|
|
use async_trait::async_trait;
|
|
use regex::Regex;
|
|
use reqwest::Client;
|
|
use url::Url;
|
|
|
|
use newsfeed_core::error::{CoreError, CoreResult};
|
|
use newsfeed_core::ingest::FeedEntry;
|
|
use newsfeed_core::ports::FeedProbe;
|
|
use newsfeed_entities::item::Media;
|
|
use newsfeed_entities::source::DiscoveredFeed;
|
|
|
|
/// Default User-Agent for outbound feed requests. Some hosts 403 an empty UA.
|
|
const DEFAULT_UA: &str = concat!(
|
|
"newsfeed-fetch/",
|
|
env!("CARGO_PKG_VERSION"),
|
|
" (feed discovery)"
|
|
);
|
|
|
|
/// Browser-like `Accept` advertising a feed preference. Sending no `Accept` (reqwest's
|
|
/// default) trips some anti-bot filters — notably Reddit, which hard-403s a header-less
|
|
/// client while only rate-limiting a browser-shaped one.
|
|
const FEED_ACCEPT: &str =
|
|
"application/rss+xml, application/atom+xml, application/xml;q=0.9, text/html;q=0.8, */*;q=0.7";
|
|
|
|
/// Attach the browser-shaped headers every feed request should carry.
|
|
fn feed_headers(rb: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
|
|
rb.header(reqwest::header::ACCEPT, FEED_ACCEPT)
|
|
.header(reqwest::header::ACCEPT_LANGUAGE, "en-US,en;q=0.9")
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Worker path: fetch a known feed URL and parse it into backend-agnostic entries.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Fetch `url` and parse it, returning one [`FeedEntry`] per feed item.
|
|
pub async fn fetch_entries(client: &Client, url: &str) -> anyhow::Result<Vec<FeedEntry>> {
|
|
let bytes = get_bytes(client, url).await?;
|
|
let feed = feed_rs::parser::parse(&bytes[..]).with_context(|| format!("parsing feed {url}"))?;
|
|
Ok(feed.entries.into_iter().map(entry_to_feed_entry).collect())
|
|
}
|
|
|
|
async fn get_bytes(client: &Client, url: &str) -> anyhow::Result<Vec<u8>> {
|
|
let bytes = feed_headers(client.get(url))
|
|
.send()
|
|
.await
|
|
.with_context(|| format!("fetching {url}"))?
|
|
.error_for_status()
|
|
.with_context(|| format!("bad status from {url}"))?
|
|
.bytes()
|
|
.await
|
|
.with_context(|| format!("reading body from {url}"))?;
|
|
Ok(bytes.to_vec())
|
|
}
|
|
|
|
fn entry_to_feed_entry(entry: feed_rs::model::Entry) -> FeedEntry {
|
|
let url = entry.links.first().map(|l| l.href.clone());
|
|
// Prefer the entry's own id; fall back to the link so de-dup still works.
|
|
let external_id = if entry.id.is_empty() {
|
|
url.clone().unwrap_or_else(|| {
|
|
entry
|
|
.title
|
|
.as_ref()
|
|
.map(|t| t.content.clone())
|
|
.unwrap_or_default()
|
|
})
|
|
} else {
|
|
entry.id.clone()
|
|
};
|
|
let title = entry
|
|
.title
|
|
.map(|t| t.content)
|
|
.unwrap_or_else(|| "(untitled)".to_string());
|
|
let summary = entry
|
|
.summary
|
|
.map(|t| t.content)
|
|
.or_else(|| entry.content.and_then(|c| c.body));
|
|
let author = entry.authors.first().map(|p| p.name.clone());
|
|
let tags = entry.categories.into_iter().map(|c| c.term).collect();
|
|
let published_at = entry.published.or(entry.updated);
|
|
|
|
let mut media = Vec::new();
|
|
for m in entry.media {
|
|
media.extend(m.content.into_iter().filter_map(|c| {
|
|
c.url.map(|u| Media {
|
|
kind: c
|
|
.content_type
|
|
.map(|m| m.to_string())
|
|
.unwrap_or_else(|| "media".to_string()),
|
|
url: u.to_string(),
|
|
width: c.width,
|
|
height: c.height,
|
|
})
|
|
}));
|
|
// media:thumbnail is the only image YouTube (and many podcast) feeds carry —
|
|
// their media:content is the video/audio itself.
|
|
media.extend(m.thumbnails.into_iter().map(|t| Media {
|
|
kind: "image/thumbnail".to_string(),
|
|
url: t.image.uri,
|
|
width: t.image.width,
|
|
height: t.image.height,
|
|
}));
|
|
}
|
|
|
|
FeedEntry {
|
|
external_id,
|
|
url,
|
|
title,
|
|
summary,
|
|
author,
|
|
tags,
|
|
media,
|
|
published_at,
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// API path: resolve an arbitrary page URL to a concrete feed.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// HTTP adapter that resolves page URLs to feeds. Cheap to clone (shares a `reqwest`
|
|
/// connection pool).
|
|
#[derive(Clone)]
|
|
pub struct HttpFeedProber {
|
|
client: Client,
|
|
}
|
|
|
|
impl HttpFeedProber {
|
|
/// Build a prober with a default HTTP client (redirect-following, 20s timeout).
|
|
pub fn new() -> anyhow::Result<Self> {
|
|
let client = Client::builder()
|
|
.user_agent(DEFAULT_UA)
|
|
.timeout(Duration::from_secs(20))
|
|
.build()
|
|
.context("building feed-discovery HTTP client")?;
|
|
Ok(Self { client })
|
|
}
|
|
|
|
/// Build a prober over a caller-supplied client.
|
|
pub fn with_client(client: Client) -> Self {
|
|
Self { client }
|
|
}
|
|
|
|
/// The discovery algorithm, as `anyhow` so callers can map it to their own error type.
|
|
async fn discover(&self, input_url: &str) -> anyhow::Result<DiscoveredFeed> {
|
|
let start = normalize_input(input_url).context("that doesn't look like a URL")?;
|
|
|
|
// Fetch the (possibly rewritten) target once. If the body itself is a feed we're
|
|
// done; otherwise treat it as HTML and look for an advertised feed link.
|
|
let (final_url, bytes) = self.fetch(start.as_str()).await?;
|
|
|
|
if let Ok(feed) = feed_rs::parser::parse(&bytes[..]) {
|
|
return Ok(DiscoveredFeed {
|
|
feed_url: final_url.to_string(),
|
|
title: feed.title.map(|t| t.content),
|
|
});
|
|
}
|
|
|
|
let html = String::from_utf8_lossy(&bytes);
|
|
let href = find_feed_link(&html)
|
|
.with_context(|| format!("no RSS/Atom feed found at {}", final_url))?;
|
|
let feed_url = final_url.join(&href).map(|u| u.to_string()).unwrap_or(href);
|
|
|
|
// Validate the discovered feed actually parses, and take its title.
|
|
let (feed_final, feed_bytes) = self.fetch(&feed_url).await?;
|
|
let feed = feed_rs::parser::parse(&feed_bytes[..])
|
|
.with_context(|| format!("discovered feed {feed_url} did not parse"))?;
|
|
Ok(DiscoveredFeed {
|
|
feed_url: feed_final.to_string(),
|
|
title: feed.title.map(|t| t.content),
|
|
})
|
|
}
|
|
|
|
/// Fetch a URL, returning the final URL (after redirects) and the body bytes.
|
|
async fn fetch(&self, url: &str) -> anyhow::Result<(Url, Vec<u8>)> {
|
|
let resp = feed_headers(self.client.get(url))
|
|
.send()
|
|
.await
|
|
.with_context(|| format!("fetching {url}"))?
|
|
.error_for_status()
|
|
.with_context(|| format!("bad status from {url}"))?;
|
|
let final_url = resp.url().clone();
|
|
let bytes = resp
|
|
.bytes()
|
|
.await
|
|
.with_context(|| format!("reading body from {url}"))?;
|
|
Ok((final_url, bytes.to_vec()))
|
|
}
|
|
}
|
|
|
|
#[async_trait]
|
|
impl FeedProbe for HttpFeedProber {
|
|
async fn probe(&self, input_url: &str) -> CoreResult<DiscoveredFeed> {
|
|
// Any failure here is a problem with the URL the user supplied, so surface it as a
|
|
// domain-validation error (→ HTTP 400) with the underlying cause chained in.
|
|
self.discover(input_url).await.map_err(|e| {
|
|
CoreError::Domain(newsfeed_entities::Error::invalid("url", format!("{e:#}")))
|
|
})
|
|
}
|
|
}
|
|
|
|
/// Normalise a user-typed URL: trim, default to `https://` when no scheme is given, then
|
|
/// apply high-confidence feed rewrites (currently Reddit `→ /.rss`). Returns the URL to
|
|
/// fetch first; if it isn't a feed the discovery step falls back to HTML link-scanning.
|
|
fn normalize_input(input: &str) -> Option<Url> {
|
|
let trimmed = input.trim();
|
|
if trimmed.is_empty() {
|
|
return None;
|
|
}
|
|
let with_scheme = if trimmed.contains("://") {
|
|
trimmed.to_string()
|
|
} else {
|
|
format!("https://{trimmed}")
|
|
};
|
|
let mut url = Url::parse(&with_scheme).ok()?;
|
|
reddit_rewrite(&mut url);
|
|
Some(url)
|
|
}
|
|
|
|
/// Reddit listing pages are JS-rendered and don't advertise their feed in HTML, but every
|
|
/// listing has a sibling `.rss`. Rewrite `/r/<x>` and `/user/<x>` to their feed.
|
|
fn reddit_rewrite(url: &mut Url) {
|
|
let host = url.host_str().unwrap_or("");
|
|
if !(host == "reddit.com" || host.ends_with(".reddit.com")) {
|
|
return;
|
|
}
|
|
let path = url.path().trim_end_matches('/');
|
|
if path.ends_with(".rss") {
|
|
return;
|
|
}
|
|
let is_listing = path.starts_with("/r/") || path.starts_with("/user/");
|
|
if is_listing {
|
|
url.set_path(&format!("{path}/.rss"));
|
|
}
|
|
}
|
|
|
|
/// Scan HTML for the first `<link rel="alternate" type="application/rss+xml|atom+xml">`
|
|
/// and return its `href`. Covers YouTube channels, Mastodon profiles, Substack, WordPress,
|
|
/// and most blogs, which all advertise their feed this way.
|
|
pub fn find_feed_link(html: &str) -> Option<String> {
|
|
static LINK_TAG: OnceLock<Regex> = OnceLock::new();
|
|
static TYPE_ATTR: OnceLock<Regex> = OnceLock::new();
|
|
static HREF_ATTR: OnceLock<Regex> = OnceLock::new();
|
|
|
|
let link_tag = LINK_TAG.get_or_init(|| Regex::new(r"(?is)<link\b[^>]*>").unwrap());
|
|
let type_attr = TYPE_ATTR
|
|
.get_or_init(|| Regex::new(r#"(?is)type\s*=\s*["'][^"']*(?:rss|atom)\+xml"#).unwrap());
|
|
let href_attr =
|
|
HREF_ATTR.get_or_init(|| Regex::new(r#"(?is)href\s*=\s*["']([^"']+)["']"#).unwrap());
|
|
|
|
for tag in link_tag.find_iter(html) {
|
|
let tag = tag.as_str();
|
|
if type_attr.is_match(tag) {
|
|
if let Some(href) = href_attr.captures(tag).and_then(|c| c.get(1)) {
|
|
let href = href.as_str().trim();
|
|
if !href.is_empty() {
|
|
return Some(href.to_string());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
None
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// OPML import.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// A feed extracted from an OPML document.
|
|
pub struct OpmlFeed {
|
|
/// Display title (`title` attr, falling back to `text`), if any.
|
|
pub title: Option<String>,
|
|
/// The feed URL (`xmlUrl` attr).
|
|
pub xml_url: String,
|
|
}
|
|
|
|
/// Parse an OPML document into its feeds, recursing through nested outline folders.
|
|
pub fn parse_opml(text: &str) -> anyhow::Result<Vec<OpmlFeed>> {
|
|
let doc = opml::OPML::from_str(text).map_err(|e| anyhow::anyhow!("invalid OPML: {e}"))?;
|
|
let mut out = Vec::new();
|
|
collect_outlines(&doc.body.outlines, &mut out);
|
|
Ok(out)
|
|
}
|
|
|
|
fn collect_outlines(outlines: &[opml::Outline], out: &mut Vec<OpmlFeed>) {
|
|
for o in outlines {
|
|
if let Some(xml_url) = &o.xml_url {
|
|
let title = o
|
|
.title
|
|
.clone()
|
|
.filter(|t| !t.is_empty())
|
|
.or_else(|| (!o.text.is_empty()).then(|| o.text.clone()));
|
|
out.push(OpmlFeed {
|
|
title,
|
|
xml_url: xml_url.clone(),
|
|
});
|
|
}
|
|
collect_outlines(&o.outlines, out);
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn reddit_listing_rewrites_to_rss() {
|
|
let u = normalize_input("reddit.com/r/rust").unwrap();
|
|
assert_eq!(u.as_str(), "https://reddit.com/r/rust/.rss");
|
|
let u = normalize_input("https://www.reddit.com/r/rust/").unwrap();
|
|
assert_eq!(u.as_str(), "https://www.reddit.com/r/rust/.rss");
|
|
let u = normalize_input("https://reddit.com/user/spez").unwrap();
|
|
assert_eq!(u.as_str(), "https://reddit.com/user/spez/.rss");
|
|
}
|
|
|
|
#[test]
|
|
fn reddit_existing_rss_untouched() {
|
|
let u = normalize_input("https://reddit.com/r/rust/.rss").unwrap();
|
|
assert_eq!(u.as_str(), "https://reddit.com/r/rust/.rss");
|
|
}
|
|
|
|
#[test]
|
|
fn non_reddit_passes_through_with_scheme() {
|
|
let u = normalize_input("example.com/blog").unwrap();
|
|
assert_eq!(u.as_str(), "https://example.com/blog");
|
|
}
|
|
|
|
#[test]
|
|
fn empty_input_is_none() {
|
|
assert!(normalize_input(" ").is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn finds_youtube_style_feed_link() {
|
|
let html = r#"<html><head>
|
|
<link rel="canonical" href="https://www.youtube.com/channel/UC123">
|
|
<link rel="alternate" type="application/rss+xml" title="RSS"
|
|
href="https://www.youtube.com/feeds/videos.xml?channel_id=UC123">
|
|
</head></html>"#;
|
|
assert_eq!(
|
|
find_feed_link(html).as_deref(),
|
|
Some("https://www.youtube.com/feeds/videos.xml?channel_id=UC123")
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn finds_atom_link_regardless_of_attr_order() {
|
|
let html = r#"<link href="/feed.atom" type="application/atom+xml" rel="alternate">"#;
|
|
assert_eq!(find_feed_link(html).as_deref(), Some("/feed.atom"));
|
|
}
|
|
|
|
#[test]
|
|
fn no_feed_link_returns_none() {
|
|
let html = r#"<head><link rel="stylesheet" href="/style.css"></head>"#;
|
|
assert!(find_feed_link(html).is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn youtube_media_thumbnail_becomes_image_media() {
|
|
// Shape of a YouTube channel feed entry: the media:group's media:content is the
|
|
// video itself; the only image is the media:thumbnail.
|
|
let atom = r#"<?xml version="1.0" encoding="UTF-8"?>
|
|
<feed xmlns="http://www.w3.org/2005/Atom"
|
|
xmlns:media="http://search.yahoo.com/mrss/">
|
|
<title>channel</title>
|
|
<entry>
|
|
<id>yt:video:abc123</id>
|
|
<title>a video</title>
|
|
<link rel="alternate" href="https://www.youtube.com/watch?v=abc123"/>
|
|
<media:group>
|
|
<media:title>a video</media:title>
|
|
<media:content url="https://www.youtube.com/v/abc123?version=3"
|
|
type="application/x-shockwave-flash" width="640" height="390"/>
|
|
<media:thumbnail url="https://i2.ytimg.com/vi/abc123/hqdefault.jpg"
|
|
width="480" height="360"/>
|
|
</media:group>
|
|
</entry>
|
|
</feed>"#;
|
|
let feed = feed_rs::parser::parse(atom.as_bytes()).unwrap();
|
|
let entry = feed.entries.into_iter().next().unwrap();
|
|
let fe = entry_to_feed_entry(entry);
|
|
let thumb = fe
|
|
.media
|
|
.iter()
|
|
.find(|m| m.kind.starts_with("image"))
|
|
.expect("thumbnail mapped as image media");
|
|
assert_eq!(thumb.url, "https://i2.ytimg.com/vi/abc123/hqdefault.jpg");
|
|
assert_eq!(thumb.width, Some(480));
|
|
assert_eq!(thumb.height, Some(360));
|
|
}
|
|
|
|
#[test]
|
|
fn parses_opml_with_nested_folders() {
|
|
let opml = r#"<?xml version="1.0"?>
|
|
<opml version="2.0"><head><title>subs</title></head><body>
|
|
<outline text="News">
|
|
<outline text="Hacker News" title="Hacker News"
|
|
type="rss" xmlUrl="https://hnrss.org/frontpage"/>
|
|
</outline>
|
|
<outline text="Blog" xmlUrl="https://example.com/feed.xml"/>
|
|
</body></opml>"#;
|
|
let feeds = parse_opml(opml).unwrap();
|
|
assert_eq!(feeds.len(), 2);
|
|
assert_eq!(feeds[0].xml_url, "https://hnrss.org/frontpage");
|
|
assert_eq!(feeds[0].title.as_deref(), Some("Hacker News"));
|
|
assert_eq!(feeds[1].xml_url, "https://example.com/feed.xml");
|
|
assert_eq!(feeds[1].title.as_deref(), Some("Blog"));
|
|
}
|
|
}
|