feat(sources): feed discovery + OPML import for the RSS rail
Make adding RSS sources painless. Instead of requiring the exact feed URL, a user can paste any page — a blog homepage, a YouTube channel, a subreddit, a Mastodon profile — and the server resolves the concrete feed it advertises. Plus OPML bulk-import to migrate a reader export or a YouTube subscriptions list in one shot. No new SourceKind, no migration; this enriches the existing worker-polled RSS rail. New crate `newsfeed-fetch` — the one place that does outbound feed HTTP: - probe(url): normalise (+ Reddit /.rss rewrite) -> fetch -> if it parses as a feed, done; else scan the HTML <head> for <link rel=alternate type=rss/atom>, resolve the relative href, and validate. Covers YouTube/Mastodon/Substack/ WordPress/blogs, which all advertise their feed this way. - parse_opml(): recurses nested outline folders. - fetch_entries()/entry mapping moved here from the worker so both the API (discovery) and worker (polling) share a single HTTP+feed-rs path. - core: add the FeedProbe port (kept I/O-free; adapter lives in newsfeed-fetch). - api: AppState carries Arc<dyn FeedProbe>; POST /v1/sources resolves a pasted homepage to its feed; add POST /v1/sources/discover (preview) and /v1/sources/import (OPML, deduped by URL, per-feed failures collected). - worker: delegate to newsfeed_fetch::fetch_entries; drop the sourcing/ module. - web: Sources page gains paste-URL + "Find feed" preview, OPML file import with a summary, and a "polled Nm ago" hint per source; new client methods and regenerated ts-rs bindings. Verified end-to-end locally: homepage URL -> discovered feed.xml + title; create stores the resolved URL; OPML import added 1/skipped 1 dup; worker polled and both items landed in the feed. fmt/clippy/test + web build/lint green. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_016fKZzDpvjiJ9eYbPGgJvUP
This commit is contained in:
25
crates/newsfeed-fetch/Cargo.toml
Normal file
25
crates/newsfeed-fetch/Cargo.toml
Normal file
@@ -0,0 +1,25 @@
|
||||
[package]
|
||||
name = "newsfeed-fetch"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
authors.workspace = true
|
||||
description = "Outbound feed I/O for newsfeed: RSS/Atom fetch, feed discovery, OPML parse."
|
||||
|
||||
[dependencies]
|
||||
newsfeed-entities.workspace = true
|
||||
newsfeed-core.workspace = true
|
||||
|
||||
reqwest.workspace = true
|
||||
feed-rs.workspace = true
|
||||
url.workspace = true
|
||||
regex.workspace = true
|
||||
opml.workspace = true
|
||||
|
||||
chrono.workspace = true
|
||||
anyhow.workspace = true
|
||||
async-trait.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tokio.workspace = true
|
||||
374
crates/newsfeed-fetch/src/lib.rs
Normal file
374
crates/newsfeed-fetch/src/lib.rs
Normal file
@@ -0,0 +1,374 @@
|
||||
//! Outbound feed I/O — the one place that talks HTTP to the wider web.
|
||||
//!
|
||||
//! Two consumers:
|
||||
//! - the **worker** polls known feed URLs via [`fetch_entries`];
|
||||
//! - the **API** resolves an arbitrary page URL to a concrete feed via [`HttpFeedProber`]
|
||||
//! (the [`newsfeed_core::ports::FeedProbe`] adapter) when a user adds a source, and
|
||||
//! bulk-imports feeds from OPML via [`parse_opml`].
|
||||
//!
|
||||
//! Kept out of `newsfeed-core` (which is I/O-free) and `newsfeed-data` (SQLite only);
|
||||
//! this is the feed adapter crate.
|
||||
|
||||
use std::sync::OnceLock;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::Context;
|
||||
use async_trait::async_trait;
|
||||
use regex::Regex;
|
||||
use reqwest::Client;
|
||||
use url::Url;
|
||||
|
||||
use newsfeed_core::error::{CoreError, CoreResult};
|
||||
use newsfeed_core::ingest::FeedEntry;
|
||||
use newsfeed_core::ports::FeedProbe;
|
||||
use newsfeed_entities::item::Media;
|
||||
use newsfeed_entities::source::DiscoveredFeed;
|
||||
|
||||
/// Default User-Agent for outbound feed requests. Some hosts 403 an empty UA.
|
||||
const DEFAULT_UA: &str = concat!(
|
||||
"newsfeed-fetch/",
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
" (feed discovery)"
|
||||
);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Worker path: fetch a known feed URL and parse it into backend-agnostic entries.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Fetch `url` and parse it, returning one [`FeedEntry`] per feed item.
|
||||
pub async fn fetch_entries(client: &Client, url: &str) -> anyhow::Result<Vec<FeedEntry>> {
|
||||
let bytes = get_bytes(client, url).await?;
|
||||
let feed = feed_rs::parser::parse(&bytes[..]).with_context(|| format!("parsing feed {url}"))?;
|
||||
Ok(feed.entries.into_iter().map(entry_to_feed_entry).collect())
|
||||
}
|
||||
|
||||
async fn get_bytes(client: &Client, url: &str) -> anyhow::Result<Vec<u8>> {
|
||||
let bytes = client
|
||||
.get(url)
|
||||
.send()
|
||||
.await
|
||||
.with_context(|| format!("fetching {url}"))?
|
||||
.error_for_status()
|
||||
.with_context(|| format!("bad status from {url}"))?
|
||||
.bytes()
|
||||
.await
|
||||
.with_context(|| format!("reading body from {url}"))?;
|
||||
Ok(bytes.to_vec())
|
||||
}
|
||||
|
||||
fn entry_to_feed_entry(entry: feed_rs::model::Entry) -> FeedEntry {
|
||||
let url = entry.links.first().map(|l| l.href.clone());
|
||||
// Prefer the entry's own id; fall back to the link so de-dup still works.
|
||||
let external_id = if entry.id.is_empty() {
|
||||
url.clone().unwrap_or_else(|| {
|
||||
entry
|
||||
.title
|
||||
.as_ref()
|
||||
.map(|t| t.content.clone())
|
||||
.unwrap_or_default()
|
||||
})
|
||||
} else {
|
||||
entry.id.clone()
|
||||
};
|
||||
let title = entry
|
||||
.title
|
||||
.map(|t| t.content)
|
||||
.unwrap_or_else(|| "(untitled)".to_string());
|
||||
let summary = entry
|
||||
.summary
|
||||
.map(|t| t.content)
|
||||
.or_else(|| entry.content.and_then(|c| c.body));
|
||||
let author = entry.authors.first().map(|p| p.name.clone());
|
||||
let tags = entry.categories.into_iter().map(|c| c.term).collect();
|
||||
let published_at = entry.published.or(entry.updated);
|
||||
|
||||
let media = entry
|
||||
.media
|
||||
.into_iter()
|
||||
.flat_map(|m| m.content)
|
||||
.filter_map(|c| {
|
||||
c.url.map(|u| Media {
|
||||
kind: c
|
||||
.content_type
|
||||
.map(|m| m.to_string())
|
||||
.unwrap_or_else(|| "media".to_string()),
|
||||
url: u.to_string(),
|
||||
width: c.width,
|
||||
height: c.height,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
FeedEntry {
|
||||
external_id,
|
||||
url,
|
||||
title,
|
||||
summary,
|
||||
author,
|
||||
tags,
|
||||
media,
|
||||
published_at,
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// API path: resolve an arbitrary page URL to a concrete feed.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// HTTP adapter that resolves page URLs to feeds. Cheap to clone (shares a `reqwest`
|
||||
/// connection pool).
|
||||
#[derive(Clone)]
|
||||
pub struct HttpFeedProber {
|
||||
client: Client,
|
||||
}
|
||||
|
||||
impl HttpFeedProber {
|
||||
/// Build a prober with a default HTTP client (redirect-following, 20s timeout).
|
||||
pub fn new() -> anyhow::Result<Self> {
|
||||
let client = Client::builder()
|
||||
.user_agent(DEFAULT_UA)
|
||||
.timeout(Duration::from_secs(20))
|
||||
.build()
|
||||
.context("building feed-discovery HTTP client")?;
|
||||
Ok(Self { client })
|
||||
}
|
||||
|
||||
/// Build a prober over a caller-supplied client.
|
||||
pub fn with_client(client: Client) -> Self {
|
||||
Self { client }
|
||||
}
|
||||
|
||||
/// The discovery algorithm, as `anyhow` so callers can map it to their own error type.
|
||||
async fn discover(&self, input_url: &str) -> anyhow::Result<DiscoveredFeed> {
|
||||
let start = normalize_input(input_url).context("that doesn't look like a URL")?;
|
||||
|
||||
// Fetch the (possibly rewritten) target once. If the body itself is a feed we're
|
||||
// done; otherwise treat it as HTML and look for an advertised feed link.
|
||||
let (final_url, bytes) = self.fetch(start.as_str()).await?;
|
||||
|
||||
if let Ok(feed) = feed_rs::parser::parse(&bytes[..]) {
|
||||
return Ok(DiscoveredFeed {
|
||||
feed_url: final_url.to_string(),
|
||||
title: feed.title.map(|t| t.content),
|
||||
});
|
||||
}
|
||||
|
||||
let html = String::from_utf8_lossy(&bytes);
|
||||
let href = find_feed_link(&html)
|
||||
.with_context(|| format!("no RSS/Atom feed found at {}", final_url))?;
|
||||
let feed_url = final_url.join(&href).map(|u| u.to_string()).unwrap_or(href);
|
||||
|
||||
// Validate the discovered feed actually parses, and take its title.
|
||||
let (feed_final, feed_bytes) = self.fetch(&feed_url).await?;
|
||||
let feed = feed_rs::parser::parse(&feed_bytes[..])
|
||||
.with_context(|| format!("discovered feed {feed_url} did not parse"))?;
|
||||
Ok(DiscoveredFeed {
|
||||
feed_url: feed_final.to_string(),
|
||||
title: feed.title.map(|t| t.content),
|
||||
})
|
||||
}
|
||||
|
||||
/// Fetch a URL, returning the final URL (after redirects) and the body bytes.
|
||||
async fn fetch(&self, url: &str) -> anyhow::Result<(Url, Vec<u8>)> {
|
||||
let resp = self
|
||||
.client
|
||||
.get(url)
|
||||
.send()
|
||||
.await
|
||||
.with_context(|| format!("fetching {url}"))?
|
||||
.error_for_status()
|
||||
.with_context(|| format!("bad status from {url}"))?;
|
||||
let final_url = resp.url().clone();
|
||||
let bytes = resp
|
||||
.bytes()
|
||||
.await
|
||||
.with_context(|| format!("reading body from {url}"))?;
|
||||
Ok((final_url, bytes.to_vec()))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl FeedProbe for HttpFeedProber {
|
||||
async fn probe(&self, input_url: &str) -> CoreResult<DiscoveredFeed> {
|
||||
// Any failure here is a problem with the URL the user supplied, so surface it as a
|
||||
// domain-validation error (→ HTTP 400) with the underlying cause chained in.
|
||||
self.discover(input_url).await.map_err(|e| {
|
||||
CoreError::Domain(newsfeed_entities::Error::invalid("url", format!("{e:#}")))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Normalise a user-typed URL: trim, default to `https://` when no scheme is given, then
|
||||
/// apply high-confidence feed rewrites (currently Reddit `→ /.rss`). Returns the URL to
|
||||
/// fetch first; if it isn't a feed the discovery step falls back to HTML link-scanning.
|
||||
fn normalize_input(input: &str) -> Option<Url> {
|
||||
let trimmed = input.trim();
|
||||
if trimmed.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let with_scheme = if trimmed.contains("://") {
|
||||
trimmed.to_string()
|
||||
} else {
|
||||
format!("https://{trimmed}")
|
||||
};
|
||||
let mut url = Url::parse(&with_scheme).ok()?;
|
||||
reddit_rewrite(&mut url);
|
||||
Some(url)
|
||||
}
|
||||
|
||||
/// Reddit listing pages are JS-rendered and don't advertise their feed in HTML, but every
|
||||
/// listing has a sibling `.rss`. Rewrite `/r/<x>` and `/user/<x>` to their feed.
|
||||
fn reddit_rewrite(url: &mut Url) {
|
||||
let host = url.host_str().unwrap_or("");
|
||||
if !(host == "reddit.com" || host.ends_with(".reddit.com")) {
|
||||
return;
|
||||
}
|
||||
let path = url.path().trim_end_matches('/');
|
||||
if path.ends_with(".rss") {
|
||||
return;
|
||||
}
|
||||
let is_listing = path.starts_with("/r/") || path.starts_with("/user/");
|
||||
if is_listing {
|
||||
url.set_path(&format!("{path}/.rss"));
|
||||
}
|
||||
}
|
||||
|
||||
/// Scan HTML for the first `<link rel="alternate" type="application/rss+xml|atom+xml">`
|
||||
/// and return its `href`. Covers YouTube channels, Mastodon profiles, Substack, WordPress,
|
||||
/// and most blogs, which all advertise their feed this way.
|
||||
pub fn find_feed_link(html: &str) -> Option<String> {
|
||||
static LINK_TAG: OnceLock<Regex> = OnceLock::new();
|
||||
static TYPE_ATTR: OnceLock<Regex> = OnceLock::new();
|
||||
static HREF_ATTR: OnceLock<Regex> = OnceLock::new();
|
||||
|
||||
let link_tag = LINK_TAG.get_or_init(|| Regex::new(r"(?is)<link\b[^>]*>").unwrap());
|
||||
let type_attr = TYPE_ATTR
|
||||
.get_or_init(|| Regex::new(r#"(?is)type\s*=\s*["'][^"']*(?:rss|atom)\+xml"#).unwrap());
|
||||
let href_attr =
|
||||
HREF_ATTR.get_or_init(|| Regex::new(r#"(?is)href\s*=\s*["']([^"']+)["']"#).unwrap());
|
||||
|
||||
for tag in link_tag.find_iter(html) {
|
||||
let tag = tag.as_str();
|
||||
if type_attr.is_match(tag) {
|
||||
if let Some(href) = href_attr.captures(tag).and_then(|c| c.get(1)) {
|
||||
let href = href.as_str().trim();
|
||||
if !href.is_empty() {
|
||||
return Some(href.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// OPML import.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// A feed extracted from an OPML document.
|
||||
pub struct OpmlFeed {
|
||||
/// Display title (`title` attr, falling back to `text`), if any.
|
||||
pub title: Option<String>,
|
||||
/// The feed URL (`xmlUrl` attr).
|
||||
pub xml_url: String,
|
||||
}
|
||||
|
||||
/// Parse an OPML document into its feeds, recursing through nested outline folders.
|
||||
pub fn parse_opml(text: &str) -> anyhow::Result<Vec<OpmlFeed>> {
|
||||
let doc = opml::OPML::from_str(text).map_err(|e| anyhow::anyhow!("invalid OPML: {e}"))?;
|
||||
let mut out = Vec::new();
|
||||
collect_outlines(&doc.body.outlines, &mut out);
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn collect_outlines(outlines: &[opml::Outline], out: &mut Vec<OpmlFeed>) {
|
||||
for o in outlines {
|
||||
if let Some(xml_url) = &o.xml_url {
|
||||
let title = o
|
||||
.title
|
||||
.clone()
|
||||
.filter(|t| !t.is_empty())
|
||||
.or_else(|| (!o.text.is_empty()).then(|| o.text.clone()));
|
||||
out.push(OpmlFeed {
|
||||
title,
|
||||
xml_url: xml_url.clone(),
|
||||
});
|
||||
}
|
||||
collect_outlines(&o.outlines, out);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn reddit_listing_rewrites_to_rss() {
|
||||
let u = normalize_input("reddit.com/r/rust").unwrap();
|
||||
assert_eq!(u.as_str(), "https://reddit.com/r/rust/.rss");
|
||||
let u = normalize_input("https://www.reddit.com/r/rust/").unwrap();
|
||||
assert_eq!(u.as_str(), "https://www.reddit.com/r/rust/.rss");
|
||||
let u = normalize_input("https://reddit.com/user/spez").unwrap();
|
||||
assert_eq!(u.as_str(), "https://reddit.com/user/spez/.rss");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reddit_existing_rss_untouched() {
|
||||
let u = normalize_input("https://reddit.com/r/rust/.rss").unwrap();
|
||||
assert_eq!(u.as_str(), "https://reddit.com/r/rust/.rss");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_reddit_passes_through_with_scheme() {
|
||||
let u = normalize_input("example.com/blog").unwrap();
|
||||
assert_eq!(u.as_str(), "https://example.com/blog");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_input_is_none() {
|
||||
assert!(normalize_input(" ").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn finds_youtube_style_feed_link() {
|
||||
let html = r#"<html><head>
|
||||
<link rel="canonical" href="https://www.youtube.com/channel/UC123">
|
||||
<link rel="alternate" type="application/rss+xml" title="RSS"
|
||||
href="https://www.youtube.com/feeds/videos.xml?channel_id=UC123">
|
||||
</head></html>"#;
|
||||
assert_eq!(
|
||||
find_feed_link(html).as_deref(),
|
||||
Some("https://www.youtube.com/feeds/videos.xml?channel_id=UC123")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn finds_atom_link_regardless_of_attr_order() {
|
||||
let html = r#"<link href="/feed.atom" type="application/atom+xml" rel="alternate">"#;
|
||||
assert_eq!(find_feed_link(html).as_deref(), Some("/feed.atom"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_feed_link_returns_none() {
|
||||
let html = r#"<head><link rel="stylesheet" href="/style.css"></head>"#;
|
||||
assert!(find_feed_link(html).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_opml_with_nested_folders() {
|
||||
let opml = r#"<?xml version="1.0"?>
|
||||
<opml version="2.0"><head><title>subs</title></head><body>
|
||||
<outline text="News">
|
||||
<outline text="Hacker News" title="Hacker News"
|
||||
type="rss" xmlUrl="https://hnrss.org/frontpage"/>
|
||||
</outline>
|
||||
<outline text="Blog" xmlUrl="https://example.com/feed.xml"/>
|
||||
</body></opml>"#;
|
||||
let feeds = parse_opml(opml).unwrap();
|
||||
assert_eq!(feeds.len(), 2);
|
||||
assert_eq!(feeds[0].xml_url, "https://hnrss.org/frontpage");
|
||||
assert_eq!(feeds[0].title.as_deref(), Some("Hacker News"));
|
||||
assert_eq!(feeds[1].xml_url, "https://example.com/feed.xml");
|
||||
assert_eq!(feeds[1].title.as_deref(), Some("Blog"));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user