feat(sources): feed discovery + OPML import for the RSS rail
Make adding RSS sources painless. Instead of requiring the exact feed URL, a user can paste any page — a blog homepage, a YouTube channel, a subreddit, a Mastodon profile — and the server resolves the concrete feed it advertises. Plus OPML bulk-import to migrate a reader export or a YouTube subscriptions list in one shot. No new SourceKind, no migration; this enriches the existing worker-polled RSS rail. New crate `newsfeed-fetch` — the one place that does outbound feed HTTP: - probe(url): normalise (+ Reddit /.rss rewrite) -> fetch -> if it parses as a feed, done; else scan the HTML <head> for <link rel=alternate type=rss/atom>, resolve the relative href, and validate. Covers YouTube/Mastodon/Substack/ WordPress/blogs, which all advertise their feed this way. - parse_opml(): recurses nested outline folders. - fetch_entries()/entry mapping moved here from the worker so both the API (discovery) and worker (polling) share a single HTTP+feed-rs path. - core: add the FeedProbe port (kept I/O-free; adapter lives in newsfeed-fetch). - api: AppState carries Arc<dyn FeedProbe>; POST /v1/sources resolves a pasted homepage to its feed; add POST /v1/sources/discover (preview) and /v1/sources/import (OPML, deduped by URL, per-feed failures collected). - worker: delegate to newsfeed_fetch::fetch_entries; drop the sourcing/ module. - web: Sources page gains paste-URL + "Find feed" preview, OPML file import with a summary, and a "polled Nm ago" hint per source; new client methods and regenerated ts-rs bindings. Verified end-to-end locally: homepage URL -> discovered feed.xml + title; create stores the resolved URL; OPML import added 1/skipped 1 dup; worker polled and both items landed in the feed. fmt/clippy/test + web build/lint green. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_016fKZzDpvjiJ9eYbPGgJvUP
This commit is contained in:
@@ -15,10 +15,10 @@ path = "src/main.rs"
|
||||
newsfeed-entities.workspace = true
|
||||
newsfeed-core.workspace = true
|
||||
newsfeed-data.workspace = true
|
||||
newsfeed-fetch.workspace = true
|
||||
|
||||
tokio.workspace = true
|
||||
reqwest.workspace = true
|
||||
feed-rs.workspace = true
|
||||
|
||||
serde.workspace = true
|
||||
chrono.workspace = true
|
||||
|
||||
@@ -6,7 +6,6 @@
|
||||
//! re-seeing an item's `external_id` is a no-op.
|
||||
|
||||
mod config;
|
||||
mod sourcing;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
@@ -25,7 +24,6 @@ use newsfeed_data::SqliteStore;
|
||||
use newsfeed_entities::interest::Interest;
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::sourcing::rss;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(
|
||||
@@ -90,7 +88,7 @@ async fn run_cycle(store: &SqliteStore, client: &Client, cfg: &Config) -> anyhow
|
||||
continue;
|
||||
};
|
||||
|
||||
match rss::fetch(client, &url).await {
|
||||
match newsfeed_fetch::fetch_entries(client, &url).await {
|
||||
Ok(entries) => {
|
||||
let interests = if let Some(cached) = interests_cache.get(&source.user_id) {
|
||||
cached
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
//! Content sourcing. Currently RSS/Atom polling; other algorithmic sources plug in here.
|
||||
|
||||
pub mod rss;
|
||||
@@ -1,80 +0,0 @@
|
||||
//! Fetch and parse an RSS/Atom feed into backend-agnostic [`FeedEntry`] values.
|
||||
|
||||
use anyhow::Context;
|
||||
use reqwest::Client;
|
||||
|
||||
use newsfeed_core::ingest::FeedEntry;
|
||||
use newsfeed_entities::item::Media;
|
||||
|
||||
/// Fetch `url` and parse it, returning one [`FeedEntry`] per feed item.
|
||||
pub async fn fetch(client: &Client, url: &str) -> anyhow::Result<Vec<FeedEntry>> {
|
||||
let bytes = client
|
||||
.get(url)
|
||||
.send()
|
||||
.await
|
||||
.with_context(|| format!("fetching {url}"))?
|
||||
.error_for_status()
|
||||
.with_context(|| format!("bad status from {url}"))?
|
||||
.bytes()
|
||||
.await
|
||||
.with_context(|| format!("reading body from {url}"))?;
|
||||
|
||||
let feed = feed_rs::parser::parse(&bytes[..]).with_context(|| format!("parsing feed {url}"))?;
|
||||
|
||||
Ok(feed.entries.into_iter().map(entry_to_feed_entry).collect())
|
||||
}
|
||||
|
||||
fn entry_to_feed_entry(entry: feed_rs::model::Entry) -> FeedEntry {
|
||||
let url = entry.links.first().map(|l| l.href.clone());
|
||||
// Prefer the entry's own id; fall back to the link so de-dup still works.
|
||||
let external_id = if entry.id.is_empty() {
|
||||
url.clone().unwrap_or_else(|| {
|
||||
entry
|
||||
.title
|
||||
.as_ref()
|
||||
.map(|t| t.content.clone())
|
||||
.unwrap_or_default()
|
||||
})
|
||||
} else {
|
||||
entry.id.clone()
|
||||
};
|
||||
let title = entry
|
||||
.title
|
||||
.map(|t| t.content)
|
||||
.unwrap_or_else(|| "(untitled)".to_string());
|
||||
let summary = entry
|
||||
.summary
|
||||
.map(|t| t.content)
|
||||
.or_else(|| entry.content.and_then(|c| c.body));
|
||||
let author = entry.authors.first().map(|p| p.name.clone());
|
||||
let tags = entry.categories.into_iter().map(|c| c.term).collect();
|
||||
let published_at = entry.published.or(entry.updated);
|
||||
|
||||
let media = entry
|
||||
.media
|
||||
.into_iter()
|
||||
.flat_map(|m| m.content)
|
||||
.filter_map(|c| {
|
||||
c.url.map(|u| Media {
|
||||
kind: c
|
||||
.content_type
|
||||
.map(|m| m.to_string())
|
||||
.unwrap_or_else(|| "media".to_string()),
|
||||
url: u.to_string(),
|
||||
width: c.width,
|
||||
height: c.height,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
FeedEntry {
|
||||
external_id,
|
||||
url,
|
||||
title,
|
||||
summary,
|
||||
author,
|
||||
tags,
|
||||
media,
|
||||
published_at,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user