feat(sources): feed discovery + OPML import for the RSS rail

Make adding RSS sources painless. Instead of requiring the exact feed URL, a user can paste any page — a blog homepage, a YouTube channel, a subreddit, a Mastodon profile — and the server resolves the concrete feed it advertises. Plus OPML bulk-import to migrate a reader export or a YouTube subscriptions list in one shot. No new SourceKind, no migration; this enriches the existing worker-polled RSS rail. New crate `newsfeed-fetch` — the one place that does outbound feed HTTP: - probe(url): normalise (+ Reddit /.rss rewrite) -> fetch -> if it parses as a feed, done; else scan the HTML <head> for <link rel=alternate type=rss/atom>, resolve the relative href, and validate. Covers YouTube/Mastodon/Substack/ WordPress/blogs, which all advertise their feed this way. - parse_opml(): recurses nested outline folders. - fetch_entries()/entry mapping moved here from the worker so both the API (discovery) and worker (polling) share a single HTTP+feed-rs path. - core: add the FeedProbe port (kept I/O-free; adapter lives in newsfeed-fetch). - api: AppState carries Arc<dyn FeedProbe>; POST /v1/sources resolves a pasted homepage to its feed; add POST /v1/sources/discover (preview) and /v1/sources/import (OPML, deduped by URL, per-feed failures collected). - worker: delegate to newsfeed_fetch::fetch_entries; drop the sourcing/ module. - web: Sources page gains paste-URL + "Find feed" preview, OPML file import with a summary, and a "polled Nm ago" hint per source; new client methods and regenerated ts-rs bindings. Verified end-to-end locally: homepage URL -> discovered feed.xml + title; create stores the resolved URL; OPML import added 1/skipped 1 dup; worker polled and both items landed in the feed. fmt/clippy/test + web build/lint green. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_016fKZzDpvjiJ9eYbPGgJvUP
2026-07-08 14:09:21 +03:00
parent ee63939151
commit 91fd03a102
21 changed files with 849 additions and 145 deletions
--- a/crates/newsfeed-api/Cargo.toml
+++ b/crates/newsfeed-api/Cargo.toml
@@ -15,6 +15,7 @@ path = "src/main.rs"
 newsfeed-entities.workspace = true
 newsfeed-core.workspace = true
 newsfeed-data.workspace = true
+newsfeed-fetch.workspace = true

 tokio.workspace = true
 axum.workspace = true
--- a/crates/newsfeed-api/src/main.rs
+++ b/crates/newsfeed-api/src/main.rs
@@ -47,8 +47,11 @@ async fn main() -> anyhow::Result<()> {
        .await
        .context("opening database")?;

+    let prober =
+        std::sync::Arc::new(newsfeed_fetch::HttpFeedProber::new().context("building feed prober")?);
+
    let bind = config.bind;
-    let state = AppState::new(store, config);
+    let state = AppState::new(store, config, prober);
    let app = routes::router(state);

    let listener = TcpListener::bind(bind)
--- a/crates/newsfeed-api/src/routes/mod.rs
+++ b/crates/newsfeed-api/src/routes/mod.rs
@@ -33,6 +33,8 @@ pub fn router(state: AppState) -> Router {
        .route("/feed/signals", post(feed::post_signal))
        // sources
        .route("/sources", get(sources::list).post(sources::create))
+        .route("/sources/discover", post(sources::discover))
+        .route("/sources/import", post(sources::import))
        .route("/sources/{id}", delete(sources::remove))
        // interests
        .route("/interests", get(interests::list).put(interests::upsert))
--- a/crates/newsfeed-api/src/routes/sources.rs
+++ b/crates/newsfeed-api/src/routes/sources.rs
@@ -1,4 +1,6 @@
-//! CRUD for a user's content sources.
+//! CRUD for a user's content sources, plus feed discovery and OPML import.
+
+use std::collections::HashSet;

 use axum::Json;
 use axum::extract::{Path, State};
@@ -6,10 +8,13 @@ use axum::http::StatusCode;
 use uuid::Uuid;

 use newsfeed_core::ports::SourceStore;
-use newsfeed_entities::source::{NewSource, Source};
+use newsfeed_entities::source::{
+    DiscoverFeedRequest, DiscoveredFeed, NewSource, OpmlImport, OpmlImportResult, Source,
+    SourceKind,
+};

 use crate::auth::CurrentUser;
-use crate::error::ApiResult;
+use crate::error::{ApiError, ApiResult};
 use crate::state::AppState;

 /// `GET /v1/sources` — the user's sources.
@@ -20,17 +25,74 @@ pub async fn list(
    Ok(Json(app.store.list_sources(user.id).await?))
 }

-/// `POST /v1/sources` — add a source.
+/// `POST /v1/sources` — add a source. For RSS the supplied URL may be any page (homepage,
+/// channel, subreddit …); it is resolved to the concrete feed it advertises and the
+/// resolved feed URL is what gets stored.
 pub async fn create(
    CurrentUser(user): CurrentUser,
    State(app): State<AppState>,
-    Json(new): Json<NewSource>,
+    Json(mut new): Json<NewSource>,
 ) -> ApiResult<(StatusCode, Json<Source>)> {
    new.validate()?;
+    if new.kind == SourceKind::Rss {
+        let input = new.url.clone().unwrap_or_default();
+        let discovered = app.prober.probe(&input).await?;
+        new.url = Some(discovered.feed_url);
+    }
    let source = app.store.create_source(user.id, &new).await?;
    Ok((StatusCode::CREATED, Json(source)))
 }

+/// `POST /v1/sources/discover` — resolve a page URL to a feed without saving anything, so
+/// the UI can preview the feed URL + title before the user commits.
+pub async fn discover(
+    CurrentUser(_user): CurrentUser,
+    State(app): State<AppState>,
+    Json(req): Json<DiscoverFeedRequest>,
+) -> ApiResult<Json<DiscoveredFeed>> {
+    Ok(Json(app.prober.probe(&req.url).await?))
+}
+
+/// `POST /v1/sources/import` — bulk-add RSS sources from an OPML document. Feeds already
+/// present (by URL) or repeated within the file are skipped; per-feed failures are
+/// collected rather than aborting the whole import.
+pub async fn import(
+    CurrentUser(user): CurrentUser,
+    State(app): State<AppState>,
+    Json(req): Json<OpmlImport>,
+) -> ApiResult<Json<OpmlImportResult>> {
+    let feeds = newsfeed_fetch::parse_opml(&req.opml)
+        .map_err(|e| ApiError::bad_request(format!("{e:#}")))?;
+
+    // Dedup against existing sources and against repeats within the file.
+    let mut seen: HashSet<String> = app
+        .store
+        .list_sources(user.id)
+        .await?
+        .into_iter()
+        .filter_map(|s| s.url)
+        .collect();
+
+    let mut result = OpmlImportResult::default();
+    for feed in feeds {
+        if !seen.insert(feed.xml_url.clone()) {
+            result.skipped += 1;
+            continue;
+        }
+        let new = NewSource {
+            kind: SourceKind::Rss,
+            name: feed.title.unwrap_or_else(|| feed.xml_url.clone()),
+            url: Some(feed.xml_url.clone()),
+            weight: None,
+        };
+        match app.store.create_source(user.id, &new).await {
+            Ok(_) => result.added += 1,
+            Err(e) => result.failed.push(format!("{}: {e}", feed.xml_url)),
+        }
+    }
+    Ok(Json(result))
+}
+
 /// `DELETE /v1/sources/{id}` — remove a source.
 pub async fn remove(
    CurrentUser(user): CurrentUser,
--- a/crates/newsfeed-api/src/state.rs
+++ b/crates/newsfeed-api/src/state.rs
@@ -3,6 +3,7 @@

 use std::sync::Arc;

+use newsfeed_core::ports::FeedProbe;
 use newsfeed_data::SqliteStore;

 use crate::config::Config;
@@ -12,13 +13,16 @@ use crate::config::Config;
 pub struct AppState {
    pub store: Arc<SqliteStore>,
    pub config: Arc<Config>,
+    /// Resolves page URLs to feeds when a user adds an RSS source.
+    pub prober: Arc<dyn FeedProbe>,
 }

 impl AppState {
-    pub fn new(store: SqliteStore, config: Config) -> Self {
+    pub fn new(store: SqliteStore, config: Config, prober: Arc<dyn FeedProbe>) -> Self {
        Self {
            store: Arc::new(store),
            config: Arc::new(config),
+            prober,
        }
    }
 }