Make adding RSS sources painless. Instead of requiring the exact feed URL, a user can paste any page — a blog homepage, a YouTube channel, a subreddit, a Mastodon profile — and the server resolves the concrete feed it advertises. Plus OPML bulk-import to migrate a reader export or a YouTube subscriptions list in one shot. No new SourceKind, no migration; this enriches the existing worker-polled RSS rail. New crate `newsfeed-fetch` — the one place that does outbound feed HTTP: - probe(url): normalise (+ Reddit /.rss rewrite) -> fetch -> if it parses as a feed, done; else scan the HTML <head> for <link rel=alternate type=rss/atom>, resolve the relative href, and validate. Covers YouTube/Mastodon/Substack/ WordPress/blogs, which all advertise their feed this way. - parse_opml(): recurses nested outline folders. - fetch_entries()/entry mapping moved here from the worker so both the API (discovery) and worker (polling) share a single HTTP+feed-rs path. - core: add the FeedProbe port (kept I/O-free; adapter lives in newsfeed-fetch). - api: AppState carries Arc<dyn FeedProbe>; POST /v1/sources resolves a pasted homepage to its feed; add POST /v1/sources/discover (preview) and /v1/sources/import (OPML, deduped by URL, per-feed failures collected). - worker: delegate to newsfeed_fetch::fetch_entries; drop the sourcing/ module. - web: Sources page gains paste-URL + "Find feed" preview, OPML file import with a summary, and a "polled Nm ago" hint per source; new client methods and regenerated ts-rs bindings. Verified end-to-end locally: homepage URL -> discovered feed.xml + title; create stores the resolved URL; OPML import added 1/skipped 1 dup; worker polled and both items landed in the feed. fmt/clippy/test + web build/lint green. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_016fKZzDpvjiJ9eYbPGgJvUP
132 lines
4.1 KiB
Rust
132 lines
4.1 KiB
Rust
//! Content sources. A source is where candidate items come from — an RSS/Atom feed the
|
|
//! worker polls, or an agentic/external producer that pushes candidates to the ingest
|
|
//! endpoint under a user's API token.
|
|
|
|
use chrono::{DateTime, Utc};
|
|
use serde::{Deserialize, Serialize};
|
|
use ts_rs::TS;
|
|
use uuid::Uuid;
|
|
|
|
use crate::error::{Error, Result};
|
|
|
|
/// How a source produces candidates.
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, TS)]
|
|
#[serde(rename_all = "snake_case")]
|
|
#[ts(export)]
|
|
pub enum SourceKind {
|
|
/// An RSS/Atom feed the worker polls on a schedule.
|
|
Rss,
|
|
/// A producer that pushes candidates to `POST /v1/ingest/candidates`
|
|
/// (agentic workloads, scrapers, bespoke algorithms).
|
|
Agentic,
|
|
}
|
|
|
|
impl SourceKind {
|
|
/// Stable string form used in the database and on the wire.
|
|
pub fn as_str(self) -> &'static str {
|
|
match self {
|
|
SourceKind::Rss => "rss",
|
|
SourceKind::Agentic => "agentic",
|
|
}
|
|
}
|
|
|
|
/// Parse from the database/wire representation.
|
|
pub fn parse(s: &str) -> Result<Self> {
|
|
match s {
|
|
"rss" => Ok(SourceKind::Rss),
|
|
"agentic" => Ok(SourceKind::Agentic),
|
|
other => Err(Error::invalid(
|
|
"kind",
|
|
format!("unknown source kind: {other}"),
|
|
)),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// A user's configured content source.
|
|
#[derive(Debug, Clone, Serialize, Deserialize, TS)]
|
|
#[ts(export)]
|
|
pub struct Source {
|
|
pub id: Uuid,
|
|
pub user_id: Uuid,
|
|
pub kind: SourceKind,
|
|
pub name: String,
|
|
/// Poll URL for [`SourceKind::Rss`]; unused/optional for agentic sources.
|
|
pub url: Option<String>,
|
|
/// Baseline weight applied to every item from this source, `0.0..=1.0`.
|
|
pub weight: f64,
|
|
pub enabled: bool,
|
|
/// When the worker last polled this source (rss only).
|
|
pub last_polled_at: Option<DateTime<Utc>>,
|
|
pub created_at: DateTime<Utc>,
|
|
}
|
|
|
|
/// Request to create a source.
|
|
#[derive(Debug, Clone, Deserialize, TS)]
|
|
#[ts(export)]
|
|
pub struct NewSource {
|
|
pub kind: SourceKind,
|
|
pub name: String,
|
|
pub url: Option<String>,
|
|
/// Defaults to `1.0` when omitted.
|
|
#[serde(default)]
|
|
pub weight: Option<f64>,
|
|
}
|
|
|
|
/// Request to resolve an arbitrary page URL to a concrete feed (`POST /v1/sources/discover`).
|
|
#[derive(Debug, Clone, Deserialize, TS)]
|
|
#[ts(export)]
|
|
pub struct DiscoverFeedRequest {
|
|
/// Any page URL: a blog homepage, a YouTube channel/video, a subreddit, a Mastodon
|
|
/// profile, or an already-direct feed URL.
|
|
pub url: String,
|
|
}
|
|
|
|
/// A feed resolved from a page URL: the concrete feed URL to poll, plus its title.
|
|
#[derive(Debug, Clone, Serialize, Deserialize, TS)]
|
|
#[ts(export)]
|
|
pub struct DiscoveredFeed {
|
|
/// The concrete RSS/Atom URL the worker should poll.
|
|
pub feed_url: String,
|
|
/// The feed's own title, when the parsed feed advertises one.
|
|
pub title: Option<String>,
|
|
}
|
|
|
|
/// Request to bulk-import RSS sources from an OPML document (`POST /v1/sources/import`).
|
|
#[derive(Debug, Clone, Deserialize, TS)]
|
|
#[ts(export)]
|
|
pub struct OpmlImport {
|
|
/// The raw OPML XML (e.g. exported from another reader or a subscriptions manager).
|
|
pub opml: String,
|
|
}
|
|
|
|
/// Outcome of an OPML import.
|
|
#[derive(Debug, Clone, Default, Serialize, Deserialize, TS)]
|
|
#[ts(export)]
|
|
pub struct OpmlImportResult {
|
|
/// Feeds newly added as sources.
|
|
pub added: u32,
|
|
/// Feeds skipped because a source with that URL already existed (or repeated in the file).
|
|
pub skipped: u32,
|
|
/// Feeds that failed to import, each as `"<url>: <reason>"`.
|
|
pub failed: Vec<String>,
|
|
}
|
|
|
|
impl NewSource {
|
|
/// Validate a create-source request.
|
|
pub fn validate(&self) -> Result<()> {
|
|
if self.name.trim().is_empty() {
|
|
return Err(Error::invalid("name", "must not be empty"));
|
|
}
|
|
if self.kind == SourceKind::Rss && self.url.as_deref().unwrap_or("").is_empty() {
|
|
return Err(Error::invalid("url", "rss sources require a url"));
|
|
}
|
|
if let Some(w) = self.weight {
|
|
if !(0.0..=1.0).contains(&w) {
|
|
return Err(Error::invalid("weight", "must be within 0.0..=1.0"));
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|