newsfeed/crates/newsfeed-worker/src/sourcing/rss.rs

//! Fetch and parse an RSS/Atom feed into backend-agnostic [`FeedEntry`] values.

use anyhow::Context;
use reqwest::Client;

use newsfeed_core::ingest::FeedEntry;
use newsfeed_entities::item::Media;

/// Fetch `url` and parse it, returning one [`FeedEntry`] per feed item.
pub async fn fetch(client: &Client, url: &str) -> anyhow::Result<Vec<FeedEntry>> {
    let bytes = client
        .get(url)
        .send()
        .await
        .with_context(|| format!("fetching {url}"))?
        .error_for_status()
        .with_context(|| format!("bad status from {url}"))?
        .bytes()
        .await
        .with_context(|| format!("reading body from {url}"))?;

    let feed = feed_rs::parser::parse(&bytes[..]).with_context(|| format!("parsing feed {url}"))?;

    Ok(feed.entries.into_iter().map(entry_to_feed_entry).collect())
}

fn entry_to_feed_entry(entry: feed_rs::model::Entry) -> FeedEntry {
    let url = entry.links.first().map(|l| l.href.clone());
    // Prefer the entry's own id; fall back to the link so de-dup still works.
    let external_id = if entry.id.is_empty() {
        url.clone().unwrap_or_else(|| {
            entry
                .title
                .as_ref()
                .map(|t| t.content.clone())
                .unwrap_or_default()
        })
    } else {
        entry.id.clone()
    };
    let title = entry
        .title
        .map(|t| t.content)
        .unwrap_or_else(|| "(untitled)".to_string());
    let summary = entry
        .summary
        .map(|t| t.content)
        .or_else(|| entry.content.and_then(|c| c.body));
    let author = entry.authors.first().map(|p| p.name.clone());
    let tags = entry.categories.into_iter().map(|c| c.term).collect();
    let published_at = entry.published.or(entry.updated);

    let media = entry
        .media
        .into_iter()
        .flat_map(|m| m.content)
        .filter_map(|c| {
            c.url.map(|u| Media {
                kind: c
                    .content_type
                    .map(|m| m.to_string())
                    .unwrap_or_else(|| "media".to_string()),
                url: u.to_string(),
                width: c.width,
                height: c.height,
            })
        })
        .collect();

    FeedEntry {
        external_id,
        url,
        title,
        summary,
        author,
        tags,
        media,
        published_at,
    }
}