feat(sources): feed discovery + OPML import for the RSS rail

Make adding RSS sources painless. Instead of requiring the exact feed URL, a user can paste any page — a blog homepage, a YouTube channel, a subreddit, a Mastodon profile — and the server resolves the concrete feed it advertises. Plus OPML bulk-import to migrate a reader export or a YouTube subscriptions list in one shot. No new SourceKind, no migration; this enriches the existing worker-polled RSS rail. New crate `newsfeed-fetch` — the one place that does outbound feed HTTP: - probe(url): normalise (+ Reddit /.rss rewrite) -> fetch -> if it parses as a feed, done; else scan the HTML <head> for <link rel=alternate type=rss/atom>, resolve the relative href, and validate. Covers YouTube/Mastodon/Substack/ WordPress/blogs, which all advertise their feed this way. - parse_opml(): recurses nested outline folders. - fetch_entries()/entry mapping moved here from the worker so both the API (discovery) and worker (polling) share a single HTTP+feed-rs path. - core: add the FeedProbe port (kept I/O-free; adapter lives in newsfeed-fetch). - api: AppState carries Arc<dyn FeedProbe>; POST /v1/sources resolves a pasted homepage to its feed; add POST /v1/sources/discover (preview) and /v1/sources/import (OPML, deduped by URL, per-feed failures collected). - worker: delegate to newsfeed_fetch::fetch_entries; drop the sourcing/ module. - web: Sources page gains paste-URL + "Find feed" preview, OPML file import with a summary, and a "polled Nm ago" hint per source; new client methods and regenerated ts-rs bindings. Verified end-to-end locally: homepage URL -> discovered feed.xml + title; create stores the resolved URL; OPML import added 1/skipped 1 dup; worker polled and both items landed in the feed. fmt/clippy/test + web build/lint green. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_016fKZzDpvjiJ9eYbPGgJvUP
2026-07-08 14:09:21 +03:00
parent ee63939151
commit 91fd03a102
21 changed files with 849 additions and 145 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -120,7 +120,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.118",
 ]

 [[package]]
@@ -217,7 +217,7 @@ checksum = "7aa268c23bfbbd2c4363b9cd302a4f504fb2a9dfe7e3451d66f35dd392e20aca"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.118",
 ]

 [[package]]
@@ -361,7 +361,7 @@ dependencies = [
 "heck",
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.118",
 ]

 [[package]]
@@ -518,7 +518,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.118",
 ]

 [[package]]
@@ -776,6 +776,31 @@ dependencies = [
 "wasm-bindgen",
 ]

+[[package]]
+name = "hard-xml"
+version = "1.41.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b07b8ba970e18a03dbb79f6786b6e4d6f198a0ac839aa5182017001bb8dee17"
+dependencies = [
+ "hard-xml-derive",
+ "jetscii",
+ "lazy_static",
+ "memchr",
+ "xmlparser",
+]
+
+[[package]]
+name = "hard-xml-derive"
+version = "1.41.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0c43e7c3212bd992c11b6b9796563388170950521ae8487f5cdf6f6e792f1c8"
+dependencies = [
+ "bitflags",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "hashbrown"
 version = "0.15.5"
@@ -1106,6 +1131,12 @@ version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"

+[[package]]
+name = "jetscii"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47f142fe24a9c9944451e8349de0a56af5f3e7226dc46f3ed4d4ecc0b85af75e"
+
 [[package]]
 name = "js-sys"
 version = "0.3.103"
@@ -1267,9 +1298,10 @@ dependencies = [
 "newsfeed-core",
 "newsfeed-data",
 "newsfeed-entities",
+ "newsfeed-fetch",
 "serde",
 "serde_json",
- "thiserror",
+ "thiserror 2.0.18",
 "tokio",
 "tower",
 "tower-http",
@@ -1290,7 +1322,7 @@ dependencies = [
 "rand 0.8.6",
 "serde",
 "sha2",
- "thiserror",
+ "thiserror 2.0.18",
 "uuid",
 ]

@@ -1317,11 +1349,28 @@ dependencies = [
 "chrono",
 "serde",
 "serde_json",
- "thiserror",
+ "thiserror 2.0.18",
 "ts-rs",
 "uuid",
 ]

+[[package]]
+name = "newsfeed-fetch"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "async-trait",
+ "chrono",
+ "feed-rs",
+ "newsfeed-core",
+ "newsfeed-entities",
+ "opml",
+ "regex",
+ "reqwest",
+ "tokio",
+ "url",
+]
+
 [[package]]
 name = "newsfeed-worker"
 version = "0.1.0"
@@ -1329,11 +1378,11 @@ dependencies = [
 "anyhow",
 "chrono",
 "clap",
- "feed-rs",
 "figment",
 "newsfeed-core",
 "newsfeed-data",
 "newsfeed-entities",
+ "newsfeed-fetch",
 "reqwest",
 "serde",
 "tokio",
@@ -1408,6 +1457,17 @@ version = "1.70.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"

+[[package]]
+name = "opml"
+version = "1.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df2f96426c857a92676dc29a9e2a181eb39321047ac994491c69eae01619ddf2"
+dependencies = [
+ "hard-xml",
+ "serde",
+ "thiserror 1.0.69",
+]
+
 [[package]]
 name = "parking"
 version = "2.2.1"
@@ -1468,7 +1528,7 @@ dependencies = [
 "proc-macro2",
 "proc-macro2-diagnostics",
 "quote",
- "syn",
+ "syn 2.0.118",
 ]

 [[package]]
@@ -1560,7 +1620,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.118",
 "version_check",
 "yansi",
 ]
@@ -1589,7 +1649,7 @@ dependencies = [
 "rustc-hash",
 "rustls",
 "socket2",
- "thiserror",
+ "thiserror 2.0.18",
 "tokio",
 "tracing",
 "web-time",
@@ -1611,7 +1671,7 @@ dependencies = [
 "rustls",
 "rustls-pki-types",
 "slab",
- "thiserror",
+ "thiserror 2.0.18",
 "tinyvec",
 "tracing",
 "web-time",
@@ -1942,7 +2002,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.118",
 ]

 [[package]]
@@ -2145,7 +2205,7 @@ dependencies = [
 "serde_json",
 "sha2",
 "smallvec",
- "thiserror",
+ "thiserror 2.0.18",
 "tokio",
 "tokio-stream",
 "tracing",
@@ -2164,7 +2224,7 @@ dependencies = [
 "quote",
 "sqlx-core",
 "sqlx-macros-core",
- "syn",
+ "syn 2.0.118",
 ]

 [[package]]
@@ -2187,7 +2247,7 @@ dependencies = [
 "sqlx-mysql",
 "sqlx-postgres",
 "sqlx-sqlite",
- "syn",
+ "syn 2.0.118",
 "tokio",
 "url",
 ]
@@ -2230,7 +2290,7 @@ dependencies = [
 "smallvec",
 "sqlx-core",
 "stringprep",
- "thiserror",
+ "thiserror 2.0.18",
 "tracing",
 "uuid",
 "whoami",
@@ -2269,7 +2329,7 @@ dependencies = [
 "smallvec",
 "sqlx-core",
 "stringprep",
- "thiserror",
+ "thiserror 2.0.18",
 "tracing",
 "uuid",
 "whoami",
@@ -2295,7 +2355,7 @@ dependencies = [
 "serde",
 "serde_urlencoded",
 "sqlx-core",
- "thiserror",
+ "thiserror 2.0.18",
 "tracing",
 "url",
 "uuid",
@@ -2330,6 +2390,17 @@ version = "2.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"

+[[package]]
+name = "syn"
+version = "1.0.109"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
 [[package]]
 name = "syn"
 version = "2.0.118"
@@ -2358,7 +2429,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.118",
 ]

 [[package]]
@@ -2370,13 +2441,33 @@ dependencies = [
 "winapi-util",
 ]

+[[package]]
+name = "thiserror"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
+dependencies = [
+ "thiserror-impl 1.0.69",
+]
+
 [[package]]
 name = "thiserror"
 version = "2.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
 dependencies = [
- "thiserror-impl",
+ "thiserror-impl 2.0.18",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.118",
 ]

 [[package]]
@@ -2387,7 +2478,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.118",
 ]

 [[package]]
@@ -2449,7 +2540,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.118",
 ]

 [[package]]
@@ -2611,7 +2702,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.118",
 ]

 [[package]]
@@ -2681,7 +2772,7 @@ dependencies = [
 "chrono",
 "lazy_static",
 "serde_json",
- "thiserror",
+ "thiserror 2.0.18",
 "ts-rs-macros",
 "uuid",
 ]
@@ -2694,7 +2785,7 @@ checksum = "0e9d8656589772eeec2cf7a8264d9cda40fb28b9bc53118ceb9e8c07f8f38730"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.118",
 "termcolor",
 ]

@@ -2711,7 +2802,7 @@ dependencies = [
 "log",
 "rand 0.9.4",
 "sha1",
- "thiserror",
+ "thiserror 2.0.18",
 ]

 [[package]]
@@ -2889,7 +2980,7 @@ dependencies = [
 "bumpalo",
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.118",
 "wasm-bindgen-shared",
 ]

@@ -2980,7 +3071,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.118",
 ]

 [[package]]
@@ -2991,7 +3082,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.118",
 ]

 [[package]]
@@ -3196,6 +3287,12 @@ version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"

+[[package]]
+name = "xmlparser"
+version = "0.13.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4"
+
 [[package]]
 name = "yansi"
 version = "1.0.1"
@@ -3221,7 +3318,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.118",
 "synstructure",
 ]

@@ -3242,7 +3339,7 @@ checksum = "4714fd92cf900833d49538023a9b3915155210801d1c1169eba513b2addefd71"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.118",
 ]

 [[package]]
@@ -3262,7 +3359,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.118",
 "synstructure",
 ]

@@ -3302,7 +3399,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.118",
 ]

 [[package]]