feat(sources): feed discovery + OPML import for the RSS rail

Make adding RSS sources painless. Instead of requiring the exact feed URL, a user can paste any page — a blog homepage, a YouTube channel, a subreddit, a Mastodon profile — and the server resolves the concrete feed it advertises. Plus OPML bulk-import to migrate a reader export or a YouTube subscriptions list in one shot. No new SourceKind, no migration; this enriches the existing worker-polled RSS rail. New crate `newsfeed-fetch` — the one place that does outbound feed HTTP: - probe(url): normalise (+ Reddit /.rss rewrite) -> fetch -> if it parses as a feed, done; else scan the HTML <head> for <link rel=alternate type=rss/atom>, resolve the relative href, and validate. Covers YouTube/Mastodon/Substack/ WordPress/blogs, which all advertise their feed this way. - parse_opml(): recurses nested outline folders. - fetch_entries()/entry mapping moved here from the worker so both the API (discovery) and worker (polling) share a single HTTP+feed-rs path. - core: add the FeedProbe port (kept I/O-free; adapter lives in newsfeed-fetch). - api: AppState carries Arc<dyn FeedProbe>; POST /v1/sources resolves a pasted homepage to its feed; add POST /v1/sources/discover (preview) and /v1/sources/import (OPML, deduped by URL, per-feed failures collected). - worker: delegate to newsfeed_fetch::fetch_entries; drop the sourcing/ module. - web: Sources page gains paste-URL + "Find feed" preview, OPML file import with a summary, and a "polled Nm ago" hint per source; new client methods and regenerated ts-rs bindings. Verified end-to-end locally: homepage URL -> discovered feed.xml + title; create stores the resolved URL; OPML import added 1/skipped 1 dup; worker polled and both items landed in the feed. fmt/clippy/test + web build/lint green. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_016fKZzDpvjiJ9eYbPGgJvUP
2026-07-08 14:09:21 +03:00
parent ee63939151
commit 91fd03a102
21 changed files with 849 additions and 145 deletions
--- a/web/src/api/bindings/DiscoverFeedRequest.ts
+++ b/web/src/api/bindings/DiscoverFeedRequest.ts
@@ -0,0 +1,11 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+/**
+ * Request to resolve an arbitrary page URL to a concrete feed (`POST /v1/sources/discover`).
+ */
+export type DiscoverFeedRequest = { 
+/**
+ * Any page URL: a blog homepage, a YouTube channel/video, a subreddit, a Mastodon
+ * profile, or an already-direct feed URL.
+ */
+url: string, };
--- a/web/src/api/bindings/DiscoveredFeed.ts
+++ b/web/src/api/bindings/DiscoveredFeed.ts
@@ -0,0 +1,14 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+/**
+ * A feed resolved from a page URL: the concrete feed URL to poll, plus its title.
+ */
+export type DiscoveredFeed = { 
+/**
+ * The concrete RSS/Atom URL the worker should poll.
+ */
+feed_url: string, 
+/**
+ * The feed's own title, when the parsed feed advertises one.
+ */
+title: string | null, };
--- a/web/src/api/bindings/OpmlImport.ts
+++ b/web/src/api/bindings/OpmlImport.ts
@@ -0,0 +1,10 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+/**
+ * Request to bulk-import RSS sources from an OPML document (`POST /v1/sources/import`).
+ */
+export type OpmlImport = { 
+/**
+ * The raw OPML XML (e.g. exported from another reader or a subscriptions manager).
+ */
+opml: string, };
--- a/web/src/api/bindings/OpmlImportResult.ts
+++ b/web/src/api/bindings/OpmlImportResult.ts
@@ -0,0 +1,18 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+/**
+ * Outcome of an OPML import.
+ */
+export type OpmlImportResult = { 
+/**
+ * Feeds newly added as sources.
+ */
+added: number, 
+/**
+ * Feeds skipped because a source with that URL already existed (or repeated in the file).
+ */
+skipped: number, 
+/**
+ * Feeds that failed to import, each as `"<url>: <reason>"`.
+ */
+failed: Array<string>, };
--- a/web/src/api/client.ts
+++ b/web/src/api/client.ts
@@ -10,6 +10,8 @@ import type { CreateApiToken } from './bindings/CreateApiToken';
 import type { CreatedApiToken } from './bindings/CreatedApiToken';
 import type { Source } from './bindings/Source';
 import type { NewSource } from './bindings/NewSource';
+import type { DiscoveredFeed } from './bindings/DiscoveredFeed';
+import type { OpmlImportResult } from './bindings/OpmlImportResult';
 import type { Interest } from './bindings/Interest';
 import type { UpsertInterest } from './bindings/UpsertInterest';
 import type { FeedPage } from './bindings/FeedPage';
@@ -71,6 +73,8 @@ export const api = {
  listSources: () => request<Source[]>('GET', '/v1/sources'),
  createSource: (s: NewSource) => request<Source>('POST', '/v1/sources', s),
  deleteSource: (id: string) => request<void>('DELETE', `/v1/sources/${id}`),
+  discoverFeed: (url: string) => request<DiscoveredFeed>('POST', '/v1/sources/discover', { url }),
+  importOpml: (opml: string) => request<OpmlImportResult>('POST', '/v1/sources/import', { opml }),

  // interests
  listInterests: () => request<Interest[]>('GET', '/v1/interests'),
--- a/web/src/routes/Sources.tsx
+++ b/web/src/routes/Sources.tsx
@@ -1,9 +1,19 @@
-import { useState, type FormEvent } from 'react';
+import { useRef, useState, type ChangeEvent, type FormEvent } from 'react';
 import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';

 import { api, ApiError } from '../api/client';
 import type { SourceKind } from '../api/bindings/SourceKind';

+/** Human "polled 5m ago" hint so the user can see the RSS rail actually working. */
+function pollStatus(iso: string | null): string {
+  if (!iso) return 'never polled';
+  const secs = (Date.now() - new Date(iso).getTime()) / 1000;
+  if (secs < 60) return 'polled just now';
+  if (secs < 3600) return `polled ${Math.floor(secs / 60)}m ago`;
+  if (secs < 86400) return `polled ${Math.floor(secs / 3600)}h ago`;
+  return `polled ${Math.floor(secs / 86400)}d ago`;
+}
+
 export function Sources() {
  const qc = useQueryClient();
  const sources = useQuery({ queryKey: ['sources'], queryFn: api.listSources });
@@ -12,28 +22,67 @@ export function Sources() {
  const [name, setName] = useState('');
  const [url, setUrl] = useState('');
  const [weight, setWeight] = useState(1);
+  const fileInput = useRef<HTMLInputElement>(null);
+
+  // Preview of the feed a pasted URL resolves to (populated by "Find feed").
+  const discover = useMutation({
+    mutationFn: () => api.discoverFeed(url),
+    onSuccess: (d) => {
+      if (d.title && !name.trim()) setName(d.title);
+    },
+  });

  const create = useMutation({
-    mutationFn: () => api.createSource({ kind, name, url: kind === 'rss' ? url : null, weight }),
+    // Send the resolved feed URL when we have one (fast path); otherwise the raw URL,
+    // which the server resolves itself. Agentic sources carry no URL.
+    mutationFn: () =>
+      api.createSource({
+        kind,
+        name,
+        url: kind === 'rss' ? (discover.data?.feed_url ?? url) : null,
+        weight,
+      }),
    onSuccess: () => {
      setName('');
      setUrl('');
      setWeight(1);
+      discover.reset();
      qc.invalidateQueries({ queryKey: ['sources'] });
    },
  });

+  const importOpml = useMutation({
+    mutationFn: (opml: string) => api.importOpml(opml),
+    onSuccess: () => qc.invalidateQueries({ queryKey: ['sources'] }),
+  });
+
  const remove = useMutation({
    mutationFn: (id: string) => api.deleteSource(id),
    onSuccess: () => qc.invalidateQueries({ queryKey: ['sources'] }),
  });

+  // Editing the URL invalidates a previous discovery preview.
+  const onUrlChange = (e: ChangeEvent<HTMLInputElement>) => {
+    setUrl(e.target.value);
+    if (discover.data || discover.error) discover.reset();
+  };
+
+  const onFile = async (e: ChangeEvent<HTMLInputElement>) => {
+    const file = e.target.files?.[0];
+    e.target.value = ''; // allow re-importing the same file
+    if (file) importOpml.mutate(await file.text());
+  };
+
  const onSubmit = (e: FormEvent) => {
    e.preventDefault();
    create.mutate();
  };

-  const error = create.error instanceof ApiError ? create.error.message : null;
+  const errOf = (e: unknown) => (e instanceof ApiError ? e.message : e ? String(e) : null);
+  const createError = errOf(create.error);
+  const discoverError = errOf(discover.error);
+  const importError = errOf(importOpml.error);
+  const summary = importOpml.data;

  return (
    <section>
@@ -57,16 +106,37 @@ export function Sources() {
        </div>

        {kind === 'rss' && (
-          <label>
-            Feed URL
-            <input
-              type="url"
-              value={url}
-              onChange={(e) => setUrl(e.target.value)}
-              placeholder="https://example.com/feed.xml"
-              required
-            />
-          </label>
+          <>
+            <label>
+              Page or feed URL
+              <div className="row">
+                <input
+                  className="grow"
+                  type="url"
+                  value={url}
+                  onChange={onUrlChange}
+                  placeholder="https://example.com — a homepage, YouTube channel, subreddit, or feed"
+                  required
+                />
+                <button
+                  type="button"
+                  className="ghost"
+                  disabled={!url || discover.isPending}
+                  onClick={() => discover.mutate()}
+                >
+                  {discover.isPending ? 'Finding…' : 'Find feed'}
+                </button>
+              </div>
+            </label>
+
+            {discover.data && (
+              <p className="muted small">
+                Found feed{discover.data.title ? ` “${discover.data.title}”` : ''}:{' '}
+                <code>{discover.data.feed_url}</code>
+              </p>
+            )}
+            {discoverError && <p className="error">{discoverError}</p>}
+          </>
        )}

        <label>
@@ -81,12 +151,52 @@ export function Sources() {
          />
        </label>

-        {error && <p className="error">{error}</p>}
+        {createError && <p className="error">{createError}</p>}
        <button type="submit" className="primary" disabled={create.isPending}>
-          Add source
+          {create.isPending ? 'Adding…' : 'Add source'}
        </button>
      </form>

+      <div className="card">
+        <div className="row">
+          <div className="grow">
+            <strong>Import OPML</strong>
+            <p className="muted small">
+              Bulk-add feeds from a reader export or a YouTube subscriptions OPML. Duplicates are skipped.
+            </p>
+          </div>
+          <button
+            type="button"
+            className="ghost"
+            disabled={importOpml.isPending}
+            onClick={() => fileInput.current?.click()}
+          >
+            {importOpml.isPending ? 'Importing…' : 'Choose OPML file'}
+          </button>
+          <input
+            ref={fileInput}
+            type="file"
+            accept=".opml,.xml,text/xml,text/x-opml"
+            hidden
+            onChange={onFile}
+          />
+        </div>
+        {summary && (
+          <p className="muted small">
+            Imported {summary.added} added, {summary.skipped} skipped
+            {summary.failed.length > 0 ? `, ${summary.failed.length} failed` : ''}.
+          </p>
+        )}
+        {summary && summary.failed.length > 0 && (
+          <ul className="muted small">
+            {summary.failed.map((f) => (
+              <li key={f}>{f}</li>
+            ))}
+          </ul>
+        )}
+        {importError && <p className="error">{importError}</p>}
+      </div>
+
      <ul className="list">
        {sources.data?.map((s) => (
          <li key={s.id} className="card list-row">
@@ -100,6 +210,7 @@ export function Sources() {
              )}
            </div>
            <div className="list-row-end">
+              {s.kind === 'rss' && <span className="muted small">{pollStatus(s.last_polled_at)}</span>}
              <span className="muted small">w {s.weight.toFixed(2)}</span>
              <button className="ghost" onClick={() => remove.mutate(s.id)}>
                Delete