rob thijssen
45ceec2ec7
Adds the first ingestion source. Page-1 polling is ETag-conditional
(304s don't count against rate limit); the very first run paginates
back through Link "next" pages up to a 10-page safety cap so the
table starts populated rather than waiting for new activity.
Hits /users/{user}/events/public — works without auth, returns the
right scope for a public timeline. Token (GITHUB_TOKEN) is optional;
when present it raises the rate limit from 60 to 5000/hr.
New plumbing:
moments-core::sources
- EventSource trait (poll() -> count)
- PollerStateStore trait (etag persistence port)
- run_poller driver: tokio interval + jittered exponential backoff
moments-data::github
- GithubSource impl, raw payload preserved as JSONB
- parse_link_next for pagination
- 4 unit tests covering parser + Link parsing
migration 0002_poller_state.sql
- one row per source: source, etag, last_modified, last_fetched
Worker binary spawns one tokio task per source (just github for now)
and aborts on SIGINT. Verified by smoke-curling the upstream endpoint:
ETag and Link headers are present; payload shape matches the parser.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
38 lines
1.3 KiB
TOML
38 lines
1.3 KiB
TOML
[workspace]
|
|
resolver = "3"
|
|
members = ["crates/*"]
|
|
|
|
[workspace.package]
|
|
version = "0.1.0"
|
|
edition = "2024"
|
|
rust-version = "1.85"
|
|
license = "GPL-3.0-or-later"
|
|
authors = ["Rob Thijssen <rthijssen@gmail.com>"]
|
|
|
|
[workspace.dependencies]
|
|
# entities
|
|
serde = { version = "1", features = ["derive"] }
|
|
serde_json = "1"
|
|
chrono = { version = "0.4", default-features = false, features = ["serde", "clock"] }
|
|
thiserror = "2"
|
|
|
|
# core / data
|
|
sqlx = { version = "0.8", default-features = false, features = ["postgres", "runtime-tokio-rustls", "macros", "migrate", "chrono", "json"] }
|
|
async-trait = "0.1"
|
|
|
|
# binaries
|
|
tokio = { version = "1", features = ["rt-multi-thread", "macros", "signal", "time"] }
|
|
axum = "0.8"
|
|
tower-http = { version = "0.6", features = ["trace", "cors"] }
|
|
tracing = "0.1"
|
|
tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
|
|
anyhow = "1"
|
|
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json", "gzip"] }
|
|
figment = { version = "0.10", features = ["toml", "env"] }
|
|
clap = { version = "4", features = ["derive", "env"] }
|
|
|
|
# internal
|
|
moments-entities = { path = "crates/moments-entities", version = "=0.1.0" }
|
|
moments-core = { path = "crates/moments-core", version = "=0.1.0" }
|
|
moments-data = { path = "crates/moments-data", version = "=0.1.0" }
|