feat: ingest private events; surface public-only
The DB now stores everything GitHub will give us, the API only ever
returns public events (for now).
Endpoint switch in the github poller: when GITHUB_TOKEN is set we
hit /users/{u}/events (public + private), otherwise fall back to
/users/{u}/events/public. Either way each event's top-level `public`
boolean is captured into a new column.
Schema:
migration 0003_event_public.sql adds events.public BOOLEAN NOT NULL
DEFAULT true, plus an index on (public, occurred_at DESC).
Wire:
Event gains a `public: bool` field.
EventQuery gains `include_private: bool` (default false).
list_events and source_summaries gate on it.
moments-api pins include_private = false at every call site —
threading it as a query param is a future-auth concern, not now.
The default-true on the column keeps existing rows correct: the 11
events already in the DB came from /events/public and are genuinely
public.
After this change, clear poller_state so the next worker run does a
fresh backfill via /events:
DELETE FROM poller_state WHERE source = 'github';
Tests: +2 in github poller (private flag captured, default-public
on missing field) — 10 total green.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -58,11 +58,17 @@ impl GithubSource {
|
||||
}
|
||||
|
||||
fn first_page_url(&self) -> String {
|
||||
// Public events endpoint: works without auth (60/hr unauth, 5000/hr authed).
|
||||
// The non-public `/users/{u}/events` endpoint now requires auth and returns
|
||||
// private-repo activity, which we don't want on a public timeline anyway.
|
||||
// With a token: hit `/events`, which returns public + private events the
|
||||
// authenticated user can see. We store everything; the API gates what
|
||||
// gets surfaced to the public timeline via the `public` column.
|
||||
// Without a token: fall back to `/events/public` (anonymous-readable).
|
||||
let endpoint = if self.config.token.is_some() {
|
||||
"events"
|
||||
} else {
|
||||
"events/public"
|
||||
};
|
||||
format!(
|
||||
"https://api.github.com/users/{}/events/public?per_page={}",
|
||||
"https://api.github.com/users/{}/{endpoint}?per_page={}",
|
||||
self.config.user, self.config.per_page
|
||||
)
|
||||
}
|
||||
@@ -172,11 +178,17 @@ fn parse_github_event(raw: serde_json::Value) -> Option<Event> {
|
||||
let occurred_at = DateTime::parse_from_rfc3339(created_at_str)
|
||||
.ok()?
|
||||
.with_timezone(&Utc);
|
||||
// GitHub marks each event with a top-level `public` boolean. Events from
|
||||
// `/events/public` are always true; `/events` may include false. Default
|
||||
// to true if missing — that matches the safer-of-the-two-mistakes (under-
|
||||
// expose) and the `/events/public` endpoint behaviour.
|
||||
let public = raw.get("public").and_then(serde_json::Value::as_bool).unwrap_or(true);
|
||||
Some(Event {
|
||||
id: format!("github:{id}"),
|
||||
source: Source::Github,
|
||||
action: event_type,
|
||||
occurred_at,
|
||||
public,
|
||||
payload: raw,
|
||||
})
|
||||
}
|
||||
@@ -208,6 +220,7 @@ mod tests {
|
||||
"id": "12345",
|
||||
"type": "PushEvent",
|
||||
"created_at": "2026-04-15T10:30:00Z",
|
||||
"public": true,
|
||||
"actor": { "login": "grenade" },
|
||||
"repo": { "name": "grenade/moments" },
|
||||
"payload": { "ref": "refs/heads/main" }
|
||||
@@ -216,9 +229,39 @@ mod tests {
|
||||
assert_eq!(ev.id, "github:12345");
|
||||
assert_eq!(ev.source, Source::Github);
|
||||
assert_eq!(ev.action, "PushEvent");
|
||||
assert!(ev.public);
|
||||
assert_eq!(ev.payload, raw);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn private_event_marked_private() {
|
||||
let raw = serde_json::json!({
|
||||
"id": "67890",
|
||||
"type": "PushEvent",
|
||||
"created_at": "2026-04-15T10:30:00Z",
|
||||
"public": false,
|
||||
"actor": { "login": "grenade" },
|
||||
"repo": { "name": "grenade/private-thing" },
|
||||
"payload": {}
|
||||
});
|
||||
let ev = parse_github_event(raw).expect("parses");
|
||||
assert!(!ev.public);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_public_field_defaults_to_public() {
|
||||
let raw = serde_json::json!({
|
||||
"id": "11111",
|
||||
"type": "PushEvent",
|
||||
"created_at": "2026-04-15T10:30:00Z",
|
||||
"actor": { "login": "grenade" },
|
||||
"repo": { "name": "grenade/x" },
|
||||
"payload": {}
|
||||
});
|
||||
let ev = parse_github_event(raw).expect("parses");
|
||||
assert!(ev.public);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_event_missing_id() {
|
||||
let raw = serde_json::json!({ "type": "PushEvent", "created_at": "2026-01-01T00:00:00Z" });
|
||||
|
||||
Reference in New Issue
Block a user