Compare commits
3 Commits
2da9461b44
...
80f3f7c5cb
| Author | SHA1 | Date | |
|---|---|---|---|
|
80f3f7c5cb
|
|||
|
a70fab4feb
|
|||
|
a71b4e6b84
|
@@ -7,6 +7,7 @@ GITHUB_USER=grenade
|
||||
GITHUB_TOKEN={{GITHUB_TOKEN}}
|
||||
POLL_INTERVAL_SECS=600
|
||||
SEARCH_POLL_INTERVAL_SECS=86400
|
||||
REPO_POLL_INTERVAL_SECS=604800
|
||||
|
||||
GITEA_HOST=git.lair.cafe
|
||||
GITEA_USER=grenade
|
||||
|
||||
@@ -11,7 +11,7 @@ use chrono::{DateTime, Utc};
|
||||
use clap::Parser;
|
||||
use moments_core::{EventReader, reshape};
|
||||
use moments_data::PgStore;
|
||||
use moments_entities::{EventQuery, Source, SourceSummary, TimelineItem};
|
||||
use moments_entities::{EventQuery, ProjectSummary, Source, SourceSummary, TimelineItem};
|
||||
use serde::Deserialize;
|
||||
use tower_http::{cors::CorsLayer, trace::TraceLayer};
|
||||
use tracing::info;
|
||||
@@ -50,6 +50,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
.route("/v1/healthz", get(healthz))
|
||||
.route("/v1/events", get(list_events))
|
||||
.route("/v1/sources", get(list_sources))
|
||||
.route("/v1/projects", get(list_projects))
|
||||
.with_state(state)
|
||||
.layer(TraceLayer::new_for_http())
|
||||
.layer(CorsLayer::permissive());
|
||||
@@ -81,6 +82,8 @@ struct EventsQueryParams {
|
||||
to: Option<DateTime<Utc>>,
|
||||
/// Comma-separated list, e.g. `source=github,gitea`.
|
||||
source: Option<String>,
|
||||
/// Filter to a specific repo, e.g. `repo=grenade/moments`.
|
||||
repo: Option<String>,
|
||||
limit: Option<u32>,
|
||||
}
|
||||
|
||||
@@ -100,6 +103,7 @@ async fn list_events(
|
||||
from: params.from,
|
||||
to: params.to,
|
||||
sources,
|
||||
repo: params.repo,
|
||||
// Public timeline only — private events stay in the DB but are never
|
||||
// surfaced. A future authenticated path can flip this.
|
||||
include_private: false,
|
||||
@@ -122,6 +126,13 @@ async fn list_sources(
|
||||
Ok(Json(summaries))
|
||||
}
|
||||
|
||||
async fn list_projects(
|
||||
State(state): State<AppState>,
|
||||
) -> Result<Json<Vec<ProjectSummary>>, ApiError> {
|
||||
let projects = state.store.list_projects().await.map_err(internal)?;
|
||||
Ok(Json(projects))
|
||||
}
|
||||
|
||||
fn parse_sources(raw: &str) -> Result<Vec<Source>, ApiError> {
|
||||
raw.split(',')
|
||||
.map(str::trim)
|
||||
|
||||
@@ -5,7 +5,7 @@ pub use presentation::reshape;
|
||||
pub use sources::{EventSource, PollerState, PollerStateStore, SourceError, run_poller};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use moments_entities::{Event, EventQuery, SourceSummary};
|
||||
use moments_entities::{Event, EventQuery, ProjectSummary, SourceSummary};
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum StoreError {
|
||||
@@ -18,6 +18,7 @@ pub enum StoreError {
|
||||
pub trait EventReader: Send + Sync {
|
||||
async fn list_events(&self, query: &EventQuery) -> Result<Vec<Event>, StoreError>;
|
||||
async fn source_summaries(&self, include_private: bool) -> Result<Vec<SourceSummary>, StoreError>;
|
||||
async fn list_projects(&self) -> Result<Vec<ProjectSummary>, StoreError>;
|
||||
}
|
||||
|
||||
/// Write-side port consumed by `moments-worker`. Idempotent upserts on `id`.
|
||||
|
||||
325
crates/moments-data/src/github_repo.rs
Normal file
325
crates/moments-data/src/github_repo.rs
Normal file
@@ -0,0 +1,325 @@
|
||||
//! Per-repo commit enumeration for full GitHub history.
|
||||
//!
|
||||
//! The Search API caps at 1000 results; this source enumerates all repos
|
||||
//! the user can access via `/user/repos` and walks each repo's commit
|
||||
//! history via `/repos/{owner}/{repo}/commits?author={user}` — no cap.
|
||||
//!
|
||||
//! Events use `github-commit:{sha}` as their ID, matching the scheme in
|
||||
//! `github_search`, so duplicates are resolved via idempotent upsert.
|
||||
//!
|
||||
//! Per-repo poller state keys (`github-repo:{owner}/{repo}`) track which
|
||||
//! repos have been fully backfilled. First run paginates the full history;
|
||||
//! subsequent runs fetch only page 1.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use moments_core::{EventSource, EventWriter, PollerStateStore, SourceError};
|
||||
use moments_entities::{Event, Source};
|
||||
use reqwest::{Client, header};
|
||||
use serde_json::Value;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
const SOURCE_NAME: &str = "github-repo";
|
||||
const USER_AGENT: &str = concat!(
|
||||
"moments/",
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
" (+https://rob.tn)"
|
||||
);
|
||||
const MAX_BACKFILL_PAGES: u32 = 100;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct GithubRepoConfig {
|
||||
pub user: String,
|
||||
pub token: Option<String>,
|
||||
pub per_page: u32,
|
||||
}
|
||||
|
||||
impl Default for GithubRepoConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
user: "grenade".into(),
|
||||
token: None,
|
||||
per_page: 100,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct GithubRepoSource {
|
||||
client: Client,
|
||||
writer: Arc<dyn EventWriter>,
|
||||
state: Arc<dyn PollerStateStore>,
|
||||
config: GithubRepoConfig,
|
||||
}
|
||||
|
||||
impl GithubRepoSource {
|
||||
pub fn new(
|
||||
client: Client,
|
||||
writer: Arc<dyn EventWriter>,
|
||||
state: Arc<dyn PollerStateStore>,
|
||||
config: GithubRepoConfig,
|
||||
) -> Self {
|
||||
Self {
|
||||
client,
|
||||
writer,
|
||||
state,
|
||||
config,
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_headers(&self, mut req: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
|
||||
req = req
|
||||
.header(header::ACCEPT, "application/vnd.github+json")
|
||||
.header("X-GitHub-Api-Version", "2022-11-28")
|
||||
.header(header::USER_AGENT, USER_AGENT);
|
||||
if let Some(token) = &self.config.token {
|
||||
req = req.header(header::AUTHORIZATION, format!("Bearer {token}"));
|
||||
}
|
||||
req
|
||||
}
|
||||
|
||||
/// Discover all repos the authenticated user can access.
|
||||
async fn discover_repos(&self) -> Result<Vec<Repo>, SourceError> {
|
||||
if self.config.token.is_none() {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
let mut repos = Vec::new();
|
||||
for page in 1..=50 {
|
||||
let url = format!(
|
||||
"https://api.github.com/user/repos?affiliation=owner,collaborator,organization_member&visibility=all&per_page={}&page={}",
|
||||
self.config.per_page, page
|
||||
);
|
||||
let req = self.apply_headers(self.client.get(&url));
|
||||
let resp = req
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| SourceError::Http(e.to_string()))?;
|
||||
if !resp.status().is_success() {
|
||||
return Err(SourceError::Http(format!("{} GET {}", resp.status(), url)));
|
||||
}
|
||||
let items: Vec<Value> = resp
|
||||
.json()
|
||||
.await
|
||||
.map_err(|e| SourceError::Parse(e.to_string()))?;
|
||||
if items.is_empty() {
|
||||
break;
|
||||
}
|
||||
for item in &items {
|
||||
if let Some(r) = parse_repo(item) {
|
||||
repos.push(r);
|
||||
}
|
||||
}
|
||||
if items.len() < self.config.per_page as usize {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(repos)
|
||||
}
|
||||
|
||||
/// Fetch commits for a single repo, paginating fully on first run.
|
||||
async fn scan_repo(&self, repo: &Repo) -> Result<usize, SourceError> {
|
||||
let state_key = format!("github-repo:{}", repo.full_name);
|
||||
let prior = self.state.load(&state_key).await?;
|
||||
let first_run = prior.is_none();
|
||||
let max_pages = if first_run { MAX_BACKFILL_PAGES } else { 1 };
|
||||
|
||||
let mut total = 0usize;
|
||||
for page in 1..=max_pages {
|
||||
let url = format!(
|
||||
"https://api.github.com/repos/{}/commits?author={}&per_page={}&page={}",
|
||||
repo.full_name, self.config.user, self.config.per_page, page
|
||||
);
|
||||
let req = self.apply_headers(self.client.get(&url));
|
||||
let resp = req
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| SourceError::Http(e.to_string()))?;
|
||||
|
||||
let status = resp.status();
|
||||
// 409 = empty repo (no commits at all), not an error
|
||||
if status.as_u16() == 409 {
|
||||
break;
|
||||
}
|
||||
if status.as_u16() == 403 || status.as_u16() == 429 {
|
||||
warn!(repo = %repo.full_name, status = %status, "rate limited; stopping early");
|
||||
return Err(SourceError::Http(format!("{} GET {}", status, url)));
|
||||
}
|
||||
if status.as_u16() == 404 {
|
||||
warn!(repo = %repo.full_name, "repo not found; skipping");
|
||||
break;
|
||||
}
|
||||
if !status.is_success() {
|
||||
return Err(SourceError::Http(format!("{} GET {}", status, url)));
|
||||
}
|
||||
|
||||
let items: Vec<Value> = resp
|
||||
.json()
|
||||
.await
|
||||
.map_err(|e| SourceError::Parse(e.to_string()))?;
|
||||
if items.is_empty() {
|
||||
break;
|
||||
}
|
||||
|
||||
let events: Vec<Event> = items
|
||||
.iter()
|
||||
.filter_map(|item| parse_commit(item, repo))
|
||||
.collect();
|
||||
total += self.writer.upsert_events(&events).await?;
|
||||
|
||||
if items.len() < self.config.per_page as usize {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
self.state.touch(&state_key).await?;
|
||||
Ok(total)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl EventSource for GithubRepoSource {
|
||||
fn name(&self) -> &'static str {
|
||||
SOURCE_NAME
|
||||
}
|
||||
|
||||
async fn poll(&self) -> Result<usize, SourceError> {
|
||||
let repos = self.discover_repos().await?;
|
||||
debug!(repos = repos.len(), "discovered github repos");
|
||||
|
||||
let mut total = 0usize;
|
||||
for repo in &repos {
|
||||
match self.scan_repo(repo).await {
|
||||
Ok(n) => {
|
||||
if n > 0 {
|
||||
debug!(repo = %repo.full_name, ingested = n, "repo commit scan complete");
|
||||
}
|
||||
total += n;
|
||||
}
|
||||
Err(SourceError::Http(ref msg)) if msg.starts_with("403") || msg.starts_with("429") => {
|
||||
warn!("rate limited during repo scan; ending poll early");
|
||||
break;
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(repo = %repo.full_name, error = %e, "repo scan failed; continuing");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
self.state.touch(SOURCE_NAME).await?;
|
||||
debug!(ingested = total, repos = repos.len(), "github-repo poll complete");
|
||||
Ok(total)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct Repo {
|
||||
full_name: String,
|
||||
private: bool,
|
||||
}
|
||||
|
||||
fn parse_repo(item: &Value) -> Option<Repo> {
|
||||
let full_name = item.get("full_name").and_then(Value::as_str)?;
|
||||
let private = item.get("private").and_then(Value::as_bool).unwrap_or(false);
|
||||
Some(Repo {
|
||||
full_name: full_name.to_string(),
|
||||
private,
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_commit(item: &Value, repo: &Repo) -> Option<Event> {
|
||||
let sha = item.get("sha").and_then(Value::as_str)?;
|
||||
let date_str = item
|
||||
.get("commit")
|
||||
.and_then(|c| c.get("author"))
|
||||
.and_then(|a| a.get("date"))
|
||||
.and_then(Value::as_str)
|
||||
.or_else(|| {
|
||||
item.get("commit")
|
||||
.and_then(|c| c.get("committer"))
|
||||
.and_then(|c| c.get("date"))
|
||||
.and_then(Value::as_str)
|
||||
})?;
|
||||
let occurred_at = DateTime::parse_from_rfc3339(date_str)
|
||||
.ok()?
|
||||
.with_timezone(&Utc);
|
||||
|
||||
Some(Event {
|
||||
id: format!("github-commit:{sha}"),
|
||||
source: Source::Github,
|
||||
action: "Commit".into(),
|
||||
occurred_at,
|
||||
public: !repo.private,
|
||||
payload: item.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn parse_commit_uses_sha_as_id() {
|
||||
let repo = Repo {
|
||||
full_name: "grenade/moments".into(),
|
||||
private: false,
|
||||
};
|
||||
let raw = json!({
|
||||
"sha": "abc123",
|
||||
"commit": {
|
||||
"author": { "date": "2024-01-15T10:30:00Z" },
|
||||
"message": "fix something"
|
||||
}
|
||||
});
|
||||
let ev = parse_commit(&raw, &repo).expect("parses");
|
||||
assert_eq!(ev.id, "github-commit:abc123");
|
||||
assert_eq!(ev.action, "Commit");
|
||||
assert!(ev.public);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_commit_private_repo() {
|
||||
let repo = Repo {
|
||||
full_name: "grenade/secret".into(),
|
||||
private: true,
|
||||
};
|
||||
let raw = json!({
|
||||
"sha": "def456",
|
||||
"commit": {
|
||||
"author": { "date": "2024-01-15T10:30:00Z" },
|
||||
"message": "secret change"
|
||||
}
|
||||
});
|
||||
let ev = parse_commit(&raw, &repo).expect("parses");
|
||||
assert!(!ev.public);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_commit_falls_back_to_committer_date() {
|
||||
let repo = Repo {
|
||||
full_name: "grenade/moments".into(),
|
||||
private: false,
|
||||
};
|
||||
let raw = json!({
|
||||
"sha": "ghi789",
|
||||
"commit": {
|
||||
"committer": { "date": "2024-02-01T12:00:00Z" },
|
||||
"message": "no author date"
|
||||
}
|
||||
});
|
||||
let ev = parse_commit(&raw, &repo).expect("parses");
|
||||
assert_eq!(ev.id, "github-commit:ghi789");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_repo_extracts_fields() {
|
||||
let raw = json!({
|
||||
"full_name": "grenade/moments",
|
||||
"private": false
|
||||
});
|
||||
let repo = parse_repo(&raw).expect("parses");
|
||||
assert_eq!(repo.full_name, "grenade/moments");
|
||||
assert!(!repo.private);
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,14 @@
|
||||
pub mod bugzilla;
|
||||
pub mod gitea;
|
||||
pub mod github;
|
||||
pub mod github_repo;
|
||||
pub mod github_search;
|
||||
pub mod hg;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use moments_core::{EventReader, EventWriter, PollerState, PollerStateStore, StoreError};
|
||||
use moments_entities::{Event, EventQuery, Source, SourceSummary};
|
||||
use moments_entities::{Event, EventQuery, ProjectSummary, Source, SourceSummary};
|
||||
use sqlx::Row;
|
||||
use sqlx::postgres::{PgPool, PgPoolOptions};
|
||||
use std::str::FromStr;
|
||||
@@ -53,6 +54,12 @@ impl EventReader for PgStore {
|
||||
AND ($2::timestamptz IS NULL OR occurred_at < $2)
|
||||
AND ($3::text[] IS NULL OR source = ANY($3))
|
||||
AND ($4::bool OR public = true)
|
||||
AND ($6::text IS NULL OR COALESCE(
|
||||
payload->'repo'->>'name',
|
||||
payload->'repository'->>'full_name',
|
||||
payload->>'_repo',
|
||||
payload->>'product'
|
||||
) = $6)
|
||||
ORDER BY occurred_at DESC
|
||||
LIMIT $5
|
||||
"#,
|
||||
@@ -62,6 +69,7 @@ impl EventReader for PgStore {
|
||||
.bind(sources.as_deref())
|
||||
.bind(query.include_private)
|
||||
.bind(query.limit as i64)
|
||||
.bind(query.repo.as_deref())
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map_err(map_err)?;
|
||||
@@ -114,6 +122,62 @@ impl EventReader for PgStore {
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn list_projects(&self) -> Result<Vec<ProjectSummary>, StoreError> {
|
||||
let rows = sqlx::query(
|
||||
r#"
|
||||
SELECT source, repo, host,
|
||||
SUM(commits)::bigint AS commit_count,
|
||||
SUM(issues)::bigint AS issue_count,
|
||||
SUM(prs)::bigint AS pr_count,
|
||||
MIN(occurred_at) AS first_activity,
|
||||
MAX(occurred_at) AS last_activity
|
||||
FROM (
|
||||
SELECT source, occurred_at,
|
||||
COALESCE(
|
||||
payload->'repo'->>'name',
|
||||
payload->'repository'->>'full_name',
|
||||
payload->>'_repo',
|
||||
payload->>'product'
|
||||
) AS repo,
|
||||
CASE source
|
||||
WHEN 'github' THEN 'github.com'
|
||||
WHEN 'gitea' THEN COALESCE(payload->>'_host', 'git.lair.cafe')
|
||||
WHEN 'hg' THEN COALESCE(payload->>'_host', 'hg-edge.mozilla.org')
|
||||
WHEN 'bugzilla' THEN 'bugzilla.mozilla.org'
|
||||
ELSE 'unknown'
|
||||
END AS host,
|
||||
CASE WHEN action IN ('Commit', 'PushEvent', 'commit_repo') THEN 1 ELSE 0 END AS commits,
|
||||
CASE WHEN action IN ('Issue', 'IssuesEvent') THEN 1 ELSE 0 END AS issues,
|
||||
CASE WHEN action IN ('PullRequest', 'PullRequestEvent') THEN 1 ELSE 0 END AS prs
|
||||
FROM events
|
||||
WHERE public = true
|
||||
) sub
|
||||
WHERE repo IS NOT NULL AND repo != ''
|
||||
GROUP BY source, repo, host
|
||||
ORDER BY MAX(occurred_at) DESC
|
||||
"#,
|
||||
)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map_err(map_err)?;
|
||||
|
||||
rows.into_iter()
|
||||
.map(|r| {
|
||||
let source_str: String = r.try_get("source").map_err(map_err)?;
|
||||
Ok(ProjectSummary {
|
||||
source: Source::from_str(&source_str).map_err(map_err)?,
|
||||
repo: r.try_get("repo").map_err(map_err)?,
|
||||
host: r.try_get("host").map_err(map_err)?,
|
||||
commit_count: r.try_get::<i64, _>("commit_count").map_err(map_err).unwrap_or(0),
|
||||
issue_count: r.try_get::<i64, _>("issue_count").map_err(map_err).unwrap_or(0),
|
||||
pr_count: r.try_get::<i64, _>("pr_count").map_err(map_err).unwrap_or(0),
|
||||
first_activity: r.try_get("first_activity").map_err(map_err)?,
|
||||
last_activity: r.try_get("last_activity").map_err(map_err)?,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
|
||||
@@ -67,6 +67,8 @@ pub struct EventQuery {
|
||||
pub from: Option<DateTime<Utc>>,
|
||||
pub to: Option<DateTime<Utc>>,
|
||||
pub sources: Option<Vec<Source>>,
|
||||
/// Filter to events matching a specific repo (matched against payload).
|
||||
pub repo: Option<String>,
|
||||
/// When false (default), only `public = true` rows are returned. The API
|
||||
/// pins this to false today; a future authenticated path can flip it.
|
||||
pub include_private: bool,
|
||||
@@ -82,6 +84,19 @@ pub struct SourceSummary {
|
||||
pub latest: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
/// Per-repo activity rollup for the dashboard.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ProjectSummary {
|
||||
pub repo: String,
|
||||
pub source: Source,
|
||||
pub host: String,
|
||||
pub commit_count: i64,
|
||||
pub issue_count: i64,
|
||||
pub pr_count: i64,
|
||||
pub first_activity: Option<DateTime<Utc>>,
|
||||
pub last_activity: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Presentation shape — what `GET /v1/events` actually returns.
|
||||
// The API reshapes raw payloads into these so the frontend stays dumb.
|
||||
|
||||
@@ -7,6 +7,7 @@ use moments_data::{
|
||||
bugzilla::{BugzillaConfig, BugzillaSource},
|
||||
gitea::{GiteaConfig, GiteaSource},
|
||||
github::{GithubConfig, GithubSource},
|
||||
github_repo::{GithubRepoConfig, GithubRepoSource},
|
||||
github_search::{GithubSearchConfig, GithubSearchSource},
|
||||
hg::{HgConfig, HgSource},
|
||||
};
|
||||
@@ -35,6 +36,11 @@ struct Args {
|
||||
#[arg(long, env = "SEARCH_POLL_INTERVAL_SECS", default_value = "86400")]
|
||||
search_interval_secs: u64,
|
||||
|
||||
/// Seconds between per-repo commit enumeration polls (full history backfill).
|
||||
/// Defaults to weekly — expensive initial scan, cheap afterwards.
|
||||
#[arg(long, env = "REPO_POLL_INTERVAL_SECS", default_value = "604800")]
|
||||
repo_interval_secs: u64,
|
||||
|
||||
#[arg(long, env = "GITEA_HOST", default_value = "git.lair.cafe")]
|
||||
gitea_host: String,
|
||||
|
||||
@@ -132,6 +138,17 @@ async fn main() -> anyhow::Result<()> {
|
||||
},
|
||||
)) as Arc<dyn EventSource>;
|
||||
|
||||
let github_repo = Arc::new(GithubRepoSource::new(
|
||||
http.clone(),
|
||||
store.clone(),
|
||||
store.clone(),
|
||||
GithubRepoConfig {
|
||||
user: args.github_user.clone(),
|
||||
token: args.github_token.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
)) as Arc<dyn EventSource>;
|
||||
|
||||
let gitea = Arc::new(GiteaSource::new(
|
||||
http.clone(),
|
||||
store.clone(),
|
||||
@@ -180,6 +197,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
bugzilla_email = args.bugzilla_email,
|
||||
events_interval_secs = args.interval_secs,
|
||||
search_interval_secs = args.search_interval_secs,
|
||||
repo_interval_secs = args.repo_interval_secs,
|
||||
gitea_interval_secs = args.gitea_interval_secs,
|
||||
hg_interval_secs = args.hg_interval_secs,
|
||||
bugzilla_interval_secs = args.bugzilla_interval_secs,
|
||||
@@ -188,6 +206,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
|
||||
let interval = Duration::from_secs(args.interval_secs);
|
||||
let search_interval = Duration::from_secs(args.search_interval_secs);
|
||||
let repo_interval = Duration::from_secs(args.repo_interval_secs);
|
||||
let gitea_interval = Duration::from_secs(args.gitea_interval_secs);
|
||||
let hg_interval = Duration::from_secs(args.hg_interval_secs);
|
||||
let bugzilla_interval = Duration::from_secs(args.bugzilla_interval_secs);
|
||||
@@ -195,6 +214,8 @@ async fn main() -> anyhow::Result<()> {
|
||||
let github_task = tokio::spawn(async move { run_poller(github, interval).await });
|
||||
let github_search_task =
|
||||
tokio::spawn(async move { run_poller(github_search, search_interval).await });
|
||||
let github_repo_task =
|
||||
tokio::spawn(async move { run_poller(github_repo, repo_interval).await });
|
||||
let gitea_task = tokio::spawn(async move { run_poller(gitea, gitea_interval).await });
|
||||
let hg_task = tokio::spawn(async move { run_poller(hg, hg_interval).await });
|
||||
let bugzilla_task =
|
||||
@@ -204,6 +225,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
info!("shutdown signal received");
|
||||
github_task.abort();
|
||||
github_search_task.abort();
|
||||
github_repo_task.abort();
|
||||
gitea_task.abort();
|
||||
hg_task.abort();
|
||||
bugzilla_task.abort();
|
||||
|
||||
@@ -19,6 +19,7 @@ WORK_DIR="${HG_WORK_DIR:-$HOME/hg}"
|
||||
|
||||
# Repos to clone (groups are expanded inline)
|
||||
REPOS=(
|
||||
mozilla-central
|
||||
integration/mozilla-inbound
|
||||
integration/autoland
|
||||
integration/fx-team
|
||||
|
||||
@@ -53,6 +53,50 @@ a.hot-pink {
|
||||
color: #1565c0;
|
||||
}
|
||||
|
||||
.site-header h1 {
|
||||
font-size: 1.75rem;
|
||||
}
|
||||
|
||||
.site-header nav a {
|
||||
color: #ecf0f1;
|
||||
opacity: 0.7;
|
||||
}
|
||||
|
||||
.site-header nav a:hover {
|
||||
color: #ff4081;
|
||||
opacity: 1;
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
.site-header nav a.active {
|
||||
color: #ff4081;
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
.nav-divider {
|
||||
opacity: 0.3;
|
||||
}
|
||||
|
||||
.project-card {
|
||||
background: rgba(255, 255, 255, 0.05);
|
||||
border: 1px solid rgba(255, 255, 255, 0.1);
|
||||
border-radius: 6px;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
.project-card h5 {
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
.project-card a {
|
||||
color: #ff4081;
|
||||
}
|
||||
|
||||
.project-card .text-muted {
|
||||
color: rgba(236, 240, 241, 0.5) !important;
|
||||
font-size: 0.7rem;
|
||||
}
|
||||
|
||||
.site-footer {
|
||||
margin-top: 3rem;
|
||||
padding: 1rem 0;
|
||||
|
||||
@@ -5,20 +5,22 @@ import 'rc-slider/assets/index.css';
|
||||
import 'react-vertical-timeline-component/style.min.css';
|
||||
import './App.css';
|
||||
|
||||
import { Layout } from './components/Layout';
|
||||
import { DashPage } from './pages/DashPage';
|
||||
import { TimelineHome } from './pages/TimelineHome';
|
||||
import { ProjectPage } from './pages/ProjectPage';
|
||||
import { CvPage } from './pages/CvPage';
|
||||
|
||||
export default function App() {
|
||||
return (
|
||||
<>
|
||||
<Routes>
|
||||
<Route path="/" element={<TimelineHome />} />
|
||||
<Routes>
|
||||
<Route element={<Layout />}>
|
||||
<Route index element={<DashPage />} />
|
||||
<Route path="/dash" element={<DashPage />} />
|
||||
<Route path="/activity" element={<TimelineHome />} />
|
||||
<Route path="/project/:source/*" element={<ProjectPage />} />
|
||||
<Route path="/cv" element={<CvPage />} />
|
||||
</Routes>
|
||||
<footer className="site-footer">
|
||||
no cookies are set or read by this site, which is why no consent banner
|
||||
is shown.
|
||||
</footer>
|
||||
</>
|
||||
</Route>
|
||||
</Routes>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -54,10 +54,22 @@ export interface SourceSummary {
|
||||
latest: string | null;
|
||||
}
|
||||
|
||||
export interface ProjectSummary {
|
||||
repo: string;
|
||||
source: Source;
|
||||
host: string;
|
||||
commit_count: number;
|
||||
issue_count: number;
|
||||
pr_count: number;
|
||||
first_activity: string | null;
|
||||
last_activity: string | null;
|
||||
}
|
||||
|
||||
export interface EventQuery {
|
||||
from?: Date;
|
||||
to?: Date;
|
||||
sources?: Source[];
|
||||
repo?: string;
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
@@ -70,6 +82,7 @@ export async function fetchEvents(q: EventQuery): Promise<TimelineItem[]> {
|
||||
if (q.sources && q.sources.length > 0) {
|
||||
params.set('source', q.sources.join(','));
|
||||
}
|
||||
if (q.repo) params.set('repo', q.repo);
|
||||
if (q.limit) params.set('limit', String(q.limit));
|
||||
|
||||
const resp = await fetch(`${API_BASE}/events?${params}`);
|
||||
@@ -82,3 +95,9 @@ export async function fetchSources(): Promise<SourceSummary[]> {
|
||||
if (!resp.ok) throw new Error(`sources: HTTP ${resp.status}`);
|
||||
return resp.json();
|
||||
}
|
||||
|
||||
export async function fetchProjects(): Promise<ProjectSummary[]> {
|
||||
const resp = await fetch(`${API_BASE}/projects`);
|
||||
if (!resp.ok) throw new Error(`projects: HTTP ${resp.status}`);
|
||||
return resp.json();
|
||||
}
|
||||
|
||||
42
ui/src/components/Layout.tsx
Normal file
42
ui/src/components/Layout.tsx
Normal file
@@ -0,0 +1,42 @@
|
||||
import { NavLink, Outlet } from 'react-router-dom';
|
||||
import Container from 'react-bootstrap/Container';
|
||||
|
||||
const externalLinks = [
|
||||
{ url: 'https://linkedin.com/in/thijssen/', label: 'linkedin' },
|
||||
{ url: 'https://stackoverflow.com/users/68115/grenade', label: 'stackoverflow' },
|
||||
{ url: 'https://github.com/grenade', label: 'github' },
|
||||
{ url: 'https://git.lair.cafe/grenade', label: 'gitea' },
|
||||
];
|
||||
|
||||
export function Layout() {
|
||||
return (
|
||||
<>
|
||||
<Container className="py-4">
|
||||
<header className="site-header d-flex flex-wrap justify-content-between align-items-center mb-4">
|
||||
<h1 className="mb-0">hi, i'm rob</h1>
|
||||
<nav className="d-flex flex-wrap gap-3 align-items-center">
|
||||
<NavLink to="/" end>dash</NavLink>
|
||||
<NavLink to="/activity">activity</NavLink>
|
||||
<NavLink to="/cv">cv</NavLink>
|
||||
<span className="nav-divider">|</span>
|
||||
{externalLinks.map((el) => (
|
||||
<a
|
||||
key={el.url}
|
||||
href={el.url}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
>
|
||||
{el.label}
|
||||
</a>
|
||||
))}
|
||||
</nav>
|
||||
</header>
|
||||
<Outlet />
|
||||
</Container>
|
||||
<footer className="site-footer">
|
||||
no cookies are set or read by this site, which is why no consent banner
|
||||
is shown.
|
||||
</footer>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -3,7 +3,6 @@ import { useLocation } from 'react-router-dom';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import Alert from 'react-bootstrap/Alert';
|
||||
import Col from 'react-bootstrap/Col';
|
||||
import Container from 'react-bootstrap/Container';
|
||||
import Row from 'react-bootstrap/Row';
|
||||
import Spinner from 'react-bootstrap/Spinner';
|
||||
|
||||
@@ -34,13 +33,13 @@ export function CvPage() {
|
||||
|
||||
if (cvQ.isLoading) {
|
||||
return (
|
||||
<Container className="py-4">
|
||||
<>
|
||||
<CvHeader />
|
||||
<div className="d-flex align-items-center gap-2">
|
||||
<Spinner animation="border" role="status" size="sm" />
|
||||
<span>loading cv…</span>
|
||||
</div>
|
||||
</Container>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -50,7 +49,7 @@ export function CvPage() {
|
||||
? ' (github limits unauthenticated requests to 60/hour per ip — try again shortly)'
|
||||
: '';
|
||||
return (
|
||||
<Container className="py-4">
|
||||
<>
|
||||
<CvHeader />
|
||||
<Alert variant="danger">
|
||||
<Alert.Heading>cv unavailable</Alert.Heading>
|
||||
@@ -62,7 +61,7 @@ export function CvPage() {
|
||||
retry
|
||||
</button>
|
||||
</Alert>
|
||||
</Container>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -72,15 +71,15 @@ export function CvPage() {
|
||||
|
||||
if (bodySections.length === 0 && navSections.length === 0) {
|
||||
return (
|
||||
<Container className="py-4">
|
||||
<>
|
||||
<CvHeader />
|
||||
<Alert variant="warning">cv unavailable: no sections in config</Alert>
|
||||
</Container>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<Container className="py-4">
|
||||
<>
|
||||
<CvHeader />
|
||||
<Row>
|
||||
<Col lg={9} className="cv-body">
|
||||
@@ -103,6 +102,6 @@ export function CvPage() {
|
||||
<CvTimeline data={data} />
|
||||
</Col>
|
||||
</Row>
|
||||
</Container>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
82
ui/src/pages/DashPage.tsx
Normal file
82
ui/src/pages/DashPage.tsx
Normal file
@@ -0,0 +1,82 @@
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import { Link } from 'react-router-dom';
|
||||
import Col from 'react-bootstrap/Col';
|
||||
import Row from 'react-bootstrap/Row';
|
||||
|
||||
import { fetchProjects, type ProjectSummary } from '../api/client';
|
||||
|
||||
export function DashPage() {
|
||||
const projectsQ = useQuery({
|
||||
queryKey: ['projects'],
|
||||
queryFn: fetchProjects,
|
||||
refetchInterval: 60_000,
|
||||
});
|
||||
|
||||
const projects = projectsQ.data ?? [];
|
||||
const ranked = rankProjects(projects).slice(0, 24);
|
||||
|
||||
return (
|
||||
<>
|
||||
<Row className="mb-3">
|
||||
<Col>
|
||||
<p>
|
||||
i rarely say anything that warrants capital letters. a peek into the
|
||||
projects i'm working on is below.
|
||||
</p>
|
||||
</Col>
|
||||
</Row>
|
||||
{projectsQ.isLoading && <p>loading...</p>}
|
||||
{projectsQ.isError && (
|
||||
<p>error: {(projectsQ.error as Error).message}</p>
|
||||
)}
|
||||
<Row xs={1} md={2} lg={3} className="g-3">
|
||||
{ranked.map((p) => (
|
||||
<Col key={`${p.source}:${p.repo}`}>
|
||||
<Link to={`/project/${p.source}/${p.repo}`} className="text-decoration-none">
|
||||
<div className="project-card p-3">
|
||||
<h5 className="mb-1">{p.repo}</h5>
|
||||
<small className="text-muted d-block mb-2">{p.source}</small>
|
||||
<div className="d-flex gap-3" style={{ fontSize: '0.8rem' }}>
|
||||
{p.commit_count > 0 && <span>{p.commit_count} commits</span>}
|
||||
{p.issue_count > 0 && <span>{p.issue_count} issues</span>}
|
||||
{p.pr_count > 0 && <span>{p.pr_count} prs</span>}
|
||||
</div>
|
||||
<div style={{ fontSize: '0.75rem', opacity: 0.6, marginTop: '0.25rem' }}>
|
||||
{formatRange(p.first_activity, p.last_activity)}
|
||||
</div>
|
||||
</div>
|
||||
</Link>
|
||||
</Col>
|
||||
))}
|
||||
</Row>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function formatRange(first: string | null, last: string | null): string {
|
||||
const fmt = (iso: string) =>
|
||||
new Date(iso).toLocaleDateString('en-GB', { month: 'short', year: 'numeric' }).toLowerCase();
|
||||
if (first && last) return `${fmt(first)} — ${fmt(last)}`;
|
||||
if (last) return fmt(last);
|
||||
return '';
|
||||
}
|
||||
|
||||
function rankProjects(projects: ProjectSummary[]): ProjectSummary[] {
|
||||
if (projects.length === 0) return [];
|
||||
const now = Date.now();
|
||||
const maxVolume = Math.max(...projects.map((p) => p.commit_count + p.issue_count + p.pr_count));
|
||||
const oldest = Math.min(
|
||||
...projects.map((p) => (p.last_activity ? new Date(p.last_activity).getTime() : 0)),
|
||||
);
|
||||
const range = now - oldest || 1;
|
||||
|
||||
return [...projects].sort((a, b) => score(b) - score(a));
|
||||
|
||||
function score(p: ProjectSummary): number {
|
||||
const volume = (p.commit_count + p.issue_count + p.pr_count) / (maxVolume || 1);
|
||||
const recency = p.last_activity
|
||||
? (new Date(p.last_activity).getTime() - oldest) / range
|
||||
: 0;
|
||||
return 0.6 * recency + 0.4 * volume;
|
||||
}
|
||||
}
|
||||
53
ui/src/pages/ProjectPage.tsx
Normal file
53
ui/src/pages/ProjectPage.tsx
Normal file
@@ -0,0 +1,53 @@
|
||||
import { useParams } from 'react-router-dom';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import Col from 'react-bootstrap/Col';
|
||||
import Row from 'react-bootstrap/Row';
|
||||
import { VerticalTimeline } from 'react-vertical-timeline-component';
|
||||
|
||||
import { fetchEvents, type Source } from '../api/client';
|
||||
import { TimelineEntry } from '../components/TimelineEntry';
|
||||
|
||||
export function ProjectPage() {
|
||||
const { source, '*': repoPath } = useParams();
|
||||
const repo = repoPath ?? '';
|
||||
|
||||
const eventsQ = useQuery({
|
||||
queryKey: ['project-events', source, repo],
|
||||
queryFn: () =>
|
||||
fetchEvents({
|
||||
sources: source ? [source as Source] : undefined,
|
||||
repo,
|
||||
limit: 500,
|
||||
}),
|
||||
refetchInterval: 60_000,
|
||||
});
|
||||
|
||||
const events = eventsQ.data ?? [];
|
||||
|
||||
return (
|
||||
<>
|
||||
<Row className="mb-3">
|
||||
<Col>
|
||||
<h2>{repo}</h2>
|
||||
<small className="text-muted">{source}</small>
|
||||
</Col>
|
||||
</Row>
|
||||
<Row>
|
||||
<Col>
|
||||
<p style={{ fontSize: '85%' }}>
|
||||
{eventsQ.isLoading
|
||||
? 'loading...'
|
||||
: eventsQ.isError
|
||||
? `error: ${(eventsQ.error as Error).message}`
|
||||
: `${events.length} ${events.length === 1 ? 'activity' : 'activities'}`}
|
||||
</p>
|
||||
<VerticalTimeline>
|
||||
{events.map((item) => (
|
||||
<TimelineEntry key={item.id} item={item} />
|
||||
))}
|
||||
</VerticalTimeline>
|
||||
</Col>
|
||||
</Row>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -1,8 +1,6 @@
|
||||
import { useMemo, useState } from 'react';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import { Link } from 'react-router-dom';
|
||||
import Col from 'react-bootstrap/Col';
|
||||
import Container from 'react-bootstrap/Container';
|
||||
import Row from 'react-bootstrap/Row';
|
||||
import { VerticalTimeline } from 'react-vertical-timeline-component';
|
||||
|
||||
@@ -13,13 +11,6 @@ import { TimelineEntry } from '../components/TimelineEntry';
|
||||
const RANGE_MIN = new Date('2010-01-01T00:00:00Z').getTime();
|
||||
const RANGE_MAX = Date.now();
|
||||
|
||||
const externalLinks: { url: string; alt: string }[] = [
|
||||
{ url: 'https://linkedin.com/in/thijssen/', alt: 'linkedin' },
|
||||
{ url: 'https://stackoverflow.com/users/68115/grenade', alt: 'stackoverflow' },
|
||||
{ url: 'https://github.com/grenade', alt: 'github' },
|
||||
{ url: 'https://git.lair.cafe/grenade', alt: 'gitea' },
|
||||
];
|
||||
|
||||
export function TimelineHome() {
|
||||
const [enabledSources, setEnabledSources] = useState<Record<Source, boolean>>({
|
||||
github: true,
|
||||
@@ -61,38 +52,7 @@ export function TimelineHome() {
|
||||
const events = eventsQ.data ?? [];
|
||||
|
||||
return (
|
||||
<Container className="py-4">
|
||||
<Row className="mb-3">
|
||||
<Col>
|
||||
<h1>hi, i'm rob</h1>
|
||||
</Col>
|
||||
<Col className="d-flex flex-wrap gap-3 justify-content-end align-items-center">
|
||||
{externalLinks.map((el) => (
|
||||
<a
|
||||
key={el.url}
|
||||
href={el.url}
|
||||
title={el.alt}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
>
|
||||
{el.alt}
|
||||
</a>
|
||||
))}
|
||||
</Col>
|
||||
</Row>
|
||||
<Row className="mb-4">
|
||||
<Col>
|
||||
<p>
|
||||
i rarely say anything that warrants capital letters. if you're here
|
||||
to see my resume, please go to{' '}
|
||||
<Link className="hot-pink" to="/cv">
|
||||
/cv
|
||||
</Link>
|
||||
. a peek into the projects i'm working on is below.
|
||||
</p>
|
||||
</Col>
|
||||
</Row>
|
||||
|
||||
<>
|
||||
<Filters
|
||||
enabledSources={enabledSources}
|
||||
onSourceToggle={(s, on) =>
|
||||
@@ -123,6 +83,6 @@ export function TimelineHome() {
|
||||
</VerticalTimeline>
|
||||
</Col>
|
||||
</Row>
|
||||
</Container>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user