feat: language stream graph on dashboard
Full-stack feature showing programming languages by commit activity as a stream graph on the dashboard. Backend: - migration: repo_languages table (source, repo, language, bytes, color) - worker: fetch language breakdowns via GitHub GraphQL (batched, 20 repos/request) and Gitea REST API during poll cycles - API: GET /v1/languages/daily (daily commit counts per language), GET /v1/languages/repos (all stored repo language data) - fix timezone bug in daily_counts and language_daily_counts: the PostgreSQL server timezone (Europe/Sofia, UTC+3) shifted day boundaries, miscounting events near midnight. Now uses explicit UTC boundaries in generate_series JOINs. - use per-source CASE for repo name extraction in language query to match gitea payload structure (repo.full_name vs repo.name) - Gitea languages use GitHub colors via COALESCE fallback Frontend: - LanguageStreamGraph component: pure SVG stream graph, weekly buckets, centered baseline, top 8 languages + Other, GitHub canonical language colors, legend with color dots - DashPage/ProjectPage: fetch repo languages once via new endpoint instead of per-repo forge proxy calls (eliminates 200+ GitHub API calls and 403 rate limit errors) - removed fetchLanguages forge proxy wrapper (dead code) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -9,12 +9,13 @@
|
||||
//! Each item carries a self-contained payload — including the event-emitting
|
||||
//! host — so the reshape layer can construct URLs without needing config.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use moments_core::{EventSource, EventWriter, PollerStateStore, SourceError};
|
||||
use moments_entities::{Event, Source};
|
||||
use moments_entities::{Event, RepoLanguage, Source};
|
||||
use reqwest::{Client, header};
|
||||
use serde_json::Value;
|
||||
use tracing::debug;
|
||||
@@ -126,17 +127,19 @@ impl GiteaSource {
|
||||
/// for org feeds which contain all members' activity).
|
||||
///
|
||||
/// `base_url` should contain everything except the `&page=N` suffix.
|
||||
/// Returns (ingested_count, set_of_repo_full_names).
|
||||
async fn poll_feed(
|
||||
&self,
|
||||
state_key: &str,
|
||||
base_url: &str,
|
||||
filter_user: bool,
|
||||
) -> Result<usize, SourceError> {
|
||||
) -> Result<(usize, HashSet<String>), SourceError> {
|
||||
let prior = self.state.load(state_key).await?;
|
||||
let first_run = prior.is_none();
|
||||
let max_pages = if first_run { MAX_BACKFILL_PAGES } else { 1 };
|
||||
|
||||
let mut total = 0usize;
|
||||
let mut repos = HashSet::new();
|
||||
for page in 1..=max_pages {
|
||||
let url = format!("{base_url}&page={page}");
|
||||
let req = self.apply_headers(self.client.get(&url));
|
||||
@@ -155,6 +158,17 @@ impl GiteaSource {
|
||||
break;
|
||||
}
|
||||
|
||||
// Collect repo names from feed items
|
||||
for item in &items {
|
||||
if let Some(name) = item
|
||||
.get("repo")
|
||||
.and_then(|r| r.get("full_name"))
|
||||
.and_then(Value::as_str)
|
||||
{
|
||||
repos.insert(name.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
let events: Vec<Event> = items
|
||||
.iter()
|
||||
.filter(|it| {
|
||||
@@ -177,6 +191,44 @@ impl GiteaSource {
|
||||
}
|
||||
|
||||
self.state.touch(state_key).await?;
|
||||
Ok((total, repos))
|
||||
}
|
||||
|
||||
/// Fetch language breakdowns for the given repos via the Gitea REST API.
|
||||
async fn fetch_languages(&self, repos: &HashSet<String>) -> Result<usize, SourceError> {
|
||||
let mut total = 0usize;
|
||||
for repo in repos {
|
||||
let url = format!(
|
||||
"https://{}/api/v1/repos/{}/languages",
|
||||
self.config.host, repo
|
||||
);
|
||||
let req = self.apply_headers(self.client.get(&url));
|
||||
let resp = req
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| SourceError::Http(e.to_string()))?;
|
||||
if !resp.status().is_success() {
|
||||
tracing::warn!(repo = %repo, status = %resp.status(), "gitea language fetch failed; skipping");
|
||||
continue;
|
||||
}
|
||||
let lang_map: std::collections::HashMap<String, i64> = resp
|
||||
.json()
|
||||
.await
|
||||
.map_err(|e| SourceError::Parse(e.to_string()))?;
|
||||
|
||||
let languages: Vec<RepoLanguage> = lang_map
|
||||
.into_iter()
|
||||
.map(|(language, bytes)| RepoLanguage {
|
||||
source: Source::Gitea,
|
||||
repo: repo.clone(),
|
||||
language,
|
||||
bytes,
|
||||
color: None, // Gitea doesn't return colors
|
||||
})
|
||||
.collect();
|
||||
total += self.writer.upsert_repo_languages(&languages).await?;
|
||||
}
|
||||
debug!(total, repos = repos.len(), "gitea repo languages updated");
|
||||
Ok(total)
|
||||
}
|
||||
}
|
||||
@@ -188,9 +240,12 @@ impl EventSource for GiteaSource {
|
||||
}
|
||||
|
||||
async fn poll(&self) -> Result<usize, SourceError> {
|
||||
let mut all_repos = HashSet::new();
|
||||
|
||||
// Poll user's own activity feed (existing behavior).
|
||||
let user_url = self.user_feed_base_url();
|
||||
let mut total = self.poll_feed(SOURCE_NAME, &user_url, false).await?;
|
||||
let (mut total, repos) = self.poll_feed(SOURCE_NAME, &user_url, false).await?;
|
||||
all_repos.extend(repos);
|
||||
|
||||
// Discover orgs and poll each org's activity feed, filtering for
|
||||
// events performed by this user.
|
||||
@@ -199,13 +254,20 @@ impl EventSource for GiteaSource {
|
||||
let state_key = format!("gitea:org:{org}");
|
||||
let org_url = self.org_feed_base_url(org);
|
||||
match self.poll_feed(&state_key, &org_url, true).await {
|
||||
Ok(n) => total += n,
|
||||
Ok((n, repos)) => {
|
||||
total += n;
|
||||
all_repos.extend(repos);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(org = %org, error = %e, "failed to poll org feed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(e) = self.fetch_languages(&all_repos).await {
|
||||
tracing::warn!(error = %e, "gitea language fetch failed; continuing");
|
||||
}
|
||||
|
||||
debug!(ingested = total, orgs = orgs.len(), "gitea poll complete");
|
||||
Ok(total)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user