feat: language stream graph on dashboard
Full-stack feature showing programming languages by commit activity as a stream graph on the dashboard. Backend: - migration: repo_languages table (source, repo, language, bytes, color) - worker: fetch language breakdowns via GitHub GraphQL (batched, 20 repos/request) and Gitea REST API during poll cycles - API: GET /v1/languages/daily (daily commit counts per language), GET /v1/languages/repos (all stored repo language data) - fix timezone bug in daily_counts and language_daily_counts: the PostgreSQL server timezone (Europe/Sofia, UTC+3) shifted day boundaries, miscounting events near midnight. Now uses explicit UTC boundaries in generate_series JOINs. - use per-source CASE for repo name extraction in language query to match gitea payload structure (repo.full_name vs repo.name) - Gitea languages use GitHub colors via COALESCE fallback Frontend: - LanguageStreamGraph component: pure SVG stream graph, weekly buckets, centered baseline, top 8 languages + Other, GitHub canonical language colors, legend with color dots - DashPage/ProjectPage: fetch repo languages once via new endpoint instead of per-repo forge proxy calls (eliminates 200+ GitHub API calls and 403 rate limit errors) - removed fetchLanguages forge proxy wrapper (dead code) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -20,7 +20,7 @@ use std::sync::Arc;
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use moments_core::{EventSource, EventWriter, PollerStateStore, SourceError};
|
||||
use moments_entities::{Event, Source};
|
||||
use moments_entities::{Event, RepoLanguage, Source};
|
||||
use reqwest::{Client, header};
|
||||
use serde_json::Value;
|
||||
use tracing::{debug, warn};
|
||||
@@ -296,6 +296,105 @@ impl GithubRepoSource {
|
||||
self.state.save(&state_key, None, newest).await?;
|
||||
Ok(total)
|
||||
}
|
||||
|
||||
/// Batch-fetch language breakdowns for repos via GraphQL, upserting
|
||||
/// into repo_languages. Repos are batched using GraphQL aliases to
|
||||
/// minimise round trips.
|
||||
async fn fetch_languages(&self, repos: &[Repo]) -> Result<usize, SourceError> {
|
||||
let token = match &self.config.token {
|
||||
Some(t) => t,
|
||||
None => return Ok(0),
|
||||
};
|
||||
|
||||
let mut total = 0usize;
|
||||
for chunk in repos.chunks(20) {
|
||||
let mut fragments = Vec::with_capacity(chunk.len());
|
||||
for (i, repo) in chunk.iter().enumerate() {
|
||||
let parts: Vec<&str> = repo.full_name.splitn(2, '/').collect();
|
||||
if parts.len() != 2 {
|
||||
continue;
|
||||
}
|
||||
fragments.push(format!(
|
||||
r#"r{i}: repository(owner: "{}", name: "{}") {{ languages(first: 20, orderBy: {{field: SIZE, direction: DESC}}) {{ edges {{ size node {{ name color }} }} }} }}"#,
|
||||
parts[0], parts[1]
|
||||
));
|
||||
}
|
||||
if fragments.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let query = format!("{{ {} }}", fragments.join(" "));
|
||||
let body = serde_json::json!({ "query": query });
|
||||
|
||||
let resp = self
|
||||
.client
|
||||
.post("https://api.github.com/graphql")
|
||||
.header(header::AUTHORIZATION, format!("Bearer {token}"))
|
||||
.header(header::USER_AGENT, USER_AGENT)
|
||||
.header(header::CONTENT_TYPE, "application/json")
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| SourceError::Http(e.to_string()))?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
warn!(status = %resp.status(), "GraphQL language fetch failed");
|
||||
break;
|
||||
}
|
||||
|
||||
let data: Value = resp
|
||||
.json()
|
||||
.await
|
||||
.map_err(|e| SourceError::Parse(e.to_string()))?;
|
||||
|
||||
if let Some(errors) = data.get("errors").and_then(Value::as_array) {
|
||||
if let Some(msg) = errors.first().and_then(|e| e.get("message")).and_then(Value::as_str) {
|
||||
warn!(error = %msg, "GraphQL language fetch had errors");
|
||||
}
|
||||
}
|
||||
|
||||
let data_obj = match data.get("data") {
|
||||
Some(d) => d,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let mut languages = Vec::new();
|
||||
for (i, repo) in chunk.iter().enumerate() {
|
||||
let alias = format!("r{i}");
|
||||
let edges = data_obj
|
||||
.get(&alias)
|
||||
.and_then(|r| r.get("languages"))
|
||||
.and_then(|l| l.get("edges"))
|
||||
.and_then(Value::as_array);
|
||||
if let Some(edges) = edges {
|
||||
for edge in edges {
|
||||
let size = edge.get("size").and_then(Value::as_i64).unwrap_or(0);
|
||||
let name = edge
|
||||
.get("node")
|
||||
.and_then(|n| n.get("name"))
|
||||
.and_then(Value::as_str);
|
||||
let color = edge
|
||||
.get("node")
|
||||
.and_then(|n| n.get("color"))
|
||||
.and_then(Value::as_str);
|
||||
if let Some(name) = name {
|
||||
languages.push(RepoLanguage {
|
||||
source: Source::Github,
|
||||
repo: repo.full_name.clone(),
|
||||
language: name.to_string(),
|
||||
bytes: size,
|
||||
color: color.map(String::from),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
total += self.writer.upsert_repo_languages(&languages).await?;
|
||||
}
|
||||
|
||||
debug!(total, "repo languages updated");
|
||||
Ok(total)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -327,6 +426,10 @@ impl EventSource for GithubRepoSource {
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(e) = self.fetch_languages(&repos).await {
|
||||
warn!(error = %e, "language fetch failed; continuing");
|
||||
}
|
||||
|
||||
self.state.touch(SOURCE_NAME).await?;
|
||||
debug!(ingested = total, repos = repos.len(), "github-repo poll complete");
|
||||
Ok(total)
|
||||
|
||||
Reference in New Issue
Block a user