fix: make the workspace pass the CI lint/test gate
Some checks failed
deploy / Build api + worker + web (push) Failing after 5m59s
deploy / Deploy moments-api to nikola (push) Has been skipped
deploy / Deploy moments-worker to frootmig (push) Has been skipped
deploy / Deploy web to oolon (push) Has been skipped

The new Gitea Actions build gate runs `cargo fmt --check`, `clippy -D warnings`,
and `cargo test` — stricter than the old deploy.sh, which only `cargo build`d.
That surfaced pre-existing drift that never compiled under the test/clippy
profile:

- apply rustfmt across the workspace (formatting only, no logic changes)
- moments-data: add the missing `prune_events` to the test-only `NoopWriter`
  stub (the EventWriter trait gained it with the blog-prune feature; a plain
  `cargo build` never compiles the `#[cfg(test)]` stub, so it went stale)
- moments-api: `.max().min()` -> `.clamp()`, and build `usvg::Options` with
  struct-update syntax instead of post-Default field assignment

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01X7zF7Kf4JqDwa6M8Qgge9M
This commit is contained in:
2026-06-25 13:00:40 +03:00
parent 1b753f991f
commit 3761333ac4
16 changed files with 378 additions and 191 deletions

View File

@@ -11,7 +11,10 @@ use chrono::{DateTime, Datelike, NaiveDate, Utc};
use clap::Parser; use clap::Parser;
use moments_core::{EventReader, reshape}; use moments_core::{EventReader, reshape};
use moments_data::PgStore; use moments_data::PgStore;
use moments_entities::{BlogPost, BlogPostSummary, DailyCount, Event, EventQuery, HourlyAvg, LanguageDailyCount, ProjectSummary, RepoLanguage, Source, SourceSummary, TimelineItem}; use moments_entities::{
BlogPost, BlogPostSummary, DailyCount, Event, EventQuery, HourlyAvg, LanguageDailyCount,
ProjectSummary, RepoLanguage, Source, SourceSummary, TimelineItem,
};
use serde::Deserialize; use serde::Deserialize;
use tower_http::{cors::CorsLayer, trace::TraceLayer}; use tower_http::{cors::CorsLayer, trace::TraceLayer};
use tracing::info; use tracing::info;
@@ -104,11 +107,7 @@ async fn list_events(
State(state): State<AppState>, State(state): State<AppState>,
Query(params): Query<EventsQueryParams>, Query(params): Query<EventsQueryParams>,
) -> Result<Json<Vec<TimelineItem>>, ApiError> { ) -> Result<Json<Vec<TimelineItem>>, ApiError> {
let sources = params let sources = params.source.as_deref().map(parse_sources).transpose()?;
.source
.as_deref()
.map(parse_sources)
.transpose()?;
let limit = params.limit.unwrap_or(100).clamp(1, 1000); let limit = params.limit.unwrap_or(100).clamp(1, 1000);
@@ -128,9 +127,7 @@ async fn list_events(
Ok(Json(items)) Ok(Json(items))
} }
async fn list_sources( async fn list_sources(State(state): State<AppState>) -> Result<Json<Vec<SourceSummary>>, ApiError> {
State(state): State<AppState>,
) -> Result<Json<Vec<SourceSummary>>, ApiError> {
let summaries = state let summaries = state
.store .store
.source_summaries(/* include_private */ true) .source_summaries(/* include_private */ true)
@@ -161,7 +158,11 @@ async fn blog_events(state: &AppState) -> Result<Vec<Event>, ApiError> {
} }
fn payload_str<'a>(event: &'a Event, key: &str) -> &'a str { fn payload_str<'a>(event: &'a Event, key: &str) -> &'a str {
event.payload.get(key).and_then(|v| v.as_str()).unwrap_or("") event
.payload
.get(key)
.and_then(|v| v.as_str())
.unwrap_or("")
} }
async fn list_blog_posts( async fn list_blog_posts(
@@ -212,8 +213,14 @@ async fn daily_counts(
Query(params): Query<DailyCountsParams>, Query(params): Query<DailyCountsParams>,
) -> Result<Json<Vec<DailyCount>>, ApiError> { ) -> Result<Json<Vec<DailyCount>>, ApiError> {
let to = params.to.unwrap_or_else(|| Utc::now().date_naive()); let to = params.to.unwrap_or_else(|| Utc::now().date_naive());
let from = params.from.unwrap_or_else(|| to - chrono::Duration::days(365)); let from = params
let counts = state.store.daily_counts(from, to, /* include_private */ true).await.map_err(internal)?; .from
.unwrap_or_else(|| to - chrono::Duration::days(365));
let counts = state
.store
.daily_counts(from, to, /* include_private */ true)
.await
.map_err(internal)?;
Ok(Json(counts)) Ok(Json(counts))
} }
@@ -222,8 +229,14 @@ async fn language_daily_counts(
Query(params): Query<DailyCountsParams>, Query(params): Query<DailyCountsParams>,
) -> Result<Json<Vec<LanguageDailyCount>>, ApiError> { ) -> Result<Json<Vec<LanguageDailyCount>>, ApiError> {
let to = params.to.unwrap_or_else(|| Utc::now().date_naive()); let to = params.to.unwrap_or_else(|| Utc::now().date_naive());
let from = params.from.unwrap_or_else(|| to - chrono::Duration::days(365)); let from = params
let counts = state.store.language_daily_counts(from, to, /* include_private */ true).await.map_err(internal)?; .from
.unwrap_or_else(|| to - chrono::Duration::days(365));
let counts = state
.store
.language_daily_counts(from, to, /* include_private */ true)
.await
.map_err(internal)?;
Ok(Json(counts)) Ok(Json(counts))
} }
@@ -242,20 +255,30 @@ async fn hourly_avgs(
Query(params): Query<HourlyAvgsParams>, Query(params): Query<HourlyAvgsParams>,
) -> Result<Json<Vec<HourlyAvg>>, ApiError> { ) -> Result<Json<Vec<HourlyAvg>>, ApiError> {
let to = params.to.unwrap_or_else(|| Utc::now().date_naive()); let to = params.to.unwrap_or_else(|| Utc::now().date_naive());
let from = params.from.unwrap_or_else(|| to - chrono::Duration::days(365)); let from = params
.from
.unwrap_or_else(|| to - chrono::Duration::days(365));
let tz = params.tz.as_deref().unwrap_or("UTC"); let tz = params.tz.as_deref().unwrap_or("UTC");
// Validate the tz string before handing it to postgres — a bad name // Validate the tz string before handing it to postgres — a bad name
// here would surface as an opaque 500 from the DB. chrono-tz would do // here would surface as an opaque 500 from the DB. chrono-tz would do
// it for free but we don't depend on it; instead reject obvious shell // it for free but we don't depend on it; instead reject obvious shell
// injection vectors (the value is bound, not interpolated, so this is // injection vectors (the value is bound, not interpolated, so this is
// belt-and-braces). // belt-and-braces).
if tz.len() > 64 || tz.chars().any(|c| !(c.is_ascii_alphanumeric() || matches!(c, '/' | '_' | '+' | '-'))) { if tz.len() > 64
|| tz
.chars()
.any(|c| !(c.is_ascii_alphanumeric() || matches!(c, '/' | '_' | '+' | '-')))
{
return Err(ApiError { return Err(ApiError {
status: StatusCode::BAD_REQUEST, status: StatusCode::BAD_REQUEST,
message: "invalid tz".into(), message: "invalid tz".into(),
}); });
} }
let avgs = state.store.hourly_avgs(from, to, tz, /* include_private */ true).await.map_err(internal)?; let avgs = state
.store
.hourly_avgs(from, to, tz, /* include_private */ true)
.await
.map_err(internal)?;
Ok(Json(avgs)) Ok(Json(avgs))
} }
@@ -266,9 +289,7 @@ async fn repo_languages(
Ok(Json(langs)) Ok(Json(langs))
} }
async fn og_contributions( async fn og_contributions(State(state): State<AppState>) -> Result<impl IntoResponse, ApiError> {
State(state): State<AppState>,
) -> Result<impl IntoResponse, ApiError> {
// Get date range from source summaries // Get date range from source summaries
let summaries = state let summaries = state
.store .store
@@ -292,7 +313,8 @@ async fn og_contributions(
let projects = state.store.list_projects().await.map_err(internal)?; let projects = state.store.list_projects().await.map_err(internal)?;
let repo_count = projects.len(); let repo_count = projects.len();
let png = render_contributions_png(&counts, earliest, today, repo_count).map_err(|e| ApiError { let png =
render_contributions_png(&counts, earliest, today, repo_count).map_err(|e| ApiError {
status: StatusCode::INTERNAL_SERVER_ERROR, status: StatusCode::INTERNAL_SERVER_ERROR,
message: e, message: e,
})?; })?;
@@ -332,7 +354,13 @@ fn render_contributions_png(
let cell = step - gap; let cell = step - gap;
let radius = cell / 2.0; let radius = cell / 2.0;
let colors = ["rgba(255,255,255,0.05)", "#0e4429", "#006d32", "#26a641", "#39d353"]; let colors = [
"rgba(255,255,255,0.05)",
"#0e4429",
"#006d32",
"#26a641",
"#39d353",
];
// Build weekly data per year // Build weekly data per year
struct YearRow { struct YearRow {
@@ -377,16 +405,24 @@ fn render_contributions_png(
let thresholds = if non_zero.is_empty() { let thresholds = if non_zero.is_empty() {
[1i64, 2, 3] [1i64, 2, 3]
} else { } else {
let p = |pct: f64| non_zero[(pct * non_zero.len() as f64).min(non_zero.len() as f64 - 1.0) as usize]; let p = |pct: f64| {
non_zero[(pct * non_zero.len() as f64).min(non_zero.len() as f64 - 1.0) as usize]
};
[p(0.25), p(0.5), p(0.75)] [p(0.25), p(0.5), p(0.75)]
}; };
let color_for = |count: i64| -> &str { let color_for = |count: i64| -> &str {
if count == 0 { colors[0] } if count == 0 {
else if count <= thresholds[0] { colors[1] } colors[0]
else if count <= thresholds[1] { colors[2] } } else if count <= thresholds[0] {
else if count <= thresholds[2] { colors[3] } colors[1]
else { colors[4] } } else if count <= thresholds[1] {
colors[2]
} else if count <= thresholds[2] {
colors[3]
} else {
colors[4]
}
}; };
let n_rows = rows.len(); let n_rows = rows.len();
@@ -425,7 +461,7 @@ fn render_contributions_png(
y = subtitle_y, y = subtitle_y,
)); ));
let label_font_size = (step * 0.7).round().max(8.0).min(14.0); let label_font_size = (step * 0.7).round().clamp(8.0, 14.0);
for (row_idx, row) in rows.iter().enumerate() { for (row_idx, row) in rows.iter().enumerate() {
let y_base = graph_y + (row_idx as f64) * step; let y_base = graph_y + (row_idx as f64) * step;
@@ -452,20 +488,23 @@ fn render_contributions_png(
// Rasterize at 1200x630 // Rasterize at 1200x630
let mut fontdb = fontdb::Database::new(); let mut fontdb = fontdb::Database::new();
fontdb.load_system_fonts(); fontdb.load_system_fonts();
let mut opts = resvg::usvg::Options::default(); let opts = resvg::usvg::Options {
opts.fontdb = std::sync::Arc::new(fontdb); fontdb: std::sync::Arc::new(fontdb),
opts.font_family = "Noto Sans".to_owned(); font_family: "Noto Sans".to_owned(),
let tree = resvg::usvg::Tree::from_str(&svg, &opts) ..Default::default()
.map_err(|e| format!("svg parse: {e}"))?; };
let tree = resvg::usvg::Tree::from_str(&svg, &opts).map_err(|e| format!("svg parse: {e}"))?;
let mut pixmap = let mut pixmap = resvg::tiny_skia::Pixmap::new(og_w as u32, og_h as u32)
resvg::tiny_skia::Pixmap::new(og_w as u32, og_h as u32).ok_or_else(|| "pixmap alloc failed".to_string())?; .ok_or_else(|| "pixmap alloc failed".to_string())?;
resvg::render(&tree, resvg::tiny_skia::Transform::default(), &mut pixmap.as_mut()); resvg::render(
&tree,
resvg::tiny_skia::Transform::default(),
&mut pixmap.as_mut(),
);
pixmap pixmap.encode_png().map_err(|e| format!("png encode: {e}"))
.encode_png()
.map_err(|e| format!("png encode: {e}"))
} }
/// Allowlisted forge hosts that the proxy may contact. /// Allowlisted forge hosts that the proxy may contact.
@@ -492,7 +531,11 @@ async fn forge_proxy(
} }
(format!("https://{host}"), "/api/v1") (format!("https://{host}"), "/api/v1")
} }
_ => return Err(ApiError::bad_request(format!("unsupported source: {source}"))), _ => {
return Err(ApiError::bad_request(format!(
"unsupported source: {source}"
)));
}
}; };
let url = format!("{base}{api_prefix}/{rest}"); let url = format!("{base}{api_prefix}/{rest}");
@@ -528,7 +571,10 @@ fn parse_sources(raw: &str) -> Result<Vec<Source>, ApiError> {
raw.split(',') raw.split(',')
.map(str::trim) .map(str::trim)
.filter(|s| !s.is_empty()) .filter(|s| !s.is_empty())
.map(|s| s.parse::<Source>().map_err(|e| ApiError::bad_request(e.to_string()))) .map(|s| {
s.parse::<Source>()
.map_err(|e| ApiError::bad_request(e.to_string()))
})
.collect() .collect()
} }
@@ -557,6 +603,10 @@ fn internal<E: std::fmt::Display>(e: E) -> ApiError {
impl IntoResponse for ApiError { impl IntoResponse for ApiError {
fn into_response(self) -> axum::response::Response { fn into_response(self) -> axum::response::Response {
(self.status, Json(serde_json::json!({ "error": self.message }))).into_response() (
self.status,
Json(serde_json::json!({ "error": self.message })),
)
.into_response()
} }
} }

View File

@@ -6,7 +6,10 @@ pub use sources::{EventSource, PollerState, PollerStateStore, SourceError, run_p
use async_trait::async_trait; use async_trait::async_trait;
use chrono::NaiveDate; use chrono::NaiveDate;
use moments_entities::{DailyCount, Event, EventQuery, HourlyAvg, LanguageDailyCount, ProjectSummary, RepoLanguage, SourceSummary}; use moments_entities::{
DailyCount, Event, EventQuery, HourlyAvg, LanguageDailyCount, ProjectSummary, RepoLanguage,
SourceSummary,
};
#[derive(Debug, thiserror::Error)] #[derive(Debug, thiserror::Error)]
pub enum StoreError { pub enum StoreError {
@@ -18,11 +21,30 @@ pub enum StoreError {
#[async_trait] #[async_trait]
pub trait EventReader: Send + Sync { pub trait EventReader: Send + Sync {
async fn list_events(&self, query: &EventQuery) -> Result<Vec<Event>, StoreError>; async fn list_events(&self, query: &EventQuery) -> Result<Vec<Event>, StoreError>;
async fn source_summaries(&self, include_private: bool) -> Result<Vec<SourceSummary>, StoreError>; async fn source_summaries(
&self,
include_private: bool,
) -> Result<Vec<SourceSummary>, StoreError>;
async fn list_projects(&self) -> Result<Vec<ProjectSummary>, StoreError>; async fn list_projects(&self) -> Result<Vec<ProjectSummary>, StoreError>;
async fn daily_counts(&self, from: NaiveDate, to: NaiveDate, include_private: bool) -> Result<Vec<DailyCount>, StoreError>; async fn daily_counts(
async fn language_daily_counts(&self, from: NaiveDate, to: NaiveDate, include_private: bool) -> Result<Vec<LanguageDailyCount>, StoreError>; &self,
async fn hourly_avgs(&self, from: NaiveDate, to: NaiveDate, tz: &str, include_private: bool) -> Result<Vec<HourlyAvg>, StoreError>; from: NaiveDate,
to: NaiveDate,
include_private: bool,
) -> Result<Vec<DailyCount>, StoreError>;
async fn language_daily_counts(
&self,
from: NaiveDate,
to: NaiveDate,
include_private: bool,
) -> Result<Vec<LanguageDailyCount>, StoreError>;
async fn hourly_avgs(
&self,
from: NaiveDate,
to: NaiveDate,
tz: &str,
include_private: bool,
) -> Result<Vec<HourlyAvg>, StoreError>;
async fn repo_languages(&self) -> Result<Vec<RepoLanguage>, StoreError>; async fn repo_languages(&self) -> Result<Vec<RepoLanguage>, StoreError>;
} }
@@ -34,5 +56,9 @@ pub trait EventWriter: Send + Sync {
/// Delete events of `source` whose id is not in `keep_ids`. For sources /// Delete events of `source` whose id is not in `keep_ids`. For sources
/// whose upstream is authoritative for the full set (e.g. the blog repo), /// whose upstream is authoritative for the full set (e.g. the blog repo),
/// this reconciles deletes and renames that upserts alone never would. /// this reconciles deletes and renames that upserts alone never would.
async fn prune_events(&self, source: moments_entities::Source, keep_ids: &[String]) -> Result<usize, StoreError>; async fn prune_events(
&self,
source: moments_entities::Source,
keep_ids: &[String],
) -> Result<usize, StoreError>;
} }

View File

@@ -108,7 +108,8 @@ mod tests {
#[test] #[test]
fn excerpt_skips_headings_and_images() { fn excerpt_skips_headings_and_images() {
let md = "# title\n\n![alt](img.jpg)\n\nfirst real paragraph\nwith a wrapped line\n\nsecond"; let md =
"# title\n\n![alt](img.jpg)\n\nfirst real paragraph\nwith a wrapped line\n\nsecond";
assert_eq!(excerpt(md), "first real paragraph with a wrapped line"); assert_eq!(excerpt(md), "first real paragraph with a wrapped line");
} }

View File

@@ -73,7 +73,9 @@ mod tests {
assert!(r.contains("filed bug #1158879 in mozilla.org"), "got: {r}"); assert!(r.contains("filed bug #1158879 in mozilla.org"), "got: {r}");
assert_eq!( assert_eq!(
item.subtitle.unwrap(), item.subtitle.unwrap(),
vec![TitleSegment::text("Commit Access (Level 1) for Rob Thijssen")] vec![TitleSegment::text(
"Commit Access (Level 1) for Rob Thijssen"
)]
); );
} }
} }

View File

@@ -40,9 +40,7 @@ pub(crate) fn reshape(event: &Event) -> TimelineItem {
"create_pull_request" => { "create_pull_request" => {
pr_action("opened", TimelineIcon::PullRequest, host, repo, content) pr_action("opened", TimelineIcon::PullRequest, host, repo, content)
} }
"close_pull_request" => { "close_pull_request" => pr_action("closed", TimelineIcon::PullRequest, host, repo, content),
pr_action("closed", TimelineIcon::PullRequest, host, repo, content)
}
"reopen_pull_request" => { "reopen_pull_request" => {
pr_action("reopened", TimelineIcon::PullRequest, host, repo, content) pr_action("reopened", TimelineIcon::PullRequest, host, repo, content)
} }
@@ -53,15 +51,13 @@ pub(crate) fn reshape(event: &Event) -> TimelineItem {
"approve_pull_request" => { "approve_pull_request" => {
pr_action("approved", TimelineIcon::PullRequest, host, repo, content) pr_action("approved", TimelineIcon::PullRequest, host, repo, content)
} }
"reject_pull_request" => { "reject_pull_request" => pr_action(
pr_action(
"requested changes on", "requested changes on",
TimelineIcon::PullRequest, TimelineIcon::PullRequest,
host, host,
repo, repo,
content, content,
) ),
}
"publish_release" => publish_release(host, repo, content), "publish_release" => publish_release(host, repo, content),
_ => fallback(host, repo, &event.action), _ => fallback(host, repo, &event.action),
}; };
@@ -303,8 +299,7 @@ fn pr_action(
TitleSegment::text(" in "), TitleSegment::text(" in "),
repo_link(host, repo), repo_link(host, repo),
]; ];
let subtitle = let subtitle = (!pr_title.is_empty()).then(|| vec![TitleSegment::text(pr_title.to_string())]);
(!pr_title.is_empty()).then(|| vec![TitleSegment::text(pr_title.to_string())]);
(icon, title, subtitle, None) (icon, title, subtitle, None)
} }
@@ -352,8 +347,7 @@ fn comment_on_pr(
TitleSegment::text(" in "), TitleSegment::text(" in "),
repo_link(host, repo), repo_link(host, repo),
]; ];
let subtitle = let subtitle = (!pr_title.is_empty()).then(|| vec![TitleSegment::text(pr_title.to_string())]);
(!pr_title.is_empty()).then(|| vec![TitleSegment::text(pr_title.to_string())]);
let body = (!body_text.is_empty()).then(|| TimelineBody::Markdown { let body = (!body_text.is_empty()).then(|| TimelineBody::Markdown {
text: body_text.to_string(), text: body_text.to_string(),
}); });
@@ -364,7 +358,10 @@ fn publish_release(host: &str, repo: Option<&str>, content: Option<&str>) -> Res
let repo = repo.unwrap_or("(unknown repo)"); let repo = repo.unwrap_or("(unknown repo)");
let name = content.unwrap_or(""); let name = content.unwrap_or("");
let title = if name.is_empty() { let title = if name.is_empty() {
vec![TitleSegment::text("published a release in "), repo_link(host, repo)] vec![
TitleSegment::text("published a release in "),
repo_link(host, repo),
]
} else { } else {
vec![ vec![
TitleSegment::text(format!("released {name} in ")), TitleSegment::text(format!("released {name} in ")),
@@ -452,10 +449,7 @@ mod tests {
let item = reshape(&ev("create_issue", raw)); let item = reshape(&ev("create_issue", raw));
assert_eq!(item.icon, TimelineIcon::Issue); assert_eq!(item.icon, TimelineIcon::Issue);
let r = render(&item); let r = render(&item);
assert!( assert!(r.contains("opened issue #1 in grenade/moments"), "got: {r}");
r.contains("opened issue #1 in grenade/moments"),
"got: {r}"
);
assert_eq!( assert_eq!(
item.subtitle.unwrap(), item.subtitle.unwrap(),
vec![TitleSegment::text( vec![TitleSegment::text(

View File

@@ -13,7 +13,10 @@ pub(crate) fn reshape(event: &Event) -> TimelineItem {
} }
let p = &event.payload; let p = &event.payload;
let repo_name = p.get("repo").and_then(|r| r.get("name")).and_then(Value::as_str); let repo_name = p
.get("repo")
.and_then(|r| r.get("name"))
.and_then(Value::as_str);
let actor_login = p let actor_login = p
.get("actor") .get("actor")
.and_then(|a| a.get("display_login").or_else(|| a.get("login"))) .and_then(|a| a.get("display_login").or_else(|| a.get("login")))
@@ -192,8 +195,14 @@ fn pull_request(repo: Option<&str>, p: Option<&Value>) -> Reshaped {
.and_then(Value::as_str) .and_then(Value::as_str)
.unwrap_or("touched"); .unwrap_or("touched");
let pr = p.and_then(|v| v.get("pull_request")); let pr = p.and_then(|v| v.get("pull_request"));
let number = p.and_then(|v| v.get("number")).and_then(Value::as_i64).unwrap_or(0); let number = p
let pr_title = pr.and_then(|v| v.get("title")).and_then(Value::as_str).unwrap_or(""); .and_then(|v| v.get("number"))
.and_then(Value::as_i64)
.unwrap_or(0);
let pr_title = pr
.and_then(|v| v.get("title"))
.and_then(Value::as_str)
.unwrap_or("");
let merged = pr let merged = pr
.and_then(|v| v.get("merged")) .and_then(|v| v.get("merged"))
.and_then(Value::as_bool) .and_then(Value::as_bool)
@@ -224,8 +233,14 @@ fn pull_request(repo: Option<&str>, p: Option<&Value>) -> Reshaped {
fn pull_request_review(repo: Option<&str>, p: Option<&Value>) -> Reshaped { fn pull_request_review(repo: Option<&str>, p: Option<&Value>) -> Reshaped {
let repo = repo.unwrap_or("(unknown repo)"); let repo = repo.unwrap_or("(unknown repo)");
let pr = p.and_then(|v| v.get("pull_request")); let pr = p.and_then(|v| v.get("pull_request"));
let number = pr.and_then(|v| v.get("number")).and_then(Value::as_i64).unwrap_or(0); let number = pr
let pr_title = pr.and_then(|v| v.get("title")).and_then(Value::as_str).unwrap_or(""); .and_then(|v| v.get("number"))
.and_then(Value::as_i64)
.unwrap_or(0);
let pr_title = pr
.and_then(|v| v.get("title"))
.and_then(Value::as_str)
.unwrap_or("");
let state = p let state = p
.and_then(|v| v.get("review")) .and_then(|v| v.get("review"))
.and_then(|r| r.get("state")) .and_then(|r| r.get("state"))
@@ -245,8 +260,14 @@ fn pull_request_review(repo: Option<&str>, p: Option<&Value>) -> Reshaped {
fn pull_request_review_comment(repo: Option<&str>, p: Option<&Value>) -> Reshaped { fn pull_request_review_comment(repo: Option<&str>, p: Option<&Value>) -> Reshaped {
let repo = repo.unwrap_or("(unknown repo)"); let repo = repo.unwrap_or("(unknown repo)");
let pr = p.and_then(|v| v.get("pull_request")); let pr = p.and_then(|v| v.get("pull_request"));
let number = pr.and_then(|v| v.get("number")).and_then(Value::as_i64).unwrap_or(0); let number = pr
let pr_title = pr.and_then(|v| v.get("title")).and_then(Value::as_str).unwrap_or(""); .and_then(|v| v.get("number"))
.and_then(Value::as_i64)
.unwrap_or(0);
let pr_title = pr
.and_then(|v| v.get("title"))
.and_then(Value::as_str)
.unwrap_or("");
let body_text = p let body_text = p
.and_then(|v| v.get("comment")) .and_then(|v| v.get("comment"))
.and_then(|c| c.get("body")) .and_then(|c| c.get("body"))
@@ -273,8 +294,14 @@ fn issues(repo: Option<&str>, p: Option<&Value>) -> Reshaped {
.and_then(Value::as_str) .and_then(Value::as_str)
.unwrap_or("touched"); .unwrap_or("touched");
let issue = p.and_then(|v| v.get("issue")); let issue = p.and_then(|v| v.get("issue"));
let number = issue.and_then(|v| v.get("number")).and_then(Value::as_i64).unwrap_or(0); let number = issue
let issue_title = issue.and_then(|v| v.get("title")).and_then(Value::as_str).unwrap_or(""); .and_then(|v| v.get("number"))
.and_then(Value::as_i64)
.unwrap_or(0);
let issue_title = issue
.and_then(|v| v.get("title"))
.and_then(Value::as_str)
.unwrap_or("");
let title = vec![ let title = vec![
TitleSegment::text(format!("{action} issue ")), TitleSegment::text(format!("{action} issue ")),
@@ -282,15 +309,22 @@ fn issues(repo: Option<&str>, p: Option<&Value>) -> Reshaped {
TitleSegment::text(" in "), TitleSegment::text(" in "),
repo_link(repo), repo_link(repo),
]; ];
let subtitle = (!issue_title.is_empty()).then(|| vec![TitleSegment::text(issue_title.to_string())]); let subtitle =
(!issue_title.is_empty()).then(|| vec![TitleSegment::text(issue_title.to_string())]);
(TimelineIcon::Issue, title, subtitle, None) (TimelineIcon::Issue, title, subtitle, None)
} }
fn issue_comment(repo: Option<&str>, p: Option<&Value>) -> Reshaped { fn issue_comment(repo: Option<&str>, p: Option<&Value>) -> Reshaped {
let repo = repo.unwrap_or("(unknown repo)"); let repo = repo.unwrap_or("(unknown repo)");
let issue = p.and_then(|v| v.get("issue")); let issue = p.and_then(|v| v.get("issue"));
let number = issue.and_then(|v| v.get("number")).and_then(Value::as_i64).unwrap_or(0); let number = issue
let issue_title = issue.and_then(|v| v.get("title")).and_then(Value::as_str).unwrap_or(""); .and_then(|v| v.get("number"))
.and_then(Value::as_i64)
.unwrap_or(0);
let issue_title = issue
.and_then(|v| v.get("title"))
.and_then(Value::as_str)
.unwrap_or("");
let body_text = p let body_text = p
.and_then(|v| v.get("comment")) .and_then(|v| v.get("comment"))
.and_then(|c| c.get("body")) .and_then(|c| c.get("body"))
@@ -303,7 +337,8 @@ fn issue_comment(repo: Option<&str>, p: Option<&Value>) -> Reshaped {
TitleSegment::text(" in "), TitleSegment::text(" in "),
repo_link(repo), repo_link(repo),
]; ];
let subtitle = (!issue_title.is_empty()).then(|| vec![TitleSegment::text(issue_title.to_string())]); let subtitle =
(!issue_title.is_empty()).then(|| vec![TitleSegment::text(issue_title.to_string())]);
let body = (!body_text.is_empty()).then(|| TimelineBody::Markdown { let body = (!body_text.is_empty()).then(|| TimelineBody::Markdown {
text: body_text.to_string(), text: body_text.to_string(),
}); });
@@ -312,7 +347,10 @@ fn issue_comment(repo: Option<&str>, p: Option<&Value>) -> Reshaped {
fn create(repo: Option<&str>, p: Option<&Value>) -> Reshaped { fn create(repo: Option<&str>, p: Option<&Value>) -> Reshaped {
let repo = repo.unwrap_or("(unknown repo)"); let repo = repo.unwrap_or("(unknown repo)");
let ref_type = p.and_then(|v| v.get("ref_type")).and_then(Value::as_str).unwrap_or("ref"); let ref_type = p
.and_then(|v| v.get("ref_type"))
.and_then(Value::as_str)
.unwrap_or("ref");
let ref_name = p.and_then(|v| v.get("ref")).and_then(Value::as_str); let ref_name = p.and_then(|v| v.get("ref")).and_then(Value::as_str);
let mut title = vec![TitleSegment::text(format!("created {ref_type} "))]; let mut title = vec![TitleSegment::text(format!("created {ref_type} "))];
@@ -327,8 +365,14 @@ fn create(repo: Option<&str>, p: Option<&Value>) -> Reshaped {
fn delete(repo: Option<&str>, p: Option<&Value>) -> Reshaped { fn delete(repo: Option<&str>, p: Option<&Value>) -> Reshaped {
let repo = repo.unwrap_or("(unknown repo)"); let repo = repo.unwrap_or("(unknown repo)");
let ref_type = p.and_then(|v| v.get("ref_type")).and_then(Value::as_str).unwrap_or("ref"); let ref_type = p
let ref_name = p.and_then(|v| v.get("ref")).and_then(Value::as_str).unwrap_or(""); .and_then(|v| v.get("ref_type"))
.and_then(Value::as_str)
.unwrap_or("ref");
let ref_name = p
.and_then(|v| v.get("ref"))
.and_then(Value::as_str)
.unwrap_or("");
let title = vec![ let title = vec![
TitleSegment::text(format!("deleted {ref_type} {ref_name} in ")), TitleSegment::text(format!("deleted {ref_type} {ref_name} in ")),
@@ -340,7 +384,9 @@ fn delete(repo: Option<&str>, p: Option<&Value>) -> Reshaped {
fn fork(repo: Option<&str>, p: Option<&Value>) -> Reshaped { fn fork(repo: Option<&str>, p: Option<&Value>) -> Reshaped {
let repo = repo.unwrap_or("(unknown repo)"); let repo = repo.unwrap_or("(unknown repo)");
let forkee = p.and_then(|v| v.get("forkee")); let forkee = p.and_then(|v| v.get("forkee"));
let forkee_full = forkee.and_then(|f| f.get("full_name")).and_then(Value::as_str); let forkee_full = forkee
.and_then(|f| f.get("full_name"))
.and_then(Value::as_str);
let mut title = vec![TitleSegment::text("forked "), repo_link(repo)]; let mut title = vec![TitleSegment::text("forked "), repo_link(repo)];
if let Some(full) = forkee_full { if let Some(full) = forkee_full {
@@ -366,7 +412,9 @@ fn release(repo: Option<&str>, p: Option<&Value>) -> Reshaped {
.and_then(|r| r.get("name").or_else(|| r.get("tag_name"))) .and_then(|r| r.get("name").or_else(|| r.get("tag_name")))
.and_then(Value::as_str) .and_then(Value::as_str)
.unwrap_or("(release)"); .unwrap_or("(release)");
let url = release.and_then(|r| r.get("html_url")).and_then(Value::as_str); let url = release
.and_then(|r| r.get("html_url"))
.and_then(Value::as_str);
let label = if let Some(u) = url { let label = if let Some(u) = url {
TitleSegment::link(name.to_string(), u.to_string()) TitleSegment::link(name.to_string(), u.to_string())
@@ -389,7 +437,10 @@ fn commit_comment(repo: Option<&str>, p: Option<&Value>) -> Reshaped {
.and_then(|c| c.get("body")) .and_then(|c| c.get("body"))
.and_then(Value::as_str) .and_then(Value::as_str)
.unwrap_or(""); .unwrap_or("");
let title = vec![TitleSegment::text("commented on a commit in "), repo_link(repo)]; let title = vec![
TitleSegment::text("commented on a commit in "),
repo_link(repo),
];
let body = (!body_text.is_empty()).then(|| TimelineBody::Markdown { let body = (!body_text.is_empty()).then(|| TimelineBody::Markdown {
text: body_text.to_string(), text: body_text.to_string(),
}); });
@@ -398,7 +449,11 @@ fn commit_comment(repo: Option<&str>, p: Option<&Value>) -> Reshaped {
fn public(repo: Option<&str>) -> Reshaped { fn public(repo: Option<&str>) -> Reshaped {
let repo = repo.unwrap_or("(unknown repo)"); let repo = repo.unwrap_or("(unknown repo)");
let title = vec![TitleSegment::text("made "), repo_link(repo), TitleSegment::text(" public")]; let title = vec![
TitleSegment::text("made "),
repo_link(repo),
TitleSegment::text(" public"),
];
(TimelineIcon::Generic, title, None, None) (TimelineIcon::Generic, title, None, None)
} }
@@ -444,11 +499,15 @@ fn search_reshape(event: &Event) -> TimelineItem {
title.push(TitleSegment::text(" ")); title.push(TitleSegment::text(" "));
} }
title.push(TitleSegment::text(format!("{verb} {kind} "))); title.push(TitleSegment::text(format!("{verb} {kind} ")));
title.push(TitleSegment::link(format!("#{number}"), html_url.to_string())); title.push(TitleSegment::link(
format!("#{number}"),
html_url.to_string(),
));
title.push(TitleSegment::text(" in ")); title.push(TitleSegment::text(" in "));
title.push(repo_link(&repo)); title.push(repo_link(&repo));
let subtitle = (!issue_title.is_empty()).then(|| vec![TitleSegment::text(issue_title.to_string())]); let subtitle =
(!issue_title.is_empty()).then(|| vec![TitleSegment::text(issue_title.to_string())]);
TimelineItem { TimelineItem {
id: event.id.clone(), id: event.id.clone(),
@@ -500,8 +559,8 @@ fn commit_reshape(event: &Event) -> TimelineItem {
title.push(TitleSegment::text(" in ")); title.push(TitleSegment::text(" in "));
title.push(repo_link(repo)); title.push(repo_link(repo));
let subtitle = (!message_first_line.is_empty()) let subtitle =
.then(|| vec![TitleSegment::text(message_first_line)]); (!message_first_line.is_empty()).then(|| vec![TitleSegment::text(message_first_line)]);
TimelineItem { TimelineItem {
id: event.id.clone(), id: event.id.clone(),
@@ -525,10 +584,7 @@ fn repo_from_url(url: &str) -> Option<String> {
fn fallback(repo: Option<&str>, action: &str) -> Reshaped { fn fallback(repo: Option<&str>, action: &str) -> Reshaped {
let title = match repo { let title = match repo {
Some(r) => vec![ Some(r) => vec![TitleSegment::text(format!("{action} on ")), repo_link(r)],
TitleSegment::text(format!("{action} on ")),
repo_link(r),
],
None => vec![TitleSegment::text(action.to_string())], None => vec![TitleSegment::text(action.to_string())],
}; };
(TimelineIcon::Generic, title, None, None) (TimelineIcon::Generic, title, None, None)
@@ -578,7 +634,10 @@ mod tests {
TitleSegment::Link { text, .. } => text.clone(), TitleSegment::Link { text, .. } => text.clone(),
}) })
.collect(); .collect();
assert!(rendered.contains("pushed 2 commits to grenade/vortex:main"), "got: {rendered}"); assert!(
rendered.contains("pushed 2 commits to grenade/vortex:main"),
"got: {rendered}"
);
match item.body.unwrap() { match item.body.unwrap() {
TimelineBody::Commits { commits } => { TimelineBody::Commits { commits } => {
assert_eq!(commits.len(), 2); assert_eq!(commits.len(), 2);
@@ -640,7 +699,10 @@ mod tests {
let item = reshape(&ev("PushEvent", raw)); let item = reshape(&ev("PushEvent", raw));
assert_eq!(item.icon, TimelineIcon::GitPush); assert_eq!(item.icon, TimelineIcon::GitPush);
let r = render(&item); let r = render(&item);
assert!(r.contains("force-pushed 1 commit to grenade/x:main"), "got: {r}"); assert!(
r.contains("force-pushed 1 commit to grenade/x:main"),
"got: {r}"
);
} }
#[test] #[test]
@@ -784,11 +846,16 @@ mod tests {
TitleSegment::Link { text, .. } => text.clone(), TitleSegment::Link { text, .. } => text.clone(),
}) })
.collect(); .collect();
assert!(rendered.contains("committed a6fcefb in faith1337z/Trade"), "got: {rendered}"); assert!(
rendered.contains("committed a6fcefb in faith1337z/Trade"),
"got: {rendered}"
);
// body of the commit message is dropped; only first line in subtitle // body of the commit message is dropped; only first line in subtitle
assert_eq!( assert_eq!(
item.subtitle.unwrap(), item.subtitle.unwrap(),
vec![TitleSegment::text("split multiline message into multiple irc messages")] vec![TitleSegment::text(
"split multiline message into multiple irc messages"
)]
); );
} }

View File

@@ -23,10 +23,7 @@ pub(crate) fn reshape(event: &Event) -> TimelineItem {
.next() .next()
.unwrap_or("") .unwrap_or("")
.to_string(); .to_string();
let author = p let author = p.get("author").and_then(Value::as_str).map(author_name);
.get("author")
.and_then(Value::as_str)
.map(author_name);
let mut title = Vec::new(); let mut title = Vec::new();
if let Some(name) = author { if let Some(name) = author {

View File

@@ -21,11 +21,7 @@ use serde_json::{Value, json};
use tracing::{debug, warn}; use tracing::{debug, warn};
const SOURCE_NAME: &str = "blog"; const SOURCE_NAME: &str = "blog";
const USER_AGENT: &str = concat!( const USER_AGENT: &str = concat!("moments/", env!("CARGO_PKG_VERSION"), " (+https://rob.tn)");
"moments/",
env!("CARGO_PKG_VERSION"),
" (+https://rob.tn)"
);
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct BlogConfig { pub struct BlogConfig {
@@ -130,8 +126,7 @@ impl BlogSource {
.filter(|e| e.get("type").and_then(Value::as_str) == Some("file")) .filter(|e| e.get("type").and_then(Value::as_str) == Some("file"))
.filter_map(|e| e.get("path").and_then(Value::as_str)) .filter_map(|e| e.get("path").and_then(Value::as_str))
.filter(|p| { .filter(|p| {
p.to_ascii_lowercase().ends_with(".md") p.to_ascii_lowercase().ends_with(".md") && !p.eq_ignore_ascii_case("readme.md")
&& !p.eq_ignore_ascii_case("readme.md")
}) })
.map(String::from) .map(String::from)
.collect()) .collect())
@@ -202,7 +197,9 @@ struct Frontmatter {
/// open with a `---` fence on the first line. /// open with a `---` fence on the first line.
fn split_frontmatter(content: &str) -> Option<(&str, &str)> { fn split_frontmatter(content: &str) -> Option<(&str, &str)> {
let rest = content.strip_prefix("---")?; let rest = content.strip_prefix("---")?;
let rest = rest.strip_prefix('\n').or_else(|| rest.strip_prefix("\r\n"))?; let rest = rest
.strip_prefix('\n')
.or_else(|| rest.strip_prefix("\r\n"))?;
// The closing fence is a `---` alone on a line. // The closing fence is a `---` alone on a line.
let mut offset = 0; let mut offset = 0;
for line in rest.split_inclusive('\n') { for line in rest.split_inclusive('\n') {

View File

@@ -21,11 +21,7 @@ use serde_json::Value;
use tracing::debug; use tracing::debug;
const SOURCE_NAME: &str = "bugzilla"; const SOURCE_NAME: &str = "bugzilla";
const USER_AGENT: &str = concat!( const USER_AGENT: &str = concat!("moments/", env!("CARGO_PKG_VERSION"), " (+https://rob.tn)");
"moments/",
env!("CARGO_PKG_VERSION"),
" (+https://rob.tn)"
);
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct BugzillaConfig { pub struct BugzillaConfig {

View File

@@ -21,11 +21,7 @@ use serde_json::Value;
use tracing::debug; use tracing::debug;
const SOURCE_NAME: &str = "gitea"; const SOURCE_NAME: &str = "gitea";
const USER_AGENT: &str = concat!( const USER_AGENT: &str = concat!("moments/", env!("CARGO_PKG_VERSION"), " (+https://rob.tn)");
"moments/",
env!("CARGO_PKG_VERSION"),
" (+https://rob.tn)"
);
const MAX_BACKFILL_PAGES: u32 = 20; const MAX_BACKFILL_PAGES: u32 = 20;
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
@@ -289,7 +285,10 @@ fn parse_gitea_event(item: &Value, host: &str) -> Option<Event> {
let occurred_at = DateTime::parse_from_rfc3339(created_str) let occurred_at = DateTime::parse_from_rfc3339(created_str)
.ok()? .ok()?
.with_timezone(&Utc); .with_timezone(&Utc);
let private = item.get("is_private").and_then(Value::as_bool).unwrap_or(false); let private = item
.get("is_private")
.and_then(Value::as_bool)
.unwrap_or(false);
let id = gitea_canonical_id(item, &op_type, created_str); let id = gitea_canonical_id(item, &op_type, created_str);
@@ -315,12 +314,20 @@ fn gitea_canonical_id(item: &Value, op_type: &str, created: &str) -> String {
let act_user_id = item let act_user_id = item
.get("act_user_id") .get("act_user_id")
.and_then(Value::as_i64) .and_then(Value::as_i64)
.or_else(|| item.get("act_user").and_then(|u| u.get("id")).and_then(Value::as_i64)) .or_else(|| {
item.get("act_user")
.and_then(|u| u.get("id"))
.and_then(Value::as_i64)
})
.unwrap_or(0); .unwrap_or(0);
let repo_id = item let repo_id = item
.get("repo_id") .get("repo_id")
.and_then(Value::as_i64) .and_then(Value::as_i64)
.or_else(|| item.get("repo").and_then(|r| r.get("id")).and_then(Value::as_i64)) .or_else(|| {
item.get("repo")
.and_then(|r| r.get("id"))
.and_then(Value::as_i64)
})
.unwrap_or(0); .unwrap_or(0);
let ref_name = item.get("ref_name").and_then(Value::as_str).unwrap_or(""); let ref_name = item.get("ref_name").and_then(Value::as_str).unwrap_or("");
let comment_id = item.get("comment_id").and_then(Value::as_i64).unwrap_or(0); let comment_id = item.get("comment_id").and_then(Value::as_i64).unwrap_or(0);
@@ -346,7 +353,10 @@ mod tests {
"repo": { "id": 7, "full_name": "grenade/moments" } "repo": { "id": 7, "full_name": "grenade/moments" }
}); });
let ev = parse_gitea_event(&raw, "git.lair.cafe").expect("parses"); let ev = parse_gitea_event(&raw, "git.lair.cafe").expect("parses");
assert_eq!(ev.id, "gitea:commit_repo:42:7:refs/heads/main:0:2026-05-03T16:37:45Z"); assert_eq!(
ev.id,
"gitea:commit_repo:42:7:refs/heads/main:0:2026-05-03T16:37:45Z"
);
assert_eq!(ev.source, Source::Gitea); assert_eq!(ev.source, Source::Gitea);
assert_eq!(ev.action, "commit_repo"); assert_eq!(ev.action, "commit_repo");
assert!(ev.public); assert!(ev.public);

View File

@@ -8,11 +8,7 @@ use reqwest::{Client, StatusCode, header};
use tracing::debug; use tracing::debug;
const SOURCE_NAME: &str = "github"; const SOURCE_NAME: &str = "github";
const USER_AGENT: &str = concat!( const USER_AGENT: &str = concat!("moments/", env!("CARGO_PKG_VERSION"), " (+https://rob.tn)");
"moments/",
env!("CARGO_PKG_VERSION"),
" (+https://rob.tn)"
);
/// Cap on initial backfill pagination. GitHub returns ~300 events max /// Cap on initial backfill pagination. GitHub returns ~300 events max
/// across pages; this is a safety net, not an expected limit. /// across pages; this is a safety net, not an expected limit.
@@ -166,7 +162,9 @@ impl EventSource for GithubSource {
} }
} }
self.state.save(SOURCE_NAME, latest_etag.as_deref(), None).await?; self.state
.save(SOURCE_NAME, latest_etag.as_deref(), None)
.await?;
Ok(total) Ok(total)
} }
} }
@@ -182,7 +180,10 @@ fn parse_github_event(raw: serde_json::Value) -> Option<Event> {
// `/events/public` are always true; `/events` may include false. Default // `/events/public` are always true; `/events` may include false. Default
// to true if missing — that matches the safer-of-the-two-mistakes (under- // to true if missing — that matches the safer-of-the-two-mistakes (under-
// expose) and the `/events/public` endpoint behaviour. // expose) and the `/events/public` endpoint behaviour.
let public = raw.get("public").and_then(serde_json::Value::as_bool).unwrap_or(true); let public = raw
.get("public")
.and_then(serde_json::Value::as_bool)
.unwrap_or(true);
Some(Event { Some(Event {
id: format!("github:{id}"), id: format!("github:{id}"),
source: Source::Github, source: Source::Github,
@@ -201,7 +202,10 @@ fn parse_link_next(header: Option<&header::HeaderValue>) -> Option<String> {
let part = part.trim(); let part = part.trim();
// Each part: `<url>; rel="next"` // Each part: `<url>; rel="next"`
let (url_part, rel_part) = part.split_once(';')?; let (url_part, rel_part) = part.split_once(';')?;
let url = url_part.trim().trim_start_matches('<').trim_end_matches('>'); let url = url_part
.trim()
.trim_start_matches('<')
.trim_end_matches('>');
let rel = rel_part.trim(); let rel = rel_part.trim();
if rel.eq_ignore_ascii_case("rel=\"next\"") { if rel.eq_ignore_ascii_case("rel=\"next\"") {
return Some(url.to_string()); return Some(url.to_string());

View File

@@ -51,11 +51,7 @@ const BRANCH_ENCODE_SET: &AsciiSet = &CONTROLS
.add(b'%'); .add(b'%');
const SOURCE_NAME: &str = "github-repo"; const SOURCE_NAME: &str = "github-repo";
const USER_AGENT: &str = concat!( const USER_AGENT: &str = concat!("moments/", env!("CARGO_PKG_VERSION"), " (+https://rob.tn)");
"moments/",
env!("CARGO_PKG_VERSION"),
" (+https://rob.tn)"
);
const MAX_BACKFILL_PAGES: u32 = 100; const MAX_BACKFILL_PAGES: u32 = 100;
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
@@ -199,10 +195,7 @@ impl GithubRepoSource {
.map_err(|e| SourceError::Http(e.to_string()))?; .map_err(|e| SourceError::Http(e.to_string()))?;
if !resp.status().is_success() { if !resp.status().is_success() {
return Err(SourceError::Http(format!( return Err(SourceError::Http(format!("{} POST graphql", resp.status())));
"{} POST graphql",
resp.status()
)));
} }
let data: Value = resp let data: Value = resp
@@ -212,7 +205,11 @@ impl GithubRepoSource {
// Check for GraphQL-level errors // Check for GraphQL-level errors
if let Some(errors) = data.get("errors").and_then(Value::as_array) { if let Some(errors) = data.get("errors").and_then(Value::as_array) {
if let Some(msg) = errors.first().and_then(|e| e.get("message")).and_then(Value::as_str) { if let Some(msg) = errors
.first()
.and_then(|e| e.get("message"))
.and_then(Value::as_str)
{
return Err(SourceError::Http(format!("GraphQL error: {msg}"))); return Err(SourceError::Http(format!("GraphQL error: {msg}")));
} }
} }
@@ -221,9 +218,7 @@ impl GithubRepoSource {
let nodes = contributed["nodes"].as_array(); let nodes = contributed["nodes"].as_array();
if let Some(nodes) = nodes { if let Some(nodes) = nodes {
for node in nodes { for node in nodes {
let full_name = node let full_name = node.get("nameWithOwner").and_then(Value::as_str);
.get("nameWithOwner")
.and_then(Value::as_str);
let private = node let private = node
.get("isPrivate") .get("isPrivate")
.and_then(Value::as_bool) .and_then(Value::as_bool)
@@ -248,7 +243,10 @@ impl GithubRepoSource {
.map(String::from); .map(String::from);
} }
debug!(repos = repos.len(), "discovered contributed repos via GraphQL"); debug!(
repos = repos.len(),
"discovered contributed repos via GraphQL"
);
Ok(repos) Ok(repos)
} }
@@ -322,8 +320,14 @@ impl GithubRepoSource {
.await .await
.map_err(|e| SourceError::Parse(e.to_string()))?; .map_err(|e| SourceError::Parse(e.to_string()))?;
if let Some(errors) = data.get("errors").and_then(Value::as_array) { if let Some(errors) = data.get("errors").and_then(Value::as_array) {
if let Some(msg) = errors.first().and_then(|e| e.get("message")).and_then(Value::as_str) { if let Some(msg) = errors
return Err(SourceError::Http(format!("GraphQL error listing branches: {msg}"))); .first()
.and_then(|e| e.get("message"))
.and_then(Value::as_str)
{
return Err(SourceError::Http(format!(
"GraphQL error listing branches: {msg}"
)));
} }
} }
let refs = &data["data"]["repository"]["refs"]; let refs = &data["data"]["repository"]["refs"];
@@ -412,7 +416,10 @@ impl GithubRepoSource {
/// noise. /// noise.
async fn scan_repo(&self, repo: &Repo) -> Result<usize, SourceError> { async fn scan_repo(&self, repo: &Repo) -> Result<usize, SourceError> {
let branches = if self.config.token.is_some() { let branches = if self.config.token.is_some() {
match self.list_branches_with_commits(repo, &self.config.user).await { match self
.list_branches_with_commits(repo, &self.config.user)
.await
{
Ok(b) => b, Ok(b) => b,
Err(e) => { Err(e) => {
warn!(repo = %repo.full_name, error = %e, "graphql branch filter failed; falling back to REST"); warn!(repo = %repo.full_name, error = %e, "graphql branch filter failed; falling back to REST");
@@ -434,7 +441,9 @@ impl GithubRepoSource {
for branch in &branches { for branch in &branches {
match self.scan_repo_branch(repo, branch, &mut seen_in_tick).await { match self.scan_repo_branch(repo, branch, &mut seen_in_tick).await {
Ok(n) => total += n, Ok(n) => total += n,
Err(SourceError::Http(ref msg)) if msg.starts_with("403") || msg.starts_with("429") => { Err(SourceError::Http(ref msg))
if msg.starts_with("403") || msg.starts_with("429") =>
{
return Err(SourceError::Http(msg.clone())); return Err(SourceError::Http(msg.clone()));
} }
Err(e) => { Err(e) => {
@@ -596,7 +605,11 @@ impl GithubRepoSource {
.map_err(|e| SourceError::Parse(e.to_string()))?; .map_err(|e| SourceError::Parse(e.to_string()))?;
if let Some(errors) = data.get("errors").and_then(Value::as_array) { if let Some(errors) = data.get("errors").and_then(Value::as_array) {
if let Some(msg) = errors.first().and_then(|e| e.get("message")).and_then(Value::as_str) { if let Some(msg) = errors
.first()
.and_then(|e| e.get("message"))
.and_then(Value::as_str)
{
warn!(error = %msg, "GraphQL language fetch had errors"); warn!(error = %msg, "GraphQL language fetch had errors");
} }
} }
@@ -664,7 +677,9 @@ impl EventSource for GithubRepoSource {
} }
total += n; total += n;
} }
Err(SourceError::Http(ref msg)) if msg.starts_with("403") || msg.starts_with("429") => { Err(SourceError::Http(ref msg))
if msg.starts_with("403") || msg.starts_with("429") =>
{
warn!("rate limited during repo scan; ending poll early"); warn!("rate limited during repo scan; ending poll early");
break; break;
} }
@@ -679,7 +694,11 @@ impl EventSource for GithubRepoSource {
} }
self.state.touch(SOURCE_NAME).await?; self.state.touch(SOURCE_NAME).await?;
debug!(ingested = total, repos = repos.len(), "github-repo poll complete"); debug!(
ingested = total,
repos = repos.len(),
"github-repo poll complete"
);
Ok(total) Ok(total)
} }
} }
@@ -692,7 +711,10 @@ struct Repo {
fn parse_repo(item: &Value) -> Option<Repo> { fn parse_repo(item: &Value) -> Option<Repo> {
let full_name = item.get("full_name").and_then(Value::as_str)?; let full_name = item.get("full_name").and_then(Value::as_str)?;
let private = item.get("private").and_then(Value::as_bool).unwrap_or(false); let private = item
.get("private")
.and_then(Value::as_bool)
.unwrap_or(false);
Some(Repo { Some(Repo {
full_name: full_name.to_string(), full_name: full_name.to_string(),
private, private,

View File

@@ -26,11 +26,7 @@ use serde_json::Value;
use tracing::{debug, warn}; use tracing::{debug, warn};
const SOURCE_NAME: &str = "github-search"; const SOURCE_NAME: &str = "github-search";
const USER_AGENT: &str = concat!( const USER_AGENT: &str = concat!("moments/", env!("CARGO_PKG_VERSION"), " (+https://rob.tn)");
"moments/",
env!("CARGO_PKG_VERSION"),
" (+https://rob.tn)"
);
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct GithubSearchConfig { pub struct GithubSearchConfig {
@@ -378,7 +374,10 @@ mod tests {
"repository": { "full_name": "faith1337z/Trade", "private": false } "repository": { "full_name": "faith1337z/Trade", "private": false }
}); });
let ev = parse_commit_event(&raw).expect("parses"); let ev = parse_commit_event(&raw).expect("parses");
assert_eq!(ev.id, "github-commit:a6fcefbe909a97ad5a049b9fa48bc74309af10d9"); assert_eq!(
ev.id,
"github-commit:a6fcefbe909a97ad5a049b9fa48bc74309af10d9"
);
assert_eq!(ev.action, "Commit"); assert_eq!(ev.action, "Commit");
assert!(ev.public); assert!(ev.public);
} }

View File

@@ -21,11 +21,7 @@ use serde_json::Value;
use tracing::{debug, warn}; use tracing::{debug, warn};
const SOURCE_NAME: &str = "hg"; const SOURCE_NAME: &str = "hg";
const USER_AGENT: &str = concat!( const USER_AGENT: &str = concat!("moments/", env!("CARGO_PKG_VERSION"), " (+https://rob.tn)");
"moments/",
env!("CARGO_PKG_VERSION"),
" (+https://rob.tn)"
);
/// Maximum changesets returned per json-log request. /// Maximum changesets returned per json-log request.
const REV_COUNT: u32 = 500; const REV_COUNT: u32 = 500;
@@ -148,10 +144,7 @@ impl HgSource {
let mut payload = entry.clone(); let mut payload = entry.clone();
if let Some(obj) = payload.as_object_mut() { if let Some(obj) = payload.as_object_mut() {
obj.insert("_repo".into(), Value::String(repo.into())); obj.insert("_repo".into(), Value::String(repo.into()));
obj.insert( obj.insert("_host".into(), Value::String(self.config.host.clone()));
"_host".into(),
Value::String(self.config.host.clone()),
);
} }
all_events.push(Event { all_events.push(Event {
id: format!("hg:{repo}:{node}"), id: format!("hg:{repo}:{node}"),
@@ -254,6 +247,13 @@ mod tests {
) -> Result<usize, moments_core::StoreError> { ) -> Result<usize, moments_core::StoreError> {
Ok(0) Ok(0)
} }
async fn prune_events(
&self,
_source: moments_entities::Source,
_keep_ids: &[String],
) -> Result<usize, moments_core::StoreError> {
Ok(0)
}
} }
struct NoopState; struct NoopState;
#[async_trait] #[async_trait]

View File

@@ -7,10 +7,13 @@ pub mod github_search;
pub mod hg; pub mod hg;
use async_trait::async_trait; use async_trait::async_trait;
use chrono::NaiveDate;
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use moments_core::{EventReader, EventWriter, PollerState, PollerStateStore, StoreError}; use moments_core::{EventReader, EventWriter, PollerState, PollerStateStore, StoreError};
use chrono::NaiveDate; use moments_entities::{
use moments_entities::{DailyCount, Event, EventQuery, HourlyAvg, LanguageDailyCount, ProjectSummary, RepoLanguage, Source, SourceSummary}; DailyCount, Event, EventQuery, HourlyAvg, LanguageDailyCount, ProjectSummary, RepoLanguage,
Source, SourceSummary,
};
use sqlx::Row; use sqlx::Row;
use sqlx::postgres::{PgPool, PgPoolOptions}; use sqlx::postgres::{PgPool, PgPoolOptions};
use std::str::FromStr; use std::str::FromStr;
@@ -99,7 +102,10 @@ impl EventReader for PgStore {
.collect() .collect()
} }
async fn source_summaries(&self, include_private: bool) -> Result<Vec<SourceSummary>, StoreError> { async fn source_summaries(
&self,
include_private: bool,
) -> Result<Vec<SourceSummary>, StoreError> {
let rows = sqlx::query( let rows = sqlx::query(
r#" r#"
SELECT source, SELECT source,
@@ -187,9 +193,18 @@ impl EventReader for PgStore {
source: Source::from_str(&source_str).map_err(map_err)?, source: Source::from_str(&source_str).map_err(map_err)?,
repo: r.try_get("repo").map_err(map_err)?, repo: r.try_get("repo").map_err(map_err)?,
host: r.try_get("host").map_err(map_err)?, host: r.try_get("host").map_err(map_err)?,
commit_count: r.try_get::<i64, _>("commit_count").map_err(map_err).unwrap_or(0), commit_count: r
issue_count: r.try_get::<i64, _>("issue_count").map_err(map_err).unwrap_or(0), .try_get::<i64, _>("commit_count")
pr_count: r.try_get::<i64, _>("pr_count").map_err(map_err).unwrap_or(0), .map_err(map_err)
.unwrap_or(0),
issue_count: r
.try_get::<i64, _>("issue_count")
.map_err(map_err)
.unwrap_or(0),
pr_count: r
.try_get::<i64, _>("pr_count")
.map_err(map_err)
.unwrap_or(0),
first_activity: r.try_get("first_activity").map_err(map_err)?, first_activity: r.try_get("first_activity").map_err(map_err)?,
last_activity: r.try_get("last_activity").map_err(map_err)?, last_activity: r.try_get("last_activity").map_err(map_err)?,
}) })
@@ -197,7 +212,12 @@ impl EventReader for PgStore {
.collect() .collect()
} }
async fn daily_counts(&self, from: NaiveDate, to: NaiveDate, include_private: bool) -> Result<Vec<DailyCount>, StoreError> { async fn daily_counts(
&self,
from: NaiveDate,
to: NaiveDate,
include_private: bool,
) -> Result<Vec<DailyCount>, StoreError> {
let rows = sqlx::query( let rows = sqlx::query(
r#" r#"
SELECT d::date AS date, SELECT d::date AS date,
@@ -228,7 +248,12 @@ impl EventReader for PgStore {
.collect() .collect()
} }
async fn language_daily_counts(&self, from: NaiveDate, to: NaiveDate, include_private: bool) -> Result<Vec<LanguageDailyCount>, StoreError> { async fn language_daily_counts(
&self,
from: NaiveDate,
to: NaiveDate,
include_private: bool,
) -> Result<Vec<LanguageDailyCount>, StoreError> {
let rows = sqlx::query( let rows = sqlx::query(
r#" r#"
SELECT date, language, color, SELECT date, language, color,

View File

@@ -126,9 +126,7 @@ async fn main() -> anyhow::Result<()> {
let store = Arc::new(PgStore::connect(&args.database_url).await?); let store = Arc::new(PgStore::connect(&args.database_url).await?);
store.migrate().await?; store.migrate().await?;
let http = Client::builder() let http = Client::builder().timeout(Duration::from_secs(30)).build()?;
.timeout(Duration::from_secs(30))
.build()?;
let github = Arc::new(GithubSource::new( let github = Arc::new(GithubSource::new(
http.clone(), http.clone(),
@@ -250,8 +248,7 @@ async fn main() -> anyhow::Result<()> {
tokio::spawn(async move { run_poller(github_repo, repo_interval).await }); tokio::spawn(async move { run_poller(github_repo, repo_interval).await });
let gitea_task = tokio::spawn(async move { run_poller(gitea, gitea_interval).await }); let gitea_task = tokio::spawn(async move { run_poller(gitea, gitea_interval).await });
let hg_task = tokio::spawn(async move { run_poller(hg, hg_interval).await }); let hg_task = tokio::spawn(async move { run_poller(hg, hg_interval).await });
let bugzilla_task = let bugzilla_task = tokio::spawn(async move { run_poller(bugzilla, bugzilla_interval).await });
tokio::spawn(async move { run_poller(bugzilla, bugzilla_interval).await });
let blog_task = let blog_task =
blog.map(|src| tokio::spawn(async move { run_poller(src, blog_interval).await })); blog.map(|src| tokio::spawn(async move { run_poller(src, blog_interval).await }));