feat(blog): add markdown blog sourced from a gitea repo
posts are markdown files with yaml frontmatter (title, slug, date;
optional draft/public) in the grenade/blog repo. the worker's new
BlogSource polls the repo — one branch-tip request when nothing
changed — and upserts posts into events with source='blog' and
occurred_at from the frontmatter date, so imported posts keep their
original publish dates and backfill the contribution graph.
- new /v1/blog and /v1/blog/{slug} endpoints over the existing
EventReader port; drafts stay hidden via the public gate
- new /blog and /blog/:slug routes, nav link, activity-feed entry
with post icon and filter toggle; relative image srcs resolve to
gitea raw urls
- shared Markdown component extracted from ProjectPage
- vite proxy target overridable via API_PROXY_TARGET for local dev
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -17,4 +17,5 @@ tracing.workspace = true
|
||||
async-trait.workspace = true
|
||||
reqwest.workspace = true
|
||||
serde.workspace = true
|
||||
serde_yaml_ng.workspace = true
|
||||
percent-encoding = "2"
|
||||
|
||||
334
crates/moments-data/src/blog.rs
Normal file
334
crates/moments-data/src/blog.rs
Normal file
@@ -0,0 +1,334 @@
|
||||
//! Blog post ingestion from a Gitea-hosted markdown repo.
|
||||
//!
|
||||
//! Posts are markdown files with YAML frontmatter (`title`, `slug`, `date`)
|
||||
//! at the root of a repo. The poll cycle is cheap: one request for the
|
||||
//! branch tip sha, compared against `poller_state.etag`; only when the repo
|
||||
//! has new commits are the file listing and contents fetched.
|
||||
//!
|
||||
//! `occurred_at` comes from the frontmatter `date`, so imported posts keep
|
||||
//! their original publish dates. Edits re-upsert under the same
|
||||
//! `blog:{slug}` id.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, NaiveDate, NaiveTime, Utc};
|
||||
use moments_core::{EventSource, EventWriter, PollerStateStore, SourceError};
|
||||
use moments_entities::{Event, Source};
|
||||
use reqwest::{Client, header};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Value, json};
|
||||
use tracing::{debug, warn};
|
||||
|
||||
const SOURCE_NAME: &str = "blog";
|
||||
const USER_AGENT: &str = concat!(
|
||||
"moments/",
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
" (+https://rob.tn)"
|
||||
);
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct BlogConfig {
|
||||
/// e.g. `git.lair.cafe`. Stamped into each payload as `_host` so the
|
||||
/// reshape layer and UI can build raw-content URLs without config.
|
||||
pub host: String,
|
||||
/// Repo holding the posts, e.g. `grenade/blog`.
|
||||
pub repo: String,
|
||||
pub branch: String,
|
||||
/// Only needed if the repo is private (which breaks raw image URLs in
|
||||
/// the UI — keep it public).
|
||||
pub token: Option<String>,
|
||||
}
|
||||
|
||||
impl Default for BlogConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
host: "git.lair.cafe".into(),
|
||||
repo: "grenade/blog".into(),
|
||||
branch: "main".into(),
|
||||
token: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct BlogSource {
|
||||
client: Client,
|
||||
writer: Arc<dyn EventWriter>,
|
||||
state: Arc<dyn PollerStateStore>,
|
||||
config: BlogConfig,
|
||||
}
|
||||
|
||||
impl BlogSource {
|
||||
pub fn new(
|
||||
client: Client,
|
||||
writer: Arc<dyn EventWriter>,
|
||||
state: Arc<dyn PollerStateStore>,
|
||||
config: BlogConfig,
|
||||
) -> Self {
|
||||
Self {
|
||||
client,
|
||||
writer,
|
||||
state,
|
||||
config,
|
||||
}
|
||||
}
|
||||
|
||||
fn api_url(&self, rest: &str) -> String {
|
||||
format!(
|
||||
"https://{}/api/v1/repos/{}/{rest}",
|
||||
self.config.host, self.config.repo
|
||||
)
|
||||
}
|
||||
|
||||
fn apply_headers(&self, mut req: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
|
||||
req = req.header(header::USER_AGENT, USER_AGENT);
|
||||
if let Some(token) = &self.config.token {
|
||||
req = req.header(header::AUTHORIZATION, format!("token {token}"));
|
||||
}
|
||||
req
|
||||
}
|
||||
|
||||
async fn get(&self, url: &str) -> Result<reqwest::Response, SourceError> {
|
||||
let resp = self
|
||||
.apply_headers(self.client.get(url))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| SourceError::Http(e.to_string()))?;
|
||||
if !resp.status().is_success() {
|
||||
return Err(SourceError::Http(format!("{} GET {}", resp.status(), url)));
|
||||
}
|
||||
Ok(resp)
|
||||
}
|
||||
|
||||
/// Tip sha of the configured branch — the change-detection key.
|
||||
async fn branch_tip(&self) -> Result<String, SourceError> {
|
||||
let url = self.api_url(&format!("branches/{}", self.config.branch));
|
||||
let body: Value = self
|
||||
.get(&url)
|
||||
.await?
|
||||
.json()
|
||||
.await
|
||||
.map_err(|e| SourceError::Parse(e.to_string()))?;
|
||||
body.get("commit")
|
||||
.and_then(|c| c.get("id"))
|
||||
.and_then(Value::as_str)
|
||||
.map(String::from)
|
||||
.ok_or_else(|| SourceError::Parse("branch response missing commit.id".into()))
|
||||
}
|
||||
|
||||
/// Markdown file paths at the repo root, excluding READMEs.
|
||||
async fn list_post_paths(&self) -> Result<Vec<String>, SourceError> {
|
||||
let url = self.api_url(&format!("contents?ref={}", self.config.branch));
|
||||
let entries: Vec<Value> = self
|
||||
.get(&url)
|
||||
.await?
|
||||
.json()
|
||||
.await
|
||||
.map_err(|e| SourceError::Parse(e.to_string()))?;
|
||||
Ok(entries
|
||||
.iter()
|
||||
.filter(|e| e.get("type").and_then(Value::as_str) == Some("file"))
|
||||
.filter_map(|e| e.get("path").and_then(Value::as_str))
|
||||
.filter(|p| {
|
||||
p.to_ascii_lowercase().ends_with(".md")
|
||||
&& !p.eq_ignore_ascii_case("readme.md")
|
||||
})
|
||||
.map(String::from)
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn fetch_raw(&self, path: &str) -> Result<String, SourceError> {
|
||||
let url = self.api_url(&format!("raw/{path}?ref={}", self.config.branch));
|
||||
self.get(&url)
|
||||
.await?
|
||||
.text()
|
||||
.await
|
||||
.map_err(|e| SourceError::Parse(e.to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl EventSource for BlogSource {
|
||||
fn name(&self) -> &'static str {
|
||||
SOURCE_NAME
|
||||
}
|
||||
|
||||
async fn poll(&self) -> Result<usize, SourceError> {
|
||||
let tip = self.branch_tip().await?;
|
||||
let prior = self.state.load(SOURCE_NAME).await?;
|
||||
if prior.as_ref().and_then(|p| p.etag.as_deref()) == Some(tip.as_str()) {
|
||||
self.state.touch(SOURCE_NAME).await?;
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let mut events = Vec::new();
|
||||
for path in self.list_post_paths().await? {
|
||||
let content = self.fetch_raw(&path).await?;
|
||||
match parse_post(&path, &content, &self.config) {
|
||||
Some(event) => events.push(event),
|
||||
None => warn!(path = %path, "blog post missing or invalid frontmatter; skipping"),
|
||||
}
|
||||
}
|
||||
|
||||
let total = self.writer.upsert_events(&events).await?;
|
||||
self.state.save(SOURCE_NAME, Some(&tip), None).await?;
|
||||
debug!(ingested = total, tip = %tip, "blog poll complete");
|
||||
Ok(total)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Frontmatter {
|
||||
title: String,
|
||||
slug: Option<String>,
|
||||
date: String,
|
||||
public: Option<bool>,
|
||||
draft: Option<bool>,
|
||||
}
|
||||
|
||||
/// Split `content` into (frontmatter yaml, markdown body). The file must
|
||||
/// open with a `---` fence on the first line.
|
||||
fn split_frontmatter(content: &str) -> Option<(&str, &str)> {
|
||||
let rest = content.strip_prefix("---")?;
|
||||
let rest = rest.strip_prefix('\n').or_else(|| rest.strip_prefix("\r\n"))?;
|
||||
// The closing fence is a `---` alone on a line.
|
||||
let mut offset = 0;
|
||||
for line in rest.split_inclusive('\n') {
|
||||
if line.trim_end() == "---" {
|
||||
let yaml = &rest[..offset];
|
||||
let body = &rest[offset + line.len()..];
|
||||
return Some((yaml, body));
|
||||
}
|
||||
offset += line.len();
|
||||
}
|
||||
// Closing fence with no trailing newline at EOF.
|
||||
if rest[offset..].trim_end() == "---" && offset > 0 {
|
||||
return Some((&rest[..offset], ""));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Accept RFC3339 (`2026-06-12T09:30:00Z`) or a bare date (`2026-06-12`,
|
||||
/// treated as midnight UTC) so imported posts keep original publish dates.
|
||||
fn parse_date(s: &str) -> Option<DateTime<Utc>> {
|
||||
if let Ok(dt) = DateTime::parse_from_rfc3339(s) {
|
||||
return Some(dt.with_timezone(&Utc));
|
||||
}
|
||||
let date = NaiveDate::parse_from_str(s, "%Y-%m-%d").ok()?;
|
||||
Some(date.and_time(NaiveTime::MIN).and_utc())
|
||||
}
|
||||
|
||||
fn parse_post(path: &str, content: &str, config: &BlogConfig) -> Option<Event> {
|
||||
let (yaml, body) = split_frontmatter(content)?;
|
||||
let fm: Frontmatter = serde_yaml_ng::from_str(yaml)
|
||||
.map_err(|e| warn!(path = %path, error = %e, "blog frontmatter parse failed"))
|
||||
.ok()?;
|
||||
let occurred_at = parse_date(&fm.date)?;
|
||||
let slug = fm.slug.unwrap_or_else(|| {
|
||||
path.rsplit('/')
|
||||
.next()
|
||||
.unwrap_or(path)
|
||||
.trim_end_matches(".md")
|
||||
.to_string()
|
||||
});
|
||||
let public = fm.public.unwrap_or(true) && !fm.draft.unwrap_or(false);
|
||||
|
||||
Some(Event {
|
||||
id: format!("blog:{slug}"),
|
||||
source: Source::Blog,
|
||||
action: "publish_post".into(),
|
||||
occurred_at,
|
||||
public,
|
||||
payload: json!({
|
||||
"title": fm.title,
|
||||
"slug": slug,
|
||||
"markdown": body.trim_start(),
|
||||
"_host": config.host,
|
||||
"_repo": config.repo,
|
||||
"_branch": config.branch,
|
||||
"_path": path,
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn config() -> BlogConfig {
|
||||
BlogConfig::default()
|
||||
}
|
||||
|
||||
const POST: &str = "---\ntitle: a watchdog, a torrent tracker, and an unhinged friday\nslug: last-week\ndate: 2026-06-12\n---\n\nthe week opened deep in helexa's neuron engine.\n";
|
||||
|
||||
#[test]
|
||||
fn parse_post_with_bare_date() {
|
||||
let ev = parse_post("last-week.md", POST, &config()).expect("parses");
|
||||
assert_eq!(ev.id, "blog:last-week");
|
||||
assert_eq!(ev.source, Source::Blog);
|
||||
assert_eq!(ev.action, "publish_post");
|
||||
assert_eq!(ev.occurred_at.to_rfc3339(), "2026-06-12T00:00:00+00:00");
|
||||
assert!(ev.public);
|
||||
assert_eq!(
|
||||
ev.payload.get("title").and_then(Value::as_str),
|
||||
Some("a watchdog, a torrent tracker, and an unhinged friday")
|
||||
);
|
||||
assert_eq!(
|
||||
ev.payload.get("markdown").and_then(Value::as_str),
|
||||
Some("the week opened deep in helexa's neuron engine.\n")
|
||||
);
|
||||
assert_eq!(
|
||||
ev.payload.get("_repo").and_then(Value::as_str),
|
||||
Some("grenade/blog")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_post_with_rfc3339_date() {
|
||||
let post = "---\ntitle: t\ndate: 2019-03-04T12:30:00+02:00\n---\nbody\n";
|
||||
let ev = parse_post("old-post.md", post, &config()).expect("parses");
|
||||
assert_eq!(ev.occurred_at.to_rfc3339(), "2019-03-04T10:30:00+00:00");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn slug_falls_back_to_filename_stem() {
|
||||
let post = "---\ntitle: t\ndate: 2020-01-01\n---\nbody\n";
|
||||
let ev = parse_post("posts/my-old-entry.md", post, &config()).expect("parses");
|
||||
assert_eq!(ev.id, "blog:my-old-entry");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn draft_is_private() {
|
||||
let post = "---\ntitle: t\ndate: 2020-01-01\ndraft: true\n---\nbody\n";
|
||||
let ev = parse_post("p.md", post, &config()).expect("parses");
|
||||
assert!(!ev.public);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_public_false_is_private() {
|
||||
let post = "---\ntitle: t\ndate: 2020-01-01\npublic: false\n---\nbody\n";
|
||||
let ev = parse_post("p.md", post, &config()).expect("parses");
|
||||
assert!(!ev.public);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_frontmatter_is_skipped() {
|
||||
assert!(parse_post("p.md", "# just a heading\n\nbody\n", &config()).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_date_is_skipped() {
|
||||
let post = "---\ntitle: t\ndate: last tuesday\n---\nbody\n";
|
||||
assert!(parse_post("p.md", post, &config()).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_handles_crlf_and_eof_fence() {
|
||||
let (yaml, body) = split_frontmatter("---\r\ntitle: t\r\n---\r\nbody").expect("splits");
|
||||
assert!(yaml.contains("title: t"));
|
||||
assert_eq!(body, "body");
|
||||
let (yaml, body) = split_frontmatter("---\ntitle: t\n---").expect("splits at eof");
|
||||
assert_eq!(yaml, "title: t\n");
|
||||
assert_eq!(body, "");
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
pub mod blog;
|
||||
pub mod bugzilla;
|
||||
pub mod gitea;
|
||||
pub mod github;
|
||||
|
||||
Reference in New Issue
Block a user