From eb3ced59050a6e5afd44e3133df1e5a48381d29a Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 11 Feb 2026 13:59:17 +0000 Subject: [PATCH] feat: raw memories as toml --- codex-rs/core/src/memories/mod.rs | 2 +- codex-rs/core/src/memories/storage.rs | 54 ++++++++++--------- codex-rs/core/src/memories/tests.rs | 18 ++++++- .../core/templates/memories/consolidation.md | 4 +- 4 files changed, 47 insertions(+), 31 deletions(-) diff --git a/codex-rs/core/src/memories/mod.rs b/codex-rs/core/src/memories/mod.rs index 6b9dad568d..c05b41b60a 100644 --- a/codex-rs/core/src/memories/mod.rs +++ b/codex-rs/core/src/memories/mod.rs @@ -21,7 +21,7 @@ use std::path::PathBuf; /// Subagent source label used to identify consolidation tasks. const MEMORY_CONSOLIDATION_SUBAGENT_LABEL: &str = "memory_consolidation"; const ROLLOUT_SUMMARIES_SUBDIR: &str = "rollout_summaries"; -const RAW_MEMORIES_FILENAME: &str = "raw_memories.md"; +const RAW_MEMORIES_FILENAME: &str = "raw_memories.yaml"; /// Maximum number of rollout candidates processed per startup pass. const MAX_ROLLOUTS_PER_STARTUP: usize = 64; /// Concurrency cap for startup memory extraction and consolidation scheduling. diff --git a/codex-rs/core/src/memories/storage.rs b/codex-rs/core/src/memories/storage.rs index 5b67e229d3..1f82d72322 100644 --- a/codex-rs/core/src/memories/storage.rs +++ b/codex-rs/core/src/memories/storage.rs @@ -1,4 +1,5 @@ use codex_state::Stage1Output; +use serde::Serialize; use std::collections::BTreeSet; use std::fmt::Write as _; use std::path::Path; @@ -10,7 +11,7 @@ use super::raw_memories_file; use super::rollout_summaries_dir; use super::text::compact_whitespace; -/// Rebuild `raw_memories.md` from DB-backed stage-1 outputs. +/// Rebuild `raw_memories.yaml` from DB-backed stage-1 outputs. pub(super) async fn rebuild_raw_memories_file_from_memories( root: &Path, memories: &[Stage1Output], @@ -62,33 +63,28 @@ pub(super) async fn sync_rollout_summaries_from_memories( } async fn rebuild_raw_memories_file(root: &Path, memories: &[Stage1Output]) -> std::io::Result<()> { - let retained = memories + let entries = memories .iter() .take(MAX_RAW_MEMORIES_FOR_GLOBAL) - .collect::>(); - let mut body = String::from("# Raw Memories\n\n"); - - if retained.is_empty() { - body.push_str("No raw memories yet.\n"); - return tokio::fs::write(raw_memories_file(root), body).await; - } - - body.push_str("Merged stage-1 raw memories (latest first):\n\n"); - for memory in retained { - writeln!(body, "## Thread `{}`", memory.thread_id) - .map_err(|err| std::io::Error::other(format!("format raw memories: {err}")))?; - writeln!( - body, - "updated_at: {}", - memory.source_updated_at.to_rfc3339() - ) - .map_err(|err| std::io::Error::other(format!("format raw memories: {err}")))?; - writeln!(body) - .map_err(|err| std::io::Error::other(format!("format raw memories: {err}")))?; - body.push_str(memory.raw_memory.trim()); - body.push_str("\n\n"); - } - + .map(|memory| -> std::io::Result { + let file_path = rollout_summaries_dir(root).join(format!("{}.md", memory.thread_id)); + let file_path = if file_path.is_absolute() { + file_path + } else { + std::env::current_dir() + .map_err(|err| { + std::io::Error::other(format!("resolve raw memory path: {err}")) + })? + .join(file_path) + }; + Ok(RawMemoryYamlEntry { + file_path: file_path.display().to_string(), + raw_memory: memory.raw_memory.clone(), + }) + }) + .collect::>>()?; + let body = serde_yaml::to_string(&entries) + .map_err(|err| std::io::Error::other(format!("serialize raw memories yaml: {err}")))?; tokio::fs::write(raw_memories_file(root), body).await } @@ -149,3 +145,9 @@ fn extract_thread_id_from_rollout_summary_filename(file_name: &str) -> Option<&s let stem = file_name.strip_suffix(".md")?; if stem.is_empty() { None } else { Some(stem) } } + +#[derive(Serialize)] +struct RawMemoryYamlEntry { + file_path: String, + raw_memory: String, +} diff --git a/codex-rs/core/src/memories/tests.rs b/codex-rs/core/src/memories/tests.rs index 1474d05f0a..58836f7037 100644 --- a/codex-rs/core/src/memories/tests.rs +++ b/codex-rs/core/src/memories/tests.rs @@ -17,6 +17,7 @@ use codex_protocol::protocol::CompactedItem; use codex_protocol::protocol::RolloutItem; use codex_state::Stage1Output; use pretty_assertions::assert_eq; +use serde::Deserialize; use tempfile::tempdir; #[test] @@ -195,6 +196,19 @@ async fn sync_rollout_summaries_and_raw_memories_file_keeps_latest_memories_only let raw_memories = tokio::fs::read_to_string(raw_memories_file(&root)) .await .expect("read raw memories"); - assert!(raw_memories.contains("raw memory")); - assert!(raw_memories.contains(&keep_id)); + let parsed: Vec = + serde_yaml::from_str(&raw_memories).expect("parse raw memories yaml"); + assert_eq!( + parsed, + vec![RawMemoryYamlEntry { + file_path: keep_path.display().to_string(), + raw_memory: "raw memory".to_string(), + }] + ); +} + +#[derive(Debug, PartialEq, Eq, Deserialize)] +struct RawMemoryYamlEntry { + file_path: String, + raw_memory: String, } diff --git a/codex-rs/core/templates/memories/consolidation.md b/codex-rs/core/templates/memories/consolidation.md index f83e1b4b47..4b3db396bb 100644 --- a/codex-rs/core/templates/memories/consolidation.md +++ b/codex-rs/core/templates/memories/consolidation.md @@ -6,7 +6,7 @@ Integrate Phase 1 artifacts into a stable, retrieval-friendly memory hierarchy w Primary inputs in this directory: - `rollout_summaries/` (per-thread summaries from Phase 1) -- `raw_memories.md` (merged Stage 1 raw memories; latest first) +- `raw_memories.yaml` (array of Stage 1 raw memories with `file_path` and `raw_memory`) - Existing outputs if present: - `MEMORY.md` - `memory_summary.md` @@ -40,7 +40,7 @@ Expected outputs (create/update only these): Workflow (order matters): 1. Determine mode (`INIT` vs `INCREMENTAL`) from artifact availability/content. -2. Read `rollout_summaries/` first for routing, then validate details in `raw_memories.md`. +2. Read `rollout_summaries/` first for routing, then validate details in `raw_memories.yaml`. 3. Read existing `MEMORY.md`, `memory_summary.md`, and `skills/` for continuity. 4. Update `skills/` only for reliable, repeatable procedures with clear verification. 5. Update `MEMORY.md` as the durable registry; add clear related-skill pointers in note bodies when useful.