From 03c5bfdd28bf57781d9814999bc4a9e9250849b4 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 29 Apr 2026 17:17:19 +0100 Subject: [PATCH] mvoes --- codex-rs/core/src/context_manager/updates.rs | 219 ++++-------- codex-rs/core/src/session/mod.rs | 109 ++++-- codex-rs/core/src/state/history.rs | 8 +- codex-rs/journal/Cargo.toml | 1 + codex-rs/journal/README.md | 86 +++++ codex-rs/journal/src/context_builder.rs | 87 +++++ codex-rs/journal/src/journal.rs | 352 +++++++++++++++---- codex-rs/journal/src/lib.rs | 27 +- codex-rs/journal/src/render.rs | 86 +++++ codex-rs/journal/src/tests.rs | 309 ++++++++++++---- codex-rs/protocol/src/journal.rs | 77 ++-- codex-rs/thread-store/src/journal_writer.rs | 12 +- 12 files changed, 985 insertions(+), 388 deletions(-) create mode 100644 codex-rs/journal/README.md create mode 100644 codex-rs/journal/src/context_builder.rs create mode 100644 codex-rs/journal/src/render.rs diff --git a/codex-rs/core/src/context_manager/updates.rs b/codex-rs/core/src/context_manager/updates.rs index ab1bf02458..28364395bb 100644 --- a/codex-rs/core/src/context_manager/updates.rs +++ b/codex-rs/core/src/context_manager/updates.rs @@ -13,23 +13,36 @@ use crate::shell::Shell; use codex_execpolicy::Policy; use codex_features::Feature; use codex_journal::Journal; -use codex_journal::JournalContextItem; -use codex_journal::JournalContextKey; use codex_journal::JournalEntry; -use codex_journal::JournalItem; +use codex_journal::KeyFilter; +use codex_journal::MetadataEntryBuilder; use codex_journal::PromptMessage; -use codex_journal::PromptMessageRole; +use codex_journal::PromptRenderer; use codex_protocol::config_types::Personality; -use codex_protocol::models::ContentItem; -use codex_protocol::models::ResponseItem; use codex_protocol::openai_models::ModelInfo; use codex_protocol::protocol::TurnContextItem; const PROMPT_BUNDLE_KEY_PREFIX: &str = "prompt"; -const DEVELOPER_BUNDLE: &str = "developer"; -const USAGE_HINT_BUNDLE: &str = "usage_hint"; -const CONTEXTUAL_USER_BUNDLE: &str = "contextual_user"; -const GUARDIAN_BUNDLE: &str = "guardian"; +pub(crate) const DEVELOPER_BUNDLE: &str = "developer"; +pub(crate) const USAGE_HINT_BUNDLE: &str = "usage_hint"; +pub(crate) const CONTEXTUAL_USER_BUNDLE: &str = "contextual_user"; +pub(crate) const GUARDIAN_BUNDLE: &str = "guardian"; + +pub(crate) fn context_prompt_renderer() -> PromptRenderer { + PromptRenderer::new() + .group(KeyFilter::prefix([ + PROMPT_BUNDLE_KEY_PREFIX, + DEVELOPER_BUNDLE, + ])) + .group(KeyFilter::prefix([ + PROMPT_BUNDLE_KEY_PREFIX, + USAGE_HINT_BUNDLE, + ])) + .group(KeyFilter::prefix([ + PROMPT_BUNDLE_KEY_PREFIX, + CONTEXTUAL_USER_BUNDLE, + ])) +} fn build_environment_update_item( previous: Option<&TurnContextItem>, @@ -186,133 +199,19 @@ pub(crate) fn build_model_instructions_update_item( Some(ModelSwitchInstructions::new(model_instructions).render()) } -pub(crate) fn developer_context_entry( - name: &str, - prompt_order: i64, - text: String, -) -> Option { - context_entry( - DEVELOPER_BUNDLE, - name, - prompt_order, - PromptMessage::developer_text(text), - ) -} - -pub(crate) fn usage_hint_context_entry( - name: &str, - prompt_order: i64, - text: String, -) -> Option { - context_entry( - USAGE_HINT_BUNDLE, - name, - prompt_order, - PromptMessage::developer_text(text), - ) -} - -pub(crate) fn contextual_user_context_entry( - name: &str, - prompt_order: i64, - text: String, -) -> Option { - context_entry( - CONTEXTUAL_USER_BUNDLE, - name, - prompt_order, - PromptMessage::user_text(text), - ) -} - -pub(crate) fn guardian_context_entry( - name: &str, - prompt_order: i64, - text: String, -) -> Option { - context_entry( - GUARDIAN_BUNDLE, - name, - prompt_order, - PromptMessage::developer_text(text), - ) -} - -fn context_entry( +pub(crate) fn context_entry( bundle: &str, name: &str, prompt_order: i64, message: PromptMessage, ) -> Option { - if message.content.is_empty() - || message.content.iter().all(|item| match item { - ContentItem::InputText { text } | ContentItem::OutputText { text } => { - text.trim().is_empty() - } - ContentItem::InputImage { .. } => false, - }) - { - return None; - } - - Some(JournalEntry::new( - [PROMPT_BUNDLE_KEY_PREFIX, bundle, name], - JournalContextItem::new(JournalContextKey::new(bundle, name, None), message) - .with_prompt_order(prompt_order), - )) + context_entry_builder(bundle, name, message) + .prompt_order(prompt_order) + .build() } -pub(crate) fn render_context_entries(entries: Vec) -> Vec { - let flattened = match Journal::from_entries(entries).flatten() { - Ok(flattened) => flattened, - Err(error) => unreachable!("context-only journal entries should flatten: {error}"), - }; - - let mut rendered = Vec::new(); - let mut current_bundle: Option> = None; - let mut current_role: Option = None; - let mut current_content = Vec::new(); - - for entry in flattened.entries() { - let JournalItem::Context(item) = &entry.item else { - continue; - }; - let bundle = entry - .key - .parts() - .iter() - .take(2) - .cloned() - .collect::>(); - - if current_bundle.as_ref() != Some(&bundle) || current_role != Some(item.message.role) { - flush_prompt_message(&mut rendered, &mut current_role, &mut current_content); - current_bundle = Some(bundle); - current_role = Some(item.message.role); - } - - current_content.extend(item.message.content.clone()); - } - - flush_prompt_message(&mut rendered, &mut current_role, &mut current_content); - rendered -} - -fn flush_prompt_message( - rendered: &mut Vec, - role: &mut Option, - content: &mut Vec, -) { - let Some(role) = role.take() else { - return; - }; - if content.is_empty() { - return; - } - rendered.push(ResponseItem::from(PromptMessage::new( - role, - std::mem::take(content), - ))); +fn context_entry_builder(bundle: &str, name: &str, message: PromptMessage) -> MetadataEntryBuilder { + Journal::metadata_entry_builder([PROMPT_BUNDLE_KEY_PREFIX, bundle, name], message) } pub(crate) fn build_settings_update_entries( @@ -333,34 +232,70 @@ pub(crate) fn build_settings_update_entries( build_model_instructions_update_item(previous_turn_settings, next).and_then(|text| { // Keep model-switch instructions first so model-specific guidance is read before // any other context diffs on this turn. - developer_context_entry("model_switch", 10, text) + context_entry( + DEVELOPER_BUNDLE, + "model_switch", + 10, + PromptMessage::developer_text(text), + ) }) { entries.push(item); } - if let Some(item) = build_permissions_update_item(previous, next, exec_policy) - .and_then(|text| developer_context_entry("permissions", 20, text)) + if let Some(item) = + build_permissions_update_item(previous, next, exec_policy).and_then(|text| { + context_entry( + DEVELOPER_BUNDLE, + "permissions", + 20, + PromptMessage::developer_text(text), + ) + }) { entries.push(item); } - if let Some(item) = build_collaboration_mode_update_item(previous, next) - .and_then(|text| developer_context_entry("collaboration_mode", 30, text)) - { + if let Some(item) = build_collaboration_mode_update_item(previous, next).and_then(|text| { + context_entry( + DEVELOPER_BUNDLE, + "collaboration_mode", + 30, + PromptMessage::developer_text(text), + ) + }) { entries.push(item); } - if let Some(item) = build_realtime_update_item(previous, previous_turn_settings, next) - .and_then(|text| developer_context_entry("realtime", 40, text)) + if let Some(item) = + build_realtime_update_item(previous, previous_turn_settings, next).and_then(|text| { + context_entry( + DEVELOPER_BUNDLE, + "realtime", + 40, + PromptMessage::developer_text(text), + ) + }) { entries.push(item); } if let Some(item) = build_personality_update_item(previous, next, personality_feature_enabled) - .and_then(|text| developer_context_entry("personality", 50, text)) + .and_then(|text| { + context_entry( + DEVELOPER_BUNDLE, + "personality", + 50, + PromptMessage::developer_text(text), + ) + }) { entries.push(item); } - if let Some(item) = build_environment_update_item(previous, next, shell) - .and_then(|text| contextual_user_context_entry("environment", 60, text)) - { + if let Some(item) = build_environment_update_item(previous, next, shell).and_then(|text| { + context_entry( + CONTEXTUAL_USER_BUNDLE, + "environment", + 60, + PromptMessage::user_text(text), + ) + }) { entries.push(item); } diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index f08c2ed36e..6f7d1233db 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -59,6 +59,7 @@ use codex_features::Feature; use codex_features::unstable_features_warning_event; use codex_hooks::Hooks; use codex_hooks::HooksConfig; +use codex_journal::PromptMessage; use codex_login::AuthManager; use codex_login::CodexAuth; use codex_login::auth_env_telemetry::collect_auth_env_telemetry; @@ -1437,7 +1438,12 @@ impl Session { exec_policy.as_ref(), self.features.enabled(Feature::Personality), ); - crate::context_manager::updates::render_context_entries(entries) + let resolved = match codex_journal::Journal::from_entries(entries).resolve() { + Ok(resolved) => resolved, + Err(error) => unreachable!("settings update entries should resolve: {error}"), + }; + crate::context_manager::updates::context_prompt_renderer() + .render_metadata(resolved.metadata()) } /// Persist the event to rollout and send it to clients. @@ -2503,9 +2509,16 @@ impl Session { &self, turn_context: &TurnContext, ) -> Vec { - crate::context_manager::updates::render_context_entries( + let resolved = match codex_journal::Journal::from_entries( self.build_initial_context_entries(turn_context).await, ) + .resolve() + { + Ok(resolved) => resolved, + Err(error) => unreachable!("initial context entries should resolve: {error}"), + }; + crate::context_manager::updates::context_prompt_renderer() + .render_metadata(resolved.metadata()) } #[expect( @@ -2539,10 +2552,11 @@ impl Session { previous_turn_settings.as_ref(), turn_context, ) - && let Some(entry) = crate::context_manager::updates::developer_context_entry( + && let Some(entry) = crate::context_manager::updates::context_entry( + crate::context_manager::updates::DEVELOPER_BUNDLE, "model_switch", 10, - model_switch_message, + PromptMessage::developer_text(model_switch_message), ) { entries.push(entry); @@ -2562,10 +2576,11 @@ impl Session { .enabled(Feature::RequestPermissionsTool), ) .render(); - if let Some(entry) = crate::context_manager::updates::developer_context_entry( + if let Some(entry) = crate::context_manager::updates::context_entry( + crate::context_manager::updates::DEVELOPER_BUNDLE, "permissions", 20, - permissions_instructions, + PromptMessage::developer_text(permissions_instructions), ) { entries.push(entry); } @@ -2577,10 +2592,11 @@ impl Session { if !separate_guardian_developer_message && let Some(developer_instructions) = turn_context.developer_instructions.as_deref() && !developer_instructions.is_empty() - && let Some(entry) = crate::context_manager::updates::developer_context_entry( + && let Some(entry) = crate::context_manager::updates::context_entry( + crate::context_manager::updates::DEVELOPER_BUNDLE, "developer_instructions", 30, - developer_instructions.to_string(), + PromptMessage::developer_text(developer_instructions.to_string()), ) { entries.push(entry); @@ -2590,10 +2606,11 @@ impl Session { && turn_context.config.memories.use_memories && let Some(memory_prompt) = build_memory_tool_developer_instructions(&turn_context.config.codex_home).await - && let Some(entry) = crate::context_manager::updates::developer_context_entry( + && let Some(entry) = crate::context_manager::updates::context_entry( + crate::context_manager::updates::DEVELOPER_BUNDLE, "memory_tool", 40, - memory_prompt, + PromptMessage::developer_text(memory_prompt), ) { entries.push(entry); @@ -2601,10 +2618,11 @@ impl Session { // Add developer instructions from collaboration_mode if they exist and are non-empty if let Some(collab_instructions) = CollaborationModeInstructions::from_collaboration_mode(&collaboration_mode) - && let Some(entry) = crate::context_manager::updates::developer_context_entry( + && let Some(entry) = crate::context_manager::updates::context_entry( + crate::context_manager::updates::DEVELOPER_BUNDLE, "collaboration_mode", 50, - collab_instructions.render(), + PromptMessage::developer_text(collab_instructions.render()), ) { entries.push(entry); @@ -2613,10 +2631,11 @@ impl Session { reference_context_item.as_ref(), previous_turn_settings.as_ref(), turn_context, - ) && let Some(entry) = crate::context_manager::updates::developer_context_entry( + ) && let Some(entry) = crate::context_manager::updates::context_entry( + crate::context_manager::updates::DEVELOPER_BUNDLE, "realtime", 60, - realtime_update, + PromptMessage::developer_text(realtime_update), ) { entries.push(entry); } @@ -2632,10 +2651,13 @@ impl Session { &model_info, personality, ) - && let Some(entry) = crate::context_manager::updates::developer_context_entry( + && let Some(entry) = crate::context_manager::updates::context_entry( + crate::context_manager::updates::DEVELOPER_BUNDLE, "personality", 70, - PersonalitySpecInstructions::new(personality_message).render(), + PromptMessage::developer_text( + PersonalitySpecInstructions::new(personality_message).render(), + ), ) { entries.push(entry); @@ -2651,10 +2673,11 @@ impl Session { .await; if let Some(apps_instructions) = AppsInstructions::from_connectors(&accessible_and_enabled_connectors) - && let Some(entry) = crate::context_manager::updates::developer_context_entry( + && let Some(entry) = crate::context_manager::updates::context_entry( + crate::context_manager::updates::DEVELOPER_BUNDLE, "apps", 80, - apps_instructions.render(), + PromptMessage::developer_text(apps_instructions.render()), ) { entries.push(entry); @@ -2680,10 +2703,11 @@ impl Session { }) .await; } - if let Some(entry) = crate::context_manager::updates::developer_context_entry( + if let Some(entry) = crate::context_manager::updates::context_entry( + crate::context_manager::updates::DEVELOPER_BUNDLE, "skills", 90, - skills_instructions.render(), + PromptMessage::developer_text(skills_instructions.render()), ) { entries.push(entry); } @@ -2696,10 +2720,11 @@ impl Session { .await; if let Some(plugin_instructions) = AvailablePluginsInstructions::from_plugins(loaded_plugins.capability_summaries()) - && let Some(entry) = crate::context_manager::updates::developer_context_entry( + && let Some(entry) = crate::context_manager::updates::context_entry( + crate::context_manager::updates::DEVELOPER_BUNDLE, "plugins", 100, - plugin_instructions.render(), + PromptMessage::developer_text(plugin_instructions.render()), ) { entries.push(entry); @@ -2708,23 +2733,27 @@ impl Session { && let Some(commit_message_instruction) = commit_message_trailer_instruction( turn_context.config.commit_attribution.as_deref(), ) - && let Some(entry) = crate::context_manager::updates::developer_context_entry( + && let Some(entry) = crate::context_manager::updates::context_entry( + crate::context_manager::updates::DEVELOPER_BUNDLE, "commit_message_trailer", 110, - commit_message_instruction, + PromptMessage::developer_text(commit_message_instruction), ) { entries.push(entry); } if let Some(user_instructions) = turn_context.user_instructions.as_deref() - && let Some(entry) = crate::context_manager::updates::contextual_user_context_entry( + && let Some(entry) = crate::context_manager::updates::context_entry( + crate::context_manager::updates::CONTEXTUAL_USER_BUNDLE, "user_instructions", 130, - UserInstructions { - text: user_instructions.to_string(), - directory: turn_context.cwd.to_string_lossy().into_owned(), - } - .render(), + PromptMessage::user_text( + UserInstructions { + text: user_instructions.to_string(), + directory: turn_context.cwd.to_string_lossy().into_owned(), + } + .render(), + ), ) { entries.push(entry); @@ -2735,12 +2764,18 @@ impl Session { .agent_control .format_environment_context_subagents(self.conversation_id) .await; - if let Some(entry) = crate::context_manager::updates::contextual_user_context_entry( + if let Some(entry) = crate::context_manager::updates::context_entry( + crate::context_manager::updates::CONTEXTUAL_USER_BUNDLE, "environment", 140, - crate::context::EnvironmentContext::from_turn_context(turn_context, shell.as_ref()) + PromptMessage::user_text( + crate::context::EnvironmentContext::from_turn_context( + turn_context, + shell.as_ref(), + ) .with_subagents(subagents) .render(), + ), ) { entries.push(entry); } @@ -2750,10 +2785,11 @@ impl Session { multi_agents::usage_hint_text(turn_context, &session_source); if let Some(usage_hint_text) = multi_agent_v2_usage_hint_text - && let Some(entry) = crate::context_manager::updates::usage_hint_context_entry( + && let Some(entry) = crate::context_manager::updates::context_entry( + crate::context_manager::updates::USAGE_HINT_BUNDLE, "multi_agent_v2", 120, - usage_hint_text.to_string(), + PromptMessage::developer_text(usage_hint_text.to_string()), ) { entries.push(entry); @@ -2763,10 +2799,11 @@ impl Session { if separate_guardian_developer_message && let Some(developer_instructions) = turn_context.developer_instructions.as_deref() && !developer_instructions.is_empty() - && let Some(entry) = crate::context_manager::updates::guardian_context_entry( + && let Some(entry) = crate::context_manager::updates::context_entry( + crate::context_manager::updates::GUARDIAN_BUNDLE, "developer_instructions", 150, - developer_instructions.to_string(), + PromptMessage::developer_text(developer_instructions.to_string()), ) { entries.push(entry); diff --git a/codex-rs/core/src/state/history.rs b/codex-rs/core/src/state/history.rs index 0daa1a3e54..14507f2488 100644 --- a/codex-rs/core/src/state/history.rs +++ b/codex-rs/core/src/state/history.rs @@ -3,9 +3,9 @@ use crate::event_mapping::is_contextual_dev_message_content; use crate::event_mapping::is_contextual_user_message_content; use crate::session::turn_context::TurnContext; use codex_journal::Journal; -use codex_journal::JournalHistoryItem; use codex_journal::JournalItem; use codex_journal::JournalKey; +use codex_journal::JournalTranscriptItem; use codex_journal::history as thread_history; use codex_journal::history::estimate_item_token_count; use codex_journal::history::estimate_response_item_model_visible_bytes; @@ -52,8 +52,8 @@ pub(crate) fn raw_items(journal: &Journal) -> Vec { .entries() .iter() .filter_map(|entry| match &entry.item { - JournalItem::History(item) => Some(item.item.clone()), - JournalItem::Context(_) | JournalItem::Checkpoint(_) => None, + JournalItem::Transcript(item) => Some(item.item.clone()), + JournalItem::Metadata(_) | JournalItem::Checkpoint(_) => None, }) .collect() } @@ -232,7 +232,7 @@ pub(crate) fn get_total_token_usage_breakdown( } fn push_history_item(journal: &mut Journal, item: ResponseItem) { - let history_item = JournalHistoryItem::new(item); + let history_item = JournalTranscriptItem::new(item); let key = JournalKey::new(vec!["history".to_string(), history_item.id.clone()]); journal.add(key, history_item); } diff --git a/codex-rs/journal/Cargo.toml b/codex-rs/journal/Cargo.toml index 14aab7d67a..a2469913da 100644 --- a/codex-rs/journal/Cargo.toml +++ b/codex-rs/journal/Cargo.toml @@ -3,6 +3,7 @@ name = "codex-journal" version.workspace = true edition.workspace = true license.workspace = true +readme = "README.md" [lints] workspace = true diff --git a/codex-rs/journal/README.md b/codex-rs/journal/README.md new file mode 100644 index 0000000000..4b0d6a86c9 --- /dev/null +++ b/codex-rs/journal/README.md @@ -0,0 +1,86 @@ +# codex-journal + +`codex-journal` is the typed journal model behind prompt metadata, transcript rewriting, forked views, +and prompt-ready projection in Codex. + +## Model + +A `Journal` stores one append-only sequence of `JournalEntry` values. Each entry is one of: + +- prompt metadata: keyed entries that can replace older entries with the same key +- transcript: ordered prompt-visible items +- checkpoints: transcript rewrite operations such as prefix replacement or truncation + +The append-only journal is the source of truth. Derived views are produced by resolving it. + +## Resolved Views + +`Journal::resolve()` returns a `ResolvedJournal` with two typed views: + +- `ResolvedMetadata`: effective prompt-metadata entries after key-based replacement +- `ResolvedTranscript`: effective transcript after checkpoint application + +These two views come from the same journal, but they behave differently: + +- metadata is deduplicated by key and ordered by `prompt_order` +- transcript preserves order and is rewritten only by checkpoints + +## Building Metadata Entries + +Use `Journal::metadata_entry_builder(...)` when you want explicit control over metadata: + +```rust +use codex_journal::Journal; +use codex_journal::JournalContextAudience; +use codex_journal::PromptMessage; + +let entry = Journal::metadata_entry_builder( + ["prompt", "developer", "permissions"], + PromptMessage::developer_text("sandbox is workspace-write"), +) +.prompt_order(20) +.audience(JournalContextAudience::All) +.build(); +``` + +`Journal::metadata_entry(...)` remains available as a shorthand for the common case. + +## Rendering + +Prompt rendering lives in `PromptRenderer`, not in `Journal` itself. Grouping is explicit and +applied only to consecutive resolved metadata entries: + +```rust +use codex_journal::Journal; +use codex_journal::KeyFilter; +use codex_journal::PromptMessage; +use codex_journal::PromptRenderer; + +let journal = Journal::from_entries(vec![ + Journal::metadata_entry( + ["prompt", "developer", "one"], + 10, + PromptMessage::developer_text("first"), + ).unwrap(), + Journal::metadata_entry( + ["prompt", "developer", "two"], + 20, + PromptMessage::developer_text("second"), + ).unwrap(), +]); + +let resolved = journal.resolve().unwrap(); +let prompt = PromptRenderer::new() + .group(KeyFilter::prefix(["prompt", "developer"])) + .render_metadata(resolved.metadata()); +``` + +## Transformations + +`Journal` also supports: + +- `filter` and `resolve_with_filter` for key-scoped views +- `flatten` for dropping obsolete entries while keeping the current effective view +- `fork` for producing child views with audience and `on_fork` filtering +- `with_history_window` for keeping only a recent hot history suffix +- `persist_jsonl` / `load_jsonl` for durable JSONL storage diff --git a/codex-rs/journal/src/context_builder.rs b/codex-rs/journal/src/context_builder.rs new file mode 100644 index 0000000000..18debcc9d1 --- /dev/null +++ b/codex-rs/journal/src/context_builder.rs @@ -0,0 +1,87 @@ +use crate::JournalContextAudience; +use crate::JournalContextForkBehavior; +use crate::JournalEntry; +use crate::JournalKey; +use crate::JournalMetadataItem; +use crate::PromptMessage; +use codex_protocol::models::ContentItem; + +/// Builder for one prompt-metadata journal entry. +#[derive(Debug, Clone)] +pub struct MetadataEntryBuilder { + key: JournalKey, + message: PromptMessage, + prompt_order: i64, + audience: JournalContextAudience, + on_fork: JournalContextForkBehavior, + tags: Vec, + source: Option, +} + +impl MetadataEntryBuilder { + pub(crate) fn new(key: impl Into, message: impl Into) -> Self { + Self { + key: key.into(), + message: message.into(), + prompt_order: 0, + audience: JournalContextAudience::default(), + on_fork: JournalContextForkBehavior::default(), + tags: Vec::new(), + source: None, + } + } + + /// Sets the prompt ordering used after resolution. + pub fn prompt_order(mut self, prompt_order: i64) -> Self { + self.prompt_order = prompt_order; + self + } + + /// Sets the audience used when projecting or forking the journal. + pub fn audience(mut self, audience: JournalContextAudience) -> Self { + self.audience = audience; + self + } + + /// Sets how this context entry behaves when creating a forked journal view. + pub fn on_fork(mut self, on_fork: JournalContextForkBehavior) -> Self { + self.on_fork = on_fork; + self + } + + /// Sets arbitrary tags for downstream classification. + pub fn tags(mut self, tags: Vec) -> Self { + self.tags = tags; + self + } + + /// Records the origin of this prompt-metadata entry. + pub fn source(mut self, source: impl Into) -> Self { + self.source = Some(source.into()); + self + } + + /// Builds one prompt-metadata entry if the message is non-empty after trimming text content. + pub fn build(self) -> Option { + if self.message.content.is_empty() + || self.message.content.iter().all(|item| match item { + ContentItem::InputText { text } | ContentItem::OutputText { text } => { + text.trim().is_empty() + } + ContentItem::InputImage { .. } => false, + }) + { + return None; + } + + let mut item = JournalMetadataItem::new(self.message) + .with_prompt_order(self.prompt_order) + .with_audience(self.audience) + .with_on_fork(self.on_fork) + .with_tags(self.tags); + if let Some(source) = self.source { + item = item.with_source(source); + } + Some(JournalEntry::new(self.key, item)) + } +} diff --git a/codex-rs/journal/src/journal.rs b/codex-rs/journal/src/journal.rs index d4f70e29bf..d0a0f523f8 100644 --- a/codex-rs/journal/src/journal.rs +++ b/codex-rs/journal/src/journal.rs @@ -1,18 +1,18 @@ use crate::JournalEntry; use crate::JournalError; +use crate::JournalItem; use crate::JournalKey; use crate::KeyFilter; +use crate::MetadataEntryBuilder; use crate::PromptView; use crate::Result; use codex_protocol::journal::JournalCheckpointItem; use codex_protocol::journal::JournalContextAudience; use codex_protocol::journal::JournalContextForkBehavior; -use codex_protocol::journal::JournalContextItem; -use codex_protocol::journal::JournalContextKey; use codex_protocol::journal::JournalHistoryCursor; -use codex_protocol::journal::JournalHistoryItem; -use codex_protocol::journal::JournalItem; +use codex_protocol::journal::JournalMetadataItem; use codex_protocol::journal::JournalReplacePrefixCheckpoint; +use codex_protocol::journal::JournalTranscriptItem; use codex_protocol::journal::JournalTruncateHistoryCheckpoint; use codex_protocol::models::ResponseItem; use indexmap::IndexMap; @@ -63,6 +63,40 @@ impl Journal { &self.entries } + /// Starts building one prompt-metadata entry. + pub fn metadata_entry_builder( + key: impl Into, + message: impl Into, + ) -> MetadataEntryBuilder { + MetadataEntryBuilder::new(key, message) + } + + /// Builds one prompt-metadata entry if the message is non-empty after trimming text content. + pub fn metadata_entry( + key: impl Into, + prompt_order: i64, + message: impl Into, + ) -> Option { + Self::metadata_entry_builder(key, message) + .prompt_order(prompt_order) + .build() + } + + pub fn context_entry_builder( + key: impl Into, + message: impl Into, + ) -> MetadataEntryBuilder { + Self::metadata_entry_builder(key, message) + } + + pub fn context_entry( + key: impl Into, + prompt_order: i64, + message: impl Into, + ) -> Option { + Self::metadata_entry(key, prompt_order, message) + } + /// Returns a journal containing only journal entries whose keys match the filter. pub fn filter(&self, filter: &KeyFilter) -> Self { let entries = self @@ -89,8 +123,8 @@ impl Journal { self.to_prompt_matching_filter(view, None) } - /// Renders the current effective journal view into model prompt items after selecting - /// only journal entries whose keys match the filter. + /// Renders the current effective journal view into model prompt items after selecting only + /// journal entries whose keys match the filter. pub fn to_prompt_with_filter( &self, view: &PromptView, @@ -115,14 +149,8 @@ impl Journal { /// This is the first building block for a rolling in-memory window: callers can /// persist the full journal elsewhere, then keep only the flattened journal hot. pub fn flatten(&self) -> Result { - let resolved = self.resolve(None)?; - Ok(Self::from_entries( - resolved - .contexts - .into_iter() - .chain(resolved.history) - .collect(), - )) + let resolved = self.resolve()?; + Ok(Self::from_entries(resolved.into_entries())) } /// Keeps only the current effective journal view plus the history suffix that starts @@ -131,13 +159,16 @@ impl Journal { /// This is a lightweight rolling-window helper: callers can persist the full /// journal on disk, then keep only a recent hot suffix in memory. pub fn with_history_window(&self, start: &JournalHistoryCursor) -> Result { - let resolved = self.resolve(None)?; - let start_index = resolve_cursor(resolved.history.as_slice(), start)?; + let resolved = self.resolve()?; + let start_index = resolve_cursor(resolved.transcript().entries(), start)?; + let ResolvedJournal { + metadata, + transcript, + } = resolved; Ok(Self::from_entries( - resolved - .contexts + metadata .into_iter() - .chain(resolved.history.into_iter().skip(start_index)) + .chain(transcript.into_iter().skip(start_index)) .collect(), )) } @@ -175,44 +206,60 @@ impl Journal { Ok(Self::from_entries(entries)) } + /// Resolves the current effective journal into prompt metadata and transcript views. + pub fn resolve(&self) -> Result { + self.resolve_filter(None) + } + + /// Resolves the current effective journal after selecting only entries whose keys match the + /// filter. + pub fn resolve_with_filter(&self, filter: &KeyFilter) -> Result { + self.resolve_filter(Some(filter)) + } + fn to_prompt_matching_filter( &self, view: &PromptView, filter: Option<&KeyFilter>, ) -> Result> { - let resolved = self.resolve(filter)?; + let resolved = self.resolve_filter(filter)?; let mut prompt: Vec = resolved - .contexts + .metadata .into_iter() - .filter(|entry| context_visible_in_view(context_item(entry), view)) + .filter(|entry| metadata_visible_in_view(metadata_item(entry), view)) .map(|entry| match entry.item { - JournalItem::Context(item) => ResponseItem::from(item), - _ => unreachable!("resolved context entries must be context items"), + JournalItem::Metadata(item) => ResponseItem::from(item), + _ => unreachable!("resolved metadata entries must be metadata items"), }) .collect(); - prompt.extend(resolved.history.into_iter().map(|entry| match entry.item { - JournalItem::History(item) => ResponseItem::from(item), - _ => unreachable!("resolved history entries must be history items"), - })); + prompt.extend( + resolved + .transcript + .into_iter() + .map(|entry| match entry.item { + JournalItem::Transcript(item) => ResponseItem::from(item), + _ => unreachable!("resolved transcript entries must be transcript items"), + }), + ); Ok(prompt) } fn fork_matching_filter(&self, view: &PromptView, filter: Option<&KeyFilter>) -> Result { - let resolved = self.resolve(filter)?; + let resolved = self.resolve_filter(filter)?; Ok(Self::from_entries( resolved - .contexts + .metadata .into_iter() - .filter(|entry| context_item(entry).on_fork == JournalContextForkBehavior::Keep) - .filter(|entry| context_visible_in_view(context_item(entry), view)) - .chain(resolved.history) + .filter(|entry| metadata_item(entry).on_fork == JournalContextForkBehavior::Keep) + .filter(|entry| metadata_visible_in_view(metadata_item(entry), view)) + .chain(resolved.transcript) .collect(), )) } - fn resolve(&self, filter: Option<&KeyFilter>) -> Result { - let mut history = Vec::::new(); - let mut latest_context_by_key = IndexMap::::new(); + fn resolve_filter(&self, filter: Option<&KeyFilter>) -> Result { + let mut transcript = Vec::::new(); + let mut latest_metadata_by_key = IndexMap::::new(); for (index, entry) in self.entries.iter().enumerate() { if let Some(filter) = filter @@ -221,39 +268,192 @@ impl Journal { continue; } match &entry.item { - JournalItem::History(_) => history.push(entry.clone()), - JournalItem::Context(context_item) => { - latest_context_by_key.insert(context_item.key.clone(), (index, entry.clone())); + JournalItem::Transcript(_) => transcript.push(entry.clone()), + JournalItem::Metadata(_) => { + latest_metadata_by_key.insert(entry.key.clone(), (index, entry.clone())); } JournalItem::Checkpoint(checkpoint) => { - apply_checkpoint(&mut history, &entry.key, checkpoint)?; + apply_checkpoint(&mut transcript, &entry.key, checkpoint)?; } } } - let mut contexts = latest_context_by_key.into_values().collect::>(); - contexts.sort_by(|(left_index, left_entry), (right_index, right_entry)| { - context_item(left_entry) + let mut metadata = latest_metadata_by_key.into_values().collect::>(); + metadata.sort_by(|(left_index, left_entry), (right_index, right_entry)| { + metadata_item(left_entry) .prompt_order - .cmp(&context_item(right_entry).prompt_order) + .cmp(&metadata_item(right_entry).prompt_order) .then_with(|| left_index.cmp(right_index)) }); - Ok(ResolvedJournal { - contexts: contexts.into_iter().map(|(_, entry)| entry).collect(), - history, - }) + Ok(ResolvedJournal::new( + metadata.into_iter().map(|(_, entry)| entry).collect(), + transcript, + )) } } -#[derive(Debug)] -struct ResolvedJournal { - contexts: Vec, - history: Vec, +/// Effective journal view after deduplicating prompt context and applying history checkpoints. +/// +/// `contexts` and `history` are derived from the same append-only source of truth, but they have +/// different semantics: +/// +/// - [`ResolvedMetadata`] contains keyed prompt-metadata entries. Later entries with the same key +/// replace earlier ones and the result is ordered by `prompt_order`. +/// - [`ResolvedTranscript`] contains ordered transcript entries after checkpoint application. +/// Transcript preserves order and can be truncated or rewritten by checkpoints. +#[derive(Debug, Clone, PartialEq)] +pub struct ResolvedJournal { + metadata: ResolvedMetadata, + transcript: ResolvedTranscript, +} + +impl ResolvedJournal { + fn new(metadata: Vec, transcript: Vec) -> Self { + Self { + metadata: ResolvedMetadata::new(metadata), + transcript: ResolvedTranscript::new(transcript), + } + } + + /// Returns the effective prompt-metadata view. + pub fn metadata(&self) -> &ResolvedMetadata { + &self.metadata + } + + /// Returns the effective transcript view. + pub fn transcript(&self) -> &ResolvedTranscript { + &self.transcript + } + + /// Returns the effective prompt-metadata view. + pub fn contexts(&self) -> &ResolvedMetadata { + self.metadata() + } + + /// Returns the effective transcript view. + pub fn history(&self) -> &ResolvedTranscript { + self.transcript() + } + + /// Consumes the resolved view and returns only the prompt-metadata entries. + pub fn into_metadata(self) -> ResolvedMetadata { + self.metadata + } + + /// Consumes the resolved view and returns only the transcript entries. + pub fn into_transcript(self) -> ResolvedTranscript { + self.transcript + } + + pub fn into_contexts(self) -> ResolvedMetadata { + self.into_metadata() + } + + pub fn into_history(self) -> ResolvedTranscript { + self.into_transcript() + } + + /// Consumes the resolved view and concatenates metadata followed by transcript entries. + pub fn into_entries(self) -> Vec { + self.metadata + .into_entries() + .into_iter() + .chain(self.transcript.into_entries()) + .collect() + } +} + +/// Effective prompt-metadata entries derived from a journal. +/// +/// These entries are deduplicated by key and sorted for prompt rendering. They are suitable for +/// rendering or for building flattened and forked journal states. +#[derive(Debug, Clone, PartialEq)] +pub struct ResolvedMetadata { + entries: Vec, +} + +impl ResolvedMetadata { + fn new(entries: Vec) -> Self { + Self { entries } + } + + /// Returns the resolved prompt-metadata entries. + pub fn entries(&self) -> &[JournalEntry] { + &self.entries + } + + /// Returns whether there are no resolved prompt-metadata entries. + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + /// Returns the number of resolved prompt-metadata entries. + pub fn len(&self) -> usize { + self.entries.len() + } + + /// Consumes the view and returns its entries. + pub fn into_entries(self) -> Vec { + self.entries + } +} + +impl IntoIterator for ResolvedMetadata { + type Item = JournalEntry; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.entries.into_iter() + } +} + +/// Effective transcript entries derived from a journal. +/// +/// These entries preserve prompt-visible order after checkpoint application. Unlike prompt +/// metadata, transcript is not deduplicated by key. +#[derive(Debug, Clone, PartialEq)] +pub struct ResolvedTranscript { + entries: Vec, +} + +impl ResolvedTranscript { + fn new(entries: Vec) -> Self { + Self { entries } + } + + /// Returns the resolved transcript entries. + pub fn entries(&self) -> &[JournalEntry] { + &self.entries + } + + /// Returns whether there are no resolved transcript entries. + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + /// Returns the number of resolved transcript entries. + pub fn len(&self) -> usize { + self.entries.len() + } + + /// Consumes the view and returns its entries. + pub fn into_entries(self) -> Vec { + self.entries + } +} + +impl IntoIterator for ResolvedTranscript { + type Item = JournalEntry; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.entries.into_iter() + } } fn apply_checkpoint( - history: &mut Vec, + transcript: &mut Vec, checkpoint_key: &JournalKey, checkpoint: &JournalCheckpointItem, ) -> Result<()> { @@ -262,40 +462,40 @@ fn apply_checkpoint( through, replacement, }) => { - let keep_from = resolve_cursor(history.as_slice(), through)?; - let mut next_history = - Vec::with_capacity(replacement.len() + history.len().saturating_sub(keep_from)); - next_history.extend( + let keep_from = resolve_cursor(transcript.as_slice(), through)?; + let mut next_transcript = + Vec::with_capacity(replacement.len() + transcript.len().saturating_sub(keep_from)); + next_transcript.extend( replacement .iter() .cloned() .enumerate() .map(|(index, item)| { JournalEntry::new( - replacement_history_key(checkpoint_key, index, &item), + replacement_transcript_key(checkpoint_key, index, &item), item, ) }), ); - next_history.extend(history[keep_from..].iter().cloned()); - *history = next_history; + next_transcript.extend(transcript[keep_from..].iter().cloned()); + *transcript = next_transcript; Ok(()) } JournalCheckpointItem::TruncateHistory(JournalTruncateHistoryCheckpoint { through }) => { - let keep_len = resolve_cursor(history.as_slice(), through)?; - history.truncate(keep_len); + let keep_len = resolve_cursor(transcript.as_slice(), through)?; + transcript.truncate(keep_len); Ok(()) } } } -fn resolve_cursor(history: &[JournalEntry], cursor: &JournalHistoryCursor) -> Result { +fn resolve_cursor(transcript: &[JournalEntry], cursor: &JournalHistoryCursor) -> Result { match cursor { JournalHistoryCursor::Start => Ok(0), - JournalHistoryCursor::End => Ok(history.len()), - JournalHistoryCursor::AfterItem(history_item_id) => history + JournalHistoryCursor::End => Ok(transcript.len()), + JournalHistoryCursor::AfterItem(history_item_id) => transcript .iter() - .position(|entry| history_item(entry).id == *history_item_id) + .position(|entry| transcript_item(entry).id == *history_item_id) .map(|index| index + 1) .ok_or_else(|| JournalError::UnknownHistoryItemId { history_item_id: history_item_id.clone(), @@ -303,32 +503,32 @@ fn resolve_cursor(history: &[JournalEntry], cursor: &JournalHistoryCursor) -> Re } } -fn replacement_history_key( +fn replacement_transcript_key( checkpoint_key: &JournalKey, index: usize, - history_item: &JournalHistoryItem, + transcript_item: &JournalTranscriptItem, ) -> JournalKey { checkpoint_key .child("replacement") .child(index.to_string()) - .child(history_item.id.clone()) + .child(transcript_item.id.clone()) } -fn context_item(entry: &JournalEntry) -> &JournalContextItem { +fn metadata_item(entry: &JournalEntry) -> &JournalMetadataItem { match &entry.item { - JournalItem::Context(item) => item, - _ => unreachable!("resolved context entries must be context items"), + JournalItem::Metadata(item) => item, + _ => unreachable!("resolved metadata entries must be metadata items"), } } -fn history_item(entry: &JournalEntry) -> &JournalHistoryItem { +fn transcript_item(entry: &JournalEntry) -> &JournalTranscriptItem { match &entry.item { - JournalItem::History(item) => item, - _ => unreachable!("resolved history entries must be history items"), + JournalItem::Transcript(item) => item, + _ => unreachable!("resolved transcript entries must be transcript items"), } } -fn context_visible_in_view(item: &JournalContextItem, view: &PromptView) -> bool { +fn metadata_visible_in_view(item: &JournalMetadataItem, view: &PromptView) -> bool { match &item.audience { JournalContextAudience::All => true, JournalContextAudience::RootOnly => view.is_root, diff --git a/codex-rs/journal/src/lib.rs b/codex-rs/journal/src/lib.rs index 5d020bffd3..383feb42c0 100644 --- a/codex-rs/journal/src/lib.rs +++ b/codex-rs/journal/src/lib.rs @@ -1,9 +1,20 @@ //! Typed journal model for prompt rendering, filtering, forking, and persistence. +//! +//! A [`Journal`] stores one append-only sequence of [`JournalEntry`] values and resolves it into +//! two derived views: +//! +//! - prompt metadata: keyed, deduplicated entries ordered by `prompt_order` +//! - transcript: ordered prompt-visible items after checkpoint application +//! +//! Callers can resolve the journal once, then choose how to project those views into prompt +//! messages with [`PromptRenderer`]. +mod context_builder; mod error; pub mod history; mod journal; mod prompt_view; +mod render; #[cfg(test)] mod tests; @@ -11,19 +22,29 @@ mod tests; pub use codex_protocol::journal::JournalCheckpointItem; pub use codex_protocol::journal::JournalContextAudience; pub use codex_protocol::journal::JournalContextForkBehavior; -pub use codex_protocol::journal::JournalContextItem; -pub use codex_protocol::journal::JournalContextKey; pub use codex_protocol::journal::JournalEntry; pub use codex_protocol::journal::JournalHistoryCursor; -pub use codex_protocol::journal::JournalHistoryItem; pub use codex_protocol::journal::JournalItem; pub use codex_protocol::journal::JournalKey; +pub use codex_protocol::journal::JournalMetadataItem; pub use codex_protocol::journal::JournalReplacePrefixCheckpoint; +pub use codex_protocol::journal::JournalTranscriptItem; pub use codex_protocol::journal::JournalTruncateHistoryCheckpoint; pub use codex_protocol::journal::KeyFilter; pub use codex_protocol::journal::PromptMessage; pub use codex_protocol::journal::PromptMessageRole; +pub use context_builder::MetadataEntryBuilder; pub use error::JournalError; pub use error::Result; pub use journal::Journal; +pub use journal::ResolvedJournal; +pub use journal::ResolvedMetadata; +pub use journal::ResolvedTranscript; pub use prompt_view::PromptView; +pub use render::PromptRenderer; + +pub type JournalContextItem = JournalMetadataItem; +pub type JournalHistoryItem = JournalTranscriptItem; +pub type ContextEntryBuilder = MetadataEntryBuilder; +pub type ResolvedContexts = ResolvedMetadata; +pub type ResolvedHistory = ResolvedTranscript; diff --git a/codex-rs/journal/src/render.rs b/codex-rs/journal/src/render.rs new file mode 100644 index 0000000000..eacd5daebf --- /dev/null +++ b/codex-rs/journal/src/render.rs @@ -0,0 +1,86 @@ +use crate::JournalItem; +use crate::KeyFilter; +use crate::PromptMessage; +use crate::PromptMessageRole; +use crate::ResolvedMetadata; +use codex_protocol::models::ContentItem; +use codex_protocol::models::ResponseItem; + +/// Renders resolved prompt metadata into model-visible prompt messages. +/// +/// Group filters are applied in declaration order. Entries are merged only when they are +/// consecutive, match the same group filter, and share the same role. +#[derive(Debug, Clone, Default)] +pub struct PromptRenderer { + group_filters: Vec, +} + +impl PromptRenderer { + /// Creates a renderer with no grouping rules. + pub fn new() -> Self { + Self::default() + } + + /// Adds one grouping rule for consecutive resolved metadata entries. + pub fn group(mut self, filter: KeyFilter) -> Self { + self.group_filters.push(filter); + self + } + + /// Renders resolved prompt metadata according to the configured grouping rules. + pub fn render_metadata(&self, metadata: &ResolvedMetadata) -> Vec { + let mut rendered = Vec::new(); + let mut current_group: Option = None; + let mut current_role: Option = None; + let mut current_content = Vec::new(); + + for entry in metadata.entries() { + let JournalItem::Metadata(item) = &entry.item else { + continue; + }; + let group = self + .group_filters + .iter() + .position(|filter| filter.matches(&entry.key)); + + if group.is_none() { + flush_prompt_message(&mut rendered, &mut current_role, &mut current_content); + rendered.push(ResponseItem::from(item.clone())); + current_group = None; + continue; + } + + if current_group != group || current_role != Some(item.message.role) { + flush_prompt_message(&mut rendered, &mut current_role, &mut current_content); + current_group = group; + current_role = Some(item.message.role); + } + + current_content.extend(item.message.content.clone()); + } + + flush_prompt_message(&mut rendered, &mut current_role, &mut current_content); + rendered + } + + pub fn render_contexts(&self, contexts: &ResolvedMetadata) -> Vec { + self.render_metadata(contexts) + } +} + +fn flush_prompt_message( + rendered: &mut Vec, + role: &mut Option, + content: &mut Vec, +) { + let Some(role) = role.take() else { + return; + }; + if content.is_empty() { + return; + } + rendered.push(ResponseItem::from(PromptMessage::new( + role, + std::mem::take(content), + ))); +} diff --git a/codex-rs/journal/src/tests.rs b/codex-rs/journal/src/tests.rs index 81bac17474..f60603ce49 100644 --- a/codex-rs/journal/src/tests.rs +++ b/codex-rs/journal/src/tests.rs @@ -2,15 +2,15 @@ use crate::Journal; use crate::JournalCheckpointItem; use crate::JournalContextAudience; use crate::JournalContextForkBehavior; -use crate::JournalContextItem; -use crate::JournalContextKey; use crate::JournalEntry; use crate::JournalHistoryCursor; -use crate::JournalHistoryItem; +use crate::JournalMetadataItem; use crate::JournalReplacePrefixCheckpoint; +use crate::JournalTranscriptItem; use crate::JournalTruncateHistoryCheckpoint; use crate::KeyFilter; use crate::PromptMessage; +use crate::PromptRenderer; use crate::PromptView; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; @@ -39,30 +39,21 @@ fn assistant_message(text: &str) -> ResponseItem { } } -fn developer_context( - namespace: &str, - name: &str, - text: &str, - prompt_order: i64, -) -> JournalContextItem { - JournalContextItem::new( - JournalContextKey::new(namespace, name, None), - PromptMessage::developer_text(text), - ) - .with_prompt_order(prompt_order) +fn developer_context(text: &str, prompt_order: i64) -> JournalMetadataItem { + JournalMetadataItem::new(PromptMessage::developer_text(text)).with_prompt_order(prompt_order) } #[test] fn to_prompt_uses_latest_context_for_key() { let mut state = Journal::new(); state.add( - ["prompt", "permissions", "older"], - developer_context("context", "permissions", "older permissions", 10), + ["prompt", "permissions"], + developer_context("older permissions", 10), ); state.add(["history", "hello"], user_message("hello")); state.add( - ["prompt", "permissions", "newer"], - developer_context("context", "permissions", "newer permissions", 10), + ["prompt", "permissions"], + developer_context("newer permissions", 10), ); let prompt = state @@ -83,13 +74,11 @@ fn to_prompt_filters_context_by_audience() { let mut state = Journal::new(); state.add( ["prompt", "root", "hint"], - developer_context("context", "root", "root-only", 0) - .with_audience(JournalContextAudience::RootOnly), + developer_context("root-only", 0).with_audience(JournalContextAudience::RootOnly), ); state.add( ["prompt", "child", "hint"], - developer_context("context", "child", "child-only", 1) - .with_audience(JournalContextAudience::SubAgentsOnly), + developer_context("child-only", 1).with_audience(JournalContextAudience::SubAgentsOnly), ); let root_prompt = state @@ -119,14 +108,8 @@ fn to_prompt_filters_context_by_audience() { #[test] fn to_prompt_with_filter_matches_key_prefix() { let mut state = Journal::new(); - state.add( - ["prompt", "root", "keep"], - developer_context("context", "keep", "keep me", 0), - ); - state.add( - ["prompt", "child", "drop"], - developer_context("context", "drop", "drop me", 1), - ); + state.add(["prompt", "root", "keep"], developer_context("keep me", 0)); + state.add(["prompt", "child", "drop"], developer_context("drop me", 1)); let prompt = state .to_prompt_with_filter(&PromptView::root(), &KeyFilter::prefix(["prompt", "root"])) @@ -140,10 +123,10 @@ fn to_prompt_with_filter_matches_key_prefix() { #[test] fn checkpoints_replace_prefix_and_then_truncate_history() { - let first = JournalHistoryItem::new(user_message("turn 1")); - let second = JournalHistoryItem::new(assistant_message("turn 1 answer")); - let third = JournalHistoryItem::new(user_message("turn 2")); - let summary = JournalHistoryItem { + let first = JournalTranscriptItem::new(user_message("turn 1")); + let second = JournalTranscriptItem::new(assistant_message("turn 1 answer")); + let third = JournalTranscriptItem::new(user_message("turn 2")); + let summary = JournalTranscriptItem { id: "summary".to_string(), turn_id: None, item: assistant_message("summary"), @@ -177,22 +160,16 @@ fn checkpoints_replace_prefix_and_then_truncate_history() { #[test] fn flatten_preserves_prompt_and_drops_obsolete_items() { - let first = JournalHistoryItem::new(user_message("turn 1")); - let answer = JournalHistoryItem::new(assistant_message("turn 1 answer")); - let summary = JournalHistoryItem { + let first = JournalTranscriptItem::new(user_message("turn 1")); + let answer = JournalTranscriptItem::new(assistant_message("turn 1 answer")); + let summary = JournalTranscriptItem { id: "summary".to_string(), turn_id: None, item: assistant_message("summary"), }; let state = Journal::from_entries(vec![ - JournalEntry::new( - ["prompt", "permissions", "old"], - developer_context("context", "permissions", "old", 0), - ), - JournalEntry::new( - ["prompt", "permissions", "new"], - developer_context("context", "permissions", "new", 0), - ), + JournalEntry::new(["prompt", "permissions"], developer_context("old", 0)), + JournalEntry::new(["prompt", "permissions"], developer_context("new", 0)), JournalEntry::new(["history", "1"], first), JournalEntry::new(["history", "2"], answer.clone()), JournalEntry::new( @@ -216,10 +193,7 @@ fn flatten_preserves_prompt_and_drops_obsolete_items() { assert_eq!( flattened.entries(), vec![ - JournalEntry::new( - ["prompt", "permissions", "new"], - developer_context("context", "permissions", "new", 0), - ), + JournalEntry::new(["prompt", "permissions"], developer_context("new", 0),), JournalEntry::new( ["checkpoint", "replace", "replacement", "0", "summary"], summary, @@ -230,13 +204,13 @@ fn flatten_preserves_prompt_and_drops_obsolete_items() { #[test] fn with_history_window_keeps_only_recent_effective_history() { - let first = JournalHistoryItem::new(user_message("turn 1")); - let second = JournalHistoryItem::new(assistant_message("turn 1 answer")); - let third = JournalHistoryItem::new(user_message("turn 2")); + let first = JournalTranscriptItem::new(user_message("turn 1")); + let second = JournalTranscriptItem::new(assistant_message("turn 1 answer")); + let third = JournalTranscriptItem::new(user_message("turn 2")); let state = Journal::from_entries(vec![ JournalEntry::new( ["prompt", "permissions", "current"], - developer_context("context", "permissions", "p", 0), + developer_context("p", 0), ), JournalEntry::new(["history", "1"], first), JournalEntry::new(["history", "2"], second.clone()), @@ -252,7 +226,7 @@ fn with_history_window_keeps_only_recent_effective_history() { vec![ JournalEntry::new( ["prompt", "permissions", "current"], - developer_context("context", "permissions", "p", 0), + developer_context("p", 0), ), JournalEntry::new(["history", "3"], third), ] @@ -261,23 +235,22 @@ fn with_history_window_keeps_only_recent_effective_history() { #[test] fn fork_drops_non_keep_context_and_respects_audience() { - let history = JournalHistoryItem::new(user_message("hello")); + let history = JournalTranscriptItem::new(user_message("hello")); let state = Journal::from_entries(vec![ JournalEntry::new( ["prompt", "child", "shared"], - developer_context("context", "shared", "shared child context", 0) + developer_context("shared child context", 0) .with_audience(JournalContextAudience::SubAgentsOnly), ), JournalEntry::new( ["prompt", "child", "regenerate"], - developer_context("context", "regenerate", "usage hint", 1) + developer_context("usage hint", 1) .with_audience(JournalContextAudience::SubAgentsOnly) .with_on_fork(JournalContextForkBehavior::Regenerate), ), JournalEntry::new( ["prompt", "root", "only"], - developer_context("context", "root", "root only", 2) - .with_audience(JournalContextAudience::RootOnly), + developer_context("root only", 2).with_audience(JournalContextAudience::RootOnly), ), JournalEntry::new(["history", "hello"], history.clone()), ]); @@ -294,7 +267,7 @@ fn fork_drops_non_keep_context_and_respects_audience() { vec![ JournalEntry::new( ["prompt", "child", "shared"], - developer_context("context", "shared", "shared child context", 0) + developer_context("shared child context", 0) .with_audience(JournalContextAudience::SubAgentsOnly), ), JournalEntry::new(["history", "hello"], history), @@ -306,11 +279,11 @@ fn fork_drops_non_keep_context_and_respects_audience() { fn persist_and_load_jsonl_round_trip() { let dir = tempdir().expect("tempdir"); let path = dir.path().join("journal.jsonl"); - let history = JournalHistoryItem::new(user_message("hello")); + let history = JournalTranscriptItem::new(user_message("hello")); let state = Journal::from_entries(vec![ JournalEntry::new( ["prompt", "permissions", "current"], - developer_context("context", "permissions", "p", 0), + developer_context("p", 0), ), JournalEntry::new(["history", "hello"], history), ]); @@ -326,14 +299,8 @@ fn persist_and_load_jsonl_round_trip() { #[test] fn filter_returns_matching_raw_entries() { let state = Journal::from_entries(vec![ - JournalEntry::new( - ["prompt", "root", "keep"], - developer_context("context", "keep", "keep me", 0), - ), - JournalEntry::new( - ["prompt", "child", "drop"], - developer_context("context", "drop", "drop me", 1), - ), + JournalEntry::new(["prompt", "root", "keep"], developer_context("keep me", 0)), + JournalEntry::new(["prompt", "child", "drop"], developer_context("drop me", 1)), JournalEntry::new(["history", "hello"], user_message("hello")), ]); @@ -343,7 +310,207 @@ fn filter_returns_matching_raw_entries() { filtered.entries(), vec![JournalEntry::new( ["prompt", "root", "keep"], - developer_context("context", "keep", "keep me", 0), + developer_context("keep me", 0), )] ); } + +#[test] +fn context_entry_skips_blank_text_messages() { + assert_eq!( + Journal::context_entry( + ["prompt", "developer", "blank"], + 10, + PromptMessage::developer_text(" "), + ), + None + ); +} + +#[test] +fn context_entry_builder_carries_optional_fields() { + let entry = Journal::context_entry_builder( + ["prompt", "developer", "one"], + PromptMessage::developer_text("first"), + ) + .prompt_order(10) + .audience(JournalContextAudience::SubAgentsOnly) + .on_fork(JournalContextForkBehavior::Regenerate) + .tags(vec!["foo".to_string(), "bar".to_string()]) + .source("unit-test") + .build() + .expect("entry should be kept"); + + assert_eq!( + entry, + JournalEntry::new( + ["prompt", "developer", "one"], + JournalMetadataItem::new(PromptMessage::developer_text("first")) + .with_prompt_order(10) + .with_audience(JournalContextAudience::SubAgentsOnly) + .with_on_fork(JournalContextForkBehavior::Regenerate) + .with_tags(vec!["foo".to_string(), "bar".to_string()]) + .with_source("unit-test"), + ) + ); +} + +#[test] +fn resolve_splits_contexts_from_history() { + let history = JournalTranscriptItem::new(user_message("hello")); + let journal = Journal::from_entries(vec![ + Journal::context_entry( + ["prompt", "developer", "one"], + 10, + PromptMessage::developer_text("first"), + ) + .expect("entry should be kept"), + Journal::context_entry( + ["prompt", "guardian", "one"], + 20, + PromptMessage::developer_text("guardian one"), + ) + .expect("entry should be kept"), + Journal::context_entry( + ["prompt", "guardian", "two"], + 30, + PromptMessage::developer_text("guardian two"), + ) + .expect("entry should be kept"), + JournalEntry::new(["history", "hello"], history.clone()), + ]); + + let resolved = journal.resolve().expect("journal should resolve"); + + assert_eq!( + resolved.metadata().entries(), + vec![ + Journal::context_entry( + ["prompt", "developer", "one"], + 10, + PromptMessage::developer_text("first"), + ) + .expect("entry should be kept"), + Journal::context_entry( + ["prompt", "guardian", "one"], + 20, + PromptMessage::developer_text("guardian one"), + ) + .expect("entry should be kept"), + Journal::context_entry( + ["prompt", "guardian", "two"], + 30, + PromptMessage::developer_text("guardian two"), + ) + .expect("entry should be kept"), + ] + ); + assert_eq!( + resolved.transcript().entries(), + vec![JournalEntry::new(["history", "hello"], history)] + ); +} + +#[test] +fn prompt_renderer_groups_by_declared_prefix_and_role() { + let resolved = Journal::from_entries(vec![ + Journal::context_entry( + ["prompt", "developer", "one"], + 10, + PromptMessage::developer_text("first"), + ) + .expect("entry should be kept"), + Journal::context_entry( + ["prompt", "developer", "two"], + 20, + PromptMessage::developer_text("second"), + ) + .expect("entry should be kept"), + Journal::context_entry( + ["prompt", "contextual_user", "one"], + 30, + PromptMessage::user_text("third"), + ) + .expect("entry should be kept"), + ]) + .resolve() + .expect("entries should resolve"); + + let rendered = PromptRenderer::new() + .group(KeyFilter::prefix(["prompt", "developer"])) + .group(KeyFilter::prefix(["prompt", "contextual_user"])) + .render_metadata(resolved.metadata()); + + assert_eq!( + rendered, + vec![ + ResponseItem::Message { + id: None, + role: "developer".to_string(), + content: vec![ + ContentItem::InputText { + text: "first".to_string(), + }, + ContentItem::InputText { + text: "second".to_string(), + }, + ], + phase: None, + }, + ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { + text: "third".to_string(), + }], + phase: None, + }, + ] + ); +} + +#[test] +fn prompt_renderer_leaves_ungrouped_entries_separate() { + let resolved = Journal::from_entries(vec![ + Journal::context_entry( + ["prompt", "developer", "one"], + 10, + PromptMessage::developer_text("first"), + ) + .expect("entry should be kept"), + Journal::context_entry( + ["prompt", "guardian", "one"], + 20, + PromptMessage::developer_text("guardian one"), + ) + .expect("entry should be kept"), + Journal::context_entry( + ["prompt", "guardian", "two"], + 30, + PromptMessage::developer_text("guardian two"), + ) + .expect("entry should be kept"), + ]) + .resolve() + .expect("entries should resolve"); + + let rendered = PromptRenderer::new() + .group(KeyFilter::prefix(["prompt", "developer"])) + .render_metadata(resolved.metadata()); + + assert_eq!( + rendered, + vec![ + ResponseItem::Message { + id: None, + role: "developer".to_string(), + content: vec![ContentItem::InputText { + text: "first".to_string(), + }], + phase: None, + }, + ResponseItem::from(PromptMessage::developer_text("guardian one")), + ResponseItem::from(PromptMessage::developer_text("guardian two")), + ] + ); +} diff --git a/codex-rs/protocol/src/journal.rs b/codex-rs/protocol/src/journal.rs index 88388f2601..0be98426b9 100644 --- a/codex-rs/protocol/src/journal.rs +++ b/codex-rs/protocol/src/journal.rs @@ -79,30 +79,6 @@ impl KeyFilter { } } -/// Stable identity for a prompt-context entry. -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema, TS)] -pub struct JournalContextKey { - pub namespace: String, - pub name: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - #[ts(optional)] - pub instance: Option, -} - -impl JournalContextKey { - pub fn new( - namespace: impl Into, - name: impl Into, - instance: Option, - ) -> Self { - Self { - namespace: namespace.into(), - name: name.into(), - instance, - } - } -} - /// Minimal message block that can be projected into a model-visible prompt item. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema, TS)] pub struct PromptMessage { @@ -170,9 +146,9 @@ impl PromptMessageRole { } } -/// Durable history item. Unlike prompt context, history keeps original ordering. +/// Durable transcript item. Unlike prompt metadata, transcript keeps original ordering. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema, TS)] -pub struct JournalHistoryItem { +pub struct JournalTranscriptItem { pub id: String, #[serde(default, skip_serializing_if = "Option::is_none")] #[ts(optional)] @@ -180,7 +156,7 @@ pub struct JournalHistoryItem { pub item: ResponseItem, } -impl JournalHistoryItem { +impl JournalTranscriptItem { pub fn new(item: ResponseItem) -> Self { Self { id: Uuid::new_v4().to_string(), @@ -195,28 +171,27 @@ impl JournalHistoryItem { } } -impl From for JournalHistoryItem { +impl From for JournalTranscriptItem { fn from(value: ResponseItem) -> Self { Self::new(value) } } -impl From for ResponseItem { - fn from(value: JournalHistoryItem) -> Self { +impl From for ResponseItem { + fn from(value: JournalTranscriptItem) -> Self { value.item } } -impl From for ResponseItem { - fn from(value: JournalContextItem) -> Self { +impl From for ResponseItem { + fn from(value: JournalMetadataItem) -> Self { value.message.into() } } -/// Prompt-context entry with stable identity and filtering metadata. +/// Prompt-metadata entry payload and filtering metadata. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema, TS)] -pub struct JournalContextItem { - pub key: JournalContextKey, +pub struct JournalMetadataItem { pub message: PromptMessage, #[serde(default)] pub prompt_order: i64, @@ -231,10 +206,9 @@ pub struct JournalContextItem { pub source: Option, } -impl JournalContextItem { - pub fn new(key: JournalContextKey, message: PromptMessage) -> Self { +impl JournalMetadataItem { + pub fn new(message: PromptMessage) -> Self { Self { - key, message, prompt_order: 0, audience: JournalContextAudience::default(), @@ -306,7 +280,7 @@ pub enum JournalHistoryCursor { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema, TS)] pub struct JournalReplacePrefixCheckpoint { pub through: JournalHistoryCursor, - pub replacement: Vec, + pub replacement: Vec, } /// Keep only the current history prefix through the resolved cursor. @@ -327,26 +301,30 @@ pub enum JournalCheckpointItem { #[serde(tag = "type", content = "payload", rename_all = "snake_case")] #[ts(tag = "type", content = "payload", rename_all = "snake_case")] pub enum JournalItem { - History(JournalHistoryItem), - Context(JournalContextItem), + #[serde(rename = "history")] + #[ts(rename = "history")] + Transcript(JournalTranscriptItem), + #[serde(rename = "context")] + #[ts(rename = "context")] + Metadata(JournalMetadataItem), Checkpoint(JournalCheckpointItem), } impl From for JournalItem { fn from(value: ResponseItem) -> Self { - Self::History(value.into()) + Self::Transcript(value.into()) } } -impl From for JournalItem { - fn from(value: JournalHistoryItem) -> Self { - Self::History(value) +impl From for JournalItem { + fn from(value: JournalTranscriptItem) -> Self { + Self::Transcript(value) } } -impl From for JournalItem { - fn from(value: JournalContextItem) -> Self { - Self::Context(value) +impl From for JournalItem { + fn from(value: JournalMetadataItem) -> Self { + Self::Metadata(value) } } @@ -381,3 +359,6 @@ where Self::new(value.0, value.1) } } + +pub type JournalHistoryItem = JournalTranscriptItem; +pub type JournalContextItem = JournalMetadataItem; diff --git a/codex-rs/thread-store/src/journal_writer.rs b/codex-rs/thread-store/src/journal_writer.rs index 8eb0546d62..8f9e5f571b 100644 --- a/codex-rs/thread-store/src/journal_writer.rs +++ b/codex-rs/thread-store/src/journal_writer.rs @@ -229,9 +229,8 @@ fn join_error(err: tokio::task::JoinError) -> ThreadStoreError { #[cfg(test)] mod tests { use codex_journal::Journal; - use codex_journal::JournalContextItem; - use codex_journal::JournalContextKey; - use codex_journal::JournalHistoryItem; + use codex_journal::JournalMetadataItem; + use codex_journal::JournalTranscriptItem; use codex_journal::PromptMessage; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; @@ -243,17 +242,14 @@ mod tests { fn developer_entry(text: &str) -> codex_journal::JournalEntry { codex_journal::JournalEntry::new( ["prompt", text], - JournalContextItem::new( - JournalContextKey::new("context", text, None), - PromptMessage::developer_text(text), - ), + JournalMetadataItem::new(PromptMessage::developer_text(text)), ) } fn user_entry(text: &str) -> codex_journal::JournalEntry { codex_journal::JournalEntry::new( ["history", text], - JournalHistoryItem { + JournalTranscriptItem { id: format!("history-{text}"), turn_id: None, item: ResponseItem::Message {