Files
codex/codex-rs/ext/memories/src/extension.rs
T
pakrym-oai ce0323f2f2 Support typed content in contextual prompt fragments (#54776)
## Why

Contextual fragments and extension-owned world-state sections currently return strings, limiting them to text content.

## What changed

- Return `ContentItem` from fragment bodies and rendering, preserving classification and attribution. Apply text markers only to `InputText`; pass other content through unchanged.
- Allow extension prompt and world-state contributions to carry typed content, and preserve media in context-window hints.
- Hash non-text world-state fragments by their serialized content and role while retaining the existing text hashing behavior.
- Add shared content-to-tool-output conversion that preserves image references, image detail, and audio.

## Testing

Add coverage for typed world-state hashing and preservation of image content and attribution. Expand code-mode output tests to cover input text, output text, file-backed and inline images, and audio.

GitOrigin-RevId: 2ed00d466a3f169b151202f3eef71e667ea64be1
2026-10-17 04:02:21 +00:00

175 lines
5.9 KiB
Rust

use std::sync::Arc;
use codex_core::config::Config;
use codex_core::context::ContextualUserFragment;
use codex_core::context::MemoryContextFragment;
use codex_extension_api::ConfigContributor;
use codex_extension_api::ContentItemKind;
use codex_extension_api::ContextContributor;
use codex_extension_api::ExtensionData;
use codex_extension_api::ExtensionFuture;
use codex_extension_api::ExtensionRegistryBuilder;
use codex_extension_api::PromptFragment;
use codex_extension_api::ThreadLifecycleContributor;
use codex_extension_api::ThreadStartInput;
use codex_extension_api::ToolContributor;
use codex_features::Feature;
use codex_otel::MetricsClient;
use codex_protocol::MemoryVersion;
use codex_utils_absolute_path::AbsolutePathBuf;
use crate::local::LocalMemoriesBackend;
use crate::prompts::build_memory_tool_developer_instructions;
use crate::tools;
/// Contributes Codex memory read-path prompt context and memory read tools.
#[derive(Clone, Default)]
pub(crate) struct MemoriesExtension {
metrics_client: Option<MetricsClient>,
}
impl MemoriesExtension {
fn new(metrics_client: Option<MetricsClient>) -> Self {
Self { metrics_client }
}
}
#[derive(Clone, Debug)]
pub(crate) struct MemoriesExtensionConfig {
pub(crate) enabled: bool,
pub(crate) dedicated_tools: bool,
pub(crate) codex_home: AbsolutePathBuf,
pub(crate) version: MemoryVersion,
}
impl MemoriesExtensionConfig {
fn from_config(config: &Config) -> Self {
Self {
enabled: config.features.enabled(Feature::MemoryTool) && config.memories.use_memories,
dedicated_tools: config.memories.dedicated_tools,
codex_home: config.codex_home.clone(),
version: config.memories.version,
}
}
}
impl ContextContributor for MemoriesExtension {
fn contribute_thread_context<'a>(
&'a self,
_session_store: &'a ExtensionData,
thread_store: &'a ExtensionData,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = Vec<PromptFragment>> + Send + 'a>> {
Box::pin(async move {
let Some(config) = thread_store.get::<MemoriesExtensionConfig>() else {
return Vec::new();
};
if !config.enabled {
return Vec::new();
}
let Some(instructions) =
build_memory_tool_developer_instructions(&config.codex_home, config.version).await
else {
return Vec::new();
};
let instructions = match config.version {
MemoryVersion::V1 => vec![instructions],
MemoryVersion::V2 => {
// Keep the complete summary while respecting each fragment's byte cap.
let mut remaining = instructions.as_str();
let mut fragments = Vec::new();
while !remaining.is_empty() {
let end = remaining.floor_char_boundary(remaining.len().min(8_900));
fragments.push(
MemoryContextFragment::ReadInstructions(remaining[..end].to_string())
.render()
.into_text()
.unwrap_or_default(),
);
remaining = &remaining[end..];
}
fragments
}
};
instructions
.into_iter()
.map(|instructions| {
PromptFragment::developer_policy(
instructions,
ContentItemKind(MemoryContextFragment::READ_INSTRUCTIONS_KIND.to_string()),
)
})
.collect()
})
}
}
impl ThreadLifecycleContributor<Config> for MemoriesExtension {
fn on_thread_start<'a>(
&'a self,
input: ThreadStartInput<'a, Config>,
) -> ExtensionFuture<'a, ()> {
Box::pin(async move {
input
.thread_store
.insert(MemoriesExtensionConfig::from_config(input.config));
})
}
}
impl ConfigContributor<Config> for MemoriesExtension {
fn on_config_changed(
&self,
_session_store: &ExtensionData,
thread_store: &ExtensionData,
_previous_config: &Config,
new_config: &Config,
) {
let mut config = MemoriesExtensionConfig::from_config(new_config);
if let Some(previous) = thread_store.get::<MemoriesExtensionConfig>() {
// The initial summary and retrieval tools must use the same namespace.
config.version = previous.version;
}
thread_store.insert(config);
}
}
impl ToolContributor for MemoriesExtension {
fn tools(
&self,
_session_store: &ExtensionData,
thread_store: &ExtensionData,
) -> Vec<
Arc<dyn for<'call> codex_extension_api::ToolExecutor<codex_extension_api::ToolCall<'call>>>,
> {
let Some(config) = thread_store.get::<MemoriesExtensionConfig>() else {
return Vec::new();
};
if !config.enabled || !config.dedicated_tools {
return Vec::new();
}
tools::memory_tools(
LocalMemoriesBackend::from_memory_root(
config
.codex_home
.join(config.version.directory_name())
.to_path_buf(),
),
self.metrics_client.clone(),
)
}
}
/// Installs the memories extension contributors into the extension registry.
pub fn install(
registry: &mut ExtensionRegistryBuilder<Config>,
metrics_client: Option<MetricsClient>,
) {
let extension = Arc::new(MemoriesExtension::new(metrics_client));
registry.thread_lifecycle_contributor(extension.clone());
registry.config_contributor(extension.clone());
registry.prompt_contributor(extension.clone());
registry.tool_contributor(extension);
}