Merge 177d7b09d9 into sapling-pr-archive-bolinfest

This commit is contained in:
Michael Bolin
2026-03-06 11:28:29 -08:00
committed by GitHub
30 changed files with 4941 additions and 4196 deletions
@@ -6281,7 +6281,7 @@
"type": "object"
},
{
"description": "Explicit mention selected by the user (name + app://connector id).",
"description": "Explicit structured mention selected by the user.\n\n`path` identifies the exact mention target, for example `app://<connector-id>` or `plugin://<plugin-name>@<marketplace-name>`.",
"properties": {
"name": {
"type": "string"
+21 -1
View File
@@ -476,6 +476,26 @@ Invoke an app by including `$<app-slug>` in the text input and adding a `mention
} } }
```
### Example: Start a turn (invoke a plugin)
Invoke a plugin by including a UI mention token such as `@sample` in the text input and adding a `mention` input item with the exact `plugin://<plugin-name>@<marketplace-name>` path returned by `plugin/list`.
```json
{ "method": "turn/start", "id": 35, "params": {
"threadId": "thr_123",
"input": [
{ "type": "text", "text": "@sample Summarize the latest updates." },
{ "type": "mention", "name": "Sample Plugin", "path": "plugin://sample@test" }
]
} }
{ "id": 35, "result": { "turn": {
"id": "turn_459",
"status": "inProgress",
"items": [],
"error": null
} } }
```
### Example: Interrupt an active turn
You can cancel a running Turn with `turn/interrupt`.
@@ -976,7 +996,7 @@ The server also emits `app/list/updated` notifications whenever either source (a
}
```
Invoke an app by inserting `$<app-slug>` in the text input. The slug is derived from the app name and lowercased with non-alphanumeric characters replaced by `-` (for example, "Demo App" becomes `$demo-app`). Add a `mention` input item (recommended) so the server uses the exact `app://<connector-id>` path rather than guessing by name.
Invoke an app by inserting `$<app-slug>` in the text input. The slug is derived from the app name and lowercased with non-alphanumeric characters replaced by `-` (for example, "Demo App" becomes `$demo-app`). Add a `mention` input item (recommended) so the server uses the exact `app://<connector-id>` path rather than guessing by name. Plugins use the same `mention` item shape, but with `plugin://<plugin-name>@<marketplace-name>` paths from `plugin/list`.
Example:
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+80 -138
View File
@@ -10,6 +10,7 @@ use crate::skills::SkillMetadata;
use crate::skills::injection::ToolMentionKind;
use crate::skills::injection::app_id_from_path;
use crate::skills::injection::extract_tool_mentions;
use crate::skills::injection::plugin_config_name_from_path;
use crate::skills::injection::tool_kind_for_path;
pub(crate) struct CollectedToolMentions {
@@ -49,20 +50,7 @@ pub(crate) fn collect_explicit_app_ids(input: &[UserInput]) -> HashSet<String> {
.collect()
}
/// Collect explicit plain-text `@plugin` mentions from user text.
///
/// This is currently the core-side fallback path for plugin mentions. It
/// matches unambiguous plugin `display_name`s from the filtered capability
/// index, case-insensitively, by scanning for exact `@display name` matches.
///
/// It is hand-rolled because core only has a `$...` / `[$...](...)` mention
/// parser today, and the existing TUI `@...` logic is file-autocomplete, not
/// turn-time parsing.
///
/// Long term, explicit plugin picks should come through structured
/// `plugin://...` mentions, likely via `UserInput::Mention`, once clients can list
/// plugins and the UI has plugin-mention support (likely a plugins/list app-server
/// endpoint). Even then, this may stay as a text fallback, similar to skills/apps.
/// Collect explicit structured `plugin://...` mentions.
pub(crate) fn collect_explicit_plugin_mentions(
input: &[UserInput],
plugins: &[PluginCapabilitySummary],
@@ -71,79 +59,34 @@ pub(crate) fn collect_explicit_plugin_mentions(
return Vec::new();
}
let mut display_name_counts = HashMap::new();
for plugin in plugins {
*display_name_counts
.entry(plugin.display_name.to_lowercase())
.or_insert(0) += 1;
}
let messages = input
.iter()
.filter_map(|item| match item {
UserInput::Text { text, .. } => Some(text.clone()),
_ => None,
})
.collect::<Vec<String>>();
let mut display_names = display_name_counts.keys().cloned().collect::<Vec<_>>();
display_names.sort_by_key(|display_name| std::cmp::Reverse(display_name.len()));
let mentioned_config_names: HashSet<String> = input
.iter()
.filter_map(|item| match item {
UserInput::Mention { path, .. } => Some(path.clone()),
_ => None,
})
.chain(collect_tool_mentions_from_messages(&messages).paths)
.filter(|path| tool_kind_for_path(path.as_str()) == ToolMentionKind::Plugin)
.filter_map(|path| plugin_config_name_from_path(path.as_str()).map(str::to_string))
.collect();
let mut mentioned_display_names = HashSet::new();
for text in input.iter().filter_map(|item| match item {
UserInput::Text { text, .. } => Some(text.as_str()),
_ => None,
}) {
let text = text.to_lowercase();
let mut index = 0;
while let Some(relative_at_sign) = text[index..].find('@') {
let at_sign = index + relative_at_sign;
if text[..at_sign]
.chars()
.next_back()
.is_some_and(is_plugin_mention_body_char)
{
index = at_sign + 1;
continue;
}
let Some((matched_display_name, matched_len)) =
display_names.iter().find_map(|display_name| {
text[at_sign + 1..].starts_with(display_name).then(|| {
let end = at_sign + 1 + display_name.len();
text[end..]
.chars()
.next()
.is_none_or(|ch| !is_plugin_mention_body_char(ch))
.then_some((display_name, display_name.len()))
})?
})
else {
index = at_sign + 1;
continue;
};
if display_name_counts
.get(matched_display_name)
.copied()
.unwrap_or(0)
== 1
{
mentioned_display_names.insert(matched_display_name.clone());
}
index = at_sign + 1 + matched_len;
}
}
if mentioned_display_names.is_empty() {
if mentioned_config_names.is_empty() {
return Vec::new();
}
let mut selected = Vec::new();
let mut seen_display_names = HashSet::new();
for plugin in plugins {
let display_name = plugin.display_name.to_lowercase();
if !mentioned_display_names.contains(&display_name) {
continue;
}
if seen_display_names.insert(display_name) {
selected.push(plugin.clone());
}
}
selected
plugins
.iter()
.filter(|plugin| mentioned_config_names.contains(plugin.config_name.as_str()))
.cloned()
.collect()
}
pub(crate) fn build_skill_name_counts(
@@ -175,10 +118,6 @@ pub(crate) fn build_connector_slug_counts(
counts
}
fn is_plugin_mention_body_char(ch: char) -> bool {
ch.is_alphanumeric() || matches!(ch, '_' | '-' | ':')
}
#[cfg(test)]
mod tests {
use std::collections::HashSet;
@@ -197,10 +136,11 @@ mod tests {
}
}
fn plugin(display_name: &str) -> PluginCapabilitySummary {
fn plugin(config_name: &str, display_name: &str) -> PluginCapabilitySummary {
PluginCapabilitySummary {
config_name: format!("{display_name}@test"),
config_name: config_name.to_string(),
display_name: display_name.to_string(),
description: None,
has_skills: true,
mcp_server_names: Vec::new(),
app_connector_ids: Vec::new(),
@@ -257,65 +197,67 @@ mod tests {
}
#[test]
fn collect_explicit_plugin_mentions_resolves_unique_display_names() {
let plugins = vec![plugin("sample"), plugin("other")];
let mentioned = collect_explicit_plugin_mentions(&[text_input("use @sample")], &plugins);
assert_eq!(mentioned, vec![plugin("sample")]);
}
#[test]
fn collect_explicit_plugin_mentions_resolves_non_slug_display_names() {
let spaced_plugins = vec![plugin("Google Calendar")];
let spaced_mentioned = collect_explicit_plugin_mentions(
&[text_input("use @Google Calendar")],
&spaced_plugins,
);
assert_eq!(spaced_mentioned, vec![plugin("Google Calendar")]);
let unicode_plugins = vec![plugin("Café")];
let unicode_mentioned =
collect_explicit_plugin_mentions(&[text_input("use @Café")], &unicode_plugins);
assert_eq!(unicode_mentioned, vec![plugin("Café")]);
}
#[test]
fn collect_explicit_plugin_mentions_prefers_longer_display_names() {
let plugins = vec![plugin("Google"), plugin("Google Calendar")];
let mentioned =
collect_explicit_plugin_mentions(&[text_input("use @Google Calendar")], &plugins);
assert_eq!(mentioned, vec![plugin("Google Calendar")]);
}
#[test]
fn collect_explicit_plugin_mentions_does_not_fall_back_from_ambiguous_longer_name() {
fn collect_explicit_plugin_mentions_from_structured_paths() {
let plugins = vec![
plugin("Google"),
PluginCapabilitySummary {
config_name: "calendar-1@test".to_string(),
..plugin("Google Calendar")
},
PluginCapabilitySummary {
config_name: "calendar-2@test".to_string(),
..plugin("Google Calendar")
},
plugin("sample@test", "sample"),
plugin("other@test", "other"),
];
let mentioned =
collect_explicit_plugin_mentions(&[text_input("use @Google Calendar")], &plugins);
let mentioned = collect_explicit_plugin_mentions(
&[UserInput::Mention {
name: "sample".to_string(),
path: "plugin://sample@test".to_string(),
}],
&plugins,
);
assert_eq!(mentioned, Vec::<PluginCapabilitySummary>::new());
assert_eq!(mentioned, vec![plugin("sample@test", "sample")]);
}
#[test]
fn collect_explicit_plugin_mentions_ignores_embedded_at_signs() {
let plugins = vec![plugin("sample")];
fn collect_explicit_plugin_mentions_from_linked_text_mentions() {
let plugins = vec![
plugin("sample@test", "sample"),
plugin("other@test", "other"),
];
let mentioned = collect_explicit_plugin_mentions(
&[text_input("contact sample@openai.com, do not use plugins")],
&[text_input("use [$sample](plugin://sample@test)")],
&plugins,
);
assert_eq!(mentioned, vec![plugin("sample@test", "sample")]);
}
#[test]
fn collect_explicit_plugin_mentions_dedupes_structured_and_linked_mentions() {
let plugins = vec![
plugin("sample@test", "sample"),
plugin("other@test", "other"),
];
let mentioned = collect_explicit_plugin_mentions(
&[
text_input("use [$sample](plugin://sample@test)"),
UserInput::Mention {
name: "sample".to_string(),
path: "plugin://sample@test".to_string(),
},
],
&plugins,
);
assert_eq!(mentioned, vec![plugin("sample@test", "sample")]);
}
#[test]
fn collect_explicit_plugin_mentions_ignores_non_plugin_paths() {
let plugins = vec![plugin("sample@test", "sample")];
let mentioned = collect_explicit_plugin_mentions(
&[text_input(
"use [$app](app://calendar) and [$skill](skill://team/skill) and [$file](/tmp/file.txt)",
)],
&plugins,
);
+1 -1
View File
@@ -19,7 +19,7 @@ pub(crate) fn build_plugin_injections(
return Vec::new();
}
// Turn each explicit @plugin mention into a developer hint that points the
// Turn each explicit plugin mention into a developer hint that points the
// model at the plugin's visible MCP servers, enabled apps, and skill prefix.
mentioned_plugins
.iter()
+28 -1
View File
@@ -66,6 +66,7 @@ pub struct ConfiguredMarketplacePluginSummary {
pub struct LoadedPlugin {
pub config_name: String,
pub manifest_name: Option<String>,
pub manifest_description: Option<String>,
pub root: AbsolutePathBuf,
pub enabled: bool,
pub skill_roots: Vec<PathBuf>,
@@ -84,6 +85,7 @@ impl LoadedPlugin {
pub struct PluginCapabilitySummary {
pub config_name: String,
pub display_name: String,
pub description: Option<String>,
pub has_skills: bool,
pub mcp_server_names: Vec<String>,
pub app_connector_ids: Vec<AppConnectorId>,
@@ -104,6 +106,7 @@ impl PluginCapabilitySummary {
.manifest_name
.clone()
.unwrap_or_else(|| plugin.config_name.clone()),
description: plugin.manifest_description.clone(),
has_skills: !plugin.skill_roots.is_empty(),
mcp_server_names,
app_connector_ids: plugin.apps.clone(),
@@ -476,6 +479,7 @@ fn load_plugin(config_name: String, plugin: &PluginConfig, store: &PluginStore)
let mut loaded_plugin = LoadedPlugin {
config_name,
manifest_name: None,
manifest_description: None,
root,
enabled: plugin.enabled,
skill_roots: Vec::new(),
@@ -507,6 +511,7 @@ fn load_plugin(config_name: String, plugin: &PluginConfig, store: &PluginStore)
};
loaded_plugin.manifest_name = Some(plugin_manifest_name(&manifest, plugin_root.as_path()));
loaded_plugin.manifest_description = manifest.description;
loaded_plugin.skill_roots = default_skill_roots(plugin_root.as_path());
let mut mcp_servers = HashMap::new();
for mcp_config_path in default_mcp_config_paths(plugin_root.as_path()) {
@@ -752,7 +757,10 @@ mod tests {
write_file(
&plugin_root.join(".codex-plugin/plugin.json"),
r#"{"name":"sample"}"#,
r#"{
"name": "sample",
"description": "Plugin that includes the sample MCP server and Skills"
}"#,
);
write_file(
&plugin_root.join("skills/sample-search/SKILL.md"),
@@ -792,6 +800,9 @@ mod tests {
vec![LoadedPlugin {
config_name: "sample@test".to_string(),
manifest_name: Some("sample".to_string()),
manifest_description: Some(
"Plugin that includes the sample MCP server and Skills".to_string(),
),
root: AbsolutePathBuf::try_from(plugin_root.clone()).unwrap(),
enabled: true,
skill_roots: vec![plugin_root.join("skills")],
@@ -819,6 +830,19 @@ mod tests {
error: None,
}]
);
assert_eq!(
outcome.capability_summaries(),
&[PluginCapabilitySummary {
config_name: "sample@test".to_string(),
display_name: "sample".to_string(),
description: Some(
"Plugin that includes the sample MCP server and Skills".to_string(),
),
has_skills: true,
mcp_server_names: vec!["sample".to_string()],
app_connector_ids: vec![AppConnectorId("connector_example".to_string())],
}]
);
assert_eq!(
outcome.effective_skill_roots(),
vec![plugin_root.join("skills")]
@@ -862,6 +886,7 @@ mod tests {
vec![LoadedPlugin {
config_name: "sample@test".to_string(),
manifest_name: None,
manifest_description: None,
root: AbsolutePathBuf::try_from(plugin_root).unwrap(),
enabled: false,
skill_roots: Vec::new(),
@@ -972,6 +997,7 @@ mod tests {
let plugin = |config_name: &str, dir_name: &str, manifest_name: &str| LoadedPlugin {
config_name: config_name.to_string(),
manifest_name: Some(manifest_name.to_string()),
manifest_description: None,
root: AbsolutePathBuf::try_from(codex_home.path().join(dir_name)).unwrap(),
enabled: true,
skill_roots: Vec::new(),
@@ -982,6 +1008,7 @@ mod tests {
let summary = |config_name: &str, display_name: &str| PluginCapabilitySummary {
config_name: config_name.to_string(),
display_name: display_name.to_string(),
description: None,
..PluginCapabilitySummary::default()
};
let outcome = PluginLoadOutcome::from_plugins(vec![
+2 -1
View File
@@ -6,7 +6,8 @@ pub(crate) const PLUGIN_MANIFEST_PATH: &str = ".codex-plugin/plugin.json";
#[derive(Debug, Default, Deserialize)]
pub(crate) struct PluginManifest {
name: String,
pub(crate) name: String,
pub(crate) description: Option<String>,
}
pub(crate) fn load_plugin_manifest(plugin_root: &Path) -> Option<PluginManifest> {
+210 -23
View File
@@ -7,6 +7,7 @@ use std::path::Path;
use std::path::PathBuf;
use chrono::SecondsFormat;
use chrono::Utc;
use codex_protocol::ThreadId;
use codex_protocol::dynamic_tools::DynamicToolSpec;
use codex_protocol::models::BaseInstructions;
@@ -448,7 +449,6 @@ impl RolloutRecorder {
// future will yield, which is fine – we only need to ensure we do not
// perform *blocking* I/O on the caller's thread.
let (tx, rx) = mpsc::channel::<RolloutCmd>(256);
// Spawn a Tokio task that owns the file handle and performs async
// writes. Using `tokio::fs::File` keeps everything on the async I/O
// driver instead of blocking the runtime.
@@ -614,14 +614,15 @@ impl RolloutRecorder {
match self.tx.send(RolloutCmd::Shutdown { ack: tx_done }).await {
Ok(_) => rx_done
.await
.map_err(|e| IoError::other(format!("failed waiting for rollout shutdown: {e}"))),
.map_err(|e| IoError::other(format!("failed waiting for rollout shutdown: {e}")))?,
Err(e) => {
warn!("failed to send rollout shutdown command: {e}");
Err(IoError::other(format!(
return Err(IoError::other(format!(
"failed to send rollout shutdown command: {e}"
)))
)));
}
}
};
Ok(())
}
}
@@ -744,25 +745,21 @@ async fn rollout_writer(
while let Some(cmd) = rx.recv().await {
match cmd {
RolloutCmd::AddItems(items) => {
let mut persisted_items = Vec::new();
for item in items {
persisted_items.push(item);
}
if persisted_items.is_empty() {
if items.is_empty() {
continue;
}
if writer.is_none() {
buffered_items.extend(persisted_items);
buffered_items.extend(items);
continue;
}
write_and_reconcile_items(
writer.as_mut(),
persisted_items.as_slice(),
items.as_slice(),
&rollout_path,
state_db_ctx.as_deref(),
&mut state_builder,
state_builder.as_ref(),
default_provider.as_str(),
)
.await?;
@@ -800,7 +797,7 @@ async fn rollout_writer(
buffered_items.as_slice(),
&rollout_path,
state_db_ctx.as_deref(),
&mut state_builder,
state_builder.as_ref(),
default_provider.as_str(),
)
.await?;
@@ -861,13 +858,12 @@ async fn write_session_meta(
if let Some(writer) = writer.as_mut() {
writer.write_rollout_item(&rollout_item).await?;
}
state_db::reconcile_rollout(
sync_thread_state_after_write(
state_db_ctx,
rollout_path,
default_provider,
state_builder.as_ref(),
std::slice::from_ref(&rollout_item),
None,
default_provider,
(!generate_memories).then_some("disabled"),
)
.await;
@@ -879,7 +875,7 @@ async fn write_and_reconcile_items(
items: &[RolloutItem],
rollout_path: &Path,
state_db_ctx: Option<&StateRuntime>,
state_builder: &mut Option<ThreadMetadataBuilder>,
state_builder: Option<&ThreadMetadataBuilder>,
default_provider: &str,
) -> std::io::Result<()> {
if let Some(writer) = writer.as_mut() {
@@ -887,20 +883,65 @@ async fn write_and_reconcile_items(
writer.write_rollout_item(item).await?;
}
}
if let Some(builder) = state_builder.as_mut() {
builder.rollout_path = rollout_path.to_path_buf();
sync_thread_state_after_write(
state_db_ctx,
rollout_path,
state_builder,
items,
default_provider,
None,
)
.await;
Ok(())
}
async fn sync_thread_state_after_write(
state_db_ctx: Option<&StateRuntime>,
rollout_path: &Path,
state_builder: Option<&ThreadMetadataBuilder>,
items: &[RolloutItem],
default_provider: &str,
new_thread_memory_mode: Option<&str>,
) {
let updated_at = Utc::now();
if new_thread_memory_mode.is_some()
|| items
.iter()
.any(codex_state::rollout_item_affects_thread_metadata)
{
state_db::apply_rollout_items(
state_db_ctx,
rollout_path,
default_provider,
state_builder,
items,
"rollout_writer",
new_thread_memory_mode,
Some(updated_at),
)
.await;
return;
}
let thread_id = state_builder
.map(|builder| builder.id)
.or_else(|| metadata::builder_from_items(items, rollout_path).map(|builder| builder.id));
if state_db::touch_thread_updated_at(state_db_ctx, thread_id, updated_at, "rollout_writer")
.await
{
return;
}
state_db::apply_rollout_items(
state_db_ctx,
rollout_path,
default_provider,
state_builder.as_ref(),
state_builder,
items,
"rollout_writer",
None,
new_thread_memory_mode,
Some(updated_at),
)
.await;
Ok(())
}
struct JsonlWriter {
@@ -1079,6 +1120,7 @@ mod tests {
use std::io::Write;
use std::path::Path;
use std::path::PathBuf;
use std::time::Duration;
use tempfile::TempDir;
use uuid::Uuid;
@@ -1200,6 +1242,151 @@ mod tests {
Ok(())
}
#[tokio::test]
async fn metadata_irrelevant_events_touch_state_db_updated_at() -> std::io::Result<()> {
let home = TempDir::new().expect("temp dir");
let mut config = ConfigBuilder::default()
.codex_home(home.path().to_path_buf())
.build()
.await?;
config
.features
.enable(Feature::Sqlite)
.expect("test config should allow sqlite");
let state_db = codex_state::StateRuntime::init(
home.path().to_path_buf(),
config.model_provider_id.clone(),
None,
)
.await
.expect("state db should initialize");
state_db
.mark_backfill_complete(None)
.await
.expect("backfill should be complete");
let thread_id = ThreadId::new();
let recorder = RolloutRecorder::new(
&config,
RolloutRecorderParams::new(
thread_id,
None,
SessionSource::Cli,
BaseInstructions::default(),
Vec::new(),
EventPersistenceMode::Limited,
),
Some(state_db.clone()),
None,
)
.await?;
recorder
.record_items(&[RolloutItem::EventMsg(EventMsg::UserMessage(
UserMessageEvent {
message: "first-user-message".to_string(),
images: None,
local_images: Vec::new(),
text_elements: Vec::new(),
},
))])
.await?;
recorder.persist().await?;
recorder.flush().await?;
let initial_thread = state_db
.get_thread(thread_id)
.await
.expect("thread should load")
.expect("thread should exist");
let initial_updated_at = initial_thread.updated_at;
let initial_title = initial_thread.title.clone();
let initial_first_user_message = initial_thread.first_user_message.clone();
tokio::time::sleep(Duration::from_secs(1)).await;
recorder
.record_items(&[RolloutItem::EventMsg(EventMsg::AgentMessage(
AgentMessageEvent {
message: "assistant text".to_string(),
phase: None,
},
))])
.await?;
recorder.flush().await?;
let updated_thread = state_db
.get_thread(thread_id)
.await
.expect("thread should load after agent message")
.expect("thread should still exist");
assert!(updated_thread.updated_at > initial_updated_at);
assert_eq!(updated_thread.title, initial_title);
assert_eq!(
updated_thread.first_user_message,
initial_first_user_message
);
recorder.shutdown().await?;
Ok(())
}
#[tokio::test]
async fn metadata_irrelevant_events_fall_back_to_upsert_when_thread_missing()
-> std::io::Result<()> {
let home = TempDir::new().expect("temp dir");
let mut config = ConfigBuilder::default()
.codex_home(home.path().to_path_buf())
.build()
.await?;
config
.features
.enable(Feature::Sqlite)
.expect("test config should allow sqlite");
let state_db = codex_state::StateRuntime::init(
home.path().to_path_buf(),
config.model_provider_id.clone(),
None,
)
.await
.expect("state db should initialize");
let thread_id = ThreadId::new();
let rollout_path = home.path().join("rollout.jsonl");
let builder = ThreadMetadataBuilder::new(
thread_id,
rollout_path.clone(),
Utc::now(),
SessionSource::Cli,
);
let items = vec![RolloutItem::EventMsg(EventMsg::AgentMessage(
AgentMessageEvent {
message: "assistant text".to_string(),
phase: None,
},
))];
sync_thread_state_after_write(
Some(state_db.as_ref()),
rollout_path.as_path(),
Some(&builder),
items.as_slice(),
config.model_provider_id.as_str(),
None,
)
.await;
let thread = state_db
.get_thread(thread_id)
.await
.expect("thread should load after fallback")
.expect("thread should be inserted after fallback");
assert_eq!(thread.id, thread_id);
Ok(())
}
#[tokio::test]
async fn list_threads_db_disabled_does_not_skip_paginated_items() -> std::io::Result<()> {
let home = TempDir::new().expect("temp dir");
+24 -8
View File
@@ -178,12 +178,14 @@ impl<'a> ToolMentions<'a> {
pub(crate) enum ToolMentionKind {
App,
Mcp,
Plugin,
Skill,
Other,
}
const APP_PATH_PREFIX: &str = "app://";
const MCP_PATH_PREFIX: &str = "mcp://";
const PLUGIN_PATH_PREFIX: &str = "plugin://";
const SKILL_PATH_PREFIX: &str = "skill://";
const SKILL_FILENAME: &str = "SKILL.md";
@@ -192,6 +194,8 @@ pub(crate) fn tool_kind_for_path(path: &str) -> ToolMentionKind {
ToolMentionKind::App
} else if path.starts_with(MCP_PATH_PREFIX) {
ToolMentionKind::Mcp
} else if path.starts_with(PLUGIN_PATH_PREFIX) {
ToolMentionKind::Plugin
} else if path.starts_with(SKILL_PATH_PREFIX) || is_skill_filename(path) {
ToolMentionKind::Skill
} else {
@@ -209,6 +213,11 @@ pub(crate) fn app_id_from_path(path: &str) -> Option<&str> {
.filter(|value| !value.is_empty())
}
pub(crate) fn plugin_config_name_from_path(path: &str) -> Option<&str> {
path.strip_prefix(PLUGIN_PATH_PREFIX)
.filter(|value| !value.is_empty())
}
pub(crate) fn normalize_skill_path(path: &str) -> &str {
path.strip_prefix(SKILL_PATH_PREFIX).unwrap_or(path)
}
@@ -219,6 +228,10 @@ pub(crate) fn normalize_skill_path(path: &str) -> &str {
/// resource path is present, it is captured for exact path matching while also tracking
/// the name for fallback matching.
pub(crate) fn extract_tool_mentions(text: &str) -> ToolMentions<'_> {
extract_tool_mentions_with_sigil(text, '$')
}
fn extract_tool_mentions_with_sigil(text: &str, sigil: char) -> ToolMentions<'_> {
let text_bytes = text.as_bytes();
let mut mentioned_names: HashSet<&str> = HashSet::new();
let mut mentioned_paths: HashSet<&str> = HashSet::new();
@@ -229,11 +242,13 @@ pub(crate) fn extract_tool_mentions(text: &str) -> ToolMentions<'_> {
let byte = text_bytes[index];
if byte == b'['
&& let Some((name, path, end_index)) =
parse_linked_tool_mention(text, text_bytes, index)
parse_linked_tool_mention(text, text_bytes, index, sigil)
{
if !is_common_env_var(name) {
let kind = tool_kind_for_path(path);
if !matches!(kind, ToolMentionKind::App | ToolMentionKind::Mcp) {
if !matches!(
tool_kind_for_path(path),
ToolMentionKind::App | ToolMentionKind::Mcp | ToolMentionKind::Plugin
) {
mentioned_names.insert(name);
}
mentioned_paths.insert(path);
@@ -242,7 +257,7 @@ pub(crate) fn extract_tool_mentions(text: &str) -> ToolMentions<'_> {
continue;
}
if byte != b'$' {
if byte != sigil as u8 {
index += 1;
continue;
}
@@ -297,7 +312,7 @@ fn select_skills_from_mentions(
.filter(|path| {
!matches!(
tool_kind_for_path(path),
ToolMentionKind::App | ToolMentionKind::Mcp
ToolMentionKind::App | ToolMentionKind::Mcp | ToolMentionKind::Plugin
)
})
.map(normalize_skill_path)
@@ -361,13 +376,14 @@ fn parse_linked_tool_mention<'a>(
text: &'a str,
text_bytes: &[u8],
start: usize,
sigil: char,
) -> Option<(&'a str, &'a str, usize)> {
let dollar_index = start + 1;
if text_bytes.get(dollar_index) != Some(&b'$') {
let sigil_index = start + 1;
if text_bytes.get(sigil_index) != Some(&(sigil as u8)) {
return None;
}
let name_start = dollar_index + 1;
let name_start = sigil_index + 1;
let first_name_byte = text_bytes.get(name_start)?;
if !is_mention_name_char(*first_name_byte) {
return None;
+30 -1
View File
@@ -361,6 +361,7 @@ pub async fn reconcile_rollout(
items,
"reconcile_rollout",
new_thread_memory_mode,
None,
)
.await;
return;
@@ -491,6 +492,7 @@ pub async fn read_repair_rollout_path(
}
/// Apply rollout items incrementally to SQLite.
#[allow(clippy::too_many_arguments)]
pub async fn apply_rollout_items(
context: Option<&codex_state::StateRuntime>,
rollout_path: &Path,
@@ -499,6 +501,7 @@ pub async fn apply_rollout_items(
items: &[RolloutItem],
stage: &str,
new_thread_memory_mode: Option<&str>,
updated_at_override: Option<DateTime<Utc>>,
) {
let Some(ctx) = context else {
return;
@@ -520,7 +523,13 @@ pub async fn apply_rollout_items(
builder.rollout_path = rollout_path.to_path_buf();
builder.cwd = normalize_cwd_for_state_db(&builder.cwd);
if let Err(err) = ctx
.apply_rollout_items(&builder, items, None, new_thread_memory_mode)
.apply_rollout_items(
&builder,
items,
None,
new_thread_memory_mode,
updated_at_override,
)
.await
{
warn!(
@@ -530,6 +539,26 @@ pub async fn apply_rollout_items(
}
}
pub async fn touch_thread_updated_at(
context: Option<&codex_state::StateRuntime>,
thread_id: Option<ThreadId>,
updated_at: DateTime<Utc>,
stage: &str,
) -> bool {
let Some(ctx) = context else {
return false;
};
let Some(thread_id) = thread_id else {
return false;
};
ctx.touch_thread_updated_at(thread_id, updated_at)
.await
.unwrap_or_else(|err| {
warn!("state db touch_thread_updated_at failed during {stage} for {thread_id}: {err}");
false
})
}
#[cfg(test)]
mod tests {
use super::*;
+1 -1
View File
@@ -684,7 +684,7 @@ fn create_collab_input_items_schema() -> JsonSchema {
"path".to_string(),
JsonSchema::String {
description: Some(
"Path when type is local_image/skill, or mention target such as app://<connector-id> when type is mention."
"Path when type is local_image/skill, or structured mention target such as app://<connector-id> or plugin://<plugin-name>@<marketplace-name> when type is mention."
.to_string(),
),
},
+3 -3
View File
@@ -243,9 +243,9 @@ async fn explicit_plugin_mentions_inject_plugin_guidance() -> Result<()> {
codex
.submit(Op::UserInput {
items: vec![codex_protocol::user_input::UserInput::Text {
text: "Use @sample for this task.".into(),
text_elements: Vec::new(),
items: vec![codex_protocol::user_input::UserInput::Mention {
name: "sample".into(),
path: format!("plugin://{SAMPLE_PLUGIN_CONFIG_NAME}"),
}],
final_output_json_schema: None,
})
+4 -1
View File
@@ -32,7 +32,10 @@ pub enum UserInput {
name: String,
path: std::path::PathBuf,
},
/// Explicit mention selected by the user (name + app://connector id).
/// Explicit structured mention selected by the user.
///
/// `path` identifies the exact mention target, for example
/// `app://<connector-id>` or `plugin://<plugin-name>@<marketplace-name>`.
Mention { name: String, path: String },
}
@@ -0,0 +1,21 @@
CREATE TABLE logs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
ts INTEGER NOT NULL,
ts_nanos INTEGER NOT NULL,
level TEXT NOT NULL,
target TEXT NOT NULL,
message TEXT,
module_path TEXT,
file TEXT,
line INTEGER,
thread_id TEXT,
process_uuid TEXT,
estimated_bytes INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX idx_logs_ts ON logs(ts DESC, ts_nanos DESC, id DESC);
CREATE INDEX idx_logs_thread_id ON logs(thread_id);
CREATE INDEX idx_logs_process_uuid ON logs(process_uuid);
CREATE INDEX idx_logs_thread_id_ts ON logs(thread_id, ts DESC, ts_nanos DESC, id DESC);
CREATE INDEX idx_logs_process_uuid_threadless_ts ON logs(process_uuid, ts DESC, ts_nanos DESC, id DESC)
WHERE thread_id IS NULL;
+3 -3
View File
@@ -12,13 +12,13 @@ use owo_colors::OwoColorize;
#[derive(Debug, Parser)]
#[command(name = "codex-state-logs")]
#[command(about = "Tail Codex logs from the state SQLite DB with simple filters")]
#[command(about = "Tail Codex logs from the dedicated logs SQLite DB with simple filters")]
struct Args {
/// Path to CODEX_HOME. Defaults to $CODEX_HOME or ~/.codex.
#[arg(long, env = "CODEX_HOME")]
codex_home: Option<PathBuf>,
/// Direct path to the SQLite database. Overrides --codex-home.
/// Direct path to the logs SQLite database. Overrides --codex-home.
#[arg(long)]
db: Option<PathBuf>,
@@ -113,7 +113,7 @@ fn resolve_db_path(args: &Args) -> anyhow::Result<PathBuf> {
}
let codex_home = args.codex_home.clone().unwrap_or_else(default_codex_home);
Ok(codex_state::state_db_path(codex_home.as_path()))
Ok(codex_state::logs_db_path(codex_home.as_path()))
}
fn default_codex_home() -> PathBuf {
+11
View File
@@ -29,6 +29,17 @@ pub fn apply_rollout_item(
}
}
/// Return whether this rollout item can mutate thread metadata stored in SQLite.
pub fn rollout_item_affects_thread_metadata(item: &RolloutItem) -> bool {
match item {
RolloutItem::SessionMeta(_) | RolloutItem::TurnContext(_) => true,
RolloutItem::EventMsg(EventMsg::TokenCount(_) | EventMsg::UserMessage(_)) => true,
RolloutItem::EventMsg(_) | RolloutItem::ResponseItem(_) | RolloutItem::Compacted(_) => {
false
}
}
}
fn apply_session_meta_from_item(metadata: &mut ThreadMetadata, meta_line: &SessionMetaLine) {
if metadata.id != meta_line.meta.id {
// Ignore session_meta lines that don't match the canonical thread ID,
+5
View File
@@ -23,6 +23,7 @@ pub use runtime::StateRuntime;
///
/// Most consumers should prefer [`StateRuntime`].
pub use extract::apply_rollout_item;
pub use extract::rollout_item_affects_thread_metadata;
pub use model::AgentJob;
pub use model::AgentJobCreateParams;
pub use model::AgentJobItem;
@@ -44,12 +45,16 @@ pub use model::Stage1StartupClaimParams;
pub use model::ThreadMetadata;
pub use model::ThreadMetadataBuilder;
pub use model::ThreadsPage;
pub use runtime::logs_db_filename;
pub use runtime::logs_db_path;
pub use runtime::state_db_filename;
pub use runtime::state_db_path;
/// Environment variable for overriding the SQLite state database home directory.
pub const SQLITE_HOME_ENV: &str = "CODEX_SQLITE_HOME";
pub const LOGS_DB_FILENAME: &str = "logs";
pub const LOGS_DB_VERSION: u32 = 1;
pub const STATE_DB_FILENAME: &str = "state";
pub const STATE_DB_VERSION: u32 = 5;
+2 -2
View File
@@ -1,7 +1,7 @@
//! Tracing log export into the state SQLite database.
//!
//! This module provides a `tracing_subscriber::Layer` that captures events and
//! inserts them into the `logs` table in `state.sqlite`. The writer runs in a
//! inserts them into the dedicated `logs` SQLite database. The writer runs in a
//! background task and batches inserts to keep logging overhead low.
//!
//! ## Usage
@@ -43,7 +43,7 @@ use crate::StateRuntime;
const LOG_QUEUE_CAPACITY: usize = 512;
const LOG_BATCH_SIZE: usize = 128;
const LOG_FLUSH_INTERVAL: Duration = Duration::from_secs(2);
const LOG_RETENTION_DAYS: i64 = 90;
const LOG_RETENTION_DAYS: i64 = 10;
pub struct LogDbLayer {
sender: mpsc::Sender<LogDbCommand>,
+2 -1
View File
@@ -1,3 +1,4 @@
use sqlx::migrate::Migrator;
pub(crate) static MIGRATOR: Migrator = sqlx::migrate!("./migrations");
pub(crate) static STATE_MIGRATOR: Migrator = sqlx::migrate!("./migrations");
pub(crate) static LOGS_MIGRATOR: Migrator = sqlx::migrate!("./logs_migrations");
+71 -21
View File
@@ -6,6 +6,8 @@ use crate::AgentJobItemStatus;
use crate::AgentJobProgress;
use crate::AgentJobStatus;
use crate::DB_ERROR_METRIC;
use crate::LOGS_DB_FILENAME;
use crate::LOGS_DB_VERSION;
use crate::LogEntry;
use crate::LogQuery;
use crate::LogRow;
@@ -17,7 +19,8 @@ use crate::ThreadMetadata;
use crate::ThreadMetadataBuilder;
use crate::ThreadsPage;
use crate::apply_rollout_item;
use crate::migrations::MIGRATOR;
use crate::migrations::LOGS_MIGRATOR;
use crate::migrations::STATE_MIGRATOR;
use crate::model::AgentJobRow;
use crate::model::ThreadRow;
use crate::model::anchor_from_item;
@@ -37,6 +40,7 @@ use sqlx::Row;
use sqlx::Sqlite;
use sqlx::SqliteConnection;
use sqlx::SqlitePool;
use sqlx::migrate::Migrator;
use sqlx::sqlite::SqliteConnectOptions;
use sqlx::sqlite::SqliteJournalMode;
use sqlx::sqlite::SqlitePoolOptions;
@@ -68,22 +72,41 @@ pub struct StateRuntime {
codex_home: PathBuf,
default_provider: String,
pool: Arc<sqlx::SqlitePool>,
logs_pool: Arc<sqlx::SqlitePool>,
}
impl StateRuntime {
/// Initialize the state runtime using the provided Codex home and default provider.
///
/// This opens (and migrates) the SQLite database at `codex_home/state.sqlite`.
/// This opens (and migrates) the SQLite databases under `codex_home`,
/// keeping logs in a dedicated file to reduce lock contention with the
/// rest of the state store.
pub async fn init(
codex_home: PathBuf,
default_provider: String,
otel: Option<OtelManager>,
) -> anyhow::Result<Arc<Self>> {
tokio::fs::create_dir_all(&codex_home).await?;
remove_legacy_state_files(&codex_home).await;
let current_state_name = state_db_filename();
let current_logs_name = logs_db_filename();
remove_legacy_db_files(
&codex_home,
current_state_name.as_str(),
STATE_DB_FILENAME,
"state",
)
.await;
remove_legacy_db_files(
&codex_home,
current_logs_name.as_str(),
LOGS_DB_FILENAME,
"logs",
)
.await;
let state_path = state_db_path(codex_home.as_path());
let logs_path = logs_db_path(codex_home.as_path());
let existed = tokio::fs::try_exists(&state_path).await.unwrap_or(false);
let pool = match open_sqlite(&state_path).await {
let pool = match open_sqlite(&state_path, &STATE_MIGRATOR).await {
Ok(db) => Arc::new(db),
Err(err) => {
warn!("failed to open state db at {}: {err}", state_path.display());
@@ -93,11 +116,22 @@ impl StateRuntime {
return Err(err);
}
};
let logs_pool = match open_sqlite(&logs_path, &LOGS_MIGRATOR).await {
Ok(db) => Arc::new(db),
Err(err) => {
warn!("failed to open logs db at {}: {err}", logs_path.display());
if let Some(otel) = otel.as_ref() {
otel.counter(METRIC_DB_INIT, 1, &[("status", "open_error")]);
}
return Err(err);
}
};
if let Some(otel) = otel.as_ref() {
otel.counter(METRIC_DB_INIT, 1, &[("status", "opened")]);
}
let runtime = Arc::new(Self {
pool,
logs_pool,
codex_home,
default_provider,
});
@@ -113,7 +147,7 @@ impl StateRuntime {
}
}
async fn open_sqlite(path: &Path) -> anyhow::Result<SqlitePool> {
async fn open_sqlite(path: &Path, migrator: &'static Migrator) -> anyhow::Result<SqlitePool> {
let options = SqliteConnectOptions::new()
.filename(path)
.create_if_missing(true)
@@ -125,26 +159,42 @@ async fn open_sqlite(path: &Path) -> anyhow::Result<SqlitePool> {
.max_connections(5)
.connect_with(options)
.await?;
MIGRATOR.run(&pool).await?;
migrator.run(&pool).await?;
Ok(pool)
}
fn db_filename(base_name: &str, version: u32) -> String {
format!("{base_name}_{version}.sqlite")
}
pub fn state_db_filename() -> String {
format!("{STATE_DB_FILENAME}_{STATE_DB_VERSION}.sqlite")
db_filename(STATE_DB_FILENAME, STATE_DB_VERSION)
}
pub fn state_db_path(codex_home: &Path) -> PathBuf {
codex_home.join(state_db_filename())
}
async fn remove_legacy_state_files(codex_home: &Path) {
let current_name = state_db_filename();
pub fn logs_db_filename() -> String {
db_filename(LOGS_DB_FILENAME, LOGS_DB_VERSION)
}
pub fn logs_db_path(codex_home: &Path) -> PathBuf {
codex_home.join(logs_db_filename())
}
async fn remove_legacy_db_files(
codex_home: &Path,
current_name: &str,
base_name: &str,
db_label: &str,
) {
let mut entries = match tokio::fs::read_dir(codex_home).await {
Ok(entries) => entries,
Err(err) => {
warn!(
"failed to read codex_home for state db cleanup {}: {err}",
codex_home.display()
"failed to read codex_home for {db_label} db cleanup {}: {err}",
codex_home.display(),
);
return;
}
@@ -160,37 +210,37 @@ async fn remove_legacy_state_files(codex_home: &Path) {
}
let file_name = entry.file_name();
let file_name = file_name.to_string_lossy();
if !should_remove_state_file(file_name.as_ref(), current_name.as_str()) {
if !should_remove_db_file(file_name.as_ref(), current_name, base_name) {
continue;
}
let legacy_path = entry.path();
if let Err(err) = tokio::fs::remove_file(&legacy_path).await {
warn!(
"failed to remove legacy state db file {}: {err}",
legacy_path.display()
"failed to remove legacy {db_label} db file {}: {err}",
legacy_path.display(),
);
}
}
}
fn should_remove_state_file(file_name: &str, current_name: &str) -> bool {
let mut base_name = file_name;
fn should_remove_db_file(file_name: &str, current_name: &str, base_name: &str) -> bool {
let mut normalized_name = file_name;
for suffix in ["-wal", "-shm", "-journal"] {
if let Some(stripped) = file_name.strip_suffix(suffix) {
base_name = stripped;
normalized_name = stripped;
break;
}
}
if base_name == current_name {
if normalized_name == current_name {
return false;
}
let unversioned_name = format!("{STATE_DB_FILENAME}.sqlite");
if base_name == unversioned_name {
let unversioned_name = format!("{base_name}.sqlite");
if normalized_name == unversioned_name {
return true;
}
let Some(version_with_extension) = base_name.strip_prefix(&format!("{STATE_DB_FILENAME}_"))
let Some(version_with_extension) = normalized_name.strip_prefix(&format!("{base_name}_"))
else {
return false;
};
+63 -5
View File
@@ -11,7 +11,7 @@ impl StateRuntime {
return Ok(());
}
let mut tx = self.pool.begin().await?;
let mut tx = self.logs_pool.begin().await?;
let mut builder = QueryBuilder::<Sqlite>::new(
"INSERT INTO logs (ts, ts_nanos, level, target, message, thread_id, process_uuid, module_path, file, line, estimated_bytes) ",
);
@@ -281,7 +281,7 @@ WHERE id IN (
pub(crate) async fn delete_logs_before(&self, cutoff_ts: i64) -> anyhow::Result<u64> {
let result = sqlx::query("DELETE FROM logs WHERE ts < ?")
.bind(cutoff_ts)
.execute(self.pool.as_ref())
.execute(self.logs_pool.as_ref())
.await?;
Ok(result.rows_affected())
}
@@ -303,7 +303,7 @@ WHERE id IN (
let rows = builder
.build_query_as::<LogRow>()
.fetch_all(self.pool.as_ref())
.fetch_all(self.logs_pool.as_ref())
.await?;
Ok(rows)
}
@@ -377,7 +377,7 @@ ORDER BY ts ASC, ts_nanos ASC, id ASC
.bind(thread_id)
.bind(thread_id)
.bind(max_bytes)
.fetch_all(self.pool.as_ref())
.fetch_all(self.logs_pool.as_ref())
.await?;
Ok(lines.concat().into_bytes())
@@ -388,7 +388,7 @@ ORDER BY ts ASC, ts_nanos ASC, id ASC
let mut builder =
QueryBuilder::<Sqlite>::new("SELECT MAX(id) AS max_id FROM logs WHERE 1 = 1");
push_log_filters(&mut builder, query);
let row = builder.build().fetch_one(self.pool.as_ref()).await?;
let row = builder.build().fetch_one(self.logs_pool.as_ref()).await?;
let max_id: Option<i64> = row.try_get("max_id")?;
Ok(max_id.unwrap_or(0))
}
@@ -465,7 +465,65 @@ mod tests {
use super::test_support::unique_temp_dir;
use crate::LogEntry;
use crate::LogQuery;
use crate::logs_db_path;
use crate::state_db_path;
use pretty_assertions::assert_eq;
use sqlx::SqlitePool;
use sqlx::sqlite::SqliteConnectOptions;
use std::path::Path;
async fn open_db_pool(path: &Path) -> SqlitePool {
SqlitePool::connect_with(
SqliteConnectOptions::new()
.filename(path)
.create_if_missing(false),
)
.await
.expect("open sqlite pool")
}
async fn log_row_count(path: &Path) -> i64 {
let pool = open_db_pool(path).await;
let count = sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM logs")
.fetch_one(&pool)
.await
.expect("count log rows");
pool.close().await;
count
}
#[tokio::test]
async fn insert_logs_use_dedicated_log_database() {
let codex_home = unique_temp_dir();
let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None)
.await
.expect("initialize runtime");
runtime
.insert_logs(&[LogEntry {
ts: 1,
ts_nanos: 0,
level: "INFO".to_string(),
target: "cli".to_string(),
message: Some("dedicated-log-db".to_string()),
thread_id: Some("thread-1".to_string()),
process_uuid: Some("proc-1".to_string()),
module_path: Some("mod".to_string()),
file: Some("main.rs".to_string()),
line: Some(7),
}])
.await
.expect("insert test logs");
let state_count = log_row_count(state_db_path(codex_home.as_path()).as_path()).await;
let logs_count = log_row_count(logs_db_path(codex_home.as_path()).as_path()).await;
assert_eq!(state_count, 0);
assert_eq!(logs_count, 1);
let _ = tokio::fs::remove_dir_all(codex_home).await;
}
#[tokio::test]
async fn query_logs_with_search_matches_substring() {
let codex_home = unique_temp_dir();
+113 -3
View File
@@ -278,6 +278,19 @@ ON CONFLICT(id) DO NOTHING
Ok(result.rows_affected() > 0)
}
pub async fn touch_thread_updated_at(
&self,
thread_id: ThreadId,
updated_at: DateTime<Utc>,
) -> anyhow::Result<bool> {
let result = sqlx::query("UPDATE threads SET updated_at = ? WHERE id = ?")
.bind(datetime_to_epoch_seconds(updated_at))
.bind(thread_id.to_string())
.execute(self.pool.as_ref())
.await?;
Ok(result.rows_affected() > 0)
}
pub async fn update_thread_git_info(
&self,
thread_id: ThreadId,
@@ -436,6 +449,7 @@ ON CONFLICT(thread_id, position) DO NOTHING
items: &[RolloutItem],
otel: Option<&OtelManager>,
new_thread_memory_mode: Option<&str>,
updated_at_override: Option<DateTime<Utc>>,
) -> anyhow::Result<()> {
if items.is_empty() {
return Ok(());
@@ -451,7 +465,11 @@ ON CONFLICT(thread_id, position) DO NOTHING
if let Some(existing_metadata) = existing_metadata.as_ref() {
metadata.prefer_existing_git_info(existing_metadata);
}
if let Some(updated_at) = file_modified_time_utc(builder.rollout_path.as_path()).await {
let updated_at = match updated_at_override {
Some(updated_at) => Some(updated_at),
None => file_modified_time_utc(builder.rollout_path.as_path()).await,
};
if let Some(updated_at) = updated_at {
metadata.updated_at = updated_at;
}
// Keep the thread upsert before dynamic tools to satisfy the foreign key constraint:
@@ -649,6 +667,7 @@ mod tests {
use super::*;
use crate::runtime::test_support::test_thread_metadata;
use crate::runtime::test_support::unique_temp_dir;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::GitInfo;
use codex_protocol::protocol::SessionMeta;
use codex_protocol::protocol::SessionMetaLine;
@@ -735,7 +754,7 @@ mod tests {
})];
runtime
.apply_rollout_items(&builder, &items, None, None)
.apply_rollout_items(&builder, &items, None, None, None)
.await
.expect("apply_rollout_items should succeed");
@@ -793,7 +812,7 @@ mod tests {
})];
runtime
.apply_rollout_items(&builder, &items, None, None)
.apply_rollout_items(&builder, &items, None, None, None)
.await
.expect("apply_rollout_items should succeed");
@@ -947,4 +966,95 @@ mod tests {
assert_eq!(persisted.git_branch, None);
assert_eq!(persisted.git_origin_url, None);
}
#[tokio::test]
async fn touch_thread_updated_at_updates_only_updated_at() {
let codex_home = unique_temp_dir();
let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None)
.await
.expect("state db should initialize");
let thread_id =
ThreadId::from_string("00000000-0000-0000-0000-000000000791").expect("valid thread id");
let mut metadata = test_thread_metadata(&codex_home, thread_id, codex_home.clone());
metadata.title = "original title".to_string();
metadata.first_user_message = Some("first-user-message".to_string());
runtime
.upsert_thread(&metadata)
.await
.expect("initial upsert should succeed");
let touched_at = DateTime::<Utc>::from_timestamp(1_700_001_111, 0).expect("timestamp");
let touched = runtime
.touch_thread_updated_at(thread_id, touched_at)
.await
.expect("touch should succeed");
assert!(touched);
let persisted = runtime
.get_thread(thread_id)
.await
.expect("thread should load")
.expect("thread should exist");
assert_eq!(persisted.updated_at, touched_at);
assert_eq!(persisted.title, "original title");
assert_eq!(
persisted.first_user_message.as_deref(),
Some("first-user-message")
);
}
#[tokio::test]
async fn apply_rollout_items_uses_override_updated_at_when_provided() {
let codex_home = unique_temp_dir();
let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None)
.await
.expect("state db should initialize");
let thread_id =
ThreadId::from_string("00000000-0000-0000-0000-000000000792").expect("valid thread id");
let metadata = test_thread_metadata(&codex_home, thread_id, codex_home.clone());
runtime
.upsert_thread(&metadata)
.await
.expect("initial upsert should succeed");
let builder = ThreadMetadataBuilder::new(
thread_id,
metadata.rollout_path.clone(),
metadata.created_at,
SessionSource::Cli,
);
let items = vec![RolloutItem::EventMsg(EventMsg::TokenCount(
codex_protocol::protocol::TokenCountEvent {
info: Some(codex_protocol::protocol::TokenUsageInfo {
total_token_usage: codex_protocol::protocol::TokenUsage {
input_tokens: 0,
cached_input_tokens: 0,
output_tokens: 0,
reasoning_output_tokens: 0,
total_tokens: 321,
},
last_token_usage: codex_protocol::protocol::TokenUsage::default(),
model_context_window: None,
}),
rate_limits: None,
},
))];
let override_updated_at =
DateTime::<Utc>::from_timestamp(1_700_001_234, 0).expect("timestamp");
runtime
.apply_rollout_items(&builder, &items, None, None, Some(override_updated_at))
.await
.expect("apply_rollout_items should succeed");
let persisted = runtime
.get_thread(thread_id)
.await
.expect("thread should load")
.expect("thread should exist");
assert_eq!(persisted.tokens_used, 321);
assert_eq!(persisted.updated_at, override_updated_at);
}
}
+122 -1
View File
@@ -213,6 +213,7 @@ use crate::tui::FrameRequester;
use crate::ui_consts::LIVE_PREFIX_COLS;
use codex_chatgpt::connectors;
use codex_chatgpt::connectors::AppInfo;
use codex_core::plugins::PluginCapabilitySummary;
use codex_core::skills::model::SkillMetadata;
use codex_file_search::FileMatch;
use std::cell::RefCell;
@@ -391,6 +392,7 @@ pub(crate) struct ChatComposer {
next_element_id: u64,
context_window_used_tokens: Option<i64>,
skills: Option<Vec<SkillMetadata>>,
plugins: Option<Vec<PluginCapabilitySummary>>,
connectors_snapshot: Option<ConnectorsSnapshot>,
dismissed_mention_popup_token: Option<String>,
mention_bindings: HashMap<u64, ComposerMentionBinding>,
@@ -510,6 +512,7 @@ impl ChatComposer {
next_element_id: 0,
context_window_used_tokens: None,
skills: None,
plugins: None,
connectors_snapshot: None,
dismissed_mention_popup_token: None,
mention_bindings: HashMap::new(),
@@ -546,6 +549,11 @@ impl ChatComposer {
self.skills = skills;
}
pub fn set_plugin_mentions(&mut self, plugins: Option<Vec<PluginCapabilitySummary>>) {
self.plugins = plugins;
self.sync_popups();
}
/// Toggle composer-side image paste handling.
///
/// This only affects whether image-like paste content is converted into attachments; the
@@ -1926,17 +1934,25 @@ impl ChatComposer {
self.skills.as_ref()
}
pub fn plugins(&self) -> Option<&Vec<PluginCapabilitySummary>> {
self.plugins.as_ref()
}
fn mentions_enabled(&self) -> bool {
let skills_ready = self
.skills
.as_ref()
.is_some_and(|skills| !skills.is_empty());
let plugins_ready = self
.plugins
.as_ref()
.is_some_and(|plugins| !plugins.is_empty());
let connectors_ready = self.connectors_enabled
&& self
.connectors_snapshot
.as_ref()
.is_some_and(|snapshot| !snapshot.connectors.is_empty());
skills_ready || connectors_ready
skills_ready || plugins_ready || connectors_ready
}
/// Extract a token prefixed with `prefix` under the cursor, if any.
@@ -3559,6 +3575,58 @@ impl ChatComposer {
}
}
if let Some(plugins) = self.plugins.as_ref() {
for plugin in plugins {
let (plugin_name, marketplace_name) = plugin
.config_name
.split_once('@')
.unwrap_or((plugin.config_name.as_str(), ""));
let mut capability_labels = Vec::new();
if plugin.has_skills {
capability_labels.push("skills".to_string());
}
if !plugin.mcp_server_names.is_empty() {
let mcp_server_count = plugin.mcp_server_names.len();
capability_labels.push(if mcp_server_count == 1 {
"1 MCP server".to_string()
} else {
format!("{mcp_server_count} MCP servers")
});
}
if !plugin.app_connector_ids.is_empty() {
let app_count = plugin.app_connector_ids.len();
capability_labels.push(if app_count == 1 {
"1 app".to_string()
} else {
format!("{app_count} apps")
});
}
let description = plugin.description.clone().or_else(|| {
Some(if capability_labels.is_empty() {
"Plugin".to_string()
} else {
format!("Plugin · {}", capability_labels.join(" · "))
})
});
let mut search_terms = vec![plugin_name.to_string(), plugin.config_name.clone()];
if plugin.display_name != plugin_name {
search_terms.push(plugin.display_name.clone());
}
if !marketplace_name.is_empty() {
search_terms.push(marketplace_name.to_string());
}
mentions.push(MentionItem {
display_name: plugin.display_name.clone(),
description,
insert_text: format!("${plugin_name}"),
search_terms,
path: Some(format!("plugin://{}", plugin.config_name)),
category_tag: (!marketplace_name.is_empty())
.then(|| format!("[{marketplace_name}]")),
});
}
}
if self.connectors_enabled
&& let Some(snapshot) = self.connectors_snapshot.as_ref()
{
@@ -5212,6 +5280,59 @@ mod tests {
assert_eq!(mention.path, Some("app://connector_1".to_string()));
}
#[test]
fn set_plugin_mentions_refreshes_open_mention_popup() {
let (tx, _rx) = unbounded_channel::<AppEvent>();
let sender = AppEventSender::new(tx);
let mut composer = ChatComposer::new(
true,
sender,
false,
"Ask Codex to do anything".to_string(),
false,
);
composer.set_text_content("$".to_string(), Vec::new(), Vec::new());
assert!(matches!(composer.active_popup, ActivePopup::None));
composer.set_plugin_mentions(Some(vec![PluginCapabilitySummary {
config_name: "sample@test".to_string(),
display_name: "Sample Plugin".to_string(),
description: None,
has_skills: true,
mcp_server_names: vec!["sample".to_string()],
app_connector_ids: Vec::new(),
}]));
let ActivePopup::Skill(popup) = &composer.active_popup else {
panic!("expected mention popup to open after plugin update");
};
let mention = popup
.selected_mention()
.expect("expected plugin mention to be selected");
assert_eq!(mention.insert_text, "$sample".to_string());
assert_eq!(mention.path, Some("plugin://sample@test".to_string()));
}
#[test]
fn plugin_mention_popup_snapshot() {
snapshot_composer_state("plugin_mention_popup", false, |composer| {
composer.set_text_content("$sa".to_string(), Vec::new(), Vec::new());
composer.set_plugin_mentions(Some(vec![PluginCapabilitySummary {
config_name: "sample@test".to_string(),
display_name: "Sample Plugin".to_string(),
description: Some(
"Plugin that includes the Figma MCP server and Skills for common workflows"
.to_string(),
),
has_skills: true,
mcp_server_names: vec!["sample".to_string()],
app_connector_ids: vec![codex_core::plugins::AppConnectorId(
"calendar".to_string(),
)],
}]));
});
}
#[test]
fn set_connector_mentions_excludes_disabled_apps_from_mention_popup() {
let (tx, _rx) = unbounded_channel::<AppEvent>();
+10
View File
@@ -28,6 +28,7 @@ use crate::render::renderable::RenderableItem;
use crate::tui::FrameRequester;
use bottom_pane_view::BottomPaneView;
use codex_core::features::Features;
use codex_core::plugins::PluginCapabilitySummary;
use codex_core::skills::model::SkillMetadata;
use codex_file_search::FileMatch;
use codex_protocol::request_user_input::RequestUserInputEvent;
@@ -254,6 +255,11 @@ impl BottomPane {
self.request_redraw();
}
pub fn set_plugin_mentions(&mut self, plugins: Option<Vec<PluginCapabilitySummary>>) {
self.composer.set_plugin_mentions(plugins);
self.request_redraw();
}
pub fn take_mention_bindings(&mut self) -> Vec<MentionBinding> {
self.composer.take_mention_bindings()
}
@@ -333,6 +339,10 @@ impl BottomPane {
self.composer.skills()
}
pub fn plugins(&self) -> Option<&Vec<PluginCapabilitySummary>> {
self.composer.plugins()
}
#[cfg(test)]
pub(crate) fn context_window_percent(&self) -> Option<i64> {
self.context_window_percent
@@ -0,0 +1,13 @@
---
source: tui/src/bottom_pane/chat_composer.rs
expression: terminal.backend()
---
" "
"› $sa "
" "
" "
" "
" "
" Sample Plugin Plugin that includes the Figma MCP server and Skills for common workflows "
" "
" Press enter to insert or esc to close "
+51
View File
@@ -1225,6 +1225,7 @@ impl ChatWidget {
self.refresh_model_display();
self.sync_fast_command_enabled();
self.sync_personality_command_enabled();
self.refresh_plugin_mentions();
let startup_tooltip_override = self.startup_tooltip_override.take();
let show_fast_status = self.should_show_fast_status(event.service_tier);
let session_info_cell = history_cell::new_session_info(
@@ -4411,6 +4412,7 @@ impl ChatWidget {
.collect();
let mut skill_names_lower: HashSet<String> = HashSet::new();
let mut selected_skill_paths: HashSet<PathBuf> = HashSet::new();
let mut selected_plugin_ids: HashSet<String> = HashSet::new();
if let Some(skills) = self.bottom_pane.skills() {
skill_names_lower = skills
@@ -4450,6 +4452,30 @@ impl ChatWidget {
}
}
if let Some(plugins) = self.plugins_for_mentions() {
for binding in &mention_bindings {
let Some(plugin_config_name) = binding
.path
.strip_prefix("plugin://")
.filter(|id| !id.is_empty())
else {
continue;
};
if !selected_plugin_ids.insert(plugin_config_name.to_string()) {
continue;
}
if let Some(plugin) = plugins
.iter()
.find(|plugin| plugin.config_name == plugin_config_name)
{
items.push(UserInput::Mention {
name: plugin.display_name.clone(),
path: binding.path.clone(),
});
}
}
}
let mut selected_app_ids: HashSet<String> = HashSet::new();
if let Some(apps) = self.connectors_for_mentions() {
for binding in &mention_bindings {
@@ -7153,6 +7179,9 @@ impl ChatWidget {
if feature == Feature::Personality {
self.sync_personality_command_enabled();
}
if feature == Feature::Plugins {
self.refresh_plugin_mentions();
}
if feature == Feature::PreventIdleSleep {
self.turn_sleep_inhibitor = SleepInhibitor::new(enabled);
self.turn_sleep_inhibitor
@@ -7578,6 +7607,14 @@ impl ChatWidget {
}
}
fn plugins_for_mentions(&self) -> Option<&[codex_core::plugins::PluginCapabilitySummary]> {
if !self.config.features.enabled(Feature::Plugins) {
return None;
}
self.bottom_pane.plugins().map(Vec::as_slice)
}
/// Build a placeholder header cell while the session is configuring.
fn placeholder_session_header_cell(config: &Config) -> Box<dyn HistoryCell> {
let placeholder_style = Style::default().add_modifier(Modifier::DIM | Modifier::ITALIC);
@@ -8077,6 +8114,7 @@ impl ChatWidget {
fn on_list_skills(&mut self, ev: ListSkillsResponseEvent) {
self.set_skills_from_response(&ev);
self.refresh_plugin_mentions();
}
pub(crate) fn on_connectors_loaded(
@@ -8160,6 +8198,19 @@ impl ChatWidget {
self.bottom_pane.set_connectors_snapshot(Some(snapshot));
}
fn refresh_plugin_mentions(&mut self) {
if !self.config.features.enabled(Feature::Plugins) {
self.bottom_pane.set_plugin_mentions(None);
return;
}
let plugins = PluginsManager::new(self.config.codex_home.clone())
.plugins_for_config(&self.config)
.capability_summaries()
.to_vec();
self.bottom_pane.set_plugin_mentions(Some(plugins));
}
pub(crate) fn open_review_popup(&mut self) {
let mut items: Vec<SelectionItem> = Vec::new();
+14 -11
View File
@@ -296,7 +296,13 @@ pub(crate) struct ToolMentions {
linked_paths: HashMap<String, String>,
}
const TOOL_MENTION_SIGIL: char = '$';
fn extract_tool_mentions_from_text(text: &str) -> ToolMentions {
extract_tool_mentions_from_text_with_sigil(text, TOOL_MENTION_SIGIL)
}
fn extract_tool_mentions_from_text_with_sigil(text: &str, sigil: char) -> ToolMentions {
let text_bytes = text.as_bytes();
let mut names: HashSet<String> = HashSet::new();
let mut linked_paths: HashMap<String, String> = HashMap::new();
@@ -306,10 +312,10 @@ fn extract_tool_mentions_from_text(text: &str) -> ToolMentions {
let byte = text_bytes[index];
if byte == b'['
&& let Some((name, path, end_index)) =
parse_linked_tool_mention(text, text_bytes, index)
parse_linked_tool_mention(text, text_bytes, index, sigil)
{
if !is_common_env_var(name) {
if !is_app_or_mcp_path(path) {
if is_skill_path(path) {
names.insert(name.to_string());
}
linked_paths
@@ -320,7 +326,7 @@ fn extract_tool_mentions_from_text(text: &str) -> ToolMentions {
continue;
}
if byte != b'$' {
if byte != sigil as u8 {
index += 1;
continue;
}
@@ -359,13 +365,14 @@ fn parse_linked_tool_mention<'a>(
text: &'a str,
text_bytes: &[u8],
start: usize,
sigil: char,
) -> Option<(&'a str, &'a str, usize)> {
let dollar_index = start + 1;
if text_bytes.get(dollar_index) != Some(&b'$') {
let sigil_index = start + 1;
if text_bytes.get(sigil_index) != Some(&(sigil as u8)) {
return None;
}
let name_start = dollar_index + 1;
let name_start = sigil_index + 1;
let first_name_byte = text_bytes.get(name_start)?;
if !is_mention_name_char(*first_name_byte) {
return None;
@@ -434,7 +441,7 @@ fn is_mention_name_char(byte: u8) -> bool {
}
fn is_skill_path(path: &str) -> bool {
!is_app_or_mcp_path(path)
!path.starts_with("app://") && !path.starts_with("mcp://") && !path.starts_with("plugin://")
}
fn normalize_skill_path(path: &str) -> &str {
@@ -445,7 +452,3 @@ fn app_id_from_path(path: &str) -> Option<&str> {
path.strip_prefix("app://")
.filter(|value| !value.is_empty())
}
fn is_app_or_mcp_path(path: &str) -> bool {
path.starts_with("app://") || path.starts_with("mcp://")
}
+67
View File
@@ -4141,6 +4141,73 @@ async fn item_completed_pops_pending_steer_with_local_image_and_text_elements()
assert!(stored_remote_image_urls.is_empty());
}
#[tokio::test(flavor = "multi_thread")]
async fn submit_user_message_emits_structured_plugin_mentions_from_bindings() {
let (mut chat, _rx, mut op_rx) = make_chatwidget_manual(None).await;
let conversation_id = ThreadId::new();
let rollout_file = NamedTempFile::new().unwrap();
let configured = codex_protocol::protocol::SessionConfiguredEvent {
session_id: conversation_id,
forked_from_id: None,
thread_name: None,
model: "test-model".to_string(),
model_provider_id: "test-provider".to_string(),
service_tier: None,
approval_policy: AskForApproval::Never,
sandbox_policy: SandboxPolicy::new_read_only_policy(),
cwd: PathBuf::from("/home/user/project"),
reasoning_effort: Some(ReasoningEffortConfig::default()),
history_log_id: 0,
history_entry_count: 0,
initial_messages: None,
network_proxy: None,
rollout_path: Some(rollout_file.path().to_path_buf()),
};
chat.handle_codex_event(Event {
id: "initial".into(),
msg: EventMsg::SessionConfigured(configured),
});
chat.set_feature_enabled(Feature::Plugins, true);
chat.bottom_pane.set_plugin_mentions(Some(vec![
codex_core::plugins::PluginCapabilitySummary {
config_name: "sample@test".to_string(),
display_name: "Sample Plugin".to_string(),
description: None,
has_skills: true,
mcp_server_names: Vec::new(),
app_connector_ids: Vec::new(),
},
]));
chat.submit_user_message(UserMessage {
text: "$sample".to_string(),
local_images: Vec::new(),
remote_image_urls: Vec::new(),
text_elements: Vec::new(),
mention_bindings: vec![MentionBinding {
mention: "sample".to_string(),
path: "plugin://sample@test".to_string(),
}],
});
let Op::UserTurn { items, .. } = next_submit_op(&mut op_rx) else {
panic!("expected Op::UserTurn");
};
assert_eq!(
items,
vec![
UserInput::Text {
text: "$sample".to_string(),
text_elements: Vec::new(),
},
UserInput::Mention {
name: "Sample Plugin".to_string(),
path: "plugin://sample@test".to_string(),
},
]
);
}
#[tokio::test]
async fn steer_enter_during_final_stream_preserves_follow_up_prompts_in_order() {
let (mut chat, mut rx, mut op_rx) = make_chatwidget_manual(None).await;
+24 -11
View File
@@ -7,6 +7,8 @@ pub(crate) struct LinkedMention {
pub(crate) path: String,
}
const TOOL_MENTION_SIGIL: char = '$';
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct DecodedHistoryText {
pub(crate) text: String,
@@ -31,7 +33,7 @@ pub(crate) fn encode_history_mentions(text: &str, mentions: &[LinkedMention]) ->
let mut index = 0usize;
while index < bytes.len() {
if bytes[index] == b'$' {
if bytes[index] == TOOL_MENTION_SIGIL as u8 {
let name_start = index + 1;
if let Some(first) = bytes.get(name_start)
&& is_mention_name_char(*first)
@@ -46,7 +48,7 @@ pub(crate) fn encode_history_mentions(text: &str, mentions: &[LinkedMention]) ->
let name = &text[name_start..name_end];
if let Some(path) = mentions_by_name.get_mut(name).and_then(VecDeque::pop_front) {
out.push('[');
out.push('$');
out.push(TOOL_MENTION_SIGIL);
out.push_str(name);
out.push_str("](");
out.push_str(path);
@@ -75,11 +77,12 @@ pub(crate) fn decode_history_mentions(text: &str) -> DecodedHistoryText {
while index < bytes.len() {
if bytes[index] == b'['
&& let Some((name, path, end_index)) = parse_linked_tool_mention(text, bytes, index)
&& let Some((name, path, end_index)) =
parse_linked_tool_mention(text, bytes, index, TOOL_MENTION_SIGIL)
&& !is_common_env_var(name)
&& is_tool_path(path)
{
out.push('$');
out.push(TOOL_MENTION_SIGIL);
out.push_str(name);
mentions.push(LinkedMention {
mention: name.to_string(),
@@ -106,13 +109,14 @@ fn parse_linked_tool_mention<'a>(
text: &'a str,
text_bytes: &[u8],
start: usize,
sigil: char,
) -> Option<(&'a str, &'a str, usize)> {
let dollar_index = start + 1;
if text_bytes.get(dollar_index) != Some(&b'$') {
let sigil_index = start + 1;
if text_bytes.get(sigil_index) != Some(&(sigil as u8)) {
return None;
}
let name_start = dollar_index + 1;
let name_start = sigil_index + 1;
let first_name_byte = text_bytes.get(name_start)?;
if !is_mention_name_char(*first_name_byte) {
return None;
@@ -183,6 +187,7 @@ fn is_common_env_var(name: &str) -> bool {
fn is_tool_path(path: &str) -> bool {
path.starts_with("app://")
|| path.starts_with("mcp://")
|| path.starts_with("plugin://")
|| path.starts_with("skill://")
|| path
.rsplit(['/', '\\'])
@@ -198,9 +203,9 @@ mod tests {
#[test]
fn decode_history_mentions_restores_visible_tokens() {
let decoded = decode_history_mentions(
"Use [$figma](app://figma-1) and [$figma](/tmp/figma/SKILL.md).",
"Use [$figma](app://figma-1), [$sample](plugin://sample@test), and [$figma](/tmp/figma/SKILL.md).",
);
assert_eq!(decoded.text, "Use $figma and $figma.");
assert_eq!(decoded.text, "Use $figma, $sample, and $figma.");
assert_eq!(
decoded.mentions,
vec![
@@ -208,6 +213,10 @@ mod tests {
mention: "figma".to_string(),
path: "app://figma-1".to_string(),
},
LinkedMention {
mention: "sample".to_string(),
path: "plugin://sample@test".to_string(),
},
LinkedMention {
mention: "figma".to_string(),
path: "/tmp/figma/SKILL.md".to_string(),
@@ -218,7 +227,7 @@ mod tests {
#[test]
fn encode_history_mentions_links_bound_mentions_in_order() {
let text = "$figma then $figma then $other";
let text = "$figma then $sample then $figma then $other";
let encoded = encode_history_mentions(
text,
&[
@@ -226,6 +235,10 @@ mod tests {
mention: "figma".to_string(),
path: "app://figma-app".to_string(),
},
LinkedMention {
mention: "sample".to_string(),
path: "plugin://sample@test".to_string(),
},
LinkedMention {
mention: "figma".to_string(),
path: "/tmp/figma/SKILL.md".to_string(),
@@ -234,7 +247,7 @@ mod tests {
);
assert_eq!(
encoded,
"[$figma](app://figma-app) then [$figma](/tmp/figma/SKILL.md) then $other"
"[$figma](app://figma-app) then [$sample](plugin://sample@test) then [$figma](/tmp/figma/SKILL.md) then $other"
);
}
}