openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-11-12 21:20:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add hybrid plugin mention fallback hints

Browse Source
This commit is contained in:
Adam Syed
2026-05-06 14:31:59 -07:00
parent 4950e7d8a6
commit 1992816729
16 changed files with 272 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/app-server-protocol/src/protocol/v2.rs
+22 -2
View File
@@ -6023,6 +6023,8 @@ pub enum UserInput {
Mention {
name: String,
path: String,
#[serde(default)]
computer_use_native_app_bundle_id: Option<String>,
},
}
@@ -6039,7 +6041,15 @@ impl UserInput {
UserInput::Image { url } => CoreUserInput::Image { image_url: url },
UserInput::LocalImage { path } => CoreUserInput::LocalImage { path },
UserInput::Skill { name, path } => CoreUserInput::Skill { name, path },
UserInput::Mention { name, path } => CoreUserInput::Mention { name, path },
UserInput::Mention {
name,
path,
computer_use_native_app_bundle_id,
} => CoreUserInput::Mention {
name,
path,
computer_use_native_app_bundle_id,
},
}
}
}
@@ -6057,7 +6067,15 @@ impl From<CoreUserInput> for UserInput {
CoreUserInput::Image { image_url } => UserInput::Image { url: image_url },
CoreUserInput::LocalImage { path } => UserInput::LocalImage { path },
CoreUserInput::Skill { name, path } => UserInput::Skill { name, path },
CoreUserInput::Mention { name, path } => UserInput::Mention { name, path },
CoreUserInput::Mention {
name,
path,
computer_use_native_app_bundle_id,
} => UserInput::Mention {
name,
path,
computer_use_native_app_bundle_id,
},
_ => unreachable!("unsupported user input variant"),
}
}
@@ -10702,6 +10720,7 @@ mod tests {
CoreUserInput::Mention {
name: "Demo App".to_string(),
path: "app://demo-app".to_string(),
computer_use_native_app_bundle_id: None,
},
],
});
@@ -10728,6 +10747,7 @@ mod tests {
UserInput::Mention {
name: "Demo App".to_string(),
path: "app://demo-app".to_string(),
computer_use_native_app_bundle_id: None,
},
],
}
codex-rs/app-server/tests/suite/v2/turn_start.rs
+1
View File
@@ -570,6 +570,7 @@ async fn turn_start_accepts_text_at_limit_with_mention_item() -> Result<()> {
V2UserInput::Mention {
name: "Demo App".to_string(),
path: "app://demo-app".to_string(),
computer_use_native_app_bundle_id: None,
},
],
..Default::default()
codex-rs/core/src/agent/control.rs
+3 -1
View File
@@ -1223,7 +1223,9 @@ pub(crate) fn render_input_preview(initial_operation: &Op) -> String {
UserInput::Image { .. } => "[image]".to_string(),
UserInput::LocalImage { path } => format!("[local_image:{}]", path.display()),
UserInput::Skill { name, path } => format!("[skill:${name}]({})", path.display()),
UserInput::Mention { name, path } => format!("[mention:${name}]({path})"),
UserInput::Mention { name, path, .. } => {
format!("[mention:${name}]({path})")
}
_ => "[input]".to_string(),
})
.collect::<Vec<_>>()
codex-rs/core/src/plugins/injection.rs
+12 -6
View File
@@ -7,13 +7,13 @@ use codex_protocol::models::ResponseItem;
use crate::connectors;
use crate::context::ContextualUserFragment;
use crate::context::PluginInstructions;
use crate::plugins::PluginCapabilitySummary;
use crate::plugins::ExplicitPluginMention;
use crate::plugins::render_explicit_plugin_instructions;
use codex_mcp::CODEX_APPS_MCP_SERVER_NAME;
use codex_mcp::ToolInfo;
pub(crate) fn build_plugin_injections(
mentioned_plugins: &[PluginCapabilitySummary],
mentioned_plugins: &[ExplicitPluginMention],
mcp_tools: &HashMap<String, ToolInfo>,
available_connectors: &[connectors::AppInfo],
) -> Vec<ResponseItem> {
@@ -25,7 +25,8 @@ pub(crate) fn build_plugin_injections(
// model at the plugin's visible MCP servers, enabled apps, and skill prefix.
mentioned_plugins
.iter()
.filter_map(|plugin| {
.filter_map(|mention| {
let plugin = &mention.plugin;
let available_mcp_servers = mcp_tools
.values()
.filter(|tool| {
@@ -52,9 +53,14 @@ pub(crate) fn build_plugin_injections(
.collect::<BTreeSet<String>>()
.into_iter()
.collect::<Vec<_>>();
render_explicit_plugin_instructions(plugin, &available_mcp_servers, &available_apps)
.map(PluginInstructions::new)
.map(ContextualUserFragment::into)
render_explicit_plugin_instructions(
plugin,
&available_mcp_servers,
&available_apps,
mention.has_computer_use_native_fallback,
)
.map(PluginInstructions::new)
.map(ContextualUserFragment::into)
})
.collect()
}
codex-rs/core/src/plugins/mentions.rs
+75 -14
View File
@@ -15,11 +15,25 @@ use crate::mention_syntax::TOOL_MENTION_SIGIL;
use super::PluginCapabilitySummary;
const COMPUTER_USE_PLUGIN_CONFIG_NAME: &str = "computer-use@openai-bundled";
pub(crate) struct CollectedToolMentions {
pub(crate) plain_names: HashSet<String>,
pub(crate) paths: HashSet<String>,
}
#[derive(Clone, Debug, PartialEq)]
pub(crate) struct ExplicitPluginMention {
pub(crate) plugin: PluginCapabilitySummary,
pub(crate) has_computer_use_native_fallback: bool,
}
#[derive(Clone, Debug)]
struct StructuredPluginMention {
config_name: String,
has_computer_use_native_fallback: bool,
}
pub(crate) fn collect_tool_mentions_from_messages(messages: &[String]) -> CollectedToolMentions {
collect_tool_mentions_from_messages_with_sigil(messages, TOOL_MENTION_SIGIL)
}
@@ -63,7 +77,7 @@ pub(crate) fn collect_explicit_app_ids(input: &[UserInput]) -> HashSet<String> {
pub(crate) fn collect_explicit_plugin_mentions(
input: &[UserInput],
plugins: &[PluginCapabilitySummary],
) -> Vec<PluginCapabilitySummary> {
) -> Vec<ExplicitPluginMention> {
if plugins.is_empty() {
return Vec::new();
}
@@ -76,30 +90,77 @@ pub(crate) fn collect_explicit_plugin_mentions(
})
.collect::<Vec<String>>();
let mentioned_config_names: HashSet<String> = input
let structured_mentions = input
.iter()
.filter_map(|item| match item {
UserInput::Mention { path, .. } => Some(path.clone()),
UserInput::Mention {
path,
computer_use_native_app_bundle_id,
..
} => Some((path.clone(), computer_use_native_app_bundle_id.clone())),
_ => None,
})
.chain(
// Plugin plaintext links use `@`, not the default `$` tool sigil.
collect_tool_mentions_from_messages_with_sigil(&messages, PLUGIN_TEXT_MENTION_SIGIL)
.paths,
)
.filter(|path| tool_kind_for_path(path.as_str()) == ToolMentionKind::Plugin)
.filter_map(|path| plugin_config_name_from_path(path.as_str()).map(str::to_string))
.collect();
.filter(|(path, _)| tool_kind_for_path(path.as_str()) == ToolMentionKind::Plugin)
.filter_map(|(path, native_app_bundle_id)| {
plugin_config_name_from_path(path.as_str()).map(|config_name| StructuredPluginMention {
config_name: config_name.to_string(),
has_computer_use_native_fallback: native_app_bundle_id
.as_deref()
.is_some_and(|bundle_id| !bundle_id.trim().is_empty()),
})
})
.collect::<Vec<_>>();
let linked_config_names =
collect_tool_mentions_from_messages_with_sigil(&messages, PLUGIN_TEXT_MENTION_SIGIL)
.paths
.into_iter()
.filter(|path| tool_kind_for_path(path.as_str()) == ToolMentionKind::Plugin)
.filter_map(|path| plugin_config_name_from_path(path.as_str()).map(str::to_string))
.collect::<HashSet<_>>();
let mentioned_config_names = structured_mentions
.iter()
.map(|mention| mention.config_name.clone())
.chain(linked_config_names)
.collect::<HashSet<_>>();
if mentioned_config_names.is_empty() {
return Vec::new();
}
plugins
let mut mentioned_plugins = plugins
.iter()
.filter(|plugin| mentioned_config_names.contains(plugin.config_name.as_str()))
.cloned()
.collect()
.map(|plugin| {
let has_computer_use_native_fallback = structured_mentions.iter().any(|mention| {
mention.config_name == plugin.config_name
&& mention.has_computer_use_native_fallback
});
ExplicitPluginMention {
plugin: plugin.clone(),
has_computer_use_native_fallback,
}
})
.collect::<Vec<_>>();
if mentioned_plugins
.iter()
.any(|mention| mention.has_computer_use_native_fallback)
&& !mentioned_plugins
.iter()
.any(|mention| mention.plugin.config_name.as_str() == COMPUTER_USE_PLUGIN_CONFIG_NAME)
&& let Some(computer_use_plugin) = plugins
.iter()
.find(|plugin| plugin.config_name.as_str() == COMPUTER_USE_PLUGIN_CONFIG_NAME)
{
mentioned_plugins.push(ExplicitPluginMention {
plugin: computer_use_plugin.clone(),
has_computer_use_native_fallback: false,
});
}
mentioned_plugins
}
pub(crate) use crate::build_skill_name_counts;
codex-rs/core/src/plugins/mentions_tests.rs
+94 -5
View File
@@ -3,6 +3,7 @@ use std::collections::HashSet;
use codex_protocol::user_input::UserInput;
use pretty_assertions::assert_eq;
use super::ExplicitPluginMention;
use super::collect_explicit_app_ids;
use super::collect_explicit_plugin_mentions;
use crate::plugins::PluginCapabilitySummary;
@@ -25,6 +26,16 @@ fn plugin(config_name: &str, display_name: &str) -> PluginCapabilitySummary {
}
}
fn explicit_plugin_mention(
plugin: PluginCapabilitySummary,
has_computer_use_native_fallback: bool,
) -> ExplicitPluginMention {
ExplicitPluginMention {
plugin,
has_computer_use_native_fallback,
}
}
#[test]
fn collect_explicit_app_ids_from_linked_text_mentions() {
let input = vec![text_input("use [$calendar](app://calendar)")];
@@ -41,6 +52,7 @@ fn collect_explicit_app_ids_dedupes_structured_and_linked_mentions() {
UserInput::Mention {
name: "calendar".to_string(),
path: "app://calendar".to_string(),
computer_use_native_app_bundle_id: None,
},
];
@@ -58,14 +70,17 @@ fn collect_explicit_app_ids_ignores_non_app_paths() {
UserInput::Mention {
name: "docs".to_string(),
path: "mcp://docs".to_string(),
computer_use_native_app_bundle_id: None,
},
UserInput::Mention {
name: "skill".to_string(),
path: "skill://team/skill".to_string(),
computer_use_native_app_bundle_id: None,
},
UserInput::Mention {
name: "file".to_string(),
path: "/tmp/file.txt".to_string(),
computer_use_native_app_bundle_id: None,
},
];
@@ -85,11 +100,18 @@ fn collect_explicit_plugin_mentions_from_structured_paths() {
&[UserInput::Mention {
name: "sample".to_string(),
path: "plugin://sample@test".to_string(),
computer_use_native_app_bundle_id: None,
}],
&plugins,
);
assert_eq!(mentioned, vec![plugin("sample@test", "sample")]);
assert_eq!(
mentioned,
vec![explicit_plugin_mention(
plugin("sample@test", "sample"),
false,
)]
);
}
#[test]
@@ -104,7 +126,13 @@ fn collect_explicit_plugin_mentions_from_linked_text_mentions() {
&plugins,
);
assert_eq!(mentioned, vec![plugin("sample@test", "sample")]);
assert_eq!(
mentioned,
vec![explicit_plugin_mention(
plugin("sample@test", "sample"),
false,
)]
);
}
#[test]
@@ -120,12 +148,19 @@ fn collect_explicit_plugin_mentions_dedupes_structured_and_linked_mentions() {
UserInput::Mention {
name: "sample".to_string(),
path: "plugin://sample@test".to_string(),
computer_use_native_app_bundle_id: None,
},
],
&plugins,
);
assert_eq!(mentioned, vec![plugin("sample@test", "sample")]);
assert_eq!(
mentioned,
vec![explicit_plugin_mention(
plugin("sample@test", "sample"),
false,
)]
);
}
#[test]
@@ -139,7 +174,7 @@ fn collect_explicit_plugin_mentions_ignores_non_plugin_paths() {
&plugins,
);
assert_eq!(mentioned, Vec::<PluginCapabilitySummary>::new());
assert_eq!(mentioned, Vec::<ExplicitPluginMention>::new());
}
#[test]
@@ -151,5 +186,59 @@ fn collect_explicit_plugin_mentions_ignores_dollar_linked_plugin_mentions() {
&plugins,
);
assert_eq!(mentioned, Vec::<PluginCapabilitySummary>::new());
assert_eq!(mentioned, Vec::<ExplicitPluginMention>::new());
}
#[test]
fn collect_explicit_plugin_mentions_marks_structured_native_fallbacks() {
let zoom = plugin("zoom@test", "Zoom");
let mentioned = collect_explicit_plugin_mentions(
&[UserInput::Mention {
name: "Zoom".to_string(),
path: "plugin://zoom@test".to_string(),
computer_use_native_app_bundle_id: Some("us.zoom.xos".to_string()),
}],
&[zoom.clone()],
);
assert_eq!(mentioned, vec![explicit_plugin_mention(zoom, true)]);
}
#[test]
fn collect_explicit_plugin_mentions_preserves_computer_use_context_for_native_fallbacks() {
let zoom = plugin("zoom@test", "Zoom");
let computer_use = plugin("computer-use@openai-bundled", "Computer Use");
let mentioned = collect_explicit_plugin_mentions(
&[UserInput::Mention {
name: "Zoom".to_string(),
path: "plugin://zoom@test".to_string(),
computer_use_native_app_bundle_id: Some("us.zoom.xos".to_string()),
}],
&[zoom.clone(), computer_use.clone()],
);
assert_eq!(
mentioned,
vec![
explicit_plugin_mention(zoom, true),
explicit_plugin_mention(computer_use, false),
]
);
}
#[test]
fn collect_explicit_plugin_mentions_ignores_empty_native_fallback_markers() {
let zoom = plugin("zoom@test", "Zoom");
let mentioned = collect_explicit_plugin_mentions(
&[UserInput::Mention {
name: "Zoom".to_string(),
path: "plugin://zoom@test".to_string(),
computer_use_native_app_bundle_id: Some(" ".to_string()),
}],
&[zoom.clone()],
);
assert_eq!(mentioned, vec![explicit_plugin_mention(zoom, false)]);
}
codex-rs/core/src/plugins/mod.rs
+1
View File
@@ -11,6 +11,7 @@ pub(crate) use discoverable::list_tool_suggest_discoverable_plugins;
pub(crate) use injection::build_plugin_injections;
pub(crate) use render::render_explicit_plugin_instructions;
pub(crate) use mentions::ExplicitPluginMention;
pub(crate) use mentions::build_connector_slug_counts;
pub(crate) use mentions::build_skill_name_counts;
pub(crate) use mentions::collect_explicit_app_ids;
codex-rs/core/src/plugins/render.rs
+8
View File
@@ -13,6 +13,7 @@ pub(crate) fn render_explicit_plugin_instructions(
plugin: &PluginCapabilitySummary,
available_mcp_servers: &[String],
available_apps: &[String],
has_computer_use_native_fallback: bool,
) -> Option<String> {
let mut lines = vec![format!(
"Capabilities from the `{}` plugin:",
@@ -48,6 +49,13 @@ pub(crate) fn render_explicit_plugin_instructions(
));
}
if has_computer_use_native_fallback {
lines.push(
"- This plugin also corresponds to a native desktop app available through Computer Use. Prefer plugin-associated capabilities first; if they are unavailable, insufficient, or fail, use the Computer Use tool surface for the native app fallback. If Computer Use tools are not already visible, use `tool_search` for `computer use` to discover them."
.to_string(),
);
}
if lines.len() == 1 {
return None;
}
codex-rs/core/src/plugins/render_tests.rs
+39
View File
@@ -21,3 +21,42 @@ fn render_plugins_section_includes_descriptions_and_skill_naming_guidance() {
assert_eq!(rendered, expected);
}
#[test]
fn render_explicit_plugin_instructions_mentions_native_app_fallback() {
let rendered = render_explicit_plugin_instructions(
&PluginCapabilitySummary {
config_name: "zoom@test".to_string(),
display_name: "Zoom".to_string(),
..PluginCapabilitySummary::default()
},
&[],
&[],
true,
)
.expect("native app fallback should render instructions");
let expected = "Capabilities from the `Zoom` plugin:\n- This plugin also corresponds to a native desktop app available through Computer Use. Prefer plugin-associated capabilities first; if they are unavailable, insufficient, or fail, use the Computer Use tool surface for the native app fallback. If Computer Use tools are not already visible, use `tool_search` for `computer use` to discover them.\nUse these plugin-associated capabilities to help solve the task.";
assert_eq!(rendered, expected);
}
#[test]
fn render_explicit_plugin_instructions_omits_native_app_fallback_without_deduped_match() {
let rendered = render_explicit_plugin_instructions(
&PluginCapabilitySummary {
config_name: "zoom@test".to_string(),
display_name: "Zoom".to_string(),
has_skills: true,
..PluginCapabilitySummary::default()
},
&[],
&[],
false,
)
.expect("skill guidance should still render instructions");
let expected = "Capabilities from the `Zoom` plugin:\n- Skills from this plugin are prefixed with `Zoom:`.\nUse these plugin-associated capabilities to help solve the task.";
assert_eq!(rendered, expected);
}
codex-rs/core/src/session/turn.rs
+1 -1
View File
@@ -276,7 +276,7 @@ pub(crate) async fn run_turn(
build_plugin_injections(&mentioned_plugins, &mcp_tools, &available_connectors);
let mentioned_plugin_metadata = mentioned_plugins
.iter()
.filter_map(crate::plugins::PluginCapabilitySummary::telemetry_metadata)
.filter_map(|mention| mention.plugin.telemetry_metadata())
.collect::<Vec<_>>();
let mut explicitly_enabled_connectors = collect_explicit_app_ids(&input);
codex-rs/core/src/tools/handlers/multi_agents_tests.rs
+1
View File
@@ -2083,6 +2083,7 @@ async fn send_input_accepts_structured_items() {
UserInput::Mention {
name: "drive".to_string(),
path: "app://google_drive".to_string(),
computer_use_native_app_bundle_id: None,
},
UserInput::Text {
text: "read the folder".to_string(),
codex-rs/core/tests/suite/plugins.rs
+2
View File
@@ -315,6 +315,7 @@ async fn explicit_plugin_mentions_inject_plugin_guidance() -> Result<()> {
items: vec![codex_protocol::user_input::UserInput::Mention {
name: "sample".into(),
path: format!("plugin://{SAMPLE_PLUGIN_CONFIG_NAME}"),
computer_use_native_app_bundle_id: None,
}],
final_output_json_schema: None,
responsesapi_client_metadata: None,
@@ -396,6 +397,7 @@ async fn explicit_plugin_mentions_track_plugin_used_analytics() -> Result<()> {
items: vec![codex_protocol::user_input::UserInput::Mention {
name: "sample".into(),
path: format!("plugin://{SAMPLE_PLUGIN_CONFIG_NAME}"),
computer_use_native_app_bundle_id: None,
}],
final_output_json_schema: None,
responsesapi_client_metadata: None,
codex-rs/protocol/src/user_input.rs
+8 -1
View File
@@ -36,7 +36,14 @@ pub enum UserInput {
///
/// `path` identifies the exact mention target, for example
/// `app://<connector-id>` or `plugin://<plugin-name>@<marketplace-name>`.
Mention { name: String, path: String },
Mention {
name: String,
path: String,
/// Present only when a plugin mention replaced a matching native app
/// mention in the UI and Computer Use remains a viable fallback.
#[serde(default, skip_serializing_if = "Option::is_none")]
computer_use_native_app_bundle_id: Option<String>,
},
}
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, TS, JsonSchema)]
codex-rs/tui/src/chatwidget.rs
+3
View File
@@ -5758,6 +5758,7 @@ impl ChatWidget {
items.push(UserInput::Mention {
name: plugin.display_name.clone(),
path: binding.path.clone(),
computer_use_native_app_bundle_id: None,
});
}
}
@@ -5780,6 +5781,7 @@ impl ChatWidget {
items.push(UserInput::Mention {
name: app.name.clone(),
path: binding.path.clone(),
computer_use_native_app_bundle_id: None,
});
}
}
@@ -5794,6 +5796,7 @@ impl ChatWidget {
items.push(UserInput::Mention {
name: app.name.clone(),
path: format!("app://{app_id}"),
computer_use_native_app_bundle_id: None,
});
}
}
codex-rs/tui/src/chatwidget/tests/composer_submission.rs
+1
View File
@@ -1160,6 +1160,7 @@ fn user_message_display_from_inputs_matches_flattened_user_message_shape() {
UserInput::Mention {
name: "repo".to_string(),
path: "app://repo".to_string(),
computer_use_native_app_bundle_id: None,
},
UserInput::Text {
text: "world".to_string(),
codex-rs/tui/src/chatwidget/tests/plan_mode.rs
+1
View File
@@ -1256,6 +1256,7 @@ async fn submit_user_message_emits_structured_plugin_mentions_from_bindings() {
UserInput::Mention {
name: "Sample Plugin".to_string(),
path: "plugin://sample@test".to_string(),
computer_use_native_app_bundle_id: None,
},
]
);