diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 7ab25a943f..b813440d46 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -6023,6 +6023,8 @@ pub enum UserInput { Mention { name: String, path: String, + #[serde(default)] + computer_use_native_app_bundle_id: Option, }, } @@ -6039,7 +6041,15 @@ impl UserInput { UserInput::Image { url } => CoreUserInput::Image { image_url: url }, UserInput::LocalImage { path } => CoreUserInput::LocalImage { path }, UserInput::Skill { name, path } => CoreUserInput::Skill { name, path }, - UserInput::Mention { name, path } => CoreUserInput::Mention { name, path }, + UserInput::Mention { + name, + path, + computer_use_native_app_bundle_id, + } => CoreUserInput::Mention { + name, + path, + computer_use_native_app_bundle_id, + }, } } } @@ -6057,7 +6067,15 @@ impl From for UserInput { CoreUserInput::Image { image_url } => UserInput::Image { url: image_url }, CoreUserInput::LocalImage { path } => UserInput::LocalImage { path }, CoreUserInput::Skill { name, path } => UserInput::Skill { name, path }, - CoreUserInput::Mention { name, path } => UserInput::Mention { name, path }, + CoreUserInput::Mention { + name, + path, + computer_use_native_app_bundle_id, + } => UserInput::Mention { + name, + path, + computer_use_native_app_bundle_id, + }, _ => unreachable!("unsupported user input variant"), } } @@ -10702,6 +10720,7 @@ mod tests { CoreUserInput::Mention { name: "Demo App".to_string(), path: "app://demo-app".to_string(), + computer_use_native_app_bundle_id: None, }, ], }); @@ -10728,6 +10747,7 @@ mod tests { UserInput::Mention { name: "Demo App".to_string(), path: "app://demo-app".to_string(), + computer_use_native_app_bundle_id: None, }, ], } diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index 3c5bbd3b61..7636cbfa61 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -570,6 +570,7 @@ async fn turn_start_accepts_text_at_limit_with_mention_item() -> Result<()> { V2UserInput::Mention { name: "Demo App".to_string(), path: "app://demo-app".to_string(), + computer_use_native_app_bundle_id: None, }, ], ..Default::default() diff --git a/codex-rs/core/src/agent/control.rs b/codex-rs/core/src/agent/control.rs index 1a13c0b00c..69609434c1 100644 --- a/codex-rs/core/src/agent/control.rs +++ b/codex-rs/core/src/agent/control.rs @@ -1223,7 +1223,9 @@ pub(crate) fn render_input_preview(initial_operation: &Op) -> String { UserInput::Image { .. } => "[image]".to_string(), UserInput::LocalImage { path } => format!("[local_image:{}]", path.display()), UserInput::Skill { name, path } => format!("[skill:${name}]({})", path.display()), - UserInput::Mention { name, path } => format!("[mention:${name}]({path})"), + UserInput::Mention { name, path, .. } => { + format!("[mention:${name}]({path})") + } _ => "[input]".to_string(), }) .collect::>() diff --git a/codex-rs/core/src/plugins/injection.rs b/codex-rs/core/src/plugins/injection.rs index 4ce50631fb..5c00352648 100644 --- a/codex-rs/core/src/plugins/injection.rs +++ b/codex-rs/core/src/plugins/injection.rs @@ -7,13 +7,13 @@ use codex_protocol::models::ResponseItem; use crate::connectors; use crate::context::ContextualUserFragment; use crate::context::PluginInstructions; -use crate::plugins::PluginCapabilitySummary; +use crate::plugins::ExplicitPluginMention; use crate::plugins::render_explicit_plugin_instructions; use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; use codex_mcp::ToolInfo; pub(crate) fn build_plugin_injections( - mentioned_plugins: &[PluginCapabilitySummary], + mentioned_plugins: &[ExplicitPluginMention], mcp_tools: &HashMap, available_connectors: &[connectors::AppInfo], ) -> Vec { @@ -25,7 +25,8 @@ pub(crate) fn build_plugin_injections( // model at the plugin's visible MCP servers, enabled apps, and skill prefix. mentioned_plugins .iter() - .filter_map(|plugin| { + .filter_map(|mention| { + let plugin = &mention.plugin; let available_mcp_servers = mcp_tools .values() .filter(|tool| { @@ -52,9 +53,14 @@ pub(crate) fn build_plugin_injections( .collect::>() .into_iter() .collect::>(); - render_explicit_plugin_instructions(plugin, &available_mcp_servers, &available_apps) - .map(PluginInstructions::new) - .map(ContextualUserFragment::into) + render_explicit_plugin_instructions( + plugin, + &available_mcp_servers, + &available_apps, + mention.has_computer_use_native_fallback, + ) + .map(PluginInstructions::new) + .map(ContextualUserFragment::into) }) .collect() } diff --git a/codex-rs/core/src/plugins/mentions.rs b/codex-rs/core/src/plugins/mentions.rs index a5e94345cb..41c0d00ba3 100644 --- a/codex-rs/core/src/plugins/mentions.rs +++ b/codex-rs/core/src/plugins/mentions.rs @@ -15,11 +15,25 @@ use crate::mention_syntax::TOOL_MENTION_SIGIL; use super::PluginCapabilitySummary; +const COMPUTER_USE_PLUGIN_CONFIG_NAME: &str = "computer-use@openai-bundled"; + pub(crate) struct CollectedToolMentions { pub(crate) plain_names: HashSet, pub(crate) paths: HashSet, } +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct ExplicitPluginMention { + pub(crate) plugin: PluginCapabilitySummary, + pub(crate) has_computer_use_native_fallback: bool, +} + +#[derive(Clone, Debug)] +struct StructuredPluginMention { + config_name: String, + has_computer_use_native_fallback: bool, +} + pub(crate) fn collect_tool_mentions_from_messages(messages: &[String]) -> CollectedToolMentions { collect_tool_mentions_from_messages_with_sigil(messages, TOOL_MENTION_SIGIL) } @@ -63,7 +77,7 @@ pub(crate) fn collect_explicit_app_ids(input: &[UserInput]) -> HashSet { pub(crate) fn collect_explicit_plugin_mentions( input: &[UserInput], plugins: &[PluginCapabilitySummary], -) -> Vec { +) -> Vec { if plugins.is_empty() { return Vec::new(); } @@ -76,30 +90,77 @@ pub(crate) fn collect_explicit_plugin_mentions( }) .collect::>(); - let mentioned_config_names: HashSet = input + let structured_mentions = input .iter() .filter_map(|item| match item { - UserInput::Mention { path, .. } => Some(path.clone()), + UserInput::Mention { + path, + computer_use_native_app_bundle_id, + .. + } => Some((path.clone(), computer_use_native_app_bundle_id.clone())), _ => None, }) - .chain( - // Plugin plaintext links use `@`, not the default `$` tool sigil. - collect_tool_mentions_from_messages_with_sigil(&messages, PLUGIN_TEXT_MENTION_SIGIL) - .paths, - ) - .filter(|path| tool_kind_for_path(path.as_str()) == ToolMentionKind::Plugin) - .filter_map(|path| plugin_config_name_from_path(path.as_str()).map(str::to_string)) - .collect(); + .filter(|(path, _)| tool_kind_for_path(path.as_str()) == ToolMentionKind::Plugin) + .filter_map(|(path, native_app_bundle_id)| { + plugin_config_name_from_path(path.as_str()).map(|config_name| StructuredPluginMention { + config_name: config_name.to_string(), + has_computer_use_native_fallback: native_app_bundle_id + .as_deref() + .is_some_and(|bundle_id| !bundle_id.trim().is_empty()), + }) + }) + .collect::>(); + + let linked_config_names = + collect_tool_mentions_from_messages_with_sigil(&messages, PLUGIN_TEXT_MENTION_SIGIL) + .paths + .into_iter() + .filter(|path| tool_kind_for_path(path.as_str()) == ToolMentionKind::Plugin) + .filter_map(|path| plugin_config_name_from_path(path.as_str()).map(str::to_string)) + .collect::>(); + + let mentioned_config_names = structured_mentions + .iter() + .map(|mention| mention.config_name.clone()) + .chain(linked_config_names) + .collect::>(); if mentioned_config_names.is_empty() { return Vec::new(); } - plugins + let mut mentioned_plugins = plugins .iter() .filter(|plugin| mentioned_config_names.contains(plugin.config_name.as_str())) - .cloned() - .collect() + .map(|plugin| { + let has_computer_use_native_fallback = structured_mentions.iter().any(|mention| { + mention.config_name == plugin.config_name + && mention.has_computer_use_native_fallback + }); + ExplicitPluginMention { + plugin: plugin.clone(), + has_computer_use_native_fallback, + } + }) + .collect::>(); + + if mentioned_plugins + .iter() + .any(|mention| mention.has_computer_use_native_fallback) + && !mentioned_plugins + .iter() + .any(|mention| mention.plugin.config_name.as_str() == COMPUTER_USE_PLUGIN_CONFIG_NAME) + && let Some(computer_use_plugin) = plugins + .iter() + .find(|plugin| plugin.config_name.as_str() == COMPUTER_USE_PLUGIN_CONFIG_NAME) + { + mentioned_plugins.push(ExplicitPluginMention { + plugin: computer_use_plugin.clone(), + has_computer_use_native_fallback: false, + }); + } + + mentioned_plugins } pub(crate) use crate::build_skill_name_counts; diff --git a/codex-rs/core/src/plugins/mentions_tests.rs b/codex-rs/core/src/plugins/mentions_tests.rs index 37c9adb886..e11eac9dd4 100644 --- a/codex-rs/core/src/plugins/mentions_tests.rs +++ b/codex-rs/core/src/plugins/mentions_tests.rs @@ -3,6 +3,7 @@ use std::collections::HashSet; use codex_protocol::user_input::UserInput; use pretty_assertions::assert_eq; +use super::ExplicitPluginMention; use super::collect_explicit_app_ids; use super::collect_explicit_plugin_mentions; use crate::plugins::PluginCapabilitySummary; @@ -25,6 +26,16 @@ fn plugin(config_name: &str, display_name: &str) -> PluginCapabilitySummary { } } +fn explicit_plugin_mention( + plugin: PluginCapabilitySummary, + has_computer_use_native_fallback: bool, +) -> ExplicitPluginMention { + ExplicitPluginMention { + plugin, + has_computer_use_native_fallback, + } +} + #[test] fn collect_explicit_app_ids_from_linked_text_mentions() { let input = vec![text_input("use [$calendar](app://calendar)")]; @@ -41,6 +52,7 @@ fn collect_explicit_app_ids_dedupes_structured_and_linked_mentions() { UserInput::Mention { name: "calendar".to_string(), path: "app://calendar".to_string(), + computer_use_native_app_bundle_id: None, }, ]; @@ -58,14 +70,17 @@ fn collect_explicit_app_ids_ignores_non_app_paths() { UserInput::Mention { name: "docs".to_string(), path: "mcp://docs".to_string(), + computer_use_native_app_bundle_id: None, }, UserInput::Mention { name: "skill".to_string(), path: "skill://team/skill".to_string(), + computer_use_native_app_bundle_id: None, }, UserInput::Mention { name: "file".to_string(), path: "/tmp/file.txt".to_string(), + computer_use_native_app_bundle_id: None, }, ]; @@ -85,11 +100,18 @@ fn collect_explicit_plugin_mentions_from_structured_paths() { &[UserInput::Mention { name: "sample".to_string(), path: "plugin://sample@test".to_string(), + computer_use_native_app_bundle_id: None, }], &plugins, ); - assert_eq!(mentioned, vec![plugin("sample@test", "sample")]); + assert_eq!( + mentioned, + vec![explicit_plugin_mention( + plugin("sample@test", "sample"), + false, + )] + ); } #[test] @@ -104,7 +126,13 @@ fn collect_explicit_plugin_mentions_from_linked_text_mentions() { &plugins, ); - assert_eq!(mentioned, vec![plugin("sample@test", "sample")]); + assert_eq!( + mentioned, + vec![explicit_plugin_mention( + plugin("sample@test", "sample"), + false, + )] + ); } #[test] @@ -120,12 +148,19 @@ fn collect_explicit_plugin_mentions_dedupes_structured_and_linked_mentions() { UserInput::Mention { name: "sample".to_string(), path: "plugin://sample@test".to_string(), + computer_use_native_app_bundle_id: None, }, ], &plugins, ); - assert_eq!(mentioned, vec![plugin("sample@test", "sample")]); + assert_eq!( + mentioned, + vec![explicit_plugin_mention( + plugin("sample@test", "sample"), + false, + )] + ); } #[test] @@ -139,7 +174,7 @@ fn collect_explicit_plugin_mentions_ignores_non_plugin_paths() { &plugins, ); - assert_eq!(mentioned, Vec::::new()); + assert_eq!(mentioned, Vec::::new()); } #[test] @@ -151,5 +186,59 @@ fn collect_explicit_plugin_mentions_ignores_dollar_linked_plugin_mentions() { &plugins, ); - assert_eq!(mentioned, Vec::::new()); + assert_eq!(mentioned, Vec::::new()); +} + +#[test] +fn collect_explicit_plugin_mentions_marks_structured_native_fallbacks() { + let zoom = plugin("zoom@test", "Zoom"); + let mentioned = collect_explicit_plugin_mentions( + &[UserInput::Mention { + name: "Zoom".to_string(), + path: "plugin://zoom@test".to_string(), + computer_use_native_app_bundle_id: Some("us.zoom.xos".to_string()), + }], + &[zoom.clone()], + ); + + assert_eq!(mentioned, vec![explicit_plugin_mention(zoom, true)]); +} + +#[test] +fn collect_explicit_plugin_mentions_preserves_computer_use_context_for_native_fallbacks() { + let zoom = plugin("zoom@test", "Zoom"); + let computer_use = plugin("computer-use@openai-bundled", "Computer Use"); + + let mentioned = collect_explicit_plugin_mentions( + &[UserInput::Mention { + name: "Zoom".to_string(), + path: "plugin://zoom@test".to_string(), + computer_use_native_app_bundle_id: Some("us.zoom.xos".to_string()), + }], + &[zoom.clone(), computer_use.clone()], + ); + + assert_eq!( + mentioned, + vec![ + explicit_plugin_mention(zoom, true), + explicit_plugin_mention(computer_use, false), + ] + ); +} + +#[test] +fn collect_explicit_plugin_mentions_ignores_empty_native_fallback_markers() { + let zoom = plugin("zoom@test", "Zoom"); + + let mentioned = collect_explicit_plugin_mentions( + &[UserInput::Mention { + name: "Zoom".to_string(), + path: "plugin://zoom@test".to_string(), + computer_use_native_app_bundle_id: Some(" ".to_string()), + }], + &[zoom.clone()], + ); + + assert_eq!(mentioned, vec![explicit_plugin_mention(zoom, false)]); } diff --git a/codex-rs/core/src/plugins/mod.rs b/codex-rs/core/src/plugins/mod.rs index 11e2c338bc..63612ddf4b 100644 --- a/codex-rs/core/src/plugins/mod.rs +++ b/codex-rs/core/src/plugins/mod.rs @@ -11,6 +11,7 @@ pub(crate) use discoverable::list_tool_suggest_discoverable_plugins; pub(crate) use injection::build_plugin_injections; pub(crate) use render::render_explicit_plugin_instructions; +pub(crate) use mentions::ExplicitPluginMention; pub(crate) use mentions::build_connector_slug_counts; pub(crate) use mentions::build_skill_name_counts; pub(crate) use mentions::collect_explicit_app_ids; diff --git a/codex-rs/core/src/plugins/render.rs b/codex-rs/core/src/plugins/render.rs index fc197dbbea..e59e9178f5 100644 --- a/codex-rs/core/src/plugins/render.rs +++ b/codex-rs/core/src/plugins/render.rs @@ -13,6 +13,7 @@ pub(crate) fn render_explicit_plugin_instructions( plugin: &PluginCapabilitySummary, available_mcp_servers: &[String], available_apps: &[String], + has_computer_use_native_fallback: bool, ) -> Option { let mut lines = vec![format!( "Capabilities from the `{}` plugin:", @@ -48,6 +49,13 @@ pub(crate) fn render_explicit_plugin_instructions( )); } + if has_computer_use_native_fallback { + lines.push( + "- This plugin also corresponds to a native desktop app available through Computer Use. Prefer plugin-associated capabilities first; if they are unavailable, insufficient, or fail, use the Computer Use tool surface for the native app fallback. If Computer Use tools are not already visible, use `tool_search` for `computer use` to discover them." + .to_string(), + ); + } + if lines.len() == 1 { return None; } diff --git a/codex-rs/core/src/plugins/render_tests.rs b/codex-rs/core/src/plugins/render_tests.rs index a0ec531209..f9fc00819e 100644 --- a/codex-rs/core/src/plugins/render_tests.rs +++ b/codex-rs/core/src/plugins/render_tests.rs @@ -21,3 +21,42 @@ fn render_plugins_section_includes_descriptions_and_skill_naming_guidance() { assert_eq!(rendered, expected); } + +#[test] +fn render_explicit_plugin_instructions_mentions_native_app_fallback() { + let rendered = render_explicit_plugin_instructions( + &PluginCapabilitySummary { + config_name: "zoom@test".to_string(), + display_name: "Zoom".to_string(), + ..PluginCapabilitySummary::default() + }, + &[], + &[], + true, + ) + .expect("native app fallback should render instructions"); + + let expected = "Capabilities from the `Zoom` plugin:\n- This plugin also corresponds to a native desktop app available through Computer Use. Prefer plugin-associated capabilities first; if they are unavailable, insufficient, or fail, use the Computer Use tool surface for the native app fallback. If Computer Use tools are not already visible, use `tool_search` for `computer use` to discover them.\nUse these plugin-associated capabilities to help solve the task."; + + assert_eq!(rendered, expected); +} + +#[test] +fn render_explicit_plugin_instructions_omits_native_app_fallback_without_deduped_match() { + let rendered = render_explicit_plugin_instructions( + &PluginCapabilitySummary { + config_name: "zoom@test".to_string(), + display_name: "Zoom".to_string(), + has_skills: true, + ..PluginCapabilitySummary::default() + }, + &[], + &[], + false, + ) + .expect("skill guidance should still render instructions"); + + let expected = "Capabilities from the `Zoom` plugin:\n- Skills from this plugin are prefixed with `Zoom:`.\nUse these plugin-associated capabilities to help solve the task."; + + assert_eq!(rendered, expected); +} diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index ce035b2cea..f723ba1921 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -276,7 +276,7 @@ pub(crate) async fn run_turn( build_plugin_injections(&mentioned_plugins, &mcp_tools, &available_connectors); let mentioned_plugin_metadata = mentioned_plugins .iter() - .filter_map(crate::plugins::PluginCapabilitySummary::telemetry_metadata) + .filter_map(|mention| mention.plugin.telemetry_metadata()) .collect::>(); let mut explicitly_enabled_connectors = collect_explicit_app_ids(&input); diff --git a/codex-rs/core/src/tools/handlers/multi_agents_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_tests.rs index 88c1698d8f..1f246dc018 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_tests.rs @@ -2083,6 +2083,7 @@ async fn send_input_accepts_structured_items() { UserInput::Mention { name: "drive".to_string(), path: "app://google_drive".to_string(), + computer_use_native_app_bundle_id: None, }, UserInput::Text { text: "read the folder".to_string(), diff --git a/codex-rs/core/tests/suite/plugins.rs b/codex-rs/core/tests/suite/plugins.rs index 5b83d3b136..c37fd8e378 100644 --- a/codex-rs/core/tests/suite/plugins.rs +++ b/codex-rs/core/tests/suite/plugins.rs @@ -315,6 +315,7 @@ async fn explicit_plugin_mentions_inject_plugin_guidance() -> Result<()> { items: vec![codex_protocol::user_input::UserInput::Mention { name: "sample".into(), path: format!("plugin://{SAMPLE_PLUGIN_CONFIG_NAME}"), + computer_use_native_app_bundle_id: None, }], final_output_json_schema: None, responsesapi_client_metadata: None, @@ -396,6 +397,7 @@ async fn explicit_plugin_mentions_track_plugin_used_analytics() -> Result<()> { items: vec![codex_protocol::user_input::UserInput::Mention { name: "sample".into(), path: format!("plugin://{SAMPLE_PLUGIN_CONFIG_NAME}"), + computer_use_native_app_bundle_id: None, }], final_output_json_schema: None, responsesapi_client_metadata: None, diff --git a/codex-rs/protocol/src/user_input.rs b/codex-rs/protocol/src/user_input.rs index 4ed112df8d..cafb030e5b 100644 --- a/codex-rs/protocol/src/user_input.rs +++ b/codex-rs/protocol/src/user_input.rs @@ -36,7 +36,14 @@ pub enum UserInput { /// /// `path` identifies the exact mention target, for example /// `app://` or `plugin://@`. - Mention { name: String, path: String }, + Mention { + name: String, + path: String, + /// Present only when a plugin mention replaced a matching native app + /// mention in the UI and Computer Use remains a viable fallback. + #[serde(default, skip_serializing_if = "Option::is_none")] + computer_use_native_app_bundle_id: Option, + }, } #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, TS, JsonSchema)] diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 2b6166ff2d..b88d961ec4 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -5758,6 +5758,7 @@ impl ChatWidget { items.push(UserInput::Mention { name: plugin.display_name.clone(), path: binding.path.clone(), + computer_use_native_app_bundle_id: None, }); } } @@ -5780,6 +5781,7 @@ impl ChatWidget { items.push(UserInput::Mention { name: app.name.clone(), path: binding.path.clone(), + computer_use_native_app_bundle_id: None, }); } } @@ -5794,6 +5796,7 @@ impl ChatWidget { items.push(UserInput::Mention { name: app.name.clone(), path: format!("app://{app_id}"), + computer_use_native_app_bundle_id: None, }); } } diff --git a/codex-rs/tui/src/chatwidget/tests/composer_submission.rs b/codex-rs/tui/src/chatwidget/tests/composer_submission.rs index 27c39f05fb..dfab8a2cad 100644 --- a/codex-rs/tui/src/chatwidget/tests/composer_submission.rs +++ b/codex-rs/tui/src/chatwidget/tests/composer_submission.rs @@ -1160,6 +1160,7 @@ fn user_message_display_from_inputs_matches_flattened_user_message_shape() { UserInput::Mention { name: "repo".to_string(), path: "app://repo".to_string(), + computer_use_native_app_bundle_id: None, }, UserInput::Text { text: "world".to_string(), diff --git a/codex-rs/tui/src/chatwidget/tests/plan_mode.rs b/codex-rs/tui/src/chatwidget/tests/plan_mode.rs index 97cc8d7fc4..d74192e594 100644 --- a/codex-rs/tui/src/chatwidget/tests/plan_mode.rs +++ b/codex-rs/tui/src/chatwidget/tests/plan_mode.rs @@ -1256,6 +1256,7 @@ async fn submit_user_message_emits_structured_plugin_mentions_from_bindings() { UserInput::Mention { name: "Sample Plugin".to_string(), path: "plugin://sample@test".to_string(), + computer_use_native_app_bundle_id: None, }, ] );