diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index 36916534c6..7b878e952c 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -66,6 +66,7 @@ use crate::facts::PluginInstallRequestSource; use crate::facts::PluginInstallRequested; use crate::facts::PluginInstallRequestedInput; use crate::facts::PluginInstallRequestedPlugin; +use crate::facts::PluginInstallSource; use crate::facts::PluginState; use crate::facts::PluginStateChangedInput; use crate::facts::PluginUsedInput; @@ -3242,6 +3243,7 @@ fn plugin_install_failed_event_serializes_expected_shape() { event_type: "codex_plugin_install_failed", event_params: CodexPluginInstallFailedMetadata { plugin: codex_plugin_metadata(sample_plugin_metadata()), + source: PluginInstallSource::Manual, error_type: "store_io".to_string(), }, }); @@ -3261,6 +3263,7 @@ fn plugin_install_failed_event_serializes_expected_shape() { "mcp_server_count": 2, "connector_ids": ["calendar", "drive"], "product_client_id": originator().value, + "source": "manual", "error_type": "store_io" } }) @@ -3692,6 +3695,7 @@ async fn reducer_ingests_plugin_install_failed_fact() { AnalyticsFact::Custom(CustomAnalyticsFact::PluginInstallFailed( PluginInstallFailedInput { plugin: sample_plugin_metadata(), + source: PluginInstallSource::ExternalAgentMigration, error_type: "invalid_plugin".to_string(), }, )), @@ -3713,6 +3717,7 @@ async fn reducer_ingests_plugin_install_failed_fact() { "mcp_server_count": 2, "connector_ids": ["calendar", "drive"], "product_client_id": originator().value, + "source": "external_agent_migration", "error_type": "invalid_plugin" } }]) @@ -3734,6 +3739,7 @@ async fn reducer_ingests_plugin_install_failed_fact_without_detail() { AnalyticsFact::Custom(CustomAnalyticsFact::PluginInstallFailed( PluginInstallFailedInput { plugin, + source: PluginInstallSource::Manual, error_type: "remote_catalog_unexpected_status".to_string(), }, )), @@ -3755,6 +3761,7 @@ async fn reducer_ingests_plugin_install_failed_fact_without_detail() { "mcp_server_count": null, "connector_ids": null, "product_client_id": originator().value, + "source": "manual", "error_type": "remote_catalog_unexpected_status" } }]) diff --git a/codex-rs/analytics/src/client.rs b/codex-rs/analytics/src/client.rs index f957a665a0..1fd77b0560 100644 --- a/codex-rs/analytics/src/client.rs +++ b/codex-rs/analytics/src/client.rs @@ -18,6 +18,7 @@ use crate::facts::HookRunInput; use crate::facts::PluginInstallFailedInput; use crate::facts::PluginInstallRequested; use crate::facts::PluginInstallRequestedInput; +use crate::facts::PluginInstallSource; use crate::facts::PluginState; use crate::facts::PluginStateChangedInput; use crate::facts::SkillInvocation; @@ -370,10 +371,16 @@ impl AnalyticsEventsClient { )); } - pub fn track_plugin_install_failed(&self, plugin: PluginTelemetryMetadata, error_type: String) { + pub fn track_plugin_install_failed( + &self, + plugin: PluginTelemetryMetadata, + source: PluginInstallSource, + error_type: String, + ) { self.record_fact(AnalyticsFact::Custom( CustomAnalyticsFact::PluginInstallFailed(PluginInstallFailedInput { plugin, + source, error_type, }), )); diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index ad8adf14cb..999ae3e1f0 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -991,6 +991,7 @@ pub(crate) struct CodexPluginEventRequest { pub(crate) struct CodexPluginInstallFailedMetadata { #[serde(flatten)] pub(crate) plugin: CodexPluginMetadata, + pub(crate) source: crate::facts::PluginInstallSource, pub(crate) error_type: String, } diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index 3346a6e19a..1f3333b8bc 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -579,8 +579,16 @@ pub(crate) struct PluginStateChangedInput { pub state: PluginState, } +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum PluginInstallSource { + Manual, + ExternalAgentMigration, +} + pub(crate) struct PluginInstallFailedInput { pub plugin: PluginTelemetryMetadata, + pub source: PluginInstallSource, pub error_type: String, } diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs index 86bc9d6829..d178d8d5bf 100644 --- a/codex-rs/analytics/src/lib.rs +++ b/codex-rs/analytics/src/lib.rs @@ -45,6 +45,7 @@ pub use facts::InvocationType; pub use facts::PluginInstallRequestSource; pub use facts::PluginInstallRequested; pub use facts::PluginInstallRequestedPlugin; +pub use facts::PluginInstallSource; pub use facts::SkillInvocation; pub use facts::SubAgentThreadStartedInput; pub use facts::ThreadInitializationMode; diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index 79649a553b..13696d670f 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -837,12 +837,17 @@ impl AnalyticsReducer { input: PluginInstallFailedInput, out: &mut Vec, ) { - let PluginInstallFailedInput { plugin, error_type } = input; + let PluginInstallFailedInput { + plugin, + source, + error_type, + } = input; out.push(TrackEventRequest::PluginInstallFailed( CodexPluginInstallFailedEventRequest { event_type: "codex_plugin_install_failed", event_params: CodexPluginInstallFailedMetadata { plugin: codex_plugin_metadata(plugin), + source, error_type, }, }, diff --git a/codex-rs/app-server/src/config/external_agent_config.rs b/codex-rs/app-server/src/config/external_agent_config.rs index ca1023e529..fa092cfb3c 100644 --- a/codex-rs/app-server/src/config/external_agent_config.rs +++ b/codex-rs/app-server/src/config/external_agent_config.rs @@ -1,8 +1,12 @@ +use codex_analytics::AnalyticsEventsClient; +use codex_analytics::PluginInstallSource; use codex_config::types::PluginConfig; use codex_core::config::Config; use codex_core::config::ConfigBuilder; +use codex_core_plugins::PluginInstallError; use codex_core_plugins::PluginInstallRequest; use codex_core_plugins::PluginsManager; +use codex_core_plugins::marketplace::MarketplaceError; use codex_core_plugins::marketplace::MarketplacePluginInstallPolicy; use codex_core_plugins::marketplace::find_marketplace_manifest_path; use codex_core_plugins::marketplace_add::MarketplaceAddRequest; @@ -36,6 +40,7 @@ const EXTERNAL_AGENT_CONFIG_DETECT_METRIC: &str = "codex.external_agent_config.d const EXTERNAL_AGENT_CONFIG_IMPORT_METRIC: &str = "codex.external_agent_config.import"; const EXTERNAL_AGENT_DIR: &str = ".claude"; const EXTERNAL_AGENT_CONFIG_MD: &str = "CLAUDE.md"; +const EXTERNAL_AGENT_KNOWN_MARKETPLACES_PATH: &str = "plugins/known_marketplaces.json"; const EXTERNAL_OFFICIAL_MARKETPLACE_NAME: &str = "claude-plugins-official"; const EXTERNAL_OFFICIAL_MARKETPLACE_SOURCE: &str = "anthropics/claude-plugins-official"; @@ -176,14 +181,16 @@ pub(crate) struct ExternalAgentConfigMigrationItem { pub(crate) struct ExternalAgentConfigService { codex_home: PathBuf, external_agent_home: PathBuf, + analytics_events_client: Option, } impl ExternalAgentConfigService { - pub(crate) fn new(codex_home: PathBuf) -> Self { + pub(crate) fn new(codex_home: PathBuf, analytics_events_client: AnalyticsEventsClient) -> Self { let external_agent_home = default_external_agent_home(); Self { codex_home, external_agent_home, + analytics_events_client: Some(analytics_events_client), } } @@ -192,6 +199,7 @@ impl ExternalAgentConfigService { Self { codex_home, external_agent_home, + analytics_events_client: None, } } @@ -805,9 +813,10 @@ impl ExternalAgentConfigService { configured_plugin_ids: &HashSet, configured_marketplace_plugins: &BTreeMap>, ) -> Option { + let import_sources = self.marketplace_import_sources(settings, source_root); let plugin_details = extract_plugin_migration_details( settings, - source_root, + &import_sources, configured_plugin_ids, configured_marketplace_plugins, )?; @@ -836,7 +845,7 @@ impl ExternalAgentConfigService { ); let source_root = cwd.unwrap_or(self.external_agent_home.as_path()); let import_sources = effective_external_settings(&source_settings)? - .map(|settings| collect_marketplace_import_sources(&settings, source_root)) + .map(|settings| self.marketplace_import_sources(&settings, source_root)) .unwrap_or_default(); let mut local_plugins = Vec::new(); @@ -893,7 +902,19 @@ impl ExternalAgentConfigService { .map_err(|err| io::Error::other(format!("failed to load config: {err}")))?; let requirements = config.config_layer_stack.requirements().clone(); let mut outcome = PluginImportOutcome::default(); - let plugins_manager = PluginsManager::new(self.codex_home.clone()); + let plugins_manager = PluginsManager::new(self.codex_home.clone()) + .with_plugin_install_source(PluginInstallSource::ExternalAgentMigration); + if let Some(analytics_events_client) = self.analytics_events_client.clone() { + plugins_manager.set_analytics_events_client(analytics_events_client); + } + let source_settings = cwd.map_or_else( + || self.external_agent_home.join("settings.json"), + |cwd| cwd.join(EXTERNAL_AGENT_DIR).join("settings.json"), + ); + let source_root = cwd.unwrap_or(self.external_agent_home.as_path()); + let import_sources = effective_external_settings(&source_settings)? + .map(|settings| self.marketplace_import_sources(&settings, source_root)) + .unwrap_or_default(); for plugin_group in plugins { let marketplace_name = plugin_group.marketplace_name.clone(); let plugin_names = plugin_group.plugin_names; @@ -901,16 +922,7 @@ impl ExternalAgentConfigService { .iter() .map(|plugin_name| format!("{plugin_name}@{marketplace_name}")) .collect::>(); - let source_settings = cwd.map_or_else( - || self.external_agent_home.join("settings.json"), - |cwd| cwd.join(EXTERNAL_AGENT_DIR).join("settings.json"), - ); - let source_root = cwd.unwrap_or(self.external_agent_home.as_path()); - let import_source = - effective_external_settings(&source_settings)?.and_then(|settings| { - collect_marketplace_import_sources(&settings, source_root) - .remove(&marketplace_name) - }); + let import_source = import_sources.get(&marketplace_name).cloned(); let Some(import_source) = import_source else { let message = format!( "external agent plugin marketplace source was not found: {marketplace_name}" @@ -1009,12 +1021,21 @@ impl ExternalAgentConfigService { Err(err) => { let plugin_id = format!("{plugin_name}@{marketplace_name}"); outcome.failed_plugin_ids.push(plugin_id.clone()); - outcome.raw_errors.push(plugin_import_raw_error( + let mut raw_error = plugin_import_raw_error( cwd, "plugin_import", err.to_string(), Some(plugin_id), - )); + ); + if matches!( + err, + PluginInstallError::Marketplace( + MarketplaceError::PluginNotFound { .. } + ) + ) { + raw_error.error_type = Some("plugin_not_found".to_string()); + } + outcome.raw_errors.push(raw_error); } } } @@ -1023,6 +1044,95 @@ impl ExternalAgentConfigService { Ok(outcome) } + fn marketplace_import_sources( + &self, + settings: &JsonValue, + source_root: &Path, + ) -> BTreeMap { + let known_marketplaces_path = self + .external_agent_home + .join(EXTERNAL_AGENT_KNOWN_MARKETPLACES_PATH); + let known_marketplaces = match read_external_settings(&known_marketplaces_path) { + Ok(known_marketplaces) => known_marketplaces, + Err(err) => { + tracing::warn!( + path = %known_marketplaces_path.display(), + error = %err, + "ignoring invalid external agent marketplace registry" + ); + None + } + }; + let mut import_sources = known_marketplaces + .as_ref() + .map(|known_marketplaces| { + collect_marketplace_import_sources( + known_marketplaces, + self.external_agent_home.as_path(), + ) + }) + .unwrap_or_default(); + + if let Some(extra_known_marketplaces) = settings + .as_object() + .and_then(|settings| settings.get("extraKnownMarketplaces")) + { + let mut scoped_marketplaces = extra_known_marketplaces.clone(); + if let Some(scoped_marketplaces) = scoped_marketplaces.as_object_mut() { + for (name, scoped_marketplace) in scoped_marketplaces { + import_sources.remove(name); + let Some(known_marketplace) = known_marketplaces + .as_ref() + .and_then(JsonValue::as_object) + .and_then(|known_marketplaces| known_marketplaces.get(name)) + else { + continue; + }; + if scoped_marketplace.get("source") != known_marketplace.get("source") { + continue; + } + let Some(install_location) = known_marketplace + .get("installLocation") + .and_then(JsonValue::as_str) + else { + continue; + }; + let install_location = Path::new(install_location); + let install_location = if install_location.is_absolute() { + install_location.to_path_buf() + } else { + self.external_agent_home.join(install_location) + }; + let Some(scoped_marketplace) = scoped_marketplace.as_object_mut() else { + continue; + }; + scoped_marketplace.insert( + "installLocation".to_string(), + JsonValue::String(install_location.display().to_string()), + ); + } + } + import_sources.extend(collect_marketplace_import_sources( + &scoped_marketplaces, + source_root, + )); + } + + if has_enabled_plugin_for_marketplace(settings, EXTERNAL_OFFICIAL_MARKETPLACE_NAME) + && !import_sources.contains_key(EXTERNAL_OFFICIAL_MARKETPLACE_NAME) + { + import_sources.insert( + EXTERNAL_OFFICIAL_MARKETPLACE_NAME.to_string(), + MarketplaceImportSource { + source: EXTERNAL_OFFICIAL_MARKETPLACE_SOURCE.to_string(), + ref_name: None, + }, + ); + } + + import_sources + } + fn import_config(&self, cwd: Option<&Path>) -> io::Result> { let repo_root = find_repo_root(cwd)?; let (source_settings, target_config) = if let Some(repo_root) = repo_root.as_ref() { @@ -1321,16 +1431,16 @@ fn merge_json_settings(existing: &mut JsonValue, incoming: &JsonValue) { } fn extract_plugin_migration_details( settings: &JsonValue, - source_root: &Path, + import_sources: &BTreeMap, configured_plugin_ids: &HashSet, configured_marketplace_plugins: &BTreeMap>, ) -> Option { - let loadable_marketplaces = collect_marketplace_import_sources(settings, source_root) - .into_iter() + let loadable_marketplaces = import_sources + .iter() .filter_map(|(marketplace_name, source)| { - is_local_marketplace_source(&source.source, source.ref_name) + is_local_marketplace_source(&source.source, source.ref_name.clone()) .ok() - .map(|_| marketplace_name) + .map(|_| marketplace_name.clone()) }) .collect::>(); let mut plugins = BTreeMap::new(); @@ -1345,9 +1455,19 @@ fn extract_plugin_migration_details( configured_marketplace_plugins.get(&plugin_id.marketplace_name) { if !installable_plugins.contains(&plugin_id.plugin_name) { + tracing::warn!( + plugin_id = %plugin_id.as_key(), + marketplace_name = %plugin_id.marketplace_name, + "enabled external agent plugin was not found in configured marketplace" + ); continue; } } else if !loadable_marketplaces.contains(&plugin_id.marketplace_name) { + tracing::warn!( + plugin_id = %plugin_id.as_key(), + marketplace_name = %plugin_id.marketplace_name, + "marketplace source was not found for enabled external agent plugin" + ); continue; } let plugin_group = plugins @@ -1444,13 +1564,11 @@ fn configured_marketplace_plugins( } fn collect_marketplace_import_sources( - settings: &JsonValue, + marketplaces: &JsonValue, source_root: &Path, ) -> BTreeMap { - let mut import_sources: BTreeMap = settings + marketplaces .as_object() - .and_then(|settings| settings.get("extraKnownMarketplaces")) - .and_then(JsonValue::as_object) .map(|extra_known_marketplaces| { extra_known_marketplaces .iter() @@ -1462,46 +1580,69 @@ fn collect_marketplace_import_sources( } else { value.as_object()? }; - let source = source_fields - .get("repo") - .or_else(|| source_fields.get("url")) - .or_else(|| source_fields.get("path")) - .or_else(|| value.get("source"))? - .as_str()? - .trim() - .to_string(); - if source.is_empty() { - return None; + let source_kind = source_fields + .get("source") + .and_then(JsonValue::as_str) + .map(str::trim); + let declared_source = match source_kind { + Some("github") => source_fields.get("repo"), + Some("git") => source_fields.get("url"), + Some("directory" | "local") => source_fields.get("path"), + Some("file" | "url" | "npm" | "settings") => None, + Some(_) => source_fields.get("source"), + None => source_fields + .get("repo") + .or_else(|| source_fields.get("url")) + .or_else(|| source_fields.get("path")) + .or_else(|| value.get("source")), } - let source = resolve_external_marketplace_source(&source, source_root); - - let ref_name = source_fields - .get("ref") - .or_else(|| value.get("ref")) + .and_then(JsonValue::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()); + let materialized_source = value + .get("installLocation") .and_then(JsonValue::as_str) .map(str::trim) .filter(|value| !value.is_empty()) - .map(ToOwned::to_owned); + .and_then(|value| { + let path = Path::new(value); + let path = if path.is_absolute() { + path.to_path_buf() + } else { + source_root.join(path) + }; + path.is_dir().then(|| path.display().to_string()) + }); + let (source, ref_name) = if let Some(source) = declared_source { + let source = if matches!(source_kind, Some("directory" | "local")) { + let path = Path::new(source); + if path.is_absolute() { + path.to_path_buf() + } else { + source_root.join(path) + } + .display() + .to_string() + } else { + resolve_external_marketplace_source(source, source_root) + }; + let ref_name = source_fields + .get("ref") + .or_else(|| value.get("ref")) + .and_then(JsonValue::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned); + (source, ref_name) + } else { + (materialized_source?, None) + }; Some((name.clone(), MarketplaceImportSource { source, ref_name })) }) .collect() }) - .unwrap_or_default(); - - if has_enabled_plugin_for_marketplace(settings, EXTERNAL_OFFICIAL_MARKETPLACE_NAME) - && !import_sources.contains_key(EXTERNAL_OFFICIAL_MARKETPLACE_NAME) - { - import_sources.insert( - EXTERNAL_OFFICIAL_MARKETPLACE_NAME.to_string(), - MarketplaceImportSource { - source: EXTERNAL_OFFICIAL_MARKETPLACE_SOURCE.to_string(), - ref_name: None, - }, - ); - } - - import_sources + .unwrap_or_default() } #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/codex-rs/app-server/src/config/external_agent_config_tests.rs b/codex-rs/app-server/src/config/external_agent_config_tests.rs index 9ab67a1379..9ecf0986ca 100644 --- a/codex-rs/app-server/src/config/external_agent_config_tests.rs +++ b/codex-rs/app-server/src/config/external_agent_config_tests.rs @@ -39,6 +39,7 @@ fn assert_single_plugin_raw_error( raw_errors: &[ExternalAgentConfigImportRawError], failure_stage: &str, source: &str, + error_type: Option<&str>, ) { assert_eq!(raw_errors.len(), 1); let raw_error = &raw_errors[0]; @@ -47,7 +48,7 @@ fn assert_single_plugin_raw_error( ExternalAgentConfigMigrationItemType::Plugins ); assert_eq!(raw_error.failure_stage, failure_stage); - assert_eq!(raw_error.error_type, None); + assert_eq!(raw_error.error_type.as_deref(), error_type); assert_eq!(raw_error.cwd, None); assert_eq!(raw_error.source.as_deref(), Some(source)); assert!(!raw_error.message.is_empty()); @@ -1742,6 +1743,169 @@ async fn detect_home_lists_enabled_plugins_from_settings() { ); } +#[tokio::test] +async fn detect_home_uses_materialized_known_marketplace_for_inline_npm_source() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + let marketplace_root = external_agent_home + .join("plugins") + .join("marketplaces") + .join("acme-tools"); + fs::create_dir_all(external_agent_home.join("plugins")) + .expect("create external agent plugins dir"); + fs::create_dir_all(&marketplace_root).expect("create installed marketplace dir"); + fs::write( + external_agent_home.join("settings.json"), + r#"{ + "enabledPlugins": { + "formatter@acme-tools": true + }, + "extraKnownMarketplaces": { + "acme-tools": { + "source": { + "source": "settings", + "name": "acme-tools", + "plugins": [{ + "name": "formatter", + "source": { + "source": "npm", + "package": "@acme/formatter" + } + }] + } + } + } + }"#, + ) + .expect("write settings"); + fs::write( + external_agent_home.join(EXTERNAL_AGENT_KNOWN_MARKETPLACES_PATH), + serde_json::to_string_pretty(&serde_json::json!({ + "acme-tools": { + "source": { + "source": "settings", + "name": "acme-tools", + "plugins": [{ + "name": "formatter", + "source": { + "source": "npm", + "package": "@acme/formatter", + }, + }], + }, + "installLocation": "plugins/marketplaces/acme-tools", + "lastUpdated": "2026-07-09T00:16:23.611Z", + } + })) + .expect("serialize known marketplaces"), + ) + .expect("write known marketplaces"); + + let items = service_for_paths(external_agent_home.clone(), codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: true, + cwds: None, + }) + .await + .expect("detect"); + + assert_eq!( + items, + vec![ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::Plugins, + description: format!( + "Migrate enabled plugins from {}", + external_agent_home.join("settings.json").display() + ), + cwd: None, + details: Some(MigrationDetails { + plugins: vec![PluginsMigration { + marketplace_name: "acme-tools".to_string(), + plugin_names: vec!["formatter".to_string()], + }], + ..Default::default() + }), + }] + ); +} + +#[test] +fn marketplace_import_sources_prefers_scoped_source_over_registry_name_collision() { + let (root, external_agent_home, codex_home) = fixture_paths(); + let source_root = root.path().join("repo"); + let scoped_marketplace = source_root.join("repo-marketplace"); + let cached_marketplace = external_agent_home.join("plugins/marketplaces/debug"); + fs::create_dir_all(&scoped_marketplace).expect("create scoped marketplace"); + fs::create_dir_all(&cached_marketplace).expect("create cached marketplace"); + fs::write( + external_agent_home.join(EXTERNAL_AGENT_KNOWN_MARKETPLACES_PATH), + serde_json::to_string_pretty(&serde_json::json!({ + "debug": { + "source": { + "source": "github", + "repo": "acme/global-marketplace", + }, + "installLocation": cached_marketplace, + } + })) + .expect("serialize known marketplaces"), + ) + .expect("write known marketplaces"); + let settings = serde_json::json!({ + "extraKnownMarketplaces": { + "debug": { + "source": { + "source": "directory", + "path": "./repo-marketplace", + } + } + } + }); + + let import_sources = service_for_paths(external_agent_home, codex_home) + .marketplace_import_sources(&settings, &source_root); + + assert_eq!( + import_sources.get("debug"), + Some(&MarketplaceImportSource { + source: source_root.join("./repo-marketplace").display().to_string(), + ref_name: None, + }) + ); +} + +#[test] +fn marketplace_import_sources_prefers_supported_declaration_over_materialization() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + let cached_marketplace = external_agent_home.join("plugins/marketplaces/acme-tools"); + fs::create_dir_all(&cached_marketplace).expect("create cached marketplace"); + fs::write( + external_agent_home.join(EXTERNAL_AGENT_KNOWN_MARKETPLACES_PATH), + serde_json::to_string_pretty(&serde_json::json!({ + "acme-tools": { + "source": { + "source": "git", + "url": "https://git.example.com/acme/tools.git", + "ref": "release", + }, + "installLocation": cached_marketplace, + } + })) + .expect("serialize known marketplaces"), + ) + .expect("write known marketplaces"); + + let import_sources = service_for_paths(external_agent_home.clone(), codex_home) + .marketplace_import_sources(&serde_json::json!({}), &external_agent_home); + + assert_eq!( + import_sources.get("acme-tools"), + Some(&MarketplaceImportSource { + source: "https://git.example.com/acme/tools.git".to_string(), + ref_name: Some("release".to_string()), + }) + ); +} + #[tokio::test] async fn detect_home_plugins_uses_local_settings_over_project_settings() { let (_root, external_agent_home, codex_home) = fixture_paths(); @@ -2255,6 +2419,7 @@ async fn import_plugins_requires_source_marketplace_details() { &outcome.raw_errors, "plugin_import", "formatter@other-tools", + /*error_type*/ None, ); } @@ -2290,7 +2455,12 @@ async fn import_plugins_defers_marketplace_source_validation_to_add_marketplace( outcome.failed_plugin_ids, vec!["formatter@acme-tools".to_string()] ); - assert_single_plugin_raw_error(&outcome.raw_errors, "plugin_import", "formatter@acme-tools"); + assert_single_plugin_raw_error( + &outcome.raw_errors, + "plugin_import", + "formatter@acme-tools", + /*error_type*/ None, + ); } #[tokio::test] @@ -2610,6 +2780,7 @@ async fn import_plugins_infers_external_official_marketplace_when_missing_from_s &outcome.raw_errors, "plugin_import", &format!("sample@{EXTERNAL_OFFICIAL_MARKETPLACE_NAME}"), + Some("plugin_not_found"), ); } diff --git a/codex-rs/app-server/src/request_processors/external_agent_config_processor.rs b/codex-rs/app-server/src/request_processors/external_agent_config_processor.rs index 5bae994ca5..c71abcb884 100644 --- a/codex-rs/app-server/src/request_processors/external_agent_config_processor.rs +++ b/codex-rs/app-server/src/request_processors/external_agent_config_processor.rs @@ -96,9 +96,11 @@ impl ExternalAgentConfigRequestProcessor { config_manager, arg0_paths, ); + let migration_service = + ExternalAgentConfigService::new(codex_home, analytics_events_client.clone()); Self { outgoing, - migration_service: ExternalAgentConfigService::new(codex_home), + migration_service, session_importer, thread_manager, config_processor, diff --git a/codex-rs/app-server/src/request_processors/plugins.rs b/codex-rs/app-server/src/request_processors/plugins.rs index ed8495d602..688cfae917 100644 --- a/codex-rs/app-server/src/request_processors/plugins.rs +++ b/codex-rs/app-server/src/request_processors/plugins.rs @@ -1,6 +1,7 @@ use super::*; use crate::error_code::internal_error; use crate::error_code::invalid_request; +use codex_analytics::PluginInstallSource; use codex_app_server_protocol::PluginAvailability; use codex_app_server_protocol::PluginInstallPolicy; use codex_app_server_protocol::PluginSharePrincipalRole; @@ -1760,8 +1761,11 @@ impl PluginRequestProcessor { capability_summary: None, } }; - self.analytics_events_client - .track_plugin_install_failed(plugin, error_type.to_string()); + self.analytics_events_client.track_plugin_install_failed( + plugin, + PluginInstallSource::Manual, + error_type.to_string(), + ); } async fn plugin_apps_needing_auth_for_install( diff --git a/codex-rs/app-server/tests/suite/v2/external_agent_config.rs b/codex-rs/app-server/tests/suite/v2/external_agent_config.rs index e83b636618..53c50ea43f 100644 --- a/codex-rs/app-server/tests/suite/v2/external_agent_config.rs +++ b/codex-rs/app-server/tests/suite/v2/external_agent_config.rs @@ -418,9 +418,18 @@ async fn external_agent_config_import_completed_tracks_analytics_event() -> Resu } #[tokio::test] -async fn external_agent_config_import_sends_completion_notification_for_local_plugins() -> Result<()> -{ +async fn external_agent_config_import_reinstalls_plugins_from_known_marketplaces() -> Result<()> { let codex_home = TempDir::new()?; + let analytics_server = start_analytics_events_server().await?; + write_analytics_config(codex_home.path(), &analytics_server.uri())?; + write_chatgpt_auth( + codex_home.path(), + ChatGptAuthFixture::new("chatgpt-token") + .account_id("account-123") + .chatgpt_user_id("user-123") + .chatgpt_account_id("account-123"), + AuthCredentialsStoreMode::File, + )?; let marketplace_root = codex_home.path().join("marketplace"); let plugin_root = marketplace_root.join("plugins").join("sample"); std::fs::create_dir_all(marketplace_root.join(".agents/plugins"))?; @@ -445,15 +454,18 @@ async fn external_agent_config_import_sends_completion_notification_for_local_pl r#"{"name":"sample","version":"0.1.0"}"#, )?; let source_home = external_agent_home(codex_home.path()); - std::fs::create_dir_all(&source_home)?; + std::fs::create_dir_all(source_home.join("plugins"))?; let settings = serde_json::json!({ "enabledPlugins": { - "sample@debug": true + "missing@debug": true, + "sample@debug": true, }, "extraKnownMarketplaces": { "debug": { - "source": "local", - "path": marketplace_root, + "source": { + "source": "file", + "path": marketplace_root.join(".agents/plugins/marketplace.json"), + } } } }); @@ -461,6 +473,19 @@ async fn external_agent_config_import_sends_completion_notification_for_local_pl source_home.join("settings.json"), serde_json::to_string_pretty(&settings)?, )?; + std::fs::write( + source_home.join("plugins/known_marketplaces.json"), + serde_json::to_string_pretty(&serde_json::json!({ + "debug": { + "source": { + "source": "file", + "path": marketplace_root.join(".agents/plugins/marketplace.json"), + }, + "installLocation": marketplace_root, + "lastUpdated": "2026-07-09T00:16:23.611Z", + } + }))?, + )?; let home_dir = codex_home.path().display().to_string(); let mut mcp = TestAppServer::builder() @@ -473,23 +498,38 @@ async fn external_agent_config_import_sends_completion_notification_for_local_pl let request_id = mcp .send_raw_request( - "externalAgentConfig/import", - Some(serde_json::json!({ - "migrationItems": [{ - "itemType": "PLUGINS", - "description": "Import plugins", - "cwd": null, - "details": { - "plugins": [{ - "marketplaceName": "debug", - "pluginNames": ["sample"] - }] - } - }] - })), + "externalAgentConfig/detect", + Some(serde_json::json!({ "includeHome": true })), ) .await?; + let response: JSONRPCResponse = timeout( + DEFAULT_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let detected: ExternalAgentConfigDetectResponse = to_response(response)?; + assert_eq!(detected.items.len(), 1); + assert_eq!( + detected.items[0].item_type, + ExternalAgentConfigMigrationItemType::Plugins + ); + assert_eq!( + detected.items[0] + .details + .as_ref() + .map(|details| details.plugins.clone()), + Some(vec![codex_app_server_protocol::PluginsMigration { + marketplace_name: "debug".to_string(), + plugin_names: vec!["missing".to_string(), "sample".to_string()], + }]) + ); + let request_id = mcp + .send_raw_request( + "externalAgentConfig/import", + Some(serde_json::json!({ "migrationItems": detected.items })), + ) + .await?; let response: JSONRPCResponse = timeout( DEFAULT_TIMEOUT, mcp.read_stream_until_response_message(RequestId::Integer(request_id)), @@ -507,6 +547,55 @@ async fn external_agent_config_import_sends_completion_notification_for_local_pl let completed: ExternalAgentConfigImportCompletedNotification = serde_json::from_value(notification.params.expect("completed params"))?; assert_eq!(completed.import_id, import_id); + assert_eq!(completed.item_type_results.len(), 1); + let plugin_result = &completed.item_type_results[0]; + assert_eq!( + plugin_result.item_type, + ExternalAgentConfigMigrationItemType::Plugins + ); + assert_eq!(plugin_result.successes.len(), 1); + assert_eq!( + plugin_result.successes[0].source.as_deref(), + Some("sample@debug") + ); + assert_eq!(plugin_result.failures.len(), 1); + assert_eq!( + plugin_result.failures[0].source.as_deref(), + Some("missing@debug") + ); + assert_eq!( + plugin_result.failures[0].error_type.as_deref(), + Some("plugin_not_found") + ); + assert_eq!(plugin_result.failures[0].failure_stage, "plugin_import"); + assert_eq!( + plugin_result.failures[0].message, + "plugin `missing` was not found in marketplace `debug`" + ); + + let event = wait_for_analytics_event( + &analytics_server, + DEFAULT_TIMEOUT, + "codex_plugin_install_failed", + ) + .await?; + let event_params = &event["event_params"]; + assert_eq!(event_params["plugin_id"], "missing@debug"); + assert_eq!(event_params["plugin_name"], "missing"); + assert_eq!(event_params["marketplace_name"], "debug"); + assert_eq!(event_params["source"], "external_agent_migration"); + assert_eq!(event_params["error_type"], "plugin_not_found"); + + let event = wait_for_analytics_event( + &analytics_server, + DEFAULT_TIMEOUT, + "codex_onboarding_external_agent_import_failure", + ) + .await?; + let event_params = &event["event_params"]; + assert_eq!(event_params["type"], "PLUGINS"); + assert_eq!(event_params["failure_stage"], "plugin_import"); + assert_eq!(event_params["error_type"], "plugin_not_found"); let request_id = mcp .send_plugin_list_request(PluginListParams { diff --git a/codex-rs/app-server/tests/suite/v2/plugin_install.rs b/codex-rs/app-server/tests/suite/v2/plugin_install.rs index 9d0a7bcff3..1d781cdc96 100644 --- a/codex-rs/app-server/tests/suite/v2/plugin_install.rs +++ b/codex-rs/app-server/tests/suite/v2/plugin_install.rs @@ -580,6 +580,7 @@ async fn plugin_install_tracks_analytics_when_remote_detail_fetch_fails() -> Res assert_eq!(event_params["remote_plugin_id"], REMOTE_PLUGIN_ID); assert_eq!(event_params["plugin_name"], json!(null)); assert_eq!(event_params["marketplace_name"], json!(null)); + assert_eq!(event_params["source"], "manual"); assert_eq!( event_params["error_type"], "remote_catalog_unexpected_status" @@ -1019,6 +1020,7 @@ async fn plugin_install_failure_tracks_analytics_event() -> Result<()> { assert_eq!(event_params["mcp_server_count"], json!(null)); assert_eq!(event_params["connector_ids"], json!(null)); assert_eq!(event_params["product_client_id"], DEFAULT_CLIENT_NAME); + assert_eq!(event_params["source"], "manual"); assert_eq!(event_params["error_type"], "store_invalid"); Ok(()) } @@ -1145,6 +1147,7 @@ async fn plugin_install_preserves_status_when_remote_bundle_error_body_is_too_la assert_eq!(event_params["plugin_id"], "linear@openai-curated-remote"); assert_eq!(event_params["remote_plugin_id"], REMOTE_PLUGIN_ID); assert_eq!(event_params["marketplace_name"], "openai-curated-remote"); + assert_eq!(event_params["source"], "manual"); assert_eq!(event_params["error_type"], "remote_bundle_download_status"); assert!( !codex_home diff --git a/codex-rs/code-mode-protocol/src/description.rs b/codex-rs/code-mode-protocol/src/description.rs index 6ae31ff209..d0504f54a4 100644 --- a/codex-rs/code-mode-protocol/src/description.rs +++ b/codex-rs/code-mode-protocol/src/description.rs @@ -250,25 +250,32 @@ pub fn is_code_mode_nested_tool(tool_name: &str) -> bool { pub fn build_exec_tool_description( enabled_tools: &[ToolDefinition], + deferred_tools: &[ToolDefinition], namespace_descriptions: &BTreeMap, code_mode_only: bool, - deferred_tools_available: bool, ) -> String { let mut sections = Vec::new(); sections.push(EXEC_DESCRIPTION_TEMPLATE.to_string()); - if deferred_tools_available { + if !deferred_tools.is_empty() { sections.push(DEFERRED_NESTED_TOOLS_GUIDANCE.to_string()); } if !code_mode_only { return sections.join("\n\n"); } + let has_mcp_tools = enabled_tools + .iter() + .chain(deferred_tools) + .any(|tool| mcp_structured_content_schema(tool.output_schema.as_ref()).is_some()); + if has_mcp_tools { + sections.push(format!( + "Shared MCP Types:\n```ts\n{MCP_TYPESCRIPT_PREAMBLE}\n```" + )); + } + if !enabled_tools.is_empty() { let mut current_namespace: Option<&str> = None; let mut nested_tool_sections = Vec::with_capacity(enabled_tools.len()); - let has_mcp_tools = enabled_tools - .iter() - .any(|tool| mcp_structured_content_schema(tool.output_schema.as_ref()).is_some()); for tool in enabled_tools { let name = tool.name.as_str(); @@ -305,11 +312,6 @@ pub fn build_exec_tool_description( } } - if has_mcp_tools { - sections.push(format!( - "Shared MCP Types:\n```ts\n{MCP_TYPESCRIPT_PREAMBLE}\n```" - )); - } let nested_tool_reference = nested_tool_sections.join("\n\n"); sections.push(nested_tool_reference); } @@ -863,9 +865,9 @@ mod tests { input_schema: None, output_schema: None, }], + &[], &BTreeMap::new(), /*code_mode_only*/ true, - /*deferred_tools_available*/ false, ); assert!(description.contains( "### `foo` @@ -876,12 +878,8 @@ bar" #[test] fn exec_description_mentions_timeout_helpers() { - let description = build_exec_tool_description( - &[], - &BTreeMap::new(), - /*code_mode_only*/ false, - /*deferred_tools_available*/ false, - ); + let description = + build_exec_tool_description(&[], &[], &BTreeMap::new(), /*code_mode_only*/ false); assert!(description.contains("`setTimeout(callback: () => void, delayMs?: number)`")); assert!(description.contains("`clearTimeout(timeoutId?: number)`")); } @@ -930,9 +928,9 @@ bar" }))), }, ], + &[], &namespace_descriptions, /*code_mode_only*/ true, - /*deferred_tools_available*/ false, ); assert_eq!(description.matches("## mcp__sample").count(), 1); assert!(description.contains("## mcp__sample\nShared namespace guidance.")); @@ -970,9 +968,9 @@ bar" "additionalProperties": false }))), }], + &[], &namespace_descriptions, /*code_mode_only*/ true, - /*deferred_tools_available*/ false, ); assert!(!description.contains("## mcp__sample")); @@ -1069,9 +1067,9 @@ bar" output_schema: second_tool.output_schema, }, ], + &[], &BTreeMap::new(), /*code_mode_only*/ true, - /*deferred_tools_available*/ false, ); assert_eq!( @@ -1083,13 +1081,51 @@ bar" assert_eq!(description.matches("Shared MCP Types:").count(), 1); } + #[test] + fn code_mode_only_description_renders_shared_mcp_types_for_deferred_tools() { + let deferred_tool = ToolDefinition { + name: "mcp__sample__alpha".to_string(), + tool_name: ToolName::namespaced("mcp__sample__", "alpha"), + description: "Deferred tool".to_string(), + kind: CodeModeToolKind::Function, + input_schema: Some(json!({ + "type": "object", + "properties": {}, + "additionalProperties": false + })), + output_schema: Some(mcp_call_tool_result_schema(json!({ + "type": "object", + "properties": {}, + "additionalProperties": false + }))), + }; + + let description = build_exec_tool_description( + &[], + &[deferred_tool], + &BTreeMap::new(), + /*code_mode_only*/ true, + ); + + assert!(description.contains("Some deferred nested tools may be omitted")); + assert!(description.contains("Shared MCP Types:")); + assert!(!description.contains("### `mcp__sample__alpha`")); + } + #[test] fn exec_description_mentions_deferred_nested_tools_when_available() { let description = build_exec_tool_description( &[], + &[ToolDefinition { + name: "deferred_tool".to_string(), + tool_name: ToolName::plain("deferred_tool"), + description: "Deferred tool".to_string(), + kind: CodeModeToolKind::Function, + input_schema: None, + output_schema: None, + }], &BTreeMap::new(), /*code_mode_only*/ false, - /*deferred_tools_available*/ true, ); assert!(description.contains("Some deferred nested tools may be omitted")); diff --git a/codex-rs/codex-mcp/src/connection_manager.rs b/codex-rs/codex-mcp/src/connection_manager.rs index 612b2ed442..c21a9f1d10 100644 --- a/codex-rs/codex-mcp/src/connection_manager.rs +++ b/codex-rs/codex-mcp/src/connection_manager.rs @@ -504,18 +504,14 @@ impl McpConnectionManager { #[instrument(level = "trace", skip_all, fields(mcp_server_count = self.clients.len()))] pub async fn list_all_tools(&self) -> Vec { let mut tools = Vec::new(); + let mut available_server_count = 0; + let mut unavailable_server_count = 0; for (server_name, managed_client) in &self.clients { managed_client.reconnect_failed_startup().await; let has_cached_tools = managed_client.has_cached_tools(); let startup_complete = managed_client .startup_complete .load(std::sync::atomic::Ordering::Acquire); - trace!( - server_name = %server_name, - has_cached_tools, - startup_complete, - "waiting for MCP server tools while building tool list" - ); let Some(server_tools) = managed_client .listed_tools() .instrument(trace_span!( @@ -526,20 +522,30 @@ impl McpConnectionManager { )) .await else { + unavailable_server_count += 1; + trace!( + server_name = %server_name, + has_cached_tools, + startup_complete, + "MCP server tools unavailable while building tool list" + ); continue; }; - trace!( - server_name = %server_name, - tool_count = server_tools.len(), - "listed MCP server tools while building tool list" - ); + available_server_count += 1; tools.extend( server_tools .into_iter() .map(|tool| self.with_server_metadata(tool)), ); } - normalize_tools_for_model_with_prefix(tools, self.prefix_mcp_tool_names) + let tools = normalize_tools_for_model_with_prefix(tools, self.prefix_mcp_tool_names); + trace!( + available_server_count, + unavailable_server_count, + tool_count = tools.len(), + "built MCP tool list" + ); + tools } /// Force-refresh codex apps tools by bypassing the in-process cache. diff --git a/codex-rs/codex-mcp/src/connection_manager_tests.rs b/codex-rs/codex-mcp/src/connection_manager_tests.rs index d8a1f1863e..d41cb76de3 100644 --- a/codex-rs/codex-mcp/src/connection_manager_tests.rs +++ b/codex-rs/codex-mcp/src/connection_manager_tests.rs @@ -1418,6 +1418,7 @@ fn server_metadata_preserves_tool_approval_policy() { let mut config = crate::codex_apps_mcp_server_config( "https://docs.example", /*apps_mcp_product_sku*/ None, + /*originator*/ None, ); config.environment_id = "remote".to_string(); config.default_tools_approval_mode = Some(AppToolApproval::Prompt); @@ -1462,6 +1463,7 @@ fn host_owned_codex_apps_matches_reserved_name_with_server_metadata() { let server = EffectiveMcpServer::configured(crate::codex_apps_mcp_server_config( "https://chatgpt.com", /*apps_mcp_product_sku*/ None, + /*originator*/ None, )); manager.server_metadata.insert( CODEX_APPS_MCP_SERVER_NAME.to_string(), diff --git a/codex-rs/codex-mcp/src/mcp/mod.rs b/codex-rs/codex-mcp/src/mcp/mod.rs index 847dbe2866..8a1847bc96 100644 --- a/codex-rs/codex-mcp/src/mcp/mod.rs +++ b/codex-rs/codex-mcp/src/mcp/mod.rs @@ -51,6 +51,7 @@ use crate::runtime::McpRuntimeContext; use crate::server::EffectiveMcpServer; pub const CODEX_APPS_MCP_SERVER_NAME: &str = "codex_apps"; +const DEFAULT_CODEX_APPS_MCP_PRODUCT_SKU: &str = "codex"; const MCP_TOOL_NAME_PREFIX: &str = "mcp"; const MCP_TOOL_NAME_DELIMITER: &str = "__"; const CODEX_CONNECTORS_TOKEN_ENV_VAR: &str = "CODEX_CONNECTORS_TOKEN"; @@ -492,10 +493,12 @@ fn codex_apps_mcp_url_for_base_url(base_url: &str) -> String { pub fn codex_apps_mcp_server_config( chatgpt_base_url: &str, apps_mcp_product_sku: Option<&str>, + originator: Option<&str>, ) -> McpServerConfig { mcp_server_config_for_url( codex_apps_mcp_url_for_base_url(chatgpt_base_url), apps_mcp_product_sku, + originator, McpServerAuth::ChatGpt, ) } @@ -504,6 +507,7 @@ pub fn codex_apps_mcp_server_config( pub fn hosted_plugin_runtime_mcp_server_config( chatgpt_base_url: &str, apps_mcp_product_sku: Option<&str>, + originator: Option<&str>, ) -> McpServerConfig { let base_url = normalize_codex_apps_base_url(chatgpt_base_url); let base_url = if base_url.contains("/backend-api") || base_url.contains("/api/codex") { @@ -514,6 +518,7 @@ pub fn hosted_plugin_runtime_mcp_server_config( mcp_server_config_for_url( format!("{base_url}/ps/mcp"), apps_mcp_product_sku, + originator, McpServerAuth::ChatGpt, ) } @@ -521,17 +526,21 @@ pub fn hosted_plugin_runtime_mcp_server_config( fn mcp_server_config_for_url( url: String, apps_mcp_product_sku: Option<&str>, + originator: Option<&str>, auth_mode: McpServerAuth, ) -> McpServerConfig { - let http_headers = apps_mcp_product_sku.map(|product_sku| { - HashMap::from([("X-OpenAI-Product-Sku".to_string(), product_sku.to_string())]) - }); + let product_sku = apps_mcp_product_sku.unwrap_or(DEFAULT_CODEX_APPS_MCP_PRODUCT_SKU); + let mut http_headers = + HashMap::from([("X-OpenAI-Product-Sku".to_string(), product_sku.to_string())]); + if let Some(originator) = originator { + http_headers.insert("originator".to_string(), originator.to_string()); + } McpServerConfig { transport: McpServerTransportConfig::StreamableHttp { url, bearer_token_env_var: codex_apps_mcp_bearer_token_env_var(), - http_headers, + http_headers: Some(http_headers), env_http_headers: None, }, auth: auth_mode, diff --git a/codex-rs/codex-mcp/src/mcp/mod_tests.rs b/codex-rs/codex-mcp/src/mcp/mod_tests.rs index 7d166cbab8..99143673e4 100644 --- a/codex-rs/codex-mcp/src/mcp/mod_tests.rs +++ b/codex-rs/codex-mcp/src/mcp/mod_tests.rs @@ -130,7 +130,11 @@ fn tool_plugin_provenance_collects_app_and_mcp_sources() { "alpha".to_string(), McpPluginAttribution::new("alpha@test".to_string(), "alpha-plugin".to_string()), /*plugin_order*/ 0, - codex_apps_mcp_server_config("https://alpha.example", /*apps_mcp_product_sku*/ None), + codex_apps_mcp_server_config( + "https://alpha.example", + /*apps_mcp_product_sku*/ None, + /*originator*/ None, + ), )); config.mcp_server_catalog = catalog.build(); config.connector_snapshot = @@ -197,7 +201,11 @@ fn selected_mcp_attribution_does_not_join_an_unrelated_local_summary() { "Executor GitHub".to_string(), ), /*selection_order*/ 0, - codex_apps_mcp_server_config("https://github.example", /*apps_mcp_product_sku*/ None), + codex_apps_mcp_server_config( + "https://github.example", + /*apps_mcp_product_sku*/ None, + /*originator*/ None, + ), )); config.mcp_server_catalog = catalog.build(); config.connector_snapshot = @@ -252,8 +260,11 @@ fn codex_apps_mcp_url_for_base_url_keeps_existing_paths() { #[test] fn codex_apps_server_config_uses_legacy_codex_apps_path() { - let config = - codex_apps_mcp_server_config("https://chatgpt.com", /*apps_mcp_product_sku*/ None); + let config = codex_apps_mcp_server_config( + "https://chatgpt.com", + /*apps_mcp_product_sku*/ None, + /*originator*/ None, + ); let url = match &config.transport { McpServerTransportConfig::StreamableHttp { url, .. } => url, _ => panic!("expected streamable http transport for codex apps"), @@ -263,8 +274,12 @@ fn codex_apps_server_config_uses_legacy_codex_apps_path() { } #[test] -fn codex_apps_server_config_forwards_configured_product_sku_header() { - let config = codex_apps_mcp_server_config("https://chatgpt.com", Some("tpp")); +fn codex_apps_server_config_forwards_thread_originator_header() { + let config = codex_apps_mcp_server_config( + "https://chatgpt.com", + /*apps_mcp_product_sku*/ None, + Some("thread_originator"), + ); match &config.transport { McpServerTransportConfig::StreamableHttp { @@ -274,10 +289,66 @@ fn codex_apps_server_config_forwards_configured_product_sku_header() { } => { assert_eq!( http_headers, - &Some(HashMap::from([( - "X-OpenAI-Product-Sku".to_string(), - "tpp".to_string(), - )])) + &Some(HashMap::from([ + ("originator".to_string(), "thread_originator".to_string()), + ("X-OpenAI-Product-Sku".to_string(), "codex".to_string()), + ])) + ); + assert!(env_http_headers.is_none()); + } + other => panic!("expected streamable http transport, got {other:?}"), + } +} + +#[test] +fn codex_apps_server_config_sets_product_sku_header() { + for (configured_product_sku, expected_product_sku) in [(None, "codex"), (Some("tpp"), "tpp")] { + let config = codex_apps_mcp_server_config( + "https://chatgpt.com", + configured_product_sku, + /*originator*/ None, + ); + + match &config.transport { + McpServerTransportConfig::StreamableHttp { + http_headers, + env_http_headers, + .. + } => { + assert_eq!( + http_headers, + &Some(HashMap::from([( + "X-OpenAI-Product-Sku".to_string(), + expected_product_sku.to_string(), + )])) + ); + assert!(env_http_headers.is_none()); + } + other => panic!("expected streamable http transport, got {other:?}"), + } + } +} + +#[test] +fn codex_apps_server_config_forwards_originator_and_configured_product_sku_headers() { + let config = codex_apps_mcp_server_config( + "https://chatgpt.com", + Some("tpp"), + Some("thread_originator"), + ); + + match &config.transport { + McpServerTransportConfig::StreamableHttp { + http_headers, + env_http_headers, + .. + } => { + assert_eq!( + http_headers, + &Some(HashMap::from([ + ("originator".to_string(), "thread_originator".to_string()), + ("X-OpenAI-Product-Sku".to_string(), "tpp".to_string()), + ])) ); assert!(env_http_headers.is_none()); } @@ -350,6 +421,7 @@ async fn effective_mcp_servers_preserve_runtime_servers() { codex_apps_mcp_server_config( &config.chatgpt_base_url, config.apps_mcp_product_sku.as_deref(), + /*originator*/ None, ), )); config.mcp_server_catalog = catalog.build(); diff --git a/codex-rs/config/src/permissions_toml.rs b/codex-rs/config/src/permissions_toml.rs index 988e21e539..92bdcfcbb3 100644 --- a/codex-rs/config/src/permissions_toml.rs +++ b/codex-rs/config/src/permissions_toml.rs @@ -501,40 +501,39 @@ impl NetworkMitmActionToml { impl NetworkToml { pub fn apply_to_network_proxy_config(&self, config: &mut NetworkProxyConfig) { if let Some(enabled) = self.enabled { - config.network.enabled = enabled; + config.enabled = enabled; } if let Some(proxy_url) = self.proxy_url.as_ref() { - config.network.proxy_url = proxy_url.clone(); + config.proxy_url = proxy_url.clone(); } if let Some(enable_socks5) = self.enable_socks5 { - config.network.enable_socks5 = enable_socks5; + config.enable_socks5 = enable_socks5; } if let Some(socks_url) = self.socks_url.as_ref() { - config.network.socks_url = socks_url.clone(); + config.socks_url = socks_url.clone(); } if let Some(enable_socks5_udp) = self.enable_socks5_udp { - config.network.enable_socks5_udp = enable_socks5_udp; + config.enable_socks5_udp = enable_socks5_udp; } if let Some(allow_upstream_proxy) = self.allow_upstream_proxy { - config.network.allow_upstream_proxy = allow_upstream_proxy; + config.allow_upstream_proxy = allow_upstream_proxy; } if let Some(dangerously_allow_non_loopback_proxy) = self.dangerously_allow_non_loopback_proxy { - config.network.dangerously_allow_non_loopback_proxy = - dangerously_allow_non_loopback_proxy; + config.dangerously_allow_non_loopback_proxy = dangerously_allow_non_loopback_proxy; } if let Some(dangerously_allow_all_unix_sockets) = self.dangerously_allow_all_unix_sockets { - config.network.dangerously_allow_all_unix_sockets = dangerously_allow_all_unix_sockets; + config.dangerously_allow_all_unix_sockets = dangerously_allow_all_unix_sockets; } if let Some(mode) = self.mode { - config.network.mode = mode; + config.mode = mode; } if let Some(domains) = self.domains.as_ref() { overlay_network_domain_permissions(config, domains); } if let Some(unix_sockets) = self.unix_sockets.as_ref() { - let mut proxy_unix_sockets = config.network.unix_sockets.take().unwrap_or_default(); + let mut proxy_unix_sockets = config.unix_sockets.take().unwrap_or_default(); for (path, permission) in &unix_sockets.entries { let permission = match permission { NetworkUnixSocketPermissionToml::Allow => { @@ -544,17 +543,16 @@ impl NetworkToml { }; proxy_unix_sockets.entries.insert(path.clone(), permission); } - config.network.unix_sockets = + config.unix_sockets = (!proxy_unix_sockets.entries.is_empty()).then_some(proxy_unix_sockets); } if let Some(allow_local_binding) = self.allow_local_binding { - config.network.allow_local_binding = allow_local_binding; + config.allow_local_binding = allow_local_binding; } if let Some(mitm) = self.mitm.as_ref() { - config.network.mitm_hooks = mitm.to_runtime_hooks(mitm.actions.as_ref()); + config.mitm_hooks = mitm.to_runtime_hooks(mitm.actions.as_ref()); } - config.network.mitm = - config.network.mode == NetworkMode::Limited || !config.network.mitm_hooks.is_empty(); + config.mitm = config.mode == NetworkMode::Limited || !config.mitm_hooks.is_empty(); } pub fn to_network_proxy_config(&self) -> NetworkProxyConfig { @@ -628,8 +626,6 @@ pub fn overlay_network_domain_permissions( NetworkDomainPermissionToml::Allow => ProxyNetworkDomainPermission::Allow, NetworkDomainPermissionToml::Deny => ProxyNetworkDomainPermission::Deny, }; - config - .network - .upsert_domain_permission(pattern.clone(), permission, normalize_host); + config.upsert_domain_permission(pattern.clone(), permission, normalize_host); } } diff --git a/codex-rs/core-plugins/src/manager.rs b/codex-rs/core-plugins/src/manager.rs index 6fac0fafe9..16fb4c8663 100644 --- a/codex-rs/core-plugins/src/manager.rs +++ b/codex-rs/core-plugins/src/manager.rs @@ -57,6 +57,7 @@ use crate::store::PluginStore; use crate::store::PluginStoreError; use crate::tool_suggest_metadata::ToolSuggestMetadataCache; use codex_analytics::AnalyticsEventsClient; +use codex_analytics::PluginInstallSource; use codex_config::ConfigLayerStack; use codex_config::clear_user_plugin; use codex_config::set_user_plugin_enabled; @@ -370,6 +371,7 @@ pub struct PluginsManager { restriction_product: Option, auth_mode: RwLock>, analytics_events_client: RwLock>, + plugin_install_source: PluginInstallSource, } #[derive(Clone)] @@ -449,9 +451,15 @@ impl PluginsManager { restriction_product, auth_mode: RwLock::new(auth_mode), analytics_events_client: RwLock::new(None), + plugin_install_source: PluginInstallSource::Manual, } } + pub fn with_plugin_install_source(mut self, source: PluginInstallSource) -> Self { + self.plugin_install_source = source; + self + } + pub fn set_auth_mode(&self, auth_mode: Option) -> bool { let mut stored_auth_mode = match self.auth_mode.write() { Ok(auth_mode_guard) => auth_mode_guard, @@ -1419,6 +1427,7 @@ impl PluginsManager { if let Some(analytics_events_client) = analytics_events_client { analytics_events_client.track_plugin_install_failed( self.telemetry_metadata_for_plugin_id(plugin_id), + self.plugin_install_source, error_type.to_string(), ); } diff --git a/codex-rs/core-skills/src/loader.rs b/codex-rs/core-skills/src/loader.rs index e74eb01a0c..aa663e3e60 100644 --- a/codex-rs/core-skills/src/loader.rs +++ b/codex-rs/core-skills/src/loader.rs @@ -1,3 +1,4 @@ +mod discovery; mod environment; mod namespace; @@ -26,15 +27,21 @@ use codex_protocol::protocol::SkillScope; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_absolute_path::AbsolutePathBufGuard; use codex_utils_path_uri::PathUri; -use codex_utils_plugins::DISCOVERABLE_PLUGIN_MANIFEST_PATHS; use codex_utils_plugins::PluginSkillRoot; use dirs::home_dir; -use futures::future::join_all; -use namespace::ResolvedSkillNamespace; +use discovery::DirectorySymlinkPolicy; +use discovery::DiscoveredSkill; +use discovery::HiddenDirectoryPolicy; +use discovery::MAX_CONCURRENT_SKILL_LOADS; +use discovery::SkillDiscovery; +use discovery::SkillDiscoveryOptions; +use discovery::SkillMetadataDiscovery; +use discovery::discover_skills; +use futures::FutureExt; +use futures::StreamExt; use namespace::SkillNamespaceResolver; use serde::Deserialize; use std::collections::HashSet; -use std::collections::VecDeque; use std::error::Error; use std::fmt; use std::io; @@ -140,17 +147,10 @@ const MAX_DEPENDENCY_URL_LEN: usize = MAX_DESCRIPTION_LEN; const MAX_SCAN_DEPTH: usize = 6; const MAX_SKILLS_DIRS_PER_ROOT: usize = 2000; -#[derive(Clone, Copy)] -enum SymlinkPolicy { - FollowDirectories, - Ignore, -} - -struct SkillFileDiscovery { - skill_files: Vec, - plugin_roots: HashSet, - namespace_roots: HashSet, - warnings: Vec, +struct ResolvedDiscoveredSkill { + skill: DiscoveredSkill, + path: AbsolutePathBuf, + path_uri: PathUri, } #[derive(Debug)] @@ -194,9 +194,12 @@ pub async fn load_skills_from_roots( plugin_skill_snapshots: Option<&crate::PluginSkillSnapshots>, ) -> SkillLoadOutcome where - I: IntoIterator, + I: IntoIterator + Send, + I::IntoIter: Send, { - crate::root_loader::load_and_merge_skill_roots(roots, plugin_skill_snapshots).await + crate::root_loader::load_and_merge_skill_roots(roots, plugin_skill_snapshots) + .boxed() + .await } #[derive(Clone)] @@ -497,172 +500,6 @@ async fn canonicalize_for_skill_identity( .unwrap_or_else(|_| path.clone()) } -async fn discover_skills_under_root( - fs: &dyn ExecutorFileSystem, - root: &PathUri, - symlink_policy: SymlinkPolicy, -) -> SkillFileDiscovery { - let root = root.clone(); - let mut discovery = SkillFileDiscovery { - skill_files: Vec::new(), - plugin_roots: HashSet::new(), - namespace_roots: HashSet::from([root.clone()]), - warnings: Vec::new(), - }; - match fs.get_metadata(&root, /*sandbox*/ None).await { - Ok(metadata) if metadata.is_directory => {} - Ok(_) => return discovery, - Err(err) if err.kind() == io::ErrorKind::NotFound => return discovery, - Err(err) => { - discovery - .warnings - .push(format!("failed to stat skills root {root}: {err:#}")); - return discovery; - } - } - - fn enqueue_dir( - queue: &mut VecDeque<(PathUri, usize)>, - visited_dirs: &mut HashSet, - truncated_by_dir_limit: &mut bool, - path: PathUri, - depth: usize, - ) { - if depth > MAX_SCAN_DEPTH { - return; - } - if visited_dirs.len() >= MAX_SKILLS_DIRS_PER_ROOT { - *truncated_by_dir_limit = true; - return; - } - if visited_dirs.insert(path.clone()) { - queue.push_back((path, depth)); - } - } - - let follow_symlinks = matches!(symlink_policy, SymlinkPolicy::FollowDirectories); - let mut visited_dirs: HashSet = HashSet::from([root.clone()]); - let mut queue: VecDeque<(PathUri, usize)> = VecDeque::from([(root.clone(), 0)]); - let mut truncated_by_dir_limit = false; - - while let Some((dir, depth)) = queue.pop_front() { - let entries = match fs.read_directory(&dir, /*sandbox*/ None).await { - Ok(entries) => entries, - Err(err) => { - discovery - .warnings - .push(format!("failed to read skills directory {dir}: {err:#}")); - continue; - } - }; - - let paths = entries - .into_iter() - .filter_map(|entry| { - let file_name = entry.file_name; - if DISCOVERABLE_PLUGIN_MANIFEST_PATHS - .iter() - .any(|path| path.split('/').next() == Some(file_name.as_str())) - { - discovery.plugin_roots.insert(dir.clone()); - } - if file_name.starts_with('.') { - return None; - } - match dir.join(&file_name) { - Ok(path) => Some((file_name, path)), - Err(err) => { - discovery.warnings.push(format!( - "failed to resolve skill path {dir}/{file_name}: {err}" - )); - None - } - } - }) - .collect::>(); - let metadata_results = join_all( - paths - .iter() - .map(|(_, path)| fs.get_metadata(path, /*sandbox*/ None)), - ) - .await; - - for ((file_name, path), metadata_result) in paths.into_iter().zip(metadata_results) { - let metadata = match metadata_result { - Ok(metadata) => metadata, - Err(err) => { - discovery - .warnings - .push(format!("failed to stat skill path {path}: {err:#}")); - continue; - } - }; - - if metadata.is_symlink { - if !follow_symlinks { - continue; - } - match fs.read_directory(&path, /*sandbox*/ None).await { - Ok(_) => { - let resolved_dir = canonicalize_uri_for_skill_identity(fs, &path).await; - discovery.namespace_roots.insert(resolved_dir.clone()); - enqueue_dir( - &mut queue, - &mut visited_dirs, - &mut truncated_by_dir_limit, - resolved_dir, - depth + 1, - ); - } - Err(err) - if matches!( - err.kind(), - io::ErrorKind::NotADirectory | io::ErrorKind::NotFound - ) => {} - Err(err) => discovery.warnings.push(format!( - "failed to read skills symlink directory {path}: {err:#}" - )), - } - continue; - } - - if metadata.is_directory { - enqueue_dir( - &mut queue, - &mut visited_dirs, - &mut truncated_by_dir_limit, - path, - depth + 1, - ); - continue; - } - - if metadata.is_file && file_name == SKILLS_FILENAME { - discovery.skill_files.push(path); - } - } - } - - if truncated_by_dir_limit { - tracing::warn!( - "skills scan truncated after {} directories (root: {})", - MAX_SKILLS_DIRS_PER_ROOT, - root - ); - } - discovery -} - -async fn canonicalize_uri_for_skill_identity( - file_system: &dyn ExecutorFileSystem, - path: &PathUri, -) -> PathUri { - file_system - .canonicalize(path, /*sandbox*/ None) - .await - .unwrap_or_else(|_| path.clone()) -} - async fn load_skills_under_root( fs: &dyn ExecutorFileSystem, root: &AbsolutePathBuf, @@ -676,56 +513,113 @@ async fn load_skills_under_root( Some(plugin_root) => Some(canonicalize_for_skill_identity(fs, plugin_root).await), None => None, }; - let symlink_policy = match scope { - SkillScope::User | SkillScope::Repo | SkillScope::Admin => SymlinkPolicy::FollowDirectories, - SkillScope::System => SymlinkPolicy::Ignore, + let directory_symlinks = match scope { + SkillScope::User | SkillScope::Repo | SkillScope::Admin => DirectorySymlinkPolicy::Follow, + SkillScope::System => DirectorySymlinkPolicy::Ignore, }; - let SkillFileDiscovery { - skill_files, + let SkillDiscovery { + skills, plugin_roots, - namespace_roots, + mut namespace_roots, warnings, - } = discover_skills_under_root(fs, &PathUri::from_abs_path(root), symlink_policy).await; + } = discover_skills( + fs, + &PathUri::from_abs_path(root), + // Preserve host discovery behavior: directory aliases are scope-dependent, while hidden + // directories are skipped unless reached through a visible alias. + SkillDiscoveryOptions { + directory_symlinks, + hidden_directories: HiddenDirectoryPolicy::Skip, + }, + ) + .await; for warning in warnings { error!("{warning}"); } let root_uri = PathUri::from_abs_path(root); - let namespace_resolver = match plugin_namespace { - Some(namespace) => SkillNamespaceResolver::with_provided_namespace(namespace), - None => { - SkillNamespaceResolver::discover( - fs, - &root_uri, - &skill_files, - plugin_roots, - namespace_roots, - ) - .await + let resolved_skills = futures::stream::iter(skills) + .map(|skill| async move { + let path_uri = fs + .canonicalize(&skill.path, /*sandbox*/ None) + .await + .unwrap_or_else(|_| skill.path.clone()); + let path = match path_uri.to_abs_path() { + Ok(path) => path, + Err(err) => { + error!("failed to convert discovered skill path {path_uri}: {err}"); + return None; + } + }; + Some(ResolvedDiscoveredSkill { + skill, + path, + path_uri, + }) + }) + .buffered(MAX_CONCURRENT_SKILL_LOADS) + .filter_map(futures::future::ready) + .collect::>() + .await; + namespace_roots.extend(resolved_skills.iter().filter_map(|skill| { + (skill.path_uri != skill.skill.path) + .then(|| skill.path_uri.parent()) + .flatten() + })); + let skill_paths = resolved_skills + .iter() + .map(|skill| skill.path_uri.clone()) + .collect::>(); + let namespace_resolver = async { + match plugin_namespace { + Some(namespace) => SkillNamespaceResolver::with_provided_namespace(namespace), + None => { + SkillNamespaceResolver::discover( + fs, + &root_uri, + &skill_paths, + plugin_roots, + namespace_roots, + ) + .await + } } }; - for path_uri in skill_files { - let path = match path_uri.to_abs_path() { - Ok(path) => path, - Err(err) => { - error!("failed to convert discovered skill path {path_uri}: {err}"); - continue; + let skill_results = futures::stream::iter(resolved_skills) + .map(|skill| { + let plugin_root = plugin_root.as_ref(); + async move { + let result = parse_skill_file( + fs, + &skill.skill, + &skill.path, + &skill.path_uri, + scope, + plugin_id, + plugin_root, + ) + .await + .map_err(|err| err.to_string()); + (skill.path, skill.path_uri, result) } - }; - match parse_skill_file( - fs, - &path, - scope, - plugin_id, - namespace_resolver.for_skill(&root_uri, &path_uri), - plugin_root.as_ref(), - ) - .await - { + }) + .buffered(MAX_CONCURRENT_SKILL_LOADS) + .collect::>() + .boxed(); + let (namespace_resolver, skill_results) = tokio::join!(namespace_resolver, skill_results); + for (path, path_uri, result) in skill_results { + let result = result.and_then(|mut skill| { + skill.name = namespace_resolver + .for_skill(&root_uri, &path_uri) + .qualify(&skill.name); + validate_len(&skill.name, MAX_QUALIFIED_NAME_LEN, "qualified name") + .map_err(|err| err.to_string())?; + Ok(skill) + }); + match result { Ok(skill) => outcome.skills.push(skill), - Err(err) if scope != SkillScope::System => outcome.errors.push(SkillError { - path, - message: err.to_string(), - }), + Err(err) if scope != SkillScope::System => { + outcome.errors.push(SkillError { path, message: err }) + } Err(_) => {} } } @@ -733,41 +627,47 @@ async fn load_skills_under_root( async fn parse_skill_file( fs: &dyn ExecutorFileSystem, + skill: &DiscoveredSkill, path: &AbsolutePathBuf, + path_uri: &PathUri, scope: SkillScope, plugin_id: Option<&str>, - namespace: &ResolvedSkillNamespace, plugin_root: Option<&AbsolutePathBuf>, ) -> Result { - let path_uri = PathUri::from_abs_path(path); - let contents = fs - .read_file_text(&path_uri, /*sandbox*/ None) - .await - .map_err(SkillParseError::Read)?; + let metadata_path = path_uri + .parent() + .and_then(|parent| parent.join(SKILLS_METADATA_DIR).ok()) + .and_then(|directory| directory.join(SKILLS_METADATA_FILENAME).ok()); + let metadata = match &skill.metadata { + SkillMetadataDiscovery::Present(_) => metadata_path.map(SkillMetadataDiscovery::Present), + SkillMetadataDiscovery::Probe(_) => metadata_path.map(SkillMetadataDiscovery::Probe), + SkillMetadataDiscovery::Absent => None, + } + .unwrap_or(SkillMetadataDiscovery::Absent); + let (contents, loaded_metadata) = tokio::join!( + fs.read_file_text(path_uri, /*sandbox*/ None), + load_skill_metadata(fs, path, &metadata, plugin_root), + ); + let contents = contents.map_err(SkillParseError::Read)?; let ParsedSkillFrontmatter { name: base_name, description, short_description, } = parse_skill_frontmatter_metadata_inner(&contents, || default_skill_name(path))?; - let name = namespace.qualify(&base_name); let LoadedSkillMetadata { interface, dependencies, policy, - } = load_skill_metadata(fs, path, plugin_root).await; - - validate_len(&name, MAX_QUALIFIED_NAME_LEN, "qualified name")?; - - let resolved_path = canonicalize_for_skill_identity(fs, path).await; + } = loaded_metadata; Ok(SkillMetadata { - name, + name: base_name, description, short_description, interface, dependencies, policy, - path_to_skills_md: resolved_path, + path_to_skills_md: path.clone(), scope, plugin_id: plugin_id.map(str::to_string), }) @@ -838,41 +738,42 @@ fn default_skill_name(path: &AbsolutePathBuf) -> String { async fn load_skill_metadata( fs: &dyn ExecutorFileSystem, skill_path: &AbsolutePathBuf, + metadata: &SkillMetadataDiscovery, plugin_root: Option<&AbsolutePathBuf>, ) -> LoadedSkillMetadata { // Fail open: optional metadata should not block loading SKILL.md. let Some(skill_dir) = skill_path.parent() else { return LoadedSkillMetadata::default(); }; - let metadata_path = skill_dir - .join(SKILLS_METADATA_DIR) - .join(SKILLS_METADATA_FILENAME); - let metadata_path_uri = PathUri::from_abs_path(&metadata_path); - match fs.get_metadata(&metadata_path_uri, /*sandbox*/ None).await { - Ok(metadata) if metadata.is_file => {} - Ok(_) => return LoadedSkillMetadata::default(), - Err(error) if error.kind() == io::ErrorKind::NotFound => { - return LoadedSkillMetadata::default(); + let metadata_path_uri = match metadata { + SkillMetadataDiscovery::Present(path) => path, + SkillMetadataDiscovery::Absent => return LoadedSkillMetadata::default(), + SkillMetadataDiscovery::Probe(path) => { + match fs.get_metadata(path, /*sandbox*/ None).await { + Ok(metadata) if metadata.is_file => {} + Ok(_) => return LoadedSkillMetadata::default(), + Err(error) if error.kind() == io::ErrorKind::NotFound => { + return LoadedSkillMetadata::default(); + } + Err(error) => { + tracing::warn!( + "ignoring {path}: failed to stat {label}: {error}", + path = path, + label = SKILLS_METADATA_FILENAME + ); + return LoadedSkillMetadata::default(); + } + } + path } - Err(error) => { - tracing::warn!( - "ignoring {path}: failed to stat {label}: {error}", - path = metadata_path.display(), - label = SKILLS_METADATA_FILENAME - ); - return LoadedSkillMetadata::default(); - } - } + }; - let contents = match fs - .read_file_text(&metadata_path_uri, /*sandbox*/ None) - .await - { + let contents = match fs.read_file_text(metadata_path_uri, /*sandbox*/ None).await { Ok(contents) => contents, Err(error) => { tracing::warn!( "ignoring {path}: failed to read {label}: {error}", - path = metadata_path.display(), + path = metadata_path_uri, label = SKILLS_METADATA_FILENAME ); return LoadedSkillMetadata::default(); @@ -886,7 +787,7 @@ async fn load_skill_metadata( Err(error) => { tracing::warn!( "ignoring {path}: invalid {label}: {error}", - path = metadata_path.display(), + path = metadata_path_uri, label = SKILLS_METADATA_FILENAME ); return LoadedSkillMetadata::default(); diff --git a/codex-rs/core-skills/src/loader/discovery.rs b/codex-rs/core-skills/src/loader/discovery.rs new file mode 100644 index 0000000000..547400bc4f --- /dev/null +++ b/codex-rs/core-skills/src/loader/discovery.rs @@ -0,0 +1,207 @@ +use std::collections::HashSet; +use std::io; + +use codex_exec_server::ExecutorFileSystem; +use codex_exec_server::WalkEntryKind; +use codex_exec_server::WalkOptions; +use codex_utils_path_uri::PathUri; +use codex_utils_plugins::DISCOVERABLE_PLUGIN_MANIFEST_PATHS; + +use super::MAX_SCAN_DEPTH; +use super::MAX_SKILLS_DIRS_PER_ROOT; +use super::SKILLS_FILENAME; +use super::SKILLS_METADATA_DIR; +use super::SKILLS_METADATA_FILENAME; + +const MAX_SKILLS_ENTRIES_PER_ROOT: usize = 20_000; +pub(super) const MAX_CONCURRENT_SKILL_LOADS: usize = 64; + +pub(super) enum DirectorySymlinkPolicy { + Follow, + Ignore, +} + +pub(super) enum HiddenDirectoryPolicy { + Include, + Skip, +} + +pub(super) struct SkillDiscoveryOptions { + pub directory_symlinks: DirectorySymlinkPolicy, + pub hidden_directories: HiddenDirectoryPolicy, +} + +pub(super) struct SkillDiscovery { + pub skills: Vec, + pub plugin_roots: HashSet, + pub namespace_roots: HashSet, + pub warnings: Vec, +} + +pub(super) struct DiscoveredSkill { + pub path: PathUri, + pub metadata: SkillMetadataDiscovery, +} + +pub(super) enum SkillMetadataDiscovery { + Present(PathUri), + Absent, + Probe(PathUri), +} + +pub(super) async fn discover_skills( + file_system: &dyn ExecutorFileSystem, + root: &PathUri, + options: SkillDiscoveryOptions, +) -> SkillDiscovery { + let empty_discovery = || SkillDiscovery { + skills: Vec::new(), + plugin_roots: HashSet::new(), + namespace_roots: HashSet::new(), + warnings: Vec::new(), + }; + let walk = match file_system + .walk( + root, + WalkOptions { + max_depth: MAX_SCAN_DEPTH, + max_directories: MAX_SKILLS_DIRS_PER_ROOT, + max_entries: MAX_SKILLS_ENTRIES_PER_ROOT, + follow_directory_symlinks: matches!( + options.directory_symlinks, + DirectorySymlinkPolicy::Follow + ), + prune_hidden_directories: matches!( + options.hidden_directories, + HiddenDirectoryPolicy::Skip + ), + }, + /*sandbox*/ None, + ) + .await + { + Ok(walk) => walk, + Err(error) if error.kind() == io::ErrorKind::NotFound => return empty_discovery(), + Err(error) => { + let mut discovery = empty_discovery(); + discovery + .warnings + .push(format!("failed to walk skills root {root}: {error:#}")); + return discovery; + } + }; + + let inventory_complete = !walk.truncated && walk.errors.is_empty(); + let mut warnings = walk + .errors + .into_iter() + .map(|error| { + format!( + "failed to scan skill path {}: {}", + error.path, error.message + ) + }) + .collect::>(); + if walk.truncated { + warnings.push(format!( + "skills scan reached its traversal limit (root: {root})" + )); + } + + let skip_hidden = matches!(options.hidden_directories, HiddenDirectoryPolicy::Skip); + let mut skill_files = Vec::new(); + let mut file_paths = HashSet::new(); + let mut metadata_directory_parents = HashSet::new(); + let mut plugin_roots = HashSet::new(); + for entry in walk.entries { + if skip_hidden && has_hidden_ancestor_below_root(&entry.path, root) { + continue; + } + match entry.kind { + WalkEntryKind::Directory => { + if entry + .path + .basename() + .is_some_and(|name| name.eq_ignore_ascii_case(SKILLS_METADATA_DIR)) + && let Some(skill_dir) = entry.path.parent() + { + metadata_directory_parents.insert(skill_dir); + } + if DISCOVERABLE_PLUGIN_MANIFEST_PATHS + .iter() + .any(|path| path.split('/').next() == entry.path.basename().as_deref()) + && let Some(plugin_root) = entry.path.parent() + { + plugin_roots.insert(plugin_root); + } + } + WalkEntryKind::File => { + file_paths.insert(entry.path.clone()); + if entry.path.basename().as_deref() == Some(SKILLS_FILENAME) { + skill_files.push(entry.path); + } + } + } + } + let skills = skill_files + .into_iter() + .map(|path| DiscoveredSkill { + metadata: discover_skill_metadata( + &path, + &file_paths, + &metadata_directory_parents, + inventory_complete, + ), + path, + }) + .collect(); + + SkillDiscovery { + skills, + plugin_roots, + namespace_roots: HashSet::from([root.clone()]), + warnings, + } +} + +fn has_hidden_ancestor_below_root(path: &PathUri, root: &PathUri) -> bool { + let mut ancestor = path.parent(); + while let Some(current) = ancestor { + if ¤t == root { + return false; + } + if current.basename().is_some_and(|name| name.starts_with('.')) { + return true; + } + ancestor = current.parent(); + } + false +} + +fn discover_skill_metadata( + skill_path: &PathUri, + file_paths: &HashSet, + metadata_directory_parents: &HashSet, + inventory_complete: bool, +) -> SkillMetadataDiscovery { + let Some(skill_dir) = skill_path.parent() else { + return SkillMetadataDiscovery::Absent; + }; + let Ok(metadata_dir) = skill_dir.join(SKILLS_METADATA_DIR) else { + return SkillMetadataDiscovery::Absent; + }; + let Ok(metadata_path) = metadata_dir.join(SKILLS_METADATA_FILENAME) else { + return SkillMetadataDiscovery::Absent; + }; + if file_paths.contains(&metadata_path) { + return SkillMetadataDiscovery::Present(metadata_path); + } + + if inventory_complete && !metadata_directory_parents.contains(&skill_dir) { + SkillMetadataDiscovery::Absent + } else { + // A complete walk proves ordinary absence, but keep a filesystem probe for case aliases, + // file symlinks omitted by the walk, and incomplete inventories. + SkillMetadataDiscovery::Probe(metadata_path) + } +} diff --git a/codex-rs/core-skills/src/loader/environment.rs b/codex-rs/core-skills/src/loader/environment.rs index 8d400d7cdc..052f292651 100644 --- a/codex-rs/core-skills/src/loader/environment.rs +++ b/codex-rs/core-skills/src/loader/environment.rs @@ -1,25 +1,23 @@ -use std::collections::HashSet; use std::io; use codex_exec_server::ExecutorFileSystem; -use codex_exec_server::WalkEntryKind; -use codex_exec_server::WalkOptions; use codex_protocol::protocol::Product; use codex_utils_path_uri::PathUri; -use codex_utils_plugins::DISCOVERABLE_PLUGIN_MANIFEST_PATHS; use futures::StreamExt; use crate::model::SkillDependencies; use crate::model::SkillPolicy; use super::MAX_QUALIFIED_NAME_LEN; -use super::MAX_SCAN_DEPTH; -use super::MAX_SKILLS_DIRS_PER_ROOT; use super::ParsedSkillFrontmatter; -use super::SKILLS_FILENAME; -use super::SKILLS_METADATA_DIR; -use super::SKILLS_METADATA_FILENAME; use super::SkillMetadataFile; +use super::discovery::DirectorySymlinkPolicy; +use super::discovery::DiscoveredSkill; +use super::discovery::HiddenDirectoryPolicy; +use super::discovery::MAX_CONCURRENT_SKILL_LOADS; +use super::discovery::SkillDiscoveryOptions; +use super::discovery::SkillMetadataDiscovery; +use super::discovery::discover_skills; use super::namespace::SkillNamespaceResolver; use super::parse_skill_frontmatter_metadata_inner; use super::resolve_dependencies; @@ -27,21 +25,6 @@ use super::resolve_policy; use super::sanitize_single_line; use super::validate_len; -const MAX_SKILLS_ENTRIES_PER_ROOT: usize = 20_000; -const MAX_CONCURRENT_SKILL_LOADS: usize = 64; - -struct EnvironmentSkillDiscovery { - skills: Vec, - plugin_roots: HashSet, - namespace_roots: HashSet, - warnings: Vec, -} - -struct DiscoveredEnvironmentSkill { - path: PathUri, - metadata: SkillMetadataDiscovery, -} - struct ParsedEnvironmentSkill { path_to_skills_md: PathUri, base_name: String, @@ -51,12 +34,6 @@ struct ParsedEnvironmentSkill { policy: Option, } -enum SkillMetadataDiscovery { - Present(PathUri), - Absent, - Probe(PathUri), -} - /// URI-native metadata for one skill owned by an execution environment. #[derive(Clone, Debug, PartialEq, Eq)] pub struct EnvironmentSkillMetadata { @@ -92,7 +69,7 @@ impl EnvironmentSkillMetadata { impl ParsedEnvironmentSkill { async fn load( file_system: &dyn ExecutorFileSystem, - skill: &DiscoveredEnvironmentSkill, + skill: &DiscoveredSkill, ) -> Result { let (contents, discovered_metadata) = match &skill.metadata { SkillMetadataDiscovery::Present(metadata_path) => { @@ -152,93 +129,17 @@ pub async fn load_environment_skills_from_root( restriction_product: Option, ) -> EnvironmentSkillLoadOutcome { let mut outcome = EnvironmentSkillLoadOutcome::default(); - let discovery = match file_system - .walk( - root, - WalkOptions { - max_depth: MAX_SCAN_DEPTH, - max_directories: MAX_SKILLS_DIRS_PER_ROOT, - max_entries: MAX_SKILLS_ENTRIES_PER_ROOT, - follow_directory_symlinks: true, - prune_hidden_directories: false, - }, - /*sandbox*/ None, - ) - .await - { - Ok(walk) => { - let inventory_complete = !walk.truncated && walk.errors.is_empty(); - let mut warnings = walk - .errors - .into_iter() - .map(|error| { - format!( - "failed to scan skill path {}: {}", - error.path, error.message - ) - }) - .collect::>(); - if walk.truncated { - warnings.push(format!( - "skills scan reached its traversal limit (root: {root})" - )); - } - let mut skill_files = Vec::new(); - let mut file_paths = HashSet::new(); - let mut directory_paths = HashSet::new(); - let mut plugin_roots = HashSet::new(); - for entry in walk.entries { - match entry.kind { - WalkEntryKind::Directory => { - directory_paths.insert(entry.path.clone()); - if DISCOVERABLE_PLUGIN_MANIFEST_PATHS - .iter() - .any(|path| path.split('/').next() == entry.path.basename().as_deref()) - && let Some(plugin_root) = entry.path.parent() - { - plugin_roots.insert(plugin_root); - } - } - WalkEntryKind::File => { - file_paths.insert(entry.path.clone()); - if entry.path.basename().as_deref() == Some(SKILLS_FILENAME) { - skill_files.push(entry.path); - } - } - } - } - let skills = skill_files - .into_iter() - .map(|path| DiscoveredEnvironmentSkill { - metadata: discover_skill_metadata( - &path, - &file_paths, - &directory_paths, - inventory_complete, - ), - path, - }) - .collect(); - EnvironmentSkillDiscovery { - skills, - plugin_roots, - namespace_roots: HashSet::from([root.clone()]), - warnings, - } - } - Err(error) if error.kind() == io::ErrorKind::NotFound => EnvironmentSkillDiscovery { - skills: Vec::new(), - plugin_roots: HashSet::new(), - namespace_roots: HashSet::new(), - warnings: Vec::new(), + let discovery = discover_skills( + file_system, + root, + // Preserve environment discovery behavior by following directory aliases and including + // hidden directories exposed by the executor. + SkillDiscoveryOptions { + directory_symlinks: DirectorySymlinkPolicy::Follow, + hidden_directories: HiddenDirectoryPolicy::Include, }, - Err(error) => EnvironmentSkillDiscovery { - skills: Vec::new(), - plugin_roots: HashSet::new(), - namespace_roots: HashSet::new(), - warnings: vec![format!("failed to walk skills root {root}: {error:#}")], - }, - }; + ) + .await; tracing::Span::current().record("skill_count", discovery.skills.len()); outcome.warnings.extend(discovery.warnings); if discovery.skills.is_empty() { @@ -311,32 +212,6 @@ pub async fn load_environment_skills_from_root( outcome } -fn discover_skill_metadata( - skill_path: &PathUri, - file_paths: &HashSet, - directory_paths: &HashSet, - inventory_complete: bool, -) -> SkillMetadataDiscovery { - let Some(skill_dir) = skill_path.parent() else { - return SkillMetadataDiscovery::Absent; - }; - let Ok(metadata_dir) = skill_dir.join(SKILLS_METADATA_DIR) else { - return SkillMetadataDiscovery::Absent; - }; - let Ok(metadata_path) = metadata_dir.join(SKILLS_METADATA_FILENAME) else { - return SkillMetadataDiscovery::Absent; - }; - if file_paths.contains(&metadata_path) { - SkillMetadataDiscovery::Present(metadata_path) - } else if inventory_complete && !directory_paths.contains(&metadata_dir) { - SkillMetadataDiscovery::Absent - } else { - // The walk can omit entries after an error or traversal limit. It also omits file - // symlinks, so keep the existing probe when the metadata directory itself was observed. - SkillMetadataDiscovery::Probe(metadata_path) - } -} - async fn read_skill_contents( file_system: &dyn ExecutorFileSystem, skill_path: &PathUri, diff --git a/codex-rs/core-skills/src/loader/namespace.rs b/codex-rs/core-skills/src/loader/namespace.rs index 34d8e9b7c4..fa00b1005d 100644 --- a/codex-rs/core-skills/src/loader/namespace.rs +++ b/codex-rs/core-skills/src/loader/namespace.rs @@ -1,10 +1,12 @@ use codex_exec_server::ExecutorFileSystem; use codex_utils_path_uri::PathUri; use codex_utils_plugins::plugin_namespace_for_root_uri; -use codex_utils_plugins::plugin_namespace_for_skill_uri; -use futures::future::join_all; +use futures::StreamExt; +use std::collections::HashMap; use std::collections::HashSet; +use super::discovery::MAX_CONCURRENT_SKILL_LOADS; + /// Resolves the namespace prefix applied to skill names during one skills scan. /// /// A plugin namespace is the plugin name from the nearest valid plugin manifest @@ -55,43 +57,89 @@ impl SkillNamespaceResolver { .filter(|plugin_root| skill_ancestors.contains(plugin_root)) .collect::>(); - // Ordinary descendants fall back to the nearest valid manifest at or above the scan root. - let inherited_namespace = plugin_namespace_for_skill_uri(fs, root) - .await - .map(ResolvedSkillNamespace::Plugin) - .unwrap_or(ResolvedSkillNamespace::Plain); // The scan root is already the fallback above if nothing else matches, exclude from the search. let namespace_roots = namespace_roots .into_iter() .filter(|namespace_root| namespace_root != root) .collect::>(); let namespace_root_set = namespace_roots.iter().cloned().collect::>(); - // Keep independent probes concurrent for remote executor latency. - let namespace_lookups = join_all(namespace_roots.into_iter().map(|namespace_root| async { - let namespace = plugin_namespace_for_skill_uri(fs, &namespace_root) - .await + let plugin_roots = plugin_roots + .into_iter() + .filter(|plugin_root| plugin_root != root && !namespace_root_set.contains(plugin_root)) + .collect::>(); + + let lookup_roots = std::iter::once(root.clone()) + .chain(namespace_roots.iter().cloned()) + .collect::>(); + let mut pending_lookups = lookup_roots + .iter() + .cloned() + .map(|lookup_root| (lookup_root.clone(), lookup_root)) + .collect::>(); + let mut direct_plugin_roots = plugin_roots.iter().cloned().collect::>(); + let mut namespaces_by_root = HashMap::new(); + let mut namespaces_by_lookup_root = HashMap::new(); + while !pending_lookups.is_empty() { + let probe_roots = pending_lookups + .iter() + .map(|(_, ancestor)| ancestor.clone()) + .chain(direct_plugin_roots.drain()) + .filter(|ancestor| !namespaces_by_root.contains_key(ancestor)) + .collect::>(); + namespaces_by_root.extend( + futures::stream::iter(probe_roots) + .map(|manifest_root| async move { + let namespace = plugin_namespace_for_root_uri(fs, &manifest_root).await; + (manifest_root, namespace) + }) + .buffered(MAX_CONCURRENT_SKILL_LOADS) + .collect::>() + .await, + ); + + let mut next_lookups = Vec::new(); + for (lookup_root, ancestor) in pending_lookups { + match namespaces_by_root.get(&ancestor) { + Some(Some(namespace)) => { + namespaces_by_lookup_root.insert(lookup_root, Some(namespace.clone())); + } + Some(None) => match ancestor.parent() { + Some(parent) => next_lookups.push((lookup_root, parent)), + None => { + namespaces_by_lookup_root.insert(lookup_root, None); + } + }, + None => unreachable!("pending namespace ancestor was not probed"), + } + } + pending_lookups = next_lookups; + } + + // Ordinary descendants fall back to the nearest valid manifest at or above the scan root. + let inherited_namespace = namespaces_by_lookup_root + .get(root) + .and_then(Option::as_ref) + .cloned() + .map(ResolvedSkillNamespace::Plugin) + .unwrap_or(ResolvedSkillNamespace::Plain); + let namespace_lookups = namespace_roots.into_iter().map(|namespace_root| { + let namespace = namespaces_by_lookup_root + .get(&namespace_root) + .and_then(Option::as_ref) + .cloned() .map(ResolvedSkillNamespace::Plugin) .unwrap_or(ResolvedSkillNamespace::Plain); (namespace_root, namespace) - })); + }); // Invalid nested manifests are omitted, so the deepest remaining match wins. - let plugin_lookups = join_all( - plugin_roots - .into_iter() - .filter(|plugin_root| { - plugin_root != root && !namespace_root_set.contains(plugin_root) - }) - .map(|plugin_root| async move { - plugin_namespace_for_root_uri(fs, &plugin_root) - .await - .map(|namespace| (plugin_root, ResolvedSkillNamespace::Plugin(namespace))) - }), - ); - let (namespace_lookups, plugin_lookups) = tokio::join!(namespace_lookups, plugin_lookups); - let nested_namespaces = namespace_lookups - .into_iter() - .chain(plugin_lookups.into_iter().flatten()) - .collect(); + let plugin_lookups = plugin_roots.into_iter().filter_map(|plugin_root| { + namespaces_by_root + .get(&plugin_root) + .and_then(Option::as_ref) + .cloned() + .map(|namespace| (plugin_root, ResolvedSkillNamespace::Plugin(namespace))) + }); + let nested_namespaces = namespace_lookups.chain(plugin_lookups).collect(); Self { inherited_namespace, diff --git a/codex-rs/core-skills/src/loader_tests.rs b/codex-rs/core-skills/src/loader_tests.rs index 1bdfe39520..3122dbaf0b 100644 --- a/codex-rs/core-skills/src/loader_tests.rs +++ b/codex-rs/core-skills/src/loader_tests.rs @@ -1021,6 +1021,32 @@ async fn loads_skills_via_symlinked_subdir_for_user_scope() { ); } +// Directory symlinks on Windows can require Developer Mode or administrator privileges. +#[tokio::test] +#[cfg(unix)] +async fn loads_skills_through_visible_alias_to_hidden_directory() { + let root = tempfile::tempdir().expect("tempdir"); + let hidden_root = root.path().join(".hidden"); + let skill_path = write_skill_at(&hidden_root, "search", "search-skill", "search description"); + symlink_dir(&hidden_root, &root.path().join("visible")); + + let outcome = load_user_skills_root(root.path()).await; + + assert!( + outcome.errors.is_empty(), + "unexpected errors: {:?}", + outcome.errors + ); + assert_eq!( + outcome.skills, + vec![expected_user_skill( + &skill_path, + "search-skill", + "search description", + )] + ); +} + #[tokio::test] #[cfg(unix)] async fn ignores_symlinked_skill_file_for_user_scope() { @@ -1485,48 +1511,6 @@ async fn invalid_nested_plugin_manifest_falls_back_to_outer_namespace() { ); } -// Directory symlinks on Windows can require Developer Mode or administrator privileges. -#[cfg(unix)] -#[tokio::test] -async fn namespaces_skills_in_symlinked_plugin_skills_dir() { - // skills/ - // └── linked-plugin -> shared-plugin/skills/ - // shared-plugin/ - // ├── .codex-plugin/plugin.json - // └── skills/search/SKILL.md - let root = tempfile::tempdir().expect("tempdir"); - let shared_plugin_root = tempfile::tempdir().expect("tempdir"); - write_plugin_manifest(shared_plugin_root.path(), r#"{"name":"linked"}"#); - let skill_path = write_skill_at( - &shared_plugin_root.path().join("skills"), - "search", - "search-skill", - "search description", - ); - let skills_root = root.path().join("skills"); - fs::create_dir_all(&skills_root).unwrap(); - symlink_dir( - &shared_plugin_root.path().join("skills"), - &skills_root.join("linked-plugin"), - ); - - let outcome = load_user_skills_root(&skills_root).await; - - assert!( - outcome.errors.is_empty(), - "unexpected errors: {:?}", - outcome.errors - ); - assert_eq!( - outcome.skills, - vec![expected_user_skill( - &skill_path, - "linked:search-skill", - "search description", - )] - ); -} - // Directory symlinks on Windows can require Developer Mode or administrator privileges. #[cfg(unix)] #[tokio::test] diff --git a/codex-rs/core-skills/tests/environment_loader.rs b/codex-rs/core-skills/tests/environment_loader.rs index 74efc6e1a3..a2c9903aea 100644 --- a/codex-rs/core-skills/tests/environment_loader.rs +++ b/codex-rs/core-skills/tests/environment_loader.rs @@ -411,3 +411,130 @@ async fn reads_skill_files_while_resolving_plugin_namespaces() { }] ); } + +#[cfg(unix)] +#[tokio::test] +async fn host_loading_reuses_walk_inventory_for_symlinked_skill_pack() { + use std::os::unix::fs::symlink; + use std::sync::Arc; + + use codex_core_skills::SkillMetadata; + use codex_core_skills::SkillPolicy; + use codex_core_skills::loader::SkillRoot; + use codex_core_skills::loader::load_skills_from_roots; + use codex_protocol::protocol::SkillScope; + use codex_utils_absolute_path::test_support::PathBufExt; + + let root = tempdir().expect("tempdir"); + let shared_plugin_root = tempdir().expect("tempdir"); + let manifest_path = shared_plugin_root.path().join(".codex-plugin/plugin.json"); + fs::create_dir_all(manifest_path.parent().expect("manifest parent")).expect("manifest dir"); + fs::write(&manifest_path, r#"{"name":"linked"}"#).expect("manifest"); + + let skills_root = shared_plugin_root.path().join("skills"); + for name in ["first", "second"] { + let skill_path = skills_root.join(name).join("SKILL.md"); + fs::create_dir_all(skill_path.parent().expect("skill parent")).expect("skill dir"); + fs::write( + &skill_path, + format!("---\nname: {name}\ndescription: {name} skill.\n---\n"), + ) + .expect("skill"); + } + let metadata_path = skills_root.join("first/agents/openai.yaml"); + fs::create_dir_all(metadata_path.parent().expect("metadata parent")).expect("metadata dir"); + fs::write( + &metadata_path, + "policy:\n allow_implicit_invocation: false\n", + ) + .expect("metadata"); + + let host_root = root.path().join("skills"); + fs::create_dir_all(&host_root).expect("host skills dir"); + let linked_root = host_root.join("linked-plugin"); + symlink(&skills_root, &linked_root).expect("skill pack symlink"); + + let recording = Arc::new(RecordingFileSystem::new( + LOCAL_FS.as_ref(), + ManifestMetadataBehavior::Immediate, + )); + let file_system: Arc = recording.clone(); + let future = load_skills_from_roots( + [SkillRoot { + path: host_root.abs(), + scope: SkillScope::User, + file_system, + plugin_id: None, + plugin_namespace: None, + plugin_root: None, + }], + /*plugin_skill_snapshots*/ None, + ); + fn assert_send(_: &T) {} + assert_send(&future); + let outcome = future.await; + + assert_eq!(outcome.errors, Vec::new()); + let first_skill_path = dunce::canonicalize(skills_root.join("first/SKILL.md")) + .unwrap() + .abs(); + let second_skill_path = dunce::canonicalize(skills_root.join("second/SKILL.md")) + .unwrap() + .abs(); + assert_eq!( + outcome.skills, + vec![ + SkillMetadata { + name: "linked:first".to_string(), + description: "first skill.".to_string(), + short_description: None, + interface: None, + dependencies: None, + policy: Some(SkillPolicy { + allow_implicit_invocation: Some(false), + products: Vec::new(), + }), + path_to_skills_md: first_skill_path, + scope: SkillScope::User, + plugin_id: None, + }, + SkillMetadata { + name: "linked:second".to_string(), + description: "second skill.".to_string(), + short_description: None, + interface: None, + dependencies: None, + policy: None, + path_to_skills_md: second_skill_path, + scope: SkillScope::User, + plugin_id: None, + }, + ] + ); + + let calls = recording.calls(); + assert_eq!(calls.walks, 1); + let linked_root = PathUri::from_host_native_path(linked_root).unwrap(); + assert!( + calls + .read_files + .iter() + .all(|path| !path.starts_with(&linked_root)) + ); + assert!( + calls + .metadata_files + .iter() + .all(|path| path.basename().as_deref() != Some("openai.yaml")) + ); + let manifest_uri = + PathUri::from_host_native_path(dunce::canonicalize(manifest_path).unwrap()).unwrap(); + assert_eq!( + calls + .metadata_files + .iter() + .filter(|path| **path == manifest_uri) + .count(), + 1 + ); +} diff --git a/codex-rs/core/src/codex_thread.rs b/codex-rs/core/src/codex_thread.rs index c80268e2f9..257a32733b 100644 --- a/codex-rs/core/src/codex_thread.rs +++ b/codex-rs/core/src/codex_thread.rs @@ -4,6 +4,7 @@ use crate::elicitation::ElicitationRegistration; use crate::session::Codex; use crate::session::SessionSettingsUpdate; use crate::session::SteerInputError; +use codex_exec_server::SelectedCapabilityRootsStatus; use codex_features::Feature; use codex_otel::SessionTelemetry; use codex_protocol::ThreadId; @@ -629,6 +630,18 @@ impl CodexThread { self.codex.thread_environment_selections().await } + /// Passively inspects the selected capability roots whose environments are ready now. + pub fn inspect_selected_capability_roots(&self) -> SelectedCapabilityRootsStatus { + self.codex + .session + .services + .turn_environments + .environment_manager() + .inspect_selected_capability_roots( + &self.codex.session.services.selected_capability_roots, + ) + } + pub async fn read_mcp_resource( &self, server: &str, diff --git a/codex-rs/core/src/config/config_tests.rs b/codex-rs/core/src/config/config_tests.rs index 0c8d0899a4..1df2bb3e3f 100644 --- a/codex-rs/core/src/config/config_tests.rs +++ b/codex-rs/core/src/config/config_tests.rs @@ -1182,16 +1182,16 @@ fn permissions_profile_network_to_proxy_config_preserves_mitm_hooks() { let config = network.to_network_proxy_config(); - assert_eq!(config.network.mode, NetworkMode::Full); - assert!(config.network.mitm); - assert_eq!(config.network.mitm_hooks.len(), 1); - assert_eq!(config.network.mitm_hooks[0].host, "api.github.com"); + assert_eq!(config.mode, NetworkMode::Full); + assert!(config.mitm); + assert_eq!(config.mitm_hooks.len(), 1); + assert_eq!(config.mitm_hooks[0].host, "api.github.com"); assert_eq!( - config.network.mitm_hooks[0].matcher.methods, + config.mitm_hooks[0].matcher.methods, vec!["POST".to_string()] ); assert_eq!( - config.network.mitm_hooks[0].actions.strip_request_headers, + config.mitm_hooks[0].actions.strip_request_headers, vec!["authorization".to_string()] ); } @@ -1228,13 +1228,13 @@ action = ["noop"] let config = network.to_network_proxy_config(); - assert_eq!(config.network.mitm_hooks.len(), 2); + assert_eq!(config.mitm_hooks.len(), 2); assert_eq!( - config.network.mitm_hooks[0].matcher.path_prefixes, + config.mitm_hooks[0].matcher.path_prefixes, vec!["/repos/openai/".to_string()] ); assert_eq!( - config.network.mitm_hooks[1].matcher.path_prefixes, + config.mitm_hooks[1].matcher.path_prefixes, vec!["/repos/".to_string()] ); } diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index 6348f14c04..5cd6748c05 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -3368,7 +3368,7 @@ impl Config { network_proxy, ); } - configured_network_proxy_config.network.enabled = true; + configured_network_proxy_config.enabled = true; } let approval_policy_was_explicit = approval_policy_override.is_some() || cfg.approval_policy.is_some(); @@ -4128,7 +4128,7 @@ impl Config { network_proxy, ); } - configured_network_proxy_config.network.enabled = true; + configured_network_proxy_config.enabled = true; } configured_network_proxy_config } else { diff --git a/codex-rs/core/src/config/network_proxy_spec.rs b/codex-rs/core/src/config/network_proxy_spec.rs index 653e6987f3..a5cef5ca72 100644 --- a/codex-rs/core/src/config/network_proxy_spec.rs +++ b/codex-rs/core/src/config/network_proxy_spec.rs @@ -74,15 +74,15 @@ impl ConfigReloader for StaticNetworkProxyReloader { impl NetworkProxySpec { pub(crate) fn enabled(&self) -> bool { - self.config.network.enabled + self.config.enabled } pub fn proxy_host_and_port(&self) -> String { - host_and_port_from_network_addr(&self.config.network.proxy_url, /*default_port*/ 3128) + host_and_port_from_network_addr(&self.config.proxy_url, /*default_port*/ 3128) } pub fn socks_enabled(&self) -> bool { - self.config.network.enable_socks5 + self.config.enable_socks5 } pub(crate) fn from_config_and_constraints( @@ -221,31 +221,30 @@ impl NetworkProxySpec { let denylist_expansion_enabled = Self::denylist_expansion_enabled(permission_profile); if let Some(enabled) = requirements.enabled { - config.network.enabled = enabled; + config.enabled = enabled; constraints.enabled = Some(enabled); } if let Some(http_port) = requirements.http_port { - config.network.proxy_url = format!("http://127.0.0.1:{http_port}"); + config.proxy_url = format!("http://127.0.0.1:{http_port}"); } if let Some(socks_port) = requirements.socks_port { - config.network.socks_url = format!("http://127.0.0.1:{socks_port}"); + config.socks_url = format!("http://127.0.0.1:{socks_port}"); } if let Some(allow_upstream_proxy) = requirements.allow_upstream_proxy { - config.network.allow_upstream_proxy = allow_upstream_proxy; + config.allow_upstream_proxy = allow_upstream_proxy; constraints.allow_upstream_proxy = Some(allow_upstream_proxy); } if let Some(dangerously_allow_non_loopback_proxy) = requirements.dangerously_allow_non_loopback_proxy { - config.network.dangerously_allow_non_loopback_proxy = - dangerously_allow_non_loopback_proxy; + config.dangerously_allow_non_loopback_proxy = dangerously_allow_non_loopback_proxy; constraints.dangerously_allow_non_loopback_proxy = Some(dangerously_allow_non_loopback_proxy); } if let Some(dangerously_allow_all_unix_sockets) = requirements.dangerously_allow_all_unix_sockets { - config.network.dangerously_allow_all_unix_sockets = dangerously_allow_all_unix_sockets; + config.dangerously_allow_all_unix_sockets = dangerously_allow_all_unix_sockets; constraints.dangerously_allow_all_unix_sockets = Some(dangerously_allow_all_unix_sockets); } @@ -270,14 +269,12 @@ impl NetworkProxySpec { let effective_allowed_domains = if allowlist_expansion_enabled { Self::merge_domain_lists( managed_allowed_domains.clone(), - config.network.allowed_domains().as_deref().unwrap_or(&[]), + config.allowed_domains().as_deref().unwrap_or(&[]), ) } else { managed_allowed_domains.clone() }; - config - .network - .set_allowed_domains(effective_allowed_domains); + config.set_allowed_domains(effective_allowed_domains); constraints.allowed_domains = Some(managed_allowed_domains); constraints.allowlist_expansion_enabled = Some(allowlist_expansion_enabled); } @@ -289,12 +286,12 @@ impl NetworkProxySpec { let effective_denied_domains = if denylist_expansion_enabled { Self::merge_domain_lists( managed_denied_domains.clone(), - config.network.denied_domains().as_deref().unwrap_or(&[]), + config.denied_domains().as_deref().unwrap_or(&[]), ) } else { managed_denied_domains.clone() }; - config.network.set_denied_domains(effective_denied_domains); + config.set_denied_domains(effective_denied_domains); constraints.denied_domains = Some(managed_denied_domains); constraints.denylist_expansion_enabled = Some(denylist_expansion_enabled); } @@ -304,13 +301,11 @@ impl NetworkProxySpec { .as_ref() .map(codex_config::NetworkUnixSocketPermissionsToml::allow_unix_sockets) .unwrap_or_default(); - config - .network - .set_allow_unix_sockets(allow_unix_sockets.clone()); + config.set_allow_unix_sockets(allow_unix_sockets.clone()); constraints.allow_unix_sockets = Some(allow_unix_sockets); } if let Some(allow_local_binding) = requirements.allow_local_binding { - config.network.allow_local_binding = allow_local_binding; + config.allow_local_binding = allow_local_binding; constraints.allow_local_binding = Some(allow_local_binding); } @@ -359,7 +354,7 @@ fn upsert_network_domains(config: &mut NetworkProxyConfig, hosts: Vec, a let mut incoming = HashSet::new(); for host in hosts { if incoming.insert(host.clone()) { - config.network.upsert_domain_permission( + config.upsert_domain_permission( host, if allow { codex_network_proxy::NetworkDomainPermission::Allow diff --git a/codex-rs/core/src/config/network_proxy_spec_tests.rs b/codex-rs/core/src/config/network_proxy_spec_tests.rs index 6dfb1e3a25..ff90af0c74 100644 --- a/codex-rs/core/src/config/network_proxy_spec_tests.rs +++ b/codex-rs/core/src/config/network_proxy_spec_tests.rs @@ -43,9 +43,7 @@ fn build_state_with_audit_metadata_threads_metadata_to_state() { #[test] fn requirements_allowed_domains_are_a_baseline_for_user_allowlist() { let mut config = NetworkProxyConfig::default(); - config - .network - .set_allowed_domains(vec!["api.example.com".to_string()]); + config.set_allowed_domains(vec!["api.example.com".to_string()]); let requirements = NetworkConstraints { domains: Some(domain_permissions([( "*.example.com", @@ -62,7 +60,7 @@ fn requirements_allowed_domains_are_a_baseline_for_user_allowlist() { .expect("config should stay within the managed allowlist"); assert_eq!( - spec.config.network.allowed_domains(), + spec.config.allowed_domains(), Some(vec![ "*.example.com".to_string(), "api.example.com".to_string() @@ -78,9 +76,7 @@ fn requirements_allowed_domains_are_a_baseline_for_user_allowlist() { #[test] fn requirements_allowed_domains_do_not_override_user_denies_for_same_pattern() { let mut config = NetworkProxyConfig::default(); - config - .network - .set_denied_domains(vec!["api.example.com".to_string()]); + config.set_denied_domains(vec!["api.example.com".to_string()]); let requirements = NetworkConstraints { domains: Some(domain_permissions([( "api.example.com", @@ -96,9 +92,9 @@ fn requirements_allowed_domains_do_not_override_user_denies_for_same_pattern() { ) .expect("managed allowlist should not erase a user deny"); - assert_eq!(spec.config.network.allowed_domains(), None); + assert_eq!(spec.config.allowed_domains(), None); assert_eq!( - spec.config.network.denied_domains(), + spec.config.denied_domains(), Some(vec!["api.example.com".to_string()]) ); assert_eq!( @@ -110,9 +106,7 @@ fn requirements_allowed_domains_do_not_override_user_denies_for_same_pattern() { #[test] fn requirements_allowlist_expansion_keeps_user_entries_mutable() { let mut config = NetworkProxyConfig::default(); - config - .network - .set_allowed_domains(vec!["api.example.com".to_string()]); + config.set_allowed_domains(vec!["api.example.com".to_string()]); let requirements = NetworkConstraints { domains: Some(domain_permissions([( "*.example.com", @@ -129,18 +123,18 @@ fn requirements_allowlist_expansion_keeps_user_entries_mutable() { .expect("managed baseline should still allow user edits"); let mut candidate = spec.config.clone(); - candidate.network.upsert_domain_permission( + candidate.upsert_domain_permission( "api.example.com".to_string(), NetworkDomainPermission::Deny, normalize_host, ); assert_eq!( - candidate.network.allowed_domains(), + candidate.allowed_domains(), Some(vec!["*.example.com".to_string()]) ); assert_eq!( - candidate.network.denied_domains(), + candidate.denied_domains(), Some(vec!["api.example.com".to_string()]) ); validate_policy_against_constraints(&candidate, &spec.constraints) @@ -150,9 +144,7 @@ fn requirements_allowlist_expansion_keeps_user_entries_mutable() { #[test] fn managed_unrestricted_profile_allows_domain_expansion() { let mut config = NetworkProxyConfig::default(); - config - .network - .set_allowed_domains(vec!["api.example.com".to_string()]); + config.set_allowed_domains(vec!["api.example.com".to_string()]); let requirements = NetworkConstraints { domains: Some(domain_permissions([( "*.example.com", @@ -173,7 +165,7 @@ fn managed_unrestricted_profile_allows_domain_expansion() { .expect("managed unrestricted filesystem should still use managed network constraints"); assert_eq!( - spec.config.network.allowed_domains(), + spec.config.allowed_domains(), Some(vec![ "*.example.com".to_string(), "api.example.com".to_string() @@ -185,12 +177,8 @@ fn managed_unrestricted_profile_allows_domain_expansion() { #[test] fn danger_full_access_keeps_managed_allowlist_and_denylist_fixed() { let mut config = NetworkProxyConfig::default(); - config - .network - .set_allowed_domains(vec!["evil.com".to_string()]); - config - .network - .set_denied_domains(vec!["more-blocked.example.com".to_string()]); + config.set_allowed_domains(vec!["evil.com".to_string()]); + config.set_denied_domains(vec!["more-blocked.example.com".to_string()]); let requirements = NetworkConstraints { domains: Some(domain_permissions([ ("*.example.com", NetworkDomainPermissionToml::Allow), @@ -207,11 +195,11 @@ fn danger_full_access_keeps_managed_allowlist_and_denylist_fixed() { .expect("yolo mode should pin the effective policy to the managed baseline"); assert_eq!( - spec.config.network.allowed_domains(), + spec.config.allowed_domains(), Some(vec!["*.example.com".to_string()]) ); assert_eq!( - spec.config.network.denied_domains(), + spec.config.denied_domains(), Some(vec!["blocked.example.com".to_string()]) ); assert_eq!(spec.constraints.allowlist_expansion_enabled, Some(false)); @@ -221,9 +209,7 @@ fn danger_full_access_keeps_managed_allowlist_and_denylist_fixed() { #[test] fn managed_allowed_domains_only_disables_default_mode_allowlist_expansion() { let mut config = NetworkProxyConfig::default(); - config - .network - .set_allowed_domains(vec!["api.example.com".to_string()]); + config.set_allowed_domains(vec!["api.example.com".to_string()]); let requirements = NetworkConstraints { domains: Some(domain_permissions([( "*.example.com", @@ -241,7 +227,7 @@ fn managed_allowed_domains_only_disables_default_mode_allowlist_expansion() { .expect("managed baseline should still load"); assert_eq!( - spec.config.network.allowed_domains(), + spec.config.allowed_domains(), Some(vec!["*.example.com".to_string()]) ); assert_eq!(spec.constraints.allowlist_expansion_enabled, Some(false)); @@ -250,9 +236,7 @@ fn managed_allowed_domains_only_disables_default_mode_allowlist_expansion() { #[test] fn managed_allowed_domains_only_ignores_user_allowlist_and_hard_denies_misses() { let mut config = NetworkProxyConfig::default(); - config - .network - .set_allowed_domains(vec!["api.example.com".to_string()]); + config.set_allowed_domains(vec!["api.example.com".to_string()]); let requirements = NetworkConstraints { domains: Some(domain_permissions([( "managed.example.com", @@ -270,7 +254,7 @@ fn managed_allowed_domains_only_ignores_user_allowlist_and_hard_denies_misses() .expect("managed-only allowlist should still load"); assert_eq!( - spec.config.network.allowed_domains(), + spec.config.allowed_domains(), Some(vec!["managed.example.com".to_string()]) ); assert_eq!( @@ -284,9 +268,7 @@ fn managed_allowed_domains_only_ignores_user_allowlist_and_hard_denies_misses() #[test] fn managed_allowed_domains_only_without_managed_allowlist_blocks_all_user_domains() { let mut config = NetworkProxyConfig::default(); - config - .network - .set_allowed_domains(vec!["api.example.com".to_string()]); + config.set_allowed_domains(vec!["api.example.com".to_string()]); let requirements = NetworkConstraints { managed_allowed_domains_only: Some(true), ..Default::default() @@ -299,7 +281,7 @@ fn managed_allowed_domains_only_without_managed_allowlist_blocks_all_user_domain ) .expect("managed-only mode should treat missing managed allowlist as empty"); - assert_eq!(spec.config.network.allowed_domains(), None); + assert_eq!(spec.config.allowed_domains(), None); assert_eq!(spec.constraints.allowed_domains, Some(Vec::new())); assert_eq!(spec.constraints.allowlist_expansion_enabled, Some(false)); assert!(spec.hard_deny_allowlist_misses); @@ -308,9 +290,7 @@ fn managed_allowed_domains_only_without_managed_allowlist_blocks_all_user_domain #[test] fn managed_allowed_domains_only_blocks_all_user_domains_in_full_access_without_managed_list() { let mut config = NetworkProxyConfig::default(); - config - .network - .set_allowed_domains(vec!["api.example.com".to_string()]); + config.set_allowed_domains(vec!["api.example.com".to_string()]); let requirements = NetworkConstraints { managed_allowed_domains_only: Some(true), ..Default::default() @@ -323,7 +303,7 @@ fn managed_allowed_domains_only_blocks_all_user_domains_in_full_access_without_m ) .expect("managed-only mode should treat missing managed allowlist as empty"); - assert_eq!(spec.config.network.allowed_domains(), None); + assert_eq!(spec.config.allowed_domains(), None); assert_eq!(spec.constraints.allowed_domains, Some(Vec::new())); assert_eq!(spec.constraints.allowlist_expansion_enabled, Some(false)); assert!(spec.hard_deny_allowlist_misses); @@ -332,9 +312,7 @@ fn managed_allowed_domains_only_blocks_all_user_domains_in_full_access_without_m #[test] fn deny_only_requirements_do_not_create_allow_constraints_in_full_access() { let mut config = NetworkProxyConfig::default(); - config - .network - .set_allowed_domains(vec!["api.example.com".to_string()]); + config.set_allowed_domains(vec!["api.example.com".to_string()]); let requirements = NetworkConstraints { domains: Some(domain_permissions([( "managed-blocked.example.com", @@ -351,13 +329,13 @@ fn deny_only_requirements_do_not_create_allow_constraints_in_full_access() { .expect("deny-only requirements should not constrain the allowlist"); assert_eq!( - spec.config.network.allowed_domains(), + spec.config.allowed_domains(), Some(vec!["api.example.com".to_string()]) ); assert_eq!(spec.constraints.allowed_domains, None); assert_eq!(spec.constraints.allowlist_expansion_enabled, None); assert_eq!( - spec.config.network.denied_domains(), + spec.config.denied_domains(), Some(vec!["managed-blocked.example.com".to_string()]) ); } @@ -365,9 +343,7 @@ fn deny_only_requirements_do_not_create_allow_constraints_in_full_access() { #[test] fn allow_only_requirements_do_not_create_deny_constraints_in_full_access() { let mut config = NetworkProxyConfig::default(); - config - .network - .set_denied_domains(vec!["blocked.example.com".to_string()]); + config.set_denied_domains(vec!["blocked.example.com".to_string()]); let requirements = NetworkConstraints { domains: Some(domain_permissions([( "managed.example.com", @@ -384,11 +360,11 @@ fn allow_only_requirements_do_not_create_deny_constraints_in_full_access() { .expect("allow-only requirements should not constrain the denylist"); assert_eq!( - spec.config.network.allowed_domains(), + spec.config.allowed_domains(), Some(vec!["managed.example.com".to_string()]) ); assert_eq!( - spec.config.network.denied_domains(), + spec.config.denied_domains(), Some(vec!["blocked.example.com".to_string()]) ); assert_eq!(spec.constraints.denied_domains, None); @@ -398,9 +374,7 @@ fn allow_only_requirements_do_not_create_deny_constraints_in_full_access() { #[test] fn requirements_denied_domains_are_a_baseline_for_default_mode() { let mut config = NetworkProxyConfig::default(); - config - .network - .set_denied_domains(vec!["blocked.example.com".to_string()]); + config.set_denied_domains(vec!["blocked.example.com".to_string()]); let requirements = NetworkConstraints { domains: Some(domain_permissions([( "managed-blocked.example.com", @@ -417,7 +391,7 @@ fn requirements_denied_domains_are_a_baseline_for_default_mode() { .expect("default mode should merge managed and user deny entries"); assert_eq!( - spec.config.network.denied_domains(), + spec.config.denied_domains(), Some(vec![ "managed-blocked.example.com".to_string(), "blocked.example.com".to_string() @@ -433,9 +407,7 @@ fn requirements_denied_domains_are_a_baseline_for_default_mode() { #[test] fn requirements_denylist_expansion_keeps_user_entries_mutable() { let mut config = NetworkProxyConfig::default(); - config - .network - .set_denied_domains(vec!["blocked.example.com".to_string()]); + config.set_denied_domains(vec!["blocked.example.com".to_string()]); let requirements = NetworkConstraints { domains: Some(domain_permissions([( "managed-blocked.example.com", @@ -452,18 +424,18 @@ fn requirements_denylist_expansion_keeps_user_entries_mutable() { .expect("managed baseline should still allow user edits"); let mut candidate = spec.config.clone(); - candidate.network.upsert_domain_permission( + candidate.upsert_domain_permission( "blocked.example.com".to_string(), NetworkDomainPermission::Allow, normalize_host, ); assert_eq!( - candidate.network.allowed_domains(), + candidate.allowed_domains(), Some(vec!["blocked.example.com".to_string()]) ); assert_eq!( - candidate.network.denied_domains(), + candidate.denied_domains(), Some(vec!["managed-blocked.example.com".to_string()]) ); validate_policy_against_constraints(&candidate, &spec.constraints) diff --git a/codex-rs/core/src/config/permissions.rs b/codex-rs/core/src/config/permissions.rs index 9c6d3ed725..1f0eb1e12d 100644 --- a/codex-rs/core/src/config/permissions.rs +++ b/codex-rs/core/src/config/permissions.rs @@ -127,7 +127,7 @@ pub(crate) fn network_proxy_config_from_profile_network( // Profile `network.enabled` controls sandbox network access. Profiles may // provide proxy settings for the feature gate to consume when that network // access is enabled, but they do not start the managed proxy on their own. - config.network.enabled = false; + config.enabled = false; config } diff --git a/codex-rs/core/src/config/permissions_tests.rs b/codex-rs/core/src/config/permissions_tests.rs index 2c1f649042..9c78a1d5e2 100644 --- a/codex-rs/core/src/config/permissions_tests.rs +++ b/codex-rs/core/src/config/permissions_tests.rs @@ -220,7 +220,7 @@ fn network_toml_overlays_unix_socket_permissions_by_path() { .apply_to_network_proxy_config(&mut config); assert_eq!( - config.network.unix_sockets, + config.unix_sockets, Some(codex_network_proxy::NetworkUnixSocketPermissions { entries: BTreeMap::from([ ( @@ -399,7 +399,7 @@ fn profile_network_proxy_config_keeps_proxy_disabled_for_bare_network_access() { ..Default::default() })); - assert!(!config.network.enabled); + assert!(!config.enabled); } #[test] @@ -417,11 +417,11 @@ fn profile_network_proxy_config_keeps_proxy_disabled_for_proxy_policy() { ..Default::default() })); - assert!(!config.network.enabled); - assert_eq!(config.network.proxy_url, "http://127.0.0.1:43128"); - assert!(!config.network.enable_socks5); + assert!(!config.enabled); + assert_eq!(config.proxy_url, "http://127.0.0.1:43128"); + assert!(!config.enable_socks5); assert_eq!( - config.network.domains, + config.domains, Some(codex_network_proxy::NetworkDomainPermissions { entries: vec![codex_network_proxy::NetworkDomainPermissionEntry { pattern: "openai.com".to_string(), diff --git a/codex-rs/core/src/guardian/tests.rs b/codex-rs/core/src/guardian/tests.rs index e23475d30e..22a11ee5f1 100644 --- a/codex-rs/core/src/guardian/tests.rs +++ b/codex-rs/core/src/guardian/tests.rs @@ -2974,11 +2974,11 @@ async fn guardian_review_session_config_clears_legacy_notify() { #[tokio::test] async fn guardian_review_session_config_uses_live_network_proxy_state() { let mut parent_config = test_config().await; - let mut parent_network = NetworkProxyConfig::default(); - parent_network.network.enabled = true; - parent_network - .network - .set_allowed_domains(vec!["parent.example".to_string()]); + let mut parent_network = NetworkProxyConfig { + enabled: true, + ..Default::default() + }; + parent_network.set_allowed_domains(vec!["parent.example".to_string()]); parent_config.permissions.network = Some( NetworkProxySpec::from_config_and_constraints( parent_network, @@ -2988,11 +2988,11 @@ async fn guardian_review_session_config_uses_live_network_proxy_state() { .expect("parent network proxy spec"), ); - let mut live_network = NetworkProxyConfig::default(); - live_network.network.enabled = true; - live_network - .network - .set_allowed_domains(vec!["github.com".to_string()]); + let mut live_network = NetworkProxyConfig { + enabled: true, + ..Default::default() + }; + live_network.set_allowed_domains(vec!["github.com".to_string()]); let guardian_config = build_guardian_review_session_config_for_test( &parent_config, diff --git a/codex-rs/core/src/mcp.rs b/codex-rs/core/src/mcp.rs index 835c47ff51..bd09ad8564 100644 --- a/codex-rs/core/src/mcp.rs +++ b/codex-rs/core/src/mcp.rs @@ -79,9 +79,15 @@ impl McpManager { /// Returns the MCP config after applying compatibility built-ins and /// runtime-only extension overlays. pub async fn runtime_config(&self, config: &Config) -> McpConfig { - self.runtime_config_with_context(McpServerContributionContext::global(config)) - .await - .config + self.runtime_config_with_context( + McpServerContributionContext::global(config), + // Threadless discovery and control-plane paths have no effective thread + // originator; active-thread tool calls use runtime_config_for_step below. + /*originator*/ + None, + ) + .await + .config } pub(crate) async fn runtime_config_for_step( @@ -89,20 +95,26 @@ impl McpManager { config: &Config, thread_init: &ExtensionDataInit, thread_store: &ExtensionData, + originator: &str, available_environment_ids: &[String], ) -> McpRuntimeProjection { - self.runtime_config_with_context(McpServerContributionContext::for_step( - config, - thread_init, - thread_store, - available_environment_ids, - )) + self.runtime_config_with_context( + McpServerContributionContext::for_step( + config, + thread_init, + thread_store, + originator, + available_environment_ids, + ), + Some(originator), + ) .await } async fn runtime_config_with_context( &self, context: McpServerContributionContext<'_, Config>, + originator: Option<&str>, ) -> McpRuntimeProjection { let config = context.config(); let mut selected_plugin_available = false; @@ -181,6 +193,7 @@ impl McpManager { codex_apps_mcp_server_config( &mcp_config.chatgpt_base_url, mcp_config.apps_mcp_product_sku.as_deref(), + originator, ), )); } else { diff --git a/codex-rs/core/src/mcp_tool_call_tests.rs b/codex-rs/core/src/mcp_tool_call_tests.rs index 99d935c7aa..dba271e2ae 100644 --- a/codex-rs/core/src/mcp_tool_call_tests.rs +++ b/codex-rs/core/src/mcp_tool_call_tests.rs @@ -1435,6 +1435,7 @@ async fn host_owned_codex_apps_manager( codex_mcp::EffectiveMcpServer::configured(codex_mcp::codex_apps_mcp_server_config( "https://chatgpt.com", /*apps_mcp_product_sku*/ None, + Some(&turn_context.originator), )), )]); let manager = codex_mcp::McpConnectionManager::new( diff --git a/codex-rs/core/src/network_proxy_loader.rs b/codex-rs/core/src/network_proxy_loader.rs index 62ed44ad81..049f55c40d 100644 --- a/codex-rs/core/src/network_proxy_loader.rs +++ b/codex-rs/core/src/network_proxy_loader.rs @@ -162,14 +162,14 @@ fn apply_network_constraints(network: NetworkToml, constraints: &mut NetworkProx if let Some(domains) = network.domains.as_ref() { let mut config = NetworkProxyConfig::default(); if let Some(allowed_domains) = constraints.allowed_domains.take() { - config.network.set_allowed_domains(allowed_domains); + config.set_allowed_domains(allowed_domains); } if let Some(denied_domains) = constraints.denied_domains.take() { - config.network.set_denied_domains(denied_domains); + config.set_denied_domains(denied_domains); } overlay_network_domain_permissions(&mut config, domains); - constraints.allowed_domains = config.network.allowed_domains(); - constraints.denied_domains = config.network.denied_domains(); + constraints.allowed_domains = config.allowed_domains(); + constraints.denied_domains = config.denied_domains(); } if let Some(unix_sockets) = network.unix_sockets.as_ref() { let allow_unix_sockets = unix_sockets.allow_unix_sockets(); @@ -256,11 +256,11 @@ impl NetworkConfigAccumulator { }; mitm.validate_action_references(&actions) .map_err(anyhow::Error::msg)?; - self.config.network.mitm_hooks = mitm.to_runtime_hooks(Some(&actions)); + self.config.mitm_hooks = mitm.to_runtime_hooks(Some(&actions)); } - self.config.network.mitm = self.config.network.mode == NetworkMode::Limited - || !self.config.network.mitm_hooks.is_empty(); + self.config.mitm = + self.config.mode == NetworkMode::Limited || !self.config.mitm_hooks.is_empty(); Ok(self.config) } } @@ -310,9 +310,7 @@ fn upsert_network_domain( host: String, permission: codex_network_proxy::NetworkDomainPermission, ) { - config - .network - .upsert_domain_permission(host, permission, normalize_host); + config.upsert_domain_permission(host, permission, normalize_host); } fn is_user_controlled_layer(layer: &ConfigLayerSource) -> bool { diff --git a/codex-rs/core/src/network_proxy_loader_tests.rs b/codex-rs/core/src/network_proxy_loader_tests.rs index f7c7c2e452..f2758e8946 100644 --- a/codex-rs/core/src/network_proxy_loader_tests.rs +++ b/codex-rs/core/src/network_proxy_loader_tests.rs @@ -57,14 +57,14 @@ default_permissions = "dev" .expect("higher layer should apply"); assert_eq!( - config.network.allowed_domains(), + config.allowed_domains(), Some(vec![ "lower.example.com".to_string(), "higher.example.com".to_string() ]) ); assert_eq!( - config.network.denied_domains(), + config.denied_domains(), Some(vec!["blocked.example.com".to_string()]) ); } @@ -108,13 +108,13 @@ default_permissions = "dev" .expect("higher layer should apply"); assert_eq!( - config.network.allowed_domains(), + config.allowed_domains(), Some(vec![ "other.example.com".to_string(), "shared.example.com".to_string() ]) ); - assert_eq!(config.network.denied_domains(), None); + assert_eq!(config.denied_domains(), None); } #[test] @@ -169,20 +169,20 @@ strip_request_headers = ["x-api-key"] .expect("higher layer should apply"); let config = accumulator.finish().expect("merged config should build"); - assert_eq!(config.network.mode, codex_network_proxy::NetworkMode::Full); - assert!(config.network.mitm); + assert_eq!(config.mode, codex_network_proxy::NetworkMode::Full); + assert!(config.mitm); assert_eq!( - config.network.allowed_domains(), + config.allowed_domains(), Some(vec![ "lower.example.com".to_string(), "higher.example.com".to_string() ]) ); - assert_eq!(config.network.mitm_hooks.len(), 1); - assert_eq!(config.network.mitm_hooks[0].host, "api.github.com"); - assert_eq!(config.network.mitm_hooks[0].matcher.methods, vec!["POST"]); + assert_eq!(config.mitm_hooks.len(), 1); + assert_eq!(config.mitm_hooks[0].host, "api.github.com"); + assert_eq!(config.mitm_hooks[0].matcher.methods, vec!["POST"]); assert_eq!( - config.network.mitm_hooks[0].actions.strip_request_headers, + config.mitm_hooks[0].actions.strip_request_headers, vec!["x-api-key"] ); } @@ -190,12 +190,8 @@ strip_request_headers = ["x-api-key"] #[test] fn execpolicy_network_rules_overlay_network_lists() { let mut config = NetworkProxyConfig::default(); - config - .network - .set_allowed_domains(vec!["config.example.com".to_string()]); - config - .network - .set_denied_domains(vec!["blocked.example.com".to_string()]); + config.set_allowed_domains(vec!["config.example.com".to_string()]); + config.set_denied_domains(vec!["blocked.example.com".to_string()]); let mut exec_policy = Policy::empty(); exec_policy @@ -218,14 +214,14 @@ fn execpolicy_network_rules_overlay_network_lists() { apply_exec_policy_network_rules(&mut config, &exec_policy); assert_eq!( - config.network.allowed_domains(), + config.allowed_domains(), Some(vec![ "config.example.com".to_string(), "blocked.example.com".to_string() ]) ); assert_eq!( - config.network.denied_domains(), + config.denied_domains(), Some(vec!["api.example.com".to_string()]) ); } @@ -270,7 +266,7 @@ async fn malformed_custom_rules_preserve_managed_denied_domain() { .expect("proxy state should tolerate malformed custom rules"); assert_eq!( - state.config.network.denied_domains(), + state.config.denied_domains(), Some(vec!["blocked.example.com".to_string()]) ); } @@ -468,7 +464,7 @@ fn config_from_layers_resolves_inherited_profiles_across_layers() { config_from_layers(&layers, &Policy::empty()).expect("inherited profiles should load"); assert_eq!( - config.network.allowed_domains(), + config.allowed_domains(), Some(vec![ "base.example.com".to_string(), "child.example.com".to_string(), @@ -507,10 +503,10 @@ fn config_from_layers_normalizes_profile_network_domains_before_merging_layers() .expect("network domain layer precedence should load"); assert_eq!( - config.network.allowed_domains(), + config.allowed_domains(), Some(vec!["example.com".to_string()]) ); - assert_eq!(config.network.denied_domains(), None); + assert_eq!(config.denied_domains(), None); } #[test] @@ -542,8 +538,8 @@ fn config_from_layers_uses_only_the_final_selected_profile_network() { let config = config_from_layers(&layers, &Policy::empty()) .expect("final built-in profile selection should load"); - assert_eq!(config.network.allowed_domains(), None); - assert_eq!(config.network.denied_domains(), None); + assert_eq!(config.allowed_domains(), None); + assert_eq!(config.denied_domains(), None); } #[test] diff --git a/codex-rs/core/src/session/mcp.rs b/codex-rs/core/src/session/mcp.rs index b6dd64a400..2c33cc041b 100644 --- a/codex-rs/core/src/session/mcp.rs +++ b/codex-rs/core/src/session/mcp.rs @@ -78,6 +78,7 @@ impl ElicitationReviewer for GuardianMcpElicitationReviewer { impl Session { pub(crate) async fn runtime_mcp_config(&self, config: &Config) -> McpConfig { + let originator = self.originator().await; let environments = self.services.turn_environments.snapshot().await; let selected_capability_roots = self .resolve_selected_capability_roots_for_step(&environments) @@ -90,6 +91,7 @@ impl Session { config, &self.services.mcp_thread_init, &self.services.thread_extension_data, + &originator, &available_environment_ids, ) .await @@ -132,6 +134,7 @@ impl Session { &turn_context.config, &self.services.mcp_thread_init, &self.services.thread_extension_data, + &turn_context.originator, &available_environment_ids, ) .await; @@ -475,6 +478,7 @@ impl Session { &refresh_config, &self.services.mcp_thread_init, &self.services.thread_extension_data, + &turn_context.originator, &available_environment_ids, ) .await; @@ -543,6 +547,7 @@ impl Session { refresh_config, &self.services.mcp_thread_init, &self.services.thread_extension_data, + &turn_context.originator, &available_environment_ids, ) .await; diff --git a/codex-rs/core/src/session/session.rs b/codex-rs/core/src/session/session.rs index 32be4813ec..7c59ec22b7 100644 --- a/codex-rs/core/src/session/session.rs +++ b/codex-rs/core/src/session/session.rs @@ -672,6 +672,7 @@ impl Session { let mcp_manager_for_mcp = Arc::clone(&mcp_manager); let mcp_thread_init_for_startup = &mcp_thread_init; let thread_extension_data_for_mcp = &thread_extension_data; + let mcp_originator = session_configuration.originator.clone(); let mcp_runtime_cwd = session_configuration .environment_selections() .first() @@ -688,6 +689,7 @@ impl Session { &config_for_mcp, mcp_thread_init_for_startup, thread_extension_data_for_mcp, + &mcp_originator, /*available_environment_ids*/ &[], ) .await; diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index 5c6d3d7bb2..696a8adbd8 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -809,7 +809,7 @@ async fn start_managed_network_proxy_applies_execpolicy_network_rules() -> anyho let current_cfg = started_proxy.proxy().current_cfg().await?; assert_eq!( - current_cfg.network.allowed_domains(), + current_cfg.allowed_domains(), Some(vec!["example.com".to_string()]) ); Ok(()) @@ -854,7 +854,7 @@ async fn start_managed_network_proxy_ignores_invalid_execpolicy_network_rules() let current_cfg = started_proxy.proxy().current_cfg().await?; assert_eq!( - current_cfg.network.allowed_domains(), + current_cfg.allowed_domains(), Some(vec!["managed.example.com".to_string()]) ); Ok(()) @@ -895,7 +895,7 @@ async fn managed_network_proxy_decider_survives_full_access_start() -> anyhow::R let spec = spec.recompute_for_permission_profile(&PermissionProfile::workspace_write())?; spec.apply_to_started_proxy(&started_proxy).await?; let current_cfg = started_proxy.proxy().current_cfg().await?; - assert_eq!(current_cfg.network.allowed_domains(), None); + assert_eq!(current_cfg.allowed_domains(), None); use tokio::io::AsyncReadExt as _; use tokio::io::AsyncWriteExt as _; @@ -934,9 +934,7 @@ async fn new_turn_refreshes_managed_network_proxy_for_sandbox_change() -> anyhow let initial_permission_profile = PermissionProfile::workspace_write(); let mut network_config = NetworkProxyConfig::default(); - network_config - .network - .set_allowed_domains(vec!["evil.com".to_string()]); + network_config.set_allowed_domains(vec!["evil.com".to_string()]); let requirements = NetworkConstraints { domains: Some(NetworkDomainPermissionsToml { entries: std::collections::BTreeMap::from([( @@ -962,12 +960,7 @@ async fn new_turn_refreshes_managed_network_proxy_for_sandbox_change() -> anyhow ) .await?; assert_eq!( - started_proxy - .proxy() - .current_cfg() - .await? - .network - .allowed_domains(), + started_proxy.proxy().current_cfg().await?.allowed_domains(), Some(vec!["*.example.com".to_string(), "evil.com".to_string()]) ); @@ -1006,12 +999,7 @@ async fn new_turn_refreshes_managed_network_proxy_for_sandbox_change() -> anyhow .load_full() .expect("managed network proxy should be present"); assert_eq!( - started_proxy - .proxy() - .current_cfg() - .await? - .network - .allowed_domains(), + started_proxy.proxy().current_cfg().await?.allowed_domains(), Some(vec!["*.example.com".to_string()]) ); diff --git a/codex-rs/core/src/stream_events_utils.rs b/codex-rs/core/src/stream_events_utils.rs index d2b87a9330..7d03a0d271 100644 --- a/codex-rs/core/src/stream_events_utils.rs +++ b/codex-rs/core/src/stream_events_utils.rs @@ -425,7 +425,26 @@ pub(crate) async fn handle_non_tool_response_item( item: &ResponseItem, plan_mode: bool, ) -> Option { - debug!(?item, "Output item"); + let item_type = match item { + ResponseItem::AdditionalTools { .. } => "additional_tools", + ResponseItem::Message { .. } => "message", + ResponseItem::AgentMessage { .. } => "agent_message", + ResponseItem::Reasoning { .. } => "reasoning", + ResponseItem::LocalShellCall { .. } => "local_shell_call", + ResponseItem::FunctionCall { .. } => "function_call", + ResponseItem::ToolSearchCall { .. } => "tool_search_call", + ResponseItem::FunctionCallOutput { .. } => "function_call_output", + ResponseItem::CustomToolCall { .. } => "custom_tool_call", + ResponseItem::CustomToolCallOutput { .. } => "custom_tool_call_output", + ResponseItem::ToolSearchOutput { .. } => "tool_search_output", + ResponseItem::WebSearchCall { .. } => "web_search_call", + ResponseItem::ImageGenerationCall { .. } => "image_generation_call", + ResponseItem::Compaction { .. } => "compaction", + ResponseItem::CompactionTrigger { .. } => "compaction_trigger", + ResponseItem::ContextCompaction { .. } => "context_compaction", + ResponseItem::Other => "other", + }; + debug!(item_type, item_id = item.id(), "Output item"); match item { ResponseItem::Message { .. } diff --git a/codex-rs/core/src/tasks/mod.rs b/codex-rs/core/src/tasks/mod.rs index c6d9d2990c..7a930bb578 100644 --- a/codex-rs/core/src/tasks/mod.rs +++ b/codex-rs/core/src/tasks/mod.rs @@ -637,7 +637,7 @@ impl Session { let network_proxy_active = match network_proxy.as_ref() { Some(started_network_proxy) => { match started_network_proxy.proxy().current_cfg().await { - Ok(config) => config.network.enabled, + Ok(config) => config.enabled, Err(err) => { warn!( "failed to read managed network proxy state for turn metrics: {err:#}" diff --git a/codex-rs/core/src/thread_manager_tests.rs b/codex-rs/core/src/thread_manager_tests.rs index bf8b7f7064..d29d6d04e5 100644 --- a/codex-rs/core/src/thread_manager_tests.rs +++ b/codex-rs/core/src/thread_manager_tests.rs @@ -553,6 +553,7 @@ async fn start_thread_seeds_extension_data_for_mcp_and_lifecycle_contributors() let mut server = codex_mcp::codex_apps_mcp_server_config( "https://selected.invalid", /*apps_mcp_product_sku*/ None, + /*originator*/ None, ); let CapabilityRootLocation::Environment { environment_id, .. } = &selected_root.location; @@ -574,6 +575,10 @@ async fn start_thread_seeds_extension_data_for_mcp_and_lifecycle_contributors() let mut config = test_config().await; config.codex_home = temp_dir.path().join("codex-home").abs(); config.cwd = config.codex_home.abs(); + config + .features + .enable(Feature::Apps) + .expect("test config should allow apps"); std::fs::create_dir_all(&config.codex_home).expect("create codex home"); let lifecycle_observed = Arc::new(std::sync::Mutex::new(Vec::new())); @@ -620,7 +625,7 @@ async fn start_thread_seeds_extension_data_for_mcp_and_lifecycle_contributors() session_source: None, thread_source: None, dynamic_tools: Vec::new(), - metrics_service_name: None, + metrics_service_name: Some("codex_work_desktop".to_string()), parent_trace: None, environments: Vec::new(), thread_extension_init: selected_root_init("selected-a", "env-a"), @@ -646,6 +651,7 @@ async fn start_thread_seeds_extension_data_for_mcp_and_lifecycle_contributors() .await .expect("start second thread"); let first_session = &first_thread.thread.codex.session; + let first_originator = first_session.originator().await; let first_resolved = first_session .services .mcp_manager @@ -653,10 +659,12 @@ async fn start_thread_seeds_extension_data_for_mcp_and_lifecycle_contributors() &config, &first_session.services.mcp_thread_init, &first_session.services.thread_extension_data, + &first_originator, /*available_environment_ids*/ &[], ) .await; let second_session = &second_thread.thread.codex.session; + let second_originator = second_session.originator().await; let second_resolved = second_session .services .mcp_manager @@ -664,6 +672,7 @@ async fn start_thread_seeds_extension_data_for_mcp_and_lifecycle_contributors() &config, &second_session.services.mcp_thread_init, &second_session.services.thread_extension_data, + &second_originator, /*available_environment_ids*/ &[], ) .await; @@ -706,6 +715,21 @@ async fn start_thread_seeds_extension_data_for_mcp_and_lifecycle_contributors() selected_servers(&second_resolved.config), std::collections::BTreeMap::from([("selected-b".to_string(), "env-b".to_string())]) ); + let codex_apps_server = codex_mcp::configured_mcp_servers(&first_resolved.config) + .remove(codex_mcp::CODEX_APPS_MCP_SERVER_NAME) + .expect("Codex Apps server should be configured"); + let codex_apps_headers = match codex_apps_server.transport { + codex_config::McpServerTransportConfig::StreamableHttp { http_headers, .. } => http_headers, + codex_config::McpServerTransportConfig::Stdio { .. } => { + panic!("Codex Apps server should use streamable HTTP") + } + }; + assert_eq!( + codex_apps_headers + .expect("Codex Apps headers should be configured") + .get("originator"), + Some(&"codex_work_desktop".to_string()) + ); } #[tokio::test] diff --git a/codex-rs/core/src/tools/code_mode/execute_spec.rs b/codex-rs/core/src/tools/code_mode/execute_spec.rs index 0a858bd206..39fbf69eb4 100644 --- a/codex-rs/core/src/tools/code_mode/execute_spec.rs +++ b/codex-rs/core/src/tools/code_mode/execute_spec.rs @@ -6,9 +6,9 @@ use std::collections::BTreeMap; pub(crate) fn create_code_mode_tool( enabled_tools: &[CodeModeToolDefinition], + deferred_tools: &[CodeModeToolDefinition], namespace_descriptions: &BTreeMap, code_mode_only: bool, - deferred_tools_available: bool, ) -> ToolSpec { const CODE_MODE_FREEFORM_GRAMMAR: &str = r#" start: pragma_source | plain_source @@ -24,9 +24,9 @@ SOURCE: /[\s\S]+/ name: codex_code_mode::PUBLIC_TOOL_NAME.to_string(), description: codex_code_mode::build_exec_tool_description( enabled_tools, + deferred_tools, namespace_descriptions, code_mode_only, - deferred_tools_available, ), format: FreeformToolFormat { r#type: "grammar".to_string(), @@ -56,17 +56,17 @@ mod tests { assert_eq!( create_code_mode_tool( &enabled_tools, + &[], &BTreeMap::new(), /*code_mode_only*/ true, - /*deferred_tools_available*/ false, ), ToolSpec::Freeform(FreeformTool { name: codex_code_mode::PUBLIC_TOOL_NAME.to_string(), description: codex_code_mode::build_exec_tool_description( &enabled_tools, + &[], &BTreeMap::new(), /*code_mode_only*/ true, - /*deferred_tools_available*/ false ), format: FreeformToolFormat { r#type: "grammar".to_string(), diff --git a/codex-rs/core/src/tools/spec_plan.rs b/codex-rs/core/src/tools/spec_plan.rs index 57879aeded..f6e50984ac 100644 --- a/codex-rs/core/src/tools/spec_plan.rs +++ b/codex-rs/core/src/tools/spec_plan.rs @@ -461,7 +461,7 @@ fn build_code_mode_executors( let mut code_mode_nested_tool_specs = Vec::new(); let mut exec_prompt_tool_specs = Vec::new(); - let mut deferred_tools_available = false; + let mut deferred_exec_prompt_tool_specs = Vec::new(); let deferred_tools_guidance_enabled = search_tool_enabled(turn_context); for executor in executors { let exposure = executor.exposure(); @@ -480,10 +480,9 @@ fn build_code_mode_executors( let spec = executor.spec(); if exposure == ToolExposure::Deferred { - // Only show deferred-tool guidance when supported and an included spec is usable by code mode. - deferred_tools_available |= deferred_tools_guidance_enabled - && !collect_code_mode_exec_prompt_tool_definitions(std::iter::once(&spec)) - .is_empty(); + if deferred_tools_guidance_enabled { + deferred_exec_prompt_tool_specs.push(spec.clone()); + } } else { exec_prompt_tool_specs.push(spec.clone()); } @@ -495,14 +494,16 @@ fn build_code_mode_executors( collect_code_mode_exec_prompt_tool_definitions(exec_prompt_tool_specs.iter()); enabled_tools .sort_by(|left, right| compare_code_mode_tools(left, right, &namespace_descriptions)); + let deferred_tools = + collect_code_mode_exec_prompt_tool_definitions(deferred_exec_prompt_tool_specs.iter()); vec![ Arc::new(CodeModeExecuteHandler::new( create_code_mode_tool( &enabled_tools, + &deferred_tools, &namespace_descriptions, tool_mode == ToolMode::CodeModeOnly, - deferred_tools_available, ), code_mode_nested_tool_specs, )), diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index 3e9d769310..52af3824d5 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -743,6 +743,7 @@ if (!tool) { }) .expect("exec description should be present"); assert!(exec_description.contains("filter `ALL_TOOLS` by `name` and `description`")); + assert!(exec_description.contains("Shared MCP Types:")); assert!(!exec_description.contains("calendar_timezone_option_99")); let request = follow_up_mock.single_request(); diff --git a/codex-rs/core/tests/suite/mcp_auth_refresh.rs b/codex-rs/core/tests/suite/mcp_auth_refresh.rs index d1fc1d85ec..7d98fd4110 100644 --- a/codex-rs/core/tests/suite/mcp_auth_refresh.rs +++ b/codex-rs/core/tests/suite/mcp_auth_refresh.rs @@ -66,6 +66,7 @@ async fn hosted_plugin_runtime_ps_mcp_tool_calls_use_current_auth_manager_token( let mut hosted_plugin_runtime_config = codex_mcp::hosted_plugin_runtime_mcp_server_config( &apps_server.chatgpt_base_url, /*apps_mcp_product_sku*/ None, + /*originator*/ None, ); let McpServerTransportConfig::StreamableHttp { bearer_token_env_var, diff --git a/codex-rs/core/tests/suite/unified_exec_process_events.rs b/codex-rs/core/tests/suite/unified_exec_process_events.rs index 8ebd332ea2..7fade19bce 100644 --- a/codex-rs/core/tests/suite/unified_exec_process_events.rs +++ b/codex-rs/core/tests/suite/unified_exec_process_events.rs @@ -137,6 +137,16 @@ async fn serve_exec_with_pushed_events( ) .await; } + Some("fs/walk") => { + send_exec_server_json( + &mut websocket, + json!({ + "id": request["id"], + "result": { "entries": [], "errors": [], "truncated": false } + }), + ) + .await; + } method => panic!("unexpected exec-server request before process/start: {method:?}"), } }; diff --git a/codex-rs/exec-server/src/client.rs b/codex-rs/exec-server/src/client.rs index 2e2f4ce450..73e49b2f54 100644 --- a/codex-rs/exec-server/src/client.rs +++ b/codex-rs/exec-server/src/client.rs @@ -113,6 +113,8 @@ const INITIALIZE_TIMEOUT: Duration = Duration::from_secs(10); const ENVIRONMENT_INFO_TIMEOUT: Duration = Duration::from_secs(30); const PROCESS_EVENT_CHANNEL_CAPACITY: usize = 256; const PROCESS_EVENT_RETAINED_BYTES: usize = 1024 * 1024; +const MAX_PENDING_PROCESS_EVENTS: usize = 256; +const MAX_PENDING_PROCESS_EVENT_BYTES: usize = 1024 * 1024; impl Default for ExecServerClientConnectOptions { fn default() -> Self { @@ -173,6 +175,7 @@ struct OrderedSessionEvents { // different tasks and can reach the client out of order. Keep future events // here until all lower sequence numbers have been published. pending: BTreeMap, + pending_bytes: usize, failure: Option, } @@ -218,9 +221,16 @@ enum ConnectionStatus { Failed(String), } +#[derive(Clone, Copy)] +enum RecoveryPolicy { + Wait, + FailFast, +} + #[derive(Clone)] pub struct ExecServerClient { inner: Arc, + recovery_policy: RecoveryPolicy, } struct ActiveProcessStart { @@ -239,6 +249,7 @@ type ConnectionAttempt = OnceCell; #[derive(Clone)] pub(crate) struct LazyRemoteExecServerClient { transport_params: ExecServerTransportParams, + recovery_policy: RecoveryPolicy, // Saves the first startup result so callers share it and failures remain final. startup: Arc, // The latest successful client, replaced whenever reconnecting succeeds. @@ -250,6 +261,7 @@ impl LazyRemoteExecServerClient { pub(crate) fn new(transport_params: ExecServerTransportParams) -> Self { Self { transport_params, + recovery_policy: RecoveryPolicy::Wait, startup: Arc::new(ConnectionAttempt::new()), current_client: Arc::new(StdMutex::new(None)), reconnect: Arc::new(StdMutex::new(None)), @@ -276,11 +288,45 @@ impl LazyRemoteExecServerClient { self.startup.get().is_some() } + pub(crate) fn readiness_result(&self) -> Option> { + if let Some(client) = self.cached_client() { + return client.readiness_result(); + } + self.startup.get().and_then(|result| match result { + Ok(client) => client.readiness_result(), + Err(error) => Some(Err(ExecServerError::ConnectionAttempt(Arc::clone(error)))), + }) + } + + pub(crate) fn fail_fast(&self) -> Self { + Self { + recovery_policy: RecoveryPolicy::FailFast, + ..self.clone() + } + } + pub(crate) async fn wait_until_ready(&self) -> Result<(), ExecServerError> { self.initial_client().await.map(drop) } pub(crate) async fn get(&self) -> Result { + if matches!(self.recovery_policy, RecoveryPolicy::FailFast) { + let client = match self.cached_client() { + Some(client) => client, + None => match self.startup.get() { + Some(Ok(client)) => client.clone(), + Some(Err(error)) => { + return Err(ExecServerError::ConnectionAttempt(Arc::clone(error))); + } + None => { + return Err(ExecServerError::Disconnected( + "exec-server environment is not ready".to_string(), + )); + } + }, + }; + return client.fail_fast(); + } if let Some(client) = self.connected_client() { return Ok(client); } @@ -456,11 +502,45 @@ pub enum ExecServerError { } impl ExecServerClient { + fn fail_fast(&self) -> Result { + self.rpc_client_without_recovery()?; + Ok(Self { + inner: Arc::clone(&self.inner), + recovery_policy: RecoveryPolicy::FailFast, + }) + } + + fn rpc_client_without_recovery(&self) -> Result, ExecServerError> { + let connection = self + .inner + .connection + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + match &connection.status { + ConnectionStatus::Connected(rpc_client) if !rpc_client.is_disconnected() => { + Ok(Arc::clone(rpc_client)) + } + ConnectionStatus::Connected(_) | ConnectionStatus::Recovering => Err( + ExecServerError::Disconnected("exec-server environment is recovering".to_string()), + ), + ConnectionStatus::Failed(message) => { + Err(ExecServerError::Disconnected(message.clone())) + } + } + } + + async fn rpc_client(&self) -> Result, ExecServerError> { + match self.recovery_policy { + RecoveryPolicy::Wait => self.inner.rpc_client().await, + RecoveryPolicy::FailFast => self.rpc_client_without_recovery(), + } + } + pub async fn initialize( &self, options: ExecServerClientConnectOptions, ) -> Result { - let rpc_client = self.inner.rpc_client().await?; + let rpc_client = self.rpc_client().await?; self.initialize_rpc(&rpc_client, options).await } @@ -511,7 +591,7 @@ impl ExecServerClient { } pub async fn environment_info(&self) -> Result { - let rpc_client = self.inner.rpc_client().await?; + let rpc_client = self.rpc_client().await?; map_rpc_call_result( rpc_client .call_with_timeout(ENVIRONMENT_INFO_METHOD, &(), ENVIRONMENT_INFO_TIMEOUT) @@ -650,7 +730,7 @@ impl ExecServerClient { params: ExecParams, ) -> Result { loop { - let rpc_client = self.inner.rpc_client().await?; + let rpc_client = self.rpc_client().await?; if !self.inner.begin_process_start(&rpc_client) { continue; } @@ -727,6 +807,23 @@ impl ExecServerClient { self.inner.is_failed() } + fn readiness_result(&self) -> Option> { + let connection = self + .inner + .connection + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + match &connection.status { + ConnectionStatus::Connected(rpc_client) if !rpc_client.is_disconnected() => { + Some(Ok(())) + } + ConnectionStatus::Connected(_) | ConnectionStatus::Recovering => None, + ConnectionStatus::Failed(message) => { + Some(Err(ExecServerError::Disconnected(message.clone()))) + } + } + } + pub(crate) async fn connect( connection: JsonRpcConnection, options: ExecServerClientConnectOptions, @@ -758,7 +855,10 @@ impl ExecServerClient { session_id, reconnect_strategy, }); - let client = Self { inner }; + let client = Self { + inner, + recovery_policy: RecoveryPolicy::Wait, + }; // An explicit resume can redirect notifications from running processes // before initialize returns. Drain them immediately so a burst cannot // fill the bounded event channel and block the initialize response. @@ -772,7 +872,7 @@ impl ExecServerClient { P: serde::Serialize, T: serde::de::DeserializeOwned, { - let rpc_client = self.inner.rpc_client().await?; + let rpc_client = self.rpc_client().await?; self.call_rpc(&rpc_client, method, params).await } @@ -881,10 +981,10 @@ impl SessionState { /// `Closed` event. The caller uses that signal to remove the session route /// after the terminal event is visible to subscribers, rather than when a /// possibly-early closed notification first arrives. - fn publish_ordered_event(&self, event: ExecProcessEvent) -> bool { + fn publish_ordered_event(&self, event: ExecProcessEvent) -> Result { let Some(seq) = event.seq() else { self.events.publish(event); - return false; + return Ok(false); }; let mut ordered_events = self @@ -897,11 +997,11 @@ impl SessionState { || ordered_events.closed_published || seq <= ordered_events.last_published_seq { - return false; + return Ok(false); } - ordered_events.pending.entry(seq).or_insert(event); - self.publish_ready(&mut ordered_events) + ordered_events.insert_pending(event)?; + Ok(self.publish_ready(&mut ordered_events)) } fn publish_ready(&self, ordered_events: &mut OrderedSessionEvents) -> bool { @@ -911,6 +1011,9 @@ impl SessionState { let Some(event) = ordered_events.pending.remove(&next_seq) else { break; }; + ordered_events.pending_bytes = ordered_events + .pending_bytes + .saturating_sub(pending_process_event_bytes(&event)); ordered_events.last_published_seq = next_seq; ordered_events.exit_published |= matches!(&event, ExecProcessEvent::Exited { .. }); let is_closed = matches!(&event, ExecProcessEvent::Closed { .. }); @@ -931,6 +1034,7 @@ impl SessionState { } ordered_events.failure = Some(message.clone()); ordered_events.pending.clear(); + ordered_events.pending_bytes = 0; self.events.publish(ExecProcessEvent::Failed(message)); drop(ordered_events); self.wake_tx @@ -966,6 +1070,68 @@ impl SessionState { } } +impl OrderedSessionEvents { + fn insert_pending(&mut self, event: ExecProcessEvent) -> Result<(), String> { + let Some(seq) = event.seq() else { + return Err("cannot reorder an unsequenced process event".to_string()); + }; + if self.pending.contains_key(&seq) { + return Ok(()); + } + + let next_seq = self.last_published_seq.saturating_add(1); + // The next expected event is synchronously published by every caller, + // so it can drain a full buffer without becoming retained state. + let closes_gap = seq == next_seq; + if !closes_gap && self.pending.len() >= MAX_PENDING_PROCESS_EVENTS { + return Err(format!( + "process event reorder buffer exceeds {MAX_PENDING_PROCESS_EVENTS} entries" + )); + } + + let event_bytes = pending_process_event_bytes(&event); + if event_bytes > MAX_PENDING_PROCESS_EVENT_BYTES { + return Err(format!( + "process event exceeds {MAX_PENDING_PROCESS_EVENT_BYTES} bytes" + )); + } + let pending_bytes = self.pending_bytes.saturating_add(event_bytes); + if !closes_gap && pending_bytes > MAX_PENDING_PROCESS_EVENT_BYTES { + return Err(format!( + "process event reorder buffer exceeds {MAX_PENDING_PROCESS_EVENT_BYTES} bytes" + )); + } + + self.pending.insert(seq, event); + self.pending_bytes = pending_bytes; + Ok(()) + } +} + +fn pending_process_event_bytes(event: &ExecProcessEvent) -> usize { + match event { + ExecProcessEvent::Output(chunk) => chunk.chunk.0.len(), + ExecProcessEvent::Failed(message) => message.len(), + ExecProcessEvent::Exited { .. } | ExecProcessEvent::Closed { .. } => 0, + } +} + +fn finish_process_event( + inner: &Inner, + process_id: &ProcessId, + session: &Arc, + result: Result, +) { + match result { + Ok(true) => inner.remove_session_if(process_id, session), + Ok(false) => {} + Err(message) => { + session.set_failure(message); + inner.remove_session_if(process_id, session); + } + } +} + impl Session { pub(crate) fn process_id(&self) -> &ProcessId { &self.process_id @@ -1161,46 +1327,46 @@ async fn handle_server_notification( let params: ExecOutputDeltaNotification = serde_json::from_value(notification.params.unwrap_or(Value::Null))?; if let Some(session) = inner.get_session(¶ms.process_id) { - session.note_change(params.seq); - let published_closed = + let result = session.publish_ordered_event(ExecProcessEvent::Output(ProcessOutputChunk { seq: params.seq, stream: params.stream, chunk: params.chunk, })); - if published_closed { - inner.remove_session_if(¶ms.process_id, &session); + if result.is_ok() { + session.note_change(params.seq); } + finish_process_event(inner, ¶ms.process_id, &session, result); } } EXEC_EXITED_METHOD => { let params: ExecExitedNotification = serde_json::from_value(notification.params.unwrap_or(Value::Null))?; if let Some(session) = inner.get_session(¶ms.process_id) { - session.note_change(params.seq); - let published_closed = session.publish_ordered_event(ExecProcessEvent::Exited { + let result = session.publish_ordered_event(ExecProcessEvent::Exited { seq: params.seq, exit_code: params.exit_code, sandbox_denied: params.sandbox_denied, }); - if published_closed { - inner.remove_session_if(¶ms.process_id, &session); + if result.is_ok() { + session.note_change(params.seq); } + finish_process_event(inner, ¶ms.process_id, &session, result); } } EXEC_CLOSED_METHOD => { let params: ExecClosedNotification = serde_json::from_value(notification.params.unwrap_or(Value::Null))?; if let Some(session) = inner.get_session(¶ms.process_id) { - session.note_change(params.seq); // Closed is terminal, but it can arrive before tail output or // exited. Keep routing this process until the ordered publisher // says Closed has actually been delivered. - let published_closed = + let result = session.publish_ordered_event(ExecProcessEvent::Closed { seq: params.seq }); - if published_closed { - inner.remove_session_if(¶ms.process_id, &session); + if result.is_ok() { + session.note_change(params.seq); } + finish_process_event(inner, ¶ms.process_id, &session, result); } } HTTP_REQUEST_BODY_DELTA_METHOD => { diff --git a/codex-rs/exec-server/src/client/rpc_http_client.rs b/codex-rs/exec-server/src/client/rpc_http_client.rs index d243a8d42f..c48ff8d29f 100644 --- a/codex-rs/exec-server/src/client/rpc_http_client.rs +++ b/codex-rs/exec-server/src/client/rpc_http_client.rs @@ -42,7 +42,7 @@ impl ExecServerClient { &self, mut params: HttpRequestParams, ) -> Result<(HttpRequestResponse, HttpResponseBodyStream), ExecServerError> { - let rpc_client = self.inner.rpc_client().await?; + let rpc_client = self.rpc_client().await?; params.stream_response = true; let request_id = self.inner.next_http_body_stream_request_id(); params.request_id = request_id.clone(); diff --git a/codex-rs/exec-server/src/client_recovery.rs b/codex-rs/exec-server/src/client_recovery.rs index 54ccf05046..a6075f3739 100644 --- a/codex-rs/exec-server/src/client_recovery.rs +++ b/codex-rs/exec-server/src/client_recovery.rs @@ -15,6 +15,7 @@ use super::ExecServerClient; use super::ExecServerError; use super::Inner; use super::OrderedSessionEvents; +use super::RecoveryPolicy; use super::SessionState; use super::disconnected_message; use super::fail_all_in_flight_work; @@ -78,30 +79,6 @@ impl SessionState { { return Ok(false); } - for chunk in chunks { - if chunk.seq > ordered_events.last_published_seq { - ordered_events - .pending - .entry(chunk.seq) - .or_insert(ExecProcessEvent::Output(chunk)); - } - } - if closed { - match ordered_events.pending.get(&target_seq) { - Some(ExecProcessEvent::Closed { .. }) => {} - Some(_) => { - return Err(ExecServerError::Protocol(format!( - "process close sequence {target_seq} conflicts with recovered output" - ))); - } - None => { - ordered_events - .pending - .insert(target_seq, ExecProcessEvent::Closed { seq: target_seq }); - } - } - } - let pending_exit = ordered_events.pending.range_mut(..=target_seq).find_map( |(_, event)| match event { ExecProcessEvent::Exited { @@ -116,12 +93,71 @@ impl SessionState { *pending_sandbox_denied = Some(pending_sandbox_denied.unwrap_or(false) || sandbox_denied); } - let exit_known = ordered_events.exit_published || exit_pending; - let event_count = target_seq - ordered_events.last_published_seq; - let retained_count = ordered_events - .pending - .range(ordered_events.last_published_seq.saturating_add(1)..=target_seq) - .count() as u64; + let mut exit_known = ordered_events.exit_published || exit_pending; + if closed + && (matches!( + ordered_events.pending.get(&target_seq), + Some(event) if !matches!(event, ExecProcessEvent::Closed { .. }) + ) || chunks.iter().any(|chunk| chunk.seq == target_seq)) + { + return Err(ExecServerError::Protocol(format!( + "process close sequence {target_seq} conflicts with recovered output" + ))); + } + let mut published_closed = false; + for chunk in chunks { + if chunk.seq > target_seq { + return Err(ExecServerError::Protocol(format!( + "recovered process output sequence {} exceeds target sequence {target_seq}", + chunk.seq + ))); + } + let next_seq = ordered_events.last_published_seq.saturating_add(1); + if exited && !exit_known && chunk.seq > next_seq { + let exit_code = exit_code.ok_or_else(|| { + ExecServerError::Protocol( + "recovering exited process did not include its exit code".to_string(), + ) + })?; + ordered_events + .insert_pending(ExecProcessEvent::Exited { + seq: next_seq, + exit_code, + sandbox_denied: Some(sandbox_denied), + }) + .map_err(ExecServerError::Protocol)?; + published_closed |= self.publish_ready(&mut ordered_events); + exit_known = true; + } + if chunk.seq > ordered_events.last_published_seq { + ordered_events + .insert_pending(ExecProcessEvent::Output(chunk)) + .map_err(ExecServerError::Protocol)?; + published_closed |= self.publish_ready(&mut ordered_events); + } + } + if closed + && !ordered_events.closed_published + && !matches!( + ordered_events.pending.get(&target_seq), + Some(ExecProcessEvent::Closed { .. }) + ) + { + ordered_events + .insert_pending(ExecProcessEvent::Closed { seq: target_seq }) + .map_err(ExecServerError::Protocol)?; + } + + let event_count = target_seq.saturating_sub(ordered_events.last_published_seq); + let first_unpublished_seq = ordered_events.last_published_seq.saturating_add(1); + let retained_count = if first_unpublished_seq <= target_seq { + ordered_events + .pending + .range(first_unpublished_seq..=target_seq) + .count() as u64 + } else { + 0 + }; let missing_count = event_count.saturating_sub(retained_count); if exited && !exit_known { if missing_count != 1 { @@ -133,18 +169,18 @@ impl SessionState { "recovering exited process did not include its exit code".to_string(), ) })?; - ordered_events.pending.insert( - seq, - ExecProcessEvent::Exited { + ordered_events + .insert_pending(ExecProcessEvent::Exited { seq, exit_code, sandbox_denied: Some(sandbox_denied), - }, - ); + }) + .map_err(ExecServerError::Protocol)?; } else if missing_count != 0 { return Err(recovery_gap_error(target_seq)); } - self.publish_ready(&mut ordered_events) + published_closed |= self.publish_ready(&mut ordered_events); + published_closed }; self.note_change(target_seq); @@ -402,6 +438,7 @@ impl Inner { let rpc_client = Arc::new(rpc_client); let client = ExecServerClient { inner: Arc::clone(self), + recovery_policy: RecoveryPolicy::Wait, }; // Resuming a session redirects notifications from its running processes // to this connection during initialize. Drain them immediately so a diff --git a/codex-rs/exec-server/src/client_recovery_tests.rs b/codex-rs/exec-server/src/client_recovery_tests.rs index d2126fd70b..7f657c59dd 100644 --- a/codex-rs/exec-server/src/client_recovery_tests.rs +++ b/codex-rs/exec-server/src/client_recovery_tests.rs @@ -1,5 +1,7 @@ use std::time::Duration; +use pretty_assertions::assert_eq; + use super::*; use crate::protocol::ExecOutputStream; use crate::protocol::ProcessOutputChunk; @@ -57,14 +59,152 @@ fn recovery_does_not_retry_other_registry_conflicts() { assert!(!is_retryable_recovery_error(&error)); } +#[test] +fn process_event_reorder_rejects_oversized_output() { + let state = SessionState::new(/*recoverable*/ true); + + let error = state + .publish_ordered_event(ExecProcessEvent::Output(ProcessOutputChunk { + seq: 1, + stream: ExecOutputStream::Stdout, + chunk: vec![0; super::super::MAX_PENDING_PROCESS_EVENT_BYTES + 1].into(), + })) + .expect_err("oversized pending process output should be rejected"); + + assert!(error.contains("bytes")); +} + +#[test] +fn process_event_reorder_accepts_gap_closing_event_at_limits() { + let state = SessionState::new(/*recoverable*/ true); + let chunk_size = + super::super::MAX_PENDING_PROCESS_EVENT_BYTES / super::super::MAX_PENDING_PROCESS_EVENTS; + let last_seq = super::super::MAX_PENDING_PROCESS_EVENTS as u64 + 1; + + for seq in 2..=last_seq { + assert!( + !state + .publish_ordered_event(ExecProcessEvent::Output(ProcessOutputChunk { + seq, + stream: ExecOutputStream::Stdout, + chunk: vec![0; chunk_size].into(), + })) + .expect("future output should fit within reorder limits") + ); + } + assert!( + !state + .publish_ordered_event(ExecProcessEvent::Output(ProcessOutputChunk { + seq: 1, + stream: ExecOutputStream::Stdout, + chunk: b"x".to_vec().into(), + })) + .expect("gap-closing output should drain the reorder buffer") + ); + + let ordered_events = state + .ordered_events + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + assert_eq!( + ( + ordered_events.last_published_seq, + ordered_events.pending.len(), + ordered_events.pending_bytes, + ), + (last_seq, 0, 0) + ); +} + +#[test] +fn recovery_handles_dense_tail_output_and_newer_notification() { + let state = SessionState::new(/*recoverable*/ true); + let last_seq = super::super::MAX_PENDING_PROCESS_EVENTS as u64 + 2; + let live_seq = last_seq + 1; + assert!( + !state + .publish_ordered_event(ExecProcessEvent::Output(ProcessOutputChunk { + seq: live_seq, + stream: ExecOutputStream::Stdout, + chunk: b"live".to_vec().into(), + })) + .expect("live output should remain bounded while recovery fills the gap") + ); + let chunks = (2..=last_seq) + .map(|seq| ProcessOutputChunk { + seq, + stream: ExecOutputStream::Stdout, + chunk: b"x".to_vec().into(), + }) + .collect(); + + assert!( + !state + .recover_events(ReadResponse { + chunks, + next_seq: last_seq + 1, + exited: true, + exit_code: Some(17), + closed: false, + failure: None, + sandbox_denied: false, + }) + .expect("dense retained output should recover") + ); + + let ordered_events = state + .ordered_events + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + assert_eq!( + ( + ordered_events.last_published_seq, + ordered_events.pending.len(), + ordered_events.pending_bytes, + ), + (live_seq, 0, 0) + ); +} + +#[test] +fn recovery_rejects_output_at_closed_sequence() { + let state = SessionState::new(/*recoverable*/ true); + + let error = state + .recover_events(ReadResponse { + chunks: vec![ProcessOutputChunk { + seq: 1, + stream: ExecOutputStream::Stdout, + chunk: b"output".to_vec().into(), + }], + next_seq: 2, + exited: false, + exit_code: None, + closed: true, + failure: None, + sandbox_denied: false, + }) + .expect_err("output should not occupy the closed sequence"); + + assert!( + error + .to_string() + .contains("conflicts with recovered output") + ); +} + #[tokio::test] async fn recovery_adds_sandbox_denial_to_pending_exit_event() { let state = SessionState::new(/*recoverable*/ true); - assert!(!state.publish_ordered_event(ExecProcessEvent::Exited { - seq: 2, - exit_code: 1, - sandbox_denied: None, - })); + assert!( + !state + .publish_ordered_event(ExecProcessEvent::Exited { + seq: 2, + exit_code: 1, + sandbox_denied: None, + }) + .expect("pending exit should fit within reorder limits") + ); state .recover_events(ReadResponse { diff --git a/codex-rs/exec-server/src/environment.rs b/codex-rs/exec-server/src/environment.rs index 66f7381103..d070be6786 100644 --- a/codex-rs/exec-server/src/environment.rs +++ b/codex-rs/exec-server/src/environment.rs @@ -608,6 +608,14 @@ impl Environment { } } + /// Returns whether the environment can serve a request without waiting or reconnecting. + pub(crate) fn readiness_result(&self) -> Option> { + match &self.remote_client { + Some(client) => client.readiness_result(), + None => Some(Ok(())), + } + } + pub fn get_exec_backend(&self) -> Arc { Arc::clone(&self.exec_backend) } @@ -619,6 +627,14 @@ impl Environment { pub fn get_filesystem(&self) -> Arc { Arc::clone(&self.filesystem) } + + /// Returns a filesystem view that fails instead of starting or waiting for a connection. + pub fn get_filesystem_without_reconnect(&self) -> Arc { + match &self.remote_client { + Some(client) => Arc::new(RemoteFileSystem::new(client.fail_fast())), + None => Arc::clone(&self.filesystem), + } + } } #[cfg(test)] @@ -1083,6 +1099,72 @@ mod tests { assert!(environment.startup_finished()); } + #[tokio::test] + async fn selected_capability_inspection_keeps_stdio_environment_lazy() { + use codex_protocol::capabilities::CapabilityRootLocation; + use codex_protocol::capabilities::SelectedCapabilityRoot; + + let environment = Environment::remote_with_transport( + ExecServerTransportParams::StdioCommand { + command: StdioExecServerCommand { + program: "codex-missing-exec-server-for-test".to_string(), + args: Vec::new(), + env: HashMap::new(), + cwd: None, + }, + initialize_timeout: Duration::from_secs(1), + }, + /*local_runtime_paths*/ None, + ); + let manager = EnvironmentManager::from_snapshot( + EnvironmentProviderSnapshot { + environments: vec![("stdio".to_string(), environment)], + default: EnvironmentDefault::Disabled, + include_local: false, + }, + /*local_runtime_paths*/ None, + ) + .expect("environment manager"); + let environment = manager.get_environment("stdio").expect("stdio environment"); + let selected_root = SelectedCapabilityRoot { + id: "demo@1".to_string(), + location: CapabilityRootLocation::Environment { + environment_id: "stdio".to_string(), + path: PathUri::parse("file:///plugins/demo").expect("plugin path URI"), + }, + }; + + let status = + manager.inspect_selected_capability_roots(std::slice::from_ref(&selected_root)); + assert!(status.ready_roots.is_empty()); + assert_eq!(status.warnings, Vec::::new()); + assert!(!environment.startup_finished()); + + let missing_root = SelectedCapabilityRoot { + id: "missing@1".to_string(), + location: CapabilityRootLocation::Environment { + environment_id: "missing".to_string(), + path: PathUri::parse("file:///plugins/missing").expect("missing plugin path URI"), + }, + }; + let status = manager.inspect_selected_capability_roots(&[missing_root]); + assert!(status.ready_roots.is_empty()); + assert_eq!( + status.warnings, + vec![ + "selected capability root `missing@1` references unavailable environment `missing`" + .to_string() + ] + ); + + assert!(environment.wait_until_ready().await.is_err()); + + let status = manager.inspect_selected_capability_roots(&[selected_root]); + assert!(status.ready_roots.is_empty()); + assert_eq!(status.warnings.len(), 1); + assert!(status.warnings[0].contains("environment `stdio` is unavailable")); + } + #[tokio::test] async fn replacing_environment_stops_its_startup_task() { let first_listener = TcpListener::bind("127.0.0.1:0") diff --git a/codex-rs/exec-server/src/lib.rs b/codex-rs/exec-server/src/lib.rs index 882c83230c..c2c8cc5194 100644 --- a/codex-rs/exec-server/src/lib.rs +++ b/codex-rs/exec-server/src/lib.rs @@ -148,6 +148,7 @@ pub use protocol::WriteStatus; pub use remote::RemoteEnvironmentConfig; pub use remote::run_remote_environment; pub use resolved_capability::ResolvedSelectedCapabilityRoot; +pub use resolved_capability::SelectedCapabilityRootsStatus; pub use runtime_paths::ExecServerRuntimePaths; pub use server::DEFAULT_LISTEN_URL; pub use server::ExecServerListenUrlParseError; diff --git a/codex-rs/exec-server/src/resolved_capability.rs b/codex-rs/exec-server/src/resolved_capability.rs index 2902cf9eef..f1fa92c2f0 100644 --- a/codex-rs/exec-server/src/resolved_capability.rs +++ b/codex-rs/exec-server/src/resolved_capability.rs @@ -20,6 +20,15 @@ pub struct ResolvedSelectedCapabilityRoot { environment: Arc, } +/// A passive view of selected capability roots and unavailable environments. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SelectedCapabilityRootsStatus { + /// Selected roots whose environments are ready. + pub ready_roots: Vec, + /// Missing environments and terminal connection failures. + pub warnings: Vec, +} + impl ResolvedSelectedCapabilityRoot { pub fn selected_root(&self) -> &SelectedCapabilityRoot { &self.selected_root @@ -35,6 +44,74 @@ impl ResolvedSelectedCapabilityRoot { } impl EnvironmentManager { + /// Inspects selected roots without starting or waiting for an environment. + /// + /// Starting or recovering environments are omitted. Missing environments and terminal + /// connection failures are returned as warnings so read-only catalog clients can distinguish + /// them from an empty catalog. + /// + /// Environment IDs are stable identities, so callers can safely resolve a returned root by + /// ID when they read it. + pub fn inspect_selected_capability_roots( + &self, + selected_roots: &[SelectedCapabilityRoot], + ) -> SelectedCapabilityRootsStatus { + let (candidates, mut warnings) = { + let environments = self + .environments + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut candidates = Vec::with_capacity(selected_roots.len()); + let mut warnings = Vec::new(); + for selected_root in selected_roots { + let CapabilityRootLocation::Environment { environment_id, .. } = + &selected_root.location; + let Some(environment) = environments.get(environment_id) else { + warnings.push(format!( + "selected capability root `{}` references unavailable environment `{environment_id}`", + selected_root.id + )); + continue; + }; + candidates.push((selected_root.clone(), Arc::clone(environment))); + } + (candidates, warnings) + }; + let mut readiness = HashMap::new(); + for (selected_root, environment) in &candidates { + let CapabilityRootLocation::Environment { environment_id, .. } = + &selected_root.location; + if readiness.contains_key(environment_id) { + continue; + } + let ready = match environment.readiness_result() { + Some(Ok(())) => true, + Some(Err(error)) => { + warnings.push(format!( + "selected capability environment `{environment_id}` is unavailable: {error}" + )); + false + } + None => false, + }; + readiness.insert(environment_id.clone(), ready); + } + + let ready_roots = candidates + .into_iter() + .filter(|(selected_root, _)| { + let CapabilityRootLocation::Environment { environment_id, .. } = + &selected_root.location; + readiness.get(environment_id).copied().unwrap_or(false) + }) + .map(|(selected_root, _)| selected_root) + .collect(); + SelectedCapabilityRootsStatus { + ready_roots, + warnings, + } + } + /// Resolves selected roots whose stable environments are ready for the current model step. /// /// Environment identity comes from the selected root's stable environment ID. A ready diff --git a/codex-rs/exec-server/tests/environment.rs b/codex-rs/exec-server/tests/environment.rs new file mode 100644 index 0000000000..4b97361ac2 --- /dev/null +++ b/codex-rs/exec-server/tests/environment.rs @@ -0,0 +1,84 @@ +mod common; + +use std::time::Duration; + +use anyhow::Context; +use codex_exec_server::EnvironmentManager; +use codex_exec_server::REMOTE_ENVIRONMENT_ID; +use codex_exec_server::SelectedCapabilityRootsStatus; +use codex_protocol::capabilities::CapabilityRootLocation; +use codex_protocol::capabilities::SelectedCapabilityRoot; +use codex_utils_path_uri::PathUri; +use common::exec_server::exec_server; +use pretty_assertions::assert_eq; +use tokio::time::sleep; +use tokio::time::timeout; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial_test::serial(remote_exec_server)] +async fn selected_capability_inspection_tracks_connection_recovery() -> anyhow::Result<()> { + let server = exec_server().await?; + let mut proxy = server.disconnectable_websocket_proxy().await?; + let manager = EnvironmentManager::create_for_tests( + Some(proxy.websocket_url().to_string()), + /*local_runtime_paths*/ None, + ) + .await; + let environment = manager + .default_environment() + .context("remote environment")?; + environment.info().await?; + + let skill_root_path = PathUri::parse("file:///plugins/demo")?; + let selected_root = SelectedCapabilityRoot { + id: "demo@1".to_string(), + location: CapabilityRootLocation::Environment { + environment_id: REMOTE_ENVIRONMENT_ID.to_string(), + path: skill_root_path.clone(), + }, + }; + assert_eq!( + manager.inspect_selected_capability_roots(std::slice::from_ref(&selected_root)), + SelectedCapabilityRootsStatus { + ready_roots: vec![selected_root.clone()], + warnings: Vec::new(), + } + ); + let file_system = environment.get_filesystem_without_reconnect(); + + proxy.pause_and_disconnect().await?; + assert_eq!( + manager.inspect_selected_capability_roots(std::slice::from_ref(&selected_root)), + SelectedCapabilityRootsStatus::default() + ); + let read_result = timeout( + Duration::from_secs(1), + file_system.read_directory(&skill_root_path, /*sandbox*/ None), + ) + .await + .context("passive filesystem read waited for recovery")?; + assert!(read_result.is_err()); + + proxy.resume()?; + let recovered_status = timeout(Duration::from_secs(5), async { + loop { + let status = + manager.inspect_selected_capability_roots(std::slice::from_ref(&selected_root)); + if !status.ready_roots.is_empty() { + break status; + } + sleep(Duration::from_millis(10)).await; + } + }) + .await + .context("environment did not recover")?; + assert_eq!( + recovered_status, + SelectedCapabilityRootsStatus { + ready_roots: vec![selected_root], + warnings: Vec::new(), + } + ); + + Ok(()) +} diff --git a/codex-rs/ext/extension-api/src/contributors/mcp.rs b/codex-rs/ext/extension-api/src/contributors/mcp.rs index 3fe648a1e9..8f7598e264 100644 --- a/codex-rs/ext/extension-api/src/contributors/mcp.rs +++ b/codex-rs/ext/extension-api/src/contributors/mcp.rs @@ -15,6 +15,8 @@ pub struct McpServerContributionContext<'a, C> { thread_store: Option<&'a ExtensionData>, /// Stable host inputs for the active thread, when resolution is thread-scoped. thread_init: Option<&'a ExtensionDataInit>, + /// Effective request originator for the active thread, when resolution is thread-scoped. + originator: Option<&'a str>, /// Environment IDs whose selected roots may contribute to this exact step. available_environment_ids: Option<&'a [String]>, } @@ -34,6 +36,7 @@ impl<'a, C> McpServerContributionContext<'a, C> { config, thread_store: None, thread_init: None, + originator: None, available_environment_ids: None, } } @@ -43,12 +46,14 @@ impl<'a, C> McpServerContributionContext<'a, C> { config: &'a C, thread_init: &'a ExtensionDataInit, thread_store: &'a ExtensionData, + originator: &'a str, available_environment_ids: &'a [String], ) -> Self { Self { config, thread_store: Some(thread_store), thread_init: Some(thread_init), + originator: Some(originator), available_environment_ids: Some(available_environment_ids), } } @@ -68,6 +73,11 @@ impl<'a, C> McpServerContributionContext<'a, C> { self.thread_init } + /// Returns the effective request originator when resolving for a running thread. + pub fn originator(&self) -> Option<&'a str> { + self.originator + } + /// Returns the exact environment availability projection for a model step. /// /// `Some` means contributors must omit selected roots whose environment ID is absent from the diff --git a/codex-rs/ext/mcp/src/lib.rs b/codex-rs/ext/mcp/src/lib.rs index 9c22bad5a0..d4f2bcf976 100644 --- a/codex-rs/ext/mcp/src/lib.rs +++ b/codex-rs/ext/mcp/src/lib.rs @@ -32,6 +32,7 @@ impl McpServerContributor for HostedPluginRuntimeExtension { config: Box::new(hosted_plugin_runtime_mcp_server_config( &config.chatgpt_base_url, config.apps_mcp_product_sku.as_deref(), + context.originator(), )), }] }) @@ -51,3 +52,7 @@ pub fn install_executor_plugins( executor_plugin::SelectedExecutorPluginMcpContributor::new(environment_manager), )); } + +#[cfg(test)] +#[path = "lib_tests.rs"] +mod tests; diff --git a/codex-rs/ext/mcp/src/lib_tests.rs b/codex-rs/ext/mcp/src/lib_tests.rs new file mode 100644 index 0000000000..7848dce7d0 --- /dev/null +++ b/codex-rs/ext/mcp/src/lib_tests.rs @@ -0,0 +1,48 @@ +use super::*; +use codex_config::McpServerTransportConfig; +use codex_core::config::ConfigBuilder; +use codex_extension_api::ExtensionData; +use codex_extension_api::ExtensionDataInit; +use pretty_assertions::assert_eq; + +#[tokio::test] +async fn hosted_plugin_runtime_forwards_thread_originator() -> Result<(), Box> +{ + let codex_home = tempfile::tempdir()?; + let config = ConfigBuilder::default() + .codex_home(codex_home.path().to_path_buf()) + .fallback_cwd(Some(codex_home.path().to_path_buf())) + .cli_overrides(vec![ + ("features.apps".to_string(), true.into()), + ("chatgpt_base_url".to_string(), "https://chatgpt.com".into()), + ]) + .build() + .await?; + let thread_init = ExtensionDataInit::new(); + let thread_store = ExtensionData::new("thread"); + + let contributions = HostedPluginRuntimeExtension + .contribute(McpServerContributionContext::for_step( + &config, + &thread_init, + &thread_store, + "codex_work_desktop", + /*available_environment_ids*/ &[], + )) + .await; + let [McpServerContribution::Set { config: server, .. }] = contributions.as_slice() else { + panic!("hosted plugin runtime should contribute one server"); + }; + let McpServerTransportConfig::StreamableHttp { http_headers, .. } = &server.transport else { + panic!("hosted plugin runtime should use streamable HTTP"); + }; + + assert_eq!( + http_headers + .as_ref() + .and_then(|headers| headers.get("originator")), + Some(&"codex_work_desktop".to_string()) + ); + + Ok(()) +} diff --git a/codex-rs/ext/mcp/tests/executor_plugin_mcp.rs b/codex-rs/ext/mcp/tests/executor_plugin_mcp.rs index da5c1fb6ad..afb6b0b923 100644 --- a/codex-rs/ext/mcp/tests/executor_plugin_mcp.rs +++ b/codex-rs/ext/mcp/tests/executor_plugin_mcp.rs @@ -202,6 +202,7 @@ async fn raw_selected_plugin_contributions( config, &thread_init, &thread_store, + "test_originator", &available_environment_ids, )) .await) diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rs index 4770456c36..716bb2e441 100644 --- a/codex-rs/features/src/lib.rs +++ b/codex-rs/features/src/lib.rs @@ -925,11 +925,7 @@ pub const FEATURES: &[FeatureSpec] = &[ FeatureSpec { id: Feature::MemoryTool, key: "memories", - stage: Stage::Experimental { - name: "Memories", - menu_description: "Allow Codex to create new memories from conversations and bring relevant memories into new conversations.", - announcement: "NEW: Codex can now generate and use memories. Try it now with `/memories`", - }, + stage: Stage::Stable, default_enabled: false, }, FeatureSpec { diff --git a/codex-rs/network-proxy/src/attribution_tests.rs b/codex-rs/network-proxy/src/attribution_tests.rs index e28110263c..fcb3c3771e 100644 --- a/codex-rs/network-proxy/src/attribution_tests.rs +++ b/codex-rs/network-proxy/src/attribution_tests.rs @@ -1,6 +1,6 @@ use super::BindConnectionAttribution; use super::write_attribution_frame; -use crate::config::NetworkProxySettings; +use crate::config::NetworkProxyConfig; use crate::runtime::network_proxy_state_for_policy; use crate::state::NetworkProxyState; use pretty_assertions::assert_eq; @@ -28,9 +28,7 @@ fn attribution_frame_has_bounded_binary_prefix() -> io::Result<()> { #[tokio::test] async fn framed_connection_receives_registered_execution_state() -> Result<(), BoxError> { - let state = Arc::new(network_proxy_state_for_policy( - NetworkProxySettings::default(), - )); + let state = Arc::new(network_proxy_state_for_policy(NetworkProxyConfig::default())); state.register_execution("token-1", "local", "execution-1"); let listener = TcpListener::bind("127.0.0.1:0").await?; diff --git a/codex-rs/network-proxy/src/config.rs b/codex-rs/network-proxy/src/config.rs index 30cdfa7b2d..3461c696ac 100644 --- a/codex-rs/network-proxy/src/config.rs +++ b/codex-rs/network-proxy/src/config.rs @@ -15,19 +15,6 @@ use url::Url; use crate::mitm_hook::MitmHookConfig; -#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] -pub struct NetworkProxyConfig { - #[serde(default)] - pub network: NetworkProxySettings, -} - -impl NetworkProxyConfig { - pub fn set_credential_broker_enabled(&mut self, enabled: bool) { - self.network.credential_broker = enabled; - self.network.mitm |= enabled; - } -} - /// Variant order encodes effective precedence for duplicate patterns: /// `None < Allow < Deny`, so deny wins over allow when entries conflict. #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] @@ -125,7 +112,7 @@ pub struct NetworkUnixSocketPermissions { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(default)] -pub struct NetworkProxySettings { +pub struct NetworkProxyConfig { #[serde(default)] pub enabled: bool, #[serde(default = "default_proxy_url")] @@ -156,7 +143,7 @@ pub struct NetworkProxySettings { pub mitm_hooks: Vec, } -impl Default for NetworkProxySettings { +impl Default for NetworkProxyConfig { fn default() -> Self { Self { enabled: false, @@ -179,7 +166,12 @@ impl Default for NetworkProxySettings { } } -impl NetworkProxySettings { +impl NetworkProxyConfig { + pub fn set_credential_broker_enabled(&mut self, enabled: bool) { + self.credential_broker = enabled; + self.mitm |= enabled; + } + pub fn allowed_domains(&self) -> Option> { self.domain_entries(NetworkDomainPermission::Allow) } @@ -340,7 +332,7 @@ fn clamp_non_loopback( pub(crate) fn clamp_bind_addrs( http_addr: SocketAddr, socks_addr: SocketAddr, - cfg: &NetworkProxySettings, + cfg: &NetworkProxyConfig, ) -> (SocketAddr, SocketAddr) { let http_addr = clamp_non_loopback( http_addr, @@ -416,7 +408,7 @@ impl ValidatedUnixSocketPath { } pub(crate) fn validate_unix_socket_allowlist_paths(cfg: &NetworkProxyConfig) -> Result<()> { - for (index, socket_path) in cfg.network.allow_unix_sockets().iter().enumerate() { + for (index, socket_path) in cfg.allow_unix_sockets().iter().enumerate() { ValidatedUnixSocketPath::parse(socket_path) .with_context(|| format!("invalid network.allow_unix_sockets[{index}]"))?; } @@ -426,11 +418,11 @@ pub(crate) fn validate_unix_socket_allowlist_paths(cfg: &NetworkProxyConfig) -> pub fn resolve_runtime(cfg: &NetworkProxyConfig) -> Result { validate_unix_socket_allowlist_paths(cfg)?; - let http_addr = resolve_addr(&cfg.network.proxy_url, /*default_port*/ 3128) - .with_context(|| format!("invalid network.proxy_url: {}", cfg.network.proxy_url))?; - let socks_addr = resolve_addr(&cfg.network.socks_url, /*default_port*/ 8081) - .with_context(|| format!("invalid network.socks_url: {}", cfg.network.socks_url))?; - let (http_addr, socks_addr) = clamp_bind_addrs(http_addr, socks_addr, &cfg.network); + let http_addr = resolve_addr(&cfg.proxy_url, /*default_port*/ 3128) + .with_context(|| format!("invalid network.proxy_url: {}", cfg.proxy_url))?; + let socks_addr = resolve_addr(&cfg.socks_url, /*default_port*/ 8081) + .with_context(|| format!("invalid network.socks_url: {}", cfg.socks_url))?; + let (http_addr, socks_addr) = clamp_bind_addrs(http_addr, socks_addr, cfg); Ok(RuntimeConfig { http_addr, @@ -575,8 +567,8 @@ mod tests { use pretty_assertions::assert_eq; - fn settings_with_unix_sockets(unix_sockets: &[&str]) -> NetworkProxySettings { - let mut settings = NetworkProxySettings::default(); + fn settings_with_unix_sockets(unix_sockets: &[&str]) -> NetworkProxyConfig { + let mut settings = NetworkProxyConfig::default(); if !unix_sockets.is_empty() { settings.set_allow_unix_sockets( unix_sockets @@ -591,8 +583,8 @@ mod tests { #[test] fn network_proxy_settings_default_matches_local_use_baseline() { assert_eq!( - NetworkProxySettings::default(), - NetworkProxySettings { + NetworkProxyConfig::default(), + NetworkProxyConfig { enabled: false, proxy_url: "http://127.0.0.1:3128".to_string(), enable_socks5: true, @@ -614,26 +606,19 @@ mod tests { } #[test] - fn partial_network_config_uses_struct_defaults_for_missing_fields() { - let config: NetworkProxyConfig = serde_json::from_str( - r#"{ - "network": { - "enabled": true - } - }"#, - ) - .unwrap(); - let expected = NetworkProxySettings { + fn network_proxy_config_uses_struct_defaults_for_missing_fields() { + let config: NetworkProxyConfig = serde_json::from_str(r#"{ "enabled": true }"#).unwrap(); + let expected = NetworkProxyConfig { enabled: true, - ..NetworkProxySettings::default() + ..NetworkProxyConfig::default() }; - assert_eq!(config.network, expected); + assert_eq!(config, expected); } #[test] fn set_allowed_domains_preserves_existing_deny_for_same_pattern() { - let mut settings = NetworkProxySettings::default(); + let mut settings = NetworkProxyConfig::default(); settings.set_denied_domains(vec!["example.com".to_string()]); settings.set_allowed_domains(vec!["example.com".to_string()]); @@ -647,36 +632,34 @@ mod tests { #[test] fn network_domain_permissions_serialize_to_effective_map_shape() { - let mut settings = NetworkProxySettings::default(); + let mut settings = NetworkProxyConfig::default(); settings.set_denied_domains(vec!["example.com".to_string()]); settings.set_allowed_domains(vec!["example.com".to_string()]); - let config = NetworkProxyConfig { network: settings }; + let config = settings; let value = serde_json::to_value(&config).unwrap(); assert_eq!( value, serde_json::json!({ - "network": { - "enabled": false, - "proxy_url": "http://127.0.0.1:3128", - "enable_socks5": true, - "socks_url": "http://127.0.0.1:8081", - "enable_socks5_udp": true, - "allow_upstream_proxy": true, - "dangerously_allow_non_loopback_proxy": false, - "dangerously_allow_all_unix_sockets": false, - "mode": "full", - "domains": { - "example.com": "deny", - }, - "unix_sockets": null, - "allow_local_binding": false, - "mitm": false, - "credential_broker": false, - "dangerously_allow_plaintext_credential_injection": false, - "mitm_hooks": [], - } + "enabled": false, + "proxy_url": "http://127.0.0.1:3128", + "enable_socks5": true, + "socks_url": "http://127.0.0.1:8081", + "enable_socks5_udp": true, + "allow_upstream_proxy": true, + "dangerously_allow_non_loopback_proxy": false, + "dangerously_allow_all_unix_sockets": false, + "mode": "full", + "domains": { + "example.com": "deny", + }, + "unix_sockets": null, + "allow_local_binding": false, + "mitm": false, + "credential_broker": false, + "dangerously_allow_plaintext_credential_injection": false, + "mitm_hooks": [], }) ); } @@ -815,7 +798,7 @@ mod tests { #[test] fn clamp_bind_addrs_allows_non_loopback_when_enabled() { - let cfg = NetworkProxySettings { + let cfg = NetworkProxyConfig { dangerously_allow_non_loopback_proxy: true, ..Default::default() }; @@ -846,7 +829,7 @@ mod tests { #[test] fn clamp_bind_addrs_forces_loopback_when_all_unix_sockets_enabled() { - let cfg = NetworkProxySettings { + let cfg = NetworkProxyConfig { dangerously_allow_non_loopback_proxy: true, dangerously_allow_all_unix_sockets: true, ..Default::default() @@ -862,9 +845,7 @@ mod tests { #[test] fn resolve_runtime_rejects_relative_allow_unix_sockets_entries() { - let cfg = NetworkProxyConfig { - network: settings_with_unix_sockets(&["relative.sock"]), - }; + let cfg = settings_with_unix_sockets(&["relative.sock"]); let err = match resolve_runtime(&cfg) { Ok(runtime) => panic!( @@ -881,9 +862,7 @@ mod tests { #[test] fn resolve_runtime_accepts_unix_style_absolute_allow_unix_sockets_entries() { - let cfg = NetworkProxyConfig { - network: settings_with_unix_sockets(&["/private/tmp/example.sock"]), - }; + let cfg = settings_with_unix_sockets(&["/private/tmp/example.sock"]); assert!( resolve_runtime(&cfg).is_ok(), diff --git a/codex-rs/network-proxy/src/connect_policy.rs b/codex-rs/network-proxy/src/connect_policy.rs index b9425db799..ed0e56f733 100644 --- a/codex-rs/network-proxy/src/connect_policy.rs +++ b/codex-rs/network-proxy/src/connect_policy.rs @@ -107,7 +107,7 @@ impl TargetPolicy { #[cfg(test)] mod tests { use super::*; - use crate::config::NetworkProxySettings; + use crate::config::NetworkProxyConfig; use crate::state::network_proxy_state_for_policy; use rama_net::address::HostWithPort; use std::net::Ipv4Addr; @@ -120,7 +120,7 @@ mod tests { .expect("bind local listener"); let target = listener.local_addr().expect("local addr"); let connector = TargetCheckedTcpConnector::new(Arc::new(network_proxy_state_for_policy( - NetworkProxySettings::default(), + NetworkProxyConfig::default(), ))); let request: rama_tcp::client::Request = @@ -142,9 +142,9 @@ mod tests { .expect("bind local listener"); let target = listener.local_addr().expect("local addr"); let connector = TargetCheckedTcpConnector::new(Arc::new(network_proxy_state_for_policy( - NetworkProxySettings { + NetworkProxyConfig { allow_local_binding: true, - ..NetworkProxySettings::default() + ..NetworkProxyConfig::default() }, ))); diff --git a/codex-rs/network-proxy/src/http_proxy.rs b/codex-rs/network-proxy/src/http_proxy.rs index 59cad9c20a..e81640b6e0 100644 --- a/codex-rs/network-proxy/src/http_proxy.rs +++ b/codex-rs/network-proxy/src/http_proxy.rs @@ -1087,7 +1087,7 @@ mod tests { use super::*; use crate::config::NetworkMode; - use crate::config::NetworkProxySettings; + use crate::config::NetworkProxyConfig; use crate::runtime::network_proxy_state_for_policy; use pretty_assertions::assert_eq; use rama_http::Method; @@ -1106,7 +1106,7 @@ mod tests { #[tokio::test] async fn http_connect_accept_blocks_in_limited_mode() { let policy = { - let mut policy = NetworkProxySettings::default(); + let mut policy = NetworkProxyConfig::default(); policy.set_allowed_domains(vec!["example.com".to_string()]); policy }; @@ -1136,9 +1136,9 @@ mod tests { #[tokio::test] async fn http_connect_accept_allows_allowlisted_host_in_full_mode() { let policy = { - let mut policy = NetworkProxySettings { + let mut policy = NetworkProxyConfig { allow_local_binding: true, - ..NetworkProxySettings::default() + ..NetworkProxyConfig::default() }; policy.set_allowed_domains(vec!["example.com".to_string()]); policy @@ -1163,9 +1163,7 @@ mod tests { #[tokio::test] async fn http_connect_accept_passes_environment_id_to_decider() { - let state = Arc::new(network_proxy_state_for_policy( - NetworkProxySettings::default(), - )); + let state = Arc::new(network_proxy_state_for_policy(NetworkProxyConfig::default())); let seen_environment_id = Arc::new(Mutex::new(None)); let decider: Arc = Arc::new({ let seen_environment_id = seen_environment_id.clone(); @@ -1202,10 +1200,10 @@ mod tests { #[tokio::test] async fn http_connect_accept_defers_brokered_host_mitm_until_protocol_detection() { - let mut policy = NetworkProxySettings { + let mut policy = NetworkProxyConfig { credential_broker: true, mitm: true, - ..NetworkProxySettings::default() + ..NetworkProxyConfig::default() }; policy.set_allowed_domains(vec!["github.com".to_string()]); let state = Arc::new(network_proxy_state_for_policy(policy)); @@ -1235,10 +1233,10 @@ mod tests { #[tokio::test] async fn plaintext_credential_injection_requires_explicit_opt_in() { let real_token = "ghp-real"; - let mut disabled_network = NetworkProxySettings { + let mut disabled_network = NetworkProxyConfig { credential_broker: true, mitm: true, - ..NetworkProxySettings::default() + ..NetworkProxyConfig::default() }; disabled_network.set_allowed_domains(vec!["api.github.com".to_string()]); let disabled_state = Arc::new(network_proxy_state_for_policy(disabled_network)); @@ -1263,11 +1261,11 @@ mod tests { Some(&HeaderValue::from_str(&format!("Bearer {dummy_token}")).unwrap()) ); - let mut enabled_network = NetworkProxySettings { + let mut enabled_network = NetworkProxyConfig { credential_broker: true, dangerously_allow_plaintext_credential_injection: true, mitm: true, - ..NetworkProxySettings::default() + ..NetworkProxyConfig::default() }; enabled_network.set_allowed_domains(vec!["api.github.com".to_string()]); let enabled_state = Arc::new(network_proxy_state_for_policy(enabled_network)); @@ -1295,7 +1293,7 @@ mod tests { #[tokio::test] async fn http_connect_accept_blocks_hooked_host_in_full_mode_without_mitm_state() { - let mut policy = NetworkProxySettings { + let mut policy = NetworkProxyConfig { mitm: true, mitm_hooks: vec![crate::mitm_hook::MitmHookConfig { host: "api.github.com".to_string(), @@ -1352,10 +1350,10 @@ mod tests { }); let state = Arc::new(network_proxy_state_for_policy({ - let mut network = NetworkProxySettings { + let mut network = NetworkProxyConfig { credential_broker: true, mitm: true, - ..NetworkProxySettings::default() + ..NetworkProxyConfig::default() }; network.set_allowed_domains(vec!["127.0.0.1".to_string()]); network.allow_local_binding = true; @@ -1416,9 +1414,7 @@ mod tests { #[tokio::test(flavor = "current_thread")] async fn http_plain_proxy_blocks_unix_socket_when_method_not_allowed() { - let state = Arc::new(network_proxy_state_for_policy( - NetworkProxySettings::default(), - )); + let state = Arc::new(network_proxy_state_for_policy(NetworkProxyConfig::default())); state .set_network_mode(NetworkMode::Limited) .await @@ -1447,9 +1443,7 @@ mod tests { #[tokio::test(flavor = "current_thread")] async fn http_plain_proxy_rejects_unix_socket_when_not_allowlisted() { - let state = Arc::new(network_proxy_state_for_policy( - NetworkProxySettings::default(), - )); + let state = Arc::new(network_proxy_state_for_policy(NetworkProxyConfig::default())); let mut req = Request::builder() .method(Method::GET) @@ -1480,7 +1474,7 @@ mod tests { #[tokio::test(flavor = "current_thread")] async fn http_plain_proxy_attempts_allowed_unix_socket_proxy() { let state = Arc::new(network_proxy_state_for_policy({ - let mut network = NetworkProxySettings::default(); + let mut network = NetworkProxyConfig::default(); network.set_allow_unix_sockets(vec!["/tmp/test.sock".to_string()]); network })); @@ -1504,7 +1498,7 @@ mod tests { #[tokio::test] async fn http_connect_accept_denies_denylisted_host() { let policy = { - let mut policy = NetworkProxySettings::default(); + let mut policy = NetworkProxyConfig::default(); policy.set_allowed_domains(vec!["**.openai.com".to_string()]); policy.set_denied_domains(vec!["api.openai.com".to_string()]); policy @@ -1533,9 +1527,7 @@ mod tests { #[tokio::test] async fn http_plain_proxy_rejects_absolute_uri_host_header_mismatch() { - let state = Arc::new(network_proxy_state_for_policy( - NetworkProxySettings::default(), - )); + let state = Arc::new(network_proxy_state_for_policy(NetworkProxyConfig::default())); let mut req = Request::builder() .method(Method::GET) .uri("http://raw.githubusercontent.com/openai/codex/main/README.md") diff --git a/codex-rs/network-proxy/src/lib.rs b/codex-rs/network-proxy/src/lib.rs index 6bf6d8eb38..753a0b0f4e 100644 --- a/codex-rs/network-proxy/src/lib.rs +++ b/codex-rs/network-proxy/src/lib.rs @@ -78,7 +78,6 @@ pub use runtime::NetworkProxyState; pub use state::NetworkProxyAuditMetadata; pub use state::NetworkProxyConstraintError; pub use state::NetworkProxyConstraints; -pub use state::PartialNetworkConfig; pub use state::PartialNetworkProxyConfig; pub use state::build_config_state; pub use state::validate_policy_against_constraints; diff --git a/codex-rs/network-proxy/src/mitm_hook.rs b/codex-rs/network-proxy/src/mitm_hook.rs index 6262dcd4ff..7ad370aaff 100644 --- a/codex-rs/network-proxy/src/mitm_hook.rs +++ b/codex-rs/network-proxy/src/mitm_hook.rs @@ -169,12 +169,12 @@ pub enum HookEvaluation { } pub(crate) fn validate_mitm_hook_config(config: &NetworkProxyConfig) -> Result<()> { - let hooks = &config.network.mitm_hooks; + let hooks = &config.mitm_hooks; if hooks.is_empty() { return Ok(()); } - if !config.network.mitm { + if !config.mitm { return Err(anyhow!("network.mitm_hooks requires network.mitm = true")); } @@ -274,7 +274,7 @@ where validate_mitm_hook_config(config)?; let mut hooks_by_host = MitmHooksByHost::new(); - for hook in &config.network.mitm_hooks { + for hook in &config.mitm_hooks { let host = normalize_hook_host(&hook.host)?; let methods = normalize_methods(&hook.matcher.methods)?; let path_prefixes = compile_path_matchers(&hook.matcher.path_prefixes)?; @@ -644,7 +644,7 @@ fn parse_secret_file(path: &str) -> Result { mod tests { use super::*; use crate::NetworkMode; - use crate::config::NetworkProxySettings; + use crate::config::NetworkProxyConfig; use pretty_assertions::assert_eq; use rama_http::Body; use rama_http::Method; @@ -652,11 +652,9 @@ mod tests { fn base_config() -> NetworkProxyConfig { NetworkProxyConfig { - network: NetworkProxySettings { - mitm: true, - mode: NetworkMode::Limited, - ..NetworkProxySettings::default() - }, + mitm: true, + mode: NetworkMode::Limited, + ..NetworkProxyConfig::default() } } @@ -683,8 +681,8 @@ mod tests { #[test] fn validate_requires_mitm_for_hooks() { let mut config = base_config(); - config.network.mitm = false; - config.network.mitm_hooks = vec![github_hook()]; + config.mitm = false; + config.mitm_hooks = vec![github_hook()]; let err = validate_mitm_hook_config(&config).expect_err("hooks require mitm"); assert!( @@ -696,8 +694,8 @@ mod tests { #[test] fn validate_allows_hooks_in_full_mode() { let mut config = base_config(); - config.network.mode = NetworkMode::Full; - config.network.mitm_hooks = vec![github_hook()]; + config.mode = NetworkMode::Full; + config.mitm_hooks = vec![github_hook()]; validate_mitm_hook_config(&config).expect("hooks should be allowed in full mode"); } @@ -709,7 +707,7 @@ mod tests { hook.matcher.body = Some(MitmHookBodyConfig(serde_json::json!({ "repository": "openai/codex" }))); - config.network.mitm_hooks = vec![hook]; + config.mitm_hooks = vec![hook]; let err = validate_mitm_hook_config(&config).expect_err("body matchers are reserved"); assert!(err.to_string().contains("match.body is reserved")); @@ -721,7 +719,7 @@ mod tests { let mut hook = github_hook(); hook.actions.inject_request_headers[0].secret_env_var = None; hook.actions.inject_request_headers[0].secret_file = Some("token.txt".to_string()); - config.network.mitm_hooks = vec![hook]; + config.mitm_hooks = vec![hook]; let err = validate_mitm_hook_config(&config).expect_err("secret file must be absolute"); assert!(format!("{err:#}").contains("secret_file must be an absolute path")); @@ -732,7 +730,7 @@ mod tests { let mut config = base_config(); let mut hook = github_hook(); hook.actions.inject_request_headers[0].secret_file = Some("/tmp/github-token".to_string()); - config.network.mitm_hooks = vec![hook]; + config.mitm_hooks = vec![hook]; let err = validate_mitm_hook_config(&config).expect_err("dual secret sources invalid"); assert!(format!("{err:#}").contains("exactly one of secret_env_var or secret_file")); @@ -741,7 +739,7 @@ mod tests { #[test] fn compile_resolves_env_backed_injected_headers() { let mut config = base_config(); - config.network.mitm_hooks = vec![github_hook()]; + config.mitm_hooks = vec![github_hook()]; let hooks = compile_mitm_hooks_with_resolvers( &config, @@ -772,7 +770,7 @@ mod tests { hook.actions.inject_request_headers[0].secret_env_var = None; hook.actions.inject_request_headers[0].secret_file = Some(secret_file.path().display().to_string()); - config.network.mitm_hooks = vec![hook]; + config.mitm_hooks = vec![hook]; let hooks = compile_mitm_hooks(&config).unwrap(); let compiled = hooks.get("api.github.com").unwrap(); @@ -789,7 +787,7 @@ mod tests { first.matcher.path_prefixes = vec!["/repos/openai/".to_string()]; let mut second = github_hook(); second.actions.inject_request_headers[0].prefix = Some("Token ".to_string()); - config.network.mitm_hooks = vec![first, second]; + config.mitm_hooks = vec![first, second]; let hooks = compile_mitm_hooks_with_resolvers( &config, @@ -827,7 +825,7 @@ mod tests { "x-github-api-version".to_string(), vec!["2022-11-28".to_string()], )]); - config.network.mitm_hooks = vec![hook]; + config.mitm_hooks = vec![hook]; let hooks = compile_mitm_hooks_with_resolvers( &config, @@ -861,7 +859,7 @@ mod tests { "x-github-api-version".to_string(), vec!["pattern:2022*preview".to_string()], )]); - config.network.mitm_hooks = vec![hook]; + config.mitm_hooks = vec![hook]; let hooks = compile_mitm_hooks_with_resolvers( &config, @@ -889,7 +887,7 @@ mod tests { let mut config = base_config(); let mut hook = github_hook(); hook.matcher.path_prefixes = vec!["pattern:/repos/[".to_string()]; - config.network.mitm_hooks = vec![hook]; + config.mitm_hooks = vec![hook]; let err = validate_mitm_hook_config(&config).expect_err("invalid glob should fail"); assert!(format!("{err:#}").contains("invalid glob pattern")); @@ -900,7 +898,7 @@ mod tests { let mut config = base_config(); let mut hook = github_hook(); hook.matcher.path_prefixes = vec!["pattern:/repos/*/codex/issues*".to_string()]; - config.network.mitm_hooks = vec![hook]; + config.mitm_hooks = vec![hook]; let hooks = compile_mitm_hooks_with_resolvers( &config, @@ -930,7 +928,7 @@ mod tests { "x-github-api-version".to_string(), vec!["2022-11-28[preview]".to_string()], )]); - config.network.mitm_hooks = vec![hook]; + config.mitm_hooks = vec![hook]; let hooks = compile_mitm_hooks_with_resolvers( &config, @@ -973,7 +971,7 @@ mod tests { "x-github-api-version".to_string(), vec!["literal:pattern:*".to_string()], )]); - config.network.mitm_hooks = vec![hook]; + config.mitm_hooks = vec![hook]; let hooks = compile_mitm_hooks_with_resolvers( &config, @@ -1011,7 +1009,7 @@ mod tests { let mut config = base_config(); let mut hook = github_hook(); hook.matcher.query = BTreeMap::from([("state".to_string(), vec!["open".to_string()])]); - config.network.mitm_hooks = vec![hook]; + config.mitm_hooks = vec![hook]; let hooks = compile_mitm_hooks_with_resolvers( &config, diff --git a/codex-rs/network-proxy/src/mitm_tests.rs b/codex-rs/network-proxy/src/mitm_tests.rs index 18e650baa3..31e9fc8aaf 100644 --- a/codex-rs/network-proxy/src/mitm_tests.rs +++ b/codex-rs/network-proxy/src/mitm_tests.rs @@ -1,6 +1,6 @@ use super::*; -use crate::config::NetworkProxySettings; +use crate::config::NetworkProxyConfig; use crate::reasons::REASON_METHOD_NOT_ALLOWED; use crate::reasons::REASON_MITM_HOOK_DENIED; use crate::reasons::REASON_NOT_ALLOWED_LOCAL; @@ -139,7 +139,7 @@ fn policy_ctx( #[tokio::test] async fn mitm_policy_blocks_disallowed_method_and_records_telemetry() { let app_state = Arc::new(network_proxy_state_for_policy({ - let mut network = NetworkProxySettings::default(); + let mut network = NetworkProxyConfig::default(); network.set_allowed_domains(vec!["example.com".to_string()]); network })); @@ -178,7 +178,7 @@ async fn mitm_policy_blocks_disallowed_method_and_records_telemetry() { #[tokio::test] async fn mitm_policy_rejects_host_mismatch() { let app_state = Arc::new(network_proxy_state_for_policy({ - let mut network = NetworkProxySettings::default(); + let mut network = NetworkProxyConfig::default(); network.set_allowed_domains(vec!["example.com".to_string()]); network })); @@ -207,7 +207,7 @@ async fn mitm_policy_rejects_host_mismatch() { #[tokio::test] async fn mitm_policy_rechecks_local_private_target_after_connect() { let app_state = Arc::new(network_proxy_state_for_policy({ - let mut network = NetworkProxySettings::default(); + let mut network = NetworkProxyConfig::default(); network.set_allowed_domains(vec!["example.com".to_string()]); network.allow_local_binding = false; network @@ -247,11 +247,11 @@ async fn mitm_policy_allows_matching_hooked_write_in_full_mode() { hook.actions.inject_request_headers[0].secret_env_var = None; hook.actions.inject_request_headers[0].secret_file = Some(secret_file.path().display().to_string()); - let mut network = NetworkProxySettings { + let mut network = NetworkProxyConfig { mitm: true, mitm_hooks: vec![hook], mode: NetworkMode::Full, - ..NetworkProxySettings::default() + ..NetworkProxyConfig::default() }; network.set_allowed_domains(vec!["api.github.com".to_string()]); let app_state = Arc::new(network_proxy_state_for_policy(network)); @@ -281,11 +281,11 @@ async fn mitm_policy_allows_matching_hooked_write_in_full_mode() { async fn mitm_policy_blocks_matching_hooked_write_in_limited_mode() { let mut hook = github_write_hook(); hook.actions.inject_request_headers.clear(); - let mut network = NetworkProxySettings { + let mut network = NetworkProxyConfig { mitm: true, mitm_hooks: vec![hook], mode: NetworkMode::Limited, - ..NetworkProxySettings::default() + ..NetworkProxyConfig::default() }; network.set_allowed_domains(vec!["api.github.com".to_string()]); let app_state = Arc::new(network_proxy_state_for_policy(network)); @@ -329,11 +329,11 @@ async fn mitm_policy_blocks_hook_miss_for_hooked_host_and_records_telemetry_in_f hook.actions.inject_request_headers[0].secret_env_var = None; hook.actions.inject_request_headers[0].secret_file = Some(secret_file.path().display().to_string()); - let mut network = NetworkProxySettings { + let mut network = NetworkProxyConfig { mitm: true, mitm_hooks: vec![hook], mode: NetworkMode::Full, - ..NetworkProxySettings::default() + ..NetworkProxyConfig::default() }; network.set_allowed_domains(vec!["api.github.com".to_string()]); let app_state = Arc::new(network_proxy_state_for_policy(network)); diff --git a/codex-rs/network-proxy/src/network_policy.rs b/codex-rs/network-proxy/src/network_policy.rs index 956aa6f682..05d9f06944 100644 --- a/codex-rs/network-proxy/src/network_policy.rs +++ b/codex-rs/network-proxy/src/network_policy.rs @@ -554,7 +554,6 @@ mod tests { use super::*; use crate::config::NetworkMode; use crate::config::NetworkProxyConfig; - use crate::config::NetworkProxySettings; use crate::reasons::REASON_DENIED; use crate::reasons::REASON_METHOD_NOT_ALLOWED; use crate::reasons::REASON_NOT_ALLOWED; @@ -595,12 +594,12 @@ mod tests { } fn state_with_metadata(metadata: NetworkProxyAuditMetadata) -> NetworkProxyState { - let network = NetworkProxySettings { + let network = NetworkProxyConfig { enabled: true, mode: NetworkMode::Full, - ..NetworkProxySettings::default() + ..NetworkProxyConfig::default() }; - let config = NetworkProxyConfig { network }; + let config = network; let state = build_config_state(config, NetworkProxyConstraints::default()).unwrap(); let reloader = Arc::new(StaticReloader { state: state.clone(), @@ -628,7 +627,7 @@ mod tests { #[tokio::test(flavor = "current_thread")] async fn evaluate_host_policy_emits_domain_event_for_decider_allow_override() { - let state = network_proxy_state_for_policy(NetworkProxySettings::default()); + let state = network_proxy_state_for_policy(NetworkProxyConfig::default()); let calls = Arc::new(AtomicUsize::new(0)); let decider: Arc = Arc::new({ let calls = calls.clone(); @@ -697,7 +696,7 @@ mod tests { #[tokio::test(flavor = "current_thread")] async fn evaluate_host_policy_emits_execution_id_for_baseline_allow() { let state = network_proxy_state_for_policy({ - let mut network = NetworkProxySettings::default(); + let mut network = NetworkProxyConfig::default(); network.set_allowed_domains(vec!["example.com".to_string()]); network }); @@ -737,7 +736,7 @@ mod tests { #[tokio::test(flavor = "current_thread")] async fn evaluate_host_policy_emits_domain_event_for_baseline_deny() { let state = network_proxy_state_for_policy({ - let mut network = NetworkProxySettings::default(); + let mut network = NetworkProxyConfig::default(); network.set_allowed_domains(vec!["example.com".to_string()]); network.set_denied_domains(vec!["blocked.com".to_string()]); network @@ -789,7 +788,7 @@ mod tests { #[tokio::test(flavor = "current_thread")] async fn evaluate_host_policy_emits_domain_event_for_decider_ask() { - let state = network_proxy_state_for_policy(NetworkProxySettings::default()); + let state = network_proxy_state_for_policy(NetworkProxyConfig::default()); let decider: Arc = Arc::new(|_req| async { NetworkDecision::ask(REASON_NOT_ALLOWED) }); let request = NetworkPolicyRequest::new(NetworkPolicyRequestArgs { @@ -876,7 +875,7 @@ mod tests { #[tokio::test(flavor = "current_thread")] async fn emit_block_decision_audit_event_emits_non_domain_event() { - let state = network_proxy_state_for_policy(NetworkProxySettings::default()); + let state = network_proxy_state_for_policy(NetworkProxyConfig::default()); let (_, events) = capture_events(|| async { emit_block_decision_audit_event( @@ -925,7 +924,7 @@ mod tests { #[tokio::test(flavor = "current_thread")] async fn evaluate_host_policy_still_denies_not_allowed_local_without_decider_override() { let state = network_proxy_state_for_policy({ - let mut network = NetworkProxySettings::default(); + let mut network = NetworkProxyConfig::default(); network.set_allowed_domains(vec!["example.com".to_string()]); network.allow_local_binding = false; network diff --git a/codex-rs/network-proxy/src/proxy.rs b/codex-rs/network-proxy/src/proxy.rs index 476c68aca8..3e6a1dbe53 100644 --- a/codex-rs/network-proxy/src/proxy.rs +++ b/codex-rs/network-proxy/src/proxy.rs @@ -183,55 +183,48 @@ impl NetworkProxyBuilder { .set_blocked_request_observer(self.blocked_request_observer.clone()) .await; let current_cfg = state.current_cfg().await?; - let (requested_http_addr, requested_socks_addr, reserved_listeners) = if self - .managed_by_codex - { - let runtime = config::resolve_runtime(¤t_cfg)?; - #[cfg(target_os = "windows")] - let (managed_http_addr, managed_socks_addr) = config::clamp_bind_addrs( - runtime.http_addr, - runtime.socks_addr, - ¤t_cfg.network, - ); - #[cfg(target_os = "windows")] - let reserved = reserve_windows_managed_listeners( - managed_http_addr, - managed_socks_addr, - current_cfg.network.enable_socks5, - ) - .context("reserve managed loopback proxy listeners")?; - #[cfg(not(target_os = "windows"))] - let reserved = reserve_loopback_ephemeral_listeners(current_cfg.network.enable_socks5) + let (requested_http_addr, requested_socks_addr, reserved_listeners) = + if self.managed_by_codex { + let runtime = config::resolve_runtime(¤t_cfg)?; + #[cfg(target_os = "windows")] + let (managed_http_addr, managed_socks_addr) = + config::clamp_bind_addrs(runtime.http_addr, runtime.socks_addr, ¤t_cfg); + #[cfg(target_os = "windows")] + let reserved = reserve_windows_managed_listeners( + managed_http_addr, + managed_socks_addr, + current_cfg.enable_socks5, + ) .context("reserve managed loopback proxy listeners")?; - let http_addr = reserved.http_addr()?; - let socks_addr = reserved.socks_addr(runtime.socks_addr)?; - ( - http_addr, - socks_addr, - Some(reserved.into_reserved_listeners()), - ) - } else { - let runtime = config::resolve_runtime(¤t_cfg)?; - ( - self.http_addr.unwrap_or(runtime.http_addr), - self.socks_addr.unwrap_or(runtime.socks_addr), - None, - ) - }; + #[cfg(not(target_os = "windows"))] + let reserved = reserve_loopback_ephemeral_listeners(current_cfg.enable_socks5) + .context("reserve managed loopback proxy listeners")?; + let http_addr = reserved.http_addr()?; + let socks_addr = reserved.socks_addr(runtime.socks_addr)?; + ( + http_addr, + socks_addr, + Some(reserved.into_reserved_listeners()), + ) + } else { + let runtime = config::resolve_runtime(¤t_cfg)?; + ( + self.http_addr.unwrap_or(runtime.http_addr), + self.socks_addr.unwrap_or(runtime.socks_addr), + None, + ) + }; // Reapply bind clamping for caller overrides so unix-socket proxying stays loopback-only. - let (http_addr, socks_addr) = config::clamp_bind_addrs( - requested_http_addr, - requested_socks_addr, - ¤t_cfg.network, - ); + let (http_addr, socks_addr) = + config::clamp_bind_addrs(requested_http_addr, requested_socks_addr, ¤t_cfg); Ok(NetworkProxy { state, http_addr, socks_addr, - socks_enabled: current_cfg.network.enable_socks5, - socks5_udp_enabled: current_cfg.network.enable_socks5_udp, + socks_enabled: current_cfg.enable_socks5, + socks5_udp_enabled: current_cfg.enable_socks5_udp, runtime_settings: Arc::new(RwLock::new(NetworkProxyRuntimeSettings::from_config( ¤t_cfg, )?)), @@ -317,16 +310,16 @@ struct NetworkProxyRuntimeSettings { impl NetworkProxyRuntimeSettings { fn from_config(config: &config::NetworkProxyConfig) -> Result { - let mitm_ca_trust_bundle = if config.network.mitm { + let mitm_ca_trust_bundle = if config.mitm { let env = crate::certs::ca_env_from_process(); Some(crate::certs::managed_ca_trust_bundle(&env)?) } else { None }; Ok(Self { - allow_local_binding: config.network.allow_local_binding, - allow_unix_sockets: config.network.allow_unix_sockets().into(), - dangerously_allow_all_unix_sockets: config.network.dangerously_allow_all_unix_sockets, + allow_local_binding: config.allow_local_binding, + allow_unix_sockets: config.allow_unix_sockets().into(), + dangerously_allow_all_unix_sockets: config.dangerously_allow_all_unix_sockets, mitm_ca_trust_bundle, }) } @@ -879,23 +872,23 @@ impl NetworkProxy { pub async fn replace_config_state(&self, new_state: ConfigState) -> Result<()> { let current_cfg = self.state.current_cfg().await?; anyhow::ensure!( - new_state.config.network.enabled == current_cfg.network.enabled, + new_state.config.enabled == current_cfg.enabled, "cannot update network.enabled on a running proxy" ); anyhow::ensure!( - new_state.config.network.proxy_url == current_cfg.network.proxy_url, + new_state.config.proxy_url == current_cfg.proxy_url, "cannot update network.proxy_url on a running proxy" ); anyhow::ensure!( - new_state.config.network.socks_url == current_cfg.network.socks_url, + new_state.config.socks_url == current_cfg.socks_url, "cannot update network.socks_url on a running proxy" ); anyhow::ensure!( - new_state.config.network.enable_socks5 == current_cfg.network.enable_socks5, + new_state.config.enable_socks5 == current_cfg.enable_socks5, "cannot update network.enable_socks5 on a running proxy" ); anyhow::ensure!( - new_state.config.network.enable_socks5_udp == current_cfg.network.enable_socks5_udp, + new_state.config.enable_socks5_udp == current_cfg.enable_socks5_udp, "cannot update network.enable_socks5_udp on a running proxy" ); @@ -922,7 +915,7 @@ impl NetworkProxy { "execution-scoped network proxy is already running" ); let current_cfg = self.state.current_cfg().await?; - if !current_cfg.network.enabled { + if !current_cfg.enabled { warn!("network.enabled is false; skipping proxy listeners"); return Ok(NetworkProxyHandle::noop()); } @@ -963,11 +956,11 @@ impl NetworkProxy { } }); - let socks_task = if current_cfg.network.enable_socks5 { + let socks_task = if current_cfg.enable_socks5 { let socks_state = self.state.clone(); let socks_decider = self.policy_decider.clone(); let socks_addr = self.socks_addr; - let enable_socks5_udp = current_cfg.network.enable_socks5_udp; + let enable_socks5_udp = current_cfg.enable_socks5_udp; Some(tokio::spawn(async move { match socks_listener { Some(listener) => { @@ -1095,7 +1088,7 @@ impl Drop for NetworkProxyHandle { #[cfg(test)] mod tests { use super::*; - use crate::config::NetworkProxySettings; + use crate::config::NetworkProxyConfig; use crate::state::network_proxy_state_for_policy; use pretty_assertions::assert_eq; use std::net::IpAddr; @@ -1111,10 +1104,10 @@ mod tests { drop(http_listener); drop(socks_listener); - let state = Arc::new(network_proxy_state_for_policy(NetworkProxySettings { + let state = Arc::new(network_proxy_state_for_policy(NetworkProxyConfig { proxy_url: format!("http://{http_addr}"), socks_url: format!("http://{socks_addr}"), - ..NetworkProxySettings::default() + ..NetworkProxyConfig::default() })); let proxy = match NetworkProxy::builder().state(state).build().await { Ok(proxy) => proxy, @@ -1145,10 +1138,10 @@ mod tests { #[tokio::test] async fn non_codex_managed_proxy_builder_uses_configured_ports() { - let settings = NetworkProxySettings { + let settings = NetworkProxyConfig { proxy_url: "http://127.0.0.1:43128".to_string(), socks_url: "http://127.0.0.1:48081".to_string(), - ..NetworkProxySettings::default() + ..NetworkProxyConfig::default() }; let state = Arc::new(network_proxy_state_for_policy(settings)); let proxy = NetworkProxy::builder() @@ -1170,9 +1163,7 @@ mod tests { #[tokio::test] async fn prepare_for_environment_keeps_env_and_sandbox_ports_in_sync() -> Result<()> { - let state = Arc::new(network_proxy_state_for_policy( - NetworkProxySettings::default(), - )); + let state = Arc::new(network_proxy_state_for_policy(NetworkProxyConfig::default())); let proxy = NetworkProxy::builder().state(state).build().await?; let handle = proxy.run().await?; @@ -1229,11 +1220,11 @@ mod tests { #[tokio::test] async fn managed_proxy_builder_does_not_reserve_socks_listener_when_disabled() { - let settings = NetworkProxySettings { + let settings = NetworkProxyConfig { enable_socks5: false, proxy_url: "http://127.0.0.1:43128".to_string(), socks_url: "http://127.0.0.1:43129".to_string(), - ..NetworkProxySettings::default() + ..NetworkProxyConfig::default() }; let state = Arc::new(network_proxy_state_for_policy(settings)); let proxy = match NetworkProxy::builder().state(state).build().await { diff --git a/codex-rs/network-proxy/src/runtime.rs b/codex-rs/network-proxy/src/runtime.rs index a51f1d8f16..25954e83e9 100644 --- a/codex-rs/network-proxy/src/runtime.rs +++ b/codex-rs/network-proxy/src/runtime.rs @@ -298,7 +298,7 @@ impl NetworkProxyState { blocked_request_observer: Option>, ) -> Self { Self { - credential_broker: CredentialBroker::new(state.config.network.credential_broker), + credential_broker: CredentialBroker::new(state.config.credential_broker), state: Arc::new(RwLock::new(state)), reloader, blocked_request_observer: Arc::new(RwLock::new(blocked_request_observer)), @@ -381,7 +381,6 @@ impl NetworkProxyState { let guard = self.state.read().await; Ok(guard .config - .network .dangerously_allow_plaintext_credential_injection) } @@ -397,15 +396,15 @@ impl NetworkProxyState { self.reload_if_needed().await?; let guard = self.state.read().await; Ok(( - guard.config.network.allowed_domains().unwrap_or_default(), - guard.config.network.denied_domains().unwrap_or_default(), + guard.config.allowed_domains().unwrap_or_default(), + guard.config.denied_domains().unwrap_or_default(), )) } pub async fn enabled(&self) -> Result { self.reload_if_needed().await?; let guard = self.state.read().await; - Ok(guard.config.network.enabled) + Ok(guard.config.enabled) } pub async fn force_reload(&self) -> Result<()> { @@ -457,11 +456,11 @@ impl NetworkProxyState { }; let (deny_set, allow_set, allow_local_binding, allowed_domains) = { let guard = self.state.read().await; - let allowed_domains = guard.config.network.allowed_domains(); + let allowed_domains = guard.config.allowed_domains(); ( guard.deny_set.clone(), guard.allow_set.clone(), - guard.config.network.allow_local_binding, + guard.config.allow_local_binding, allowed_domains, ) }; @@ -594,7 +593,7 @@ impl NetworkProxyState { } let guard = self.state.read().await; - if guard.config.network.dangerously_allow_all_unix_sockets { + if guard.config.dangerously_allow_all_unix_sockets { return Ok(true); } @@ -604,7 +603,7 @@ impl NetworkProxyState { Err(_) => return Ok(false), }; let requested_canonical = std::fs::canonicalize(requested_abs.as_path()).ok(); - for allowed in &guard.config.network.allow_unix_sockets() { + for allowed in &guard.config.allow_unix_sockets() { let allowed_path = match ValidatedUnixSocketPath::parse(allowed) { Ok(ValidatedUnixSocketPath::Native(path)) => path, Ok(ValidatedUnixSocketPath::UnixStyleAbsolute(_)) => continue, @@ -635,25 +634,25 @@ impl NetworkProxyState { pub async fn method_allowed(&self, method: &str) -> Result { self.reload_if_needed().await?; let guard = self.state.read().await; - Ok(guard.config.network.mode.allows_method(method)) + Ok(guard.config.mode.allows_method(method)) } pub async fn allow_upstream_proxy(&self) -> Result { self.reload_if_needed().await?; let guard = self.state.read().await; - Ok(guard.config.network.allow_upstream_proxy) + Ok(guard.config.allow_upstream_proxy) } pub async fn allow_local_binding(&self) -> Result { self.reload_if_needed().await?; let guard = self.state.read().await; - Ok(guard.config.network.allow_local_binding) + Ok(guard.config.allow_local_binding) } pub async fn network_mode(&self) -> Result { self.reload_if_needed().await?; let guard = self.state.read().await; - Ok(guard.config.network.mode) + Ok(guard.config.mode) } pub async fn set_network_mode(&self, mode: NetworkMode) -> Result<()> { @@ -662,7 +661,7 @@ impl NetworkProxyState { let (candidate, constraints) = { let guard = self.state.read().await; let mut candidate = guard.config.clone(); - candidate.network.mode = mode; + candidate.mode = mode; (candidate, guard.constraints.clone()) }; @@ -675,7 +674,7 @@ impl NetworkProxyState { drop(guard); continue; } - guard.config.network.mode = mode; + guard.config.mode = mode; info!("updated network mode to {mode:?}"); return Ok(()); } @@ -740,8 +739,8 @@ impl NetworkProxyState { }; let mut candidate = previous_cfg.clone(); - let target_entries = target.entries(&candidate.network); - let opposite_entries = target.opposite_entries(&candidate.network); + let target_entries = target.entries(&candidate); + let opposite_entries = target.opposite_entries(&candidate); let target_contains = target_entries .iter() .any(|entry| normalize_host(entry) == normalized_host); @@ -752,7 +751,7 @@ impl NetworkProxyState { return Ok(()); } - candidate.network.upsert_domain_permission( + candidate.upsert_domain_permission( normalized_host.clone(), target.permission(), normalize_host, @@ -809,7 +808,7 @@ impl NetworkProxyState { fn ensure_credential_broker_enablement_unchanged(&self, new_state: &ConfigState) -> Result<()> { anyhow::ensure!( - self.credential_broker.enabled() == new_state.config.network.credential_broker, + self.credential_broker.enabled() == new_state.config.credential_broker, "network.credential_broker cannot change while the proxy is running" ); Ok(()) @@ -844,14 +843,14 @@ impl DomainListKind { } } - fn entries(self, network: &crate::config::NetworkProxySettings) -> Vec { + fn entries(self, network: &crate::config::NetworkProxyConfig) -> Vec { match self { Self::Allow => network.allowed_domains().unwrap_or_default(), Self::Deny => network.denied_domains().unwrap_or_default(), } } - fn opposite_entries(self, network: &crate::config::NetworkProxySettings) -> Vec { + fn opposite_entries(self, network: &crate::config::NetworkProxyConfig) -> Vec { match self { Self::Allow => network.denied_domains().unwrap_or_default(), Self::Deny => network.allowed_domains().unwrap_or_default(), @@ -905,15 +904,15 @@ where } fn log_policy_changes(previous: &NetworkProxyConfig, next: &NetworkProxyConfig) { - let previous_allowed_domains = previous.network.allowed_domains().unwrap_or_default(); - let next_allowed_domains = next.network.allowed_domains().unwrap_or_default(); + let previous_allowed_domains = previous.allowed_domains().unwrap_or_default(); + let next_allowed_domains = next.allowed_domains().unwrap_or_default(); log_domain_list_changes( "allowlist", &previous_allowed_domains, &next_allowed_domains, ); - let previous_denied_domains = previous.network.denied_domains().unwrap_or_default(); - let next_denied_domains = next.network.denied_domains().unwrap_or_default(); + let previous_denied_domains = previous.denied_domains().unwrap_or_default(); + let next_denied_domains = next.denied_domains().unwrap_or_default(); log_domain_list_changes("denylist", &previous_denied_domains, &next_denied_domains); } @@ -980,13 +979,13 @@ fn unix_timestamp() -> i64 { #[cfg(test)] pub(crate) fn network_proxy_state_for_policy( - mut network: crate::config::NetworkProxySettings, + mut network: crate::config::NetworkProxyConfig, ) -> NetworkProxyState { network.enabled = true; - let config = NetworkProxyConfig { network }; + let config = network; let state = ConfigState { allow_set: crate::policy::compile_allowlist_globset( - &config.network.allowed_domains().unwrap_or_default(), + &config.allowed_domains().unwrap_or_default(), ) .unwrap(), blocked: VecDeque::new(), @@ -994,7 +993,7 @@ pub(crate) fn network_proxy_state_for_policy( config: config.clone(), constraints: NetworkProxyConstraints::default(), deny_set: crate::policy::compile_denylist_globset( - &config.network.denied_domains().unwrap_or_default(), + &config.denied_domains().unwrap_or_default(), ) .unwrap(), mitm: None, @@ -1027,7 +1026,6 @@ mod tests { use super::*; use crate::config::NetworkProxyConfig; - use crate::config::NetworkProxySettings; use crate::policy::compile_allowlist_globset; use crate::policy::compile_denylist_globset; use crate::state::NetworkProxyConstraints; @@ -1060,8 +1058,8 @@ mod tests { entries.iter().map(|entry| (*entry).to_string()).collect() } - fn network_settings(allowed_domains: &[&str], denied_domains: &[&str]) -> NetworkProxySettings { - let mut network = NetworkProxySettings::default(); + fn network_settings(allowed_domains: &[&str], denied_domains: &[&str]) -> NetworkProxyConfig { + let mut network = NetworkProxyConfig::default(); if !allowed_domains.is_empty() { network.set_allowed_domains(strings(allowed_domains)); } @@ -1075,7 +1073,7 @@ mod tests { allowed_domains: &[&str], denied_domains: &[&str], unix_sockets: &[String], - ) -> NetworkProxySettings { + ) -> NetworkProxyConfig { let mut network = network_settings(allowed_domains, denied_domains); if !unix_sockets.is_empty() { network.set_allow_unix_sockets(unix_sockets.to_vec()); @@ -1209,13 +1207,8 @@ mod tests { #[tokio::test] async fn add_allowed_domain_succeeds_when_managed_baseline_allows_expansion() { - let config = NetworkProxyConfig { - network: { - let mut network = network_settings(&["managed.example.com"], &[]); - network.enabled = true; - network - }, - }; + let mut config = network_settings(&["managed.example.com"], &[]); + config.enabled = true; let constraints = NetworkProxyConstraints { allowed_domains: Some(vec!["managed.example.com".to_string()]), allowlist_expansion_enabled: Some(true), @@ -1241,13 +1234,8 @@ mod tests { #[tokio::test] async fn add_allowed_domain_rejects_expansion_when_managed_baseline_is_fixed() { - let config = NetworkProxyConfig { - network: { - let mut network = network_settings(&["managed.example.com"], &[]); - network.enabled = true; - network - }, - }; + let mut config = network_settings(&["managed.example.com"], &[]); + config.enabled = true; let constraints = NetworkProxyConstraints { allowed_domains: Some(vec!["managed.example.com".to_string()]), allowlist_expansion_enabled: Some(false), @@ -1271,13 +1259,8 @@ mod tests { #[tokio::test] async fn add_denied_domain_rejects_expansion_when_managed_baseline_is_fixed() { - let config = NetworkProxyConfig { - network: { - let mut network = network_settings(&[], &["managed.example.com"]); - network.enabled = true; - network - }, - }; + let mut config = network_settings(&[], &["managed.example.com"]); + config.enabled = true; let constraints = NetworkProxyConstraints { denied_domains: Some(vec!["managed.example.com".to_string()]), denylist_expansion_enabled: Some(false), @@ -1301,7 +1284,7 @@ mod tests { #[tokio::test] async fn blocked_snapshot_does_not_consume_entries() { - let state = network_proxy_state_for_policy(NetworkProxySettings::default()); + let state = network_proxy_state_for_policy(NetworkProxyConfig::default()); state .record_blocked(BlockedRequest::new(BlockedRequestArgs { @@ -1340,7 +1323,7 @@ mod tests { #[tokio::test] async fn drain_blocked_returns_buffered_window() { - let state = network_proxy_state_for_policy(NetworkProxySettings::default()); + let state = network_proxy_state_for_policy(NetworkProxyConfig::default()); for idx in 0..(MAX_BLOCKED_EVENTS + 5) { state @@ -1492,7 +1475,7 @@ mod tests { #[tokio::test] async fn host_blocked_requires_exact_scoped_ipv6_allowlist_match() { - let state = network_proxy_state_for_policy(NetworkProxySettings { + let state = network_proxy_state_for_policy(NetworkProxyConfig { allow_local_binding: true, ..network_settings(&["fe80::1%eth0"], &[]) }); @@ -1515,7 +1498,7 @@ mod tests { #[tokio::test] async fn host_blocked_denies_scoped_ipv6_literal_before_local_binding() { - let state = network_proxy_state_for_policy(NetworkProxySettings { + let state = network_proxy_state_for_policy(NetworkProxyConfig { allow_local_binding: true, ..network_settings(&["*"], &["fd00::1"]) }); @@ -1531,7 +1514,7 @@ mod tests { #[tokio::test] async fn host_blocked_requires_exact_scoped_ipv6_denylist_match() { - let state = network_proxy_state_for_policy(NetworkProxySettings { + let state = network_proxy_state_for_policy(NetworkProxyConfig { allow_local_binding: true, ..network_settings(&["*"], &["fd00::1%eth0"]) }); @@ -1564,7 +1547,7 @@ mod tests { #[tokio::test] async fn host_blocked_rejects_loopback_when_allowlist_empty() { - let state = network_proxy_state_for_policy(NetworkProxySettings::default()); + let state = network_proxy_state_for_policy(NetworkProxyConfig::default()); assert_eq!( state.host_blocked("127.0.0.1", /*port*/ 80).await.unwrap(), @@ -1574,7 +1557,7 @@ mod tests { #[tokio::test] async fn host_blocked_rejects_allowlisted_hostname_when_dns_lookup_fails() { - let mut network = NetworkProxySettings::default(); + let mut network = NetworkProxyConfig::default(); network.set_allowed_domains(vec!["does-not-resolve.invalid".to_string()]); let state = network_proxy_state_for_policy(network); @@ -1653,13 +1636,8 @@ mod tests { ..NetworkProxyConstraints::default() }; - let config = NetworkProxyConfig { - network: { - let mut network = network_settings(&["example.com", "evil.com"], &[]); - network.enabled = true; - network - }, - }; + let mut config = network_settings(&["example.com", "evil.com"], &[]); + config.enabled = true; assert!(validate_policy_against_constraints(&config, &constraints).is_err()); } @@ -1672,13 +1650,8 @@ mod tests { ..NetworkProxyConstraints::default() }; - let config = NetworkProxyConfig { - network: { - let mut network = network_settings(&["example.com", "api.openai.com"], &[]); - network.enabled = true; - network - }, - }; + let mut config = network_settings(&["example.com", "api.openai.com"], &[]); + config.enabled = true; assert!(validate_policy_against_constraints(&config, &constraints).is_ok()); } @@ -1691,11 +1664,9 @@ mod tests { }; let config = NetworkProxyConfig { - network: NetworkProxySettings { - enabled: true, - mode: NetworkMode::Full, - ..NetworkProxySettings::default() - }, + enabled: true, + mode: NetworkMode::Full, + ..NetworkProxyConfig::default() }; assert!(validate_policy_against_constraints(&config, &constraints).is_err()); @@ -1708,13 +1679,8 @@ mod tests { ..NetworkProxyConstraints::default() }; - let config = NetworkProxyConfig { - network: { - let mut network = network_settings(&["api.example.com"], &[]); - network.enabled = true; - network - }, - }; + let mut config = network_settings(&["api.example.com"], &[]); + config.enabled = true; assert!(validate_policy_against_constraints(&config, &constraints).is_ok()); } @@ -1726,13 +1692,8 @@ mod tests { ..NetworkProxyConstraints::default() }; - let config = NetworkProxyConfig { - network: { - let mut network = network_settings(&["**.example.com"], &[]); - network.enabled = true; - network - }, - }; + let mut config = network_settings(&["**.example.com"], &[]); + config.enabled = true; assert!(validate_policy_against_constraints(&config, &constraints).is_err()); } @@ -1744,13 +1705,8 @@ mod tests { ..NetworkProxyConstraints::default() }; - let config = NetworkProxyConfig { - network: { - let mut network = network_settings(&["api.example.com"], &[]); - network.enabled = true; - network - }, - }; + let mut config = network_settings(&["api.example.com"], &[]); + config.enabled = true; assert!(validate_policy_against_constraints(&config, &constraints).is_err()); } @@ -1763,13 +1719,8 @@ mod tests { ..NetworkProxyConstraints::default() }; - let config = NetworkProxyConfig { - network: { - let mut network = network_settings(&["api.example.com"], &[]); - network.enabled = true; - network - }, - }; + let mut config = network_settings(&["api.example.com"], &[]); + config.enabled = true; assert!(validate_policy_against_constraints(&config, &constraints).is_err()); } @@ -1782,13 +1733,8 @@ mod tests { ..NetworkProxyConstraints::default() }; - let config = NetworkProxyConfig { - network: { - let mut network = network_settings(&["api.example.com"], &[]); - network.enabled = true; - network - }, - }; + let mut config = network_settings(&["api.example.com"], &[]); + config.enabled = true; assert!(validate_policy_against_constraints(&config, &constraints).is_err()); } @@ -1801,10 +1747,8 @@ mod tests { }; let config = NetworkProxyConfig { - network: NetworkProxySettings { - enabled: true, - ..NetworkProxySettings::default() - }, + enabled: true, + ..NetworkProxyConfig::default() }; assert!(validate_policy_against_constraints(&config, &constraints).is_err()); @@ -1818,13 +1762,8 @@ mod tests { ..NetworkProxyConstraints::default() }; - let config = NetworkProxyConfig { - network: { - let mut network = network_settings(&[], &["evil.com", "more-evil.com"]); - network.enabled = true; - network - }, - }; + let mut config = network_settings(&[], &["evil.com", "more-evil.com"]); + config.enabled = true; assert!(validate_policy_against_constraints(&config, &constraints).is_err()); } @@ -1837,10 +1776,8 @@ mod tests { }; let config = NetworkProxyConfig { - network: NetworkProxySettings { - enabled: true, - ..NetworkProxySettings::default() - }, + enabled: true, + ..NetworkProxyConfig::default() }; assert!(validate_policy_against_constraints(&config, &constraints).is_err()); @@ -1854,11 +1791,9 @@ mod tests { }; let config = NetworkProxyConfig { - network: NetworkProxySettings { - enabled: true, - allow_local_binding: true, - ..NetworkProxySettings::default() - }, + enabled: true, + allow_local_binding: true, + ..NetworkProxyConfig::default() }; assert!(validate_policy_against_constraints(&config, &constraints).is_err()); @@ -1873,11 +1808,9 @@ mod tests { }; let config = NetworkProxyConfig { - network: NetworkProxySettings { - enabled: true, - dangerously_allow_all_unix_sockets: true, - ..NetworkProxySettings::default() - }, + enabled: true, + dangerously_allow_all_unix_sockets: true, + ..NetworkProxyConfig::default() }; assert!(validate_policy_against_constraints(&config, &constraints).is_err()); @@ -1892,11 +1825,9 @@ mod tests { }; let config = NetworkProxyConfig { - network: NetworkProxySettings { - enabled: true, - dangerously_allow_all_unix_sockets: true, - ..NetworkProxySettings::default() - }, + enabled: true, + dangerously_allow_all_unix_sockets: true, + ..NetworkProxyConfig::default() }; assert!(validate_policy_against_constraints(&config, &constraints).is_err()); @@ -1910,11 +1841,9 @@ mod tests { }; let config = NetworkProxyConfig { - network: NetworkProxySettings { - enabled: true, - dangerously_allow_all_unix_sockets: true, - ..NetworkProxySettings::default() - }, + enabled: true, + dangerously_allow_all_unix_sockets: true, + ..NetworkProxyConfig::default() }; assert!(validate_policy_against_constraints(&config, &constraints).is_ok()); @@ -1925,11 +1854,9 @@ mod tests { let constraints = NetworkProxyConstraints::default(); let config = NetworkProxyConfig { - network: NetworkProxySettings { - enabled: true, - dangerously_allow_all_unix_sockets: true, - ..NetworkProxySettings::default() - }, + enabled: true, + dangerously_allow_all_unix_sockets: true, + ..NetworkProxyConfig::default() }; assert!(validate_policy_against_constraints(&config, &constraints).is_ok()); @@ -2005,52 +1932,32 @@ mod tests { #[test] fn build_config_state_allows_global_wildcard_allowed_domains() { - let config = NetworkProxyConfig { - network: { - let mut network = network_settings(&["*"], &[]); - network.enabled = true; - network - }, - }; + let mut config = network_settings(&["*"], &[]); + config.enabled = true; assert!(build_config_state(config, NetworkProxyConstraints::default()).is_ok()); } #[test] fn build_config_state_allows_bracketed_global_wildcard_allowed_domains() { - let config = NetworkProxyConfig { - network: { - let mut network = network_settings(&["[*]"], &[]); - network.enabled = true; - network - }, - }; + let mut config = network_settings(&["[*]"], &[]); + config.enabled = true; assert!(build_config_state(config, NetworkProxyConstraints::default()).is_ok()); } #[test] fn build_config_state_rejects_global_wildcard_denied_domains() { - let config = NetworkProxyConfig { - network: { - let mut network = network_settings(&["example.com"], &["*"]); - network.enabled = true; - network - }, - }; + let mut config = network_settings(&["example.com"], &["*"]); + config.enabled = true; assert!(build_config_state(config, NetworkProxyConstraints::default()).is_err()); } #[test] fn build_config_state_rejects_bracketed_global_wildcard_denied_domains() { - let config = NetworkProxyConfig { - network: { - let mut network = network_settings(&["example.com"], &["[*]"]); - network.enabled = true; - network - }, - }; + let mut config = network_settings(&["example.com"], &["[*]"]); + config.enabled = true; assert!(build_config_state(config, NetworkProxyConstraints::default()).is_err()); } diff --git a/codex-rs/network-proxy/src/socks5.rs b/codex-rs/network-proxy/src/socks5.rs index d1c193198f..f2fa36b8ef 100644 --- a/codex-rs/network-proxy/src/socks5.rs +++ b/codex-rs/network-proxy/src/socks5.rs @@ -809,7 +809,6 @@ mod tests { use super::*; use crate::config::NetworkMode; use crate::config::NetworkProxyConfig; - use crate::config::NetworkProxySettings; use crate::mitm_hook::MitmHookConfig; use crate::mitm_hook::MitmHookMatchConfig; use crate::network_policy::test_support::POLICY_DECISION_EVENT_NAME; @@ -855,12 +854,9 @@ mod tests { } } - fn state_for_settings(network: NetworkProxySettings) -> Arc { - let config = NetworkProxyConfig { network }; - let _mitm_config_state_guard = config - .network - .mitm - .then(|| MITM_CONFIG_STATE_LOCK.lock().unwrap()); + fn state_for_settings(network: NetworkProxyConfig) -> Arc { + let config = network; + let _mitm_config_state_guard = config.mitm.then(|| MITM_CONFIG_STATE_LOCK.lock().unwrap()); let state = build_config_state(config, NetworkProxyConstraints::default()).unwrap(); let reloader = Arc::new(StaticReloader { state: state.clone(), @@ -870,10 +866,10 @@ mod tests { #[tokio::test(flavor = "current_thread")] async fn handle_socks5_tcp_emits_block_decision_for_proxy_disabled() { - let state = state_for_settings(NetworkProxySettings { + let state = state_for_settings(NetworkProxyConfig { enabled: false, mode: NetworkMode::Full, - ..NetworkProxySettings::default() + ..NetworkProxyConfig::default() }); let mut request = TcpRequest::new(HostWithPort::try_from("example.com:443").expect("valid authority")); @@ -912,11 +908,11 @@ mod tests { #[tokio::test(flavor = "current_thread")] async fn handle_socks5_tcp_uses_mitm_in_limited_mode() { - let mut settings = NetworkProxySettings { + let mut settings = NetworkProxyConfig { enabled: true, mode: NetworkMode::Limited, mitm: true, - ..NetworkProxySettings::default() + ..NetworkProxyConfig::default() }; settings.set_allowed_domains(vec!["example.com".to_string()]); let state = state_for_settings(settings); @@ -938,10 +934,10 @@ mod tests { #[tokio::test(flavor = "current_thread")] async fn handle_socks5_tcp_blocks_non_https_in_limited_mode() { - let mut settings = NetworkProxySettings { + let mut settings = NetworkProxyConfig { enabled: true, mode: NetworkMode::Limited, - ..NetworkProxySettings::default() + ..NetworkProxyConfig::default() }; settings.set_allowed_domains(vec!["example.com".to_string()]); let state = state_for_settings(settings); @@ -985,12 +981,12 @@ mod tests { #[tokio::test(flavor = "current_thread")] async fn handle_socks5_tcp_detects_tls_for_brokered_nonstandard_port_in_full_mode() { - let mut settings = NetworkProxySettings { + let mut settings = NetworkProxyConfig { enabled: true, mode: NetworkMode::Full, mitm: true, credential_broker: true, - ..NetworkProxySettings::default() + ..NetworkProxyConfig::default() }; settings.set_allowed_domains(vec!["api.openai.com".to_string()]); let state = state_for_settings(settings); @@ -1015,10 +1011,10 @@ mod tests { #[tokio::test(flavor = "current_thread")] async fn handle_socks5_tcp_blocks_limited_mode_without_mitm_state() { - let mut settings = NetworkProxySettings { + let mut settings = NetworkProxyConfig { enabled: true, mode: NetworkMode::Limited, - ..NetworkProxySettings::default() + ..NetworkProxyConfig::default() }; settings.set_allowed_domains(vec!["example.com".to_string()]); let state = state_for_settings(settings); @@ -1043,7 +1039,7 @@ mod tests { #[tokio::test(flavor = "current_thread")] async fn handle_socks5_tcp_uses_mitm_for_hooked_host_in_full_mode() { - let mut settings = NetworkProxySettings { + let mut settings = NetworkProxyConfig { enabled: true, mode: NetworkMode::Full, mitm: true, @@ -1056,7 +1052,7 @@ mod tests { }, ..MitmHookConfig::default() }], - ..NetworkProxySettings::default() + ..NetworkProxyConfig::default() }; settings.set_allowed_domains(vec!["api.github.com".to_string()]); let state = state_for_settings(settings); @@ -1078,7 +1074,7 @@ mod tests { #[tokio::test(flavor = "current_thread")] async fn handle_socks5_tcp_blocks_hooked_non_https_host_in_full_mode() { - let mut settings = NetworkProxySettings { + let mut settings = NetworkProxyConfig { enabled: true, mode: NetworkMode::Full, mitm: true, @@ -1091,7 +1087,7 @@ mod tests { }, ..MitmHookConfig::default() }], - ..NetworkProxySettings::default() + ..NetworkProxyConfig::default() }; settings.set_allowed_domains(vec!["api.github.com".to_string()]); let state = state_for_settings(settings); @@ -1116,10 +1112,10 @@ mod tests { #[tokio::test(flavor = "current_thread")] async fn inspect_socks5_udp_emits_block_decision_for_mode_guard_deny() { - let state = state_for_settings(NetworkProxySettings { + let state = state_for_settings(NetworkProxyConfig { enabled: true, mode: NetworkMode::Limited, - ..NetworkProxySettings::default() + ..NetworkProxyConfig::default() }); let request = RelayRequest { direction: RelayDirection::South, diff --git a/codex-rs/network-proxy/src/state.rs b/codex-rs/network-proxy/src/state.rs index 9a5287dac4..c61f6b1617 100644 --- a/codex-rs/network-proxy/src/state.rs +++ b/codex-rs/network-proxy/src/state.rs @@ -40,12 +40,6 @@ pub struct NetworkProxyConstraints { #[derive(Debug, Clone, Deserialize)] pub struct PartialNetworkProxyConfig { - #[serde(default)] - pub network: PartialNetworkConfig, -} - -#[derive(Debug, Default, Clone, Deserialize)] -pub struct PartialNetworkConfig { pub enabled: Option, pub mode: Option, pub allow_upstream_proxy: Option, @@ -69,20 +63,20 @@ pub fn build_config_state( ) -> anyhow::Result { crate::config::validate_unix_socket_allowlist_paths(&config)?; anyhow::ensure!( - !config.network.credential_broker || config.network.mitm, + !config.credential_broker || config.mitm, "network.credential_broker requires network.mitm = true" ); - let allowed_domains = config.network.allowed_domains().unwrap_or_default(); - let denied_domains = config.network.denied_domains().unwrap_or_default(); + let allowed_domains = config.allowed_domains().unwrap_or_default(); + let denied_domains = config.denied_domains().unwrap_or_default(); validate_non_global_wildcard_domain_patterns("network.denied_domains", &denied_domains) .map_err(NetworkProxyConstraintError::into_anyhow)?; let deny_set = compile_denylist_globset(&denied_domains)?; let allow_set = compile_allowlist_globset(&allowed_domains)?; let mitm_hooks = compile_mitm_hooks(&config)?; - let mitm = if config.network.mitm { + let mitm = if config.mitm { Some(Arc::new(MitmState::new(MitmUpstreamConfig { - allow_upstream_proxy: config.network.allow_upstream_proxy, - allow_local_binding: config.network.allow_local_binding, + allow_upstream_proxy: config.allow_upstream_proxy, + allow_local_binding: config.allow_local_binding, })?)) } else { None @@ -122,14 +116,14 @@ pub fn validate_policy_against_constraints( validator(&candidate) } - let enabled = config.network.enabled; - let config_allowed_domains = config.network.allowed_domains().unwrap_or_default(); - let config_denied_domains = config.network.denied_domains().unwrap_or_default(); + let enabled = config.enabled; + let config_allowed_domains = config.allowed_domains().unwrap_or_default(); + let config_denied_domains = config.denied_domains().unwrap_or_default(); let denied_domain_overrides: HashSet = config_denied_domains .iter() .map(|entry| entry.to_ascii_lowercase()) .collect(); - let config_allow_unix_sockets = config.network.allow_unix_sockets(); + let config_allow_unix_sockets = config.allow_unix_sockets(); validate_mitm_hook_config(config).map_err(invalid_mitm_hook_configuration)?; validate_non_global_wildcard_domain_patterns("network.denied_domains", &config_denied_domains)?; if let Some(max_enabled) = constraints.enabled { @@ -147,7 +141,7 @@ pub fn validate_policy_against_constraints( } if let Some(max_mode) = constraints.mode { - validate(config.network.mode, move |candidate| { + validate(config.mode, move |candidate| { if network_mode_rank(*candidate) > network_mode_rank(max_mode) { Err(invalid_value( "network.mode", @@ -162,7 +156,7 @@ pub fn validate_policy_against_constraints( let allow_upstream_proxy = constraints.allow_upstream_proxy; validate( - config.network.allow_upstream_proxy, + config.allow_upstream_proxy, move |candidate| match allow_upstream_proxy { Some(true) | None => Ok(()), Some(false) => { @@ -181,7 +175,7 @@ pub fn validate_policy_against_constraints( let allow_non_loopback_proxy = constraints.dangerously_allow_non_loopback_proxy; validate( - config.network.dangerously_allow_non_loopback_proxy, + config.dangerously_allow_non_loopback_proxy, move |candidate| match allow_non_loopback_proxy { Some(true) | None => Ok(()), Some(false) => { @@ -202,7 +196,7 @@ pub fn validate_policy_against_constraints( .dangerously_allow_all_unix_sockets .unwrap_or(constraints.allow_unix_sockets.is_none()); validate( - config.network.dangerously_allow_all_unix_sockets, + config.dangerously_allow_all_unix_sockets, move |candidate| { if *candidate && !allow_all_unix_sockets { Err(invalid_value( @@ -217,7 +211,7 @@ pub fn validate_policy_against_constraints( )?; if let Some(allow_local_binding) = constraints.allow_local_binding { - validate(config.network.allow_local_binding, move |candidate| { + validate(config.allow_local_binding, move |candidate| { if *candidate && !allow_local_binding { Err(invalid_value( "network.allow_local_binding", diff --git a/codex-rs/sandboxing/src/seatbelt_tests.rs b/codex-rs/sandboxing/src/seatbelt_tests.rs index b15019a8cb..fef2da4405 100644 --- a/codex-rs/sandboxing/src/seatbelt_tests.rs +++ b/codex-rs/sandboxing/src/seatbelt_tests.rs @@ -658,11 +658,9 @@ async fn create_seatbelt_args_merges_proxy_and_explicit_unix_socket_paths() -> a let network_socket = "/tmp/codex-proxy-use"; let explicit_socket = "/tmp/codex-browser-use"; let mut network_config = NetworkProxyConfig::default(); - network_config.network.enabled = true; - network_config.network.mode = NetworkMode::Full; - network_config - .network - .set_allow_unix_sockets(vec![network_socket.to_string()]); + network_config.enabled = true; + network_config.mode = NetworkMode::Full; + network_config.set_allow_unix_sockets(vec![network_socket.to_string()]); let state = build_config_state(network_config, NetworkProxyConstraints::default())?; let network_proxy = NetworkProxy::builder() .state(Arc::new(NetworkProxyState::with_reloader( diff --git a/codex-rs/state/src/log_db.rs b/codex-rs/state/src/log_db.rs index 2cf0f5d9af..01c8ac3951 100644 --- a/codex-rs/state/src/log_db.rs +++ b/codex-rs/state/src/log_db.rs @@ -53,9 +53,11 @@ const LOG_FLUSH_INTERVAL: Duration = Duration::from_secs(2); pub fn default_filter() -> Targets { Targets::new() .with_default(LevelFilter::TRACE) + .with_target("hyper_util", LevelFilter::WARN) .with_target("log", LevelFilter::OFF) .with_target("codex_otel.log_only", LevelFilter::OFF) .with_target("codex_otel.trace_safe", LevelFilter::OFF) + .with_target("rmcp::service", LevelFilter::INFO) } #[derive(Clone, Copy, Debug, Eq, PartialEq)] diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index fa745c652e..0775daa5d5 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -622,8 +622,8 @@ pub(crate) struct ChatWidget { interrupts: InterruptManager, // Accumulates the current reasoning block text to extract a header reasoning_buffer: String, - // Accumulates full reasoning content for transcript-only recording - full_reasoning_buffer: String, + // Preserves reasoning-summary part boundaries for transcript-only recording. + reasoning_summary_parts: Vec, status_state: StatusState, review: ReviewState, // Active hook runs render in a dedicated live cell so they can run alongside tools. diff --git a/codex-rs/tui/src/chatwidget/constructor.rs b/codex-rs/tui/src/chatwidget/constructor.rs index 7fb6fe54e9..5e5286962e 100644 --- a/codex-rs/tui/src/chatwidget/constructor.rs +++ b/codex-rs/tui/src/chatwidget/constructor.rs @@ -176,7 +176,7 @@ impl ChatWidget { newly_installed_marketplace_tab_id: None, interrupts: InterruptManager::new(), reasoning_buffer: String::new(), - full_reasoning_buffer: String::new(), + reasoning_summary_parts: Vec::new(), status_state: StatusState::default(), review: ReviewState::default(), active_hook_cell: None, diff --git a/codex-rs/tui/src/chatwidget/input_flow.rs b/codex-rs/tui/src/chatwidget/input_flow.rs index 916c54cd5e..eea54352ba 100644 --- a/codex-rs/tui/src/chatwidget/input_flow.rs +++ b/codex-rs/tui/src/chatwidget/input_flow.rs @@ -37,7 +37,7 @@ impl ChatWidget { // Submitted is emitted when user submits. // Reset any reasoning header only when we are actually submitting a turn. self.reasoning_buffer.clear(); - self.full_reasoning_buffer.clear(); + self.reasoning_summary_parts.clear(); self.set_status_header(String::from("Working")); self.submit_user_message(user_message); } else { diff --git a/codex-rs/tui/src/chatwidget/replay.rs b/codex-rs/tui/src/chatwidget/replay.rs index 91cb33c30c..e639e46c7f 100644 --- a/codex-rs/tui/src/chatwidget/replay.rs +++ b/codex-rs/tui/src/chatwidget/replay.rs @@ -113,13 +113,18 @@ impl ChatWidget { summary, content, .. } => { if from_replay { - for delta in summary { - self.on_agent_reasoning_delta(delta); - } - if self.config.show_raw_agent_reasoning { - for delta in content { - self.on_agent_reasoning_delta(delta); + let reasoning_parts = summary.into_iter().chain( + self.config + .show_raw_agent_reasoning + .then_some(content) + .into_iter() + .flatten(), + ); + for (index, delta) in reasoning_parts.enumerate() { + if index > 0 { + self.on_reasoning_section_break(); } + self.on_agent_reasoning_delta(delta); } } self.on_agent_reasoning_final(); diff --git a/codex-rs/tui/src/chatwidget/slash_dispatch.rs b/codex-rs/tui/src/chatwidget/slash_dispatch.rs index df9c65f8ce..f729a52547 100644 --- a/codex-rs/tui/src/chatwidget/slash_dispatch.rs +++ b/codex-rs/tui/src/chatwidget/slash_dispatch.rs @@ -735,7 +735,7 @@ impl ChatWidget { ); if self.is_session_configured() { self.reasoning_buffer.clear(); - self.full_reasoning_buffer.clear(); + self.reasoning_summary_parts.clear(); self.set_status_header(String::from("Working")); self.submit_user_message(user_message); } else { diff --git a/codex-rs/tui/src/chatwidget/streaming.rs b/codex-rs/tui/src/chatwidget/streaming.rs index f8b0124203..e0d90dad9b 100644 --- a/codex-rs/tui/src/chatwidget/streaming.rs +++ b/codex-rs/tui/src/chatwidget/streaming.rs @@ -226,24 +226,26 @@ impl ChatWidget { pub(super) fn on_agent_reasoning_final(&mut self) { // At the end of a reasoning block, record transcript-only content. - self.full_reasoning_buffer.push_str(&self.reasoning_buffer); - if !self.full_reasoning_buffer.is_empty() { - let cell = history_cell::new_reasoning_summary_block( - self.full_reasoning_buffer.clone(), - &self.config.cwd, - ); + if !self.reasoning_buffer.is_empty() { + self.reasoning_summary_parts + .push(std::mem::take(&mut self.reasoning_buffer)); + } + if !self.reasoning_summary_parts.is_empty() { + let reasoning_parts = std::mem::take(&mut self.reasoning_summary_parts); + let cell = history_cell::new_reasoning_summary_block(reasoning_parts, &self.config.cwd); self.add_boxed_history(cell); } self.reasoning_buffer.clear(); - self.full_reasoning_buffer.clear(); + self.reasoning_summary_parts.clear(); self.request_redraw(); } pub(super) fn on_reasoning_section_break(&mut self) { // Start a new reasoning block for header extraction and accumulate transcript. - self.full_reasoning_buffer.push_str(&self.reasoning_buffer); - self.full_reasoning_buffer.push_str("\n\n"); - self.reasoning_buffer.clear(); + if !self.reasoning_buffer.is_empty() { + self.reasoning_summary_parts + .push(std::mem::take(&mut self.reasoning_buffer)); + } } pub(super) fn on_stream_error(&mut self, message: String, additional_details: Option) { diff --git a/codex-rs/tui/src/chatwidget/tests/history_replay.rs b/codex-rs/tui/src/chatwidget/tests/history_replay.rs index f8626dfcc0..8126cc89b1 100644 --- a/codex-rs/tui/src/chatwidget/tests/history_replay.rs +++ b/codex-rs/tui/src/chatwidget/tests/history_replay.rs @@ -867,7 +867,7 @@ async fn replayed_thread_closed_notification_does_not_exit_tui() { } #[tokio::test] -async fn replayed_reasoning_item_hides_raw_reasoning_when_disabled() { +async fn replayed_reasoning_item_preserves_summary_parts_and_hides_raw_reasoning_when_disabled() { let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; chat.config.show_raw_agent_reasoning = false; chat.handle_thread_session(crate::session_state::ThreadSessionState { @@ -897,7 +897,10 @@ async fn replayed_reasoning_item_hides_raw_reasoning_when_disabled() { chat.replay_thread_item( AppServerThreadItem::Reasoning { id: "reasoning-1".to_string(), - summary: vec!["Summary only".to_string()], + summary: vec![ + "**Plan**\n\ndone".to_string(), + "**Checking tests**\n\n".to_string(), + ], content: vec!["Raw reasoning".to_string()], }, "turn-1".to_string(), @@ -910,7 +913,7 @@ async fn replayed_reasoning_item_hides_raw_reasoning_when_disabled() { } other => panic!("expected InsertHistoryCell, got {other:?}"), }; - assert!(!rendered.trim().is_empty()); + assert_eq!(rendered, "• done\n"); assert!(!rendered.contains("Raw reasoning")); } @@ -1047,6 +1050,82 @@ async fn live_reasoning_summary_is_not_rendered_twice_when_item_completes() { assert_eq!(rendered.matches("Summary only").count(), 1); } +#[tokio::test] +async fn live_reasoning_summary_drops_empty_parts_without_losing_content() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.show_welcome_banner = false; + + chat.handle_server_notification( + ServerNotification::TurnStarted(TurnStartedNotification { + thread_id: "thread-1".to_string(), + turn: AppServerTurn { + id: "turn-1".to_string(), + items_view: codex_app_server_protocol::TurnItemsView::Full, + items: Vec::new(), + status: AppServerTurnStatus::InProgress, + error: None, + started_at: Some(0), + completed_at: None, + duration_ms: None, + }, + }), + /*replay_kind*/ None, + ); + let _ = drain_insert_history(&mut rx); + + for (summary_index, delta) in [ + (0, "**Plan**\n\ndone"), + (1, "**Checking tests**\n\n"), + ] { + chat.handle_server_notification( + ServerNotification::ReasoningSummaryPartAdded( + codex_app_server_protocol::ReasoningSummaryPartAddedNotification { + thread_id: "thread-1".to_string(), + turn_id: "turn-1".to_string(), + item_id: "reasoning-1".to_string(), + summary_index, + }, + ), + /*replay_kind*/ None, + ); + chat.handle_server_notification( + ServerNotification::ReasoningSummaryTextDelta(ReasoningSummaryTextDeltaNotification { + thread_id: "thread-1".to_string(), + turn_id: "turn-1".to_string(), + item_id: "reasoning-1".to_string(), + delta: delta.to_string(), + summary_index, + }), + /*replay_kind*/ None, + ); + } + + chat.handle_server_notification( + ServerNotification::ItemCompleted(ItemCompletedNotification { + thread_id: "thread-1".to_string(), + turn_id: "turn-1".to_string(), + completed_at_ms: 0, + item: AppServerThreadItem::Reasoning { + id: "reasoning-1".to_string(), + summary: vec![ + "**Plan**\n\ndone".to_string(), + "**Checking tests**\n\n".to_string(), + ], + content: Vec::new(), + }, + }), + /*replay_kind*/ None, + ); + + let rendered = match rx.try_recv() { + Ok(AppEvent::InsertHistoryCell(cell)) => { + lines_to_single_string(&cell.transcript_lines(/*width*/ 80)) + } + other => panic!("expected InsertHistoryCell, got {other:?}"), + }; + assert_eq!(rendered, "• done\n"); +} + #[tokio::test] async fn thread_snapshot_replayed_turn_started_marks_task_running() { let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; diff --git a/codex-rs/tui/src/chatwidget/turn_runtime.rs b/codex-rs/tui/src/chatwidget/turn_runtime.rs index d71a60820e..6ab68ff15f 100644 --- a/codex-rs/tui/src/chatwidget/turn_runtime.rs +++ b/codex-rs/tui/src/chatwidget/turn_runtime.rs @@ -80,7 +80,7 @@ impl ChatWidget { if self.mcp_startup_status.is_none() || !self.status_header_is_mcp_startup_owned() { self.set_status_header(String::from("Working")); } - self.full_reasoning_buffer.clear(); + self.reasoning_summary_parts.clear(); self.reasoning_buffer.clear(); self.set_ambient_pet_notification( crate::pets::PetNotificationKind::Running, diff --git a/codex-rs/tui/src/history_cell/messages.rs b/codex-rs/tui/src/history_cell/messages.rs index 350ae81db1..60cadf4341 100644 --- a/codex-rs/tui/src/history_cell/messages.rs +++ b/codex-rs/tui/src/history_cell/messages.rs @@ -495,37 +495,70 @@ pub(crate) fn new_user_prompt( /// Create the reasoning history cell emitted at the end of a reasoning block. /// /// The helper snapshots `cwd` into the returned cell so local file links render the same way they -/// did while the turn was live, even if rendering happens after other app state has advanced. +/// did while the turn was live, even if rendering happens after other app state has advanced. Part +/// boundaries are preserved so standalone empty placeholders can be removed without changing +/// literal HTML comments or bold-only summary content. pub(crate) fn new_reasoning_summary_block( - full_reasoning_buffer: String, + reasoning_parts: Vec, cwd: &Path, ) -> Box { - let cwd = cwd.to_path_buf(); - let full_reasoning_buffer = full_reasoning_buffer.trim(); - if let Some(open) = full_reasoning_buffer.find("**") { - let after_open = &full_reasoning_buffer[(open + 2)..]; - if let Some(close) = after_open.find("**") { - let after_close_idx = open + 2 + close + 2; - // if we don't have anything beyond `after_close_idx` - // then we don't have a summary to inject into history - if after_close_idx < full_reasoning_buffer.len() { - let header_buffer = full_reasoning_buffer[..after_close_idx].to_string(); - let summary_buffer = full_reasoning_buffer[after_close_idx..].to_string(); - // Preserve the session cwd so local file links render the same way in the - // collapsed reasoning block as they did while streaming live content. - return Box::new(ReasoningSummaryCell::new( - header_buffer, - summary_buffer, - &cwd, - /*transcript_only*/ false, - )); - } - } - } + let (header, content) = split_reasoning_summary_parts(&reasoning_parts); + let transcript_only = header.is_empty(); Box::new(ReasoningSummaryCell::new( - "".to_string(), - full_reasoning_buffer.to_string(), - &cwd, - /*transcript_only*/ true, + header, + content, + cwd, + transcript_only, )) } + +/// Split structured reasoning-summary parts into the status header and renderable content. +pub(crate) fn split_reasoning_summary_parts(reasoning_parts: &[String]) -> (String, String) { + let mut leading_empty_part_header = None; + let mut content_parts = Vec::with_capacity(reasoning_parts.len()); + + for part in reasoning_parts { + let part = part.trim(); + if part.is_empty() { + continue; + } + + let header_end = part.strip_prefix("**").and_then(|after_open| { + after_open + .find("**") + .and_then(|close| (close > 0).then_some(close + 4)) + }); + let body = header_end.map_or(part, |header_end| &part[header_end..]); + if body.trim() == "" { + if content_parts.is_empty() + && leading_empty_part_header.is_none() + && let Some(header_end) = header_end + { + leading_empty_part_header = Some(part[..header_end].to_string()); + } + continue; + } + + content_parts.push(part); + } + + let content = content_parts.join("\n\n"); + if content.is_empty() { + return (leading_empty_part_header.unwrap_or_default(), content); + } + + if let Some(after_open) = content.strip_prefix("**") + && let Some(close) = after_open.find("**") + { + let after_close_idx = 2 + close + 2; + let after_close = &content[after_close_idx..]; + if after_close.starts_with('\n') || after_close.starts_with('\r') { + return ( + content[..after_close_idx].to_string(), + after_close.to_string(), + ); + } + } + + (leading_empty_part_header.unwrap_or_default(), content) +} diff --git a/codex-rs/tui/src/history_cell/tests.rs b/codex-rs/tui/src/history_cell/tests.rs index b5c9739239..e6e4649de3 100644 --- a/codex-rs/tui/src/history_cell/tests.rs +++ b/codex-rs/tui/src/history_cell/tests.rs @@ -2249,7 +2249,7 @@ fn plan_update_does_not_split_url_like_tokens_in_note_or_step() { #[test] fn reasoning_summary_block() { let cell = new_reasoning_summary_block( - "**High level reasoning**\n\nDetailed reasoning goes here.".to_string(), + vec!["**High level reasoning**\n\nDetailed reasoning goes here.".to_string()], &test_cwd(), ); @@ -2307,8 +2307,10 @@ fn reasoning_summary_height_matches_wrapped_rendering_for_url_like_content() { #[test] fn reasoning_summary_block_returns_reasoning_cell_when_feature_disabled() { - let cell = - new_reasoning_summary_block("Detailed reasoning goes here.".to_string(), &test_cwd()); + let cell = new_reasoning_summary_block( + vec!["Detailed reasoning goes here.".to_string()], + &test_cwd(), + ); let rendered = render_transcript(cell.as_ref()); assert_eq!(rendered, vec!["• Detailed reasoning goes here."]); @@ -2320,7 +2322,7 @@ async fn reasoning_summary_block_respects_config_overrides() { config.model = Some("gpt-3.5-turbo".to_string()); config.model_supports_reasoning_summaries = Some(true); let cell = new_reasoning_summary_block( - "**High level reasoning**\n\nDetailed reasoning goes here.".to_string(), + vec!["**High level reasoning**\n\nDetailed reasoning goes here.".to_string()], &test_cwd(), ); @@ -2331,7 +2333,7 @@ async fn reasoning_summary_block_respects_config_overrides() { #[test] fn reasoning_summary_block_falls_back_when_header_is_missing() { let cell = new_reasoning_summary_block( - "**High level reasoning without closing".to_string(), + vec!["**High level reasoning without closing".to_string()], &test_cwd(), ); @@ -2342,7 +2344,7 @@ fn reasoning_summary_block_falls_back_when_header_is_missing() { #[test] fn reasoning_summary_block_falls_back_when_summary_is_missing() { let cell = new_reasoning_summary_block( - "**High level reasoning without closing**".to_string(), + vec!["**High level reasoning without closing**".to_string()], &test_cwd(), ); @@ -2350,7 +2352,7 @@ fn reasoning_summary_block_falls_back_when_summary_is_missing() { assert_eq!(rendered, vec!["• High level reasoning without closing"]); let cell = new_reasoning_summary_block( - "**High level reasoning without closing**\n\n ".to_string(), + vec!["**High level reasoning without closing**\n\n ".to_string()], &test_cwd(), ); @@ -2361,7 +2363,7 @@ fn reasoning_summary_block_falls_back_when_summary_is_missing() { #[test] fn reasoning_summary_block_splits_header_and_summary_when_present() { let cell = new_reasoning_summary_block( - "**High level plan**\n\nWe should fix the bug next.".to_string(), + vec!["**High level plan**\n\nWe should fix the bug next.".to_string()], &test_cwd(), ); @@ -2372,6 +2374,100 @@ fn reasoning_summary_block_splits_header_and_summary_when_present() { assert_eq!(rendered_transcript, vec!["• We should fix the bug next."]); } +#[test] +fn reasoning_summary_block_hides_empty_html_comment_parts() { + let cell = new_reasoning_summary_block( + vec![ + "**Checking the first thing**\n\n".to_string(), + "**Checking the second thing**\n\n".to_string(), + ], + &test_cwd(), + ); + + let rendered_display = render_lines(&cell.display_lines(/*width*/ 80)); + insta::assert_snapshot!(rendered_display.join("\n"), @""); + + let rendered_transcript = render_transcript(cell.as_ref()); + assert_eq!(rendered_transcript, Vec::::new()); +} + +#[test] +fn reasoning_summary_block_preserves_bold_content_after_empty_html_comment_part() { + let cell = new_reasoning_summary_block( + vec![ + "**Status**\n\n".to_string(), + "**Important conclusion**".to_string(), + "".to_string(), + ], + &test_cwd(), + ); + + let rendered_display = render_lines(&cell.display_lines(/*width*/ 80)); + insta::assert_snapshot!(rendered_display.join("\n"), @"• Important conclusion"); + + let rendered_transcript = render_transcript(cell.as_ref()); + assert_eq!(rendered_transcript, vec!["• Important conclusion"]); + + let cell = new_reasoning_summary_block( + vec![ + "**Status**\n\n".to_string(), + "**Result:** keep **this**".to_string(), + ], + &test_cwd(), + ); + + let rendered_transcript = render_transcript(cell.as_ref()); + assert_eq!(rendered_transcript, vec!["• Result: keep this"]); +} + +#[test] +fn reasoning_summary_block_strips_header_after_leading_empty_part() { + let cell = new_reasoning_summary_block( + vec![ + "**Status**\n\n".to_string(), + "**Checking tests**\n\nTests passed".to_string(), + ], + &test_cwd(), + ); + + let rendered_display = render_lines(&cell.display_lines(/*width*/ 80)); + insta::assert_snapshot!(rendered_display.join("\n"), @"• Tests passed"); + + let rendered_transcript = render_transcript(cell.as_ref()); + assert_eq!(rendered_transcript, vec!["• Tests passed"]); +} + +#[test] +fn reasoning_summary_block_drops_empty_part_after_real_content() { + let cell = new_reasoning_summary_block( + vec![ + "**Plan**\n\ndone".to_string(), + "**Checking tests**\n\n".to_string(), + ], + &test_cwd(), + ); + + let rendered_display = render_lines(&cell.display_lines(/*width*/ 80)); + insta::assert_snapshot!(rendered_display.join("\n"), @"• done"); + + let rendered_transcript = render_transcript(cell.as_ref()); + assert_eq!(rendered_transcript, vec!["• done"]); +} + +#[test] +fn reasoning_summary_block_preserves_literal_html_comment() { + let cell = new_reasoning_summary_block( + vec!["**Plan**\n\nUse `` in JSX.".to_string()], + &test_cwd(), + ); + + let rendered_display = render_lines(&cell.display_lines(/*width*/ 80)); + insta::assert_snapshot!(rendered_display.join("\n"), @"• Use in JSX."); + + let rendered_transcript = render_transcript(cell.as_ref()); + assert_eq!(rendered_transcript, vec!["• Use in JSX."]); +} + #[test] fn deprecation_notice_renders_summary_with_details() { let cell = new_deprecation_notice( diff --git a/codex-rs/tui/src/thread_transcript.rs b/codex-rs/tui/src/thread_transcript.rs index 132a9e3c72..cee7959168 100644 --- a/codex-rs/tui/src/thread_transcript.rs +++ b/codex-rs/tui/src/thread_transcript.rs @@ -9,6 +9,7 @@ use crate::history_cell::HistoryCell; use crate::history_cell::PlainHistoryCell; use crate::history_cell::ReasoningSummaryCell; use crate::history_cell::UserHistoryCell; +use crate::history_cell::split_reasoning_summary_parts; use crate::multi_agents::sub_agent_activity_summary; use codex_app_server_protocol::Thread; use codex_app_server_protocol::ThreadItem; @@ -89,19 +90,17 @@ pub(crate) fn thread_to_transcript_cells( ThreadItem::Reasoning { summary, content, .. } => { - let text = if matches!(raw_reasoning_visibility, RawReasoningVisibility::Visible) - && !content.is_empty() - { - content.join("\n\n") - } else { - summary.join("\n\n") - }; + let (header, text) = + if matches!(raw_reasoning_visibility, RawReasoningVisibility::Visible) + && !content.is_empty() + { + ("Reasoning".to_string(), content.join("\n\n")) + } else { + split_reasoning_summary_parts(summary) + }; if !text.trim().is_empty() { cells.push(Arc::new(ReasoningSummaryCell::new( - "Reasoning".to_string(), - text, - cwd, - /*transcript_only*/ false, + header, text, cwd, /*transcript_only*/ false, ))); } }