From a11f4795e3f1a54189cdd55f0b68c49a7b7978b6 Mon Sep 17 00:00:00 2001 From: starr-openai Date: Fri, 20 Mar 2026 19:21:42 -0700 Subject: [PATCH] Draft environment-backed skills and project docs Co-authored-by: Codex --- .../app-server-protocol/src/protocol/v2.rs | 35 ++ codex-rs/app-server/README.md | 8 +- .../app-server/src/codex_message_processor.rs | 45 ++- .../app-server/tests/suite/v2/skills_list.rs | 36 +- codex-rs/core/src/codex.rs | 35 +- codex-rs/core/src/project_doc.rs | 214 ++++++++++++ codex-rs/core/src/skills/injection.rs | 65 ++++ codex-rs/core/src/skills/loader.rs | 321 +++++++++++++++++- codex-rs/core/src/skills/manager.rs | 164 +++++++++ .../tools/runtimes/shell/unix_escalation.rs | 8 +- codex-rs/exec-server/src/environment.rs | 4 + 11 files changed, 911 insertions(+), 24 deletions(-) diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index c351060c72..6a6f16be7f 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -3081,6 +3081,11 @@ pub struct ThreadReadResponse { #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct SkillsListParams { + /// Optional execution environment that owns the skill snapshot. + #[serde(default)] + #[ts(optional = nullable)] + pub environment_id: Option, + /// When empty, defaults to the current session working directory. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub cwds: Vec, @@ -3107,6 +3112,7 @@ pub struct SkillsListExtraRootsForCwd { #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct SkillsListResponse { + pub environment_id: String, pub data: Vec, } @@ -7987,18 +7993,21 @@ mod tests { fn skills_list_params_serialization_uses_force_reload() { assert_eq!( serde_json::to_value(SkillsListParams { + environment_id: None, cwds: Vec::new(), force_reload: false, per_cwd_extra_user_roots: None, }) .unwrap(), json!({ + "environmentId": null, "perCwdExtraUserRoots": null, }), ); assert_eq!( serde_json::to_value(SkillsListParams { + environment_id: Some("env-1".to_string()), cwds: vec![PathBuf::from("/repo")], force_reload: true, per_cwd_extra_user_roots: Some(vec![SkillsListExtraRootsForCwd { @@ -8011,6 +8020,7 @@ mod tests { }) .unwrap(), json!({ + "environmentId": "env-1", "cwds": ["/repo"], "forceReload": true, "perCwdExtraUserRoots": [ @@ -8023,6 +8033,31 @@ mod tests { ); } + #[test] + fn skills_list_response_serialization_includes_environment_id() { + assert_eq!( + serde_json::to_value(SkillsListResponse { + environment_id: "env-1".to_string(), + data: vec![SkillsListEntry { + cwd: PathBuf::from("/repo"), + skills: Vec::new(), + errors: Vec::new(), + }], + }) + .unwrap(), + json!({ + "environmentId": "env-1", + "data": [ + { + "cwd": "/repo", + "skills": [], + "errors": [], + } + ], + }), + ); + } + #[test] fn plugin_list_params_serialization_uses_force_remote_sync() { assert_eq!( diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index bee64f0674..f4716535f1 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -163,7 +163,7 @@ Example with notification opt-out: - `model/list` — list available models (set `includeHidden: true` to include entries with `hidden: true`), with reasoning effort options, optional legacy `upgrade` model ids, optional `upgradeInfo` metadata (`model`, `upgradeCopy`, `modelLink`, `migrationMarkdown`), and optional `availabilityNux` metadata. - `experimentalFeature/list` — list feature flags with stage metadata (`beta`, `underDevelopment`, `stable`, etc.), enabled/default-enabled state, and cursor pagination. For non-beta flags, `displayName`/`description`/`announcement` are `null`. - `collaborationMode/list` — list available collaboration mode presets (experimental, no pagination). This response omits built-in developer instructions; clients should either pass `settings.developer_instructions: null` when setting a mode to use Codex's built-in instructions, or provide their own instructions explicitly. -- `skills/list` — list skills for one or more `cwd` values (optional `forceReload`). +- `skills/list` — list skills for one or more `cwd` values (optional `forceReload`) within an optional `environmentId`; the response echoes the resolved `environmentId`. - `plugin/list` — list discovered plugin marketplaces and plugin state, including effective marketplace install/auth policy metadata and best-effort `featuredPluginIds` for the official curated marketplace. `interface.category` uses the marketplace category when present; otherwise it falls back to the plugin manifest category. Pass `forceRemoteSync: true` to refresh curated plugin state before listing (**under development; do not call from production clients yet**). - `plugin/read` — read one plugin by `marketplacePath` plus `pluginName`, returning marketplace info, a list-style `summary`, manifest descriptions/interface metadata, and bundled skills/apps/MCP server names (**under development; do not call from production clients yet**). - `skills/changed` — notification emitted when watched local skill files change. @@ -1093,14 +1093,15 @@ Example: $skill-creator Add a new skill for triaging flaky CI and include step-by-step usage. ``` -Use `skills/list` to fetch the available skills (optionally scoped by `cwds`, with `forceReload`). +Use `skills/list` to fetch the available skills (optionally scoped by `cwds`, with `forceReload`, and optionally bound to an `environmentId`). You can also add `perCwdExtraUserRoots` to scan additional absolute paths as `user` scope for specific `cwd` entries. Entries whose `cwd` is not present in `cwds` are ignored. -`skills/list` might reuse a cached skills result per `cwd`; setting `forceReload` to `true` refreshes the result from disk. +`skills/list` might reuse a cached skills result per `cwd`; setting `forceReload` to `true` refreshes the result from disk. The response includes the resolved `environmentId` so clients can cache and invalidate by execution environment. The server also emits `skills/changed` notifications when watched local skill files change. Treat this as an invalidation signal and re-run `skills/list` with your current params when needed. ```json { "method": "skills/list", "id": 25, "params": { + "environmentId": "local", "cwds": ["/Users/me/project", "/Users/me/other-project"], "forceReload": true, "perCwdExtraUserRoots": [ @@ -1111,6 +1112,7 @@ The server also emits `skills/changed` notifications when watched local skill fi ] } } { "id": 25, "result": { + "environmentId": "local", "data": [{ "cwd": "/Users/me/project", "skills": [ diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 6983add6d5..9148235ee4 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -5455,10 +5455,26 @@ impl CodexMessageProcessor { async fn skills_list(&self, request_id: ConnectionRequestId, params: SkillsListParams) { let SkillsListParams { + environment_id, cwds, force_reload, per_cwd_extra_user_roots, } = params; + let configured_environment_id = Environment::default_environment_id( + self.config.experimental_exec_server_url.as_deref(), + ); + if let Some(environment_id) = environment_id + && environment_id != configured_environment_id + { + self.send_invalid_request_error( + request_id, + format!( + "unsupported environmentId `{environment_id}`; configured environment is `{configured_environment_id}`" + ), + ) + .await; + return; + } let cwds = if cwds.is_empty() { vec![self.config.cwd.clone()] } else { @@ -5504,6 +5520,19 @@ impl CodexMessageProcessor { return; } }; + let environment = + match Environment::create(self.config.experimental_exec_server_url.clone()).await { + Ok(environment) => environment, + Err(error) => { + self.send_invalid_request_error( + request_id, + format!("failed to bind environment for skills/list: {error}"), + ) + .await; + return; + } + }; + let environment_filesystem = environment.get_filesystem(); let skills_manager = self.thread_manager.skills_manager(); let mut data = Vec::new(); for cwd in cwds { @@ -5511,7 +5540,13 @@ impl CodexMessageProcessor { .get(&cwd) .map_or(&[][..], std::vec::Vec::as_slice); let outcome = skills_manager - .skills_for_cwd_with_extra_user_roots(&cwd, &config, force_reload, extra_roots) + .skills_for_cwd_with_extra_user_roots_with_filesystem( + &cwd, + &config, + force_reload, + extra_roots, + &environment_filesystem, + ) .await; let errors = errors_to_info(&outcome.errors); let skills = skills_to_info(&outcome.skills, &outcome.disabled_paths); @@ -5522,7 +5557,13 @@ impl CodexMessageProcessor { }); } self.outgoing - .send_response(request_id, SkillsListResponse { data }) + .send_response( + request_id, + SkillsListResponse { + environment_id: configured_environment_id, + data, + }, + ) .await; } diff --git a/codex-rs/app-server/tests/suite/v2/skills_list.rs b/codex-rs/app-server/tests/suite/v2/skills_list.rs index 2bf0a17964..be94410e1d 100644 --- a/codex-rs/app-server/tests/suite/v2/skills_list.rs +++ b/codex-rs/app-server/tests/suite/v2/skills_list.rs @@ -38,6 +38,7 @@ async fn skills_list_includes_skills_from_per_cwd_extra_user_roots() -> Result<( let request_id = mcp .send_skills_list_request(SkillsListParams { + environment_id: None, cwds: vec![cwd.path().to_path_buf()], force_reload: true, per_cwd_extra_user_roots: Some(vec![SkillsListExtraRootsForCwd { @@ -52,7 +53,11 @@ async fn skills_list_includes_skills_from_per_cwd_extra_user_roots() -> Result<( mcp.read_stream_until_response_message(RequestId::Integer(request_id)), ) .await??; - let SkillsListResponse { data } = to_response(response)?; + let SkillsListResponse { + environment_id, + data, + } = to_response(response)?; + assert_eq!(environment_id, "local"); assert_eq!(data.len(), 1); assert_eq!(data[0].cwd, cwd.path().to_path_buf()); assert!( @@ -74,6 +79,7 @@ async fn skills_list_rejects_relative_extra_user_roots() -> Result<()> { let request_id = mcp .send_skills_list_request(SkillsListParams { + environment_id: None, cwds: vec![cwd.path().to_path_buf()], force_reload: true, per_cwd_extra_user_roots: Some(vec![SkillsListExtraRootsForCwd { @@ -111,6 +117,7 @@ async fn skills_list_ignores_per_cwd_extra_roots_for_unknown_cwd() -> Result<()> let request_id = mcp .send_skills_list_request(SkillsListParams { + environment_id: None, cwds: vec![requested_cwd.path().to_path_buf()], force_reload: true, per_cwd_extra_user_roots: Some(vec![SkillsListExtraRootsForCwd { @@ -125,7 +132,11 @@ async fn skills_list_ignores_per_cwd_extra_roots_for_unknown_cwd() -> Result<()> mcp.read_stream_until_response_message(RequestId::Integer(request_id)), ) .await??; - let SkillsListResponse { data } = to_response(response)?; + let SkillsListResponse { + environment_id, + data, + } = to_response(response)?; + assert_eq!(environment_id, "local"); assert_eq!(data.len(), 1); assert_eq!(data[0].cwd, requested_cwd.path().to_path_buf()); assert!( @@ -150,6 +161,7 @@ async fn skills_list_uses_cached_result_until_force_reload() -> Result<()> { // Seed the cwd cache first without extra roots. let first_request_id = mcp .send_skills_list_request(SkillsListParams { + environment_id: None, cwds: vec![cwd.path().to_path_buf()], force_reload: false, per_cwd_extra_user_roots: None, @@ -160,7 +172,11 @@ async fn skills_list_uses_cached_result_until_force_reload() -> Result<()> { mcp.read_stream_until_response_message(RequestId::Integer(first_request_id)), ) .await??; - let SkillsListResponse { data: first_data } = to_response(first_response)?; + let SkillsListResponse { + environment_id: first_environment_id, + data: first_data, + } = to_response(first_response)?; + assert_eq!(first_environment_id, "local"); assert_eq!(first_data.len(), 1); assert!( first_data[0] @@ -171,6 +187,7 @@ async fn skills_list_uses_cached_result_until_force_reload() -> Result<()> { let second_request_id = mcp .send_skills_list_request(SkillsListParams { + environment_id: None, cwds: vec![cwd.path().to_path_buf()], force_reload: false, per_cwd_extra_user_roots: Some(vec![SkillsListExtraRootsForCwd { @@ -184,7 +201,11 @@ async fn skills_list_uses_cached_result_until_force_reload() -> Result<()> { mcp.read_stream_until_response_message(RequestId::Integer(second_request_id)), ) .await??; - let SkillsListResponse { data: second_data } = to_response(second_response)?; + let SkillsListResponse { + environment_id: second_environment_id, + data: second_data, + } = to_response(second_response)?; + assert_eq!(second_environment_id, "local"); assert_eq!(second_data.len(), 1); assert!( second_data[0] @@ -195,6 +216,7 @@ async fn skills_list_uses_cached_result_until_force_reload() -> Result<()> { let third_request_id = mcp .send_skills_list_request(SkillsListParams { + environment_id: None, cwds: vec![cwd.path().to_path_buf()], force_reload: true, per_cwd_extra_user_roots: Some(vec![SkillsListExtraRootsForCwd { @@ -208,7 +230,11 @@ async fn skills_list_uses_cached_result_until_force_reload() -> Result<()> { mcp.read_stream_until_response_message(RequestId::Integer(third_request_id)), ) .await??; - let SkillsListResponse { data: third_data } = to_response(third_response)?; + let SkillsListResponse { + environment_id: third_environment_id, + data: third_data, + } = to_response(third_response)?; + assert_eq!(third_environment_id, "local"); assert_eq!(third_data.len(), 1); assert!( third_data[0] diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 9d2ddd4c4d..9e48281778 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -231,7 +231,7 @@ use crate::network_policy_decision::execpolicy_network_rule_amendment; use crate::plugins::PluginsManager; use crate::plugins::build_plugin_injections; use crate::plugins::render_plugins_section; -use crate::project_doc::get_user_instructions; +use crate::project_doc::get_user_instructions_with_filesystem; use crate::protocol::AgentMessageContentDeltaEvent; use crate::protocol::AgentReasoningSectionBreakEvent; use crate::protocol::ApplyPatchApprovalRequestEvent; @@ -282,7 +282,7 @@ use crate::skills::SkillInjections; use crate::skills::SkillLoadOutcome; use crate::skills::SkillMetadata; use crate::skills::SkillsManager; -use crate::skills::build_skill_injections; +use crate::skills::build_skill_injections_with_filesystem; use crate::skills::collect_env_var_dependencies; use crate::skills::collect_explicit_skill_mentions; use crate::skills::injection::ToolMentionKind; @@ -437,7 +437,13 @@ impl Codex { let (tx_sub, rx_sub) = async_channel::bounded(SUBMISSION_CHANNEL_CAPACITY); let (tx_event, rx_event) = async_channel::unbounded(); - let loaded_skills = skills_manager.skills_for_config(&config); + let environment = + Arc::new(Environment::create(config.experimental_exec_server_url.clone()).await?); + let environment_filesystem = environment.get_filesystem(); + + let loaded_skills = skills_manager + .skills_for_config_with_filesystem(&config, &environment_filesystem) + .await; for err in &loaded_skills.errors { error!( @@ -482,7 +488,8 @@ impl Codex { config.startup_warnings.push(message); } - let user_instructions = get_user_instructions(&config).await; + let user_instructions = + get_user_instructions_with_filesystem(&config, &environment_filesystem).await; let exec_policy = if crate::guardian::is_guardian_reviewer_source(&session_source) { // Guardian review should rely on the built-in shell safety checks, @@ -1833,9 +1840,7 @@ impl Session { code_mode_service: crate::tools::code_mode::CodeModeService::new( config.js_repl_node_path.clone(), ), - environment: Arc::new( - Environment::create(config.experimental_exec_server_url.clone()).await?, - ), + environment: Arc::clone(&environment), }; let js_repl = Arc::new(JsReplHandle::with_node_path( config.js_repl_node_path.clone(), @@ -2395,10 +2400,12 @@ impl Session { &per_turn_config, ) .await; + let environment_filesystem = self.services.environment.get_filesystem(); let skills_outcome = Arc::new( self.services .skills_manager - .skills_for_config(&per_turn_config), + .skills_for_config_with_filesystem(&per_turn_config, &environment_filesystem) + .await, ); let mut turn_context: TurnContext = Self::make_turn_context( Some(Arc::clone(&self.services.auth_manager)), @@ -4792,10 +4799,16 @@ mod handlers { let skills_manager = &sess.services.skills_manager; let config = sess.get_config().await; + let environment_filesystem = sess.services.environment.get_filesystem(); let mut skills = Vec::new(); for cwd in cwds { let outcome = skills_manager - .skills_for_cwd(&cwd, config.as_ref(), force_reload) + .skills_for_cwd_with_filesystem( + &cwd, + config.as_ref(), + force_reload, + &environment_filesystem, + ) .await; let errors = super::errors_to_info(&outcome.errors); let skills_metadata = super::skills_to_info(&outcome.skills, &outcome.disabled_paths); @@ -5471,11 +5484,13 @@ pub(crate) async fn run_turn( thread_id, turn_context.sub_id.clone(), ); + let environment_filesystem = sess.services.environment.get_filesystem(); let SkillInjections { items: skill_items, warnings: skill_warnings, - } = build_skill_injections( + } = build_skill_injections_with_filesystem( &mentioned_skills, + &environment_filesystem, Some(&session_telemetry), &sess.services.analytics_events_client, tracking.clone(), diff --git a/codex-rs/core/src/project_doc.rs b/codex-rs/core/src/project_doc.rs index aa6c3b3e73..ca5aab3f1f 100644 --- a/codex-rs/core/src/project_doc.rs +++ b/codex-rs/core/src/project_doc.rs @@ -22,6 +22,8 @@ use crate::config_loader::merge_toml_values; use crate::config_loader::project_root_markers_from_config; use crate::features::Feature; use codex_app_server_protocol::ConfigLayerSource; +use codex_exec_server::ExecutorFileSystem; +use codex_utils_absolute_path::AbsolutePathBuf; use dunce::canonicalize as normalize_path; use std::path::PathBuf; use tokio::io::AsyncReadExt; @@ -119,6 +121,56 @@ pub(crate) async fn get_user_instructions(config: &Config) -> Option { } } +/// Combines `Config::instructions` and `AGENTS.md` using an explicit executor filesystem. +pub(crate) async fn get_user_instructions_with_filesystem( + config: &Config, + filesystem: &F, +) -> Option +where + F: ExecutorFileSystem + ?Sized, +{ + let project_docs = read_project_docs_with_filesystem(config, filesystem).await; + + let mut output = String::new(); + + if let Some(instructions) = config.user_instructions.clone() { + output.push_str(&instructions); + } + + match project_docs { + Ok(Some(docs)) => { + if !output.is_empty() { + output.push_str(PROJECT_DOC_SEPARATOR); + } + output.push_str(&docs); + } + Ok(None) => {} + Err(e) => { + error!("error trying to find project doc: {e:#}"); + } + }; + + if let Some(js_repl_section) = render_js_repl_instructions(config) { + if !output.is_empty() { + output.push_str("\n\n"); + } + output.push_str(&js_repl_section); + } + + if config.features.enabled(Feature::ChildAgentsMd) { + if !output.is_empty() { + output.push_str("\n\n"); + } + output.push_str(HIERARCHICAL_AGENTS_MESSAGE); + } + + if !output.is_empty() { + Some(output) + } else { + None + } +} + /// Attempt to locate and load the project documentation. /// /// On success returns `Ok(Some(contents))` where `contents` is the @@ -178,6 +230,69 @@ pub async fn read_project_docs(config: &Config) -> std::io::Result( + config: &Config, + filesystem: &F, +) -> std::io::Result> +where + F: ExecutorFileSystem + ?Sized, +{ + let max_total = config.project_doc_max_bytes; + + if max_total == 0 { + return Ok(None); + } + + let paths = discover_project_doc_paths_with_filesystem(config, filesystem).await?; + if paths.is_empty() { + return Ok(None); + } + + let mut remaining: u64 = max_total as u64; + let mut parts: Vec = Vec::new(); + + for p in paths { + if remaining == 0 { + break; + } + + let file_path = AbsolutePathBuf::from_absolute_path(&p).map_err(|err| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("invalid project doc path `{}`: {err}", p.display()), + ) + })?; + let data = match filesystem.read_file(&file_path).await { + Ok(bytes) => bytes, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue, + Err(e) => return Err(e), + }; + + let limit = usize::try_from(remaining).unwrap_or(usize::MAX); + let slice_len = data.len().min(limit); + if data.len() as u64 > remaining { + tracing::warn!( + "Project doc `{}` exceeds remaining budget ({} bytes) - truncating.", + p.display(), + remaining, + ); + } + + let text = String::from_utf8_lossy(&data[..slice_len]).to_string(); + if !text.trim().is_empty() { + parts.push(text); + remaining = remaining.saturating_sub(slice_len as u64); + } + } + + if parts.is_empty() { + Ok(None) + } else { + Ok(Some(parts.join("\n\n"))) + } +} + /// Discover the list of AGENTS.md files using the same search rules as /// `read_project_docs`, but return the file paths instead of concatenated /// contents. The list is ordered from project root to the current working @@ -270,6 +385,105 @@ pub fn discover_project_doc_paths(config: &Config) -> std::io::Result( + config: &Config, + filesystem: &F, +) -> std::io::Result> +where + F: ExecutorFileSystem + ?Sized, +{ + let dir = config.cwd.clone(); + let mut merged = TomlValue::Table(toml::map::Map::new()); + for layer in config.config_layer_stack.get_layers( + ConfigLayerStackOrdering::LowestPrecedenceFirst, + /*include_disabled*/ false, + ) { + if matches!(layer.name, ConfigLayerSource::Project { .. }) { + continue; + } + merge_toml_values(&mut merged, &layer.config); + } + let project_root_markers = match project_root_markers_from_config(&merged) { + Ok(Some(markers)) => markers, + Ok(None) => default_project_root_markers(), + Err(err) => { + tracing::warn!("invalid project_root_markers: {err}"); + default_project_root_markers() + } + }; + let mut project_root = None; + if !project_root_markers.is_empty() { + for ancestor in dir.ancestors() { + for marker in &project_root_markers { + let marker_path = ancestor.join(marker); + let marker_path_abs = + AbsolutePathBuf::from_absolute_path(&marker_path).map_err(|err| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!( + "invalid project marker path `{}`: {err}", + marker_path.display() + ), + ) + })?; + if filesystem.get_metadata(&marker_path_abs).await.is_ok() { + project_root = Some(ancestor.to_path_buf()); + break; + } + } + if project_root.is_some() { + break; + } + } + } + + let search_dirs: Vec = if let Some(root) = project_root { + let mut dirs = Vec::new(); + let mut cursor = dir.as_path(); + loop { + dirs.push(cursor.to_path_buf()); + if cursor == root { + break; + } + let Some(parent) = cursor.parent() else { + break; + }; + cursor = parent; + } + dirs.reverse(); + dirs + } else { + vec![dir] + }; + + let mut found: Vec = Vec::new(); + let candidate_filenames = candidate_filenames(config); + for d in search_dirs { + for name in &candidate_filenames { + let candidate = d.join(name); + let candidate = AbsolutePathBuf::from_absolute_path(&candidate).map_err(|err| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("invalid project doc path `{}`: {err}", candidate.display()), + ) + })?; + match filesystem.get_metadata(&candidate).await { + Ok(metadata) => { + if metadata.is_file { + found.push(candidate.as_path().to_path_buf()); + break; + } + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue, + Err(e) => return Err(e), + } + } + } + + Ok(found) +} + fn candidate_filenames<'a>(config: &'a Config) -> Vec<&'a str> { let mut names: Vec<&'a str> = Vec::with_capacity(2 + config.project_doc_fallback_filenames.len()); diff --git a/codex-rs/core/src/skills/injection.rs b/codex-rs/core/src/skills/injection.rs index b83be2322c..696ac05720 100644 --- a/codex-rs/core/src/skills/injection.rs +++ b/codex-rs/core/src/skills/injection.rs @@ -10,9 +10,11 @@ use crate::instructions::SkillInstructions; use crate::mention_syntax::TOOL_MENTION_SIGIL; use crate::mentions::build_skill_name_counts; use crate::skills::SkillMetadata; +use codex_exec_server::ExecutorFileSystem; use codex_otel::SessionTelemetry; use codex_protocol::models::ResponseItem; use codex_protocol::user_input::UserInput; +use codex_utils_absolute_path::AbsolutePathBuf; use tokio::fs; #[derive(Debug, Default)] @@ -70,6 +72,69 @@ pub(crate) async fn build_skill_injections( result } +pub(crate) async fn build_skill_injections_with_filesystem( + mentioned_skills: &[SkillMetadata], + filesystem: &F, + otel: Option<&SessionTelemetry>, + analytics_client: &AnalyticsEventsClient, + tracking: TrackEventsContext, +) -> SkillInjections +where + F: ExecutorFileSystem + ?Sized, +{ + if mentioned_skills.is_empty() { + return SkillInjections::default(); + } + + let mut result = SkillInjections { + items: Vec::with_capacity(mentioned_skills.len()), + warnings: Vec::new(), + }; + let mut invocations = Vec::new(); + + for skill in mentioned_skills { + let Ok(skill_path) = AbsolutePathBuf::from_absolute_path(&skill.path_to_skills_md) else { + emit_skill_injected_metric(otel, skill, "error"); + result.warnings.push(format!( + "Failed to load skill {} at {}: invalid path", + skill.name, + skill.path_to_skills_md.display() + )); + continue; + }; + + match filesystem.read_file(&skill_path).await { + Ok(contents) => { + emit_skill_injected_metric(otel, skill, "ok"); + invocations.push(SkillInvocation { + skill_name: skill.name.clone(), + skill_scope: skill.scope, + skill_path: skill.path_to_skills_md.clone(), + invocation_type: InvocationType::Explicit, + }); + result.items.push(ResponseItem::from(SkillInstructions { + name: skill.name.clone(), + path: skill.path_to_skills_md.to_string_lossy().into_owned(), + contents: String::from_utf8_lossy(&contents).to_string(), + })); + } + Err(err) => { + emit_skill_injected_metric(otel, skill, "error"); + let message = format!( + "Failed to load skill {name} at {path}: {err:#}", + name = skill.name, + path = skill.path_to_skills_md.display() + ); + result.warnings.push(message); + } + } + } + + analytics_client.track_skill_invocations(tracking, invocations); + + result +} + fn emit_skill_injected_metric( otel: Option<&SessionTelemetry>, skill: &SkillMetadata, diff --git a/codex-rs/core/src/skills/loader.rs b/codex-rs/core/src/skills/loader.rs index 5672bdb0a3..38781fa67c 100644 --- a/codex-rs/core/src/skills/loader.rs +++ b/codex-rs/core/src/skills/loader.rs @@ -14,12 +14,14 @@ use crate::skills::model::SkillPolicy; use crate::skills::model::SkillToolDependency; use crate::skills::system::system_cache_root_dir; use codex_app_server_protocol::ConfigLayerSource; +use codex_exec_server::ExecutorFileSystem; use codex_protocol::models::FileSystemPermissions; use codex_protocol::models::MacOsSeatbeltProfileExtensions; use codex_protocol::models::NetworkPermissions; use codex_protocol::models::PermissionProfile; use codex_protocol::protocol::Product; use codex_protocol::protocol::SkillScope; +use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_absolute_path::AbsolutePathBufGuard; use dirs::home_dir; use dunce::canonicalize as canonicalize_path; @@ -215,6 +217,49 @@ where outcome } +pub(crate) async fn load_skills_from_roots_with_filesystem( + roots: I, + filesystem: &F, +) -> SkillLoadOutcome +where + I: IntoIterator, + F: ExecutorFileSystem + ?Sized, +{ + let mut outcome = SkillLoadOutcome::default(); + for root in roots { + discover_skills_under_root_with_filesystem( + &root.path, + root.scope, + &mut outcome, + filesystem, + ) + .await; + } + + let mut seen: HashSet = HashSet::new(); + outcome + .skills + .retain(|skill| seen.insert(skill.path_to_skills_md.clone())); + + fn scope_rank(scope: SkillScope) -> u8 { + match scope { + SkillScope::Repo => 0, + SkillScope::User => 1, + SkillScope::System => 2, + SkillScope::Admin => 3, + } + } + + outcome.skills.sort_by(|a, b| { + scope_rank(a.scope) + .cmp(&scope_rank(b.scope)) + .then_with(|| a.name.cmp(&b.name)) + .then_with(|| a.path_to_skills_md.cmp(&b.path_to_skills_md)) + }); + + outcome +} + pub(crate) fn skill_roots( config_layer_stack: &ConfigLayerStack, cwd: &Path, @@ -228,6 +273,26 @@ pub(crate) fn skill_roots( ) } +pub(crate) async fn skill_roots_with_filesystem( + config_layer_stack: &ConfigLayerStack, + cwd: &Path, + plugin_skill_roots: Vec, + filesystem: &F, +) -> Vec +where + F: ExecutorFileSystem + ?Sized, +{ + let mut roots = skill_roots_from_layer_stack_inner(config_layer_stack, home_dir().as_deref()); + roots.extend(plugin_skill_roots.into_iter().map(|path| SkillRoot { + path, + scope: SkillScope::User, + })); + roots + .extend(repo_agents_skill_roots_with_filesystem(config_layer_stack, cwd, filesystem).await); + dedupe_skill_roots_by_path(&mut roots); + roots +} + fn skill_roots_with_home_dir( config_layer_stack: &ConfigLayerStack, cwd: &Path, @@ -323,6 +388,31 @@ fn repo_agents_skill_roots(config_layer_stack: &ConfigLayerStack, cwd: &Path) -> roots } +async fn repo_agents_skill_roots_with_filesystem( + config_layer_stack: &ConfigLayerStack, + cwd: &Path, + filesystem: &F, +) -> Vec +where + F: ExecutorFileSystem + ?Sized, +{ + let project_root_markers = project_root_markers_from_stack(config_layer_stack); + let project_root = + find_project_root_with_filesystem(cwd, &project_root_markers, filesystem).await; + let dirs = dirs_between_project_root_and_cwd(cwd, &project_root); + let mut roots = Vec::new(); + for dir in dirs { + let agents_skills = dir.join(AGENTS_DIR_NAME).join(SKILLS_DIR_NAME); + if path_is_directory_with_filesystem(&agents_skills, filesystem).await { + roots.push(SkillRoot { + path: agents_skills, + scope: SkillScope::Repo, + }); + } + } + roots +} + fn project_root_markers_from_stack(config_layer_stack: &ConfigLayerStack) -> Vec { let mut merged = TomlValue::Table(toml::map::Map::new()); for layer in config_layer_stack.get_layers( @@ -362,6 +452,30 @@ fn find_project_root(cwd: &Path, project_root_markers: &[String]) -> PathBuf { cwd.to_path_buf() } +async fn find_project_root_with_filesystem( + cwd: &Path, + project_root_markers: &[String], + filesystem: &F, +) -> PathBuf +where + F: ExecutorFileSystem + ?Sized, +{ + if project_root_markers.is_empty() { + return cwd.to_path_buf(); + } + + for ancestor in cwd.ancestors() { + for marker in project_root_markers { + let marker_path = ancestor.join(marker); + if path_exists_with_filesystem(&marker_path, filesystem).await { + return ancestor.to_path_buf(); + } + } + } + + cwd.to_path_buf() +} + fn dirs_between_project_root_and_cwd(cwd: &Path, project_root: &Path) -> Vec { let mut dirs = cwd .ancestors() @@ -385,6 +499,30 @@ fn dedupe_skill_roots_by_path(roots: &mut Vec) { roots.retain(|root| seen.insert(root.path.clone())); } +async fn path_exists_with_filesystem(path: &Path, filesystem: &F) -> bool +where + F: ExecutorFileSystem + ?Sized, +{ + let Ok(path) = AbsolutePathBuf::from_absolute_path(path) else { + return false; + }; + filesystem.get_metadata(&path).await.is_ok() +} + +async fn path_is_directory_with_filesystem(path: &Path, filesystem: &F) -> bool +where + F: ExecutorFileSystem + ?Sized, +{ + let Ok(path) = AbsolutePathBuf::from_absolute_path(path) else { + return false; + }; + filesystem + .get_metadata(&path) + .await + .map(|metadata| metadata.is_directory) + .unwrap_or(false) +} + fn discover_skills_under_root(root: &Path, scope: SkillScope, outcome: &mut SkillLoadOutcome) { let Ok(root) = canonicalize_path(root) else { return; @@ -524,10 +662,142 @@ fn discover_skills_under_root(root: &Path, scope: SkillScope, outcome: &mut Skil } } +async fn discover_skills_under_root_with_filesystem( + root: &Path, + scope: SkillScope, + outcome: &mut SkillLoadOutcome, + filesystem: &F, +) where + F: ExecutorFileSystem + ?Sized, +{ + let Ok(root) = AbsolutePathBuf::from_absolute_path(root) else { + return; + }; + + let metadata = match filesystem.get_metadata(&root).await { + Ok(metadata) => metadata, + Err(_) => return, + }; + if !metadata.is_directory { + return; + } + + fn enqueue_dir( + queue: &mut VecDeque<(PathBuf, usize)>, + visited_dirs: &mut HashSet, + truncated_by_dir_limit: &mut bool, + path: PathBuf, + depth: usize, + ) { + if depth > MAX_SCAN_DEPTH { + return; + } + if visited_dirs.len() >= MAX_SKILLS_DIRS_PER_ROOT { + *truncated_by_dir_limit = true; + return; + } + if visited_dirs.insert(path.clone()) { + queue.push_back((path, depth)); + } + } + + let mut visited_dirs: HashSet = HashSet::new(); + visited_dirs.insert(root.as_path().to_path_buf()); + + let mut queue: VecDeque<(PathBuf, usize)> = VecDeque::from([(root.as_path().to_path_buf(), 0)]); + let mut truncated_by_dir_limit = false; + + while let Some((dir, depth)) = queue.pop_front() { + let dir_abs = match AbsolutePathBuf::from_absolute_path(&dir) { + Ok(dir_abs) => dir_abs, + Err(_) => continue, + }; + let entries = match filesystem.read_directory(&dir_abs).await { + Ok(entries) => entries, + Err(e) => { + error!("failed to read skills dir {}: {e:#}", dir.display()); + continue; + } + }; + + for entry in entries { + let path = dir.join(&entry.file_name); + let file_name = entry.file_name.as_str(); + + if file_name.starts_with('.') { + continue; + } + + if entry.is_directory { + enqueue_dir( + &mut queue, + &mut visited_dirs, + &mut truncated_by_dir_limit, + path, + depth + 1, + ); + continue; + } + + if entry.is_file && file_name == SKILLS_FILENAME { + match parse_skill_file_with_filesystem(&path, scope, filesystem).await { + Ok(skill) => { + outcome.skills.push(skill); + } + Err(err) => { + if scope != SkillScope::System { + outcome.errors.push(SkillError { + path, + message: err.to_string(), + }); + } + } + } + } + } + } + + if truncated_by_dir_limit { + tracing::warn!( + "skills scan truncated after {} directories (root: {})", + MAX_SKILLS_DIRS_PER_ROOT, + root.display() + ); + } +} + fn parse_skill_file(path: &Path, scope: SkillScope) -> Result { let contents = fs::read_to_string(path).map_err(SkillParseError::Read)?; + let loaded_metadata = load_skill_metadata(path); + parse_skill_file_from_contents(path, scope, &contents, loaded_metadata) +} - let frontmatter = extract_frontmatter(&contents).ok_or(SkillParseError::MissingFrontmatter)?; +async fn parse_skill_file_with_filesystem( + path: &Path, + scope: SkillScope, + filesystem: &F, +) -> Result +where + F: ExecutorFileSystem + ?Sized, +{ + let skill_path = AbsolutePathBuf::from_absolute_path(path) + .map_err(|err| SkillParseError::Read(std::io::Error::other(err.to_string())))?; + let contents = filesystem + .read_file(&skill_path) + .await + .map_err(SkillParseError::Read)?; + let contents = String::from_utf8_lossy(&contents).to_string(); + let loaded_metadata = load_skill_metadata_with_filesystem(path, filesystem).await; + parse_skill_file_from_contents(path, scope, &contents, loaded_metadata) +} + +fn parse_skill_file_from_contents( + path: &Path, + scope: SkillScope, + contents: &str, + loaded_metadata: LoadedSkillMetadata, +) -> Result { + let frontmatter = extract_frontmatter(contents).ok_or(SkillParseError::MissingFrontmatter)?; let parsed: SkillFrontmatter = serde_yaml::from_str(&frontmatter).map_err(SkillParseError::InvalidYaml)?; @@ -556,7 +826,7 @@ fn parse_skill_file(path: &Path, scope: SkillScope) -> Result LoadedSkillMetadata { } }; + load_skill_metadata_from_contents(skill_dir, &metadata_path, &contents) +} + +async fn load_skill_metadata_with_filesystem( + skill_path: &Path, + filesystem: &F, +) -> LoadedSkillMetadata +where + F: ExecutorFileSystem + ?Sized, +{ + // Fail open: optional metadata should not block loading SKILL.md. + let Some(skill_dir) = skill_path.parent() else { + return LoadedSkillMetadata::default(); + }; + let metadata_path = skill_dir + .join(SKILLS_METADATA_DIR) + .join(SKILLS_METADATA_FILENAME); + let Ok(metadata_path) = AbsolutePathBuf::from_absolute_path(&metadata_path) else { + return LoadedSkillMetadata::default(); + }; + + let contents = match filesystem.read_file(&metadata_path).await { + Ok(contents) => contents, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return LoadedSkillMetadata::default(); + } + Err(error) => { + tracing::warn!( + "ignoring {path}: failed to read {label}: {error}", + path = metadata_path.display(), + label = SKILLS_METADATA_FILENAME + ); + return LoadedSkillMetadata::default(); + } + }; + + let contents = String::from_utf8_lossy(&contents).to_string(); + load_skill_metadata_from_contents(skill_dir, metadata_path.as_path(), &contents) +} + +fn load_skill_metadata_from_contents( + skill_dir: &Path, + metadata_path: &Path, + contents: &str, +) -> LoadedSkillMetadata { let parsed: SkillMetadataFile = { let _guard = AbsolutePathBufGuard::new(skill_dir); - match serde_yaml::from_str(&contents) { + match serde_yaml::from_str(contents) { Ok(parsed) => parsed, Err(error) => { tracing::warn!( diff --git a/codex-rs/core/src/skills/manager.rs b/codex-rs/core/src/skills/manager.rs index 982780f821..1ab7eecac6 100644 --- a/codex-rs/core/src/skills/manager.rs +++ b/codex-rs/core/src/skills/manager.rs @@ -24,9 +24,12 @@ use crate::skills::SkillLoadOutcome; use crate::skills::build_implicit_skill_path_indexes; use crate::skills::loader::SkillRoot; use crate::skills::loader::load_skills_from_roots; +use crate::skills::loader::load_skills_from_roots_with_filesystem; use crate::skills::loader::skill_roots; +use crate::skills::loader::skill_roots_with_filesystem; use crate::skills::system::install_system_skills; use crate::skills::system::uninstall_system_skills; +use codex_exec_server::ExecutorFileSystem; pub struct SkillsManager { codex_home: PathBuf, @@ -98,6 +101,59 @@ impl SkillsManager { outcome } + pub async fn skills_for_config_with_filesystem( + &self, + config: &Config, + filesystem: &F, + ) -> SkillLoadOutcome + where + F: ExecutorFileSystem + ?Sized, + { + let roots = self + .skill_roots_for_config_with_filesystem(config, filesystem) + .await; + let cache_key = config_skills_cache_key(&roots, &config.config_layer_stack); + if let Some(outcome) = self.cached_outcome_for_config(&cache_key) { + return outcome; + } + + let outcome = crate::skills::filter_skill_load_outcome_for_product( + finalize_skill_outcome( + load_skills_from_roots_with_filesystem(roots, filesystem).await, + &config.config_layer_stack, + ), + self.restriction_product, + ); + let mut cache = self + .cache_by_config + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + cache.insert(cache_key, outcome.clone()); + outcome + } + + pub(crate) async fn skill_roots_for_config_with_filesystem( + &self, + config: &Config, + filesystem: &F, + ) -> Vec + where + F: ExecutorFileSystem + ?Sized, + { + let loaded_plugins = self.plugins_manager.plugins_for_config(config); + let mut roots = skill_roots_with_filesystem( + &config.config_layer_stack, + &config.cwd, + loaded_plugins.effective_skill_roots(), + filesystem, + ) + .await; + if !config.bundled_skills_enabled() { + roots.retain(|root| root.scope != SkillScope::System); + } + roots + } + pub(crate) fn skill_roots_for_config(&self, config: &Config) -> Vec { let loaded_plugins = self.plugins_manager.plugins_for_config(config); let mut roots = skill_roots( @@ -125,6 +181,26 @@ impl SkillsManager { .await } + pub async fn skills_for_cwd_with_filesystem( + &self, + cwd: &Path, + config: &Config, + force_reload: bool, + filesystem: &F, + ) -> SkillLoadOutcome + where + F: ExecutorFileSystem + ?Sized, + { + self.skills_for_cwd_with_extra_user_roots_with_filesystem( + cwd, + config, + force_reload, + &[], + filesystem, + ) + .await + } + pub async fn skills_for_cwd_with_extra_user_roots( &self, cwd: &Path, @@ -201,6 +277,94 @@ impl SkillsManager { outcome } + pub async fn skills_for_cwd_with_extra_user_roots_with_filesystem( + &self, + cwd: &Path, + config: &Config, + force_reload: bool, + extra_user_roots: &[PathBuf], + filesystem: &F, + ) -> SkillLoadOutcome + where + F: ExecutorFileSystem + ?Sized, + { + if !force_reload && let Some(outcome) = self.cached_outcome_for_cwd(cwd) { + return outcome; + } + let normalized_extra_user_roots = normalize_extra_user_roots(extra_user_roots); + + let cwd_abs = match AbsolutePathBuf::try_from(cwd) { + Ok(cwd_abs) => cwd_abs, + Err(err) => { + return SkillLoadOutcome { + errors: vec![crate::skills::model::SkillError { + path: cwd.to_path_buf(), + message: err.to_string(), + }], + ..Default::default() + }; + } + }; + + let cli_overrides: Vec<(String, TomlValue)> = Vec::new(); + let config_layer_stack = match load_config_layers_state( + &self.codex_home, + Some(cwd_abs), + &cli_overrides, + LoaderOverrides::default(), + CloudRequirementsLoader::default(), + ) + .await + { + Ok(config_layer_stack) => config_layer_stack, + Err(err) => { + return SkillLoadOutcome { + errors: vec![crate::skills::model::SkillError { + path: cwd.to_path_buf(), + message: err.to_string(), + }], + ..Default::default() + }; + } + }; + + let loaded_plugins = self + .plugins_manager + .plugins_for_config_with_force_reload(config, force_reload); + let mut roots = skill_roots_with_filesystem( + &config_layer_stack, + cwd, + loaded_plugins.effective_skill_roots(), + filesystem, + ) + .await; + if !bundled_skills_enabled_from_stack(&config_layer_stack) { + roots.retain(|root| root.scope != SkillScope::System); + } + roots.extend( + normalized_extra_user_roots + .iter() + .cloned() + .map(|path| SkillRoot { + path, + scope: SkillScope::User, + }), + ); + let outcome = crate::skills::filter_skill_load_outcome_for_product( + finalize_skill_outcome( + load_skills_from_roots_with_filesystem(roots, filesystem).await, + &config_layer_stack, + ), + self.restriction_product, + ); + let mut cache = self + .cache_by_cwd + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + cache.insert(cwd.to_path_buf(), outcome.clone()); + outcome + } + fn build_skill_outcome( &self, roots: Vec, diff --git a/codex-rs/core/src/tools/runtimes/shell/unix_escalation.rs b/codex-rs/core/src/tools/runtimes/shell/unix_escalation.rs index afad1da2ab..79e71ad969 100644 --- a/codex-rs/core/src/tools/runtimes/shell/unix_escalation.rs +++ b/codex-rs/core/src/tools/runtimes/shell/unix_escalation.rs @@ -485,11 +485,17 @@ impl CoreShellActionProvider { /// any skills. async fn find_skill(&self, program: &AbsolutePathBuf) -> Option { let force_reload = false; + let environment_filesystem = self.turn.environment.get_filesystem(); let skills_outcome = self .session .services .skills_manager - .skills_for_cwd(&self.turn.cwd, self.turn.config.as_ref(), force_reload) + .skills_for_cwd_with_filesystem( + &self.turn.cwd, + self.turn.config.as_ref(), + force_reload, + &environment_filesystem, + ) .await; let program_path = program.as_path(); diff --git a/codex-rs/exec-server/src/environment.rs b/codex-rs/exec-server/src/environment.rs index 0fca2c2fa8..448066a539 100644 --- a/codex-rs/exec-server/src/environment.rs +++ b/codex-rs/exec-server/src/environment.rs @@ -340,6 +340,10 @@ impl Environment { Arc::clone(&self.executor) } + pub fn filesystem(&self) -> Arc { + Arc::clone(&self.filesystem) + } + pub fn get_filesystem(&self) -> impl ExecutorFileSystem + use<> { EnvironmentFileSystem { filesystem: Arc::clone(&self.filesystem),