From 653f07c421439bd94a3f8ec204c38aeb51b2d1d0 Mon Sep 17 00:00:00 2001 From: starr-openai Date: Sat, 21 Mar 2026 15:39:25 -0700 Subject: [PATCH] Port executor-backed file tools and plugin auth handling Co-authored-by: Codex --- .../app-server/src/codex_message_processor.rs | 48 ++-- codex-rs/app-server/src/command_exec.rs | 40 ++- codex-rs/core/src/connectors.rs | 8 + codex-rs/core/src/plugins/manager.rs | 8 +- codex-rs/core/src/plugins/manager_tests.rs | 1 + codex-rs/core/src/plugins/mod.rs | 1 + .../core/src/tools/handlers/grep_files.rs | 140 +++++++++- codex-rs/core/src/tools/handlers/list_dir.rs | 125 ++++++++- codex-rs/core/src/tools/handlers/read_file.rs | 244 +++++++++++++++++- 9 files changed, 577 insertions(+), 38 deletions(-) diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 152c8fa126..9462d4e987 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -232,7 +232,6 @@ use codex_core::plugins::PluginInstallError as CorePluginInstallError; use codex_core::plugins::PluginInstallRequest; use codex_core::plugins::PluginReadRequest; use codex_core::plugins::PluginUninstallError as CorePluginUninstallError; -use codex_core::plugins::load_plugin_apps; use codex_core::read_head_for_summary; use codex_core::read_session_meta_line; use codex_core::rollout_date_parts; @@ -6081,6 +6080,15 @@ impl CodexMessageProcessor { return; } let config_cwd = marketplace_path.as_path().parent().map(Path::to_path_buf); + let config_for_auth = match self.load_latest_config(config_cwd.clone()).await { + Ok(config) => config, + Err(err) => { + warn!( + "failed to load config for plugin/install auth context, using current config: {err:?}" + ); + self.config.as_ref().clone() + } + }; let environment = match Environment::create(self.config.experimental_exec_server_url.clone()).await { Ok(environment) => environment, @@ -6102,17 +6110,10 @@ impl CodexMessageProcessor { }; let install_result = if force_remote_sync { - let config = match self.load_latest_config(config_cwd.clone()).await { - Ok(config) => config, - Err(err) => { - self.outgoing.send_error(request_id, err).await; - return; - } - }; let auth = self.auth_manager.auth().await; plugins_manager .install_plugin_with_remote_sync_with_filesystem( - &config, + &config_for_auth, auth.as_ref(), request, &environment_filesystem, @@ -6126,16 +6127,8 @@ impl CodexMessageProcessor { match install_result { Ok(result) => { - let config = match self.load_latest_config(config_cwd).await { - Ok(config) => config, - Err(err) => { - warn!( - "failed to reload config after plugin install, using current config: {err:?}" - ); - self.config.as_ref().clone() - } - }; - let plugin_apps = load_plugin_apps(result.installed_path.as_path()); + let config = config_for_auth; + let plugin_apps = result.apps.clone(); let apps_needing_auth = if plugin_apps.is_empty() || !config.features.apps_enabled(Some(&self.auth_manager)).await { @@ -6186,7 +6179,6 @@ impl CodexMessageProcessor { "codex_apps MCP not ready after plugin install; skipping appsNeedingAuth check" ); } - plugin_app_helpers::plugin_apps_needing_auth( &all_connectors, &accessible_connectors, @@ -7399,21 +7391,29 @@ impl CodexMessageProcessor { "" => vec![], _ if self.config.experimental_exec_server_url.is_some() => { let environment = - match Environment::create(self.config.experimental_exec_server_url.clone()).await + match Environment::create(self.config.experimental_exec_server_url.clone()) + .await { Ok(environment) => environment, Err(error) => { self.send_invalid_request_error( request_id, - format!("failed to bind environment for fuzzy file search: {error}"), + format!( + "failed to bind environment for fuzzy file search: {error}" + ), ) .await; return; } }; let filesystem = environment.get_filesystem(); - run_fuzzy_file_search_with_filesystem(query, roots, &filesystem, cancel_flag.clone()) - .await + run_fuzzy_file_search_with_filesystem( + query, + roots, + &filesystem, + cancel_flag.clone(), + ) + .await } _ => run_fuzzy_file_search(query, roots, cancel_flag.clone()).await, }; diff --git a/codex-rs/app-server/src/command_exec.rs b/codex-rs/app-server/src/command_exec.rs index a77bb8748c..92687dfa67 100644 --- a/codex-rs/app-server/src/command_exec.rs +++ b/codex-rs/app-server/src/command_exec.rs @@ -582,7 +582,21 @@ async fn run_command_with_executor(params: RunExecutorCommandParams) { handle_executor_resize(&executor, &executor_process_id, tty, size).await } CommandControl::Terminate => { - handle_executor_terminate(&executor, &executor_process_id).await + let result = + handle_executor_terminate(&executor, &executor_process_id) + .await; + if result.is_ok() + && let Some(waited_exit_code) = wait_for_executor_exit( + &executor, + &executor_process_id, + /*wait_ms*/ 500, + ) + .await + { + exit_code = waited_exit_code; + exit_deadline.get_or_insert_with(Instant::now); + } + result } }; if let Some(response_tx) = response_tx { @@ -598,6 +612,15 @@ async fn run_command_with_executor(params: RunExecutorCommandParams) { _ = &mut expiration, if !timed_out => { timed_out = true; let _ = handle_executor_terminate(&executor, &executor_process_id).await; + if wait_for_executor_exit(&executor, &executor_process_id, /*wait_ms*/ 50) + .await + .is_some() + { + exit_code = EXEC_TIMEOUT_EXIT_CODE; + exit_deadline.get_or_insert_with(|| { + Instant::now() + Duration::from_millis(IO_DRAIN_TIMEOUT_MS) + }); + } } read = executor.read(ExecutorReadParams { process_id: executor_process_id.clone(), @@ -764,6 +787,21 @@ async fn handle_executor_terminate( Ok(()) } +async fn wait_for_executor_exit( + executor: &Arc, + process_id: &str, + wait_ms: u64, +) -> Option { + let response = executor + .wait(ExecutorExecWaitParams { + process_id: process_id.to_string(), + wait_ms: Some(wait_ms), + }) + .await + .ok()?; + response.exited.then_some(response.exit_code.unwrap_or(-1)) +} + fn spawn_process_output(params: SpawnProcessOutputParams) -> tokio::task::JoinHandle { let SpawnProcessOutputParams { connection_id, diff --git a/codex-rs/core/src/connectors.rs b/codex-rs/core/src/connectors.rs index 1d7700229f..045470e99c 100644 --- a/codex-rs/core/src/connectors.rs +++ b/codex-rs/core/src/connectors.rs @@ -264,6 +264,9 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_options_and_status( mcp_connection_manager.list_all_tools().await }; let mut should_reload_tools = false; + let has_codex_apps_tools = tools + .values() + .any(|tool| tool.server_name == CODEX_APPS_MCP_SERVER_NAME); let codex_apps_ready = if refreshed_tools_succeeded { true } else if let Some(cfg) = mcp_servers.get(CODEX_APPS_MCP_SERVER_NAME) { @@ -272,6 +275,11 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_options_and_status( .await; if immediate_ready { true + } else if has_codex_apps_tools { + // If codex_apps tools are already visible, treat the server as + // ready enough for auth-state derivation instead of returning a + // false negative appsNeedingAuth result. + true } else if tools.is_empty() { let timeout = cfg .startup_timeout_sec diff --git a/codex-rs/core/src/plugins/manager.rs b/codex-rs/core/src/plugins/manager.rs index 09b35ed922..e6485a7adc 100644 --- a/codex-rs/core/src/plugins/manager.rs +++ b/codex-rs/core/src/plugins/manager.rs @@ -202,6 +202,7 @@ pub struct PluginInstallOutcome { pub plugin_version: String, pub installed_path: AbsolutePathBuf, pub auth_policy: MarketplacePluginAuthPolicy, + pub apps: Vec, } #[derive(Debug, Clone, PartialEq)] @@ -796,6 +797,7 @@ impl PluginsManager { &self, resolved: ResolvedMarketplacePlugin, ) -> Result { + let apps = load_plugin_apps(resolved.source_path.as_path()); let auth_policy = resolved.auth_policy; let plugin_version = if resolved.plugin_id.marketplace_name == OPENAI_CURATED_MARKETPLACE_NAME { @@ -851,6 +853,7 @@ impl PluginsManager { plugin_version: result.plugin_version, installed_path: result.installed_path, auth_policy, + apps, }) } @@ -862,6 +865,8 @@ impl PluginsManager { where F: ExecutorFileSystem + ?Sized, { + let apps = + load_plugin_apps_with_filesystem(resolved.source_path.as_path(), filesystem).await; let auth_policy = resolved.auth_policy; let plugin_version = if resolved.plugin_id.marketplace_name == OPENAI_CURATED_MARKETPLACE_NAME { @@ -922,6 +927,7 @@ impl PluginsManager { plugin_version: result.plugin_version, installed_path: result.installed_path, auth_policy, + apps, }) } @@ -2451,7 +2457,7 @@ pub fn load_plugin_apps(plugin_root: &Path) -> Vec { load_apps_from_paths(plugin_root, default_app_config_paths(plugin_root)) } -async fn load_plugin_apps_with_filesystem( +pub async fn load_plugin_apps_with_filesystem( plugin_root: &Path, filesystem: &F, ) -> Vec diff --git a/codex-rs/core/src/plugins/manager_tests.rs b/codex-rs/core/src/plugins/manager_tests.rs index 49a63eee42..775dedf5c3 100644 --- a/codex-rs/core/src/plugins/manager_tests.rs +++ b/codex-rs/core/src/plugins/manager_tests.rs @@ -843,6 +843,7 @@ async fn install_plugin_updates_config_with_relative_path_and_plugin_key() { plugin_version: "local".to_string(), installed_path: AbsolutePathBuf::try_from(installed_path).unwrap(), auth_policy: MarketplacePluginAuthPolicy::OnUse, + apps: Vec::new(), } ); diff --git a/codex-rs/core/src/plugins/mod.rs b/codex-rs/core/src/plugins/mod.rs index 14d641ae82..8f00d6ca89 100644 --- a/codex-rs/core/src/plugins/mod.rs +++ b/codex-rs/core/src/plugins/mod.rs @@ -37,6 +37,7 @@ pub use manager::PluginsManager; pub use manager::RemotePluginSyncResult; pub use manager::installed_plugin_telemetry_metadata; pub use manager::load_plugin_apps; +pub use manager::load_plugin_apps_with_filesystem; pub(crate) use manager::plugin_namespace_for_skill_path; pub use manager::plugin_telemetry_metadata_from_root; pub use manifest::PluginManifestInterface; diff --git a/codex-rs/core/src/tools/handlers/grep_files.rs b/codex-rs/core/src/tools/handlers/grep_files.rs index fdb0fce7be..ef478461a4 100644 --- a/codex-rs/core/src/tools/handlers/grep_files.rs +++ b/codex-rs/core/src/tools/handlers/grep_files.rs @@ -2,9 +2,13 @@ use std::path::Path; use std::time::Duration; use async_trait::async_trait; +use codex_exec_server::ExecutorFileSystem; +use codex_utils_absolute_path::AbsolutePathBuf; +use regex_lite::Regex; use serde::Deserialize; use tokio::process::Command; use tokio::time::timeout; +use wildmatch::WildMatch; use crate::function_tool::FunctionCallError; use crate::tools::context::FunctionToolOutput; @@ -73,8 +77,6 @@ impl ToolHandler for GrepFilesHandler { let limit = args.limit.min(MAX_LIMIT); let search_path = turn.resolve_path(args.path.clone()); - verify_path_exists(&search_path).await?; - let include = args.include.as_deref().map(str::trim).and_then(|val| { if val.is_empty() { None @@ -83,8 +85,27 @@ impl ToolHandler for GrepFilesHandler { } }); - let search_results = - run_rg_search(pattern, include.as_deref(), &search_path, limit, &turn.cwd).await?; + let search_results = if turn.environment.experimental_exec_server_url().is_some() { + let absolute_path = AbsolutePathBuf::try_from(search_path.clone()).map_err(|err| { + FunctionCallError::RespondToModel(format!( + "failed to normalize search path `{}`: {err}", + search_path.display() + )) + })?; + let filesystem = turn.environment.get_filesystem(); + verify_path_exists_with_filesystem(&filesystem, &absolute_path).await?; + run_search_with_filesystem( + pattern, + include.as_deref(), + &absolute_path, + limit, + &filesystem, + ) + .await? + } else { + verify_path_exists(&search_path).await?; + run_rg_search(pattern, include.as_deref(), &search_path, limit, &turn.cwd).await? + }; if search_results.is_empty() { Ok(FunctionToolOutput::from_text( @@ -107,6 +128,19 @@ async fn verify_path_exists(path: &Path) -> Result<(), FunctionCallError> { Ok(()) } +async fn verify_path_exists_with_filesystem( + filesystem: &F, + path: &AbsolutePathBuf, +) -> Result<(), FunctionCallError> +where + F: ExecutorFileSystem + ?Sized, +{ + filesystem.get_metadata(path).await.map_err(|err| { + FunctionCallError::RespondToModel(format!("unable to access `{}`: {err}", path.display())) + })?; + Ok(()) +} + async fn run_rg_search( pattern: &str, include: Option<&str>, @@ -152,6 +186,104 @@ async fn run_rg_search( } } +async fn run_search_with_filesystem( + pattern: &str, + include: Option<&str>, + search_path: &AbsolutePathBuf, + limit: usize, + filesystem: &F, +) -> Result, FunctionCallError> +where + F: ExecutorFileSystem + ?Sized, +{ + let regex = Regex::new(pattern).map_err(|err| { + FunctionCallError::RespondToModel(format!("invalid regex pattern `{pattern}`: {err}")) + })?; + let include = include.map(WildMatch::new); + let search_metadata = filesystem.get_metadata(search_path).await.map_err(|err| { + FunctionCallError::RespondToModel(format!( + "unable to access `{}`: {err}", + search_path.display() + )) + })?; + + let mut matches = Vec::new(); + if search_metadata.is_file { + if file_matches_regex(filesystem, search_path, include.as_ref(), ®ex).await { + let modified_at_ms = filesystem + .get_metadata(search_path) + .await + .map(|metadata| metadata.modified_at_ms) + .unwrap_or_default(); + matches.push((modified_at_ms, search_path.to_string_lossy().to_string())); + } + } else { + let mut pending = vec![search_path.clone()]; + while let Some(current_dir) = pending.pop() { + let entries = match filesystem.read_directory(¤t_dir).await { + Ok(entries) => entries, + Err(_) => continue, + }; + for entry in entries { + let Ok(entry_path) = + AbsolutePathBuf::try_from(current_dir.as_path().join(&entry.file_name)) + else { + continue; + }; + if entry.is_directory { + pending.push(entry_path); + continue; + } + if !entry.is_file { + continue; + } + if !file_matches_regex(filesystem, &entry_path, include.as_ref(), ®ex).await { + continue; + } + let modified_at_ms = filesystem + .get_metadata(&entry_path) + .await + .map(|metadata| metadata.modified_at_ms) + .unwrap_or_default(); + matches.push((modified_at_ms, entry_path.to_string_lossy().to_string())); + } + } + } + + matches.sort_unstable_by(|a, b| b.0.cmp(&a.0).then_with(|| a.1.cmp(&b.1))); + Ok(matches + .into_iter() + .take(limit) + .map(|(_, path)| path) + .collect()) +} + +async fn file_matches_regex( + filesystem: &F, + path: &AbsolutePathBuf, + include: Option<&WildMatch>, + regex: &Regex, +) -> bool +where + F: ExecutorFileSystem + ?Sized, +{ + if let Some(glob) = include { + let full_path = path.to_string_lossy(); + let Some(file_name) = path.as_path().file_name().and_then(|name| name.to_str()) else { + return false; + }; + if !glob.matches(&full_path) && !glob.matches(file_name) { + return false; + } + } + + let Ok(bytes) = filesystem.read_file(path).await else { + return false; + }; + let contents = String::from_utf8_lossy(&bytes); + regex.is_match(&contents) +} + fn parse_results(stdout: &[u8], limit: usize) -> Vec { let mut results = Vec::new(); for line in stdout.split(|byte| *byte == b'\n') { diff --git a/codex-rs/core/src/tools/handlers/list_dir.rs b/codex-rs/core/src/tools/handlers/list_dir.rs index fd461e82e5..99434a15b2 100644 --- a/codex-rs/core/src/tools/handlers/list_dir.rs +++ b/codex-rs/core/src/tools/handlers/list_dir.rs @@ -5,6 +5,8 @@ use std::path::Path; use std::path::PathBuf; use async_trait::async_trait; +use codex_exec_server::ExecutorFileSystem; +use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_string::take_bytes_at_char_boundary; use serde::Deserialize; use tokio::fs; @@ -54,7 +56,7 @@ impl ToolHandler for ListDirHandler { } async fn handle(&self, invocation: ToolInvocation) -> Result { - let ToolInvocation { payload, .. } = invocation; + let ToolInvocation { payload, turn, .. } = invocation; let arguments = match payload { ToolPayload::Function { arguments } => arguments, @@ -99,7 +101,16 @@ impl ToolHandler for ListDirHandler { )); } - let entries = list_dir_slice(&path, offset, limit, depth).await?; + let entries = if turn.environment.experimental_exec_server_url().is_some() { + let absolute_path = AbsolutePathBuf::try_from(path.clone()).map_err(|err| { + FunctionCallError::RespondToModel(format!("failed to normalize dir_path: {err}")) + })?; + let filesystem = turn.environment.get_filesystem(); + list_dir_slice_with_filesystem(&filesystem, &absolute_path, offset, limit, depth) + .await? + } else { + list_dir_slice(&path, offset, limit, depth).await? + }; let mut output = Vec::with_capacity(entries.len() + 1); output.push(format!("Absolute path: {}", path.display())); output.extend(entries); @@ -146,6 +157,49 @@ async fn list_dir_slice( Ok(formatted) } +async fn list_dir_slice_with_filesystem( + filesystem: &F, + path: &AbsolutePathBuf, + offset: usize, + limit: usize, + depth: usize, +) -> Result, FunctionCallError> +where + F: ExecutorFileSystem + ?Sized, +{ + let mut entries = Vec::new(); + collect_entries_with_filesystem(filesystem, path, Path::new(""), depth, &mut entries).await?; + + if entries.is_empty() { + return Ok(Vec::new()); + } + + entries.sort_unstable_by(|a, b| a.name.cmp(&b.name)); + + let start_index = offset - 1; + if start_index >= entries.len() { + return Err(FunctionCallError::RespondToModel( + "offset exceeds directory entry count".to_string(), + )); + } + + let remaining_entries = entries.len() - start_index; + let capped_limit = limit.min(remaining_entries); + let end_index = start_index + capped_limit; + let selected_entries = &entries[start_index..end_index]; + let mut formatted = Vec::with_capacity(selected_entries.len()); + + for entry in selected_entries { + formatted.push(format_entry_line(entry)); + } + + if end_index < entries.len() { + formatted.push(format!("More than {capped_limit} entries found")); + } + + Ok(formatted) +} + async fn collect_entries( dir_path: &Path, relative_prefix: &Path, @@ -206,6 +260,73 @@ async fn collect_entries( Ok(()) } +async fn collect_entries_with_filesystem( + filesystem: &F, + dir_path: &AbsolutePathBuf, + relative_prefix: &Path, + depth: usize, + entries: &mut Vec, +) -> Result<(), FunctionCallError> +where + F: ExecutorFileSystem + ?Sized, +{ + let mut queue = VecDeque::new(); + queue.push_back((dir_path.clone(), relative_prefix.to_path_buf(), depth)); + + while let Some((current_dir, prefix, remaining_depth)) = queue.pop_front() { + let mut dir_entries = filesystem + .read_directory(¤t_dir) + .await + .map_err(|err| { + FunctionCallError::RespondToModel(format!("failed to read directory: {err}")) + })? + .into_iter() + .filter_map(|entry| { + let relative_path = if prefix.as_os_str().is_empty() { + PathBuf::from(&entry.file_name) + } else { + prefix.join(&entry.file_name) + }; + let display_name = format_entry_component(OsStr::new(&entry.file_name)); + let display_depth = prefix.components().count(); + let sort_key = format_entry_name(&relative_path); + let kind = if entry.is_directory { + DirEntryKind::Directory + } else { + DirEntryKind::File + }; + let Ok(entry_path) = + AbsolutePathBuf::try_from(current_dir.as_path().join(&entry.file_name)) + else { + return None; + }; + Some(( + entry_path, + relative_path, + kind, + DirEntry { + name: sort_key, + display_name, + depth: display_depth, + kind, + }, + )) + }) + .collect::>(); + + dir_entries.sort_unstable_by(|a, b| a.3.name.cmp(&b.3.name)); + + for (entry_path, relative_path, kind, dir_entry) in dir_entries { + if kind == DirEntryKind::Directory && remaining_depth > 1 { + queue.push_back((entry_path, relative_path, remaining_depth - 1)); + } + entries.push(dir_entry); + } + } + + Ok(()) +} + fn format_entry_name(path: &Path) -> String { let normalized = path.to_string_lossy().replace("\\", "/"); if normalized.len() > MAX_ENTRY_LENGTH { diff --git a/codex-rs/core/src/tools/handlers/read_file.rs b/codex-rs/core/src/tools/handlers/read_file.rs index b868edf5b9..2afcdaa823 100644 --- a/codex-rs/core/src/tools/handlers/read_file.rs +++ b/codex-rs/core/src/tools/handlers/read_file.rs @@ -2,6 +2,7 @@ use std::collections::VecDeque; use std::path::PathBuf; use async_trait::async_trait; +use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_string::take_bytes_at_char_boundary; use serde::Deserialize; @@ -100,7 +101,7 @@ impl ToolHandler for ReadFileHandler { } async fn handle(&self, invocation: ToolInvocation) -> Result { - let ToolInvocation { payload, .. } = invocation; + let ToolInvocation { payload, turn, .. } = invocation; let arguments = match payload { ToolPayload::Function { arguments } => arguments, @@ -140,11 +141,34 @@ impl ToolHandler for ReadFileHandler { )); } - let collected = match mode { - ReadMode::Slice => slice::read(&path, offset, limit).await?, - ReadMode::Indentation => { - let indentation = indentation.unwrap_or_default(); - indentation::read_block(&path, offset, limit, indentation).await? + let collected = if turn.environment.experimental_exec_server_url().is_some() { + let absolute_path = AbsolutePathBuf::try_from(path.clone()).map_err(|err| { + FunctionCallError::RespondToModel(format!("failed to normalize file_path: {err}")) + })?; + let filesystem = turn.environment.get_filesystem(); + match mode { + ReadMode::Slice => { + slice::read_with_filesystem(&filesystem, &absolute_path, offset, limit).await? + } + ReadMode::Indentation => { + let indentation = indentation.unwrap_or_default(); + indentation::read_block_with_filesystem( + &filesystem, + &absolute_path, + offset, + limit, + indentation, + ) + .await? + } + } + } else { + match mode { + ReadMode::Slice => slice::read(&path, offset, limit).await?, + ReadMode::Indentation => { + let indentation = indentation.unwrap_or_default(); + indentation::read_block(&path, offset, limit, indentation).await? + } } }; Ok(FunctionToolOutput::from_text( @@ -156,7 +180,10 @@ impl ToolHandler for ReadFileHandler { mod slice { use crate::function_tool::FunctionCallError; + use crate::tools::handlers::read_file::collect_line_bytes; use crate::tools::handlers::read_file::format_line; + use codex_exec_server::ExecutorFileSystem; + use codex_utils_absolute_path::AbsolutePathBuf; use std::path::Path; use tokio::fs::File; use tokio::io::AsyncBufReadExt; @@ -219,6 +246,35 @@ mod slice { Ok(collected) } + + pub async fn read_with_filesystem( + filesystem: &F, + path: &AbsolutePathBuf, + offset: usize, + limit: usize, + ) -> Result, FunctionCallError> + where + F: ExecutorFileSystem + ?Sized, + { + let bytes = filesystem.read_file(path).await.map_err(|err| { + FunctionCallError::RespondToModel(format!("failed to read file: {err}")) + })?; + + let lines = collect_line_bytes(&bytes); + if lines.len() < offset { + return Err(FunctionCallError::RespondToModel( + "offset exceeds file length".to_string(), + )); + } + + Ok(lines + .into_iter() + .enumerate() + .skip(offset.saturating_sub(1)) + .take(limit) + .map(|(idx, line)| format!("L{}: {}", idx + 1, format_line(line))) + .collect()) + } } mod indentation { @@ -226,8 +282,11 @@ mod indentation { use crate::tools::handlers::read_file::IndentationArgs; use crate::tools::handlers::read_file::LineRecord; use crate::tools::handlers::read_file::TAB_WIDTH; + use crate::tools::handlers::read_file::collect_line_bytes; use crate::tools::handlers::read_file::format_line; use crate::tools::handlers::read_file::trim_empty_lines; + use codex_exec_server::ExecutorFileSystem; + use codex_utils_absolute_path::AbsolutePathBuf; use std::collections::VecDeque; use std::path::Path; use tokio::fs::File; @@ -366,6 +425,127 @@ mod indentation { .collect()) } + pub async fn read_block_with_filesystem( + filesystem: &F, + path: &AbsolutePathBuf, + offset: usize, + limit: usize, + options: IndentationArgs, + ) -> Result, FunctionCallError> + where + F: ExecutorFileSystem + ?Sized, + { + let anchor_line = options.anchor_line.unwrap_or(offset); + if anchor_line == 0 { + return Err(FunctionCallError::RespondToModel( + "anchor_line must be a 1-indexed line number".to_string(), + )); + } + + let guard_limit = options.max_lines.unwrap_or(limit); + if guard_limit == 0 { + return Err(FunctionCallError::RespondToModel( + "max_lines must be greater than zero".to_string(), + )); + } + + let collected = collect_file_lines_with_filesystem(filesystem, path).await?; + if collected.is_empty() || anchor_line > collected.len() { + return Err(FunctionCallError::RespondToModel( + "anchor_line exceeds file length".to_string(), + )); + } + + let anchor_index = anchor_line - 1; + let effective_indents = compute_effective_indents(&collected); + let anchor_indent = effective_indents[anchor_index]; + let min_indent = if options.max_levels == 0 { + 0 + } else { + anchor_indent.saturating_sub(options.max_levels * TAB_WIDTH) + }; + let final_limit = limit.min(guard_limit).min(collected.len()); + + if final_limit == 1 { + return Ok(vec![format!( + "L{}: {}", + collected[anchor_index].number, collected[anchor_index].display + )]); + } + + let mut i: isize = anchor_index as isize - 1; + let mut j: usize = anchor_index + 1; + let mut i_counter_min_indent = 0; + let mut j_counter_min_indent = 0; + + let mut out = VecDeque::with_capacity(limit); + out.push_back(&collected[anchor_index]); + + while out.len() < final_limit { + let mut progressed = 0; + + if i >= 0 { + let iu = i as usize; + if effective_indents[iu] >= min_indent { + out.push_front(&collected[iu]); + progressed += 1; + i -= 1; + + if effective_indents[iu] == min_indent && !options.include_siblings { + let allow_header_comment = + options.include_header && collected[iu].is_comment(); + let can_take_line = allow_header_comment || i_counter_min_indent == 0; + + if can_take_line { + i_counter_min_indent += 1; + } else { + out.pop_front(); + progressed -= 1; + i = -1; + } + } + + if out.len() >= final_limit { + break; + } + } else { + i = -1; + } + } + + if j < collected.len() { + let ju = j; + if effective_indents[ju] >= min_indent { + out.push_back(&collected[ju]); + progressed += 1; + j += 1; + + if effective_indents[ju] == min_indent && !options.include_siblings { + if j_counter_min_indent > 0 { + out.pop_back(); + progressed -= 1; + j = collected.len(); + } + j_counter_min_indent += 1; + } + } else { + j = collected.len(); + } + } + + if progressed == 0 { + break; + } + } + + trim_empty_lines(&mut out); + + Ok(out + .into_iter() + .map(|record| format!("L{}: {}", record.number, record.display)) + .collect()) + } + async fn collect_file_lines(path: &Path) -> Result, FunctionCallError> { let file = File::open(path).await.map_err(|err| { FunctionCallError::RespondToModel(format!("failed to read file: {err}")) @@ -408,6 +588,34 @@ mod indentation { Ok(lines) } + async fn collect_file_lines_with_filesystem( + filesystem: &F, + path: &AbsolutePathBuf, + ) -> Result, FunctionCallError> + where + F: ExecutorFileSystem + ?Sized, + { + let bytes = filesystem.read_file(path).await.map_err(|err| { + FunctionCallError::RespondToModel(format!("failed to read file: {err}")) + })?; + + Ok(collect_line_bytes(&bytes) + .into_iter() + .enumerate() + .map(|(idx, line)| { + let raw = String::from_utf8_lossy(line).into_owned(); + let indent = measure_indent(&raw); + let display = format_line(line); + LineRecord { + number: idx + 1, + raw, + display, + indent, + } + }) + .collect()) + } + fn compute_effective_indents(records: &[LineRecord]) -> Vec { let mut effective = Vec::with_capacity(records.len()); let mut previous_indent = 0usize; @@ -439,6 +647,30 @@ fn format_line(bytes: &[u8]) -> String { } } +fn collect_line_bytes(bytes: &[u8]) -> Vec<&[u8]> { + let mut lines = Vec::new(); + let mut start = 0usize; + + for idx in 0..=bytes.len() { + if idx != bytes.len() && bytes[idx] != b'\n' { + continue; + } + + if idx == bytes.len() && start == bytes.len() { + break; + } + + let mut line = &bytes[start..idx]; + if line.last() == Some(&b'\r') { + line = &line[..line.len() - 1]; + } + lines.push(line); + start = idx.saturating_add(1); + } + + lines +} + fn trim_empty_lines(out: &mut VecDeque<&LineRecord>) { while matches!(out.front(), Some(line) if line.raw.trim().is_empty()) { out.pop_front();