mirror of
https://github.com/openai/codex.git
synced 2026-11-17 22:10:37 +00:00
Port executor-backed file tools and plugin auth handling
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -232,7 +232,6 @@ use codex_core::plugins::PluginInstallError as CorePluginInstallError;
|
||||
use codex_core::plugins::PluginInstallRequest;
|
||||
use codex_core::plugins::PluginReadRequest;
|
||||
use codex_core::plugins::PluginUninstallError as CorePluginUninstallError;
|
||||
use codex_core::plugins::load_plugin_apps;
|
||||
use codex_core::read_head_for_summary;
|
||||
use codex_core::read_session_meta_line;
|
||||
use codex_core::rollout_date_parts;
|
||||
@@ -6081,6 +6080,15 @@ impl CodexMessageProcessor {
|
||||
return;
|
||||
}
|
||||
let config_cwd = marketplace_path.as_path().parent().map(Path::to_path_buf);
|
||||
let config_for_auth = match self.load_latest_config(config_cwd.clone()).await {
|
||||
Ok(config) => config,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
"failed to load config for plugin/install auth context, using current config: {err:?}"
|
||||
);
|
||||
self.config.as_ref().clone()
|
||||
}
|
||||
};
|
||||
let environment =
|
||||
match Environment::create(self.config.experimental_exec_server_url.clone()).await {
|
||||
Ok(environment) => environment,
|
||||
@@ -6102,17 +6110,10 @@ impl CodexMessageProcessor {
|
||||
};
|
||||
|
||||
let install_result = if force_remote_sync {
|
||||
let config = match self.load_latest_config(config_cwd.clone()).await {
|
||||
Ok(config) => config,
|
||||
Err(err) => {
|
||||
self.outgoing.send_error(request_id, err).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
let auth = self.auth_manager.auth().await;
|
||||
plugins_manager
|
||||
.install_plugin_with_remote_sync_with_filesystem(
|
||||
&config,
|
||||
&config_for_auth,
|
||||
auth.as_ref(),
|
||||
request,
|
||||
&environment_filesystem,
|
||||
@@ -6126,16 +6127,8 @@ impl CodexMessageProcessor {
|
||||
|
||||
match install_result {
|
||||
Ok(result) => {
|
||||
let config = match self.load_latest_config(config_cwd).await {
|
||||
Ok(config) => config,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
"failed to reload config after plugin install, using current config: {err:?}"
|
||||
);
|
||||
self.config.as_ref().clone()
|
||||
}
|
||||
};
|
||||
let plugin_apps = load_plugin_apps(result.installed_path.as_path());
|
||||
let config = config_for_auth;
|
||||
let plugin_apps = result.apps.clone();
|
||||
let apps_needing_auth = if plugin_apps.is_empty()
|
||||
|| !config.features.apps_enabled(Some(&self.auth_manager)).await
|
||||
{
|
||||
@@ -6186,7 +6179,6 @@ impl CodexMessageProcessor {
|
||||
"codex_apps MCP not ready after plugin install; skipping appsNeedingAuth check"
|
||||
);
|
||||
}
|
||||
|
||||
plugin_app_helpers::plugin_apps_needing_auth(
|
||||
&all_connectors,
|
||||
&accessible_connectors,
|
||||
@@ -7399,21 +7391,29 @@ impl CodexMessageProcessor {
|
||||
"" => vec![],
|
||||
_ if self.config.experimental_exec_server_url.is_some() => {
|
||||
let environment =
|
||||
match Environment::create(self.config.experimental_exec_server_url.clone()).await
|
||||
match Environment::create(self.config.experimental_exec_server_url.clone())
|
||||
.await
|
||||
{
|
||||
Ok(environment) => environment,
|
||||
Err(error) => {
|
||||
self.send_invalid_request_error(
|
||||
request_id,
|
||||
format!("failed to bind environment for fuzzy file search: {error}"),
|
||||
format!(
|
||||
"failed to bind environment for fuzzy file search: {error}"
|
||||
),
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
let filesystem = environment.get_filesystem();
|
||||
run_fuzzy_file_search_with_filesystem(query, roots, &filesystem, cancel_flag.clone())
|
||||
.await
|
||||
run_fuzzy_file_search_with_filesystem(
|
||||
query,
|
||||
roots,
|
||||
&filesystem,
|
||||
cancel_flag.clone(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
_ => run_fuzzy_file_search(query, roots, cancel_flag.clone()).await,
|
||||
};
|
||||
|
||||
@@ -582,7 +582,21 @@ async fn run_command_with_executor(params: RunExecutorCommandParams) {
|
||||
handle_executor_resize(&executor, &executor_process_id, tty, size).await
|
||||
}
|
||||
CommandControl::Terminate => {
|
||||
handle_executor_terminate(&executor, &executor_process_id).await
|
||||
let result =
|
||||
handle_executor_terminate(&executor, &executor_process_id)
|
||||
.await;
|
||||
if result.is_ok()
|
||||
&& let Some(waited_exit_code) = wait_for_executor_exit(
|
||||
&executor,
|
||||
&executor_process_id,
|
||||
/*wait_ms*/ 500,
|
||||
)
|
||||
.await
|
||||
{
|
||||
exit_code = waited_exit_code;
|
||||
exit_deadline.get_or_insert_with(Instant::now);
|
||||
}
|
||||
result
|
||||
}
|
||||
};
|
||||
if let Some(response_tx) = response_tx {
|
||||
@@ -598,6 +612,15 @@ async fn run_command_with_executor(params: RunExecutorCommandParams) {
|
||||
_ = &mut expiration, if !timed_out => {
|
||||
timed_out = true;
|
||||
let _ = handle_executor_terminate(&executor, &executor_process_id).await;
|
||||
if wait_for_executor_exit(&executor, &executor_process_id, /*wait_ms*/ 50)
|
||||
.await
|
||||
.is_some()
|
||||
{
|
||||
exit_code = EXEC_TIMEOUT_EXIT_CODE;
|
||||
exit_deadline.get_or_insert_with(|| {
|
||||
Instant::now() + Duration::from_millis(IO_DRAIN_TIMEOUT_MS)
|
||||
});
|
||||
}
|
||||
}
|
||||
read = executor.read(ExecutorReadParams {
|
||||
process_id: executor_process_id.clone(),
|
||||
@@ -764,6 +787,21 @@ async fn handle_executor_terminate(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn wait_for_executor_exit(
|
||||
executor: &Arc<dyn ExecProcess>,
|
||||
process_id: &str,
|
||||
wait_ms: u64,
|
||||
) -> Option<i32> {
|
||||
let response = executor
|
||||
.wait(ExecutorExecWaitParams {
|
||||
process_id: process_id.to_string(),
|
||||
wait_ms: Some(wait_ms),
|
||||
})
|
||||
.await
|
||||
.ok()?;
|
||||
response.exited.then_some(response.exit_code.unwrap_or(-1))
|
||||
}
|
||||
|
||||
fn spawn_process_output(params: SpawnProcessOutputParams) -> tokio::task::JoinHandle<String> {
|
||||
let SpawnProcessOutputParams {
|
||||
connection_id,
|
||||
|
||||
@@ -264,6 +264,9 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_options_and_status(
|
||||
mcp_connection_manager.list_all_tools().await
|
||||
};
|
||||
let mut should_reload_tools = false;
|
||||
let has_codex_apps_tools = tools
|
||||
.values()
|
||||
.any(|tool| tool.server_name == CODEX_APPS_MCP_SERVER_NAME);
|
||||
let codex_apps_ready = if refreshed_tools_succeeded {
|
||||
true
|
||||
} else if let Some(cfg) = mcp_servers.get(CODEX_APPS_MCP_SERVER_NAME) {
|
||||
@@ -272,6 +275,11 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_options_and_status(
|
||||
.await;
|
||||
if immediate_ready {
|
||||
true
|
||||
} else if has_codex_apps_tools {
|
||||
// If codex_apps tools are already visible, treat the server as
|
||||
// ready enough for auth-state derivation instead of returning a
|
||||
// false negative appsNeedingAuth result.
|
||||
true
|
||||
} else if tools.is_empty() {
|
||||
let timeout = cfg
|
||||
.startup_timeout_sec
|
||||
|
||||
@@ -202,6 +202,7 @@ pub struct PluginInstallOutcome {
|
||||
pub plugin_version: String,
|
||||
pub installed_path: AbsolutePathBuf,
|
||||
pub auth_policy: MarketplacePluginAuthPolicy,
|
||||
pub apps: Vec<AppConnectorId>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
@@ -796,6 +797,7 @@ impl PluginsManager {
|
||||
&self,
|
||||
resolved: ResolvedMarketplacePlugin,
|
||||
) -> Result<PluginInstallOutcome, PluginInstallError> {
|
||||
let apps = load_plugin_apps(resolved.source_path.as_path());
|
||||
let auth_policy = resolved.auth_policy;
|
||||
let plugin_version =
|
||||
if resolved.plugin_id.marketplace_name == OPENAI_CURATED_MARKETPLACE_NAME {
|
||||
@@ -851,6 +853,7 @@ impl PluginsManager {
|
||||
plugin_version: result.plugin_version,
|
||||
installed_path: result.installed_path,
|
||||
auth_policy,
|
||||
apps,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -862,6 +865,8 @@ impl PluginsManager {
|
||||
where
|
||||
F: ExecutorFileSystem + ?Sized,
|
||||
{
|
||||
let apps =
|
||||
load_plugin_apps_with_filesystem(resolved.source_path.as_path(), filesystem).await;
|
||||
let auth_policy = resolved.auth_policy;
|
||||
let plugin_version =
|
||||
if resolved.plugin_id.marketplace_name == OPENAI_CURATED_MARKETPLACE_NAME {
|
||||
@@ -922,6 +927,7 @@ impl PluginsManager {
|
||||
plugin_version: result.plugin_version,
|
||||
installed_path: result.installed_path,
|
||||
auth_policy,
|
||||
apps,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2451,7 +2457,7 @@ pub fn load_plugin_apps(plugin_root: &Path) -> Vec<AppConnectorId> {
|
||||
load_apps_from_paths(plugin_root, default_app_config_paths(plugin_root))
|
||||
}
|
||||
|
||||
async fn load_plugin_apps_with_filesystem<F>(
|
||||
pub async fn load_plugin_apps_with_filesystem<F>(
|
||||
plugin_root: &Path,
|
||||
filesystem: &F,
|
||||
) -> Vec<AppConnectorId>
|
||||
|
||||
@@ -843,6 +843,7 @@ async fn install_plugin_updates_config_with_relative_path_and_plugin_key() {
|
||||
plugin_version: "local".to_string(),
|
||||
installed_path: AbsolutePathBuf::try_from(installed_path).unwrap(),
|
||||
auth_policy: MarketplacePluginAuthPolicy::OnUse,
|
||||
apps: Vec::new(),
|
||||
}
|
||||
);
|
||||
|
||||
|
||||
@@ -37,6 +37,7 @@ pub use manager::PluginsManager;
|
||||
pub use manager::RemotePluginSyncResult;
|
||||
pub use manager::installed_plugin_telemetry_metadata;
|
||||
pub use manager::load_plugin_apps;
|
||||
pub use manager::load_plugin_apps_with_filesystem;
|
||||
pub(crate) use manager::plugin_namespace_for_skill_path;
|
||||
pub use manager::plugin_telemetry_metadata_from_root;
|
||||
pub use manifest::PluginManifestInterface;
|
||||
|
||||
@@ -2,9 +2,13 @@ use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use codex_exec_server::ExecutorFileSystem;
|
||||
use codex_utils_absolute_path::AbsolutePathBuf;
|
||||
use regex_lite::Regex;
|
||||
use serde::Deserialize;
|
||||
use tokio::process::Command;
|
||||
use tokio::time::timeout;
|
||||
use wildmatch::WildMatch;
|
||||
|
||||
use crate::function_tool::FunctionCallError;
|
||||
use crate::tools::context::FunctionToolOutput;
|
||||
@@ -73,8 +77,6 @@ impl ToolHandler for GrepFilesHandler {
|
||||
let limit = args.limit.min(MAX_LIMIT);
|
||||
let search_path = turn.resolve_path(args.path.clone());
|
||||
|
||||
verify_path_exists(&search_path).await?;
|
||||
|
||||
let include = args.include.as_deref().map(str::trim).and_then(|val| {
|
||||
if val.is_empty() {
|
||||
None
|
||||
@@ -83,8 +85,27 @@ impl ToolHandler for GrepFilesHandler {
|
||||
}
|
||||
});
|
||||
|
||||
let search_results =
|
||||
run_rg_search(pattern, include.as_deref(), &search_path, limit, &turn.cwd).await?;
|
||||
let search_results = if turn.environment.experimental_exec_server_url().is_some() {
|
||||
let absolute_path = AbsolutePathBuf::try_from(search_path.clone()).map_err(|err| {
|
||||
FunctionCallError::RespondToModel(format!(
|
||||
"failed to normalize search path `{}`: {err}",
|
||||
search_path.display()
|
||||
))
|
||||
})?;
|
||||
let filesystem = turn.environment.get_filesystem();
|
||||
verify_path_exists_with_filesystem(&filesystem, &absolute_path).await?;
|
||||
run_search_with_filesystem(
|
||||
pattern,
|
||||
include.as_deref(),
|
||||
&absolute_path,
|
||||
limit,
|
||||
&filesystem,
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
verify_path_exists(&search_path).await?;
|
||||
run_rg_search(pattern, include.as_deref(), &search_path, limit, &turn.cwd).await?
|
||||
};
|
||||
|
||||
if search_results.is_empty() {
|
||||
Ok(FunctionToolOutput::from_text(
|
||||
@@ -107,6 +128,19 @@ async fn verify_path_exists(path: &Path) -> Result<(), FunctionCallError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn verify_path_exists_with_filesystem<F>(
|
||||
filesystem: &F,
|
||||
path: &AbsolutePathBuf,
|
||||
) -> Result<(), FunctionCallError>
|
||||
where
|
||||
F: ExecutorFileSystem + ?Sized,
|
||||
{
|
||||
filesystem.get_metadata(path).await.map_err(|err| {
|
||||
FunctionCallError::RespondToModel(format!("unable to access `{}`: {err}", path.display()))
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run_rg_search(
|
||||
pattern: &str,
|
||||
include: Option<&str>,
|
||||
@@ -152,6 +186,104 @@ async fn run_rg_search(
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_search_with_filesystem<F>(
|
||||
pattern: &str,
|
||||
include: Option<&str>,
|
||||
search_path: &AbsolutePathBuf,
|
||||
limit: usize,
|
||||
filesystem: &F,
|
||||
) -> Result<Vec<String>, FunctionCallError>
|
||||
where
|
||||
F: ExecutorFileSystem + ?Sized,
|
||||
{
|
||||
let regex = Regex::new(pattern).map_err(|err| {
|
||||
FunctionCallError::RespondToModel(format!("invalid regex pattern `{pattern}`: {err}"))
|
||||
})?;
|
||||
let include = include.map(WildMatch::new);
|
||||
let search_metadata = filesystem.get_metadata(search_path).await.map_err(|err| {
|
||||
FunctionCallError::RespondToModel(format!(
|
||||
"unable to access `{}`: {err}",
|
||||
search_path.display()
|
||||
))
|
||||
})?;
|
||||
|
||||
let mut matches = Vec::new();
|
||||
if search_metadata.is_file {
|
||||
if file_matches_regex(filesystem, search_path, include.as_ref(), ®ex).await {
|
||||
let modified_at_ms = filesystem
|
||||
.get_metadata(search_path)
|
||||
.await
|
||||
.map(|metadata| metadata.modified_at_ms)
|
||||
.unwrap_or_default();
|
||||
matches.push((modified_at_ms, search_path.to_string_lossy().to_string()));
|
||||
}
|
||||
} else {
|
||||
let mut pending = vec![search_path.clone()];
|
||||
while let Some(current_dir) = pending.pop() {
|
||||
let entries = match filesystem.read_directory(¤t_dir).await {
|
||||
Ok(entries) => entries,
|
||||
Err(_) => continue,
|
||||
};
|
||||
for entry in entries {
|
||||
let Ok(entry_path) =
|
||||
AbsolutePathBuf::try_from(current_dir.as_path().join(&entry.file_name))
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if entry.is_directory {
|
||||
pending.push(entry_path);
|
||||
continue;
|
||||
}
|
||||
if !entry.is_file {
|
||||
continue;
|
||||
}
|
||||
if !file_matches_regex(filesystem, &entry_path, include.as_ref(), ®ex).await {
|
||||
continue;
|
||||
}
|
||||
let modified_at_ms = filesystem
|
||||
.get_metadata(&entry_path)
|
||||
.await
|
||||
.map(|metadata| metadata.modified_at_ms)
|
||||
.unwrap_or_default();
|
||||
matches.push((modified_at_ms, entry_path.to_string_lossy().to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
matches.sort_unstable_by(|a, b| b.0.cmp(&a.0).then_with(|| a.1.cmp(&b.1)));
|
||||
Ok(matches
|
||||
.into_iter()
|
||||
.take(limit)
|
||||
.map(|(_, path)| path)
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn file_matches_regex<F>(
|
||||
filesystem: &F,
|
||||
path: &AbsolutePathBuf,
|
||||
include: Option<&WildMatch>,
|
||||
regex: &Regex,
|
||||
) -> bool
|
||||
where
|
||||
F: ExecutorFileSystem + ?Sized,
|
||||
{
|
||||
if let Some(glob) = include {
|
||||
let full_path = path.to_string_lossy();
|
||||
let Some(file_name) = path.as_path().file_name().and_then(|name| name.to_str()) else {
|
||||
return false;
|
||||
};
|
||||
if !glob.matches(&full_path) && !glob.matches(file_name) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
let Ok(bytes) = filesystem.read_file(path).await else {
|
||||
return false;
|
||||
};
|
||||
let contents = String::from_utf8_lossy(&bytes);
|
||||
regex.is_match(&contents)
|
||||
}
|
||||
|
||||
fn parse_results(stdout: &[u8], limit: usize) -> Vec<String> {
|
||||
let mut results = Vec::new();
|
||||
for line in stdout.split(|byte| *byte == b'\n') {
|
||||
|
||||
@@ -5,6 +5,8 @@ use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use codex_exec_server::ExecutorFileSystem;
|
||||
use codex_utils_absolute_path::AbsolutePathBuf;
|
||||
use codex_utils_string::take_bytes_at_char_boundary;
|
||||
use serde::Deserialize;
|
||||
use tokio::fs;
|
||||
@@ -54,7 +56,7 @@ impl ToolHandler for ListDirHandler {
|
||||
}
|
||||
|
||||
async fn handle(&self, invocation: ToolInvocation) -> Result<Self::Output, FunctionCallError> {
|
||||
let ToolInvocation { payload, .. } = invocation;
|
||||
let ToolInvocation { payload, turn, .. } = invocation;
|
||||
|
||||
let arguments = match payload {
|
||||
ToolPayload::Function { arguments } => arguments,
|
||||
@@ -99,7 +101,16 @@ impl ToolHandler for ListDirHandler {
|
||||
));
|
||||
}
|
||||
|
||||
let entries = list_dir_slice(&path, offset, limit, depth).await?;
|
||||
let entries = if turn.environment.experimental_exec_server_url().is_some() {
|
||||
let absolute_path = AbsolutePathBuf::try_from(path.clone()).map_err(|err| {
|
||||
FunctionCallError::RespondToModel(format!("failed to normalize dir_path: {err}"))
|
||||
})?;
|
||||
let filesystem = turn.environment.get_filesystem();
|
||||
list_dir_slice_with_filesystem(&filesystem, &absolute_path, offset, limit, depth)
|
||||
.await?
|
||||
} else {
|
||||
list_dir_slice(&path, offset, limit, depth).await?
|
||||
};
|
||||
let mut output = Vec::with_capacity(entries.len() + 1);
|
||||
output.push(format!("Absolute path: {}", path.display()));
|
||||
output.extend(entries);
|
||||
@@ -146,6 +157,49 @@ async fn list_dir_slice(
|
||||
Ok(formatted)
|
||||
}
|
||||
|
||||
async fn list_dir_slice_with_filesystem<F>(
|
||||
filesystem: &F,
|
||||
path: &AbsolutePathBuf,
|
||||
offset: usize,
|
||||
limit: usize,
|
||||
depth: usize,
|
||||
) -> Result<Vec<String>, FunctionCallError>
|
||||
where
|
||||
F: ExecutorFileSystem + ?Sized,
|
||||
{
|
||||
let mut entries = Vec::new();
|
||||
collect_entries_with_filesystem(filesystem, path, Path::new(""), depth, &mut entries).await?;
|
||||
|
||||
if entries.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
entries.sort_unstable_by(|a, b| a.name.cmp(&b.name));
|
||||
|
||||
let start_index = offset - 1;
|
||||
if start_index >= entries.len() {
|
||||
return Err(FunctionCallError::RespondToModel(
|
||||
"offset exceeds directory entry count".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let remaining_entries = entries.len() - start_index;
|
||||
let capped_limit = limit.min(remaining_entries);
|
||||
let end_index = start_index + capped_limit;
|
||||
let selected_entries = &entries[start_index..end_index];
|
||||
let mut formatted = Vec::with_capacity(selected_entries.len());
|
||||
|
||||
for entry in selected_entries {
|
||||
formatted.push(format_entry_line(entry));
|
||||
}
|
||||
|
||||
if end_index < entries.len() {
|
||||
formatted.push(format!("More than {capped_limit} entries found"));
|
||||
}
|
||||
|
||||
Ok(formatted)
|
||||
}
|
||||
|
||||
async fn collect_entries(
|
||||
dir_path: &Path,
|
||||
relative_prefix: &Path,
|
||||
@@ -206,6 +260,73 @@ async fn collect_entries(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn collect_entries_with_filesystem<F>(
|
||||
filesystem: &F,
|
||||
dir_path: &AbsolutePathBuf,
|
||||
relative_prefix: &Path,
|
||||
depth: usize,
|
||||
entries: &mut Vec<DirEntry>,
|
||||
) -> Result<(), FunctionCallError>
|
||||
where
|
||||
F: ExecutorFileSystem + ?Sized,
|
||||
{
|
||||
let mut queue = VecDeque::new();
|
||||
queue.push_back((dir_path.clone(), relative_prefix.to_path_buf(), depth));
|
||||
|
||||
while let Some((current_dir, prefix, remaining_depth)) = queue.pop_front() {
|
||||
let mut dir_entries = filesystem
|
||||
.read_directory(¤t_dir)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
FunctionCallError::RespondToModel(format!("failed to read directory: {err}"))
|
||||
})?
|
||||
.into_iter()
|
||||
.filter_map(|entry| {
|
||||
let relative_path = if prefix.as_os_str().is_empty() {
|
||||
PathBuf::from(&entry.file_name)
|
||||
} else {
|
||||
prefix.join(&entry.file_name)
|
||||
};
|
||||
let display_name = format_entry_component(OsStr::new(&entry.file_name));
|
||||
let display_depth = prefix.components().count();
|
||||
let sort_key = format_entry_name(&relative_path);
|
||||
let kind = if entry.is_directory {
|
||||
DirEntryKind::Directory
|
||||
} else {
|
||||
DirEntryKind::File
|
||||
};
|
||||
let Ok(entry_path) =
|
||||
AbsolutePathBuf::try_from(current_dir.as_path().join(&entry.file_name))
|
||||
else {
|
||||
return None;
|
||||
};
|
||||
Some((
|
||||
entry_path,
|
||||
relative_path,
|
||||
kind,
|
||||
DirEntry {
|
||||
name: sort_key,
|
||||
display_name,
|
||||
depth: display_depth,
|
||||
kind,
|
||||
},
|
||||
))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
dir_entries.sort_unstable_by(|a, b| a.3.name.cmp(&b.3.name));
|
||||
|
||||
for (entry_path, relative_path, kind, dir_entry) in dir_entries {
|
||||
if kind == DirEntryKind::Directory && remaining_depth > 1 {
|
||||
queue.push_back((entry_path, relative_path, remaining_depth - 1));
|
||||
}
|
||||
entries.push(dir_entry);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn format_entry_name(path: &Path) -> String {
|
||||
let normalized = path.to_string_lossy().replace("\\", "/");
|
||||
if normalized.len() > MAX_ENTRY_LENGTH {
|
||||
|
||||
@@ -2,6 +2,7 @@ use std::collections::VecDeque;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use codex_utils_absolute_path::AbsolutePathBuf;
|
||||
use codex_utils_string::take_bytes_at_char_boundary;
|
||||
use serde::Deserialize;
|
||||
|
||||
@@ -100,7 +101,7 @@ impl ToolHandler for ReadFileHandler {
|
||||
}
|
||||
|
||||
async fn handle(&self, invocation: ToolInvocation) -> Result<Self::Output, FunctionCallError> {
|
||||
let ToolInvocation { payload, .. } = invocation;
|
||||
let ToolInvocation { payload, turn, .. } = invocation;
|
||||
|
||||
let arguments = match payload {
|
||||
ToolPayload::Function { arguments } => arguments,
|
||||
@@ -140,11 +141,34 @@ impl ToolHandler for ReadFileHandler {
|
||||
));
|
||||
}
|
||||
|
||||
let collected = match mode {
|
||||
ReadMode::Slice => slice::read(&path, offset, limit).await?,
|
||||
ReadMode::Indentation => {
|
||||
let indentation = indentation.unwrap_or_default();
|
||||
indentation::read_block(&path, offset, limit, indentation).await?
|
||||
let collected = if turn.environment.experimental_exec_server_url().is_some() {
|
||||
let absolute_path = AbsolutePathBuf::try_from(path.clone()).map_err(|err| {
|
||||
FunctionCallError::RespondToModel(format!("failed to normalize file_path: {err}"))
|
||||
})?;
|
||||
let filesystem = turn.environment.get_filesystem();
|
||||
match mode {
|
||||
ReadMode::Slice => {
|
||||
slice::read_with_filesystem(&filesystem, &absolute_path, offset, limit).await?
|
||||
}
|
||||
ReadMode::Indentation => {
|
||||
let indentation = indentation.unwrap_or_default();
|
||||
indentation::read_block_with_filesystem(
|
||||
&filesystem,
|
||||
&absolute_path,
|
||||
offset,
|
||||
limit,
|
||||
indentation,
|
||||
)
|
||||
.await?
|
||||
}
|
||||
}
|
||||
} else {
|
||||
match mode {
|
||||
ReadMode::Slice => slice::read(&path, offset, limit).await?,
|
||||
ReadMode::Indentation => {
|
||||
let indentation = indentation.unwrap_or_default();
|
||||
indentation::read_block(&path, offset, limit, indentation).await?
|
||||
}
|
||||
}
|
||||
};
|
||||
Ok(FunctionToolOutput::from_text(
|
||||
@@ -156,7 +180,10 @@ impl ToolHandler for ReadFileHandler {
|
||||
|
||||
mod slice {
|
||||
use crate::function_tool::FunctionCallError;
|
||||
use crate::tools::handlers::read_file::collect_line_bytes;
|
||||
use crate::tools::handlers::read_file::format_line;
|
||||
use codex_exec_server::ExecutorFileSystem;
|
||||
use codex_utils_absolute_path::AbsolutePathBuf;
|
||||
use std::path::Path;
|
||||
use tokio::fs::File;
|
||||
use tokio::io::AsyncBufReadExt;
|
||||
@@ -219,6 +246,35 @@ mod slice {
|
||||
|
||||
Ok(collected)
|
||||
}
|
||||
|
||||
pub async fn read_with_filesystem<F>(
|
||||
filesystem: &F,
|
||||
path: &AbsolutePathBuf,
|
||||
offset: usize,
|
||||
limit: usize,
|
||||
) -> Result<Vec<String>, FunctionCallError>
|
||||
where
|
||||
F: ExecutorFileSystem + ?Sized,
|
||||
{
|
||||
let bytes = filesystem.read_file(path).await.map_err(|err| {
|
||||
FunctionCallError::RespondToModel(format!("failed to read file: {err}"))
|
||||
})?;
|
||||
|
||||
let lines = collect_line_bytes(&bytes);
|
||||
if lines.len() < offset {
|
||||
return Err(FunctionCallError::RespondToModel(
|
||||
"offset exceeds file length".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
Ok(lines
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.skip(offset.saturating_sub(1))
|
||||
.take(limit)
|
||||
.map(|(idx, line)| format!("L{}: {}", idx + 1, format_line(line)))
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
|
||||
mod indentation {
|
||||
@@ -226,8 +282,11 @@ mod indentation {
|
||||
use crate::tools::handlers::read_file::IndentationArgs;
|
||||
use crate::tools::handlers::read_file::LineRecord;
|
||||
use crate::tools::handlers::read_file::TAB_WIDTH;
|
||||
use crate::tools::handlers::read_file::collect_line_bytes;
|
||||
use crate::tools::handlers::read_file::format_line;
|
||||
use crate::tools::handlers::read_file::trim_empty_lines;
|
||||
use codex_exec_server::ExecutorFileSystem;
|
||||
use codex_utils_absolute_path::AbsolutePathBuf;
|
||||
use std::collections::VecDeque;
|
||||
use std::path::Path;
|
||||
use tokio::fs::File;
|
||||
@@ -366,6 +425,127 @@ mod indentation {
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub async fn read_block_with_filesystem<F>(
|
||||
filesystem: &F,
|
||||
path: &AbsolutePathBuf,
|
||||
offset: usize,
|
||||
limit: usize,
|
||||
options: IndentationArgs,
|
||||
) -> Result<Vec<String>, FunctionCallError>
|
||||
where
|
||||
F: ExecutorFileSystem + ?Sized,
|
||||
{
|
||||
let anchor_line = options.anchor_line.unwrap_or(offset);
|
||||
if anchor_line == 0 {
|
||||
return Err(FunctionCallError::RespondToModel(
|
||||
"anchor_line must be a 1-indexed line number".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let guard_limit = options.max_lines.unwrap_or(limit);
|
||||
if guard_limit == 0 {
|
||||
return Err(FunctionCallError::RespondToModel(
|
||||
"max_lines must be greater than zero".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let collected = collect_file_lines_with_filesystem(filesystem, path).await?;
|
||||
if collected.is_empty() || anchor_line > collected.len() {
|
||||
return Err(FunctionCallError::RespondToModel(
|
||||
"anchor_line exceeds file length".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let anchor_index = anchor_line - 1;
|
||||
let effective_indents = compute_effective_indents(&collected);
|
||||
let anchor_indent = effective_indents[anchor_index];
|
||||
let min_indent = if options.max_levels == 0 {
|
||||
0
|
||||
} else {
|
||||
anchor_indent.saturating_sub(options.max_levels * TAB_WIDTH)
|
||||
};
|
||||
let final_limit = limit.min(guard_limit).min(collected.len());
|
||||
|
||||
if final_limit == 1 {
|
||||
return Ok(vec![format!(
|
||||
"L{}: {}",
|
||||
collected[anchor_index].number, collected[anchor_index].display
|
||||
)]);
|
||||
}
|
||||
|
||||
let mut i: isize = anchor_index as isize - 1;
|
||||
let mut j: usize = anchor_index + 1;
|
||||
let mut i_counter_min_indent = 0;
|
||||
let mut j_counter_min_indent = 0;
|
||||
|
||||
let mut out = VecDeque::with_capacity(limit);
|
||||
out.push_back(&collected[anchor_index]);
|
||||
|
||||
while out.len() < final_limit {
|
||||
let mut progressed = 0;
|
||||
|
||||
if i >= 0 {
|
||||
let iu = i as usize;
|
||||
if effective_indents[iu] >= min_indent {
|
||||
out.push_front(&collected[iu]);
|
||||
progressed += 1;
|
||||
i -= 1;
|
||||
|
||||
if effective_indents[iu] == min_indent && !options.include_siblings {
|
||||
let allow_header_comment =
|
||||
options.include_header && collected[iu].is_comment();
|
||||
let can_take_line = allow_header_comment || i_counter_min_indent == 0;
|
||||
|
||||
if can_take_line {
|
||||
i_counter_min_indent += 1;
|
||||
} else {
|
||||
out.pop_front();
|
||||
progressed -= 1;
|
||||
i = -1;
|
||||
}
|
||||
}
|
||||
|
||||
if out.len() >= final_limit {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
i = -1;
|
||||
}
|
||||
}
|
||||
|
||||
if j < collected.len() {
|
||||
let ju = j;
|
||||
if effective_indents[ju] >= min_indent {
|
||||
out.push_back(&collected[ju]);
|
||||
progressed += 1;
|
||||
j += 1;
|
||||
|
||||
if effective_indents[ju] == min_indent && !options.include_siblings {
|
||||
if j_counter_min_indent > 0 {
|
||||
out.pop_back();
|
||||
progressed -= 1;
|
||||
j = collected.len();
|
||||
}
|
||||
j_counter_min_indent += 1;
|
||||
}
|
||||
} else {
|
||||
j = collected.len();
|
||||
}
|
||||
}
|
||||
|
||||
if progressed == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
trim_empty_lines(&mut out);
|
||||
|
||||
Ok(out
|
||||
.into_iter()
|
||||
.map(|record| format!("L{}: {}", record.number, record.display))
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn collect_file_lines(path: &Path) -> Result<Vec<LineRecord>, FunctionCallError> {
|
||||
let file = File::open(path).await.map_err(|err| {
|
||||
FunctionCallError::RespondToModel(format!("failed to read file: {err}"))
|
||||
@@ -408,6 +588,34 @@ mod indentation {
|
||||
Ok(lines)
|
||||
}
|
||||
|
||||
async fn collect_file_lines_with_filesystem<F>(
|
||||
filesystem: &F,
|
||||
path: &AbsolutePathBuf,
|
||||
) -> Result<Vec<LineRecord>, FunctionCallError>
|
||||
where
|
||||
F: ExecutorFileSystem + ?Sized,
|
||||
{
|
||||
let bytes = filesystem.read_file(path).await.map_err(|err| {
|
||||
FunctionCallError::RespondToModel(format!("failed to read file: {err}"))
|
||||
})?;
|
||||
|
||||
Ok(collect_line_bytes(&bytes)
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(idx, line)| {
|
||||
let raw = String::from_utf8_lossy(line).into_owned();
|
||||
let indent = measure_indent(&raw);
|
||||
let display = format_line(line);
|
||||
LineRecord {
|
||||
number: idx + 1,
|
||||
raw,
|
||||
display,
|
||||
indent,
|
||||
}
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn compute_effective_indents(records: &[LineRecord]) -> Vec<usize> {
|
||||
let mut effective = Vec::with_capacity(records.len());
|
||||
let mut previous_indent = 0usize;
|
||||
@@ -439,6 +647,30 @@ fn format_line(bytes: &[u8]) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
fn collect_line_bytes(bytes: &[u8]) -> Vec<&[u8]> {
|
||||
let mut lines = Vec::new();
|
||||
let mut start = 0usize;
|
||||
|
||||
for idx in 0..=bytes.len() {
|
||||
if idx != bytes.len() && bytes[idx] != b'\n' {
|
||||
continue;
|
||||
}
|
||||
|
||||
if idx == bytes.len() && start == bytes.len() {
|
||||
break;
|
||||
}
|
||||
|
||||
let mut line = &bytes[start..idx];
|
||||
if line.last() == Some(&b'\r') {
|
||||
line = &line[..line.len() - 1];
|
||||
}
|
||||
lines.push(line);
|
||||
start = idx.saturating_add(1);
|
||||
}
|
||||
|
||||
lines
|
||||
}
|
||||
|
||||
fn trim_empty_lines(out: &mut VecDeque<&LineRecord>) {
|
||||
while matches!(out.front(), Some(line) if line.raw.trim().is_empty()) {
|
||||
out.pop_front();
|
||||
|
||||
Reference in New Issue
Block a user