mirror of
https://github.com/openai/codex.git
synced 2026-09-05 15:18:41 +00:00
## Why `fs/readFile` buffers the entire file in one response, which makes large remote reads expensive and prevents callers from applying backpressure. We need an opt-in streaming path with bounded block sizes while preserving the existing single-call API for small and sandboxed reads. ## What changed - Add `ExecServerClient::stream`, returning a named `FileReadStream` that implements `futures::Stream` and yields immutable 1 MiB byte blocks. - Add internal `fs/open`, `fs/readBlock`, and `fs/close` RPCs. `fs/readBlock` accepts an explicit offset and length. - Keep unsandboxed files open between block reads, cap open handles per connection, and clean them up on EOF, error, stream drop, explicit close, or connection shutdown. - Reject platform-sandboxed streaming opens instead of turning the one-shot sandbox helper into a persistent server. Existing `fs/readFile` behavior is unchanged. ## Testing - `just test -p codex-exec-server` - Integration coverage for 1 MiB chunking, exact block-boundary EOF, sandbox rejection, and continued reads from the opened file after path replacement. - Handle-manager coverage for non-sequential offsets, variable block lengths, the 128-handle limit, and capacity release after close.
257 lines
7.7 KiB
Rust
257 lines
7.7 KiB
Rust
use super::*;
|
|
use codex_file_system::CopyOptions;
|
|
use codex_file_system::CreateDirectoryOptions;
|
|
use codex_file_system::ExecutorFileSystemFuture;
|
|
use codex_file_system::FileMetadata;
|
|
use codex_file_system::FileSystemReadStream;
|
|
use codex_file_system::FileSystemSandboxContext;
|
|
use codex_file_system::ReadDirectoryEntry;
|
|
use codex_file_system::RemoveOptions;
|
|
use codex_utils_path_uri::PathUri;
|
|
use pretty_assertions::assert_eq;
|
|
use tempfile::tempdir;
|
|
|
|
struct TestFileSystem;
|
|
|
|
impl ExecutorFileSystem for TestFileSystem {
|
|
fn canonicalize<'a>(
|
|
&'a self,
|
|
path: &'a PathUri,
|
|
_sandbox: Option<&'a FileSystemSandboxContext>,
|
|
) -> ExecutorFileSystemFuture<'a, PathUri> {
|
|
Box::pin(async move {
|
|
let path = path.to_abs_path()?;
|
|
let canonicalized = path.canonicalize()?;
|
|
Ok(PathUri::from_abs_path(&canonicalized))
|
|
})
|
|
}
|
|
|
|
fn read_file<'a>(
|
|
&'a self,
|
|
path: &'a PathUri,
|
|
_sandbox: Option<&'a FileSystemSandboxContext>,
|
|
) -> ExecutorFileSystemFuture<'a, Vec<u8>> {
|
|
Box::pin(async move {
|
|
let path = path.to_abs_path()?;
|
|
tokio::fs::read(path.as_path()).await
|
|
})
|
|
}
|
|
|
|
fn read_file_stream<'a>(
|
|
&'a self,
|
|
_path: &'a PathUri,
|
|
_sandbox: Option<&'a FileSystemSandboxContext>,
|
|
) -> ExecutorFileSystemFuture<'a, FileSystemReadStream> {
|
|
Box::pin(async {
|
|
Err(std::io::Error::new(
|
|
std::io::ErrorKind::Unsupported,
|
|
"test filesystem does not support streaming reads",
|
|
))
|
|
})
|
|
}
|
|
|
|
fn write_file<'a>(
|
|
&'a self,
|
|
_path: &'a PathUri,
|
|
_contents: Vec<u8>,
|
|
_sandbox: Option<&'a FileSystemSandboxContext>,
|
|
) -> ExecutorFileSystemFuture<'a, ()> {
|
|
Box::pin(async move { unimplemented!("test filesystem only supports reads") })
|
|
}
|
|
|
|
fn create_directory<'a>(
|
|
&'a self,
|
|
_path: &'a PathUri,
|
|
_create_directory_options: CreateDirectoryOptions,
|
|
_sandbox: Option<&'a FileSystemSandboxContext>,
|
|
) -> ExecutorFileSystemFuture<'a, ()> {
|
|
Box::pin(async move { unimplemented!("test filesystem only supports reads") })
|
|
}
|
|
|
|
fn get_metadata<'a>(
|
|
&'a self,
|
|
_path: &'a PathUri,
|
|
_sandbox: Option<&'a FileSystemSandboxContext>,
|
|
) -> ExecutorFileSystemFuture<'a, FileMetadata> {
|
|
Box::pin(async move { unimplemented!("test filesystem only supports reads") })
|
|
}
|
|
|
|
fn read_directory<'a>(
|
|
&'a self,
|
|
_path: &'a PathUri,
|
|
_sandbox: Option<&'a FileSystemSandboxContext>,
|
|
) -> ExecutorFileSystemFuture<'a, Vec<ReadDirectoryEntry>> {
|
|
Box::pin(async move { unimplemented!("test filesystem only supports reads") })
|
|
}
|
|
|
|
fn remove<'a>(
|
|
&'a self,
|
|
_path: &'a PathUri,
|
|
_remove_options: RemoveOptions,
|
|
_sandbox: Option<&'a FileSystemSandboxContext>,
|
|
) -> ExecutorFileSystemFuture<'a, ()> {
|
|
Box::pin(async move { unimplemented!("test filesystem only supports reads") })
|
|
}
|
|
|
|
fn copy<'a>(
|
|
&'a self,
|
|
_source_path: &'a PathUri,
|
|
_destination_path: &'a PathUri,
|
|
_copy_options: CopyOptions,
|
|
_sandbox: Option<&'a FileSystemSandboxContext>,
|
|
) -> ExecutorFileSystemFuture<'a, ()> {
|
|
Box::pin(async move { unimplemented!("test filesystem only supports reads") })
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn profile_v2_rejects_matching_legacy_profile_in_base_user_config() {
|
|
let tmp = tempdir().expect("tempdir");
|
|
let selected_config = tmp.path().join("work.config.toml");
|
|
|
|
std::fs::write(
|
|
tmp.path().join(CONFIG_TOML_FILE),
|
|
r#"
|
|
model = "gpt-main"
|
|
|
|
[profiles.work]
|
|
model = "gpt-work"
|
|
"#,
|
|
)
|
|
.expect("write default user config");
|
|
std::fs::write(&selected_config, r#"model = "gpt-work-v2""#)
|
|
.expect("write selected user config");
|
|
|
|
let mut overrides = LoaderOverrides::without_managed_config_for_tests();
|
|
overrides.user_config_path = Some(AbsolutePathBuf::resolve_path_against_base(
|
|
"work.config.toml",
|
|
tmp.path(),
|
|
));
|
|
overrides.user_config_profile = Some("work".parse().expect("profile-v2 name"));
|
|
|
|
let err = load_config_layers_state(
|
|
&TestFileSystem,
|
|
tmp.path(),
|
|
/*cwd*/ None,
|
|
&[],
|
|
overrides,
|
|
&crate::NoopThreadConfigLoader,
|
|
)
|
|
.await
|
|
.expect_err("profile-v2 should reject a matching legacy profile in base user config");
|
|
|
|
assert_eq!(
|
|
err.kind(),
|
|
io::ErrorKind::InvalidData,
|
|
"a matching legacy profile should be a hard config error"
|
|
);
|
|
let message = err.to_string();
|
|
assert!(
|
|
message.contains("--profile `work` cannot be used"),
|
|
"unexpected error message: {message}"
|
|
);
|
|
assert!(
|
|
message.contains("config.toml"),
|
|
"unexpected error message: {message}"
|
|
);
|
|
assert!(
|
|
message.contains("[profiles.work]"),
|
|
"unexpected error message: {message}"
|
|
);
|
|
assert!(
|
|
message.contains("https://developers.openai.com/codex/config-advanced#profiles"),
|
|
"unexpected error message: {message}"
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn profile_v2_rejects_matching_legacy_profile_selector_in_base_user_config() {
|
|
let tmp = tempdir().expect("tempdir");
|
|
let selected_config = tmp.path().join("work.config.toml");
|
|
|
|
std::fs::write(
|
|
tmp.path().join(CONFIG_TOML_FILE),
|
|
r#"
|
|
profile = "work"
|
|
model = "gpt-main"
|
|
"#,
|
|
)
|
|
.expect("write default user config");
|
|
std::fs::write(&selected_config, r#"model = "gpt-work-v2""#)
|
|
.expect("write selected user config");
|
|
|
|
let mut overrides = LoaderOverrides::without_managed_config_for_tests();
|
|
overrides.user_config_path = Some(AbsolutePathBuf::resolve_path_against_base(
|
|
"work.config.toml",
|
|
tmp.path(),
|
|
));
|
|
overrides.user_config_profile = Some("work".parse().expect("profile-v2 name"));
|
|
|
|
let err = load_config_layers_state(
|
|
&TestFileSystem,
|
|
tmp.path(),
|
|
/*cwd*/ None,
|
|
&[],
|
|
overrides,
|
|
&crate::NoopThreadConfigLoader,
|
|
)
|
|
.await
|
|
.expect_err("profile-v2 should reject a matching legacy profile selector");
|
|
|
|
assert_eq!(
|
|
err.kind(),
|
|
io::ErrorKind::InvalidData,
|
|
"a matching legacy profile selector should be a hard config error"
|
|
);
|
|
let message = err.to_string();
|
|
assert!(
|
|
message.contains("--profile `work` cannot be used"),
|
|
"unexpected error message: {message}"
|
|
);
|
|
assert!(
|
|
message.contains("profile = \"work\""),
|
|
"unexpected error message: {message}"
|
|
);
|
|
assert!(
|
|
message.contains("work.config.toml"),
|
|
"unexpected error message: {message}"
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn profile_v2_allows_unrelated_legacy_profiles_in_base_user_config() {
|
|
let tmp = tempdir().expect("tempdir");
|
|
let selected_config = tmp.path().join("work.config.toml");
|
|
|
|
std::fs::write(
|
|
tmp.path().join(CONFIG_TOML_FILE),
|
|
r#"
|
|
model = "gpt-main"
|
|
|
|
[profiles.dev]
|
|
model = "gpt-dev"
|
|
"#,
|
|
)
|
|
.expect("write default user config");
|
|
std::fs::write(&selected_config, r#"model = "gpt-work-v2""#)
|
|
.expect("write selected user config");
|
|
|
|
let mut overrides = LoaderOverrides::without_managed_config_for_tests();
|
|
overrides.user_config_path = Some(AbsolutePathBuf::resolve_path_against_base(
|
|
"work.config.toml",
|
|
tmp.path(),
|
|
));
|
|
overrides.user_config_profile = Some("work".parse().expect("profile-v2 name"));
|
|
|
|
load_config_layers_state(
|
|
&TestFileSystem,
|
|
tmp.path(),
|
|
/*cwd*/ None,
|
|
&[],
|
|
overrides,
|
|
&crate::NoopThreadConfigLoader,
|
|
)
|
|
.await
|
|
.expect("profile-v2 should allow unrelated legacy profiles in base user config");
|
|
}
|