Files
codex/codex-rs/config/src/loader/tests.rs
pakrym-oai a4711b88dd [codex] exec-server: stream files in chunks (#28354)
## Why

`fs/readFile` buffers the entire file in one response, which makes large
remote reads expensive and prevents callers from applying backpressure.
We need an opt-in streaming path with bounded block sizes while
preserving the existing single-call API for small and sandboxed reads.

## What changed

- Add `ExecServerClient::stream`, returning a named `FileReadStream`
that implements `futures::Stream` and yields immutable 1 MiB byte
blocks.
- Add internal `fs/open`, `fs/readBlock`, and `fs/close` RPCs.
`fs/readBlock` accepts an explicit offset and length.
- Keep unsandboxed files open between block reads, cap open handles per
connection, and clean them up on EOF, error, stream drop, explicit
close, or connection shutdown.
- Reject platform-sandboxed streaming opens instead of turning the
one-shot sandbox helper into a persistent server. Existing `fs/readFile`
behavior is unchanged.

## Testing

- `just test -p codex-exec-server`
- Integration coverage for 1 MiB chunking, exact block-boundary EOF,
sandbox rejection, and continued reads from the opened file after path
replacement.
- Handle-manager coverage for non-sequential offsets, variable block
lengths, the 128-handle limit, and capacity release after close.
2026-06-16 09:50:55 -07:00

257 lines
7.7 KiB
Rust

use super::*;
use codex_file_system::CopyOptions;
use codex_file_system::CreateDirectoryOptions;
use codex_file_system::ExecutorFileSystemFuture;
use codex_file_system::FileMetadata;
use codex_file_system::FileSystemReadStream;
use codex_file_system::FileSystemSandboxContext;
use codex_file_system::ReadDirectoryEntry;
use codex_file_system::RemoveOptions;
use codex_utils_path_uri::PathUri;
use pretty_assertions::assert_eq;
use tempfile::tempdir;
struct TestFileSystem;
impl ExecutorFileSystem for TestFileSystem {
fn canonicalize<'a>(
&'a self,
path: &'a PathUri,
_sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, PathUri> {
Box::pin(async move {
let path = path.to_abs_path()?;
let canonicalized = path.canonicalize()?;
Ok(PathUri::from_abs_path(&canonicalized))
})
}
fn read_file<'a>(
&'a self,
path: &'a PathUri,
_sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, Vec<u8>> {
Box::pin(async move {
let path = path.to_abs_path()?;
tokio::fs::read(path.as_path()).await
})
}
fn read_file_stream<'a>(
&'a self,
_path: &'a PathUri,
_sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, FileSystemReadStream> {
Box::pin(async {
Err(std::io::Error::new(
std::io::ErrorKind::Unsupported,
"test filesystem does not support streaming reads",
))
})
}
fn write_file<'a>(
&'a self,
_path: &'a PathUri,
_contents: Vec<u8>,
_sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, ()> {
Box::pin(async move { unimplemented!("test filesystem only supports reads") })
}
fn create_directory<'a>(
&'a self,
_path: &'a PathUri,
_create_directory_options: CreateDirectoryOptions,
_sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, ()> {
Box::pin(async move { unimplemented!("test filesystem only supports reads") })
}
fn get_metadata<'a>(
&'a self,
_path: &'a PathUri,
_sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, FileMetadata> {
Box::pin(async move { unimplemented!("test filesystem only supports reads") })
}
fn read_directory<'a>(
&'a self,
_path: &'a PathUri,
_sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, Vec<ReadDirectoryEntry>> {
Box::pin(async move { unimplemented!("test filesystem only supports reads") })
}
fn remove<'a>(
&'a self,
_path: &'a PathUri,
_remove_options: RemoveOptions,
_sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, ()> {
Box::pin(async move { unimplemented!("test filesystem only supports reads") })
}
fn copy<'a>(
&'a self,
_source_path: &'a PathUri,
_destination_path: &'a PathUri,
_copy_options: CopyOptions,
_sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, ()> {
Box::pin(async move { unimplemented!("test filesystem only supports reads") })
}
}
#[tokio::test]
async fn profile_v2_rejects_matching_legacy_profile_in_base_user_config() {
let tmp = tempdir().expect("tempdir");
let selected_config = tmp.path().join("work.config.toml");
std::fs::write(
tmp.path().join(CONFIG_TOML_FILE),
r#"
model = "gpt-main"
[profiles.work]
model = "gpt-work"
"#,
)
.expect("write default user config");
std::fs::write(&selected_config, r#"model = "gpt-work-v2""#)
.expect("write selected user config");
let mut overrides = LoaderOverrides::without_managed_config_for_tests();
overrides.user_config_path = Some(AbsolutePathBuf::resolve_path_against_base(
"work.config.toml",
tmp.path(),
));
overrides.user_config_profile = Some("work".parse().expect("profile-v2 name"));
let err = load_config_layers_state(
&TestFileSystem,
tmp.path(),
/*cwd*/ None,
&[],
overrides,
&crate::NoopThreadConfigLoader,
)
.await
.expect_err("profile-v2 should reject a matching legacy profile in base user config");
assert_eq!(
err.kind(),
io::ErrorKind::InvalidData,
"a matching legacy profile should be a hard config error"
);
let message = err.to_string();
assert!(
message.contains("--profile `work` cannot be used"),
"unexpected error message: {message}"
);
assert!(
message.contains("config.toml"),
"unexpected error message: {message}"
);
assert!(
message.contains("[profiles.work]"),
"unexpected error message: {message}"
);
assert!(
message.contains("https://developers.openai.com/codex/config-advanced#profiles"),
"unexpected error message: {message}"
);
}
#[tokio::test]
async fn profile_v2_rejects_matching_legacy_profile_selector_in_base_user_config() {
let tmp = tempdir().expect("tempdir");
let selected_config = tmp.path().join("work.config.toml");
std::fs::write(
tmp.path().join(CONFIG_TOML_FILE),
r#"
profile = "work"
model = "gpt-main"
"#,
)
.expect("write default user config");
std::fs::write(&selected_config, r#"model = "gpt-work-v2""#)
.expect("write selected user config");
let mut overrides = LoaderOverrides::without_managed_config_for_tests();
overrides.user_config_path = Some(AbsolutePathBuf::resolve_path_against_base(
"work.config.toml",
tmp.path(),
));
overrides.user_config_profile = Some("work".parse().expect("profile-v2 name"));
let err = load_config_layers_state(
&TestFileSystem,
tmp.path(),
/*cwd*/ None,
&[],
overrides,
&crate::NoopThreadConfigLoader,
)
.await
.expect_err("profile-v2 should reject a matching legacy profile selector");
assert_eq!(
err.kind(),
io::ErrorKind::InvalidData,
"a matching legacy profile selector should be a hard config error"
);
let message = err.to_string();
assert!(
message.contains("--profile `work` cannot be used"),
"unexpected error message: {message}"
);
assert!(
message.contains("profile = \"work\""),
"unexpected error message: {message}"
);
assert!(
message.contains("work.config.toml"),
"unexpected error message: {message}"
);
}
#[tokio::test]
async fn profile_v2_allows_unrelated_legacy_profiles_in_base_user_config() {
let tmp = tempdir().expect("tempdir");
let selected_config = tmp.path().join("work.config.toml");
std::fs::write(
tmp.path().join(CONFIG_TOML_FILE),
r#"
model = "gpt-main"
[profiles.dev]
model = "gpt-dev"
"#,
)
.expect("write default user config");
std::fs::write(&selected_config, r#"model = "gpt-work-v2""#)
.expect("write selected user config");
let mut overrides = LoaderOverrides::without_managed_config_for_tests();
overrides.user_config_path = Some(AbsolutePathBuf::resolve_path_against_base(
"work.config.toml",
tmp.path(),
));
overrides.user_config_profile = Some("work".parse().expect("profile-v2 name"));
load_config_layers_state(
&TestFileSystem,
tmp.path(),
/*cwd*/ None,
&[],
overrides,
&crate::NoopThreadConfigLoader,
)
.await
.expect("profile-v2 should allow unrelated legacy profiles in base user config");
}