Add streaming file replacement to filesystem abstractions (#54505)

## What changed

Add `write_file_stream` to `ExecutorFileSystem` and `EnvironmentAccess`, preserving the accessor's captured sandbox permissions. Local and streaming-capable remote filesystems replace files one bounded chunk at a time and return the byte count. Older executors buffer the input before sending a single `writeFile` request.

Close remote handles on completion or failure, perform best-effort cleanup after cancellation, and invalidate pending metadata lookups. Document that failure or cancellation can leave a truncated or partial file and that in-flight I/O may outlive cancellation.

## Testing

Add coverage for sequential writes, binary and empty files across local and remote filesystems, source failures, handle cleanup, metadata invalidation after cancellation, and compatibility with older executors.

GitOrigin-RevId: d3eb44e0ac2f0f336e9e157d686464f4d45a3f26
This commit is contained in:
Adam Perry @ OpenAI
2026-10-16 04:42:32 +00:00
committed by copyberry
parent e790ce2717
commit a45b8ccb01
6 changed files with 505 additions and 28 deletions
+151 -27
View File
@@ -1,5 +1,6 @@
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::future::Future;
use std::sync::Arc;
use std::sync::Mutex as StdMutex;
use std::sync::OnceLock;
@@ -969,21 +970,32 @@ impl ExecServerClient {
.await
}
pub async fn fs_open(&self, params: FsOpenParams) -> Result<FsOpenResponse, ExecServerError> {
// Older executors ignore the mode field and could silently return a read-only handle.
if params.mode == FsOpenMode::Replace
&& !self
.environment_info()
.await?
.capabilities
.file_write_streaming
{
return Err(ExecServerError::Protocol(
"exec-server does not support writable file streams".to_string(),
));
pub(crate) async fn supports_file_write_streaming(&self) -> Result<bool, ExecServerError> {
Ok(self
.environment_info()
.await?
.capabilities
.file_write_streaming)
}
/// Returns an owned future so callers can finish an open after they stop waiting for it.
pub fn fs_open(
&self,
params: FsOpenParams,
) -> impl Future<Output = Result<FsOpenResponse, ExecServerError>> + Send + 'static {
let client = self.clone();
async move {
// Older executors ignore the mode field and could silently return a read-only handle.
if params.mode == FsOpenMode::Replace && !client.supports_file_write_streaming().await?
{
return Err(ExecServerError::Protocol(
"exec-server does not support writable file streams".to_string(),
));
}
client
.call(FS_OPEN_METHOD, &WireFsOpenParams::from(params))
.await
}
self.call(FS_OPEN_METHOD, &WireFsOpenParams::from(params))
.await
}
pub async fn fs_read_block(
@@ -997,12 +1009,7 @@ impl ExecServerClient {
&self,
params: FsWriteBlockParams,
) -> Result<FsWriteBlockResponse, ExecServerError> {
if !self
.environment_info()
.await?
.capabilities
.file_write_streaming
{
if !self.supports_file_write_streaming().await? {
return Err(ExecServerError::Protocol(
"exec-server does not support writable file streams".to_string(),
));
@@ -1958,6 +1965,7 @@ mod tests {
use codex_exec_server_protocol::JSONRPCResponse;
use codex_http_client::HttpClientFactory;
use codex_http_client::OutboundProxyPolicy;
use codex_protocol::models::PermissionProfile;
use codex_utils_path_uri::PathUri;
use futures::SinkExt;
use futures::StreamExt;
@@ -1996,8 +2004,12 @@ mod tests {
use super::ExecServerClientConnectOptions;
use super::ExecServerError;
use super::LazyRemoteExecServerClient;
use crate::Environment;
use crate::EnvironmentObservedStatus;
use crate::ExecutorFileSystem;
use crate::FileSystemSandboxContext;
use crate::ProcessId;
use crate::client_api::DEFAULT_REMOTE_EXEC_SERVER_CONNECT_TIMEOUT;
#[cfg(not(windows))]
use crate::client_api::DEFAULT_REMOTE_EXEC_SERVER_INITIALIZE_TIMEOUT;
use crate::client_api::ExecServerTransportParams;
@@ -2019,19 +2031,24 @@ mod tests {
use crate::protocol::ExecOutputStream;
use crate::protocol::ExecParams;
use crate::protocol::ExecResponse;
use crate::protocol::FS_CLOSE_METHOD;
use crate::protocol::FS_OPEN_METHOD;
use crate::protocol::FS_WRITE_FILE_METHOD;
use crate::protocol::FsOpenMode;
use crate::protocol::FsOpenParams;
use crate::protocol::FsWriteBlockParams;
use crate::protocol::FsWriteFileParams;
use crate::protocol::INITIALIZE_METHOD;
use crate::protocol::INITIALIZED_METHOD;
use crate::protocol::InitializeResponse;
use crate::protocol::ProcessOutputChunk;
use crate::protocol::ProcessSandboxType;
use crate::protocol::ReadResponse;
use crate::protocol::WireFsWriteFileParams;
use crate::protocol::WriteParams;
use crate::protocol::WriteResponse;
use crate::protocol::WriteStatus;
use crate::remote_file_system::RemoteFileSystem;
async fn read_jsonrpc_line<R>(lines: &mut tokio::io::Lines<BufReader<R>>) -> JSONRPCMessage
where
@@ -2758,9 +2775,10 @@ mod tests {
Ok(())
}
/// Old executors must never receive writable opens they would silently treat as reads.
/// Old executors receive one sandboxed writeFile, never an open they would mistake for a read.
#[tokio::test]
async fn writable_file_streams_require_executor_capability() -> anyhow::Result<()> {
async fn old_executor_receives_buffered_replacement_and_keeps_read_only_opens()
-> anyhow::Result<()> {
let listener = TcpListener::bind("127.0.0.1:0").await?;
let websocket_url = format!("ws://{}", listener.local_addr()?);
let read_open = FsOpenParams {
@@ -2769,6 +2787,16 @@ mod tests {
mode: FsOpenMode::Read,
sandbox: None,
};
let sandbox = FileSystemSandboxContext::from_permission_profile(
PermissionProfile::workspace_write(),
PathUri::parse("file:///tmp")?,
);
let expected_write = WireFsWriteFileParams::from(FsWriteFileParams {
path: read_open.path.clone(),
data_base64: "AP9oaQ==".to_string(),
follow_symlinks: None,
sandbox: Some(sandbox.clone()),
});
let server = tokio::spawn(async move {
let mut websocket = accept_websocket(&listener).await;
let mut info = EnvironmentInfo::local();
@@ -2780,6 +2808,23 @@ mod tests {
Some(info),
)
.await;
let JSONRPCMessage::Request(write) = read_jsonrpc_websocket(&mut websocket).await
else {
panic!("expected buffered write");
};
assert_eq!(write.method, FS_WRITE_FILE_METHOD);
assert_eq!(
write.params,
Some(serde_json::to_value(expected_write).expect("serialize legacy write"))
);
write_jsonrpc_websocket(
&mut websocket,
JSONRPCMessage::Response(JSONRPCResponse {
id: write.id,
result: serde_json::json!({}),
}),
)
.await;
let JSONRPCMessage::Request(request) = read_jsonrpc_websocket(&mut websocket).await
else {
panic!("expected read-only open");
@@ -2803,12 +2848,32 @@ mod tests {
)
.await;
});
let client = ExecServerClient::connect_websocket(RemoteExecServerConnectArgs::new(
websocket_url,
"file-stream-test".to_string(),
let remote = LazyRemoteExecServerClient::new(
ExecServerTransportParams::websocket_url(
websocket_url,
DEFAULT_REMOTE_EXEC_SERVER_CONNECT_TIMEOUT,
),
HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
))
.await?;
);
let client = remote.get().await?;
let fs = RemoteFileSystem::new(remote);
// A source error precedes any legacy write, so the server must see only the complete file.
let broken = futures::stream::iter([
Ok(bytes::Bytes::from_static(b"partial")),
Err(std::io::Error::from(std::io::ErrorKind::UnexpectedEof)),
]);
let error = fs
.write_file_stream(&read_open.path, broken.boxed(), Some(&sandbox))
.await
.expect_err("source error");
assert_eq!(error.kind(), std::io::ErrorKind::UnexpectedEof);
let contents = futures::stream::iter([b"\x00\xff", b"hi"])
.map(|chunk| Ok(bytes::Bytes::from_static(chunk)));
let written = fs
.write_file_stream(&read_open.path, contents.boxed(), Some(&sandbox))
.await?;
assert_eq!(written, 4);
let error = client
.fs_open(FsOpenParams {
mode: FsOpenMode::Replace,
@@ -2826,11 +2891,70 @@ mod tests {
.await
.unwrap_err();
assert!(matches!(error, ExecServerError::Protocol(_)));
// This read-only open must be the next request; there is no stream handle to close.
client.fs_open(read_open).await?;
server.await?;
Ok(())
}
/// An open that completes after cancellation must close that exact handle.
#[tokio::test]
async fn cancelled_streaming_replacement_closes_pending_open() -> anyhow::Result<()> {
let listener = TcpListener::bind("127.0.0.1:0").await?;
let websocket_url = format!("ws://{}", listener.local_addr()?);
let fs = Environment::create_for_tests(Some(websocket_url))?.get_filesystem();
let path = PathUri::parse("file:///tmp/cancelled.bin")?;
let contents = futures::stream::poll_fn(|_| panic!("cancelled write polled its source"));
let mut write = fs.write_file_stream(&path, contents.boxed(), /*sandbox*/ None);
let receive_open = async {
let mut ws = accept_websocket(&listener).await;
complete_websocket_initialize_with_environment_info(
&mut ws,
"session-1",
/*expected_resume_session_id*/ None,
Some(EnvironmentInfo::local()),
)
.await;
let JSONRPCMessage::Request(open) = read_jsonrpc_websocket(&mut ws).await else {
panic!("expected fs/open");
};
assert_eq!(open.method, FS_OPEN_METHOD);
(ws, open)
};
let (mut ws, open) = tokio::select! {
open = receive_open => open,
result = &mut write => panic!("write must wait for open: {result:?}"),
};
drop(write);
let params = open.params.expect("fs/open params");
let handle = params["handleId"].as_str().expect("handleId");
let handle = serde_json::json!({"handleId": handle});
write_jsonrpc_websocket(
&mut ws,
JSONRPCMessage::Response(JSONRPCResponse {
id: open.id,
result: handle.clone(),
}),
)
.await;
let JSONRPCMessage::Request(close) = read_jsonrpc_websocket(&mut ws).await else {
panic!("expected fs/close");
};
assert_eq!(close.method, FS_CLOSE_METHOD);
assert_eq!(close.params, Some(handle));
write_jsonrpc_websocket(
&mut ws,
JSONRPCMessage::Response(JSONRPCResponse {
id: close.id,
result: serde_json::json!({}),
}),
)
.await;
Ok(())
}
#[test_case::test_case(false; "socket_closed")]
#[test_case::test_case(true; "health_check_timed_out")]
#[tokio::test]
@@ -1,3 +1,5 @@
use bytes::Bytes;
use codex_file_system::FILE_WRITE_CHUNK_SIZE;
use codex_file_system::MAX_WALK_DEPTH;
use codex_file_system::MAX_WALK_DIRECTORIES;
use codex_file_system::MAX_WALK_ENTRIES;
@@ -5,6 +7,8 @@ use codex_file_system::MAX_WALK_RESPONSE_BYTES;
use codex_file_system::WALK_RESPONSE_ITEM_OVERHEAD_BYTES;
use codex_utils_absolute_path::AbsolutePathBuf;
use codex_utils_path_uri::PathUri;
use futures::StreamExt;
use futures::stream::BoxStream;
use std::collections::HashSet;
use std::collections::VecDeque;
use std::path::Path;
@@ -14,6 +18,7 @@ use std::sync::LazyLock;
use std::time::SystemTime;
use std::time::UNIX_EPOCH;
use tokio::io;
use tokio::io::AsyncWriteExt;
use tokio_util::io::ReaderStream;
use tokio_util::sync::CancellationToken;
@@ -253,6 +258,35 @@ impl LocalFileSystem {
}
impl ExecutorFileSystem for LocalFileSystem {
fn write_file_stream<'a>(
&'a self,
path: &'a PathUri,
mut contents: BoxStream<'a, FileSystemResult<Bytes>>,
sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, u64> {
Box::pin(async move {
let mut file = self.open_file(path, FsOpenMode::Replace, sandbox).await?;
let mut written = 0_u64;
while let Some(chunk) = contents.next().await {
let chunk = chunk?;
if !(1..=FILE_WRITE_CHUNK_SIZE).contains(&chunk.len()) {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"invalid file stream block length",
));
}
let next_offset = written.checked_add(chunk.len() as u64).ok_or_else(|| {
io::Error::new(io::ErrorKind::InvalidData, "file stream offset overflow")
})?;
file.write_all(&chunk).await?;
// Tokio can return before its blocking write finishes; finish this block before reading another.
file.flush().await?;
written = next_offset;
}
Ok(written)
})
}
fn canonicalize<'a>(
&'a self,
path: &'a PathUri,
+135 -1
View File
@@ -5,13 +5,22 @@ use std::sync::MutexGuard;
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD;
use bytes::Bytes;
use codex_file_system::FILE_WRITE_CHUNK_SIZE;
use codex_utils_path_uri::PathUri;
use futures::StreamExt;
use futures::TryFutureExt;
use futures::stream::BoxStream;
use tokio::io;
use tokio::sync::OnceCell;
use tracing::Instrument;
use tracing::instrument::WithSubscriber;
use tracing::trace;
use uuid::Uuid;
use crate::CopyOptions;
use crate::CreateDirectoryOptions;
use crate::ExecServerClient;
use crate::ExecServerError;
use crate::ExecutorFileSystem;
use crate::ExecutorFileSystemFuture;
@@ -28,13 +37,17 @@ use crate::WalkOutcome;
use crate::WriteFileOptions;
use crate::client::LazyRemoteExecServerClient;
use crate::protocol::FsCanonicalizeParams;
use crate::protocol::FsCloseParams;
use crate::protocol::FsCopyParams;
use crate::protocol::FsCreateDirectoryParams;
use crate::protocol::FsGetMetadataParams;
use crate::protocol::FsOpenMode;
use crate::protocol::FsOpenParams;
use crate::protocol::FsReadDirectoryParams;
use crate::protocol::FsReadFileParams;
use crate::protocol::FsRemoveParams;
use crate::protocol::FsWalkParams;
use crate::protocol::FsWriteBlockParams;
use crate::protocol::FsWriteFileParams;
const INVALID_REQUEST_ERROR_CODE: i64 = -32600;
@@ -317,6 +330,98 @@ impl RemoteFileSystem {
}
impl ExecutorFileSystem for RemoteFileSystem {
fn write_file_stream<'a>(
&'a self,
path: &'a PathUri,
contents: BoxStream<'a, FileSystemResult<Bytes>>,
sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, u64> {
Box::pin(async move {
let client = self.client.get().await.map_err(map_remote_error)?;
let _invalidation = MetadataInvalidation(self);
let mut contents = contents.map(|chunk| {
let chunk = chunk?;
if !(1..=FILE_WRITE_CHUNK_SIZE).contains(&chunk.len()) {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"invalid file stream block length",
));
}
Ok(chunk)
});
// Choose the legacy write before opening a stream, so an open or write failure
// can never cause us to replay a replacement that may have already modified the file.
if !client
.supports_file_write_streaming()
.await
.map_err(map_remote_error)?
{
let mut bytes = Vec::new();
while let Some(chunk) = contents.next().await {
bytes.extend_from_slice(&chunk?);
}
let len = bytes.len() as u64;
self.write_file(path, bytes, WriteFileOptions::default(), sandbox)
.await?;
return Ok(len);
}
// Dropping the JoinHandle lets the open finish. If it succeeds after cancellation,
// Tokio drops the unclaimed registration, which closes that exact handle.
let open = client
.fs_open(FsOpenParams {
handle_id: Uuid::new_v4().simple().to_string(),
path: path.clone(),
mode: FsOpenMode::Replace,
sandbox: sandbox.cloned(),
})
.map_ok(move |opened| FileWriteRegistration {
client,
handle_id: opened.handle_id,
active: true,
runtime: tokio::runtime::Handle::current(),
});
let open = tokio::spawn(open.in_current_span().with_current_subscriber());
let mut registration = open
.await
.map_err(io::Error::other)?
.map_err(map_remote_error)?;
let result: io::Result<u64> = async {
let mut written = 0_u64;
while let Some(chunk) = contents.next().await {
let chunk = chunk?;
let next_offset = written.checked_add(chunk.len() as u64).ok_or_else(|| {
io::Error::new(io::ErrorKind::InvalidData, "file stream offset overflow")
})?;
registration
.client
.fs_write_block(FsWriteBlockParams {
handle_id: registration.handle_id.clone(),
offset: written,
chunk: chunk.to_vec().into(),
})
.await
.map_err(map_remote_error)?;
written = next_offset;
}
Ok(written)
}
.await;
let close_result = registration
.client
.fs_close(FsCloseParams {
handle_id: registration.handle_id.clone(),
})
.await
.map_err(map_remote_error);
registration.active = close_result.is_err();
let written = result?;
close_result?;
Ok(written)
})
}
fn canonicalize<'a>(
&'a self,
path: &'a PathUri,
@@ -417,7 +522,36 @@ impl ExecutorFileSystem for RemoteFileSystem {
}
}
fn map_remote_error(error: ExecServerError) -> io::Error {
/// A cancelled write must stop later lookups from joining pre-cancellation metadata requests.
struct MetadataInvalidation<'a>(&'a RemoteFileSystem);
impl Drop for MetadataInvalidation<'_> {
fn drop(&mut self) {
self.0.lock_metadata_requests().clear();
}
}
/// Best-effort remote handle cleanup may outlive a cancelled transfer.
struct FileWriteRegistration {
client: ExecServerClient,
handle_id: String,
active: bool,
runtime: tokio::runtime::Handle,
}
impl Drop for FileWriteRegistration {
fn drop(&mut self) {
if self.active {
let client = self.client.clone();
let handle_id = self.handle_id.clone();
self.runtime.spawn(async move {
let _ = client.fs_close(FsCloseParams { handle_id }).await;
});
}
}
}
pub(crate) fn map_remote_error(error: ExecServerError) -> io::Error {
match error {
ExecServerError::Server { code, message } if code == NOT_FOUND_ERROR_CODE => {
io::Error::new(io::ErrorKind::NotFound, message)
+135
View File
@@ -18,6 +18,7 @@ use codex_exec_server::RemoteExecServerConnectArgs;
use codex_http_client::HttpClientFactory;
use codex_http_client::OutboundProxyPolicy;
use codex_utils_path_uri::PathUri;
use futures::StreamExt;
use futures::TryStreamExt;
use pretty_assertions::assert_eq;
use std::sync::Arc;
@@ -35,6 +36,140 @@ use crate::common::exec_server::exec_server;
const BLOCK_SIZE: usize = 1024 * 1024;
const OPEN_FILE_LIMIT: usize = 128;
/// The writer must finish a block before asking its source for more data.
#[tokio::test]
async fn streaming_replacement_writes_before_reading_the_next_block() -> Result<()> {
let server = exec_server().await?;
let local = Environment::create_for_tests(/*exec_server_url*/ None)?.get_filesystem();
let remote = connect_file_system(server.websocket_url())?;
let tmp = TempDir::new()?;
let target = tmp.path().join("sequential.bin");
let target_uri = PathUri::from_host_native_path(&target)?;
for fs in [&local, &remote] {
// This stream borrows the target path rather than requiring an owned filesystem stream.
let contents = futures::stream::iter(0..3_u8).map(|block| {
assert_eq!(
std::fs::metadata(&target).unwrap().len(),
u64::from(block) * BLOCK_SIZE as u64
);
Ok(bytes::Bytes::from(vec![block; BLOCK_SIZE]))
});
assert_eq!(
fs.write_file_stream(&target_uri, contents.boxed(), /*sandbox*/ None)
.await?,
3 * BLOCK_SIZE as u64
);
}
Ok(())
}
/// Streaming replacement preserves binary data across local and remote filesystem combinations.
#[tokio::test]
async fn streaming_replacement_copies_binary_and_empty_files() -> Result<()> {
let server = exec_server().await?;
let local = Environment::create_for_tests(/*exec_server_url*/ None)?.get_filesystem();
let remote = connect_file_system(server.websocket_url())?;
let tmp = TempDir::new()?;
let source = tmp.path().join("source.bin");
let target = tmp.path().join("target.bin");
let source_uri = PathUri::from_host_native_path(&source)?;
let target_uri = PathUri::from_host_native_path(&target)?;
for source_fs in [&local, &remote] {
for target_fs in [&local, &remote] {
for bytes in [[0, 255, 13, 10].repeat(BLOCK_SIZE), Vec::new()] {
std::fs::write(&source, &bytes)?;
std::fs::write(&target, vec![42; bytes.len() + 10])?;
let contents = source_fs
.read_file_stream(&source_uri, /*sandbox*/ None)
.await?;
let written = target_fs
.write_file_stream(&target_uri, contents.boxed(), /*sandbox*/ None)
.await?;
assert_eq!(
(written, std::fs::read(&target)?),
(bytes.len() as u64, bytes)
);
}
}
}
Ok(())
}
/// A failed source leaves only confirmed blocks at the target and releases remote handle capacity.
#[tokio::test]
async fn streaming_replacement_preserves_partial_failure_and_releases_handles() -> Result<()> {
let server = exec_server().await?;
let fs = connect_file_system(server.websocket_url())?;
let tmp = TempDir::new()?;
let target = tmp.path().join("partial.bin");
let target_uri = PathUri::from_host_native_path(&target)?;
for _ in 0..=OPEN_FILE_LIMIT {
std::fs::write(&target, b"original target contents")?;
let contents = futures::stream::iter([
Ok(bytes::Bytes::from_static(b"partial")),
Err(std::io::Error::other("source failed")),
]);
let error = fs
.write_file_stream(&target_uri, contents.boxed(), /*sandbox*/ None)
.await
.unwrap_err();
assert_eq!(
(error.to_string(), std::fs::read(&target)?),
("source failed".to_string(), b"partial".to_vec())
);
}
Ok(())
}
/// Cancelling a partial replacement must not let later metadata lookups reuse an older response.
#[tokio::test]
async fn cancelled_streaming_replacement_invalidates_metadata() -> Result<()> {
let server = exec_server().await?;
let fs = connect_file_system(server.websocket_url())?;
let tmp = TempDir::new()?;
let target = tmp.path().join("metadata.bin");
let path = PathUri::from_host_native_path(&target)?;
let original = b"original target contents";
std::fs::write(&target, original)?;
// Connect before polling metadata so its first poll sends the request.
fs.read_file(&path, Default::default(), /*sandbox*/ None)
.await?;
let stale = fs.get_metadata(&path, Default::default(), /*sandbox*/ None);
tokio::pin!(stale);
assert!(futures::poll!(stale.as_mut()).is_pending());
// The fixture dispatches requests in order. This read confirms the old stat finished,
// while leaving its caller unpolled so that another lookup could still join it.
assert_eq!(
fs.read_file(&path, Default::default(), /*sandbox*/ None)
.await?,
original
);
let (polled_tx, polled_rx) = tokio::sync::oneshot::channel();
let contents = futures::stream::iter([Ok(bytes::Bytes::from_static(b"partial"))]).chain(
futures::stream::once(async move {
let _ = polled_tx.send(());
futures::future::pending::<std::io::Result<bytes::Bytes>>().await
}),
);
let mut write = fs.write_file_stream(&path, contents.boxed(), /*sandbox*/ None);
tokio::select! {
result = &mut write => panic!("copy should await its source: {result:?}"),
_ = polled_rx => {},
}
drop(write);
let (refreshed, stale) = tokio::join!(
fs.get_metadata(&path, Default::default(), /*sandbox*/ None),
stale.as_mut(),
);
assert_eq!(
(stale?.size, refreshed?.size, std::fs::read(&target)?),
(original.len() as u64, 7, b"partial".to_vec())
);
Ok(())
}
#[tokio::test]
async fn stream_stops_after_an_exact_block_boundary() -> Result<()> {
let server = exec_server().await?;