diff --git a/codex-rs/exec-server/src/client.rs b/codex-rs/exec-server/src/client.rs index 2ce504dabb..6f1f74114b 100644 --- a/codex-rs/exec-server/src/client.rs +++ b/codex-rs/exec-server/src/client.rs @@ -1,5 +1,6 @@ use std::collections::BTreeMap; use std::collections::HashMap; +use std::future::Future; use std::sync::Arc; use std::sync::Mutex as StdMutex; use std::sync::OnceLock; @@ -969,21 +970,32 @@ impl ExecServerClient { .await } - pub async fn fs_open(&self, params: FsOpenParams) -> Result { - // Older executors ignore the mode field and could silently return a read-only handle. - if params.mode == FsOpenMode::Replace - && !self - .environment_info() - .await? - .capabilities - .file_write_streaming - { - return Err(ExecServerError::Protocol( - "exec-server does not support writable file streams".to_string(), - )); + pub(crate) async fn supports_file_write_streaming(&self) -> Result { + Ok(self + .environment_info() + .await? + .capabilities + .file_write_streaming) + } + + /// Returns an owned future so callers can finish an open after they stop waiting for it. + pub fn fs_open( + &self, + params: FsOpenParams, + ) -> impl Future> + Send + 'static { + let client = self.clone(); + async move { + // Older executors ignore the mode field and could silently return a read-only handle. + if params.mode == FsOpenMode::Replace && !client.supports_file_write_streaming().await? + { + return Err(ExecServerError::Protocol( + "exec-server does not support writable file streams".to_string(), + )); + } + client + .call(FS_OPEN_METHOD, &WireFsOpenParams::from(params)) + .await } - self.call(FS_OPEN_METHOD, &WireFsOpenParams::from(params)) - .await } pub async fn fs_read_block( @@ -997,12 +1009,7 @@ impl ExecServerClient { &self, params: FsWriteBlockParams, ) -> Result { - if !self - .environment_info() - .await? - .capabilities - .file_write_streaming - { + if !self.supports_file_write_streaming().await? { return Err(ExecServerError::Protocol( "exec-server does not support writable file streams".to_string(), )); @@ -1958,6 +1965,7 @@ mod tests { use codex_exec_server_protocol::JSONRPCResponse; use codex_http_client::HttpClientFactory; use codex_http_client::OutboundProxyPolicy; + use codex_protocol::models::PermissionProfile; use codex_utils_path_uri::PathUri; use futures::SinkExt; use futures::StreamExt; @@ -1996,8 +2004,12 @@ mod tests { use super::ExecServerClientConnectOptions; use super::ExecServerError; use super::LazyRemoteExecServerClient; + use crate::Environment; use crate::EnvironmentObservedStatus; + use crate::ExecutorFileSystem; + use crate::FileSystemSandboxContext; use crate::ProcessId; + use crate::client_api::DEFAULT_REMOTE_EXEC_SERVER_CONNECT_TIMEOUT; #[cfg(not(windows))] use crate::client_api::DEFAULT_REMOTE_EXEC_SERVER_INITIALIZE_TIMEOUT; use crate::client_api::ExecServerTransportParams; @@ -2019,19 +2031,24 @@ mod tests { use crate::protocol::ExecOutputStream; use crate::protocol::ExecParams; use crate::protocol::ExecResponse; + use crate::protocol::FS_CLOSE_METHOD; use crate::protocol::FS_OPEN_METHOD; + use crate::protocol::FS_WRITE_FILE_METHOD; use crate::protocol::FsOpenMode; use crate::protocol::FsOpenParams; use crate::protocol::FsWriteBlockParams; + use crate::protocol::FsWriteFileParams; use crate::protocol::INITIALIZE_METHOD; use crate::protocol::INITIALIZED_METHOD; use crate::protocol::InitializeResponse; use crate::protocol::ProcessOutputChunk; use crate::protocol::ProcessSandboxType; use crate::protocol::ReadResponse; + use crate::protocol::WireFsWriteFileParams; use crate::protocol::WriteParams; use crate::protocol::WriteResponse; use crate::protocol::WriteStatus; + use crate::remote_file_system::RemoteFileSystem; async fn read_jsonrpc_line(lines: &mut tokio::io::Lines>) -> JSONRPCMessage where @@ -2758,9 +2775,10 @@ mod tests { Ok(()) } - /// Old executors must never receive writable opens they would silently treat as reads. + /// Old executors receive one sandboxed writeFile, never an open they would mistake for a read. #[tokio::test] - async fn writable_file_streams_require_executor_capability() -> anyhow::Result<()> { + async fn old_executor_receives_buffered_replacement_and_keeps_read_only_opens() + -> anyhow::Result<()> { let listener = TcpListener::bind("127.0.0.1:0").await?; let websocket_url = format!("ws://{}", listener.local_addr()?); let read_open = FsOpenParams { @@ -2769,6 +2787,16 @@ mod tests { mode: FsOpenMode::Read, sandbox: None, }; + let sandbox = FileSystemSandboxContext::from_permission_profile( + PermissionProfile::workspace_write(), + PathUri::parse("file:///tmp")?, + ); + let expected_write = WireFsWriteFileParams::from(FsWriteFileParams { + path: read_open.path.clone(), + data_base64: "AP9oaQ==".to_string(), + follow_symlinks: None, + sandbox: Some(sandbox.clone()), + }); let server = tokio::spawn(async move { let mut websocket = accept_websocket(&listener).await; let mut info = EnvironmentInfo::local(); @@ -2780,6 +2808,23 @@ mod tests { Some(info), ) .await; + let JSONRPCMessage::Request(write) = read_jsonrpc_websocket(&mut websocket).await + else { + panic!("expected buffered write"); + }; + assert_eq!(write.method, FS_WRITE_FILE_METHOD); + assert_eq!( + write.params, + Some(serde_json::to_value(expected_write).expect("serialize legacy write")) + ); + write_jsonrpc_websocket( + &mut websocket, + JSONRPCMessage::Response(JSONRPCResponse { + id: write.id, + result: serde_json::json!({}), + }), + ) + .await; let JSONRPCMessage::Request(request) = read_jsonrpc_websocket(&mut websocket).await else { panic!("expected read-only open"); @@ -2803,12 +2848,32 @@ mod tests { ) .await; }); - let client = ExecServerClient::connect_websocket(RemoteExecServerConnectArgs::new( - websocket_url, - "file-stream-test".to_string(), + let remote = LazyRemoteExecServerClient::new( + ExecServerTransportParams::websocket_url( + websocket_url, + DEFAULT_REMOTE_EXEC_SERVER_CONNECT_TIMEOUT, + ), HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault), - )) - .await?; + ); + let client = remote.get().await?; + let fs = RemoteFileSystem::new(remote); + // A source error precedes any legacy write, so the server must see only the complete file. + let broken = futures::stream::iter([ + Ok(bytes::Bytes::from_static(b"partial")), + Err(std::io::Error::from(std::io::ErrorKind::UnexpectedEof)), + ]); + let error = fs + .write_file_stream(&read_open.path, broken.boxed(), Some(&sandbox)) + .await + .expect_err("source error"); + assert_eq!(error.kind(), std::io::ErrorKind::UnexpectedEof); + + let contents = futures::stream::iter([b"\x00\xff", b"hi"]) + .map(|chunk| Ok(bytes::Bytes::from_static(chunk))); + let written = fs + .write_file_stream(&read_open.path, contents.boxed(), Some(&sandbox)) + .await?; + assert_eq!(written, 4); let error = client .fs_open(FsOpenParams { mode: FsOpenMode::Replace, @@ -2826,11 +2891,70 @@ mod tests { .await .unwrap_err(); assert!(matches!(error, ExecServerError::Protocol(_))); + // This read-only open must be the next request; there is no stream handle to close. client.fs_open(read_open).await?; server.await?; Ok(()) } + /// An open that completes after cancellation must close that exact handle. + #[tokio::test] + async fn cancelled_streaming_replacement_closes_pending_open() -> anyhow::Result<()> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let websocket_url = format!("ws://{}", listener.local_addr()?); + let fs = Environment::create_for_tests(Some(websocket_url))?.get_filesystem(); + let path = PathUri::parse("file:///tmp/cancelled.bin")?; + let contents = futures::stream::poll_fn(|_| panic!("cancelled write polled its source")); + let mut write = fs.write_file_stream(&path, contents.boxed(), /*sandbox*/ None); + + let receive_open = async { + let mut ws = accept_websocket(&listener).await; + complete_websocket_initialize_with_environment_info( + &mut ws, + "session-1", + /*expected_resume_session_id*/ None, + Some(EnvironmentInfo::local()), + ) + .await; + let JSONRPCMessage::Request(open) = read_jsonrpc_websocket(&mut ws).await else { + panic!("expected fs/open"); + }; + assert_eq!(open.method, FS_OPEN_METHOD); + (ws, open) + }; + let (mut ws, open) = tokio::select! { + open = receive_open => open, + result = &mut write => panic!("write must wait for open: {result:?}"), + }; + drop(write); + + let params = open.params.expect("fs/open params"); + let handle = params["handleId"].as_str().expect("handleId"); + let handle = serde_json::json!({"handleId": handle}); + write_jsonrpc_websocket( + &mut ws, + JSONRPCMessage::Response(JSONRPCResponse { + id: open.id, + result: handle.clone(), + }), + ) + .await; + let JSONRPCMessage::Request(close) = read_jsonrpc_websocket(&mut ws).await else { + panic!("expected fs/close"); + }; + assert_eq!(close.method, FS_CLOSE_METHOD); + assert_eq!(close.params, Some(handle)); + write_jsonrpc_websocket( + &mut ws, + JSONRPCMessage::Response(JSONRPCResponse { + id: close.id, + result: serde_json::json!({}), + }), + ) + .await; + Ok(()) + } + #[test_case::test_case(false; "socket_closed")] #[test_case::test_case(true; "health_check_timed_out")] #[tokio::test] diff --git a/codex-rs/exec-server/src/local_file_system.rs b/codex-rs/exec-server/src/local_file_system.rs index ec8b81c52a..abe2a0a6f1 100644 --- a/codex-rs/exec-server/src/local_file_system.rs +++ b/codex-rs/exec-server/src/local_file_system.rs @@ -1,3 +1,5 @@ +use bytes::Bytes; +use codex_file_system::FILE_WRITE_CHUNK_SIZE; use codex_file_system::MAX_WALK_DEPTH; use codex_file_system::MAX_WALK_DIRECTORIES; use codex_file_system::MAX_WALK_ENTRIES; @@ -5,6 +7,8 @@ use codex_file_system::MAX_WALK_RESPONSE_BYTES; use codex_file_system::WALK_RESPONSE_ITEM_OVERHEAD_BYTES; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_path_uri::PathUri; +use futures::StreamExt; +use futures::stream::BoxStream; use std::collections::HashSet; use std::collections::VecDeque; use std::path::Path; @@ -14,6 +18,7 @@ use std::sync::LazyLock; use std::time::SystemTime; use std::time::UNIX_EPOCH; use tokio::io; +use tokio::io::AsyncWriteExt; use tokio_util::io::ReaderStream; use tokio_util::sync::CancellationToken; @@ -253,6 +258,35 @@ impl LocalFileSystem { } impl ExecutorFileSystem for LocalFileSystem { + fn write_file_stream<'a>( + &'a self, + path: &'a PathUri, + mut contents: BoxStream<'a, FileSystemResult>, + sandbox: Option<&'a FileSystemSandboxContext>, + ) -> ExecutorFileSystemFuture<'a, u64> { + Box::pin(async move { + let mut file = self.open_file(path, FsOpenMode::Replace, sandbox).await?; + let mut written = 0_u64; + while let Some(chunk) = contents.next().await { + let chunk = chunk?; + if !(1..=FILE_WRITE_CHUNK_SIZE).contains(&chunk.len()) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "invalid file stream block length", + )); + } + let next_offset = written.checked_add(chunk.len() as u64).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "file stream offset overflow") + })?; + file.write_all(&chunk).await?; + // Tokio can return before its blocking write finishes; finish this block before reading another. + file.flush().await?; + written = next_offset; + } + Ok(written) + }) + } + fn canonicalize<'a>( &'a self, path: &'a PathUri, diff --git a/codex-rs/exec-server/src/remote_file_system.rs b/codex-rs/exec-server/src/remote_file_system.rs index cdcaa78bca..729b885366 100644 --- a/codex-rs/exec-server/src/remote_file_system.rs +++ b/codex-rs/exec-server/src/remote_file_system.rs @@ -5,13 +5,22 @@ use std::sync::MutexGuard; use base64::Engine as _; use base64::engine::general_purpose::STANDARD; +use bytes::Bytes; +use codex_file_system::FILE_WRITE_CHUNK_SIZE; use codex_utils_path_uri::PathUri; +use futures::StreamExt; +use futures::TryFutureExt; +use futures::stream::BoxStream; use tokio::io; use tokio::sync::OnceCell; +use tracing::Instrument; +use tracing::instrument::WithSubscriber; use tracing::trace; +use uuid::Uuid; use crate::CopyOptions; use crate::CreateDirectoryOptions; +use crate::ExecServerClient; use crate::ExecServerError; use crate::ExecutorFileSystem; use crate::ExecutorFileSystemFuture; @@ -28,13 +37,17 @@ use crate::WalkOutcome; use crate::WriteFileOptions; use crate::client::LazyRemoteExecServerClient; use crate::protocol::FsCanonicalizeParams; +use crate::protocol::FsCloseParams; use crate::protocol::FsCopyParams; use crate::protocol::FsCreateDirectoryParams; use crate::protocol::FsGetMetadataParams; +use crate::protocol::FsOpenMode; +use crate::protocol::FsOpenParams; use crate::protocol::FsReadDirectoryParams; use crate::protocol::FsReadFileParams; use crate::protocol::FsRemoveParams; use crate::protocol::FsWalkParams; +use crate::protocol::FsWriteBlockParams; use crate::protocol::FsWriteFileParams; const INVALID_REQUEST_ERROR_CODE: i64 = -32600; @@ -317,6 +330,98 @@ impl RemoteFileSystem { } impl ExecutorFileSystem for RemoteFileSystem { + fn write_file_stream<'a>( + &'a self, + path: &'a PathUri, + contents: BoxStream<'a, FileSystemResult>, + sandbox: Option<&'a FileSystemSandboxContext>, + ) -> ExecutorFileSystemFuture<'a, u64> { + Box::pin(async move { + let client = self.client.get().await.map_err(map_remote_error)?; + let _invalidation = MetadataInvalidation(self); + let mut contents = contents.map(|chunk| { + let chunk = chunk?; + if !(1..=FILE_WRITE_CHUNK_SIZE).contains(&chunk.len()) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "invalid file stream block length", + )); + } + Ok(chunk) + }); + // Choose the legacy write before opening a stream, so an open or write failure + // can never cause us to replay a replacement that may have already modified the file. + if !client + .supports_file_write_streaming() + .await + .map_err(map_remote_error)? + { + let mut bytes = Vec::new(); + while let Some(chunk) = contents.next().await { + bytes.extend_from_slice(&chunk?); + } + let len = bytes.len() as u64; + self.write_file(path, bytes, WriteFileOptions::default(), sandbox) + .await?; + return Ok(len); + } + + // Dropping the JoinHandle lets the open finish. If it succeeds after cancellation, + // Tokio drops the unclaimed registration, which closes that exact handle. + let open = client + .fs_open(FsOpenParams { + handle_id: Uuid::new_v4().simple().to_string(), + path: path.clone(), + mode: FsOpenMode::Replace, + sandbox: sandbox.cloned(), + }) + .map_ok(move |opened| FileWriteRegistration { + client, + handle_id: opened.handle_id, + active: true, + runtime: tokio::runtime::Handle::current(), + }); + let open = tokio::spawn(open.in_current_span().with_current_subscriber()); + let mut registration = open + .await + .map_err(io::Error::other)? + .map_err(map_remote_error)?; + + let result: io::Result = async { + let mut written = 0_u64; + while let Some(chunk) = contents.next().await { + let chunk = chunk?; + let next_offset = written.checked_add(chunk.len() as u64).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "file stream offset overflow") + })?; + registration + .client + .fs_write_block(FsWriteBlockParams { + handle_id: registration.handle_id.clone(), + offset: written, + chunk: chunk.to_vec().into(), + }) + .await + .map_err(map_remote_error)?; + written = next_offset; + } + Ok(written) + } + .await; + let close_result = registration + .client + .fs_close(FsCloseParams { + handle_id: registration.handle_id.clone(), + }) + .await + .map_err(map_remote_error); + registration.active = close_result.is_err(); + let written = result?; + close_result?; + Ok(written) + }) + } + fn canonicalize<'a>( &'a self, path: &'a PathUri, @@ -417,7 +522,36 @@ impl ExecutorFileSystem for RemoteFileSystem { } } -fn map_remote_error(error: ExecServerError) -> io::Error { +/// A cancelled write must stop later lookups from joining pre-cancellation metadata requests. +struct MetadataInvalidation<'a>(&'a RemoteFileSystem); + +impl Drop for MetadataInvalidation<'_> { + fn drop(&mut self) { + self.0.lock_metadata_requests().clear(); + } +} + +/// Best-effort remote handle cleanup may outlive a cancelled transfer. +struct FileWriteRegistration { + client: ExecServerClient, + handle_id: String, + active: bool, + runtime: tokio::runtime::Handle, +} + +impl Drop for FileWriteRegistration { + fn drop(&mut self) { + if self.active { + let client = self.client.clone(); + let handle_id = self.handle_id.clone(); + self.runtime.spawn(async move { + let _ = client.fs_close(FsCloseParams { handle_id }).await; + }); + } + } +} + +pub(crate) fn map_remote_error(error: ExecServerError) -> io::Error { match error { ExecServerError::Server { code, message } if code == NOT_FOUND_ERROR_CODE => { io::Error::new(io::ErrorKind::NotFound, message) diff --git a/codex-rs/exec-server/tests/file_stream.rs b/codex-rs/exec-server/tests/file_stream.rs index 2e37bb729b..8191c542cc 100644 --- a/codex-rs/exec-server/tests/file_stream.rs +++ b/codex-rs/exec-server/tests/file_stream.rs @@ -18,6 +18,7 @@ use codex_exec_server::RemoteExecServerConnectArgs; use codex_http_client::HttpClientFactory; use codex_http_client::OutboundProxyPolicy; use codex_utils_path_uri::PathUri; +use futures::StreamExt; use futures::TryStreamExt; use pretty_assertions::assert_eq; use std::sync::Arc; @@ -35,6 +36,140 @@ use crate::common::exec_server::exec_server; const BLOCK_SIZE: usize = 1024 * 1024; const OPEN_FILE_LIMIT: usize = 128; +/// The writer must finish a block before asking its source for more data. +#[tokio::test] +async fn streaming_replacement_writes_before_reading_the_next_block() -> Result<()> { + let server = exec_server().await?; + let local = Environment::create_for_tests(/*exec_server_url*/ None)?.get_filesystem(); + let remote = connect_file_system(server.websocket_url())?; + let tmp = TempDir::new()?; + let target = tmp.path().join("sequential.bin"); + let target_uri = PathUri::from_host_native_path(&target)?; + for fs in [&local, &remote] { + // This stream borrows the target path rather than requiring an owned filesystem stream. + let contents = futures::stream::iter(0..3_u8).map(|block| { + assert_eq!( + std::fs::metadata(&target).unwrap().len(), + u64::from(block) * BLOCK_SIZE as u64 + ); + Ok(bytes::Bytes::from(vec![block; BLOCK_SIZE])) + }); + assert_eq!( + fs.write_file_stream(&target_uri, contents.boxed(), /*sandbox*/ None) + .await?, + 3 * BLOCK_SIZE as u64 + ); + } + Ok(()) +} + +/// Streaming replacement preserves binary data across local and remote filesystem combinations. +#[tokio::test] +async fn streaming_replacement_copies_binary_and_empty_files() -> Result<()> { + let server = exec_server().await?; + let local = Environment::create_for_tests(/*exec_server_url*/ None)?.get_filesystem(); + let remote = connect_file_system(server.websocket_url())?; + let tmp = TempDir::new()?; + let source = tmp.path().join("source.bin"); + let target = tmp.path().join("target.bin"); + let source_uri = PathUri::from_host_native_path(&source)?; + let target_uri = PathUri::from_host_native_path(&target)?; + for source_fs in [&local, &remote] { + for target_fs in [&local, &remote] { + for bytes in [[0, 255, 13, 10].repeat(BLOCK_SIZE), Vec::new()] { + std::fs::write(&source, &bytes)?; + std::fs::write(&target, vec![42; bytes.len() + 10])?; + let contents = source_fs + .read_file_stream(&source_uri, /*sandbox*/ None) + .await?; + let written = target_fs + .write_file_stream(&target_uri, contents.boxed(), /*sandbox*/ None) + .await?; + assert_eq!( + (written, std::fs::read(&target)?), + (bytes.len() as u64, bytes) + ); + } + } + } + Ok(()) +} + +/// A failed source leaves only confirmed blocks at the target and releases remote handle capacity. +#[tokio::test] +async fn streaming_replacement_preserves_partial_failure_and_releases_handles() -> Result<()> { + let server = exec_server().await?; + let fs = connect_file_system(server.websocket_url())?; + let tmp = TempDir::new()?; + let target = tmp.path().join("partial.bin"); + let target_uri = PathUri::from_host_native_path(&target)?; + for _ in 0..=OPEN_FILE_LIMIT { + std::fs::write(&target, b"original target contents")?; + let contents = futures::stream::iter([ + Ok(bytes::Bytes::from_static(b"partial")), + Err(std::io::Error::other("source failed")), + ]); + let error = fs + .write_file_stream(&target_uri, contents.boxed(), /*sandbox*/ None) + .await + .unwrap_err(); + assert_eq!( + (error.to_string(), std::fs::read(&target)?), + ("source failed".to_string(), b"partial".to_vec()) + ); + } + Ok(()) +} + +/// Cancelling a partial replacement must not let later metadata lookups reuse an older response. +#[tokio::test] +async fn cancelled_streaming_replacement_invalidates_metadata() -> Result<()> { + let server = exec_server().await?; + let fs = connect_file_system(server.websocket_url())?; + let tmp = TempDir::new()?; + let target = tmp.path().join("metadata.bin"); + let path = PathUri::from_host_native_path(&target)?; + let original = b"original target contents"; + std::fs::write(&target, original)?; + // Connect before polling metadata so its first poll sends the request. + fs.read_file(&path, Default::default(), /*sandbox*/ None) + .await?; + let stale = fs.get_metadata(&path, Default::default(), /*sandbox*/ None); + tokio::pin!(stale); + assert!(futures::poll!(stale.as_mut()).is_pending()); + // The fixture dispatches requests in order. This read confirms the old stat finished, + // while leaving its caller unpolled so that another lookup could still join it. + assert_eq!( + fs.read_file(&path, Default::default(), /*sandbox*/ None) + .await?, + original + ); + + let (polled_tx, polled_rx) = tokio::sync::oneshot::channel(); + let contents = futures::stream::iter([Ok(bytes::Bytes::from_static(b"partial"))]).chain( + futures::stream::once(async move { + let _ = polled_tx.send(()); + futures::future::pending::>().await + }), + ); + let mut write = fs.write_file_stream(&path, contents.boxed(), /*sandbox*/ None); + tokio::select! { + result = &mut write => panic!("copy should await its source: {result:?}"), + _ = polled_rx => {}, + } + drop(write); + + let (refreshed, stale) = tokio::join!( + fs.get_metadata(&path, Default::default(), /*sandbox*/ None), + stale.as_mut(), + ); + assert_eq!( + (stale?.size, refreshed?.size, std::fs::read(&target)?), + (original.len() as u64, 7, b"partial".to_vec()) + ); + Ok(()) +} + #[tokio::test] async fn stream_stops_after_an_exact_block_boundary() -> Result<()> { let server = exec_server().await?; diff --git a/codex-rs/file-system/src/environment_accessor.rs b/codex-rs/file-system/src/environment_accessor.rs index 159edeb446..06f63ede9e 100644 --- a/codex-rs/file-system/src/environment_accessor.rs +++ b/codex-rs/file-system/src/environment_accessor.rs @@ -6,6 +6,7 @@ use crate::ExecutorFileSystem; use crate::ExecutorFileSystemFuture; use crate::FileMetadata; use crate::FileSystemReadStream; +use crate::FileSystemResult; use crate::FileSystemSandboxContext; use crate::GetMetadataOptions; use crate::ReadDirectoryEntry; @@ -14,7 +15,9 @@ use crate::RemoveOptions; use crate::WalkOptions; use crate::WalkOutcome; use crate::WriteFileOptions; +use bytes::Bytes; use codex_utils_path_uri::PathUri; +use futures::stream::BoxStream; use std::fmt; use std::hash::Hash; use std::hash::Hasher; @@ -43,6 +46,13 @@ pub trait EnvironmentAccess: Send + Sync { path: &'a PathUri, ) -> ExecutorFileSystemFuture<'a, FileSystemReadStream>; + /// Replaces a file through [`ExecutorFileSystem::write_file_stream`] with captured permissions. + fn write_file_stream<'a>( + &'a self, + path: &'a PathUri, + contents: BoxStream<'a, FileSystemResult>, + ) -> ExecutorFileSystemFuture<'a, u64>; + fn write_file<'a>( &'a self, path: &'a PathUri, @@ -210,6 +220,15 @@ impl EnvironmentAccess for FileSystemEnvironmentAccessor<'_> { .read_file_stream(path, self.key.sandbox.as_deref()) } + fn write_file_stream<'a>( + &'a self, + path: &'a PathUri, + contents: BoxStream<'a, FileSystemResult>, + ) -> ExecutorFileSystemFuture<'a, u64> { + self.file_system + .write_file_stream(path, contents, self.key.sandbox.as_deref()) + } + fn write_file<'a>( &'a self, path: &'a PathUri, diff --git a/codex-rs/file-system/src/lib.rs b/codex-rs/file-system/src/lib.rs index 1687d006f5..7b95299b5e 100644 --- a/codex-rs/file-system/src/lib.rs +++ b/codex-rs/file-system/src/lib.rs @@ -29,6 +29,7 @@ pub use find_up::FindUpErrorPolicy; pub use find_up::find_nearest_ancestor_with_markers; pub use find_up::find_nearest_native_ancestor_with_markers; use futures::Stream; +use futures::stream::BoxStream; pub use nfs::is_nfs; use serde::Deserialize; use serde::Serialize; @@ -660,6 +661,36 @@ pub trait ExecutorFileSystem: Send + Sync { sandbox: Option<&'a FileSystemSandboxContext>, ) -> ExecutorFileSystemFuture<'a, FileSystemReadStream>; + /// Replaces a regular file from a stream of bounded chunks. + /// + /// Accepts any sendable byte stream, boxed to keep this trait object-safe. + /// Input chunks must be nonempty and no larger than [`FILE_WRITE_CHUNK_SIZE`]. + /// Returns the number of bytes written after closing the target. Parents must already exist. + /// Failure or cancellation may leave a truncated or partial target. + /// + /// Cancellation is best effort: dropping the future stops consuming input and issuing new + /// blocks, but an in-flight open or write may still truncate or modify the target. Dropping + /// does not wait for that I/O or remote handle cleanup, so a subsequent replacement may overlap + /// with the abandoned operation. + /// + /// Local and streaming-capable remote filesystems write one chunk at a time. For compatibility, + /// a remote filesystem may buffer the entire input and send one write to an older executor. + /// The default implementation returns `Unsupported` without consuming the input. + fn write_file_stream<'a>( + &'a self, + path: &'a PathUri, + contents: BoxStream<'a, FileSystemResult>, + sandbox: Option<&'a FileSystemSandboxContext>, + ) -> ExecutorFileSystemFuture<'a, u64> { + Box::pin(async move { + let _ = (path, contents, sandbox); + Err(io::Error::new( + io::ErrorKind::Unsupported, + "filesystem does not support streaming writes", + )) + }) + } + /// Reads a file and decodes it as UTF-8 text. fn read_file_text<'a>( &'a self,