mirror of
https://github.com/openai/codex.git
synced 2026-11-16 22:00:36 +00:00
Harden streamed file handle lifecycle
This commit is contained in:
@@ -735,7 +735,7 @@ client_request_definitions! {
|
||||
},
|
||||
FsReadFileClose => "fs/readFile/close" {
|
||||
params: v2::FsReadFileCloseParams,
|
||||
serialization: None,
|
||||
serialization: file_handle_id(params.handle_id),
|
||||
response: v2::FsReadFileCloseResponse,
|
||||
},
|
||||
FsWriteFileOpen => "fs/writeFile/open" {
|
||||
@@ -750,7 +750,7 @@ client_request_definitions! {
|
||||
},
|
||||
FsWriteFileClose => "fs/writeFile/close" {
|
||||
params: v2::FsWriteFileCloseParams,
|
||||
serialization: None,
|
||||
serialization: file_handle_id(params.handle_id),
|
||||
response: v2::FsWriteFileCloseResponse,
|
||||
},
|
||||
FsCreateDirectory => "fs/createDirectory" {
|
||||
@@ -1789,6 +1789,32 @@ mod tests {
|
||||
})
|
||||
);
|
||||
|
||||
let read_file_close = ClientRequest::FsReadFileClose {
|
||||
request_id: request_id(),
|
||||
params: v2::FsReadFileCloseParams {
|
||||
handle_id: "file-1".to_string(),
|
||||
},
|
||||
};
|
||||
assert_eq!(
|
||||
read_file_close.serialization_scope(),
|
||||
Some(ClientRequestSerializationScope::FileHandle {
|
||||
handle_id: "file-1".to_string()
|
||||
})
|
||||
);
|
||||
|
||||
let write_file_close = ClientRequest::FsWriteFileClose {
|
||||
request_id: request_id(),
|
||||
params: v2::FsWriteFileCloseParams {
|
||||
handle_id: "file-1".to_string(),
|
||||
},
|
||||
};
|
||||
assert_eq!(
|
||||
write_file_close.serialization_scope(),
|
||||
Some(ClientRequestSerializationScope::FileHandle {
|
||||
handle_id: "file-1".to_string()
|
||||
})
|
||||
);
|
||||
|
||||
let plugin_install = ClientRequest::PluginInstall {
|
||||
request_id: request_id(),
|
||||
params: v2::PluginInstallParams {
|
||||
|
||||
@@ -1160,11 +1160,11 @@ All filesystem paths in this section must be absolute.
|
||||
- `fs/createDirectory` defaults `recursive` to `true` when omitted.
|
||||
- `fs/remove` defaults both `recursive` and `force` to `true` when omitted.
|
||||
- `fs/readFile` always returns base64 bytes via `dataBase64`, and `fs/writeFile` always expects base64 bytes in `dataBase64`.
|
||||
- Streaming file handles use caller-supplied `handleId` values scoped to one connection. The server advertises a 262144-byte decoded chunk limit from each open response.
|
||||
- Streaming file handles use caller-supplied `handleId` values scoped to one connection. Each connection may have at most 64 opening or open file handles. The server advertises a 262144-byte decoded chunk limit from each open response.
|
||||
- `fs/readFile/read` is positional: pass an absolute byte `offset` and optional `maxBytes`. `eof` reports whether more bytes were available at that read; a growing file may later return more data.
|
||||
- `fs/readFile/stat` reads metadata from the opened file object, so it remains valid if the original path is renamed or replaced.
|
||||
- Opening a streamed write creates or truncates the destination immediately. Completed chunks are visible immediately, and cancellation, disconnection, or an I/O error may leave a partially written file. Existing file identity and metadata are preserved like `fs/writeFile`.
|
||||
- `fs/readFile/close` and `fs/writeFile/close` are idempotent cancellation operations. Disconnecting closes all handles owned by that connection.
|
||||
- `fs/readFile/close` and `fs/writeFile/close` are idempotent and serialized with other operations for the same handle. Close waits for an active operation to finish and prevents a pending open from becoming live. Disconnecting closes all handles owned by that connection.
|
||||
- `fs/copy` handles both file copies and directory-tree copies; it requires `recursive: true` when `sourcePath` is a directory. Recursive copies traverse regular files, directories, and symlinks; other entry types are skipped.
|
||||
|
||||
### Example: Filesystem watch
|
||||
|
||||
@@ -193,7 +193,10 @@ impl FsRequestProcessor {
|
||||
params: FsReadFileCloseParams,
|
||||
) -> Result<FsReadFileCloseResponse, JSONRPCErrorError> {
|
||||
if let Some(handles) = self.existing_handles(connection_id).await {
|
||||
handles.close(¶ms.handle_id).await;
|
||||
handles
|
||||
.close(¶ms.handle_id)
|
||||
.await
|
||||
.map_err(map_file_handle_error)?;
|
||||
}
|
||||
Ok(FsReadFileCloseResponse {})
|
||||
}
|
||||
@@ -244,7 +247,10 @@ impl FsRequestProcessor {
|
||||
params: FsWriteFileCloseParams,
|
||||
) -> Result<FsWriteFileCloseResponse, JSONRPCErrorError> {
|
||||
if let Some(handles) = self.existing_handles(connection_id).await {
|
||||
handles.close(¶ms.handle_id).await;
|
||||
handles
|
||||
.close(¶ms.handle_id)
|
||||
.await
|
||||
.map_err(map_file_handle_error)?;
|
||||
}
|
||||
Ok(FsWriteFileCloseResponse {})
|
||||
}
|
||||
|
||||
@@ -3,25 +3,34 @@ use std::io;
|
||||
use std::sync::Arc;
|
||||
|
||||
use codex_file_system::ExecutorFileSystem;
|
||||
use codex_file_system::FileReadChunk;
|
||||
use codex_file_system::FileReadHandle;
|
||||
use codex_file_system::FileSystemSandboxContext;
|
||||
use codex_file_system::FileWriteHandle;
|
||||
use codex_file_system::OpenFileMetadata;
|
||||
use codex_utils_absolute_path::AbsolutePathBuf;
|
||||
use tokio::sync::Mutex;
|
||||
use tokio::sync::Semaphore;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
#[derive(Debug, Clone, Eq, PartialEq)]
|
||||
pub struct FileReadChunk {
|
||||
pub data: Vec<u8>,
|
||||
pub eof: bool,
|
||||
}
|
||||
pub const MAX_OPEN_FILE_HANDLES: usize = 64;
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
#[derive(Clone)]
|
||||
pub struct FileHandleManager {
|
||||
handles: Arc<Mutex<HashMap<String, FileHandle>>>,
|
||||
max_handles: usize,
|
||||
}
|
||||
|
||||
impl Default for FileHandleManager {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
handles: Arc::new(Mutex::new(HashMap::new())),
|
||||
max_handles: MAX_OPEN_FILE_HANDLES,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
enum FileHandle {
|
||||
Opening(Arc<OpeningHandleEntry>),
|
||||
Read(Arc<ReadHandleEntry>),
|
||||
@@ -30,21 +39,32 @@ enum FileHandle {
|
||||
|
||||
struct OpeningHandleEntry {
|
||||
cancellation: CancellationToken,
|
||||
completed: CancellationToken,
|
||||
}
|
||||
|
||||
struct ReadHandleEntry {
|
||||
handle: Box<dyn FileReadHandle>,
|
||||
cancellation: CancellationToken,
|
||||
operation: Semaphore,
|
||||
max_chunk_bytes: usize,
|
||||
}
|
||||
|
||||
struct WriteHandleEntry {
|
||||
handle: Box<dyn FileWriteHandle>,
|
||||
cancellation: CancellationToken,
|
||||
operation: Semaphore,
|
||||
max_chunk_bytes: usize,
|
||||
}
|
||||
|
||||
impl FileHandleManager {
|
||||
#[cfg(test)]
|
||||
fn with_max_handles(max_handles: usize) -> Self {
|
||||
Self {
|
||||
handles: Arc::new(Mutex::new(HashMap::new())),
|
||||
max_handles,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn open_read(
|
||||
&self,
|
||||
file_system: Arc<dyn ExecutorFileSystem>,
|
||||
@@ -53,6 +73,7 @@ impl FileHandleManager {
|
||||
sandbox: Option<&FileSystemSandboxContext>,
|
||||
) -> io::Result<usize> {
|
||||
let opening = self.reserve_opening(&handle_id).await?;
|
||||
let _completion = OpeningCompletionGuard(opening.completed.clone());
|
||||
let handle = match file_system.open_file_for_read(path, sandbox).await {
|
||||
Ok(handle) => handle,
|
||||
Err(err) => {
|
||||
@@ -68,10 +89,16 @@ impl FileHandleManager {
|
||||
let entry = FileHandle::Read(Arc::new(ReadHandleEntry {
|
||||
handle,
|
||||
cancellation: opening.cancellation.clone(),
|
||||
operation: Semaphore::new(1),
|
||||
max_chunk_bytes,
|
||||
}));
|
||||
self.promote_opening(handle_id, &opening, entry).await?;
|
||||
Ok(max_chunk_bytes)
|
||||
match self.promote_opening(handle_id, &opening, entry).await {
|
||||
Ok(()) => Ok(max_chunk_bytes),
|
||||
Err(handle) => {
|
||||
let _ = close_file_handle(handle).await;
|
||||
Err(cancelled_handle_error())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn read(
|
||||
@@ -90,33 +117,16 @@ impl FileHandleManager {
|
||||
"file read max bytes must be greater than zero",
|
||||
));
|
||||
}
|
||||
let result = tokio::select! {
|
||||
_ = entry.cancellation.cancelled() => Err(cancelled_handle_error()),
|
||||
result = entry.handle.read(offset, max_bytes) => result,
|
||||
};
|
||||
let operation = acquire_operation(&entry.operation).await?;
|
||||
if entry.cancellation.is_cancelled() {
|
||||
return Err(cancelled_handle_error());
|
||||
}
|
||||
let result = entry.handle.read(offset, max_bytes).await;
|
||||
drop(operation);
|
||||
match result {
|
||||
Ok(data) => {
|
||||
let eof = if data.len() < max_bytes {
|
||||
true
|
||||
} else {
|
||||
let metadata = tokio::select! {
|
||||
_ = entry.cancellation.cancelled() => Err(cancelled_handle_error()),
|
||||
result = entry.handle.metadata() => result,
|
||||
};
|
||||
match metadata {
|
||||
Ok(metadata) => {
|
||||
offset.saturating_add(data.len() as u64) >= metadata.size_bytes
|
||||
}
|
||||
Err(err) => {
|
||||
self.remove_if_read(handle_id, &entry).await;
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
};
|
||||
Ok(FileReadChunk { data, eof })
|
||||
}
|
||||
Ok(chunk) => Ok(chunk),
|
||||
Err(err) => {
|
||||
self.remove_if_read(handle_id, &entry).await;
|
||||
self.remove_and_close_read(handle_id, &entry).await;
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
@@ -124,12 +134,14 @@ impl FileHandleManager {
|
||||
|
||||
pub async fn stat_read(&self, handle_id: &str) -> io::Result<OpenFileMetadata> {
|
||||
let entry = self.read_entry(handle_id).await?;
|
||||
let result = tokio::select! {
|
||||
_ = entry.cancellation.cancelled() => Err(cancelled_handle_error()),
|
||||
result = entry.handle.metadata() => result,
|
||||
};
|
||||
let operation = acquire_operation(&entry.operation).await?;
|
||||
if entry.cancellation.is_cancelled() {
|
||||
return Err(cancelled_handle_error());
|
||||
}
|
||||
let result = entry.handle.metadata().await;
|
||||
drop(operation);
|
||||
if result.is_err() {
|
||||
self.remove_if_read(handle_id, &entry).await;
|
||||
self.remove_and_close_read(handle_id, &entry).await;
|
||||
}
|
||||
result
|
||||
}
|
||||
@@ -142,6 +154,7 @@ impl FileHandleManager {
|
||||
sandbox: Option<&FileSystemSandboxContext>,
|
||||
) -> io::Result<usize> {
|
||||
let opening = self.reserve_opening(&handle_id).await?;
|
||||
let _completion = OpeningCompletionGuard(opening.completed.clone());
|
||||
let handle = match file_system.open_file_for_write(path, sandbox).await {
|
||||
Ok(handle) => handle,
|
||||
Err(err) => {
|
||||
@@ -157,10 +170,16 @@ impl FileHandleManager {
|
||||
let entry = FileHandle::Write(Arc::new(WriteHandleEntry {
|
||||
handle,
|
||||
cancellation: opening.cancellation.clone(),
|
||||
operation: Semaphore::new(1),
|
||||
max_chunk_bytes,
|
||||
}));
|
||||
self.promote_opening(handle_id, &opening, entry).await?;
|
||||
Ok(max_chunk_bytes)
|
||||
match self.promote_opening(handle_id, &opening, entry).await {
|
||||
Ok(()) => Ok(max_chunk_bytes),
|
||||
Err(handle) => {
|
||||
let _ = close_file_handle(handle).await;
|
||||
Err(cancelled_handle_error())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn write(&self, handle_id: &str, data: &[u8]) -> io::Result<()> {
|
||||
@@ -174,34 +193,33 @@ impl FileHandleManager {
|
||||
),
|
||||
));
|
||||
}
|
||||
let result = tokio::select! {
|
||||
_ = entry.cancellation.cancelled() => Err(cancelled_handle_error()),
|
||||
result = entry.handle.write(data) => result,
|
||||
};
|
||||
let operation = acquire_operation(&entry.operation).await?;
|
||||
if entry.cancellation.is_cancelled() {
|
||||
return Err(cancelled_handle_error());
|
||||
}
|
||||
let result = entry.handle.write(data).await;
|
||||
drop(operation);
|
||||
if result.is_err() {
|
||||
self.remove_if_write(handle_id, &entry).await;
|
||||
self.remove_and_close_write(handle_id, &entry).await;
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub async fn close(&self, handle_id: &str) {
|
||||
let handle = self.handles.lock().await.remove(handle_id);
|
||||
match handle {
|
||||
Some(FileHandle::Opening(entry)) => entry.cancellation.cancel(),
|
||||
Some(FileHandle::Read(entry)) => entry.cancellation.cancel(),
|
||||
Some(FileHandle::Write(entry)) => entry.cancellation.cancel(),
|
||||
None => {}
|
||||
}
|
||||
pub async fn close(&self, handle_id: &str) -> io::Result<()> {
|
||||
let handle = self.handles.lock().await.get(handle_id).cloned();
|
||||
let Some(handle) = handle else {
|
||||
return Ok(());
|
||||
};
|
||||
let result = close_file_handle(handle.clone()).await;
|
||||
self.remove_if_handle(handle_id, &handle).await;
|
||||
result
|
||||
}
|
||||
|
||||
pub async fn close_all(&self) {
|
||||
let handles = std::mem::take(&mut *self.handles.lock().await);
|
||||
for handle in handles.into_values() {
|
||||
match handle {
|
||||
FileHandle::Opening(entry) => entry.cancellation.cancel(),
|
||||
FileHandle::Read(entry) => entry.cancellation.cancel(),
|
||||
FileHandle::Write(entry) => entry.cancellation.cancel(),
|
||||
}
|
||||
let handles = self.handles.lock().await.clone();
|
||||
for (handle_id, handle) in handles {
|
||||
let _ = close_file_handle(handle.clone()).await;
|
||||
self.remove_if_handle(&handle_id, &handle).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -219,8 +237,15 @@ impl FileHandleManager {
|
||||
format!("file handle `{handle_id}` is already open"),
|
||||
));
|
||||
}
|
||||
if handles.len() >= self.max_handles {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
format!("open file handle limit of {} reached", self.max_handles),
|
||||
));
|
||||
}
|
||||
let opening = Arc::new(OpeningHandleEntry {
|
||||
cancellation: CancellationToken::new(),
|
||||
completed: CancellationToken::new(),
|
||||
});
|
||||
handles.insert(
|
||||
handle_id.to_string(),
|
||||
@@ -234,17 +259,14 @@ impl FileHandleManager {
|
||||
handle_id: String,
|
||||
opening: &Arc<OpeningHandleEntry>,
|
||||
handle: FileHandle,
|
||||
) -> io::Result<()> {
|
||||
) -> Result<(), FileHandle> {
|
||||
let mut handles = self.handles.lock().await;
|
||||
let owns_reservation = matches!(
|
||||
handles.get(&handle_id),
|
||||
Some(FileHandle::Opening(current)) if Arc::ptr_eq(current, opening)
|
||||
);
|
||||
if opening.cancellation.is_cancelled() || !owns_reservation {
|
||||
if owns_reservation {
|
||||
handles.remove(&handle_id);
|
||||
}
|
||||
return Err(cancelled_handle_error());
|
||||
return Err(handle);
|
||||
}
|
||||
handles.insert(handle_id, handle);
|
||||
Ok(())
|
||||
@@ -278,27 +300,105 @@ impl FileHandleManager {
|
||||
}
|
||||
}
|
||||
|
||||
async fn remove_if_read(&self, handle_id: &str, expected: &Arc<ReadHandleEntry>) {
|
||||
async fn remove_if_handle(&self, handle_id: &str, expected: &FileHandle) {
|
||||
let mut handles = self.handles.lock().await;
|
||||
if matches!(
|
||||
handles.get(handle_id),
|
||||
Some(FileHandle::Read(entry)) if Arc::ptr_eq(entry, expected)
|
||||
) {
|
||||
if handles
|
||||
.get(handle_id)
|
||||
.is_some_and(|current| current.ptr_eq(expected))
|
||||
{
|
||||
handles.remove(handle_id);
|
||||
expected.cancellation.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
async fn remove_if_write(&self, handle_id: &str, expected: &Arc<WriteHandleEntry>) {
|
||||
let mut handles = self.handles.lock().await;
|
||||
if matches!(
|
||||
handles.get(handle_id),
|
||||
Some(FileHandle::Write(entry)) if Arc::ptr_eq(entry, expected)
|
||||
) {
|
||||
handles.remove(handle_id);
|
||||
async fn remove_and_close_read(&self, handle_id: &str, expected: &Arc<ReadHandleEntry>) {
|
||||
let handle = FileHandle::Read(Arc::clone(expected));
|
||||
let owns_handle = self
|
||||
.handles
|
||||
.lock()
|
||||
.await
|
||||
.get(handle_id)
|
||||
.is_some_and(|current| current.ptr_eq(&handle));
|
||||
if owns_handle {
|
||||
expected.cancellation.cancel();
|
||||
let Ok(operation) = acquire_operation(&expected.operation).await else {
|
||||
return;
|
||||
};
|
||||
let _ = expected.handle.close().await;
|
||||
drop(operation);
|
||||
self.remove_if_handle(handle_id, &handle).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn remove_and_close_write(&self, handle_id: &str, expected: &Arc<WriteHandleEntry>) {
|
||||
let handle = FileHandle::Write(Arc::clone(expected));
|
||||
let owns_handle = self
|
||||
.handles
|
||||
.lock()
|
||||
.await
|
||||
.get(handle_id)
|
||||
.is_some_and(|current| current.ptr_eq(&handle));
|
||||
if owns_handle {
|
||||
expected.cancellation.cancel();
|
||||
let Ok(operation) = acquire_operation(&expected.operation).await else {
|
||||
return;
|
||||
};
|
||||
let _ = expected.handle.close().await;
|
||||
drop(operation);
|
||||
self.remove_if_handle(handle_id, &handle).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FileHandle {
|
||||
fn ptr_eq(&self, other: &Self) -> bool {
|
||||
match (self, other) {
|
||||
(Self::Opening(left), Self::Opening(right)) => Arc::ptr_eq(left, right),
|
||||
(Self::Read(left), Self::Read(right)) => Arc::ptr_eq(left, right),
|
||||
(Self::Write(left), Self::Write(right)) => Arc::ptr_eq(left, right),
|
||||
(Self::Opening(_), Self::Read(_) | Self::Write(_))
|
||||
| (Self::Read(_), Self::Opening(_) | Self::Write(_))
|
||||
| (Self::Write(_), Self::Opening(_) | Self::Read(_)) => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn close_file_handle(handle: FileHandle) -> io::Result<()> {
|
||||
match handle {
|
||||
FileHandle::Opening(entry) => {
|
||||
entry.cancellation.cancel();
|
||||
entry.completed.cancelled().await;
|
||||
Ok(())
|
||||
}
|
||||
FileHandle::Read(entry) => {
|
||||
entry.cancellation.cancel();
|
||||
let operation = acquire_operation(&entry.operation).await?;
|
||||
let result = entry.handle.close().await;
|
||||
drop(operation);
|
||||
result
|
||||
}
|
||||
FileHandle::Write(entry) => {
|
||||
entry.cancellation.cancel();
|
||||
let operation = acquire_operation(&entry.operation).await?;
|
||||
let result = entry.handle.close().await;
|
||||
drop(operation);
|
||||
result
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn acquire_operation(operation: &Semaphore) -> io::Result<tokio::sync::SemaphorePermit<'_>> {
|
||||
operation
|
||||
.acquire()
|
||||
.await
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "file handle is closed"))
|
||||
}
|
||||
|
||||
struct OpeningCompletionGuard(CancellationToken);
|
||||
|
||||
impl Drop for OpeningCompletionGuard {
|
||||
fn drop(&mut self) {
|
||||
self.0.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
fn unknown_handle_error(handle_id: &str) -> io::Error {
|
||||
|
||||
@@ -5,6 +5,7 @@ use codex_file_system::CreateDirectoryOptions;
|
||||
use codex_file_system::FileMetadata;
|
||||
use codex_file_system::ReadDirectoryEntry;
|
||||
use codex_file_system::RemoveOptions;
|
||||
use pretty_assertions::assert_eq;
|
||||
use std::path::Path;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::atomic::Ordering;
|
||||
@@ -161,9 +162,15 @@ impl FileReadHandle for TestReadHandle {
|
||||
fn read(
|
||||
&self,
|
||||
_offset: u64,
|
||||
_max_bytes: usize,
|
||||
) -> std::pin::Pin<Box<dyn std::future::Future<Output = io::Result<Vec<u8>>> + Send + '_>> {
|
||||
unreachable!("read is not used by these tests")
|
||||
max_bytes: usize,
|
||||
) -> std::pin::Pin<Box<dyn std::future::Future<Output = io::Result<FileReadChunk>> + Send + '_>>
|
||||
{
|
||||
Box::pin(async move {
|
||||
Ok(FileReadChunk {
|
||||
data: vec![b'x'; max_bytes],
|
||||
eof: true,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn metadata(
|
||||
@@ -173,6 +180,12 @@ impl FileReadHandle for TestReadHandle {
|
||||
> {
|
||||
unreachable!("metadata is not used by these tests")
|
||||
}
|
||||
|
||||
fn close(
|
||||
&self,
|
||||
) -> std::pin::Pin<Box<dyn std::future::Future<Output = io::Result<()>> + Send + '_>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TestReadHandle {
|
||||
@@ -196,6 +209,12 @@ impl FileWriteHandle for TestWriteHandle {
|
||||
) -> std::pin::Pin<Box<dyn std::future::Future<Output = io::Result<()>> + Send + 'a>> {
|
||||
unreachable!("write is not used by these tests")
|
||||
}
|
||||
|
||||
fn close(
|
||||
&self,
|
||||
) -> std::pin::Pin<Box<dyn std::future::Future<Output = io::Result<()>> + Send + '_>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TestWriteHandle {
|
||||
@@ -220,7 +239,12 @@ async fn close_during_read_open_prevents_handle_from_becoming_live() {
|
||||
};
|
||||
|
||||
file_system.read_started.notified().await;
|
||||
manager.close("read-1").await;
|
||||
let close = {
|
||||
let manager = manager.clone();
|
||||
tokio::spawn(async move { manager.close("read-1").await })
|
||||
};
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!close.is_finished());
|
||||
file_system.read_release.add_permits(1);
|
||||
|
||||
let err = open
|
||||
@@ -228,6 +252,7 @@ async fn close_during_read_open_prevents_handle_from_becoming_live() {
|
||||
.expect("join read open")
|
||||
.expect_err("closed read open should fail");
|
||||
assert_eq!(err.kind(), io::ErrorKind::Interrupted);
|
||||
close.await.expect("join read close").expect("close read");
|
||||
assert!(file_system.read_handle_dropped.load(Ordering::SeqCst));
|
||||
assert_eq!(
|
||||
manager
|
||||
@@ -255,7 +280,12 @@ async fn close_during_write_open_prevents_handle_from_becoming_live() {
|
||||
};
|
||||
|
||||
file_system.write_started.notified().await;
|
||||
manager.close("write-1").await;
|
||||
let close = {
|
||||
let manager = manager.clone();
|
||||
tokio::spawn(async move { manager.close("write-1").await })
|
||||
};
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!close.is_finished());
|
||||
file_system.write_release.add_permits(1);
|
||||
|
||||
let err = open
|
||||
@@ -263,6 +293,7 @@ async fn close_during_write_open_prevents_handle_from_becoming_live() {
|
||||
.expect("join write open")
|
||||
.expect_err("closed write open should fail");
|
||||
assert_eq!(err.kind(), io::ErrorKind::Interrupted);
|
||||
close.await.expect("join write close").expect("close write");
|
||||
assert!(file_system.write_handle_dropped.load(Ordering::SeqCst));
|
||||
assert_eq!(
|
||||
manager
|
||||
@@ -275,7 +306,7 @@ async fn close_during_write_open_prevents_handle_from_becoming_live() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delayed_open_cannot_overwrite_reused_handle_id() {
|
||||
async fn closing_open_reserves_handle_id_until_cleanup_finishes() {
|
||||
let manager = FileHandleManager::default();
|
||||
let first_file_system = Arc::new(BlockingFileSystem::default());
|
||||
let path = AbsolutePathBuf::from_absolute_path(std::env::temp_dir()).expect("absolute path");
|
||||
@@ -291,9 +322,36 @@ async fn delayed_open_cannot_overwrite_reused_handle_id() {
|
||||
};
|
||||
|
||||
first_file_system.read_started.notified().await;
|
||||
manager.close("read-1").await;
|
||||
let close = {
|
||||
let manager = manager.clone();
|
||||
tokio::spawn(async move { manager.close("read-1").await })
|
||||
};
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
let second_file_system = Arc::new(BlockingFileSystem::default());
|
||||
assert_eq!(
|
||||
manager
|
||||
.open_read(
|
||||
second_file_system.clone(),
|
||||
"read-1".to_string(),
|
||||
&path,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect_err("closing handle id should remain reserved")
|
||||
.kind(),
|
||||
io::ErrorKind::AlreadyExists
|
||||
);
|
||||
|
||||
first_file_system.read_release.add_permits(1);
|
||||
let err = first_open
|
||||
.await
|
||||
.expect("join first read open")
|
||||
.expect_err("closed first read open should fail");
|
||||
assert_eq!(err.kind(), io::ErrorKind::Interrupted);
|
||||
close.await.expect("join read close").expect("close read");
|
||||
assert!(first_file_system.read_handle_dropped.load(Ordering::SeqCst));
|
||||
|
||||
let second_open = {
|
||||
let manager = manager.clone();
|
||||
let file_system = Arc::clone(&second_file_system);
|
||||
@@ -313,19 +371,73 @@ async fn delayed_open_cannot_overwrite_reused_handle_id() {
|
||||
16
|
||||
);
|
||||
|
||||
first_file_system.read_release.add_permits(1);
|
||||
let err = first_open
|
||||
.await
|
||||
.expect("join first read open")
|
||||
.expect_err("closed first read open should fail");
|
||||
assert_eq!(err.kind(), io::ErrorKind::Interrupted);
|
||||
assert!(first_file_system.read_handle_dropped.load(Ordering::SeqCst));
|
||||
assert!(manager.read_entry("read-1").await.is_ok());
|
||||
|
||||
manager.close("read-1").await;
|
||||
manager.close("read-1").await.expect("close second read");
|
||||
assert!(
|
||||
second_file_system
|
||||
.read_handle_dropped
|
||||
.load(Ordering::SeqCst)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_uses_handle_authoritative_eof() {
|
||||
let manager = FileHandleManager::default();
|
||||
let file_system = Arc::new(BlockingFileSystem::default());
|
||||
let path = AbsolutePathBuf::from_absolute_path(std::env::temp_dir()).expect("absolute path");
|
||||
let open = {
|
||||
let manager = manager.clone();
|
||||
let file_system = Arc::clone(&file_system);
|
||||
tokio::spawn(async move {
|
||||
manager
|
||||
.open_read(file_system, "read-1".to_string(), &path, None)
|
||||
.await
|
||||
})
|
||||
};
|
||||
|
||||
file_system.read_started.notified().await;
|
||||
file_system.read_release.add_permits(1);
|
||||
open.await
|
||||
.expect("join read open")
|
||||
.expect("open read handle");
|
||||
|
||||
assert_eq!(
|
||||
manager.read("read-1", 0, Some(16)).await.expect("read"),
|
||||
FileReadChunk {
|
||||
data: vec![b'x'; 16],
|
||||
eof: true,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn open_handle_count_is_bounded() {
|
||||
let manager = FileHandleManager::with_max_handles(1);
|
||||
let file_system = Arc::new(BlockingFileSystem::default());
|
||||
let path = AbsolutePathBuf::from_absolute_path(std::env::temp_dir()).expect("absolute path");
|
||||
let open = {
|
||||
let manager = manager.clone();
|
||||
let file_system = Arc::clone(&file_system);
|
||||
let path = path.clone();
|
||||
tokio::spawn(async move {
|
||||
manager
|
||||
.open_read(file_system, "read-1".to_string(), &path, None)
|
||||
.await
|
||||
})
|
||||
};
|
||||
|
||||
file_system.read_started.notified().await;
|
||||
assert_eq!(
|
||||
manager
|
||||
.open_read(file_system.clone(), "read-2".to_string(), &path, None)
|
||||
.await
|
||||
.expect_err("second handle should exceed limit")
|
||||
.kind(),
|
||||
io::ErrorKind::InvalidInput
|
||||
);
|
||||
|
||||
file_system.read_release.add_permits(1);
|
||||
open.await
|
||||
.expect("join read open")
|
||||
.expect("open read handle");
|
||||
manager.close("read-1").await.expect("close read");
|
||||
}
|
||||
|
||||
@@ -36,6 +36,7 @@ pub use codex_file_system::CopyOptions;
|
||||
pub use codex_file_system::CreateDirectoryOptions;
|
||||
pub use codex_file_system::ExecutorFileSystem;
|
||||
pub use codex_file_system::FileMetadata;
|
||||
pub use codex_file_system::FileReadChunk;
|
||||
pub use codex_file_system::FileReadHandle;
|
||||
pub use codex_file_system::FileSystemResult;
|
||||
pub use codex_file_system::FileSystemSandboxContext;
|
||||
@@ -52,7 +53,6 @@ pub use environment_path::EnvironmentPathRef;
|
||||
pub use environment_provider::DefaultEnvironmentProvider;
|
||||
pub use environment_provider::EnvironmentProvider;
|
||||
pub use file_handle_manager::FileHandleManager;
|
||||
pub use file_handle_manager::FileReadChunk;
|
||||
pub use fs_helper::CODEX_FS_HELPER_ARG1;
|
||||
pub use fs_helper_main::main as run_fs_helper_main;
|
||||
pub use local_file_system::LOCAL_FS;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use async_trait::async_trait;
|
||||
use codex_file_system::DEFAULT_FILE_STREAM_CHUNK_BYTES;
|
||||
use codex_file_system::FileReadChunk;
|
||||
use codex_file_system::FileReadHandle;
|
||||
use codex_file_system::FileWriteHandle;
|
||||
use codex_file_system::OpenFileMetadata;
|
||||
@@ -377,7 +378,10 @@ impl ExecutorFileSystem for DirectFileSystem {
|
||||
sandbox: Option<&FileSystemSandboxContext>,
|
||||
) -> FileSystemResult<Box<dyn FileReadHandle>> {
|
||||
reject_sandbox_context(sandbox)?;
|
||||
let file = std::fs::File::open(path.as_path())?;
|
||||
let path = path.as_path().to_path_buf();
|
||||
let file = tokio::task::spawn_blocking(move || std::fs::File::open(path))
|
||||
.await
|
||||
.map_err(|err| io::Error::other(format!("filesystem task failed: {err}")))??;
|
||||
Ok(Box::new(DirectFileReadHandle { file }))
|
||||
}
|
||||
|
||||
@@ -387,8 +391,16 @@ impl ExecutorFileSystem for DirectFileSystem {
|
||||
sandbox: Option<&FileSystemSandboxContext>,
|
||||
) -> FileSystemResult<Box<dyn FileWriteHandle>> {
|
||||
reject_sandbox_context(sandbox)?;
|
||||
let file = std::fs::File::create(path.as_path())?;
|
||||
Ok(Box::new(DirectFileWriteHandle { file }))
|
||||
let path = path.as_path().to_path_buf();
|
||||
let file = tokio::task::spawn_blocking(move || std::fs::File::create(path))
|
||||
.await
|
||||
.map_err(|err| io::Error::other(format!("filesystem task failed: {err}")))??;
|
||||
Ok(Box::new(DirectFileWriteHandle {
|
||||
state: tokio::sync::Mutex::new(DirectFileWriteState {
|
||||
file: Some(file),
|
||||
write_task: None,
|
||||
}),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn create_directory(
|
||||
@@ -537,21 +549,26 @@ impl FileReadHandle for DirectFileReadHandle {
|
||||
&self,
|
||||
offset: u64,
|
||||
max_bytes: usize,
|
||||
) -> std::pin::Pin<Box<dyn std::future::Future<Output = FileSystemResult<Vec<u8>>> + Send + '_>>
|
||||
{
|
||||
) -> std::pin::Pin<
|
||||
Box<dyn std::future::Future<Output = FileSystemResult<FileReadChunk>> + Send + '_>,
|
||||
> {
|
||||
let file = self.file.try_clone();
|
||||
Box::pin(async move {
|
||||
let file = file?;
|
||||
let mut bytes = vec![0; max_bytes.min(DEFAULT_FILE_STREAM_CHUNK_BYTES)];
|
||||
let (bytes_read, mut bytes) =
|
||||
tokio::task::spawn_blocking(move || -> io::Result<(usize, Vec<u8>)> {
|
||||
let bytes_read = read_file_at(&file, &mut bytes, offset)?;
|
||||
Ok((bytes_read, bytes))
|
||||
})
|
||||
.await
|
||||
.map_err(|err| io::Error::other(format!("filesystem task failed: {err}")))??;
|
||||
bytes.truncate(bytes_read);
|
||||
Ok(bytes)
|
||||
tokio::task::spawn_blocking(move || -> io::Result<FileReadChunk> {
|
||||
let requested_bytes = bytes.len();
|
||||
let bytes_read = read_file_at(&file, &mut bytes, offset)?;
|
||||
bytes.truncate(bytes_read);
|
||||
let eof = if bytes_read < requested_bytes {
|
||||
true
|
||||
} else {
|
||||
offset.saturating_add(bytes_read as u64) >= file.metadata()?.len()
|
||||
};
|
||||
Ok(FileReadChunk { data: bytes, eof })
|
||||
})
|
||||
.await
|
||||
.map_err(|err| io::Error::other(format!("filesystem task failed: {err}")))?
|
||||
})
|
||||
}
|
||||
|
||||
@@ -573,6 +590,13 @@ impl FileReadHandle for DirectFileReadHandle {
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn close(
|
||||
&self,
|
||||
) -> std::pin::Pin<Box<dyn std::future::Future<Output = FileSystemResult<()>> + Send + '_>>
|
||||
{
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
@@ -592,8 +616,13 @@ fn read_file_at(file: &std::fs::File, bytes: &mut [u8], offset: u64) -> io::Resu
|
||||
std::io::Read::read(&mut file, bytes)
|
||||
}
|
||||
|
||||
struct DirectFileWriteState {
|
||||
file: Option<std::fs::File>,
|
||||
write_task: Option<tokio::task::JoinHandle<(std::fs::File, FileSystemResult<()>)>>,
|
||||
}
|
||||
|
||||
struct DirectFileWriteHandle {
|
||||
file: std::fs::File,
|
||||
state: tokio::sync::Mutex<DirectFileWriteState>,
|
||||
}
|
||||
|
||||
impl FileWriteHandle for DirectFileWriteHandle {
|
||||
@@ -601,12 +630,15 @@ impl FileWriteHandle for DirectFileWriteHandle {
|
||||
DEFAULT_FILE_STREAM_CHUNK_BYTES
|
||||
}
|
||||
|
||||
#[expect(
|
||||
clippy::await_holding_invalid_type,
|
||||
reason = "the state lock keeps the tracked blocking write attached across cancellation"
|
||||
)]
|
||||
fn write<'a>(
|
||||
&'a self,
|
||||
data: &'a [u8],
|
||||
) -> std::pin::Pin<Box<dyn std::future::Future<Output = FileSystemResult<()>> + Send + 'a>>
|
||||
{
|
||||
let file = self.file.try_clone();
|
||||
Box::pin(async move {
|
||||
if data.len() > DEFAULT_FILE_STREAM_CHUNK_BYTES {
|
||||
return Err(io::Error::new(
|
||||
@@ -616,16 +648,48 @@ impl FileWriteHandle for DirectFileWriteHandle {
|
||||
),
|
||||
));
|
||||
}
|
||||
let file = file?;
|
||||
let mut state = self.state.lock().await;
|
||||
finish_direct_write(&mut state).await?;
|
||||
let file = state.file.take().ok_or_else(|| {
|
||||
io::Error::new(io::ErrorKind::BrokenPipe, "file write handle is closed")
|
||||
})?;
|
||||
let data = data.to_vec();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
state.write_task = Some(tokio::task::spawn_blocking(move || {
|
||||
let mut file = file;
|
||||
std::io::Write::write_all(&mut file, &data)
|
||||
})
|
||||
.await
|
||||
.map_err(|err| io::Error::other(format!("filesystem task failed: {err}")))?
|
||||
let result = std::io::Write::write_all(&mut file, &data);
|
||||
(file, result)
|
||||
}));
|
||||
finish_direct_write(&mut state).await
|
||||
})
|
||||
}
|
||||
|
||||
#[expect(
|
||||
clippy::await_holding_invalid_type,
|
||||
reason = "close must join the tracked blocking write before dropping the file"
|
||||
)]
|
||||
fn close(
|
||||
&self,
|
||||
) -> std::pin::Pin<Box<dyn std::future::Future<Output = FileSystemResult<()>> + Send + '_>>
|
||||
{
|
||||
Box::pin(async move {
|
||||
let mut state = self.state.lock().await;
|
||||
let result = finish_direct_write(&mut state).await;
|
||||
state.file.take();
|
||||
result
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async fn finish_direct_write(state: &mut DirectFileWriteState) -> FileSystemResult<()> {
|
||||
let Some(write_task) = state.write_task.as_mut() else {
|
||||
return Ok(());
|
||||
};
|
||||
let (file, result) = write_task
|
||||
.await
|
||||
.map_err(|err| io::Error::other(format!("filesystem task failed: {err}")))?;
|
||||
state.write_task = None;
|
||||
state.file = Some(file);
|
||||
result
|
||||
}
|
||||
|
||||
fn reject_sandbox_context(sandbox: Option<&FileSystemSandboxContext>) -> io::Result<()> {
|
||||
@@ -799,3 +863,7 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "local_file_system_streaming_tests.rs"]
|
||||
mod streaming_tests;
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
use super::*;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
#[tokio::test]
|
||||
async fn close_waits_for_in_flight_blocking_write() -> io::Result<()> {
|
||||
let temp_dir = tempfile::TempDir::new()?;
|
||||
let path = temp_dir.path().join("output");
|
||||
let file = std::fs::File::create(&path)?;
|
||||
let (started_tx, started_rx) = std::sync::mpsc::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
let write_task = tokio::task::spawn_blocking(move || {
|
||||
started_tx.send(()).expect("signal write start");
|
||||
release_rx.recv().expect("release write");
|
||||
let mut file = file;
|
||||
let result = std::io::Write::write_all(&mut file, b"x");
|
||||
(file, result)
|
||||
});
|
||||
let handle = Arc::new(DirectFileWriteHandle {
|
||||
state: tokio::sync::Mutex::new(DirectFileWriteState {
|
||||
file: None,
|
||||
write_task: Some(write_task),
|
||||
}),
|
||||
});
|
||||
tokio::task::spawn_blocking(move || started_rx.recv())
|
||||
.await
|
||||
.expect("join write start")
|
||||
.expect("wait for write start");
|
||||
|
||||
let close = {
|
||||
let handle = Arc::clone(&handle);
|
||||
tokio::spawn(async move { handle.close().await })
|
||||
};
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!close.is_finished());
|
||||
|
||||
release_tx.send(()).expect("release write");
|
||||
close.await.expect("join close")?;
|
||||
assert_eq!(std::fs::read(path)?, b"x");
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
use async_trait::async_trait;
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose::STANDARD;
|
||||
use codex_file_system::FileReadChunk;
|
||||
use codex_file_system::FileReadHandle;
|
||||
use codex_file_system::FileWriteHandle;
|
||||
use codex_file_system::OpenFileMetadata;
|
||||
@@ -306,8 +307,9 @@ impl FileReadHandle for RemoteFileReadHandle {
|
||||
&self,
|
||||
offset: u64,
|
||||
max_bytes: usize,
|
||||
) -> std::pin::Pin<Box<dyn std::future::Future<Output = FileSystemResult<Vec<u8>>> + Send + '_>>
|
||||
{
|
||||
) -> std::pin::Pin<
|
||||
Box<dyn std::future::Future<Output = FileSystemResult<FileReadChunk>> + Send + '_>,
|
||||
> {
|
||||
Box::pin(async move {
|
||||
let response = self
|
||||
.client
|
||||
@@ -318,11 +320,15 @@ impl FileReadHandle for RemoteFileReadHandle {
|
||||
})
|
||||
.await
|
||||
.map_err(map_remote_error)?;
|
||||
STANDARD.decode(response.data_base64).map_err(|err| {
|
||||
let data = STANDARD.decode(response.data_base64).map_err(|err| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!("remote fs/readFile/read returned invalid base64 dataBase64: {err}"),
|
||||
)
|
||||
})?;
|
||||
Ok(FileReadChunk {
|
||||
data,
|
||||
eof: response.eof,
|
||||
})
|
||||
})
|
||||
}
|
||||
@@ -347,19 +353,20 @@ impl FileReadHandle for RemoteFileReadHandle {
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RemoteFileReadHandle {
|
||||
fn drop(&mut self) {
|
||||
let client = self.client.clone();
|
||||
let handle_id = self.handle_id.clone();
|
||||
if let Ok(runtime) = tokio::runtime::Handle::try_current() {
|
||||
runtime.spawn(async move {
|
||||
let _ = client
|
||||
.fs_read_file_close(FsReadFileCloseParams { handle_id })
|
||||
.await;
|
||||
});
|
||||
}
|
||||
fn close(
|
||||
&self,
|
||||
) -> std::pin::Pin<Box<dyn std::future::Future<Output = FileSystemResult<()>> + Send + '_>>
|
||||
{
|
||||
Box::pin(async move {
|
||||
self.client
|
||||
.fs_read_file_close(FsReadFileCloseParams {
|
||||
handle_id: self.handle_id.clone(),
|
||||
})
|
||||
.await
|
||||
.map_err(map_remote_error)?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -390,19 +397,20 @@ impl FileWriteHandle for RemoteFileWriteHandle {
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RemoteFileWriteHandle {
|
||||
fn drop(&mut self) {
|
||||
let client = self.client.clone();
|
||||
let handle_id = self.handle_id.clone();
|
||||
if let Ok(runtime) = tokio::runtime::Handle::try_current() {
|
||||
runtime.spawn(async move {
|
||||
let _ = client
|
||||
.fs_write_file_close(FsWriteFileCloseParams { handle_id })
|
||||
.await;
|
||||
});
|
||||
}
|
||||
fn close(
|
||||
&self,
|
||||
) -> std::pin::Pin<Box<dyn std::future::Future<Output = FileSystemResult<()>> + Send + '_>>
|
||||
{
|
||||
Box::pin(async move {
|
||||
self.client
|
||||
.fs_write_file_close(FsWriteFileCloseParams {
|
||||
handle_id: self.handle_id.clone(),
|
||||
})
|
||||
.await
|
||||
.map_err(map_remote_error)?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -151,7 +151,10 @@ impl FileSystemHandler {
|
||||
&self,
|
||||
params: FsReadFileCloseParams,
|
||||
) -> Result<FsReadFileCloseResponse, JSONRPCErrorError> {
|
||||
self.handles.close(¶ms.handle_id).await;
|
||||
self.handles
|
||||
.close(¶ms.handle_id)
|
||||
.await
|
||||
.map_err(map_fs_error)?;
|
||||
Ok(FsReadFileCloseResponse {})
|
||||
}
|
||||
|
||||
@@ -192,7 +195,10 @@ impl FileSystemHandler {
|
||||
&self,
|
||||
params: FsWriteFileCloseParams,
|
||||
) -> Result<FsWriteFileCloseResponse, JSONRPCErrorError> {
|
||||
self.handles.close(¶ms.handle_id).await;
|
||||
self.handles
|
||||
.close(¶ms.handle_id)
|
||||
.await
|
||||
.map_err(map_fs_error)?;
|
||||
Ok(FsWriteFileCloseResponse {})
|
||||
}
|
||||
|
||||
|
||||
@@ -48,6 +48,12 @@ pub struct OpenFileMetadata {
|
||||
pub modified_at_ms: i64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct FileReadChunk {
|
||||
pub data: Vec<u8>,
|
||||
pub eof: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct ReadDirectoryEntry {
|
||||
pub file_name: String,
|
||||
@@ -151,11 +157,13 @@ pub trait FileReadHandle: Send + Sync {
|
||||
&self,
|
||||
offset: u64,
|
||||
max_bytes: usize,
|
||||
) -> Pin<Box<dyn Future<Output = FileSystemResult<Vec<u8>>> + Send + '_>>;
|
||||
) -> Pin<Box<dyn Future<Output = FileSystemResult<FileReadChunk>> + Send + '_>>;
|
||||
|
||||
fn metadata(
|
||||
&self,
|
||||
) -> Pin<Box<dyn Future<Output = FileSystemResult<OpenFileMetadata>> + Send + '_>>;
|
||||
|
||||
fn close(&self) -> Pin<Box<dyn Future<Output = FileSystemResult<()>> + Send + '_>>;
|
||||
}
|
||||
|
||||
/// An open file used for bounded sequential writes.
|
||||
@@ -170,6 +178,8 @@ pub trait FileWriteHandle: Send + Sync {
|
||||
&'a self,
|
||||
data: &'a [u8],
|
||||
) -> Pin<Box<dyn Future<Output = FileSystemResult<()>> + Send + 'a>>;
|
||||
|
||||
fn close(&self) -> Pin<Box<dyn Future<Output = FileSystemResult<()>> + Send + '_>>;
|
||||
}
|
||||
|
||||
/// Abstract filesystem access used by components that may operate locally or via
|
||||
|
||||
@@ -163,6 +163,7 @@ Response:
|
||||
|
||||
- Handle IDs are client-supplied strings scoped to one connection.
|
||||
- Opening a duplicate active handle ID returns `INVALID_REQUEST`.
|
||||
- Each connection may have at most 64 opening or open file handles.
|
||||
- Operations for one handle are serialized. Different handles may run
|
||||
concurrently.
|
||||
- Reads are positional and do not maintain a server-side cursor.
|
||||
@@ -173,11 +174,9 @@ Response:
|
||||
- Backpressure comes from awaiting bounded read and write responses. Clients
|
||||
should use a bounded pipeline of up to two read requests to hide transport
|
||||
round-trip latency without accumulating unbounded response data.
|
||||
- Close operations are idempotent. App-server close requests bypass queued
|
||||
operations and cancel an active open or chunk operation. A close received
|
||||
while open is pending prevents that handle from becoming live. Exec-server
|
||||
handles bounded file RPCs in request order, so close takes effect before the
|
||||
next file operation.
|
||||
- Close operations are idempotent and serialized with other operations for the
|
||||
same handle. Close waits for an active operation to finish, rejects queued
|
||||
operations, and prevents a pending open from becoming live.
|
||||
- Closing a connection closes all of its handles.
|
||||
- Any filesystem or I/O error closes the affected handle. Protocol errors such
|
||||
as an unknown handle do not affect other handles.
|
||||
|
||||
Reference in New Issue
Block a user