mirror of
https://github.com/openai/codex.git
synced 2026-11-19 22:30:42 +00:00
Stream app-server fs/readFile responses over stdio (#62599)
## Why Large file reads over stdio previously allocated both a base64 string and a complete JSON response before writing to stdout. ## What changed Keep file contents as bytes until the stdio writer encodes them directly into the newline-delimited JSON response through a bounded buffer. Clients still receive the same `dataBase64` field. ## Testing Compare streamed output byte for byte with the eagerly encoded response for empty files, inputs that require base64 padding, and files larger than the write buffer. Check that JSON-RPC conversion preserves the request ID and base64 payload. GitOrigin-RevId: 730f201f0905fd15e32b6e9c5b2ef02ca21c7532
This commit is contained in:
Generated
+1
@@ -2290,6 +2290,7 @@ name = "codex-app-server-protocol"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64 0.22.1",
|
||||
"codex-app-server-protocol-noop-macros",
|
||||
"codex-experimental-api-macros",
|
||||
"codex-extension-items",
|
||||
|
||||
@@ -14,6 +14,7 @@ workspace = true
|
||||
|
||||
[dependencies]
|
||||
anyhow = { workspace = true }
|
||||
base64 = { workspace = true }
|
||||
codex-experimental-api-macros = { workspace = true }
|
||||
codex-app-server-protocol-noop-macros = { workspace = true }
|
||||
codex-extension-items = { workspace = true }
|
||||
|
||||
@@ -339,6 +339,7 @@ macro_rules! client_request_definitions {
|
||||
#[allow(clippy::large_enum_variant)]
|
||||
pub enum ClientResponsePayload {
|
||||
$( $variant($response), )*
|
||||
FsReadFileBytes(v2::FsReadFileBytes),
|
||||
InterruptConversation(v1::InterruptConversationResponse),
|
||||
}
|
||||
|
||||
@@ -353,6 +354,10 @@ macro_rules! client_request_definitions {
|
||||
})
|
||||
}
|
||||
)*
|
||||
Self::FsReadFileBytes(file) => Some(ClientResponse::FsReadFile {
|
||||
request_id,
|
||||
response: file.into(),
|
||||
}),
|
||||
Self::InterruptConversation(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,6 +41,10 @@ fn interrupt_conversation_payload_stays_jsonrpc_only() -> Result<()> {
|
||||
#[test]
|
||||
fn client_response_jsonrpc_parts_preserve_payloads_and_request_ids() -> Result<()> {
|
||||
let payloads = [
|
||||
(
|
||||
ClientResponsePayload::FsReadFileBytes(v2::FsReadFileBytes { data: vec![0, 255] }),
|
||||
json!({ "dataBase64": "AP8=" }),
|
||||
),
|
||||
(
|
||||
ClientResponsePayload::GetAuthStatus(v1::GetAuthStatusResponse {
|
||||
auth_method: Some(AuthMode::Chatgpt),
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
use crate::JsonSchema;
|
||||
use crate::TS;
|
||||
use base64::Engine;
|
||||
use base64::display::Base64Display;
|
||||
use base64::engine::general_purpose::STANDARD;
|
||||
use codex_utils_absolute_path::AbsolutePathBuf;
|
||||
use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
use serde::Serializer;
|
||||
|
||||
/// Read a file from the host filesystem.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
|
||||
@@ -22,6 +26,26 @@ pub struct FsReadFileResponse {
|
||||
pub data_base64: String,
|
||||
}
|
||||
|
||||
/// Outgoing `fs/readFile` payload that encodes the file as base64 during serialization.
|
||||
#[derive(Serialize, Debug, Clone)]
|
||||
pub struct FsReadFileBytes {
|
||||
#[serde(rename = "dataBase64", serialize_with = "serialize_file_data")]
|
||||
pub data: Vec<u8>,
|
||||
}
|
||||
|
||||
fn serialize_file_data<S: Serializer>(data: &[u8], serializer: S) -> Result<S::Ok, S::Error> {
|
||||
// serde_json writes Display fragments directly into the JSON string.
|
||||
serializer.collect_str(&Base64Display::new(data, &STANDARD))
|
||||
}
|
||||
|
||||
impl From<FsReadFileBytes> for FsReadFileResponse {
|
||||
fn from(file: FsReadFileBytes) -> Self {
|
||||
Self {
|
||||
data_base64: STANDARD.encode(file.data),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Write a file on the host filesystem.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
|
||||
@@ -22,9 +22,9 @@ pub enum OutgoingMessage {
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct OutgoingResponse {
|
||||
pub struct OutgoingResponse<T = Box<ClientResponsePayload>> {
|
||||
pub id: RequestId,
|
||||
pub result: Box<ClientResponsePayload>,
|
||||
pub result: T,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize)]
|
||||
|
||||
@@ -9,11 +9,16 @@ use super::TransportEvent;
|
||||
use super::forward_incoming_message;
|
||||
use super::next_connection_id;
|
||||
use super::serialize_outgoing_message;
|
||||
use crate::outgoing_message::OutgoingMessage;
|
||||
use crate::outgoing_message::OutgoingResponse;
|
||||
use crate::outgoing_message::QueuedOutgoingMessage;
|
||||
use codex_app_server_protocol::ClientResponsePayload;
|
||||
use codex_app_server_protocol::FsReadFileBytes;
|
||||
use codex_app_server_protocol::InitializeParams;
|
||||
use codex_app_server_protocol::JSONRPCMessage;
|
||||
use codex_app_server_protocol::JSONRPCRequest;
|
||||
use std::io::BufRead;
|
||||
use std::io::BufWriter;
|
||||
use std::io::ErrorKind;
|
||||
use std::io::Result as IoResult;
|
||||
use std::io::Write;
|
||||
@@ -25,6 +30,48 @@ use tracing::debug;
|
||||
use tracing::error;
|
||||
use tracing::info;
|
||||
|
||||
// Batch the small chunks emitted by the streaming base64 encoder.
|
||||
const FILE_RESPONSE_BUFFER_SIZE: usize = 64 * 1024;
|
||||
|
||||
enum StdoutMessage {
|
||||
Line(String),
|
||||
FileRead(OutgoingResponse<FsReadFileBytes>),
|
||||
}
|
||||
|
||||
impl StdoutMessage {
|
||||
fn new(message: OutgoingMessage) -> Option<Self> {
|
||||
let message = match message {
|
||||
OutgoingMessage::Response(response) => {
|
||||
// Encoding the request ID and file bytes can only fail on I/O. Other
|
||||
// responses need a complete frame for serialization error recovery.
|
||||
if let ClientResponsePayload::FsReadFileBytes(result) = *response.result {
|
||||
return Some(Self::FileRead(OutgoingResponse {
|
||||
id: response.id,
|
||||
result,
|
||||
}));
|
||||
}
|
||||
OutgoingMessage::Response(response)
|
||||
}
|
||||
message => message,
|
||||
};
|
||||
let mut line = serialize_outgoing_message(message)?;
|
||||
line.push('\n');
|
||||
Some(Self::Line(line))
|
||||
}
|
||||
|
||||
fn write_to(self, output: &mut BufWriter<impl Write>) -> IoResult<()> {
|
||||
match self {
|
||||
// The previous frame was flushed. Forward a complete line without copying it.
|
||||
Self::Line(line) => output.get_mut().write_all(line.as_bytes()),
|
||||
Self::FileRead(response) => {
|
||||
serde_json::to_writer(&mut *output, &response)?;
|
||||
output.write_all(b"\n")?;
|
||||
output.flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn start_stdio_connection(
|
||||
transport_event_tx: mpsc::Sender<TransportEvent>,
|
||||
initialize_client_name_tx: oneshot::Sender<String>,
|
||||
@@ -80,13 +127,16 @@ pub async fn start_stdio_connection(
|
||||
// Keep stdout's blocking writes off Tokio's pool too. The forwarding future
|
||||
// owns writer_rx so cancelling it also releases producers stuck on a full queue.
|
||||
let (stdout_tx, mut stdout_rx) =
|
||||
mpsc::channel::<(String, oneshot::Sender<()>)>(/*buffer*/ 1);
|
||||
mpsc::channel::<(StdoutMessage, oneshot::Sender<()>)>(/*buffer*/ 1);
|
||||
std::thread::Builder::new()
|
||||
.name("app-server-stdout".to_string())
|
||||
.spawn(move || {
|
||||
let mut stdout = std::io::stdout().lock();
|
||||
while let Some((json, written_tx)) = stdout_rx.blocking_recv() {
|
||||
if let Err(err) = stdout.write_all(json.as_bytes()) {
|
||||
let mut stdout =
|
||||
BufWriter::with_capacity(FILE_RESPONSE_BUFFER_SIZE, std::io::stdout().lock());
|
||||
while let Some((message, written_tx)) = stdout_rx.blocking_recv() {
|
||||
if let Err(err) = message.write_to(&mut stdout) {
|
||||
// Discard any buffered tail so dropping the writer cannot retry it.
|
||||
let _ = stdout.into_parts();
|
||||
error!("Failed to write to stdout: {err}");
|
||||
break;
|
||||
}
|
||||
@@ -135,12 +185,11 @@ pub async fn start_stdio_connection(
|
||||
|
||||
let write_messages = async move {
|
||||
while let Some(queued_message) = writer_rx.recv().await {
|
||||
let Some(mut json) = serialize_outgoing_message(queued_message.message) else {
|
||||
let Some(message) = StdoutMessage::new(queued_message.message) else {
|
||||
continue;
|
||||
};
|
||||
json.push('\n');
|
||||
let (written_tx, written_rx) = oneshot::channel();
|
||||
if stdout_tx.send((json, written_tx)).await.is_err() || written_rx.await.is_err() {
|
||||
if stdout_tx.send((message, written_tx)).await.is_err() || written_rx.await.is_err() {
|
||||
break;
|
||||
}
|
||||
if let Some(write_complete_tx) = queued_message.write_complete_tx {
|
||||
@@ -192,3 +241,7 @@ fn stdio_initialize_client_name(line: &str) -> Option<String> {
|
||||
let params = serde_json::from_value::<InitializeParams>(params?).ok()?;
|
||||
Some(params.client_info.name)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "stdio_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
#![cfg(test)]
|
||||
|
||||
use super::*;
|
||||
use base64::Engine;
|
||||
use base64::engine::general_purpose::STANDARD;
|
||||
use codex_app_server_protocol::FsReadFileResponse;
|
||||
use codex_app_server_protocol::RequestId;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
#[test]
|
||||
fn file_read_writes_the_same_json_line_directly() {
|
||||
let mut output = BufWriter::with_capacity(FILE_RESPONSE_BUFFER_SIZE, Vec::new());
|
||||
let mut expected = Vec::new();
|
||||
for len in [0, 1, 2, FILE_RESPONSE_BUFFER_SIZE + 1] {
|
||||
let data: Vec<_> = (1..=u8::MAX).cycle().take(len).collect();
|
||||
let id = RequestId::String("id\"\n".to_string());
|
||||
let eager = OutgoingResponse {
|
||||
id: id.clone(),
|
||||
result: FsReadFileResponse {
|
||||
data_base64: STANDARD.encode(&data),
|
||||
},
|
||||
};
|
||||
serde_json::to_writer(&mut expected, &eager).expect("serialize eager response");
|
||||
expected.push(b'\n');
|
||||
|
||||
let message = OutgoingMessage::Response(OutgoingResponse {
|
||||
id,
|
||||
result: Box::new(ClientResponsePayload::FsReadFileBytes(FsReadFileBytes {
|
||||
data,
|
||||
})),
|
||||
});
|
||||
let message = StdoutMessage::new(message).expect("file response");
|
||||
assert!(matches!(message, StdoutMessage::FileRead(_)));
|
||||
message.write_to(&mut output).expect("write file response");
|
||||
assert_eq!(output.get_ref(), &expected);
|
||||
}
|
||||
}
|
||||
@@ -1360,11 +1360,18 @@ impl MessageProcessor {
|
||||
ClientRequest::EnvironmentStatus { params, .. } => {
|
||||
self.environment_processor.environment_status(params).await
|
||||
}
|
||||
ClientRequest::FsReadFile { params, .. } => self
|
||||
.fs_processor
|
||||
.read_file(params)
|
||||
.await
|
||||
.map(|response| Some(response.into())),
|
||||
ClientRequest::FsReadFile { params, .. } => {
|
||||
self.fs_processor.read_file(params).await.map(|file| {
|
||||
// Defer encoding to the stdio writer. Remote control serializes
|
||||
// for both segment sizing and transmission.
|
||||
let payload = if session.origin == crate::transport::ConnectionOrigin::Stdio {
|
||||
ClientResponsePayload::FsReadFileBytes(file)
|
||||
} else {
|
||||
ClientResponsePayload::FsReadFile(file.into())
|
||||
};
|
||||
Some(payload)
|
||||
})
|
||||
}
|
||||
ClientRequest::FsWriteFile { params, .. } => self
|
||||
.fs_processor
|
||||
.write_file(params)
|
||||
|
||||
@@ -13,8 +13,8 @@ use codex_app_server_protocol::FsGetMetadataResponse;
|
||||
use codex_app_server_protocol::FsReadDirectoryEntry;
|
||||
use codex_app_server_protocol::FsReadDirectoryParams;
|
||||
use codex_app_server_protocol::FsReadDirectoryResponse;
|
||||
use codex_app_server_protocol::FsReadFileBytes;
|
||||
use codex_app_server_protocol::FsReadFileParams;
|
||||
use codex_app_server_protocol::FsReadFileResponse;
|
||||
use codex_app_server_protocol::FsRemoveParams;
|
||||
use codex_app_server_protocol::FsRemoveResponse;
|
||||
use codex_app_server_protocol::FsUnwatchParams;
|
||||
@@ -64,16 +64,14 @@ impl FsRequestProcessor {
|
||||
pub(crate) async fn read_file(
|
||||
&self,
|
||||
params: FsReadFileParams,
|
||||
) -> Result<FsReadFileResponse, JSONRPCErrorError> {
|
||||
) -> Result<FsReadFileBytes, JSONRPCErrorError> {
|
||||
let path = PathUri::from_abs_path(¶ms.path);
|
||||
let bytes = self
|
||||
let data = self
|
||||
.file_system()?
|
||||
.read_file(&path, Default::default(), /*sandbox*/ None)
|
||||
.await
|
||||
.map_err(map_fs_error)?;
|
||||
Ok(FsReadFileResponse {
|
||||
data_base64: STANDARD.encode(bytes),
|
||||
})
|
||||
Ok(FsReadFileBytes { data })
|
||||
}
|
||||
|
||||
pub(crate) async fn write_file(
|
||||
|
||||
Reference in New Issue
Block a user