From ceb36928049ecb5efe6e0adf3fa98292adb90539 Mon Sep 17 00:00:00 2001 From: Soheil Norouzi Date: Mon, 22 Jun 2026 13:23:00 +0000 Subject: [PATCH] exec-server: execute prepared HTTPS uploads --- codex-rs/exec-server/src/client.rs | 17 + codex-rs/exec-server/src/environment.rs | 24 ++ codex-rs/exec-server/src/protocol.rs | 1 + codex-rs/exec-server/src/server.rs | 1 + .../src/server/file_transfer_handler.rs | 171 +++++++++- .../src/server/file_transfer_handler_tests.rs | 194 ++++++++++- .../src/server/file_transfer_http.rs | 300 ++++++++++++++++++ codex-rs/exec-server/src/server/handler.rs | 16 +- codex-rs/exec-server/src/server/registry.rs | 8 + codex-rs/exec-server/tests/health.rs | 10 +- 10 files changed, 719 insertions(+), 23 deletions(-) create mode 100644 codex-rs/exec-server/src/server/file_transfer_http.rs diff --git a/codex-rs/exec-server/src/client.rs b/codex-rs/exec-server/src/client.rs index c568d5ddac..8b1b26f925 100644 --- a/codex-rs/exec-server/src/client.rs +++ b/codex-rs/exec-server/src/client.rs @@ -50,6 +50,7 @@ use crate::protocol::ExecParams; use crate::protocol::ExecResponse; use crate::protocol::FILE_TRANSFER_CANCEL_METHOD; use crate::protocol::FILE_TRANSFER_PREPARE_UPLOAD_METHOD; +use crate::protocol::FILE_TRANSFER_START_UPLOAD_METHOD; use crate::protocol::FILE_TRANSFER_STATUS_METHOD; use crate::protocol::FS_CANONICALIZE_METHOD; use crate::protocol::FS_CLOSE_METHOD; @@ -66,6 +67,8 @@ use crate::protocol::FileTransferCancelParams; use crate::protocol::FileTransferCancelResponse; use crate::protocol::FileTransferPrepareUploadParams; use crate::protocol::FileTransferPrepareUploadResponse; +use crate::protocol::FileTransferStartUploadParams; +use crate::protocol::FileTransferStartUploadResponse; use crate::protocol::FileTransferStatusParams; use crate::protocol::FileTransferStatusResponse; use crate::protocol::FsCanonicalizeParams; @@ -423,6 +426,13 @@ impl LazyRemoteExecServerClient { self.get().await?.file_transfer_prepare_upload(params).await } + pub(crate) async fn file_transfer_start_upload( + &self, + params: FileTransferStartUploadParams, + ) -> Result { + self.get().await?.file_transfer_start_upload(params).await + } + pub(crate) async fn file_transfer_status( &self, transfer_id: String, @@ -547,6 +557,13 @@ impl ExecServerClient { .await } + pub async fn file_transfer_start_upload( + &self, + params: FileTransferStartUploadParams, + ) -> Result { + self.call(FILE_TRANSFER_START_UPLOAD_METHOD, ¶ms).await + } + pub async fn file_transfer_status( &self, transfer_id: String, diff --git a/codex-rs/exec-server/src/environment.rs b/codex-rs/exec-server/src/environment.rs index 1c7a2dcae7..e257c55e61 100644 --- a/codex-rs/exec-server/src/environment.rs +++ b/codex-rs/exec-server/src/environment.rs @@ -27,7 +27,10 @@ use crate::protocol::EnvironmentInfo; use crate::protocol::FileTransferCancelResponse; use crate::protocol::FileTransferPrepareUploadParams; use crate::protocol::FileTransferPrepareUploadResponse; +use crate::protocol::FileTransferStartUploadParams; +use crate::protocol::FileTransferStartUploadResponse; use crate::protocol::FileTransferStatusResponse; +use crate::protocol::PreparedFileUploadCapability; use crate::protocol::ShellInfo; use crate::remote::NoiseRendezvousEnvironmentConfig; use crate::remote_file_system::RemoteFileSystem; @@ -578,6 +581,18 @@ impl Environment { client.file_transfer_prepare_upload(params).await } + pub async fn file_transfer_start_upload( + &self, + params: FileTransferStartUploadParams, + ) -> Result { + let client = self.remote_client.as_ref().ok_or_else(|| { + ExecServerError::Protocol( + "executor-owned file transfer requires a remote environment".to_string(), + ) + })?; + client.file_transfer_start_upload(params).await + } + pub async fn file_transfer_status( &self, transfer_id: String, @@ -652,6 +667,15 @@ impl EnvironmentInfo { capabilities: EnvironmentCapabilities::default(), } } + + pub(crate) fn executor(prepared_file_upload: Option) -> Self { + Self { + shell: codex_shell_command::shell_detect::default_user_shell().into(), + capabilities: EnvironmentCapabilities { + prepared_file_upload, + }, + } + } } impl From for ShellInfo { diff --git a/codex-rs/exec-server/src/protocol.rs b/codex-rs/exec-server/src/protocol.rs index 0eb48696db..72b1eb7321 100644 --- a/codex-rs/exec-server/src/protocol.rs +++ b/codex-rs/exec-server/src/protocol.rs @@ -36,6 +36,7 @@ pub const HTTP_REQUEST_METHOD: &str = "http/request"; /// JSON-RPC notification method for streamed executor HTTP response bodies. pub const HTTP_REQUEST_BODY_DELTA_METHOD: &str = "http/request/bodyDelta"; pub const FILE_TRANSFER_PREPARE_UPLOAD_METHOD: &str = "fileTransfer/prepareUpload"; +pub const FILE_TRANSFER_START_UPLOAD_METHOD: &str = "fileTransfer/startUpload"; pub const FILE_TRANSFER_STATUS_METHOD: &str = "fileTransfer/status"; pub const FILE_TRANSFER_CANCEL_METHOD: &str = "fileTransfer/cancel"; diff --git a/codex-rs/exec-server/src/server.rs b/codex-rs/exec-server/src/server.rs index 3b70f60223..90379b0297 100644 --- a/codex-rs/exec-server/src/server.rs +++ b/codex-rs/exec-server/src/server.rs @@ -1,5 +1,6 @@ mod file_system_handler; mod file_transfer_handler; +mod file_transfer_http; mod handler; mod process_handler; mod processor; diff --git a/codex-rs/exec-server/src/server/file_transfer_handler.rs b/codex-rs/exec-server/src/server/file_transfer_handler.rs index a97dbcec12..572997659a 100644 --- a/codex-rs/exec-server/src/server/file_transfer_handler.rs +++ b/codex-rs/exec-server/src/server/file_transfer_handler.rs @@ -26,19 +26,28 @@ use crate::protocol::FileTransferDigestAlgorithm; use crate::protocol::FileTransferOperationState; use crate::protocol::FileTransferPrepareUploadParams; use crate::protocol::FileTransferPrepareUploadResponse; +use crate::protocol::FileTransferStartUploadParams; +use crate::protocol::FileTransferStartUploadResponse; use crate::protocol::FileTransferStatusParams; use crate::protocol::FileTransferStatusResponse; +use crate::protocol::FileTransferUploadDescriptorKind; use crate::protocol::MAX_PREPARED_FILE_UPLOAD_BYTES; +use crate::protocol::PREPARED_FILE_UPLOAD_PROTOCOL_VERSION; +use crate::protocol::PreparedFileUploadCapability; use crate::rpc::file_transfer_session_lost; use crate::rpc::internal_error; use crate::rpc::invalid_params; use crate::rpc::invalid_request; use crate::rpc::not_found; +use crate::server::file_transfer_http::UploadOutcome; +use crate::server::file_transfer_http::upload_bytes; +use crate::server::file_transfer_http::validate_upload_descriptor; pub(crate) const FILE_TRANSFER_ENABLED_ENV_VAR: &str = "CODEX_EXEC_SERVER_PREPARED_FILE_UPLOAD_ENABLED"; const MAX_PREPARED_BYTES_PER_SESSION: u64 = 32 * 1024 * 1024; const MAX_OPERATIONS_PER_SESSION: usize = 32; +const MAX_ACTIVE_UPLOADS_PER_SESSION: usize = 2; #[cfg(test)] const PREPARED_UPLOAD_TTL: Duration = Duration::from_millis(100); #[cfg(not(test))] @@ -72,6 +81,7 @@ struct UploadOperation { state: FileTransferOperationState, error: Option, terminal_at: Option, + cancellation: CancellationToken, } impl FileTransferHandler { @@ -96,6 +106,19 @@ impl FileTransferHandler { handler } + pub(crate) fn capability(&self) -> Option { + matches!( + self.inner.availability, + PreparedFileUploadAvailability::EnabledForDevelopment + ) + .then_some(PreparedFileUploadCapability { + protocol_version: PREPARED_FILE_UPLOAD_PROTOCOL_VERSION, + max_upload_bytes: MAX_PREPARED_FILE_UPLOAD_BYTES, + descriptor_kinds: vec![FileTransferUploadDescriptorKind::HttpsPut], + supports_status_reconciliation: true, + }) + } + pub(crate) async fn prepare_upload( &self, params: FileTransferPrepareUploadParams, @@ -137,6 +160,7 @@ impl FileTransferHandler { state: FileTransferOperationState::Prepared, error: None, terminal_at: None, + cancellation: CancellationToken::new(), }; let mut operations = self.inner.operations.lock().await; @@ -154,6 +178,70 @@ impl FileTransferHandler { }) } + pub(crate) async fn start_upload( + &self, + params: FileTransferStartUploadParams, + ) -> Result { + self.require_enabled()?; + self.validate_transfer_id(¶ms.transfer_id)?; + { + let mut operations = self.inner.operations.lock().await; + let active_upload_count = active_uploads(&operations); + let operation = operations + .get_mut(¶ms.transfer_id) + .ok_or_else(|| not_found("unknown prepared upload".to_string()))?; + expire_operation(operation, Instant::now()); + if operation.state != FileTransferOperationState::Prepared { + return Err(invalid_request(format!( + "prepared upload is {}", + state_name(operation.state) + ))); + } + if active_upload_count >= MAX_ACTIVE_UPLOADS_PER_SESSION { + return Err(invalid_request( + "active file upload quota exceeded".to_string(), + )); + } + } + let descriptor = validate_upload_descriptor(params.descriptor).await?; + let (bytes, cancellation) = { + let mut operations = self.inner.operations.lock().await; + let active_upload_count = active_uploads(&operations); + let operation = operations + .get_mut(¶ms.transfer_id) + .ok_or_else(|| not_found("unknown prepared upload".to_string()))?; + expire_operation(operation, Instant::now()); + if operation.state != FileTransferOperationState::Prepared { + return Err(invalid_request(format!( + "prepared upload is {}", + state_name(operation.state) + ))); + } + if active_upload_count >= MAX_ACTIVE_UPLOADS_PER_SESSION { + return Err(invalid_request( + "active file upload quota exceeded".to_string(), + )); + } + let bytes = operation + .bytes + .take() + .ok_or_else(|| internal_error("prepared upload bytes are missing".to_string()))?; + operation.state = FileTransferOperationState::Uploading; + (bytes, operation.cancellation.clone()) + }; + self.inner.expiry_changed.notify_one(); + + let transfer_id = params.transfer_id; + let handler = self.clone(); + let task_transfer_id = transfer_id.clone(); + let _task = self.inner.tasks.spawn(async move { + let outcome = upload_bytes(bytes, descriptor, cancellation).await; + handler.finish_upload(&task_transfer_id, outcome).await; + }); + + Ok(FileTransferStartUploadResponse { transfer_id }) + } + pub(crate) async fn status( &self, params: FileTransferStatusParams, @@ -179,14 +267,22 @@ impl FileTransferHandler { .get_mut(¶ms.transfer_id) .ok_or_else(|| not_found("unknown file transfer operation".to_string()))?; expire_operation(operation, Instant::now()); - if operation.state == FileTransferOperationState::Prepared { - operation.bytes = None; - set_terminal( - operation, - FileTransferOperationState::Canceled, - /*error*/ None, - ); - self.inner.expiry_changed.notify_one(); + match operation.state { + FileTransferOperationState::Prepared => { + operation.bytes = None; + set_terminal( + operation, + FileTransferOperationState::Canceled, + /*error*/ None, + ); + self.inner.expiry_changed.notify_one(); + } + FileTransferOperationState::Uploading => { + operation.cancellation.cancel(); + operation.state = FileTransferOperationState::CancelRequested; + operation.error = None; + } + _ => {} } Ok(FileTransferCancelResponse { state: operation.state, @@ -199,12 +295,46 @@ impl FileTransferHandler { { let mut operations = self.inner.operations.lock().await; for operation in operations.values_mut() { + operation.cancellation.cancel(); operation.bytes = None; + if matches!( + operation.state, + FileTransferOperationState::Uploading + | FileTransferOperationState::CancelRequested + ) { + set_terminal( + operation, + FileTransferOperationState::CompletionUnknown, + Some("upload completion was lost with the executor session".to_string()), + ); + } } } self.inner.tasks.wait().await; } + async fn finish_upload(&self, transfer_id: &str, outcome: UploadOutcome) { + let mut operations = self.inner.operations.lock().await; + let Some(operation) = operations.get_mut(transfer_id) else { + return; + }; + match outcome { + UploadOutcome::Succeeded => set_terminal( + operation, + FileTransferOperationState::Succeeded, + /*error*/ None, + ), + UploadOutcome::Failed(error) => { + set_terminal(operation, FileTransferOperationState::Failed, Some(error)) + } + UploadOutcome::CompletionUnknown(error) => set_terminal( + operation, + FileTransferOperationState::CompletionUnknown, + Some(error), + ), + } + } + fn require_enabled(&self) -> Result<(), JSONRPCErrorError> { if matches!( self.inner.availability, @@ -321,6 +451,18 @@ fn prepared_bytes(operations: &HashMap) -> u64 { .sum() } +fn active_uploads(operations: &HashMap) -> usize { + operations + .values() + .filter(|operation| { + matches!( + operation.state, + FileTransferOperationState::Uploading | FileTransferOperationState::CancelRequested + ) + }) + .count() +} + fn status_response(transfer_id: &str, operation: &UploadOperation) -> FileTransferStatusResponse { FileTransferStatusResponse { transfer_id: transfer_id.to_string(), @@ -350,6 +492,19 @@ fn set_terminal( operation.terminal_at = Some(Instant::now()); } +fn state_name(state: FileTransferOperationState) -> &'static str { + match state { + FileTransferOperationState::Prepared => "prepared", + FileTransferOperationState::Uploading => "uploading", + FileTransferOperationState::CancelRequested => "cancel requested", + FileTransferOperationState::Succeeded => "succeeded", + FileTransferOperationState::Failed => "failed", + FileTransferOperationState::Canceled => "canceled", + FileTransferOperationState::CompletionUnknown => "completion unknown", + FileTransferOperationState::Expired => "expired", + } +} + fn unix_seconds(time: SystemTime) -> i64 { time.duration_since(UNIX_EPOCH) .map(|duration| duration.as_secs() as i64) diff --git a/codex-rs/exec-server/src/server/file_transfer_handler_tests.rs b/codex-rs/exec-server/src/server/file_transfer_handler_tests.rs index 2d6b1ca47f..301ac525e4 100644 --- a/codex-rs/exec-server/src/server/file_transfer_handler_tests.rs +++ b/codex-rs/exec-server/src/server/file_transfer_handler_tests.rs @@ -5,9 +5,18 @@ use codex_protocol::models::PermissionProfile; use codex_utils_path_uri::PathUri; use pretty_assertions::assert_eq; use tokio::time::timeout; +use wiremock::Mock; +use wiremock::MockServer; +use wiremock::ResponseTemplate; +use wiremock::matchers::body_bytes; +use wiremock::matchers::method; +use wiremock::matchers::path; use super::*; +use crate::protocol::FileTransferHeader; +use crate::protocol::FileTransferUploadDescriptor; use crate::rpc::FILE_TRANSFER_SESSION_LOST_ERROR_CODE; +use crate::server::file_transfer_http::validate_upload_descriptor; fn test_runtime_paths() -> ExecServerRuntimePaths { ExecServerRuntimePaths::new( @@ -36,8 +45,25 @@ fn full_access_context() -> FileSystemSandboxContext { FileSystemSandboxContext::from_permission_profile(PermissionProfile::Disabled) } +fn upload_descriptor(url: String) -> FileTransferUploadDescriptor { + FileTransferUploadDescriptor::HttpsPut { + url, + headers: Vec::new(), + expires_at_unix_seconds: unix_seconds(SystemTime::now() + Duration::from_secs(60)), + } +} + #[tokio::test] -async fn prepare_captures_stable_bytes_and_metadata() { +async fn prepared_upload_uses_stable_executor_owned_bytes() { + let server = MockServer::start().await; + Mock::given(method("PUT")) + .and(path("/upload")) + .and(body_bytes(b"prepared bytes".as_slice())) + .respond_with(ResponseTemplate::new(200)) + .expect(1) + .mount(&server) + .await; + let source_dir = tempfile::tempdir().expect("source tempdir"); let source = source_dir.path().join("report.txt"); tokio::fs::write(&source, b"prepared bytes") @@ -58,14 +84,48 @@ async fn prepare_captures_stable_bytes_and_metadata() { tokio::fs::write(&source, b"different bytes") .await .expect("mutate source"); - { - let operations = handler.inner.operations.lock().await; - let snapshot = operations - .get(&prepared.transfer_id) - .and_then(|operation| operation.bytes.as_ref()) - .expect("prepared snapshot should remain present"); - assert_eq!(snapshot.as_slice(), b"prepared bytes"); - } + let started = handler + .start_upload(FileTransferStartUploadParams { + transfer_id: prepared.transfer_id.clone(), + descriptor: upload_descriptor(format!("{}/upload", server.uri())), + }) + .await + .expect("start upload"); + assert_eq!(started.transfer_id, prepared.transfer_id); + + let status = timeout(Duration::from_secs(2), async { + loop { + let status = handler + .status(FileTransferStatusParams { + transfer_id: started.transfer_id.clone(), + }) + .await + .expect("upload status"); + if status.state != FileTransferOperationState::Uploading { + break status; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("upload should finish"); + assert_eq!( + status, + FileTransferStatusResponse { + transfer_id: started.transfer_id.clone(), + state: FileTransferOperationState::Succeeded, + error: None, + } + ); + + let replay = handler + .start_upload(FileTransferStartUploadParams { + transfer_id: started.transfer_id, + descriptor: upload_descriptor(format!("{}/replay", server.uri())), + }) + .await + .expect_err("prepared upload is single use"); + assert_eq!(replay.code, -32600); handler.shutdown().await; } @@ -97,7 +157,7 @@ async fn prepare_enforces_byte_limit_and_cancel_drops_snapshot() { assert_eq!(canceled.state, FileTransferOperationState::Canceled); let status = handler .status(FileTransferStatusParams { - transfer_id: prepared.transfer_id, + transfer_id: prepared.transfer_id.clone(), }) .await .expect("canceled status"); @@ -106,6 +166,84 @@ async fn prepare_enforces_byte_limit_and_cancel_drops_snapshot() { let operations = handler.inner.operations.lock().await; assert_eq!(prepared_bytes(&operations), 0); } + + let start = handler + .start_upload(FileTransferStartUploadParams { + transfer_id: prepared.transfer_id, + descriptor: upload_descriptor("https://safe.blob.core.windows.net/object".to_string()), + }) + .await + .expect_err("canceled snapshot cannot be uploaded"); + assert_eq!(start.code, -32600); + handler.shutdown().await; +} + +#[tokio::test] +async fn cancel_after_remote_receives_body_reports_unknown_completion() { + let server = MockServer::start().await; + Mock::given(method("PUT")) + .and(path("/delayed")) + .and(body_bytes(b"committable bytes".as_slice())) + .respond_with(ResponseTemplate::new(200).set_delay(Duration::from_secs(2))) + .expect(1) + .mount(&server) + .await; + let source_dir = tempfile::tempdir().expect("source tempdir"); + let source = source_dir.path().join("report.txt"); + tokio::fs::write(&source, b"committable bytes") + .await + .expect("write source"); + let handler = test_handler(); + let prepared = handler + .prepare_upload(prepare_params(&source, /*max_bytes*/ 1024)) + .await + .expect("prepare upload"); + handler + .start_upload(FileTransferStartUploadParams { + transfer_id: prepared.transfer_id.clone(), + descriptor: upload_descriptor(format!("{}/delayed", server.uri())), + }) + .await + .expect("start upload"); + timeout(Duration::from_secs(1), async { + loop { + if server + .received_requests() + .await + .is_some_and(|requests| !requests.is_empty()) + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("server should receive the complete request before cancellation"); + + let canceled = handler + .cancel(FileTransferCancelParams { + transfer_id: prepared.transfer_id.clone(), + }) + .await + .expect("request cancellation"); + assert_eq!(canceled.state, FileTransferOperationState::CancelRequested); + let status = timeout(Duration::from_secs(1), async { + loop { + let status = handler + .status(FileTransferStatusParams { + transfer_id: prepared.transfer_id.clone(), + }) + .await + .expect("upload status"); + if status.state != FileTransferOperationState::CancelRequested { + break status; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("cancellation should settle"); + assert_eq!(status.state, FileTransferOperationState::CompletionUnknown); handler.shutdown().await; } @@ -144,6 +282,7 @@ async fn prepared_snapshot_expires_without_a_follow_up_rpc() { let operation = operations .get(&prepared.transfer_id) .expect("terminal status should be retained"); + assert_eq!(operation.state, FileTransferOperationState::Expired); assert!(operation.bytes.is_none()); } handler.shutdown().await; @@ -174,7 +313,7 @@ async fn terminal_records_do_not_poison_the_session_quota() { } #[tokio::test] -async fn disabled_handler_rejects_before_reading_and_detects_old_session_ids() { +async fn disabled_handler_rejects_before_reading_and_does_not_accept_old_session_ids() { let disabled = FileTransferHandler::new( test_runtime_paths(), PreparedFileUploadAvailability::Disabled, @@ -190,7 +329,7 @@ async fn disabled_handler_rejects_before_reading_and_detects_old_session_ids() { let handler = test_handler(); let error = handler .status(FileTransferStatusParams { - transfer_id: format!("old-generation:{}", Uuid::new_v4()), + transfer_id: format!("old-session:{}", Uuid::new_v4()), }) .await .expect_err("old session ID should be distinguishable from an unknown operation"); @@ -198,3 +337,34 @@ async fn disabled_handler_rejects_before_reading_and_detects_old_session_ids() { disabled.shutdown().await; handler.shutdown().await; } + +#[tokio::test] +async fn descriptor_policy_rejects_unsafe_transport_and_headers() { + let insecure = + validate_upload_descriptor(upload_descriptor("http://example.com/upload".to_string())) + .await + .expect_err("non-local HTTP URL must be rejected"); + assert_eq!(insecure.code, -32602); + + let server = MockServer::start().await; + let forbidden_header = validate_upload_descriptor(FileTransferUploadDescriptor::HttpsPut { + url: format!("{}/upload", server.uri()), + headers: vec![FileTransferHeader { + name: "Authorization".to_string(), + value: "secret".to_string(), + }], + expires_at_unix_seconds: unix_seconds(SystemTime::now() + Duration::from_secs(60)), + }) + .await + .expect_err("authorization header must be rejected"); + assert_eq!(forbidden_header.code, -32602); + + let missing_expiry = validate_upload_descriptor(FileTransferUploadDescriptor::HttpsPut { + url: format!("{}/upload", server.uri()), + headers: Vec::new(), + expires_at_unix_seconds: 0, + }) + .await + .expect_err("expired descriptor must be rejected"); + assert_eq!(missing_expiry.code, -32602); +} diff --git a/codex-rs/exec-server/src/server/file_transfer_http.rs b/codex-rs/exec-server/src/server/file_transfer_http.rs new file mode 100644 index 0000000000..0147933ab4 --- /dev/null +++ b/codex-rs/exec-server/src/server/file_transfer_http.rs @@ -0,0 +1,300 @@ +use std::net::IpAddr; +use std::time::Duration; +use std::time::SystemTime; +use std::time::UNIX_EPOCH; + +use codex_app_server_protocol::JSONRPCErrorError; +use codex_client::build_reqwest_client_with_custom_ca; +use futures::stream; +use reqwest::Method; +use reqwest::Url; +use reqwest::header::CONTENT_LENGTH; +use reqwest::header::HeaderMap; +use reqwest::header::HeaderName; +use reqwest::header::HeaderValue; +use reqwest::redirect::Policy; +use tokio_util::sync::CancellationToken; +use zeroize::Zeroizing; + +use crate::protocol::FileTransferHeader; +use crate::protocol::FileTransferUploadDescriptor; +use crate::rpc::invalid_params; + +const MAX_DESCRIPTOR_URL_BYTES: usize = 8 * 1024; +const MAX_DESCRIPTOR_HEADERS: usize = 16; +const MAX_DESCRIPTOR_HEADER_BYTES: usize = 16 * 1024; +const MIN_DESCRIPTOR_LIFETIME: Duration = Duration::from_secs(30); +const MAX_DESCRIPTOR_LIFETIME: Duration = Duration::from_secs(10 * 60); +const DNS_TIMEOUT: Duration = Duration::from_secs(5); +const TRANSFER_CONNECT_TIMEOUT: Duration = Duration::from_secs(10); +const TRANSFER_TIMEOUT: Duration = Duration::from_secs(5 * 60); + +pub(super) enum UploadOutcome { + Succeeded, + Failed(String), + CompletionUnknown(String), +} + +pub(super) struct ValidatedUploadDescriptor { + url: Url, + addresses: Vec, + headers: HeaderMap, +} + +impl std::fmt::Debug for ValidatedUploadDescriptor { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ValidatedUploadDescriptor") + .field("url", &"[REDACTED]") + .field("address_count", &self.addresses.len()) + .field("header_count", &self.headers.len()) + .finish() + } +} + +pub(super) async fn validate_upload_descriptor( + descriptor: FileTransferUploadDescriptor, +) -> Result { + let FileTransferUploadDescriptor::HttpsPut { + url, + headers, + expires_at_unix_seconds, + } = descriptor; + validate_expiry(expires_at_unix_seconds)?; + if url.len() > MAX_DESCRIPTOR_URL_BYTES { + return Err(invalid_params( + "file transfer URL exceeds the size limit".to_string(), + )); + } + let url = + Url::parse(&url).map_err(|_| invalid_params("file transfer URL is invalid".to_string()))?; + let local_development_url = is_local_development_url(&url); + if url.scheme() != "https" && !local_development_url { + return Err(invalid_params( + "file transfer URL must use HTTPS".to_string(), + )); + } + if !url.username().is_empty() || url.password().is_some() || url.fragment().is_some() { + return Err(invalid_params( + "file transfer URL must not contain credentials or a fragment".to_string(), + )); + } + let host = url + .host_str() + .ok_or_else(|| invalid_params("file transfer URL must contain a host".to_string()))?; + if !local_development_url && !is_trusted_transfer_host(host) { + return Err(invalid_params( + "file transfer URL host is not trusted".to_string(), + )); + } + let port = url + .port_or_known_default() + .ok_or_else(|| invalid_params("file transfer URL has no valid port".to_string()))?; + if !local_development_url && port != 443 { + return Err(invalid_params( + "file transfer URL must use port 443".to_string(), + )); + } + let addresses = tokio::time::timeout(DNS_TIMEOUT, tokio::net::lookup_host((host, port))) + .await + .map_err(|_| invalid_params("file transfer host resolution timed out".to_string()))? + .map_err(|_| invalid_params("file transfer host resolution failed".to_string()))? + .collect::>(); + if addresses.is_empty() + || addresses + .iter() + .any(|address| is_disallowed_transfer_address(address.ip()) && !local_development_url) + { + return Err(invalid_params( + "file transfer URL resolved to a disallowed address".to_string(), + )); + } + let azure_blob = host + .trim_end_matches('.') + .to_ascii_lowercase() + .ends_with(".blob.core.windows.net"); + let headers = validate_headers(headers, azure_blob)?; + Ok(ValidatedUploadDescriptor { + url, + addresses, + headers, + }) +} + +pub(super) async fn upload_bytes( + bytes: Zeroizing>, + descriptor: ValidatedUploadDescriptor, + cancellation: CancellationToken, +) -> UploadOutcome { + let Some(host) = descriptor.url.host_str() else { + return UploadOutcome::Failed("transfer URL omitted host".to_string()); + }; + let mut builder = reqwest::Client::builder() + .connect_timeout(TRANSFER_CONNECT_TIMEOUT) + .timeout(TRANSFER_TIMEOUT) + .no_proxy() + .redirect(Policy::none()); + builder = builder.resolve_to_addrs(host, &descriptor.addresses); + let client = match build_reqwest_client_with_custom_ca(builder) { + Ok(client) => client, + Err(_) => return UploadOutcome::Failed("failed to build transfer client".to_string()), + }; + let size = bytes.len() as u64; + let body_stream = stream::unfold((bytes, 0usize), |(bytes, offset)| async move { + if offset >= bytes.len() { + return None; + } + let end = (offset + 64 * 1024).min(bytes.len()); + let chunk = bytes::Bytes::copy_from_slice(&bytes[offset..end]); + Some((Ok::<_, std::convert::Infallible>(chunk), (bytes, end))) + }); + let request = match client + .request(Method::PUT, descriptor.url) + .headers(descriptor.headers) + .header(CONTENT_LENGTH, size) + .body(reqwest::Body::wrap_stream(body_stream)) + .build() + { + Ok(request) => request, + Err(_) => return UploadOutcome::Failed("failed to build transfer request".to_string()), + }; + let response = tokio::select! { + _ = cancellation.cancelled() => { + return UploadOutcome::CompletionUnknown( + "upload cancellation was requested after dispatch".to_string(), + ); + } + response = client.execute(request) => response, + }; + let response = match response { + Ok(response) => response, + Err(_) => { + return UploadOutcome::CompletionUnknown( + "upload completion could not be confirmed".to_string(), + ); + } + }; + if response.status().is_success() { + UploadOutcome::Succeeded + } else { + UploadOutcome::CompletionUnknown(format!( + "transfer returned HTTP {}", + response.status().as_u16() + )) + } +} + +fn validate_expiry(expires_at_unix_seconds: i64) -> Result<(), JSONRPCErrorError> { + let now = unix_seconds(SystemTime::now()); + let minimum = now.saturating_add(MIN_DESCRIPTOR_LIFETIME.as_secs() as i64); + let maximum = now.saturating_add(MAX_DESCRIPTOR_LIFETIME.as_secs() as i64); + if expires_at_unix_seconds < minimum || expires_at_unix_seconds > maximum { + return Err(invalid_params( + "file transfer descriptor expiry must be between 30 seconds and 10 minutes".to_string(), + )); + } + Ok(()) +} + +fn validate_headers( + headers: Vec, + azure_blob: bool, +) -> Result { + if headers.len() > MAX_DESCRIPTOR_HEADERS { + return Err(invalid_params( + "file transfer descriptor has too many headers".to_string(), + )); + } + let mut header_bytes = 0usize; + let mut result = HeaderMap::new(); + for header in headers { + header_bytes = header_bytes.saturating_add(header.name.len() + header.value.len()); + if header_bytes > MAX_DESCRIPTOR_HEADER_BYTES { + return Err(invalid_params( + "file transfer descriptor headers exceed the size limit".to_string(), + )); + } + let name = HeaderName::from_bytes(header.name.as_bytes()) + .map_err(|_| invalid_params("file transfer header name is invalid".to_string()))?; + if !is_allowed_header(&name) { + return Err(invalid_params(format!( + "file transfer header `{name}` is not permitted" + ))); + } + if result.contains_key(&name) { + return Err(invalid_params(format!( + "file transfer header `{name}` must not be repeated" + ))); + } + let value = HeaderValue::from_str(&header.value) + .map_err(|_| invalid_params("file transfer header value is invalid".to_string()))?; + if name == "x-ms-blob-type" && value != "BlockBlob" { + return Err(invalid_params( + "file transfer x-ms-blob-type must be BlockBlob".to_string(), + )); + } + result.insert(name, value); + } + if azure_blob && !result.contains_key("x-ms-blob-type") { + result.insert("x-ms-blob-type", HeaderValue::from_static("BlockBlob")); + } + Ok(result) +} + +fn is_allowed_header(name: &HeaderName) -> bool { + matches!( + name.as_str(), + "content-type" | "content-disposition" | "x-ms-blob-type" | "x-ms-date" | "x-ms-version" + ) || name.as_str().starts_with("x-ms-meta-") +} + +fn is_trusted_transfer_host(host: &str) -> bool { + let host = host.trim_end_matches('.').to_ascii_lowercase(); + host.ends_with(".blob.core.windows.net") || host.ends_with(".oaiusercontent.com") +} + +fn is_local_development_url(url: &Url) -> bool { + cfg!(debug_assertions) + && url.scheme() == "http" + && url + .host_str() + .is_some_and(|host| matches!(host, "localhost" | "127.0.0.1" | "::1")) +} + +fn is_disallowed_transfer_address(address: IpAddr) -> bool { + match address { + IpAddr::V4(address) => { + let octets = address.octets(); + address.is_private() + || address.is_link_local() + || address.is_loopback() + || address.is_unspecified() + || address.is_multicast() + || address.is_broadcast() + || address.is_documentation() + || octets[0] == 0 + || (octets[0] == 100 && (64..=127).contains(&octets[1])) + || (octets[0] == 192 && octets[1] == 0 && octets[2] == 0) + || (octets[0] == 198 && (18..=19).contains(&octets[1])) + || octets[0] >= 240 + } + IpAddr::V6(address) => { + let segments = address.segments(); + address.is_loopback() + || address.is_unspecified() + || address.is_multicast() + || (segments[0] & 0xfe00) == 0xfc00 + || (segments[0] & 0xffc0) == 0xfe80 + || (segments[0] == 0x2001 && segments[1] == 0x0db8) + || address + .to_ipv4_mapped() + .is_some_and(|address| is_disallowed_transfer_address(IpAddr::V4(address))) + } + } +} + +fn unix_seconds(time: SystemTime) -> i64 { + time.duration_since(UNIX_EPOCH) + .map(|duration| duration.as_secs() as i64) + .unwrap_or(0) +} diff --git a/codex-rs/exec-server/src/server/handler.rs b/codex-rs/exec-server/src/server/handler.rs index edaf094555..7f0ea4d9f5 100644 --- a/codex-rs/exec-server/src/server/handler.rs +++ b/codex-rs/exec-server/src/server/handler.rs @@ -21,6 +21,8 @@ use crate::protocol::FileTransferCancelParams; use crate::protocol::FileTransferCancelResponse; use crate::protocol::FileTransferPrepareUploadParams; use crate::protocol::FileTransferPrepareUploadResponse; +use crate::protocol::FileTransferStartUploadParams; +use crate::protocol::FileTransferStartUploadResponse; use crate::protocol::FileTransferStatusParams; use crate::protocol::FileTransferStatusResponse; use crate::protocol::FsCanonicalizeParams; @@ -166,8 +168,10 @@ impl ExecServerHandler { } pub(crate) fn environment_info(&self) -> Result { - self.require_initialized_for("environment info")?; - Ok(EnvironmentInfo::local()) + let session = self.require_initialized_for("environment info")?; + Ok(EnvironmentInfo::executor( + session.file_transfer().capability(), + )) } pub(crate) async fn file_transfer_prepare_upload( @@ -178,6 +182,14 @@ impl ExecServerHandler { session.file_transfer().prepare_upload(params).await } + pub(crate) async fn file_transfer_start_upload( + &self, + params: FileTransferStartUploadParams, + ) -> Result { + let session = self.require_initialized_for("file transfer")?; + session.file_transfer().start_upload(params).await + } + pub(crate) async fn file_transfer_status( &self, params: FileTransferStatusParams, diff --git a/codex-rs/exec-server/src/server/registry.rs b/codex-rs/exec-server/src/server/registry.rs index ddfc5a8f22..276268e59c 100644 --- a/codex-rs/exec-server/src/server/registry.rs +++ b/codex-rs/exec-server/src/server/registry.rs @@ -9,6 +9,7 @@ use crate::protocol::EXEC_WRITE_METHOD; use crate::protocol::ExecParams; use crate::protocol::FILE_TRANSFER_CANCEL_METHOD; use crate::protocol::FILE_TRANSFER_PREPARE_UPLOAD_METHOD; +use crate::protocol::FILE_TRANSFER_START_UPLOAD_METHOD; use crate::protocol::FILE_TRANSFER_STATUS_METHOD; use crate::protocol::FS_CANONICALIZE_METHOD; use crate::protocol::FS_CLOSE_METHOD; @@ -23,6 +24,7 @@ use crate::protocol::FS_REMOVE_METHOD; use crate::protocol::FS_WRITE_FILE_METHOD; use crate::protocol::FileTransferCancelParams; use crate::protocol::FileTransferPrepareUploadParams; +use crate::protocol::FileTransferStartUploadParams; use crate::protocol::FileTransferStatusParams; use crate::protocol::FsCanonicalizeParams; use crate::protocol::FsCloseParams; @@ -81,6 +83,12 @@ pub(crate) fn build_router() -> RpcRouter { handler.file_transfer_prepare_upload(params).await }, ); + router.request( + FILE_TRANSFER_START_UPLOAD_METHOD, + |handler: Arc, params: FileTransferStartUploadParams| async move { + handler.file_transfer_start_upload(params).await + }, + ); router.request( FILE_TRANSFER_STATUS_METHOD, |handler: Arc, params: FileTransferStatusParams| async move { diff --git a/codex-rs/exec-server/tests/health.rs b/codex-rs/exec-server/tests/health.rs index 1fcfb049b5..85f755d026 100644 --- a/codex-rs/exec-server/tests/health.rs +++ b/codex-rs/exec-server/tests/health.rs @@ -3,6 +3,8 @@ mod common; use codex_exec_server::Environment; +use codex_exec_server::EnvironmentCapabilities; +use codex_exec_server::EnvironmentInfo; use common::exec_server::exec_server; use pretty_assertions::assert_eq; @@ -29,7 +31,13 @@ async fn remote_environment_fetches_info_from_exec_server() -> anyhow::Result<() let remote_info = environment.info().await?; let local_info = Environment::default_for_tests().info().await?; - assert_eq!(remote_info, local_info); + assert_eq!( + remote_info, + EnvironmentInfo { + shell: local_info.shell, + capabilities: EnvironmentCapabilities::default(), + } + ); server.shutdown().await?; Ok(())