exec-server: execute prepared HTTPS uploads

This commit is contained in:
Soheil Norouzi
2026-06-22 14:41:45 +00:00
parent 46abd7900d
commit ceb3692804
10 changed files with 719 additions and 23 deletions
+17
View File
@@ -50,6 +50,7 @@ use crate::protocol::ExecParams;
use crate::protocol::ExecResponse;
use crate::protocol::FILE_TRANSFER_CANCEL_METHOD;
use crate::protocol::FILE_TRANSFER_PREPARE_UPLOAD_METHOD;
use crate::protocol::FILE_TRANSFER_START_UPLOAD_METHOD;
use crate::protocol::FILE_TRANSFER_STATUS_METHOD;
use crate::protocol::FS_CANONICALIZE_METHOD;
use crate::protocol::FS_CLOSE_METHOD;
@@ -66,6 +67,8 @@ use crate::protocol::FileTransferCancelParams;
use crate::protocol::FileTransferCancelResponse;
use crate::protocol::FileTransferPrepareUploadParams;
use crate::protocol::FileTransferPrepareUploadResponse;
use crate::protocol::FileTransferStartUploadParams;
use crate::protocol::FileTransferStartUploadResponse;
use crate::protocol::FileTransferStatusParams;
use crate::protocol::FileTransferStatusResponse;
use crate::protocol::FsCanonicalizeParams;
@@ -423,6 +426,13 @@ impl LazyRemoteExecServerClient {
self.get().await?.file_transfer_prepare_upload(params).await
}
pub(crate) async fn file_transfer_start_upload(
&self,
params: FileTransferStartUploadParams,
) -> Result<FileTransferStartUploadResponse, ExecServerError> {
self.get().await?.file_transfer_start_upload(params).await
}
pub(crate) async fn file_transfer_status(
&self,
transfer_id: String,
@@ -547,6 +557,13 @@ impl ExecServerClient {
.await
}
pub async fn file_transfer_start_upload(
&self,
params: FileTransferStartUploadParams,
) -> Result<FileTransferStartUploadResponse, ExecServerError> {
self.call(FILE_TRANSFER_START_UPLOAD_METHOD, &params).await
}
pub async fn file_transfer_status(
&self,
transfer_id: String,
+24
View File
@@ -27,7 +27,10 @@ use crate::protocol::EnvironmentInfo;
use crate::protocol::FileTransferCancelResponse;
use crate::protocol::FileTransferPrepareUploadParams;
use crate::protocol::FileTransferPrepareUploadResponse;
use crate::protocol::FileTransferStartUploadParams;
use crate::protocol::FileTransferStartUploadResponse;
use crate::protocol::FileTransferStatusResponse;
use crate::protocol::PreparedFileUploadCapability;
use crate::protocol::ShellInfo;
use crate::remote::NoiseRendezvousEnvironmentConfig;
use crate::remote_file_system::RemoteFileSystem;
@@ -578,6 +581,18 @@ impl Environment {
client.file_transfer_prepare_upload(params).await
}
pub async fn file_transfer_start_upload(
&self,
params: FileTransferStartUploadParams,
) -> Result<FileTransferStartUploadResponse, ExecServerError> {
let client = self.remote_client.as_ref().ok_or_else(|| {
ExecServerError::Protocol(
"executor-owned file transfer requires a remote environment".to_string(),
)
})?;
client.file_transfer_start_upload(params).await
}
pub async fn file_transfer_status(
&self,
transfer_id: String,
@@ -652,6 +667,15 @@ impl EnvironmentInfo {
capabilities: EnvironmentCapabilities::default(),
}
}
pub(crate) fn executor(prepared_file_upload: Option<PreparedFileUploadCapability>) -> Self {
Self {
shell: codex_shell_command::shell_detect::default_user_shell().into(),
capabilities: EnvironmentCapabilities {
prepared_file_upload,
},
}
}
}
impl From<DetectedShell> for ShellInfo {
+1
View File
@@ -36,6 +36,7 @@ pub const HTTP_REQUEST_METHOD: &str = "http/request";
/// JSON-RPC notification method for streamed executor HTTP response bodies.
pub const HTTP_REQUEST_BODY_DELTA_METHOD: &str = "http/request/bodyDelta";
pub const FILE_TRANSFER_PREPARE_UPLOAD_METHOD: &str = "fileTransfer/prepareUpload";
pub const FILE_TRANSFER_START_UPLOAD_METHOD: &str = "fileTransfer/startUpload";
pub const FILE_TRANSFER_STATUS_METHOD: &str = "fileTransfer/status";
pub const FILE_TRANSFER_CANCEL_METHOD: &str = "fileTransfer/cancel";
+1
View File
@@ -1,5 +1,6 @@
mod file_system_handler;
mod file_transfer_handler;
mod file_transfer_http;
mod handler;
mod process_handler;
mod processor;
@@ -26,19 +26,28 @@ use crate::protocol::FileTransferDigestAlgorithm;
use crate::protocol::FileTransferOperationState;
use crate::protocol::FileTransferPrepareUploadParams;
use crate::protocol::FileTransferPrepareUploadResponse;
use crate::protocol::FileTransferStartUploadParams;
use crate::protocol::FileTransferStartUploadResponse;
use crate::protocol::FileTransferStatusParams;
use crate::protocol::FileTransferStatusResponse;
use crate::protocol::FileTransferUploadDescriptorKind;
use crate::protocol::MAX_PREPARED_FILE_UPLOAD_BYTES;
use crate::protocol::PREPARED_FILE_UPLOAD_PROTOCOL_VERSION;
use crate::protocol::PreparedFileUploadCapability;
use crate::rpc::file_transfer_session_lost;
use crate::rpc::internal_error;
use crate::rpc::invalid_params;
use crate::rpc::invalid_request;
use crate::rpc::not_found;
use crate::server::file_transfer_http::UploadOutcome;
use crate::server::file_transfer_http::upload_bytes;
use crate::server::file_transfer_http::validate_upload_descriptor;
pub(crate) const FILE_TRANSFER_ENABLED_ENV_VAR: &str =
"CODEX_EXEC_SERVER_PREPARED_FILE_UPLOAD_ENABLED";
const MAX_PREPARED_BYTES_PER_SESSION: u64 = 32 * 1024 * 1024;
const MAX_OPERATIONS_PER_SESSION: usize = 32;
const MAX_ACTIVE_UPLOADS_PER_SESSION: usize = 2;
#[cfg(test)]
const PREPARED_UPLOAD_TTL: Duration = Duration::from_millis(100);
#[cfg(not(test))]
@@ -72,6 +81,7 @@ struct UploadOperation {
state: FileTransferOperationState,
error: Option<String>,
terminal_at: Option<Instant>,
cancellation: CancellationToken,
}
impl FileTransferHandler {
@@ -96,6 +106,19 @@ impl FileTransferHandler {
handler
}
pub(crate) fn capability(&self) -> Option<PreparedFileUploadCapability> {
matches!(
self.inner.availability,
PreparedFileUploadAvailability::EnabledForDevelopment
)
.then_some(PreparedFileUploadCapability {
protocol_version: PREPARED_FILE_UPLOAD_PROTOCOL_VERSION,
max_upload_bytes: MAX_PREPARED_FILE_UPLOAD_BYTES,
descriptor_kinds: vec![FileTransferUploadDescriptorKind::HttpsPut],
supports_status_reconciliation: true,
})
}
pub(crate) async fn prepare_upload(
&self,
params: FileTransferPrepareUploadParams,
@@ -137,6 +160,7 @@ impl FileTransferHandler {
state: FileTransferOperationState::Prepared,
error: None,
terminal_at: None,
cancellation: CancellationToken::new(),
};
let mut operations = self.inner.operations.lock().await;
@@ -154,6 +178,70 @@ impl FileTransferHandler {
})
}
pub(crate) async fn start_upload(
&self,
params: FileTransferStartUploadParams,
) -> Result<FileTransferStartUploadResponse, JSONRPCErrorError> {
self.require_enabled()?;
self.validate_transfer_id(&params.transfer_id)?;
{
let mut operations = self.inner.operations.lock().await;
let active_upload_count = active_uploads(&operations);
let operation = operations
.get_mut(&params.transfer_id)
.ok_or_else(|| not_found("unknown prepared upload".to_string()))?;
expire_operation(operation, Instant::now());
if operation.state != FileTransferOperationState::Prepared {
return Err(invalid_request(format!(
"prepared upload is {}",
state_name(operation.state)
)));
}
if active_upload_count >= MAX_ACTIVE_UPLOADS_PER_SESSION {
return Err(invalid_request(
"active file upload quota exceeded".to_string(),
));
}
}
let descriptor = validate_upload_descriptor(params.descriptor).await?;
let (bytes, cancellation) = {
let mut operations = self.inner.operations.lock().await;
let active_upload_count = active_uploads(&operations);
let operation = operations
.get_mut(&params.transfer_id)
.ok_or_else(|| not_found("unknown prepared upload".to_string()))?;
expire_operation(operation, Instant::now());
if operation.state != FileTransferOperationState::Prepared {
return Err(invalid_request(format!(
"prepared upload is {}",
state_name(operation.state)
)));
}
if active_upload_count >= MAX_ACTIVE_UPLOADS_PER_SESSION {
return Err(invalid_request(
"active file upload quota exceeded".to_string(),
));
}
let bytes = operation
.bytes
.take()
.ok_or_else(|| internal_error("prepared upload bytes are missing".to_string()))?;
operation.state = FileTransferOperationState::Uploading;
(bytes, operation.cancellation.clone())
};
self.inner.expiry_changed.notify_one();
let transfer_id = params.transfer_id;
let handler = self.clone();
let task_transfer_id = transfer_id.clone();
let _task = self.inner.tasks.spawn(async move {
let outcome = upload_bytes(bytes, descriptor, cancellation).await;
handler.finish_upload(&task_transfer_id, outcome).await;
});
Ok(FileTransferStartUploadResponse { transfer_id })
}
pub(crate) async fn status(
&self,
params: FileTransferStatusParams,
@@ -179,14 +267,22 @@ impl FileTransferHandler {
.get_mut(&params.transfer_id)
.ok_or_else(|| not_found("unknown file transfer operation".to_string()))?;
expire_operation(operation, Instant::now());
if operation.state == FileTransferOperationState::Prepared {
operation.bytes = None;
set_terminal(
operation,
FileTransferOperationState::Canceled,
/*error*/ None,
);
self.inner.expiry_changed.notify_one();
match operation.state {
FileTransferOperationState::Prepared => {
operation.bytes = None;
set_terminal(
operation,
FileTransferOperationState::Canceled,
/*error*/ None,
);
self.inner.expiry_changed.notify_one();
}
FileTransferOperationState::Uploading => {
operation.cancellation.cancel();
operation.state = FileTransferOperationState::CancelRequested;
operation.error = None;
}
_ => {}
}
Ok(FileTransferCancelResponse {
state: operation.state,
@@ -199,12 +295,46 @@ impl FileTransferHandler {
{
let mut operations = self.inner.operations.lock().await;
for operation in operations.values_mut() {
operation.cancellation.cancel();
operation.bytes = None;
if matches!(
operation.state,
FileTransferOperationState::Uploading
| FileTransferOperationState::CancelRequested
) {
set_terminal(
operation,
FileTransferOperationState::CompletionUnknown,
Some("upload completion was lost with the executor session".to_string()),
);
}
}
}
self.inner.tasks.wait().await;
}
async fn finish_upload(&self, transfer_id: &str, outcome: UploadOutcome) {
let mut operations = self.inner.operations.lock().await;
let Some(operation) = operations.get_mut(transfer_id) else {
return;
};
match outcome {
UploadOutcome::Succeeded => set_terminal(
operation,
FileTransferOperationState::Succeeded,
/*error*/ None,
),
UploadOutcome::Failed(error) => {
set_terminal(operation, FileTransferOperationState::Failed, Some(error))
}
UploadOutcome::CompletionUnknown(error) => set_terminal(
operation,
FileTransferOperationState::CompletionUnknown,
Some(error),
),
}
}
fn require_enabled(&self) -> Result<(), JSONRPCErrorError> {
if matches!(
self.inner.availability,
@@ -321,6 +451,18 @@ fn prepared_bytes(operations: &HashMap<String, UploadOperation>) -> u64 {
.sum()
}
fn active_uploads(operations: &HashMap<String, UploadOperation>) -> usize {
operations
.values()
.filter(|operation| {
matches!(
operation.state,
FileTransferOperationState::Uploading | FileTransferOperationState::CancelRequested
)
})
.count()
}
fn status_response(transfer_id: &str, operation: &UploadOperation) -> FileTransferStatusResponse {
FileTransferStatusResponse {
transfer_id: transfer_id.to_string(),
@@ -350,6 +492,19 @@ fn set_terminal(
operation.terminal_at = Some(Instant::now());
}
fn state_name(state: FileTransferOperationState) -> &'static str {
match state {
FileTransferOperationState::Prepared => "prepared",
FileTransferOperationState::Uploading => "uploading",
FileTransferOperationState::CancelRequested => "cancel requested",
FileTransferOperationState::Succeeded => "succeeded",
FileTransferOperationState::Failed => "failed",
FileTransferOperationState::Canceled => "canceled",
FileTransferOperationState::CompletionUnknown => "completion unknown",
FileTransferOperationState::Expired => "expired",
}
}
fn unix_seconds(time: SystemTime) -> i64 {
time.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_secs() as i64)
@@ -5,9 +5,18 @@ use codex_protocol::models::PermissionProfile;
use codex_utils_path_uri::PathUri;
use pretty_assertions::assert_eq;
use tokio::time::timeout;
use wiremock::Mock;
use wiremock::MockServer;
use wiremock::ResponseTemplate;
use wiremock::matchers::body_bytes;
use wiremock::matchers::method;
use wiremock::matchers::path;
use super::*;
use crate::protocol::FileTransferHeader;
use crate::protocol::FileTransferUploadDescriptor;
use crate::rpc::FILE_TRANSFER_SESSION_LOST_ERROR_CODE;
use crate::server::file_transfer_http::validate_upload_descriptor;
fn test_runtime_paths() -> ExecServerRuntimePaths {
ExecServerRuntimePaths::new(
@@ -36,8 +45,25 @@ fn full_access_context() -> FileSystemSandboxContext {
FileSystemSandboxContext::from_permission_profile(PermissionProfile::Disabled)
}
fn upload_descriptor(url: String) -> FileTransferUploadDescriptor {
FileTransferUploadDescriptor::HttpsPut {
url,
headers: Vec::new(),
expires_at_unix_seconds: unix_seconds(SystemTime::now() + Duration::from_secs(60)),
}
}
#[tokio::test]
async fn prepare_captures_stable_bytes_and_metadata() {
async fn prepared_upload_uses_stable_executor_owned_bytes() {
let server = MockServer::start().await;
Mock::given(method("PUT"))
.and(path("/upload"))
.and(body_bytes(b"prepared bytes".as_slice()))
.respond_with(ResponseTemplate::new(200))
.expect(1)
.mount(&server)
.await;
let source_dir = tempfile::tempdir().expect("source tempdir");
let source = source_dir.path().join("report.txt");
tokio::fs::write(&source, b"prepared bytes")
@@ -58,14 +84,48 @@ async fn prepare_captures_stable_bytes_and_metadata() {
tokio::fs::write(&source, b"different bytes")
.await
.expect("mutate source");
{
let operations = handler.inner.operations.lock().await;
let snapshot = operations
.get(&prepared.transfer_id)
.and_then(|operation| operation.bytes.as_ref())
.expect("prepared snapshot should remain present");
assert_eq!(snapshot.as_slice(), b"prepared bytes");
}
let started = handler
.start_upload(FileTransferStartUploadParams {
transfer_id: prepared.transfer_id.clone(),
descriptor: upload_descriptor(format!("{}/upload", server.uri())),
})
.await
.expect("start upload");
assert_eq!(started.transfer_id, prepared.transfer_id);
let status = timeout(Duration::from_secs(2), async {
loop {
let status = handler
.status(FileTransferStatusParams {
transfer_id: started.transfer_id.clone(),
})
.await
.expect("upload status");
if status.state != FileTransferOperationState::Uploading {
break status;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("upload should finish");
assert_eq!(
status,
FileTransferStatusResponse {
transfer_id: started.transfer_id.clone(),
state: FileTransferOperationState::Succeeded,
error: None,
}
);
let replay = handler
.start_upload(FileTransferStartUploadParams {
transfer_id: started.transfer_id,
descriptor: upload_descriptor(format!("{}/replay", server.uri())),
})
.await
.expect_err("prepared upload is single use");
assert_eq!(replay.code, -32600);
handler.shutdown().await;
}
@@ -97,7 +157,7 @@ async fn prepare_enforces_byte_limit_and_cancel_drops_snapshot() {
assert_eq!(canceled.state, FileTransferOperationState::Canceled);
let status = handler
.status(FileTransferStatusParams {
transfer_id: prepared.transfer_id,
transfer_id: prepared.transfer_id.clone(),
})
.await
.expect("canceled status");
@@ -106,6 +166,84 @@ async fn prepare_enforces_byte_limit_and_cancel_drops_snapshot() {
let operations = handler.inner.operations.lock().await;
assert_eq!(prepared_bytes(&operations), 0);
}
let start = handler
.start_upload(FileTransferStartUploadParams {
transfer_id: prepared.transfer_id,
descriptor: upload_descriptor("https://safe.blob.core.windows.net/object".to_string()),
})
.await
.expect_err("canceled snapshot cannot be uploaded");
assert_eq!(start.code, -32600);
handler.shutdown().await;
}
#[tokio::test]
async fn cancel_after_remote_receives_body_reports_unknown_completion() {
let server = MockServer::start().await;
Mock::given(method("PUT"))
.and(path("/delayed"))
.and(body_bytes(b"committable bytes".as_slice()))
.respond_with(ResponseTemplate::new(200).set_delay(Duration::from_secs(2)))
.expect(1)
.mount(&server)
.await;
let source_dir = tempfile::tempdir().expect("source tempdir");
let source = source_dir.path().join("report.txt");
tokio::fs::write(&source, b"committable bytes")
.await
.expect("write source");
let handler = test_handler();
let prepared = handler
.prepare_upload(prepare_params(&source, /*max_bytes*/ 1024))
.await
.expect("prepare upload");
handler
.start_upload(FileTransferStartUploadParams {
transfer_id: prepared.transfer_id.clone(),
descriptor: upload_descriptor(format!("{}/delayed", server.uri())),
})
.await
.expect("start upload");
timeout(Duration::from_secs(1), async {
loop {
if server
.received_requests()
.await
.is_some_and(|requests| !requests.is_empty())
{
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("server should receive the complete request before cancellation");
let canceled = handler
.cancel(FileTransferCancelParams {
transfer_id: prepared.transfer_id.clone(),
})
.await
.expect("request cancellation");
assert_eq!(canceled.state, FileTransferOperationState::CancelRequested);
let status = timeout(Duration::from_secs(1), async {
loop {
let status = handler
.status(FileTransferStatusParams {
transfer_id: prepared.transfer_id.clone(),
})
.await
.expect("upload status");
if status.state != FileTransferOperationState::CancelRequested {
break status;
}
tokio::task::yield_now().await;
}
})
.await
.expect("cancellation should settle");
assert_eq!(status.state, FileTransferOperationState::CompletionUnknown);
handler.shutdown().await;
}
@@ -144,6 +282,7 @@ async fn prepared_snapshot_expires_without_a_follow_up_rpc() {
let operation = operations
.get(&prepared.transfer_id)
.expect("terminal status should be retained");
assert_eq!(operation.state, FileTransferOperationState::Expired);
assert!(operation.bytes.is_none());
}
handler.shutdown().await;
@@ -174,7 +313,7 @@ async fn terminal_records_do_not_poison_the_session_quota() {
}
#[tokio::test]
async fn disabled_handler_rejects_before_reading_and_detects_old_session_ids() {
async fn disabled_handler_rejects_before_reading_and_does_not_accept_old_session_ids() {
let disabled = FileTransferHandler::new(
test_runtime_paths(),
PreparedFileUploadAvailability::Disabled,
@@ -190,7 +329,7 @@ async fn disabled_handler_rejects_before_reading_and_detects_old_session_ids() {
let handler = test_handler();
let error = handler
.status(FileTransferStatusParams {
transfer_id: format!("old-generation:{}", Uuid::new_v4()),
transfer_id: format!("old-session:{}", Uuid::new_v4()),
})
.await
.expect_err("old session ID should be distinguishable from an unknown operation");
@@ -198,3 +337,34 @@ async fn disabled_handler_rejects_before_reading_and_detects_old_session_ids() {
disabled.shutdown().await;
handler.shutdown().await;
}
#[tokio::test]
async fn descriptor_policy_rejects_unsafe_transport_and_headers() {
let insecure =
validate_upload_descriptor(upload_descriptor("http://example.com/upload".to_string()))
.await
.expect_err("non-local HTTP URL must be rejected");
assert_eq!(insecure.code, -32602);
let server = MockServer::start().await;
let forbidden_header = validate_upload_descriptor(FileTransferUploadDescriptor::HttpsPut {
url: format!("{}/upload", server.uri()),
headers: vec![FileTransferHeader {
name: "Authorization".to_string(),
value: "secret".to_string(),
}],
expires_at_unix_seconds: unix_seconds(SystemTime::now() + Duration::from_secs(60)),
})
.await
.expect_err("authorization header must be rejected");
assert_eq!(forbidden_header.code, -32602);
let missing_expiry = validate_upload_descriptor(FileTransferUploadDescriptor::HttpsPut {
url: format!("{}/upload", server.uri()),
headers: Vec::new(),
expires_at_unix_seconds: 0,
})
.await
.expect_err("expired descriptor must be rejected");
assert_eq!(missing_expiry.code, -32602);
}
@@ -0,0 +1,300 @@
use std::net::IpAddr;
use std::time::Duration;
use std::time::SystemTime;
use std::time::UNIX_EPOCH;
use codex_app_server_protocol::JSONRPCErrorError;
use codex_client::build_reqwest_client_with_custom_ca;
use futures::stream;
use reqwest::Method;
use reqwest::Url;
use reqwest::header::CONTENT_LENGTH;
use reqwest::header::HeaderMap;
use reqwest::header::HeaderName;
use reqwest::header::HeaderValue;
use reqwest::redirect::Policy;
use tokio_util::sync::CancellationToken;
use zeroize::Zeroizing;
use crate::protocol::FileTransferHeader;
use crate::protocol::FileTransferUploadDescriptor;
use crate::rpc::invalid_params;
const MAX_DESCRIPTOR_URL_BYTES: usize = 8 * 1024;
const MAX_DESCRIPTOR_HEADERS: usize = 16;
const MAX_DESCRIPTOR_HEADER_BYTES: usize = 16 * 1024;
const MIN_DESCRIPTOR_LIFETIME: Duration = Duration::from_secs(30);
const MAX_DESCRIPTOR_LIFETIME: Duration = Duration::from_secs(10 * 60);
const DNS_TIMEOUT: Duration = Duration::from_secs(5);
const TRANSFER_CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
const TRANSFER_TIMEOUT: Duration = Duration::from_secs(5 * 60);
pub(super) enum UploadOutcome {
Succeeded,
Failed(String),
CompletionUnknown(String),
}
pub(super) struct ValidatedUploadDescriptor {
url: Url,
addresses: Vec<std::net::SocketAddr>,
headers: HeaderMap,
}
impl std::fmt::Debug for ValidatedUploadDescriptor {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("ValidatedUploadDescriptor")
.field("url", &"[REDACTED]")
.field("address_count", &self.addresses.len())
.field("header_count", &self.headers.len())
.finish()
}
}
pub(super) async fn validate_upload_descriptor(
descriptor: FileTransferUploadDescriptor,
) -> Result<ValidatedUploadDescriptor, JSONRPCErrorError> {
let FileTransferUploadDescriptor::HttpsPut {
url,
headers,
expires_at_unix_seconds,
} = descriptor;
validate_expiry(expires_at_unix_seconds)?;
if url.len() > MAX_DESCRIPTOR_URL_BYTES {
return Err(invalid_params(
"file transfer URL exceeds the size limit".to_string(),
));
}
let url =
Url::parse(&url).map_err(|_| invalid_params("file transfer URL is invalid".to_string()))?;
let local_development_url = is_local_development_url(&url);
if url.scheme() != "https" && !local_development_url {
return Err(invalid_params(
"file transfer URL must use HTTPS".to_string(),
));
}
if !url.username().is_empty() || url.password().is_some() || url.fragment().is_some() {
return Err(invalid_params(
"file transfer URL must not contain credentials or a fragment".to_string(),
));
}
let host = url
.host_str()
.ok_or_else(|| invalid_params("file transfer URL must contain a host".to_string()))?;
if !local_development_url && !is_trusted_transfer_host(host) {
return Err(invalid_params(
"file transfer URL host is not trusted".to_string(),
));
}
let port = url
.port_or_known_default()
.ok_or_else(|| invalid_params("file transfer URL has no valid port".to_string()))?;
if !local_development_url && port != 443 {
return Err(invalid_params(
"file transfer URL must use port 443".to_string(),
));
}
let addresses = tokio::time::timeout(DNS_TIMEOUT, tokio::net::lookup_host((host, port)))
.await
.map_err(|_| invalid_params("file transfer host resolution timed out".to_string()))?
.map_err(|_| invalid_params("file transfer host resolution failed".to_string()))?
.collect::<Vec<_>>();
if addresses.is_empty()
|| addresses
.iter()
.any(|address| is_disallowed_transfer_address(address.ip()) && !local_development_url)
{
return Err(invalid_params(
"file transfer URL resolved to a disallowed address".to_string(),
));
}
let azure_blob = host
.trim_end_matches('.')
.to_ascii_lowercase()
.ends_with(".blob.core.windows.net");
let headers = validate_headers(headers, azure_blob)?;
Ok(ValidatedUploadDescriptor {
url,
addresses,
headers,
})
}
pub(super) async fn upload_bytes(
bytes: Zeroizing<Vec<u8>>,
descriptor: ValidatedUploadDescriptor,
cancellation: CancellationToken,
) -> UploadOutcome {
let Some(host) = descriptor.url.host_str() else {
return UploadOutcome::Failed("transfer URL omitted host".to_string());
};
let mut builder = reqwest::Client::builder()
.connect_timeout(TRANSFER_CONNECT_TIMEOUT)
.timeout(TRANSFER_TIMEOUT)
.no_proxy()
.redirect(Policy::none());
builder = builder.resolve_to_addrs(host, &descriptor.addresses);
let client = match build_reqwest_client_with_custom_ca(builder) {
Ok(client) => client,
Err(_) => return UploadOutcome::Failed("failed to build transfer client".to_string()),
};
let size = bytes.len() as u64;
let body_stream = stream::unfold((bytes, 0usize), |(bytes, offset)| async move {
if offset >= bytes.len() {
return None;
}
let end = (offset + 64 * 1024).min(bytes.len());
let chunk = bytes::Bytes::copy_from_slice(&bytes[offset..end]);
Some((Ok::<_, std::convert::Infallible>(chunk), (bytes, end)))
});
let request = match client
.request(Method::PUT, descriptor.url)
.headers(descriptor.headers)
.header(CONTENT_LENGTH, size)
.body(reqwest::Body::wrap_stream(body_stream))
.build()
{
Ok(request) => request,
Err(_) => return UploadOutcome::Failed("failed to build transfer request".to_string()),
};
let response = tokio::select! {
_ = cancellation.cancelled() => {
return UploadOutcome::CompletionUnknown(
"upload cancellation was requested after dispatch".to_string(),
);
}
response = client.execute(request) => response,
};
let response = match response {
Ok(response) => response,
Err(_) => {
return UploadOutcome::CompletionUnknown(
"upload completion could not be confirmed".to_string(),
);
}
};
if response.status().is_success() {
UploadOutcome::Succeeded
} else {
UploadOutcome::CompletionUnknown(format!(
"transfer returned HTTP {}",
response.status().as_u16()
))
}
}
fn validate_expiry(expires_at_unix_seconds: i64) -> Result<(), JSONRPCErrorError> {
let now = unix_seconds(SystemTime::now());
let minimum = now.saturating_add(MIN_DESCRIPTOR_LIFETIME.as_secs() as i64);
let maximum = now.saturating_add(MAX_DESCRIPTOR_LIFETIME.as_secs() as i64);
if expires_at_unix_seconds < minimum || expires_at_unix_seconds > maximum {
return Err(invalid_params(
"file transfer descriptor expiry must be between 30 seconds and 10 minutes".to_string(),
));
}
Ok(())
}
fn validate_headers(
headers: Vec<FileTransferHeader>,
azure_blob: bool,
) -> Result<HeaderMap, JSONRPCErrorError> {
if headers.len() > MAX_DESCRIPTOR_HEADERS {
return Err(invalid_params(
"file transfer descriptor has too many headers".to_string(),
));
}
let mut header_bytes = 0usize;
let mut result = HeaderMap::new();
for header in headers {
header_bytes = header_bytes.saturating_add(header.name.len() + header.value.len());
if header_bytes > MAX_DESCRIPTOR_HEADER_BYTES {
return Err(invalid_params(
"file transfer descriptor headers exceed the size limit".to_string(),
));
}
let name = HeaderName::from_bytes(header.name.as_bytes())
.map_err(|_| invalid_params("file transfer header name is invalid".to_string()))?;
if !is_allowed_header(&name) {
return Err(invalid_params(format!(
"file transfer header `{name}` is not permitted"
)));
}
if result.contains_key(&name) {
return Err(invalid_params(format!(
"file transfer header `{name}` must not be repeated"
)));
}
let value = HeaderValue::from_str(&header.value)
.map_err(|_| invalid_params("file transfer header value is invalid".to_string()))?;
if name == "x-ms-blob-type" && value != "BlockBlob" {
return Err(invalid_params(
"file transfer x-ms-blob-type must be BlockBlob".to_string(),
));
}
result.insert(name, value);
}
if azure_blob && !result.contains_key("x-ms-blob-type") {
result.insert("x-ms-blob-type", HeaderValue::from_static("BlockBlob"));
}
Ok(result)
}
fn is_allowed_header(name: &HeaderName) -> bool {
matches!(
name.as_str(),
"content-type" | "content-disposition" | "x-ms-blob-type" | "x-ms-date" | "x-ms-version"
) || name.as_str().starts_with("x-ms-meta-")
}
fn is_trusted_transfer_host(host: &str) -> bool {
let host = host.trim_end_matches('.').to_ascii_lowercase();
host.ends_with(".blob.core.windows.net") || host.ends_with(".oaiusercontent.com")
}
fn is_local_development_url(url: &Url) -> bool {
cfg!(debug_assertions)
&& url.scheme() == "http"
&& url
.host_str()
.is_some_and(|host| matches!(host, "localhost" | "127.0.0.1" | "::1"))
}
fn is_disallowed_transfer_address(address: IpAddr) -> bool {
match address {
IpAddr::V4(address) => {
let octets = address.octets();
address.is_private()
|| address.is_link_local()
|| address.is_loopback()
|| address.is_unspecified()
|| address.is_multicast()
|| address.is_broadcast()
|| address.is_documentation()
|| octets[0] == 0
|| (octets[0] == 100 && (64..=127).contains(&octets[1]))
|| (octets[0] == 192 && octets[1] == 0 && octets[2] == 0)
|| (octets[0] == 198 && (18..=19).contains(&octets[1]))
|| octets[0] >= 240
}
IpAddr::V6(address) => {
let segments = address.segments();
address.is_loopback()
|| address.is_unspecified()
|| address.is_multicast()
|| (segments[0] & 0xfe00) == 0xfc00
|| (segments[0] & 0xffc0) == 0xfe80
|| (segments[0] == 0x2001 && segments[1] == 0x0db8)
|| address
.to_ipv4_mapped()
.is_some_and(|address| is_disallowed_transfer_address(IpAddr::V4(address)))
}
}
}
fn unix_seconds(time: SystemTime) -> i64 {
time.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_secs() as i64)
.unwrap_or(0)
}
+14 -2
View File
@@ -21,6 +21,8 @@ use crate::protocol::FileTransferCancelParams;
use crate::protocol::FileTransferCancelResponse;
use crate::protocol::FileTransferPrepareUploadParams;
use crate::protocol::FileTransferPrepareUploadResponse;
use crate::protocol::FileTransferStartUploadParams;
use crate::protocol::FileTransferStartUploadResponse;
use crate::protocol::FileTransferStatusParams;
use crate::protocol::FileTransferStatusResponse;
use crate::protocol::FsCanonicalizeParams;
@@ -166,8 +168,10 @@ impl ExecServerHandler {
}
pub(crate) fn environment_info(&self) -> Result<EnvironmentInfo, JSONRPCErrorError> {
self.require_initialized_for("environment info")?;
Ok(EnvironmentInfo::local())
let session = self.require_initialized_for("environment info")?;
Ok(EnvironmentInfo::executor(
session.file_transfer().capability(),
))
}
pub(crate) async fn file_transfer_prepare_upload(
@@ -178,6 +182,14 @@ impl ExecServerHandler {
session.file_transfer().prepare_upload(params).await
}
pub(crate) async fn file_transfer_start_upload(
&self,
params: FileTransferStartUploadParams,
) -> Result<FileTransferStartUploadResponse, JSONRPCErrorError> {
let session = self.require_initialized_for("file transfer")?;
session.file_transfer().start_upload(params).await
}
pub(crate) async fn file_transfer_status(
&self,
params: FileTransferStatusParams,
@@ -9,6 +9,7 @@ use crate::protocol::EXEC_WRITE_METHOD;
use crate::protocol::ExecParams;
use crate::protocol::FILE_TRANSFER_CANCEL_METHOD;
use crate::protocol::FILE_TRANSFER_PREPARE_UPLOAD_METHOD;
use crate::protocol::FILE_TRANSFER_START_UPLOAD_METHOD;
use crate::protocol::FILE_TRANSFER_STATUS_METHOD;
use crate::protocol::FS_CANONICALIZE_METHOD;
use crate::protocol::FS_CLOSE_METHOD;
@@ -23,6 +24,7 @@ use crate::protocol::FS_REMOVE_METHOD;
use crate::protocol::FS_WRITE_FILE_METHOD;
use crate::protocol::FileTransferCancelParams;
use crate::protocol::FileTransferPrepareUploadParams;
use crate::protocol::FileTransferStartUploadParams;
use crate::protocol::FileTransferStatusParams;
use crate::protocol::FsCanonicalizeParams;
use crate::protocol::FsCloseParams;
@@ -81,6 +83,12 @@ pub(crate) fn build_router() -> RpcRouter<ExecServerHandler> {
handler.file_transfer_prepare_upload(params).await
},
);
router.request(
FILE_TRANSFER_START_UPLOAD_METHOD,
|handler: Arc<ExecServerHandler>, params: FileTransferStartUploadParams| async move {
handler.file_transfer_start_upload(params).await
},
);
router.request(
FILE_TRANSFER_STATUS_METHOD,
|handler: Arc<ExecServerHandler>, params: FileTransferStatusParams| async move {
+9 -1
View File
@@ -3,6 +3,8 @@
mod common;
use codex_exec_server::Environment;
use codex_exec_server::EnvironmentCapabilities;
use codex_exec_server::EnvironmentInfo;
use common::exec_server::exec_server;
use pretty_assertions::assert_eq;
@@ -29,7 +31,13 @@ async fn remote_environment_fetches_info_from_exec_server() -> anyhow::Result<()
let remote_info = environment.info().await?;
let local_info = Environment::default_for_tests().info().await?;
assert_eq!(remote_info, local_info);
assert_eq!(
remote_info,
EnvironmentInfo {
shell: local_info.shell,
capabilities: EnvironmentCapabilities::default(),
}
);
server.shutdown().await?;
Ok(())