From c1ed51ca2cd71cbbd4e6734f21cc3c9a9136d5d2 Mon Sep 17 00:00:00 2001 From: richardopenai Date: Wed, 18 Nov 2026 17:27:38 +0000 Subject: [PATCH] Accept inbound Noise connections in remote exec-server (#62325) ## Why Remote Noise executors previously connected outbound to a rendezvous relay. An inbound listener lets harnesses that can reach the executor connect directly while retaining registry-backed authorization. ## What changed Use `--listen ws://IP:PORT` with `--remote` in Noise mode to accept inbound WebSockets. The listener serves RPCs over encrypted Noise streams after the registry authorizes the harness. It limits concurrent connections and handshake attempts, closes sockets that miss the first authorization deadline, and drains connections on shutdown. Noise clients can attach optional admission headers from the connection bundle to `wss://` requests. ## Testing The inbound integration test exercises a registry-authorized file read, rejection of a registry-denied harness, the absence of a plaintext RPC response, and connection closure on shutdown. Unit tests cover handshake limits and rejection of admission headers before dialing a plaintext WebSocket. GitOrigin-RevId: b388672e49a3fe806e38f6db38825435ba25d27a --- codex-rs/cli/tests/exec_server.rs | 2 + codex-rs/core/tests/suite/remote_env.rs | 1 + codex-rs/exec-server-cli/src/command.rs | 24 ++- .../src/exec_server_args_tests.rs | 27 +++ codex-rs/exec-server/src/client_api.rs | 2 + .../exec-server/src/client_refresh_tests.rs | 3 +- codex-rs/exec-server/src/client_transport.rs | 16 ++ .../exec-server/src/client_transport_tests.rs | 31 +++ .../exec-server/src/noise_relay/admission.rs | 81 +++++++ .../src/noise_relay/executor_stream.rs | 19 +- .../src/noise_relay/executor_stream_tests.rs | 7 +- .../src/noise_relay/forward_stream_tests.rs | 7 +- codex-rs/exec-server/src/noise_relay/mod.rs | 3 +- codex-rs/exec-server/src/relay.rs | 35 +++- codex-rs/exec-server/src/relay_noise_tests.rs | 8 + codex-rs/exec-server/src/remote.rs | 83 +++++++- codex-rs/exec-server/src/server.rs | 2 + codex-rs/exec-server/src/server/transport.rs | 198 ++++++++++++++++-- codex-rs/exec-server/tests/common/relay.rs | 5 +- codex-rs/exec-server/tests/relay.rs | 139 ++++++++++++ .../tests/relay/registration_retry_tests.rs | 1 + 21 files changed, 646 insertions(+), 48 deletions(-) create mode 100644 codex-rs/exec-server/src/noise_relay/admission.rs diff --git a/codex-rs/cli/tests/exec_server.rs b/codex-rs/cli/tests/exec_server.rs index 8469b2ffee..372f79ecc4 100644 --- a/codex-rs/cli/tests/exec_server.rs +++ b/codex-rs/cli/tests/exec_server.rs @@ -580,6 +580,7 @@ metrics_exporter = {{ otlp-http = {{ endpoint = "{collector_url}/v1/metrics", pr let executor_public_key = registered_parent_lifetime_executor_public_key(®istry).await?; let harness_args = NoiseRendezvousConnectArgs { bundle: NoiseRendezvousConnectBundle { + http_headers: Default::default(), websocket_url: format!("{rendezvous_url}/relay?role=harness"), environment_id: ENVIRONMENT_ID.to_string(), executor_registration_id: EXECUTOR_REGISTRATION_ID.to_string(), @@ -870,6 +871,7 @@ metrics_exporter = {{ otlp-http = {{ endpoint = "{}/v1/metrics", protocol = "jso let executor_public_key = registered_parent_lifetime_executor_public_key(®istry).await?; let args = NoiseRendezvousConnectArgs { bundle: NoiseRendezvousConnectBundle { + http_headers: Default::default(), websocket_url: format!("{rendezvous}/relay?role=harness"), environment_id: ENVIRONMENT.to_string(), executor_registration_id: "registration-scoped".to_string(), diff --git a/codex-rs/core/tests/suite/remote_env.rs b/codex-rs/core/tests/suite/remote_env.rs index 25083ae5e9..936fde9087 100644 --- a/codex-rs/core/tests/suite/remote_env.rs +++ b/codex-rs/core/tests/suite/remote_env.rs @@ -1642,6 +1642,7 @@ impl NoiseRendezvousConnectProvider for OfflineThenReadyNoiseConnectProvider { }); } let bundle = NoiseRendezvousConnectBundle { + http_headers: Default::default(), websocket_url: self.websocket_url.clone(), environment_id: REMOTE_ENVIRONMENT_ID.to_string(), executor_registration_id: "ready-first-registration".to_string(), diff --git a/codex-rs/exec-server-cli/src/command.rs b/codex-rs/exec-server-cli/src/command.rs index fd630310c0..14bfe3ed67 100644 --- a/codex-rs/exec-server-cli/src/command.rs +++ b/codex-rs/exec-server-cli/src/command.rs @@ -68,11 +68,7 @@ pub struct ExecServerCommand { request_dispatch_mode: codex_exec_server::RequestDispatchMode, /// Transport endpoint URL. Supported values: `ws://IP:PORT` (default), `stdio`, `stdio://`. - #[arg( - long = "listen", - value_name = "URL", - conflicts_with = "exec_server_remote" - )] + #[arg(long = "listen", value_name = "URL")] listen: Option, #[command(flatten)] @@ -303,7 +299,13 @@ impl ExecServerCommand { let mut remote_config = codex_exec_server::RemoteEnvironmentConfig::new_with_transport( base_url, environment_id, - if direct_transport { + if let Some(listen) = self.listen.as_deref() { + let address = listen + .strip_prefix("ws://") + .and_then(|s| s.parse().ok()) + .ok_or_else(|| anyhow::anyhow!("remote --listen requires ws://IP:PORT"))?; + codex_exec_server::RemoteEnvironmentTransport::NoiseInbound(address) + } else if direct_transport { codex_exec_server::RemoteEnvironmentTransport::Direct } else { codex_exec_server::RemoteEnvironmentTransport::Noise @@ -428,6 +430,16 @@ impl ExecServerCommand { !self.chatgpt_token_stdin || self.command.is_none(), "--chatgpt-token-stdin cannot be used with forward" ); + anyhow::ensure!( + self.remote.is_none() + || self.listen.is_none() + || (self.remote_transport == ExecServerRemoteTransport::Noise + && self.command.is_none() + && self.listen.as_deref().is_some_and(|s| s + .strip_prefix("ws://") + .is_some_and(|v| v.parse::().is_ok()))), + "remote --listen requires ws://IP:PORT, Noise transport, and no forward" + ); match (self.remote_transport, self.aws_sigv4) { (ExecServerRemoteTransport::Noise, true) => { anyhow::bail!("--aws-sigv4 requires --remote-transport direct"); diff --git a/codex-rs/exec-server-cli/src/exec_server_args_tests.rs b/codex-rs/exec-server-cli/src/exec_server_args_tests.rs index d000825a2d..6d165701fb 100644 --- a/codex-rs/exec-server-cli/src/exec_server_args_tests.rs +++ b/codex-rs/exec-server-cli/src/exec_server_args_tests.rs @@ -351,3 +351,30 @@ fn standalone_help_documents_executor_options() { .join("\n"); insta::assert_snapshot!(help); } + +/// An inbound Noise listener requires remote registration and cannot expose direct RPC transport. +#[test] +fn inbound_noise_requires_registered_noise_transport() { + let command = exec_server_from_args(&[ + "--remote", + "https://registry.example.com", + "--environment-id", + "env-1", + "--listen", + "ws://0.0.0.0:9001", + ]); + assert_eq!(command.listen.as_deref(), Some("ws://0.0.0.0:9001")); + command.validate_remote_transport().unwrap(); + let command = exec_server_from_args(&[ + "--remote", + "https://registry.example.com", + "--environment-id", + "env-1", + "--remote-transport", + "direct", + "--aws-sigv4", + "--listen", + "ws://0.0.0.0:9001", + ]); + assert!(command.validate_remote_transport().is_err()); +} diff --git a/codex-rs/exec-server/src/client_api.rs b/codex-rs/exec-server/src/client_api.rs index e870f12471..5b1cce00e3 100644 --- a/codex-rs/exec-server/src/client_api.rs +++ b/codex-rs/exec-server/src/client_api.rs @@ -44,6 +44,8 @@ pub struct RemoteExecServerConnectArgs { /// physical connection attempt and must not be mixed with values from another /// registry response. pub struct NoiseRendezvousConnectBundle { + /// Admission headers for the authorized WebSocket destination. Never log them. + pub http_headers: HeaderMap, pub websocket_url: String, pub environment_id: String, pub executor_registration_id: String, diff --git a/codex-rs/exec-server/src/client_refresh_tests.rs b/codex-rs/exec-server/src/client_refresh_tests.rs index 255f99b599..db0e21b7c3 100644 --- a/codex-rs/exec-server/src/client_refresh_tests.rs +++ b/codex-rs/exec-server/src/client_refresh_tests.rs @@ -90,6 +90,7 @@ impl NoiseRendezvousConnectProvider for Registry { .map_err(|_| ExecServerError::Protocol("test lookup failed".to_owned()))?; } Ok(NoiseRendezvousConnectBundle { + http_headers: Default::default(), websocket_url: target.url, environment_id: "environment".to_owned(), executor_registration_id: target.registration, @@ -206,7 +207,7 @@ impl Executor { let validator = validator.clone(); connections.spawn(async move { let socket = tokio_tungstenite::accept_async(socket).await.unwrap(); - run_multiplexed_environment(socket, processor, "environment".to_owned(), executor.registration, executor.identity, validator).await; + run_multiplexed_environment(socket, processor, "environment".to_owned(), executor.registration, executor.identity, validator, /*admission*/ None).await; }); } _ = connections.join_next(), if !connections.is_empty() => {} diff --git a/codex-rs/exec-server/src/client_transport.rs b/codex-rs/exec-server/src/client_transport.rs index 51c2eb1482..ad7d7b418d 100644 --- a/codex-rs/exec-server/src/client_transport.rs +++ b/codex-rs/exec-server/src/client_transport.rs @@ -658,6 +658,7 @@ impl ExecServerClient { http_client_factory, } = args; let NoiseRendezvousConnectBundle { + http_headers, websocket_url, environment_id, executor_registration_id, @@ -676,6 +677,21 @@ impl ExecServerClient { url: diagnostic_url.clone(), source, })?; + if !http_headers.is_empty() && request.uri().scheme_str() != Some("wss") { + return Err(ExecServerError::Protocol( + "Noise WebSocket admission headers require wss://".to_string(), + )); + } + for (name, value) in &http_headers { + if is_websocket_handshake_header(name) { + return Err(ExecServerError::Protocol( + "invalid Noise WebSocket admission header".to_string(), + )); + } + let mut value = value.clone(); + value.set_sensitive(true); + request.headers_mut().insert(name, value); + } request.headers_mut().extend(current_rendezvous_headers()); let (stream, _) = timeout( connect_timeout, diff --git a/codex-rs/exec-server/src/client_transport_tests.rs b/codex-rs/exec-server/src/client_transport_tests.rs index 3ae10252bc..ad6f1ba672 100644 --- a/codex-rs/exec-server/src/client_transport_tests.rs +++ b/codex-rs/exec-server/src/client_transport_tests.rs @@ -161,6 +161,7 @@ impl NoiseRendezvousConnectProvider for SequenceNoiseConnectProvider { fn test_bundle(websocket_url: String) -> Result { Ok(NoiseRendezvousConnectBundle { + http_headers: Default::default(), websocket_url, environment_id: "environment".to_string(), executor_registration_id: "registration".to_string(), @@ -237,6 +238,35 @@ async fn noise_handshake_reports_disconnect_reason() -> Result<()> { Ok(()) } +#[tokio::test] +async fn noise_admission_headers_are_rejected_before_dialing_plaintext() -> Result<()> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let mut bundle = test_bundle(format!("ws://{}", listener.local_addr()?))?; + bundle.http_headers.insert( + http::header::AUTHORIZATION, + http::HeaderValue::from_static("fake-token"), + ); + let result = ExecServerClient::open_noise_rendezvous_connection(NoiseRendezvousConnectArgs { + bundle, + harness_identity: NoiseChannelIdentity::generate()?, + client_name: "test".to_string(), + connect_timeout: std::time::Duration::from_secs(1), + initialize_timeout: std::time::Duration::from_secs(1), + resume_session_id: None, + http_client_factory: codex_http_client::HttpClientFactory::new( + codex_http_client::OutboundProxyPolicy::ReqwestDefault, + ), + }) + .await; + assert!(matches!(result, Err(ExecServerError::Protocol(_)))); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(10), listener.accept()) + .await + .is_err() + ); + Ok(()) +} + #[tokio::test] async fn noise_handshake_uses_initialize_timeout() -> Result<()> { let listener = TcpListener::bind("127.0.0.1:0").await?; @@ -262,6 +292,7 @@ async fn noise_handshake_uses_initialize_timeout() -> Result<()> { let initialize_timeout = std::time::Duration::from_millis(1); let opened = ExecServerClient::open_noise_rendezvous_connection(NoiseRendezvousConnectArgs { bundle: NoiseRendezvousConnectBundle { + http_headers: Default::default(), websocket_url, environment_id: "environment".to_string(), executor_registration_id: "registration".to_string(), diff --git a/codex-rs/exec-server/src/noise_relay/admission.rs b/codex-rs/exec-server/src/noise_relay/admission.rs new file mode 100644 index 0000000000..8d8528e30e --- /dev/null +++ b/codex-rs/exec-server/src/noise_relay/admission.rs @@ -0,0 +1,81 @@ +//! Listener-wide admission for inbound Noise handshakes. The budget is charged +//! before parsing a hybrid handshake and held through its ERS validation. + +use std::collections::VecDeque; +use std::sync::Arc; +use std::sync::Mutex; +use std::time::Duration; + +use tokio::sync::Notify; +use tokio::sync::OwnedSemaphorePermit; +use tokio::sync::Semaphore; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; + +const MAX_CONCURRENT_HANDSHAKES: usize = 16; +const MAX_HANDSHAKES_PER_SECOND: usize = 32; +const HANDSHAKE_WINDOW: Duration = Duration::from_secs(1); + +pub(crate) struct HandshakeBudget { + permits: Arc, + attempts: Mutex>, +} + +impl Default for HandshakeBudget { + fn default() -> Self { + Self { + permits: Arc::new(Semaphore::new(MAX_CONCURRENT_HANDSHAKES)), + attempts: Mutex::new(VecDeque::new()), + } + } +} + +impl HandshakeBudget { + pub(crate) fn try_admit(&self) -> Option { + let permit = Arc::clone(&self.permits).try_acquire_owned().ok()?; + let now = Instant::now(); + let mut attempts = self + .attempts + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + while attempts + .front() + .is_some_and(|time| now.duration_since(*time) >= HANDSHAKE_WINDOW) + { + attempts.pop_front(); + } + if attempts.len() >= MAX_HANDSHAKES_PER_SECOND { + return None; + } + attempts.push_back(now); + Some(permit) + } +} + +#[derive(Clone)] +pub(crate) struct InboundAdmission { + pub(crate) budget: Arc, + pub(crate) authorized: Arc, + pub(crate) shutdown: CancellationToken, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test(start_paused = true)] + async fn budget_bounds_concurrency_and_reconnect_rate() { + let budget = HandshakeBudget::default(); + let permits = (0..MAX_CONCURRENT_HANDSHAKES) + .map(|_| budget.try_admit().unwrap()) + .collect::>(); + assert!(budget.try_admit().is_none()); + drop(permits); + for _ in MAX_CONCURRENT_HANDSHAKES..MAX_HANDSHAKES_PER_SECOND { + drop(budget.try_admit().unwrap()); + } + assert!(budget.try_admit().is_none()); + tokio::time::advance(HANDSHAKE_WINDOW).await; + assert!(budget.try_admit().is_some()); + } +} diff --git a/codex-rs/exec-server/src/noise_relay/executor_stream.rs b/codex-rs/exec-server/src/noise_relay/executor_stream.rs index 612cc9781f..091c43edce 100644 --- a/codex-rs/exec-server/src/noise_relay/executor_stream.rs +++ b/codex-rs/exec-server/src/noise_relay/executor_stream.rs @@ -9,6 +9,7 @@ use std::sync::Mutex; use tokio::sync::mpsc; use tokio::sync::watch; +use tokio_util::task::TaskTracker; use tracing::warn; use crate::ExecServerError; @@ -86,8 +87,8 @@ impl NoiseVirtualStream { /// The returned value is the read half; the spawned task owns outbound framing /// and reports its instance ID on exit so stream-ID reuse is safe. pub(crate) fn spawn_noise_virtual_stream( - stream_id: String, - instance_id: u64, + instance: ClosedNoiseVirtualStream, + tasks: Option<&TaskTracker>, handler: H, physical_outgoing_tx: mpsc::Sender>, closed_stream_tx: mpsc::Sender, @@ -99,9 +100,10 @@ pub(crate) fn spawn_noise_virtual_stream( let (disconnected_tx, disconnected_rx) = watch::channel(false); let transport = Arc::new(Mutex::new(transport)); let writer_transport = Arc::clone(&transport); - let owner_stream_id = stream_id.clone(); + let owner_stream_id = instance.stream_id.clone(); + let instance_id = instance.instance_id; let owner_closed_stream_tx = closed_stream_tx.clone(); - let writer_stream_id = stream_id; + let writer_stream_id = instance.stream_id; let writer_task = tokio::spawn(async move { let mut next_seq = 0u32; 'writer: while let Some(message) = outgoing_rx.recv().await { @@ -166,7 +168,7 @@ pub(crate) fn spawn_noise_virtual_stream( writer_task, executor_registration, }; - tokio::spawn(async move { + let owner = async move { handler.run_connection(connection).await; let _ = owner_closed_stream_tx .send(ClosedNoiseVirtualStream { @@ -174,7 +176,12 @@ pub(crate) fn spawn_noise_virtual_stream( instance_id, }) .await; - }); + }; + if let Some(tasks) = tasks { + tasks.spawn(owner); + } else { + tokio::spawn(owner); + } NoiseVirtualStream { incoming_tx, diff --git a/codex-rs/exec-server/src/noise_relay/executor_stream_tests.rs b/codex-rs/exec-server/src/noise_relay/executor_stream_tests.rs index 7adfd34860..ad127f1e92 100644 --- a/codex-rs/exec-server/src/noise_relay/executor_stream_tests.rs +++ b/codex-rs/exec-server/src/noise_relay/executor_stream_tests.rs @@ -90,8 +90,11 @@ async fn processor_exit_reports_closed_virtual_stream() -> Result<()> { let (physical_outgoing_tx, _physical_outgoing_rx) = mpsc::channel(CHANNEL_CAPACITY); let (closed_stream_tx, mut closed_stream_rx) = mpsc::channel(1); let mut stream = spawn_noise_virtual_stream( - "stream-1".to_string(), - /*instance_id*/ 7, + ClosedNoiseVirtualStream { + stream_id: "stream-1".to_string(), + instance_id: 7, + }, + /*tasks*/ None, ConnectionProcessor::new(ExecServerRuntimeOptions::new( std::env::current_exe()?, /*codex_linux_sandbox_exe*/ None, diff --git a/codex-rs/exec-server/src/noise_relay/forward_stream_tests.rs b/codex-rs/exec-server/src/noise_relay/forward_stream_tests.rs index d19c3232a5..a1da79ee7e 100644 --- a/codex-rs/exec-server/src/noise_relay/forward_stream_tests.rs +++ b/codex-rs/exec-server/src/noise_relay/forward_stream_tests.rs @@ -53,8 +53,11 @@ async fn forwards_opaque_noise_payloads_and_drains_before_closing() -> Result<() let (physical_outgoing_tx, mut physical_outgoing_rx) = mpsc::channel(1); let (closed_stream_tx, mut closed_stream_rx) = mpsc::channel(2); let mut stream = spawn_noise_virtual_stream( - "stream-1".to_string(), - /*instance_id*/ 7, + ClosedNoiseVirtualStream { + stream_id: "stream-1".to_string(), + instance_id: 7, + }, + /*tasks*/ None, forwarder, physical_outgoing_tx, closed_stream_tx, diff --git a/codex-rs/exec-server/src/noise_relay/mod.rs b/codex-rs/exec-server/src/noise_relay/mod.rs index dc6030e13b..26aa6a18de 100644 --- a/codex-rs/exec-server/src/noise_relay/mod.rs +++ b/codex-rs/exec-server/src/noise_relay/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod admission; pub(crate) mod executor_stream; mod harness; pub(crate) mod message_framing; @@ -15,7 +16,7 @@ pub(crate) const NOISE_RELAY_RESET_REASON: &str = "noise_relay_protocol_error"; // This bounds allocation in tungstenite before protobuf and Noise record // validation run. It comfortably fits one maximum Noise record plus metadata. -const MAX_NOISE_RELAY_WEBSOCKET_MESSAGE_SIZE: usize = 256 * 1024; +pub(crate) const MAX_NOISE_RELAY_WEBSOCKET_MESSAGE_SIZE: usize = 256 * 1024; /// Return the websocket limits required by every Noise relay endpoint. pub(crate) fn noise_relay_websocket_config() -> WebSocketConfig { diff --git a/codex-rs/exec-server/src/relay.rs b/codex-rs/exec-server/src/relay.rs index 3b8c6f2dfe..e067a516d5 100644 --- a/codex-rs/exec-server/src/relay.rs +++ b/codex-rs/exec-server/src/relay.rs @@ -34,6 +34,7 @@ use crate::noise_channel::NoiseChannelPublicKey; use crate::noise_channel::PendingResponderHandshake; use crate::noise_channel::noise_channel_prologue; use crate::noise_relay::NOISE_RELAY_RESET_REASON; +use crate::noise_relay::admission::InboundAdmission; use crate::noise_relay::executor_stream::ClosedNoiseVirtualStream; use crate::noise_relay::executor_stream::NoiseVirtualStream; use crate::noise_relay::executor_stream::spawn_noise_virtual_stream; @@ -48,6 +49,7 @@ use crate::relay_proto::relay_message_frame; use crate::server::ConnectionProcessor; use crate::telemetry::ExecutorRegistration; use crate::websocket_pong_watchdog::WEBSOCKET_PONG_TIMEOUT_REASON; +use tokio_util::task::TaskTracker; #[path = "relay_writer.rs"] mod writer; @@ -517,6 +519,7 @@ pub(crate) async fn run_multiplexed_environment( executor_registration_id: String, identity: NoiseChannelIdentity, validator: V, + admission: Option, ) -> RendezvousDisconnectReason where T: Sink + Stream> + Unpin + Send + 'static, @@ -545,6 +548,11 @@ where let mut streams: HashMap> = HashMap::new(); let mut pending_handshakes: HashMap = HashMap::new(); let mut validation_tasks: JoinSet = JoinSet::new(); + let stream_tasks = admission.as_ref().map(|_| TaskTracker::new()); + let shutdown = admission + .as_ref() + .map(|a| a.shutdown.clone()) + .unwrap_or_default(); let mut failed_handshakes = 0usize; let mut handshake_cooldown_until = None; let mut next_validation_id = 0u64; @@ -589,6 +597,7 @@ where loop { // Registry calls run separately so a slow check does not block the relay. let frame = tokio::select! { + _ = shutdown.cancelled() => break, writer_result = &mut physical_writer_task => { match writer_result { Ok(reason) => disconnect_reason = reason, @@ -715,8 +724,8 @@ where streams.insert( validation_result.stream_id.clone(), spawn_noise_virtual_stream( - validation_result.stream_id, - validation_result.validation_id, + ClosedNoiseVirtualStream { stream_id: validation_result.stream_id, instance_id: validation_result.validation_id }, + stream_tasks.as_ref(), handler.clone(), physical_outgoing_tx.clone(), closed_stream_tx.clone(), @@ -724,6 +733,9 @@ where executor_registration.clone(), ), ); + if let Some(admission) = &admission { + admission.authorized.notify_one(); + } } Some(Err(error)) => { warn!("Noise relay harness key validation task failed: {error}"); @@ -815,6 +827,15 @@ where send_reset(&physical_outgoing_tx, stream_id); continue; } + let handshake_permit = if let Some(admission) = &admission { + let Some(permit) = admission.budget.try_admit() else { + send_reset(&physical_outgoing_tx, stream_id); + continue; + }; + Some(permit) + } else { + None + }; let prologue = noise_channel_prologue(&environment_id, &executor_registration_id, &stream_id); let request = match frame.into_handshake_payload() { @@ -880,6 +901,7 @@ where // Failed validation leaves no transport state. Preserve the // trusted registry status for request-scoped rejection handling. validation_tasks.spawn(async move { + let _handshake_permit = handshake_permit; let result = match timeout( HARNESS_KEY_VALIDATION_TIMEOUT, validator.validate_harness_key( @@ -954,7 +976,13 @@ where for (_stream_id, stream) in streams { stream.disconnect(); } - // Dropping the JoinSet aborts any registry checks still running. + // Reporters must not block on a closed physical connection while handlers drain. + drop(closed_stream_rx); + if let Some(stream_tasks) = stream_tasks { + validation_tasks.shutdown().await; + stream_tasks.close(); + stream_tasks.wait().await; + } if !physical_writer_task.is_finished() { physical_writer_task.abort(); let _ = physical_writer_task.await; @@ -1259,6 +1287,7 @@ mod tests { "test-registration".to_string(), NoiseChannelIdentity::generate()?, AllowHarnessKeyValidator, + /*admission*/ None, )); assert!(matches!(outbound_rx.next().await, Some(Message::Ping(_)))); diff --git a/codex-rs/exec-server/src/relay_noise_tests.rs b/codex-rs/exec-server/src/relay_noise_tests.rs index 2defe4bdd3..3a417cc84c 100644 --- a/codex-rs/exec-server/src/relay_noise_tests.rs +++ b/codex-rs/exec-server/src/relay_noise_tests.rs @@ -164,6 +164,7 @@ async fn missing_pong_disconnects_physical_relay() -> Result<()> { calls: Arc::new(AtomicUsize::new(0)), release: Arc::new(Notify::new()), }, + /*admission*/ None, )); assert_eq!( @@ -195,6 +196,7 @@ async fn pong_keeps_physical_relay_connected() -> Result<()> { calls: Arc::new(AtomicUsize::new(0)), release: Arc::new(Notify::new()), }, + /*admission*/ None, )); timeout(Duration::from_secs(1), async { @@ -307,6 +309,7 @@ async fn processor_exit_resets_noise_harness_stream() -> Result<()> { calls: Arc::new(AtomicUsize::new(0)), release, }, + /*admission*/ None, ))); let mut connection = noise_harness_connection_from_websocket_with_readiness( harness_websocket, @@ -375,6 +378,7 @@ async fn pending_harness_key_validation_does_not_block_new_handshakes() -> Resul calls: Arc::clone(&calls), release: Arc::new(Notify::new()), }, + /*admission*/ None, )); for stream_id in ["stream-1", "stream-2"] { @@ -429,6 +433,7 @@ async fn duplicate_handshakes_pause_admission_without_disconnecting() -> Result< calls: Arc::clone(&calls), release: Arc::clone(&release), }, + /*admission*/ None, )); let stream_id = "stream-1"; @@ -538,6 +543,7 @@ async fn oversized_harness_authorization_is_rejected_before_validation() -> Resu calls: Arc::clone(&calls), release: Arc::new(Notify::new()), }, + /*admission*/ None, )); let stream_id = "stream-1"; @@ -617,6 +623,7 @@ async fn malformed_handshakes_preserve_active_streams_and_admission_recovers() - }, timeout_started: Arc::clone(&timeout_started), }, + /*admission*/ None, ) .with_subscriber(subscriber), )); @@ -896,6 +903,7 @@ async fn early_data_during_validation_pauses_admission_without_disconnecting() - calls: Arc::new(AtomicUsize::new(0)), release: Arc::new(Notify::new()), }, + /*admission*/ None, )); for attempt in 0..MAX_FAILED_NOISE_HANDSHAKES { diff --git a/codex-rs/exec-server/src/remote.rs b/codex-rs/exec-server/src/remote.rs index ed1fa073fc..33c42bd3ae 100644 --- a/codex-rs/exec-server/src/remote.rs +++ b/codex-rs/exec-server/src/remote.rs @@ -70,10 +70,11 @@ mod registration_retry; /// Wire transport used after registering a remote exec-server. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -/// The transport used to connect a remote exec-server environment. pub enum RemoteEnvironmentTransport { #[default] Noise, + /// Accept authenticated Noise connections on this published guest address. + NoiseInbound(std::net::SocketAddr), Direct, } @@ -181,12 +182,9 @@ impl EnvironmentRegistryClient { security_profile: NOISE_RELAY_SECURITY_PROFILE.to_string(), executor_public_key: executor_public_key.clone(), }; - let (response, executor_credential) = self - .send_authenticated( - Request::new(Method::POST, url).with_json(&body), - Some(deadline), - ) - .await?; + let request = Request::new(Method::POST, url).with_json(&body); + let (response, executor_credential) = + self.send_authenticated(request, Some(deadline)).await?; let status = response.status(); // Read diagnostics within the same attempt budget, preserving a known error status. let response: EnvironmentRegistryRegistrationResponse = timeout_at(deadline, async { @@ -294,6 +292,7 @@ impl EnvironmentRegistryClient { )); } Ok(NoiseRendezvousConnectBundle { + http_headers: Default::default(), websocket_url: response.url, environment_id: response.environment_id, executor_registration_id: response.executor_registration_id, @@ -379,7 +378,7 @@ impl EnvironmentRegistryClient { } #[derive(Clone)] -struct RegistryHarnessKeyValidator { +pub(crate) struct RegistryHarnessKeyValidator { client: EnvironmentRegistryClient, environment_id: String, executor_registration_id: String, @@ -774,6 +773,35 @@ where }) .await } + RemoteEnvironmentTransport::NoiseInbound(address) => { + let inbound_processor = processor.clone(); + run_remote_transport( + config, + std::future::pending(), + |config, client| async move { + let control = crate::server::NoiseListenerControl::default(); + let run = run_inbound_noise_environment( + config, + client, + inbound_processor, + address, + control.clone(), + ); + tokio::pin!(run, shutdown); + let result = tokio::select! { + result = &mut run => result, + _ = &mut shutdown => { + control.stop(); + run.await + } + }; + control.stop(); + control.wait().await; + result + }, + ) + .await + } RemoteEnvironmentTransport::Direct => { run_remote_transport(config, shutdown, |config, client| { run_direct_environment(config, client, processor.clone()) @@ -798,7 +826,7 @@ where // separate Direct-compatible bridge, so reject it rather than use the // Noise-specific path. // Remove this guard when a Direct-compatible forwarder is added. - if config.transport == RemoteEnvironmentTransport::Direct { + if config.transport != RemoteEnvironmentTransport::Noise { return Err(ExecServerError::EnvironmentRegistryConfig( "direct exec-server transport does not support forwarding".to_string(), )); @@ -814,6 +842,42 @@ where .await } +async fn run_inbound_noise_environment( + config: RemoteEnvironmentConfig, + client: EnvironmentRegistryClient, + processor: ConnectionProcessor, + address: std::net::SocketAddr, + control: crate::server::NoiseListenerControl, +) -> Result<(), ExecServerError> { + let listener = tokio::net::TcpListener::bind(address) + .await + .map_err(ExecServerError::Spawn)?; + let identity = NoiseChannelIdentity::generate().map_err(|error| { + ExecServerError::Protocol(format!("failed to generate Noise identity: {error}")) + })?; + let public_key = identity.public_key(); + let registration = tokio::select! { + result = client.register_environment_with_retry(&config.environment_id, &public_key) => result?, + _ = control.cancelled() => return Ok(()), + }; + let validator = RegistryHarnessKeyValidator { + client, + environment_id: config.environment_id.clone(), + executor_registration_id: registration.executor_registration_id.clone(), + }; + crate::server::run_noise_websocket_listener( + listener, + processor, + config.environment_id, + registration.executor_registration_id, + identity, + validator, + control, + ) + .await + .map_err(ExecServerError::Spawn) +} + async fn run_remote_transport( config: RemoteEnvironmentConfig, shutdown: F, @@ -886,6 +950,7 @@ async fn run_remote_environment_connections( environment_id: config.environment_id.clone(), executor_registration_id, }, + /*admission*/ None, ) .await; backoff.connection_closed(connected_at.elapsed()); diff --git a/codex-rs/exec-server/src/server.rs b/codex-rs/exec-server/src/server.rs index 29f1440d29..f2c76a88fc 100644 --- a/codex-rs/exec-server/src/server.rs +++ b/codex-rs/exec-server/src/server.rs @@ -19,6 +19,8 @@ pub use request_dispatcher::ConcurrentRequestLimit; pub use request_dispatcher::RequestDispatchMode; pub use transport::DEFAULT_LISTEN_URL; pub use transport::ExecServerListenUrlParseError; +pub(crate) use transport::NoiseListenerControl; +pub(crate) use transport::run_noise_websocket_listener; use crate::ExecServerRuntimeOptions; use crate::ExecServerTelemetry; diff --git a/codex-rs/exec-server/src/server/transport.rs b/codex-rs/exec-server/src/server/transport.rs index 4bb1d1f120..8731af5511 100644 --- a/codex-rs/exec-server/src/server/transport.rs +++ b/codex-rs/exec-server/src/server/transport.rs @@ -1,10 +1,16 @@ //! Exec-server stdio and WebSocket listeners. Configured authentication gates //! WebSocket upgrades; unauthenticated listeners remain supported for compatibility. +use crate::noise_channel::NoiseChannelIdentity; +use crate::noise_relay::MAX_NOISE_RELAY_WEBSOCKET_MESSAGE_SIZE; +use crate::noise_relay::admission::HandshakeBudget; +use crate::noise_relay::admission::InboundAdmission; +use crate::relay::run_multiplexed_environment; use axum::Router; use axum::body::Body; use axum::extract::ConnectInfo; use axum::extract::State; +use axum::extract::ws::Message as AxumMessage; use axum::extract::ws::WebSocketUpgrade; use axum::http::HeaderMap; use axum::http::Request; @@ -21,12 +27,21 @@ use codex_websocket_auth::WebsocketAuthPolicy; use codex_websocket_auth::WebsocketAuthSettings; use codex_websocket_auth::authorize_upgrade; use codex_websocket_auth::policy_from_settings; +use futures::SinkExt; +use futures::StreamExt; use std::io::Write as _; use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; use tokio::io; use tokio::io::AsyncRead; use tokio::io::AsyncWrite; use tokio::net::TcpListener; +use tokio::sync::Notify; +use tokio::sync::Semaphore; +use tokio_tungstenite::tungstenite::Message as NoiseMessage; +use tokio_util::sync::CancellationToken; +use tokio_util::task::TaskTracker; use tracing::info; use tracing::warn; @@ -195,26 +210,108 @@ async fn run_websocket_listener( println!("ws://{local_addr}"); std::io::stdout().flush()?; + serve_websocket( + listener, + ExecServerWebSocketState { + processor, + auth_policy, + noise: None, + }, + ) + .await?; + Ok(()) +} + +const FIRST_NOISE_AUTHORIZATION_TIMEOUT: Duration = Duration::from_secs(20); + +#[derive(Clone, Default)] +pub(crate) struct NoiseListenerControl { + shutdown: CancellationToken, + tasks: TaskTracker, + budget: Arc, +} + +impl NoiseListenerControl { + pub(crate) fn stop(&self) { + self.shutdown.cancel(); + } + pub(crate) async fn cancelled(&self) { + self.shutdown.cancelled().await; + } + pub(crate) async fn wait(&self) { + self.tasks.close(); + self.tasks.wait().await; + } +} + +#[derive(Clone)] +struct NoiseListener { + environment_id: String, + registration_id: String, + identity: NoiseChannelIdentity, + validator: crate::remote::RegistryHarnessKeyValidator, + permits: Arc, + control: NoiseListenerControl, +} + +pub(crate) async fn run_noise_websocket_listener( + listener: TcpListener, + processor: ConnectionProcessor, + environment_id: String, + registration_id: String, + identity: NoiseChannelIdentity, + validator: crate::remote::RegistryHarnessKeyValidator, + control: NoiseListenerControl, +) -> std::io::Result<()> { + serve_websocket( + listener, + ExecServerWebSocketState { + processor, + auth_policy: WebsocketAuthPolicy::default(), + noise: Some(NoiseListener { + environment_id, + registration_id, + identity, + validator, + permits: Arc::new(Semaphore::new(64)), + control, + }), + }, + ) + .await +} + +async fn serve_websocket( + listener: TcpListener, + state: ExecServerWebSocketState, +) -> std::io::Result<()> { + let shutdown = state + .noise + .as_ref() + .map(|noise| noise.control.shutdown.clone()); let router = Router::new() .route("/", any(websocket_upgrade_handler)) .route("/readyz", get(readiness_handler)) .layer(middleware::from_fn(reject_requests_with_origin_header)) - .with_state(ExecServerWebSocketState { - processor, - auth_policy, - }); + .with_state(state); axum::serve( listener, router.into_make_service_with_connect_info::(), ) - .await?; - Ok(()) + .with_graceful_shutdown(async move { + match shutdown { + Some(shutdown) => shutdown.cancelled().await, + None => std::future::pending::<()>().await, + } + }) + .await } #[derive(Clone)] struct ExecServerWebSocketState { processor: ConnectionProcessor, auth_policy: WebsocketAuthPolicy, + noise: Option, } async fn readiness_handler() -> StatusCode { @@ -246,19 +343,88 @@ async fn websocket_upgrade_handler( if let Err(err) = authorize_upgrade(&headers, &state.auth_policy) { return (err.status_code(), err.message()).into_response(); } + let (websocket, permit) = if let Some(noise) = &state.noise { + let Ok(permit) = Arc::clone(&noise.permits).try_acquire_owned() else { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + }; + ( + websocket + .max_message_size(MAX_NOISE_RELAY_WEBSOCKET_MESSAGE_SIZE) + .max_frame_size(MAX_NOISE_RELAY_WEBSOCKET_MESSAGE_SIZE), + Some(permit), + ) + } else { + (websocket, None) + }; info!(%peer_addr, "exec-server websocket client connected"); + // Register before Axum spawns the upgrade callback, including upgrades still pending. + let task = state + .noise + .as_ref() + .map(|noise| noise.control.tasks.token()); let mut response = websocket .on_upgrade(move |stream| async move { - state - .processor - .run_connection( - JsonRpcConnection::from_axum_websocket( - stream, - format!("exec-server websocket {peer_addr}"), - ), - ConnectionTransport::WebSocket, - ) - .await; + let _task = task; + let _permit = permit; + if let Some(noise) = state.noise { + let stream = stream + .with(|message: NoiseMessage| { + futures::future::ready(Ok::<_, axum::Error>(match message { + NoiseMessage::Binary(v) => AxumMessage::Binary(v), + NoiseMessage::Text(v) => AxumMessage::Text(v.as_str().into()), + NoiseMessage::Ping(v) => AxumMessage::Ping(v), + NoiseMessage::Pong(v) => AxumMessage::Pong(v), + NoiseMessage::Close(_) | NoiseMessage::Frame(_) => { + AxumMessage::Close(None) + } + })) + }) + .map(|result| { + result.map(|message| match message { + AxumMessage::Binary(v) => NoiseMessage::Binary(v), + AxumMessage::Text(v) => NoiseMessage::Text(v.as_str().into()), + AxumMessage::Ping(v) => NoiseMessage::Ping(v), + AxumMessage::Pong(v) => NoiseMessage::Pong(v), + AxumMessage::Close(_) => NoiseMessage::Close(None), + }) + }); + let admission = InboundAdmission { + budget: Arc::clone(&noise.control.budget), + authorized: Arc::new(Notify::new()), + shutdown: noise.control.shutdown.child_token(), + }; + let authorized = Arc::clone(&admission.authorized); + let shutdown = admission.shutdown.clone(); + let run = run_multiplexed_environment( + stream, + state.processor, + noise.environment_id, + noise.registration_id, + noise.identity, + noise.validator, + Some(admission), + ); + tokio::pin!(run); + tokio::select! { + _ = &mut run => {}, + _ = authorized.notified() => { run.await; }, + _ = tokio::time::sleep(FIRST_NOISE_AUTHORIZATION_TIMEOUT) => { + shutdown.cancel(); + run.await; + }, + } + } else { + state + .processor + .run_connection( + JsonRpcConnection::from_axum_websocket( + stream, + format!("exec-server websocket {peer_addr}"), + ), + ConnectionTransport::WebSocket, + ) + .await; + } }) .into_response(); if let Some(request_id) = headers.get("x-request-id") { diff --git a/codex-rs/exec-server/tests/common/relay.rs b/codex-rs/exec-server/tests/common/relay.rs index d10f06c145..f20440b504 100644 --- a/codex-rs/exec-server/tests/common/relay.rs +++ b/codex-rs/exec-server/tests/common/relay.rs @@ -52,8 +52,8 @@ pub(crate) fn static_registry_auth_provider() -> codex_api::SharedAuthProvider { } pub(crate) struct RelayTest { - registry: MockServer, - listener: TcpListener, + pub(crate) registry: MockServer, + pub(crate) listener: TcpListener, } pub(crate) struct RelayConnection { @@ -109,6 +109,7 @@ impl RelayTest { let harness_identity = NoiseChannelIdentity::generate()?; let client_args = NoiseRendezvousConnectArgs { bundle: NoiseRendezvousConnectBundle { + http_headers: Default::default(), websocket_url: format!("{rendezvous_url}/relay?role=harness"), environment_id: ENVIRONMENT_ID.to_string(), executor_registration_id: EXECUTOR_REGISTRATION_ID.to_string(), diff --git a/codex-rs/exec-server/tests/relay.rs b/codex-rs/exec-server/tests/relay.rs index 8bca066ef1..a0b1491293 100644 --- a/codex-rs/exec-server/tests/relay.rs +++ b/codex-rs/exec-server/tests/relay.rs @@ -21,10 +21,13 @@ use codex_exec_server::EnvironmentManager; use codex_exec_server::EnvironmentReadyInfo; use codex_exec_server::ExecParams; use codex_exec_server::ExecResponse; +use codex_exec_server::ExecServerClient; use codex_exec_server::ExecServerError; use codex_exec_server::ExecServerRuntimeOptions; use codex_exec_server::FsReadFileParams; +use codex_exec_server::NoiseChannelIdentity; use codex_exec_server::NoiseChannelPublicKey; +use codex_exec_server::NoiseRendezvousConnectArgs; use codex_exec_server::NoiseRendezvousConnectBundle; use codex_exec_server::NoiseRendezvousConnectProvider; use codex_exec_server::ProcessId; @@ -82,6 +85,7 @@ impl NoiseRendezvousConnectProvider for FreshBundleNoiseConnectProvider { ) -> BoxFuture<'_, Result> { let call = self.calls.fetch_add(1, Ordering::Relaxed) + 1; let bundle = NoiseRendezvousConnectBundle { + http_headers: Default::default(), websocket_url: self.websocket_url.clone(), environment_id: ENVIRONMENT_ID.to_string(), executor_registration_id: EXECUTOR_REGISTRATION_ID.to_string(), @@ -421,3 +425,138 @@ async fn remote_environment_routes_encrypted_exec_server_rpc() -> Result<()> { timeout(TEST_TIMEOUT, remote_environment).await???; Ok(()) } + +/// An inbound executor serves encrypted RPCs directly and rejects a registry-denied harness. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn inbound_noise_executes_only_after_registry_authorization() -> Result<()> { + use codex_exec_server::RemoteEnvironmentTransport; + use futures::SinkExt; + use futures::StreamExt; + use tokio_tungstenite::tungstenite::Message; + use wiremock::matchers::body_partial_json; + + let relay = RelayTest::new().await?; + let address = relay.listener.local_addr()?; + let registry = relay.registry; + drop(relay.listener); + Mock::given(method("POST")) + .and(path(format!( + "/cloud/environment/{ENVIRONMENT_ID}/validate" + ))) + .and(body_partial_json( + serde_json::json!({"harness_key_authorization": "denied"}), + )) + .respond_with(ResponseTemplate::new(403)) + .with_priority(1) + .mount(®istry) + .await; + let config = RemoteEnvironmentConfig::new_with_transport( + registry.uri(), + ENVIRONMENT_ID.to_string(), + RemoteEnvironmentTransport::NoiseInbound(address), + static_registry_auth_provider(), + HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault), + )?; + let (codex_exe, sandbox_exe) = common::current_test_binary_helper_paths()?; + let runtime = ExecServerRuntimeOptions::new(codex_exe, sandbox_exe)?; + let (stop, stopped) = tokio::sync::oneshot::channel(); + let server = AbortOnDropHandle::new(tokio::spawn( + codex_exec_server::run_remote_environment_until_shutdown(config, runtime, async move { + let _ = stopped.await; + }), + )); + let executor_key = timeout(TEST_TIMEOUT, async { + loop { + if let Ok(key) = registered_executor_public_key(®istry).await { + break key; + } + tokio::task::yield_now().await; + } + }) + .await?; + let (mut plaintext, _) = tokio_tungstenite::connect_async(format!("ws://{address}/")).await?; + plaintext + .send(Message::Text( + r#"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"clientName":"plaintext"}}"# + .into(), + )) + .await?; + assert!( + timeout(std::time::Duration::from_millis(150), plaintext.next()) + .await + .is_err() + ); + let idle = tokio::spawn(async move { + while let Some(message) = plaintext.next().await { + match message { + Ok(Message::Ping(payload)) => plaintext.send(Message::Pong(payload)).await?, + Ok(Message::Close(_)) | Err(_) => break, + _ => {} + } + } + anyhow::Ok(()) + }); + let connect = |authorization: &str| -> Result { + Ok(NoiseRendezvousConnectArgs { + bundle: NoiseRendezvousConnectBundle { + http_headers: Default::default(), + websocket_url: format!("ws://{address}/"), + environment_id: ENVIRONMENT_ID.to_string(), + executor_registration_id: EXECUTOR_REGISTRATION_ID.to_string(), + executor_public_key: executor_key.clone(), + harness_key_authorization: authorization.to_string(), + }, + harness_identity: NoiseChannelIdentity::generate()?, + client_name: "inbound-test".to_string(), + connect_timeout: TEST_TIMEOUT, + initialize_timeout: TEST_TIMEOUT, + resume_session_id: None, + http_client_factory: HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault), + }) + }; + assert!( + ExecServerClient::connect_noise_rendezvous(connect("denied")?) + .await + .is_err() + ); + let client = + ExecServerClient::connect_noise_rendezvous(connect(HARNESS_KEY_AUTHORIZATION)?).await?; + let temp = TempDir::new()?; + let path = temp.path().join("proof"); + std::fs::write(&path, b"inbound-noise-proof")?; + let response = client + .fs_read_file(FsReadFileParams { + path: PathUri::from_host_native_path(path.clone())?, + follow_symlinks: None, + sandbox: None, + }) + .await?; + assert_eq!( + STANDARD.decode(response.data_base64)?, + b"inbound-noise-proof" + ); + timeout(std::time::Duration::from_secs(25), idle).await???; + // The first-authorization deadline must not close an established Noise session. + client + .fs_read_file(FsReadFileParams { + path: PathUri::from_host_native_path(path.clone())?, + follow_symlinks: None, + sandbox: None, + }) + .await?; + let _ = stop.send(()); + timeout(TEST_TIMEOUT, server).await???; + assert!( + timeout( + TEST_TIMEOUT, + client.fs_read_file(FsReadFileParams { + path: PathUri::from_host_native_path(path)?, + follow_symlinks: None, + sandbox: None, + }) + ) + .await? + .is_err() + ); + Ok(()) +} diff --git a/codex-rs/exec-server/tests/relay/registration_retry_tests.rs b/codex-rs/exec-server/tests/relay/registration_retry_tests.rs index 987b4b34a8..84afb52ed1 100644 --- a/codex-rs/exec-server/tests/relay/registration_retry_tests.rs +++ b/codex-rs/exec-server/tests/relay/registration_retry_tests.rs @@ -106,6 +106,7 @@ impl RegistryFixture { let environment = accept_websocket(&self.listener, "environment").await?; let args = NoiseRendezvousConnectArgs { bundle: NoiseRendezvousConnectBundle { + http_headers: Default::default(), websocket_url: format!("ws://{}/relay?role=harness", self.listener.local_addr()?), environment_id: ENVIRONMENT_ID.to_string(), executor_registration_id: registration_id.to_string(),