diff --git a/codex-rs/cli/tests/exec_server.rs b/codex-rs/cli/tests/exec_server.rs index 8469b2ffee..372f79ecc4 100644 --- a/codex-rs/cli/tests/exec_server.rs +++ b/codex-rs/cli/tests/exec_server.rs @@ -580,6 +580,7 @@ metrics_exporter = {{ otlp-http = {{ endpoint = "{collector_url}/v1/metrics", pr let executor_public_key = registered_parent_lifetime_executor_public_key(®istry).await?; let harness_args = NoiseRendezvousConnectArgs { bundle: NoiseRendezvousConnectBundle { + http_headers: Default::default(), websocket_url: format!("{rendezvous_url}/relay?role=harness"), environment_id: ENVIRONMENT_ID.to_string(), executor_registration_id: EXECUTOR_REGISTRATION_ID.to_string(), @@ -870,6 +871,7 @@ metrics_exporter = {{ otlp-http = {{ endpoint = "{}/v1/metrics", protocol = "jso let executor_public_key = registered_parent_lifetime_executor_public_key(®istry).await?; let args = NoiseRendezvousConnectArgs { bundle: NoiseRendezvousConnectBundle { + http_headers: Default::default(), websocket_url: format!("{rendezvous}/relay?role=harness"), environment_id: ENVIRONMENT.to_string(), executor_registration_id: "registration-scoped".to_string(), diff --git a/codex-rs/core/tests/suite/remote_env.rs b/codex-rs/core/tests/suite/remote_env.rs index 25083ae5e9..936fde9087 100644 --- a/codex-rs/core/tests/suite/remote_env.rs +++ b/codex-rs/core/tests/suite/remote_env.rs @@ -1642,6 +1642,7 @@ impl NoiseRendezvousConnectProvider for OfflineThenReadyNoiseConnectProvider { }); } let bundle = NoiseRendezvousConnectBundle { + http_headers: Default::default(), websocket_url: self.websocket_url.clone(), environment_id: REMOTE_ENVIRONMENT_ID.to_string(), executor_registration_id: "ready-first-registration".to_string(), diff --git a/codex-rs/exec-server-cli/src/command.rs b/codex-rs/exec-server-cli/src/command.rs index fd630310c0..14bfe3ed67 100644 --- a/codex-rs/exec-server-cli/src/command.rs +++ b/codex-rs/exec-server-cli/src/command.rs @@ -68,11 +68,7 @@ pub struct ExecServerCommand { request_dispatch_mode: codex_exec_server::RequestDispatchMode, /// Transport endpoint URL. Supported values: `ws://IP:PORT` (default), `stdio`, `stdio://`. - #[arg( - long = "listen", - value_name = "URL", - conflicts_with = "exec_server_remote" - )] + #[arg(long = "listen", value_name = "URL")] listen: Option, #[command(flatten)] @@ -303,7 +299,13 @@ impl ExecServerCommand { let mut remote_config = codex_exec_server::RemoteEnvironmentConfig::new_with_transport( base_url, environment_id, - if direct_transport { + if let Some(listen) = self.listen.as_deref() { + let address = listen + .strip_prefix("ws://") + .and_then(|s| s.parse().ok()) + .ok_or_else(|| anyhow::anyhow!("remote --listen requires ws://IP:PORT"))?; + codex_exec_server::RemoteEnvironmentTransport::NoiseInbound(address) + } else if direct_transport { codex_exec_server::RemoteEnvironmentTransport::Direct } else { codex_exec_server::RemoteEnvironmentTransport::Noise @@ -428,6 +430,16 @@ impl ExecServerCommand { !self.chatgpt_token_stdin || self.command.is_none(), "--chatgpt-token-stdin cannot be used with forward" ); + anyhow::ensure!( + self.remote.is_none() + || self.listen.is_none() + || (self.remote_transport == ExecServerRemoteTransport::Noise + && self.command.is_none() + && self.listen.as_deref().is_some_and(|s| s + .strip_prefix("ws://") + .is_some_and(|v| v.parse::().is_ok()))), + "remote --listen requires ws://IP:PORT, Noise transport, and no forward" + ); match (self.remote_transport, self.aws_sigv4) { (ExecServerRemoteTransport::Noise, true) => { anyhow::bail!("--aws-sigv4 requires --remote-transport direct"); diff --git a/codex-rs/exec-server-cli/src/exec_server_args_tests.rs b/codex-rs/exec-server-cli/src/exec_server_args_tests.rs index d000825a2d..6d165701fb 100644 --- a/codex-rs/exec-server-cli/src/exec_server_args_tests.rs +++ b/codex-rs/exec-server-cli/src/exec_server_args_tests.rs @@ -351,3 +351,30 @@ fn standalone_help_documents_executor_options() { .join("\n"); insta::assert_snapshot!(help); } + +/// An inbound Noise listener requires remote registration and cannot expose direct RPC transport. +#[test] +fn inbound_noise_requires_registered_noise_transport() { + let command = exec_server_from_args(&[ + "--remote", + "https://registry.example.com", + "--environment-id", + "env-1", + "--listen", + "ws://0.0.0.0:9001", + ]); + assert_eq!(command.listen.as_deref(), Some("ws://0.0.0.0:9001")); + command.validate_remote_transport().unwrap(); + let command = exec_server_from_args(&[ + "--remote", + "https://registry.example.com", + "--environment-id", + "env-1", + "--remote-transport", + "direct", + "--aws-sigv4", + "--listen", + "ws://0.0.0.0:9001", + ]); + assert!(command.validate_remote_transport().is_err()); +} diff --git a/codex-rs/exec-server/src/client_api.rs b/codex-rs/exec-server/src/client_api.rs index e870f12471..5b1cce00e3 100644 --- a/codex-rs/exec-server/src/client_api.rs +++ b/codex-rs/exec-server/src/client_api.rs @@ -44,6 +44,8 @@ pub struct RemoteExecServerConnectArgs { /// physical connection attempt and must not be mixed with values from another /// registry response. pub struct NoiseRendezvousConnectBundle { + /// Admission headers for the authorized WebSocket destination. Never log them. + pub http_headers: HeaderMap, pub websocket_url: String, pub environment_id: String, pub executor_registration_id: String, diff --git a/codex-rs/exec-server/src/client_refresh_tests.rs b/codex-rs/exec-server/src/client_refresh_tests.rs index 255f99b599..db0e21b7c3 100644 --- a/codex-rs/exec-server/src/client_refresh_tests.rs +++ b/codex-rs/exec-server/src/client_refresh_tests.rs @@ -90,6 +90,7 @@ impl NoiseRendezvousConnectProvider for Registry { .map_err(|_| ExecServerError::Protocol("test lookup failed".to_owned()))?; } Ok(NoiseRendezvousConnectBundle { + http_headers: Default::default(), websocket_url: target.url, environment_id: "environment".to_owned(), executor_registration_id: target.registration, @@ -206,7 +207,7 @@ impl Executor { let validator = validator.clone(); connections.spawn(async move { let socket = tokio_tungstenite::accept_async(socket).await.unwrap(); - run_multiplexed_environment(socket, processor, "environment".to_owned(), executor.registration, executor.identity, validator).await; + run_multiplexed_environment(socket, processor, "environment".to_owned(), executor.registration, executor.identity, validator, /*admission*/ None).await; }); } _ = connections.join_next(), if !connections.is_empty() => {} diff --git a/codex-rs/exec-server/src/client_transport.rs b/codex-rs/exec-server/src/client_transport.rs index 51c2eb1482..ad7d7b418d 100644 --- a/codex-rs/exec-server/src/client_transport.rs +++ b/codex-rs/exec-server/src/client_transport.rs @@ -658,6 +658,7 @@ impl ExecServerClient { http_client_factory, } = args; let NoiseRendezvousConnectBundle { + http_headers, websocket_url, environment_id, executor_registration_id, @@ -676,6 +677,21 @@ impl ExecServerClient { url: diagnostic_url.clone(), source, })?; + if !http_headers.is_empty() && request.uri().scheme_str() != Some("wss") { + return Err(ExecServerError::Protocol( + "Noise WebSocket admission headers require wss://".to_string(), + )); + } + for (name, value) in &http_headers { + if is_websocket_handshake_header(name) { + return Err(ExecServerError::Protocol( + "invalid Noise WebSocket admission header".to_string(), + )); + } + let mut value = value.clone(); + value.set_sensitive(true); + request.headers_mut().insert(name, value); + } request.headers_mut().extend(current_rendezvous_headers()); let (stream, _) = timeout( connect_timeout, diff --git a/codex-rs/exec-server/src/client_transport_tests.rs b/codex-rs/exec-server/src/client_transport_tests.rs index 3ae10252bc..ad6f1ba672 100644 --- a/codex-rs/exec-server/src/client_transport_tests.rs +++ b/codex-rs/exec-server/src/client_transport_tests.rs @@ -161,6 +161,7 @@ impl NoiseRendezvousConnectProvider for SequenceNoiseConnectProvider { fn test_bundle(websocket_url: String) -> Result { Ok(NoiseRendezvousConnectBundle { + http_headers: Default::default(), websocket_url, environment_id: "environment".to_string(), executor_registration_id: "registration".to_string(), @@ -237,6 +238,35 @@ async fn noise_handshake_reports_disconnect_reason() -> Result<()> { Ok(()) } +#[tokio::test] +async fn noise_admission_headers_are_rejected_before_dialing_plaintext() -> Result<()> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let mut bundle = test_bundle(format!("ws://{}", listener.local_addr()?))?; + bundle.http_headers.insert( + http::header::AUTHORIZATION, + http::HeaderValue::from_static("fake-token"), + ); + let result = ExecServerClient::open_noise_rendezvous_connection(NoiseRendezvousConnectArgs { + bundle, + harness_identity: NoiseChannelIdentity::generate()?, + client_name: "test".to_string(), + connect_timeout: std::time::Duration::from_secs(1), + initialize_timeout: std::time::Duration::from_secs(1), + resume_session_id: None, + http_client_factory: codex_http_client::HttpClientFactory::new( + codex_http_client::OutboundProxyPolicy::ReqwestDefault, + ), + }) + .await; + assert!(matches!(result, Err(ExecServerError::Protocol(_)))); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(10), listener.accept()) + .await + .is_err() + ); + Ok(()) +} + #[tokio::test] async fn noise_handshake_uses_initialize_timeout() -> Result<()> { let listener = TcpListener::bind("127.0.0.1:0").await?; @@ -262,6 +292,7 @@ async fn noise_handshake_uses_initialize_timeout() -> Result<()> { let initialize_timeout = std::time::Duration::from_millis(1); let opened = ExecServerClient::open_noise_rendezvous_connection(NoiseRendezvousConnectArgs { bundle: NoiseRendezvousConnectBundle { + http_headers: Default::default(), websocket_url, environment_id: "environment".to_string(), executor_registration_id: "registration".to_string(), diff --git a/codex-rs/exec-server/src/noise_relay/admission.rs b/codex-rs/exec-server/src/noise_relay/admission.rs new file mode 100644 index 0000000000..8d8528e30e --- /dev/null +++ b/codex-rs/exec-server/src/noise_relay/admission.rs @@ -0,0 +1,81 @@ +//! Listener-wide admission for inbound Noise handshakes. The budget is charged +//! before parsing a hybrid handshake and held through its ERS validation. + +use std::collections::VecDeque; +use std::sync::Arc; +use std::sync::Mutex; +use std::time::Duration; + +use tokio::sync::Notify; +use tokio::sync::OwnedSemaphorePermit; +use tokio::sync::Semaphore; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; + +const MAX_CONCURRENT_HANDSHAKES: usize = 16; +const MAX_HANDSHAKES_PER_SECOND: usize = 32; +const HANDSHAKE_WINDOW: Duration = Duration::from_secs(1); + +pub(crate) struct HandshakeBudget { + permits: Arc, + attempts: Mutex>, +} + +impl Default for HandshakeBudget { + fn default() -> Self { + Self { + permits: Arc::new(Semaphore::new(MAX_CONCURRENT_HANDSHAKES)), + attempts: Mutex::new(VecDeque::new()), + } + } +} + +impl HandshakeBudget { + pub(crate) fn try_admit(&self) -> Option { + let permit = Arc::clone(&self.permits).try_acquire_owned().ok()?; + let now = Instant::now(); + let mut attempts = self + .attempts + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + while attempts + .front() + .is_some_and(|time| now.duration_since(*time) >= HANDSHAKE_WINDOW) + { + attempts.pop_front(); + } + if attempts.len() >= MAX_HANDSHAKES_PER_SECOND { + return None; + } + attempts.push_back(now); + Some(permit) + } +} + +#[derive(Clone)] +pub(crate) struct InboundAdmission { + pub(crate) budget: Arc, + pub(crate) authorized: Arc, + pub(crate) shutdown: CancellationToken, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test(start_paused = true)] + async fn budget_bounds_concurrency_and_reconnect_rate() { + let budget = HandshakeBudget::default(); + let permits = (0..MAX_CONCURRENT_HANDSHAKES) + .map(|_| budget.try_admit().unwrap()) + .collect::>(); + assert!(budget.try_admit().is_none()); + drop(permits); + for _ in MAX_CONCURRENT_HANDSHAKES..MAX_HANDSHAKES_PER_SECOND { + drop(budget.try_admit().unwrap()); + } + assert!(budget.try_admit().is_none()); + tokio::time::advance(HANDSHAKE_WINDOW).await; + assert!(budget.try_admit().is_some()); + } +} diff --git a/codex-rs/exec-server/src/noise_relay/executor_stream.rs b/codex-rs/exec-server/src/noise_relay/executor_stream.rs index 612cc9781f..091c43edce 100644 --- a/codex-rs/exec-server/src/noise_relay/executor_stream.rs +++ b/codex-rs/exec-server/src/noise_relay/executor_stream.rs @@ -9,6 +9,7 @@ use std::sync::Mutex; use tokio::sync::mpsc; use tokio::sync::watch; +use tokio_util::task::TaskTracker; use tracing::warn; use crate::ExecServerError; @@ -86,8 +87,8 @@ impl NoiseVirtualStream { /// The returned value is the read half; the spawned task owns outbound framing /// and reports its instance ID on exit so stream-ID reuse is safe. pub(crate) fn spawn_noise_virtual_stream( - stream_id: String, - instance_id: u64, + instance: ClosedNoiseVirtualStream, + tasks: Option<&TaskTracker>, handler: H, physical_outgoing_tx: mpsc::Sender>, closed_stream_tx: mpsc::Sender, @@ -99,9 +100,10 @@ pub(crate) fn spawn_noise_virtual_stream( let (disconnected_tx, disconnected_rx) = watch::channel(false); let transport = Arc::new(Mutex::new(transport)); let writer_transport = Arc::clone(&transport); - let owner_stream_id = stream_id.clone(); + let owner_stream_id = instance.stream_id.clone(); + let instance_id = instance.instance_id; let owner_closed_stream_tx = closed_stream_tx.clone(); - let writer_stream_id = stream_id; + let writer_stream_id = instance.stream_id; let writer_task = tokio::spawn(async move { let mut next_seq = 0u32; 'writer: while let Some(message) = outgoing_rx.recv().await { @@ -166,7 +168,7 @@ pub(crate) fn spawn_noise_virtual_stream( writer_task, executor_registration, }; - tokio::spawn(async move { + let owner = async move { handler.run_connection(connection).await; let _ = owner_closed_stream_tx .send(ClosedNoiseVirtualStream { @@ -174,7 +176,12 @@ pub(crate) fn spawn_noise_virtual_stream( instance_id, }) .await; - }); + }; + if let Some(tasks) = tasks { + tasks.spawn(owner); + } else { + tokio::spawn(owner); + } NoiseVirtualStream { incoming_tx, diff --git a/codex-rs/exec-server/src/noise_relay/executor_stream_tests.rs b/codex-rs/exec-server/src/noise_relay/executor_stream_tests.rs index 7adfd34860..ad127f1e92 100644 --- a/codex-rs/exec-server/src/noise_relay/executor_stream_tests.rs +++ b/codex-rs/exec-server/src/noise_relay/executor_stream_tests.rs @@ -90,8 +90,11 @@ async fn processor_exit_reports_closed_virtual_stream() -> Result<()> { let (physical_outgoing_tx, _physical_outgoing_rx) = mpsc::channel(CHANNEL_CAPACITY); let (closed_stream_tx, mut closed_stream_rx) = mpsc::channel(1); let mut stream = spawn_noise_virtual_stream( - "stream-1".to_string(), - /*instance_id*/ 7, + ClosedNoiseVirtualStream { + stream_id: "stream-1".to_string(), + instance_id: 7, + }, + /*tasks*/ None, ConnectionProcessor::new(ExecServerRuntimeOptions::new( std::env::current_exe()?, /*codex_linux_sandbox_exe*/ None, diff --git a/codex-rs/exec-server/src/noise_relay/forward_stream_tests.rs b/codex-rs/exec-server/src/noise_relay/forward_stream_tests.rs index d19c3232a5..a1da79ee7e 100644 --- a/codex-rs/exec-server/src/noise_relay/forward_stream_tests.rs +++ b/codex-rs/exec-server/src/noise_relay/forward_stream_tests.rs @@ -53,8 +53,11 @@ async fn forwards_opaque_noise_payloads_and_drains_before_closing() -> Result<() let (physical_outgoing_tx, mut physical_outgoing_rx) = mpsc::channel(1); let (closed_stream_tx, mut closed_stream_rx) = mpsc::channel(2); let mut stream = spawn_noise_virtual_stream( - "stream-1".to_string(), - /*instance_id*/ 7, + ClosedNoiseVirtualStream { + stream_id: "stream-1".to_string(), + instance_id: 7, + }, + /*tasks*/ None, forwarder, physical_outgoing_tx, closed_stream_tx, diff --git a/codex-rs/exec-server/src/noise_relay/mod.rs b/codex-rs/exec-server/src/noise_relay/mod.rs index dc6030e13b..26aa6a18de 100644 --- a/codex-rs/exec-server/src/noise_relay/mod.rs +++ b/codex-rs/exec-server/src/noise_relay/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod admission; pub(crate) mod executor_stream; mod harness; pub(crate) mod message_framing; @@ -15,7 +16,7 @@ pub(crate) const NOISE_RELAY_RESET_REASON: &str = "noise_relay_protocol_error"; // This bounds allocation in tungstenite before protobuf and Noise record // validation run. It comfortably fits one maximum Noise record plus metadata. -const MAX_NOISE_RELAY_WEBSOCKET_MESSAGE_SIZE: usize = 256 * 1024; +pub(crate) const MAX_NOISE_RELAY_WEBSOCKET_MESSAGE_SIZE: usize = 256 * 1024; /// Return the websocket limits required by every Noise relay endpoint. pub(crate) fn noise_relay_websocket_config() -> WebSocketConfig { diff --git a/codex-rs/exec-server/src/relay.rs b/codex-rs/exec-server/src/relay.rs index 3b8c6f2dfe..e067a516d5 100644 --- a/codex-rs/exec-server/src/relay.rs +++ b/codex-rs/exec-server/src/relay.rs @@ -34,6 +34,7 @@ use crate::noise_channel::NoiseChannelPublicKey; use crate::noise_channel::PendingResponderHandshake; use crate::noise_channel::noise_channel_prologue; use crate::noise_relay::NOISE_RELAY_RESET_REASON; +use crate::noise_relay::admission::InboundAdmission; use crate::noise_relay::executor_stream::ClosedNoiseVirtualStream; use crate::noise_relay::executor_stream::NoiseVirtualStream; use crate::noise_relay::executor_stream::spawn_noise_virtual_stream; @@ -48,6 +49,7 @@ use crate::relay_proto::relay_message_frame; use crate::server::ConnectionProcessor; use crate::telemetry::ExecutorRegistration; use crate::websocket_pong_watchdog::WEBSOCKET_PONG_TIMEOUT_REASON; +use tokio_util::task::TaskTracker; #[path = "relay_writer.rs"] mod writer; @@ -517,6 +519,7 @@ pub(crate) async fn run_multiplexed_environment( executor_registration_id: String, identity: NoiseChannelIdentity, validator: V, + admission: Option, ) -> RendezvousDisconnectReason where T: Sink + Stream> + Unpin + Send + 'static, @@ -545,6 +548,11 @@ where let mut streams: HashMap> = HashMap::new(); let mut pending_handshakes: HashMap = HashMap::new(); let mut validation_tasks: JoinSet = JoinSet::new(); + let stream_tasks = admission.as_ref().map(|_| TaskTracker::new()); + let shutdown = admission + .as_ref() + .map(|a| a.shutdown.clone()) + .unwrap_or_default(); let mut failed_handshakes = 0usize; let mut handshake_cooldown_until = None; let mut next_validation_id = 0u64; @@ -589,6 +597,7 @@ where loop { // Registry calls run separately so a slow check does not block the relay. let frame = tokio::select! { + _ = shutdown.cancelled() => break, writer_result = &mut physical_writer_task => { match writer_result { Ok(reason) => disconnect_reason = reason, @@ -715,8 +724,8 @@ where streams.insert( validation_result.stream_id.clone(), spawn_noise_virtual_stream( - validation_result.stream_id, - validation_result.validation_id, + ClosedNoiseVirtualStream { stream_id: validation_result.stream_id, instance_id: validation_result.validation_id }, + stream_tasks.as_ref(), handler.clone(), physical_outgoing_tx.clone(), closed_stream_tx.clone(), @@ -724,6 +733,9 @@ where executor_registration.clone(), ), ); + if let Some(admission) = &admission { + admission.authorized.notify_one(); + } } Some(Err(error)) => { warn!("Noise relay harness key validation task failed: {error}"); @@ -815,6 +827,15 @@ where send_reset(&physical_outgoing_tx, stream_id); continue; } + let handshake_permit = if let Some(admission) = &admission { + let Some(permit) = admission.budget.try_admit() else { + send_reset(&physical_outgoing_tx, stream_id); + continue; + }; + Some(permit) + } else { + None + }; let prologue = noise_channel_prologue(&environment_id, &executor_registration_id, &stream_id); let request = match frame.into_handshake_payload() { @@ -880,6 +901,7 @@ where // Failed validation leaves no transport state. Preserve the // trusted registry status for request-scoped rejection handling. validation_tasks.spawn(async move { + let _handshake_permit = handshake_permit; let result = match timeout( HARNESS_KEY_VALIDATION_TIMEOUT, validator.validate_harness_key( @@ -954,7 +976,13 @@ where for (_stream_id, stream) in streams { stream.disconnect(); } - // Dropping the JoinSet aborts any registry checks still running. + // Reporters must not block on a closed physical connection while handlers drain. + drop(closed_stream_rx); + if let Some(stream_tasks) = stream_tasks { + validation_tasks.shutdown().await; + stream_tasks.close(); + stream_tasks.wait().await; + } if !physical_writer_task.is_finished() { physical_writer_task.abort(); let _ = physical_writer_task.await; @@ -1259,6 +1287,7 @@ mod tests { "test-registration".to_string(), NoiseChannelIdentity::generate()?, AllowHarnessKeyValidator, + /*admission*/ None, )); assert!(matches!(outbound_rx.next().await, Some(Message::Ping(_)))); diff --git a/codex-rs/exec-server/src/relay_noise_tests.rs b/codex-rs/exec-server/src/relay_noise_tests.rs index 2defe4bdd3..3a417cc84c 100644 --- a/codex-rs/exec-server/src/relay_noise_tests.rs +++ b/codex-rs/exec-server/src/relay_noise_tests.rs @@ -164,6 +164,7 @@ async fn missing_pong_disconnects_physical_relay() -> Result<()> { calls: Arc::new(AtomicUsize::new(0)), release: Arc::new(Notify::new()), }, + /*admission*/ None, )); assert_eq!( @@ -195,6 +196,7 @@ async fn pong_keeps_physical_relay_connected() -> Result<()> { calls: Arc::new(AtomicUsize::new(0)), release: Arc::new(Notify::new()), }, + /*admission*/ None, )); timeout(Duration::from_secs(1), async { @@ -307,6 +309,7 @@ async fn processor_exit_resets_noise_harness_stream() -> Result<()> { calls: Arc::new(AtomicUsize::new(0)), release, }, + /*admission*/ None, ))); let mut connection = noise_harness_connection_from_websocket_with_readiness( harness_websocket, @@ -375,6 +378,7 @@ async fn pending_harness_key_validation_does_not_block_new_handshakes() -> Resul calls: Arc::clone(&calls), release: Arc::new(Notify::new()), }, + /*admission*/ None, )); for stream_id in ["stream-1", "stream-2"] { @@ -429,6 +433,7 @@ async fn duplicate_handshakes_pause_admission_without_disconnecting() -> Result< calls: Arc::clone(&calls), release: Arc::clone(&release), }, + /*admission*/ None, )); let stream_id = "stream-1"; @@ -538,6 +543,7 @@ async fn oversized_harness_authorization_is_rejected_before_validation() -> Resu calls: Arc::clone(&calls), release: Arc::new(Notify::new()), }, + /*admission*/ None, )); let stream_id = "stream-1"; @@ -617,6 +623,7 @@ async fn malformed_handshakes_preserve_active_streams_and_admission_recovers() - }, timeout_started: Arc::clone(&timeout_started), }, + /*admission*/ None, ) .with_subscriber(subscriber), )); @@ -896,6 +903,7 @@ async fn early_data_during_validation_pauses_admission_without_disconnecting() - calls: Arc::new(AtomicUsize::new(0)), release: Arc::new(Notify::new()), }, + /*admission*/ None, )); for attempt in 0..MAX_FAILED_NOISE_HANDSHAKES { diff --git a/codex-rs/exec-server/src/remote.rs b/codex-rs/exec-server/src/remote.rs index ed1fa073fc..33c42bd3ae 100644 --- a/codex-rs/exec-server/src/remote.rs +++ b/codex-rs/exec-server/src/remote.rs @@ -70,10 +70,11 @@ mod registration_retry; /// Wire transport used after registering a remote exec-server. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -/// The transport used to connect a remote exec-server environment. pub enum RemoteEnvironmentTransport { #[default] Noise, + /// Accept authenticated Noise connections on this published guest address. + NoiseInbound(std::net::SocketAddr), Direct, } @@ -181,12 +182,9 @@ impl EnvironmentRegistryClient { security_profile: NOISE_RELAY_SECURITY_PROFILE.to_string(), executor_public_key: executor_public_key.clone(), }; - let (response, executor_credential) = self - .send_authenticated( - Request::new(Method::POST, url).with_json(&body), - Some(deadline), - ) - .await?; + let request = Request::new(Method::POST, url).with_json(&body); + let (response, executor_credential) = + self.send_authenticated(request, Some(deadline)).await?; let status = response.status(); // Read diagnostics within the same attempt budget, preserving a known error status. let response: EnvironmentRegistryRegistrationResponse = timeout_at(deadline, async { @@ -294,6 +292,7 @@ impl EnvironmentRegistryClient { )); } Ok(NoiseRendezvousConnectBundle { + http_headers: Default::default(), websocket_url: response.url, environment_id: response.environment_id, executor_registration_id: response.executor_registration_id, @@ -379,7 +378,7 @@ impl EnvironmentRegistryClient { } #[derive(Clone)] -struct RegistryHarnessKeyValidator { +pub(crate) struct RegistryHarnessKeyValidator { client: EnvironmentRegistryClient, environment_id: String, executor_registration_id: String, @@ -774,6 +773,35 @@ where }) .await } + RemoteEnvironmentTransport::NoiseInbound(address) => { + let inbound_processor = processor.clone(); + run_remote_transport( + config, + std::future::pending(), + |config, client| async move { + let control = crate::server::NoiseListenerControl::default(); + let run = run_inbound_noise_environment( + config, + client, + inbound_processor, + address, + control.clone(), + ); + tokio::pin!(run, shutdown); + let result = tokio::select! { + result = &mut run => result, + _ = &mut shutdown => { + control.stop(); + run.await + } + }; + control.stop(); + control.wait().await; + result + }, + ) + .await + } RemoteEnvironmentTransport::Direct => { run_remote_transport(config, shutdown, |config, client| { run_direct_environment(config, client, processor.clone()) @@ -798,7 +826,7 @@ where // separate Direct-compatible bridge, so reject it rather than use the // Noise-specific path. // Remove this guard when a Direct-compatible forwarder is added. - if config.transport == RemoteEnvironmentTransport::Direct { + if config.transport != RemoteEnvironmentTransport::Noise { return Err(ExecServerError::EnvironmentRegistryConfig( "direct exec-server transport does not support forwarding".to_string(), )); @@ -814,6 +842,42 @@ where .await } +async fn run_inbound_noise_environment( + config: RemoteEnvironmentConfig, + client: EnvironmentRegistryClient, + processor: ConnectionProcessor, + address: std::net::SocketAddr, + control: crate::server::NoiseListenerControl, +) -> Result<(), ExecServerError> { + let listener = tokio::net::TcpListener::bind(address) + .await + .map_err(ExecServerError::Spawn)?; + let identity = NoiseChannelIdentity::generate().map_err(|error| { + ExecServerError::Protocol(format!("failed to generate Noise identity: {error}")) + })?; + let public_key = identity.public_key(); + let registration = tokio::select! { + result = client.register_environment_with_retry(&config.environment_id, &public_key) => result?, + _ = control.cancelled() => return Ok(()), + }; + let validator = RegistryHarnessKeyValidator { + client, + environment_id: config.environment_id.clone(), + executor_registration_id: registration.executor_registration_id.clone(), + }; + crate::server::run_noise_websocket_listener( + listener, + processor, + config.environment_id, + registration.executor_registration_id, + identity, + validator, + control, + ) + .await + .map_err(ExecServerError::Spawn) +} + async fn run_remote_transport( config: RemoteEnvironmentConfig, shutdown: F, @@ -886,6 +950,7 @@ async fn run_remote_environment_connections( environment_id: config.environment_id.clone(), executor_registration_id, }, + /*admission*/ None, ) .await; backoff.connection_closed(connected_at.elapsed()); diff --git a/codex-rs/exec-server/src/server.rs b/codex-rs/exec-server/src/server.rs index 29f1440d29..f2c76a88fc 100644 --- a/codex-rs/exec-server/src/server.rs +++ b/codex-rs/exec-server/src/server.rs @@ -19,6 +19,8 @@ pub use request_dispatcher::ConcurrentRequestLimit; pub use request_dispatcher::RequestDispatchMode; pub use transport::DEFAULT_LISTEN_URL; pub use transport::ExecServerListenUrlParseError; +pub(crate) use transport::NoiseListenerControl; +pub(crate) use transport::run_noise_websocket_listener; use crate::ExecServerRuntimeOptions; use crate::ExecServerTelemetry; diff --git a/codex-rs/exec-server/src/server/transport.rs b/codex-rs/exec-server/src/server/transport.rs index 4bb1d1f120..8731af5511 100644 --- a/codex-rs/exec-server/src/server/transport.rs +++ b/codex-rs/exec-server/src/server/transport.rs @@ -1,10 +1,16 @@ //! Exec-server stdio and WebSocket listeners. Configured authentication gates //! WebSocket upgrades; unauthenticated listeners remain supported for compatibility. +use crate::noise_channel::NoiseChannelIdentity; +use crate::noise_relay::MAX_NOISE_RELAY_WEBSOCKET_MESSAGE_SIZE; +use crate::noise_relay::admission::HandshakeBudget; +use crate::noise_relay::admission::InboundAdmission; +use crate::relay::run_multiplexed_environment; use axum::Router; use axum::body::Body; use axum::extract::ConnectInfo; use axum::extract::State; +use axum::extract::ws::Message as AxumMessage; use axum::extract::ws::WebSocketUpgrade; use axum::http::HeaderMap; use axum::http::Request; @@ -21,12 +27,21 @@ use codex_websocket_auth::WebsocketAuthPolicy; use codex_websocket_auth::WebsocketAuthSettings; use codex_websocket_auth::authorize_upgrade; use codex_websocket_auth::policy_from_settings; +use futures::SinkExt; +use futures::StreamExt; use std::io::Write as _; use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; use tokio::io; use tokio::io::AsyncRead; use tokio::io::AsyncWrite; use tokio::net::TcpListener; +use tokio::sync::Notify; +use tokio::sync::Semaphore; +use tokio_tungstenite::tungstenite::Message as NoiseMessage; +use tokio_util::sync::CancellationToken; +use tokio_util::task::TaskTracker; use tracing::info; use tracing::warn; @@ -195,26 +210,108 @@ async fn run_websocket_listener( println!("ws://{local_addr}"); std::io::stdout().flush()?; + serve_websocket( + listener, + ExecServerWebSocketState { + processor, + auth_policy, + noise: None, + }, + ) + .await?; + Ok(()) +} + +const FIRST_NOISE_AUTHORIZATION_TIMEOUT: Duration = Duration::from_secs(20); + +#[derive(Clone, Default)] +pub(crate) struct NoiseListenerControl { + shutdown: CancellationToken, + tasks: TaskTracker, + budget: Arc, +} + +impl NoiseListenerControl { + pub(crate) fn stop(&self) { + self.shutdown.cancel(); + } + pub(crate) async fn cancelled(&self) { + self.shutdown.cancelled().await; + } + pub(crate) async fn wait(&self) { + self.tasks.close(); + self.tasks.wait().await; + } +} + +#[derive(Clone)] +struct NoiseListener { + environment_id: String, + registration_id: String, + identity: NoiseChannelIdentity, + validator: crate::remote::RegistryHarnessKeyValidator, + permits: Arc, + control: NoiseListenerControl, +} + +pub(crate) async fn run_noise_websocket_listener( + listener: TcpListener, + processor: ConnectionProcessor, + environment_id: String, + registration_id: String, + identity: NoiseChannelIdentity, + validator: crate::remote::RegistryHarnessKeyValidator, + control: NoiseListenerControl, +) -> std::io::Result<()> { + serve_websocket( + listener, + ExecServerWebSocketState { + processor, + auth_policy: WebsocketAuthPolicy::default(), + noise: Some(NoiseListener { + environment_id, + registration_id, + identity, + validator, + permits: Arc::new(Semaphore::new(64)), + control, + }), + }, + ) + .await +} + +async fn serve_websocket( + listener: TcpListener, + state: ExecServerWebSocketState, +) -> std::io::Result<()> { + let shutdown = state + .noise + .as_ref() + .map(|noise| noise.control.shutdown.clone()); let router = Router::new() .route("/", any(websocket_upgrade_handler)) .route("/readyz", get(readiness_handler)) .layer(middleware::from_fn(reject_requests_with_origin_header)) - .with_state(ExecServerWebSocketState { - processor, - auth_policy, - }); + .with_state(state); axum::serve( listener, router.into_make_service_with_connect_info::(), ) - .await?; - Ok(()) + .with_graceful_shutdown(async move { + match shutdown { + Some(shutdown) => shutdown.cancelled().await, + None => std::future::pending::<()>().await, + } + }) + .await } #[derive(Clone)] struct ExecServerWebSocketState { processor: ConnectionProcessor, auth_policy: WebsocketAuthPolicy, + noise: Option, } async fn readiness_handler() -> StatusCode { @@ -246,19 +343,88 @@ async fn websocket_upgrade_handler( if let Err(err) = authorize_upgrade(&headers, &state.auth_policy) { return (err.status_code(), err.message()).into_response(); } + let (websocket, permit) = if let Some(noise) = &state.noise { + let Ok(permit) = Arc::clone(&noise.permits).try_acquire_owned() else { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + }; + ( + websocket + .max_message_size(MAX_NOISE_RELAY_WEBSOCKET_MESSAGE_SIZE) + .max_frame_size(MAX_NOISE_RELAY_WEBSOCKET_MESSAGE_SIZE), + Some(permit), + ) + } else { + (websocket, None) + }; info!(%peer_addr, "exec-server websocket client connected"); + // Register before Axum spawns the upgrade callback, including upgrades still pending. + let task = state + .noise + .as_ref() + .map(|noise| noise.control.tasks.token()); let mut response = websocket .on_upgrade(move |stream| async move { - state - .processor - .run_connection( - JsonRpcConnection::from_axum_websocket( - stream, - format!("exec-server websocket {peer_addr}"), - ), - ConnectionTransport::WebSocket, - ) - .await; + let _task = task; + let _permit = permit; + if let Some(noise) = state.noise { + let stream = stream + .with(|message: NoiseMessage| { + futures::future::ready(Ok::<_, axum::Error>(match message { + NoiseMessage::Binary(v) => AxumMessage::Binary(v), + NoiseMessage::Text(v) => AxumMessage::Text(v.as_str().into()), + NoiseMessage::Ping(v) => AxumMessage::Ping(v), + NoiseMessage::Pong(v) => AxumMessage::Pong(v), + NoiseMessage::Close(_) | NoiseMessage::Frame(_) => { + AxumMessage::Close(None) + } + })) + }) + .map(|result| { + result.map(|message| match message { + AxumMessage::Binary(v) => NoiseMessage::Binary(v), + AxumMessage::Text(v) => NoiseMessage::Text(v.as_str().into()), + AxumMessage::Ping(v) => NoiseMessage::Ping(v), + AxumMessage::Pong(v) => NoiseMessage::Pong(v), + AxumMessage::Close(_) => NoiseMessage::Close(None), + }) + }); + let admission = InboundAdmission { + budget: Arc::clone(&noise.control.budget), + authorized: Arc::new(Notify::new()), + shutdown: noise.control.shutdown.child_token(), + }; + let authorized = Arc::clone(&admission.authorized); + let shutdown = admission.shutdown.clone(); + let run = run_multiplexed_environment( + stream, + state.processor, + noise.environment_id, + noise.registration_id, + noise.identity, + noise.validator, + Some(admission), + ); + tokio::pin!(run); + tokio::select! { + _ = &mut run => {}, + _ = authorized.notified() => { run.await; }, + _ = tokio::time::sleep(FIRST_NOISE_AUTHORIZATION_TIMEOUT) => { + shutdown.cancel(); + run.await; + }, + } + } else { + state + .processor + .run_connection( + JsonRpcConnection::from_axum_websocket( + stream, + format!("exec-server websocket {peer_addr}"), + ), + ConnectionTransport::WebSocket, + ) + .await; + } }) .into_response(); if let Some(request_id) = headers.get("x-request-id") { diff --git a/codex-rs/exec-server/tests/common/relay.rs b/codex-rs/exec-server/tests/common/relay.rs index d10f06c145..f20440b504 100644 --- a/codex-rs/exec-server/tests/common/relay.rs +++ b/codex-rs/exec-server/tests/common/relay.rs @@ -52,8 +52,8 @@ pub(crate) fn static_registry_auth_provider() -> codex_api::SharedAuthProvider { } pub(crate) struct RelayTest { - registry: MockServer, - listener: TcpListener, + pub(crate) registry: MockServer, + pub(crate) listener: TcpListener, } pub(crate) struct RelayConnection { @@ -109,6 +109,7 @@ impl RelayTest { let harness_identity = NoiseChannelIdentity::generate()?; let client_args = NoiseRendezvousConnectArgs { bundle: NoiseRendezvousConnectBundle { + http_headers: Default::default(), websocket_url: format!("{rendezvous_url}/relay?role=harness"), environment_id: ENVIRONMENT_ID.to_string(), executor_registration_id: EXECUTOR_REGISTRATION_ID.to_string(), diff --git a/codex-rs/exec-server/tests/relay.rs b/codex-rs/exec-server/tests/relay.rs index 8bca066ef1..a0b1491293 100644 --- a/codex-rs/exec-server/tests/relay.rs +++ b/codex-rs/exec-server/tests/relay.rs @@ -21,10 +21,13 @@ use codex_exec_server::EnvironmentManager; use codex_exec_server::EnvironmentReadyInfo; use codex_exec_server::ExecParams; use codex_exec_server::ExecResponse; +use codex_exec_server::ExecServerClient; use codex_exec_server::ExecServerError; use codex_exec_server::ExecServerRuntimeOptions; use codex_exec_server::FsReadFileParams; +use codex_exec_server::NoiseChannelIdentity; use codex_exec_server::NoiseChannelPublicKey; +use codex_exec_server::NoiseRendezvousConnectArgs; use codex_exec_server::NoiseRendezvousConnectBundle; use codex_exec_server::NoiseRendezvousConnectProvider; use codex_exec_server::ProcessId; @@ -82,6 +85,7 @@ impl NoiseRendezvousConnectProvider for FreshBundleNoiseConnectProvider { ) -> BoxFuture<'_, Result> { let call = self.calls.fetch_add(1, Ordering::Relaxed) + 1; let bundle = NoiseRendezvousConnectBundle { + http_headers: Default::default(), websocket_url: self.websocket_url.clone(), environment_id: ENVIRONMENT_ID.to_string(), executor_registration_id: EXECUTOR_REGISTRATION_ID.to_string(), @@ -421,3 +425,138 @@ async fn remote_environment_routes_encrypted_exec_server_rpc() -> Result<()> { timeout(TEST_TIMEOUT, remote_environment).await???; Ok(()) } + +/// An inbound executor serves encrypted RPCs directly and rejects a registry-denied harness. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn inbound_noise_executes_only_after_registry_authorization() -> Result<()> { + use codex_exec_server::RemoteEnvironmentTransport; + use futures::SinkExt; + use futures::StreamExt; + use tokio_tungstenite::tungstenite::Message; + use wiremock::matchers::body_partial_json; + + let relay = RelayTest::new().await?; + let address = relay.listener.local_addr()?; + let registry = relay.registry; + drop(relay.listener); + Mock::given(method("POST")) + .and(path(format!( + "/cloud/environment/{ENVIRONMENT_ID}/validate" + ))) + .and(body_partial_json( + serde_json::json!({"harness_key_authorization": "denied"}), + )) + .respond_with(ResponseTemplate::new(403)) + .with_priority(1) + .mount(®istry) + .await; + let config = RemoteEnvironmentConfig::new_with_transport( + registry.uri(), + ENVIRONMENT_ID.to_string(), + RemoteEnvironmentTransport::NoiseInbound(address), + static_registry_auth_provider(), + HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault), + )?; + let (codex_exe, sandbox_exe) = common::current_test_binary_helper_paths()?; + let runtime = ExecServerRuntimeOptions::new(codex_exe, sandbox_exe)?; + let (stop, stopped) = tokio::sync::oneshot::channel(); + let server = AbortOnDropHandle::new(tokio::spawn( + codex_exec_server::run_remote_environment_until_shutdown(config, runtime, async move { + let _ = stopped.await; + }), + )); + let executor_key = timeout(TEST_TIMEOUT, async { + loop { + if let Ok(key) = registered_executor_public_key(®istry).await { + break key; + } + tokio::task::yield_now().await; + } + }) + .await?; + let (mut plaintext, _) = tokio_tungstenite::connect_async(format!("ws://{address}/")).await?; + plaintext + .send(Message::Text( + r#"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"clientName":"plaintext"}}"# + .into(), + )) + .await?; + assert!( + timeout(std::time::Duration::from_millis(150), plaintext.next()) + .await + .is_err() + ); + let idle = tokio::spawn(async move { + while let Some(message) = plaintext.next().await { + match message { + Ok(Message::Ping(payload)) => plaintext.send(Message::Pong(payload)).await?, + Ok(Message::Close(_)) | Err(_) => break, + _ => {} + } + } + anyhow::Ok(()) + }); + let connect = |authorization: &str| -> Result { + Ok(NoiseRendezvousConnectArgs { + bundle: NoiseRendezvousConnectBundle { + http_headers: Default::default(), + websocket_url: format!("ws://{address}/"), + environment_id: ENVIRONMENT_ID.to_string(), + executor_registration_id: EXECUTOR_REGISTRATION_ID.to_string(), + executor_public_key: executor_key.clone(), + harness_key_authorization: authorization.to_string(), + }, + harness_identity: NoiseChannelIdentity::generate()?, + client_name: "inbound-test".to_string(), + connect_timeout: TEST_TIMEOUT, + initialize_timeout: TEST_TIMEOUT, + resume_session_id: None, + http_client_factory: HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault), + }) + }; + assert!( + ExecServerClient::connect_noise_rendezvous(connect("denied")?) + .await + .is_err() + ); + let client = + ExecServerClient::connect_noise_rendezvous(connect(HARNESS_KEY_AUTHORIZATION)?).await?; + let temp = TempDir::new()?; + let path = temp.path().join("proof"); + std::fs::write(&path, b"inbound-noise-proof")?; + let response = client + .fs_read_file(FsReadFileParams { + path: PathUri::from_host_native_path(path.clone())?, + follow_symlinks: None, + sandbox: None, + }) + .await?; + assert_eq!( + STANDARD.decode(response.data_base64)?, + b"inbound-noise-proof" + ); + timeout(std::time::Duration::from_secs(25), idle).await???; + // The first-authorization deadline must not close an established Noise session. + client + .fs_read_file(FsReadFileParams { + path: PathUri::from_host_native_path(path.clone())?, + follow_symlinks: None, + sandbox: None, + }) + .await?; + let _ = stop.send(()); + timeout(TEST_TIMEOUT, server).await???; + assert!( + timeout( + TEST_TIMEOUT, + client.fs_read_file(FsReadFileParams { + path: PathUri::from_host_native_path(path)?, + follow_symlinks: None, + sandbox: None, + }) + ) + .await? + .is_err() + ); + Ok(()) +} diff --git a/codex-rs/exec-server/tests/relay/registration_retry_tests.rs b/codex-rs/exec-server/tests/relay/registration_retry_tests.rs index 987b4b34a8..84afb52ed1 100644 --- a/codex-rs/exec-server/tests/relay/registration_retry_tests.rs +++ b/codex-rs/exec-server/tests/relay/registration_retry_tests.rs @@ -106,6 +106,7 @@ impl RegistryFixture { let environment = accept_websocket(&self.listener, "environment").await?; let args = NoiseRendezvousConnectArgs { bundle: NoiseRendezvousConnectBundle { + http_headers: Default::default(), websocket_url: format!("ws://{}/relay?role=harness", self.listener.local_addr()?), environment_id: ENVIRONMENT_ID.to_string(), executor_registration_id: registration_id.to_string(),