mirror of
https://github.com/openai/codex.git
synced 2026-11-08 20:40:33 +00:00
Stop Noise relay recovery on rejected executor credentials (#59750)
## Why Registry `401` responses during harness key validation previously produced generic relay resets, causing recovery to retry rejected executor credentials instead of surfacing the authentication failure. ## What changed - Send an authenticated Noise handshake rejection for registry `401` responses without opening an application transport. - Propagate the rejection as `EnvironmentRegistryAuth` and preserve it in the terminal connection state so recovery stops with a specific authentication error. - Preserve registry HTTP status codes so `403` responses for stale harness grants and transient service failures can continue through recovery. ## Testing Add tests that reject tampered authentication rejections and exercise session recovery: `401` stops after one rejected recovery attempt, while `403`, `502`, and `503` retry with fresh harness grants and reconnect successfully. GitOrigin-RevId: 9a55412e583146c094948618c7c9aeb4fb4b953e
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
//! Preserve policy denials through terminal connection state and allow later policy recovery.
|
||||
//! Preserve policy denials and registry auth errors through terminal connection state.
|
||||
|
||||
use codex_http_client::NetworkPolicyDenied;
|
||||
use codex_http_client::RouteAwareRequestError;
|
||||
@@ -11,6 +11,8 @@ pub(super) enum ConnectionFailure {
|
||||
Disconnected(String),
|
||||
#[error(transparent)]
|
||||
Policy(NetworkPolicyDenied),
|
||||
#[error("environment registry authentication error: {0}")]
|
||||
RegistryAuth(String),
|
||||
}
|
||||
|
||||
impl From<ConnectionFailure> for ExecServerError {
|
||||
@@ -18,6 +20,7 @@ impl From<ConnectionFailure> for ExecServerError {
|
||||
match failure {
|
||||
ConnectionFailure::Disconnected(message) => Self::Disconnected(message),
|
||||
ConnectionFailure::Policy(denial) => Self::ApplicationNetworkPolicy(denial),
|
||||
ConnectionFailure::RegistryAuth(message) => Self::EnvironmentRegistryAuth(message),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -456,6 +456,11 @@ impl Inner {
|
||||
return;
|
||||
}
|
||||
if !is_retryable_recovery_error(&error) {
|
||||
if let ExecServerError::EnvironmentRegistryAuth(message) = error {
|
||||
self.fail(super::ConnectionFailure::RegistryAuth(message))
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
break error.to_string();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -207,7 +207,7 @@ struct OpenNoiseRendezvousConnection {
|
||||
connection: JsonRpcConnection,
|
||||
pending_tasks: Vec<AbortOnDropHandle<()>>,
|
||||
options: ExecServerClientConnectOptions,
|
||||
handshake_ready: tokio::sync::oneshot::Receiver<()>,
|
||||
handshake_ready: tokio::sync::oneshot::Receiver<Result<(), ExecServerError>>,
|
||||
}
|
||||
|
||||
struct ReadyNoiseRendezvousConnection {
|
||||
@@ -749,7 +749,7 @@ impl ExecServerClient {
|
||||
initialize_span: &tracing::Span,
|
||||
) -> Result<(), ExecServerError> {
|
||||
match timeout_at(deadline, &mut connection.handshake_ready).await {
|
||||
Ok(Ok(())) => Ok(()),
|
||||
Ok(Ok(result)) => result,
|
||||
Ok(Err(_)) => Err(ExecServerError::Disconnected(
|
||||
"Noise harness handshake failed before connection became ready".to_string(),
|
||||
)),
|
||||
|
||||
@@ -3,7 +3,8 @@
|
||||
//! The harness initiates hybrid IK and pins the exec-server static key returned
|
||||
//! by the registry. The first handshake message lets the exec-server authenticate
|
||||
//! the harness static key; the exec-server then asks the registry whether that
|
||||
//! key is authorized before completing the handshake.
|
||||
//! key is authorized before opening a transport. A denied request can receive an
|
||||
//! authenticated rejection without creating a transport on either peer.
|
||||
//!
|
||||
//! "Hybrid" means the session keys include both X25519 and ML-KEM-768 key
|
||||
//! agreement. Once the two-message handshake finishes, AES-GCM protects the
|
||||
@@ -33,6 +34,7 @@ use serde::Serialize;
|
||||
pub(crate) const NOISE_CHANNEL_SUITE: &str = "Noise_hybridIK_X25519+MLKEM768_AESGCM_SHA256";
|
||||
|
||||
const PROLOGUE_DOMAIN: &[u8] = b"codex-exec-server-relay-noise/v1";
|
||||
const REGISTRY_AUTH_REJECTED: &[u8] = b"codex-registry-authentication-rejected/v1";
|
||||
|
||||
type Handshake = HybridHandshake<X25519, MlKem768, MlKem768, AesGcm, Sha256>;
|
||||
type Transport = TransportState<AesGcm, Sha256>;
|
||||
@@ -152,12 +154,16 @@ impl InitiatorHandshake {
|
||||
}
|
||||
|
||||
/// Consume the executor response and enter transport mode.
|
||||
/// The v1 response does not carry an application payload.
|
||||
/// Successful responses have no payload. A fixed authenticated rejection
|
||||
/// fails this request without opening an application transport.
|
||||
pub(crate) fn finish(mut self, response: &[u8]) -> Result<NoiseTransport, NoiseChannelError> {
|
||||
ensure_noise_frame_len(response.len(), "handshake response is too large")?;
|
||||
let overhead = self.handshake.get_next_message_overhead()?;
|
||||
let mut payload = vec![0u8; response.len().saturating_sub(overhead)];
|
||||
let payload_len = self.handshake.read_message(response, &mut payload)?;
|
||||
if &payload[..payload_len] == REGISTRY_AUTH_REJECTED {
|
||||
return Err(NoiseChannelError::RegistryAuthenticationRejected);
|
||||
}
|
||||
if payload_len != 0 {
|
||||
return Err(NoiseChannelError::InvalidMessage(
|
||||
"handshake response payload must be empty",
|
||||
@@ -227,6 +233,18 @@ impl PendingResponderHandshake {
|
||||
response,
|
||||
))
|
||||
}
|
||||
|
||||
/// Authenticate a terminal registry 401 to the harness without authorizing
|
||||
/// its key or constructing a transport. Older peers reject nonempty payloads.
|
||||
pub(crate) fn reject_registry_authentication(mut self) -> Result<Vec<u8>, NoiseChannelError> {
|
||||
let overhead = self.handshake.get_next_message_overhead()?;
|
||||
let mut response = vec![0u8; overhead + REGISTRY_AUTH_REJECTED.len()];
|
||||
let response_len = self
|
||||
.handshake
|
||||
.write_message(REGISTRY_AUTH_REJECTED, &mut response)?;
|
||||
response.truncate(response_len);
|
||||
Ok(response)
|
||||
}
|
||||
}
|
||||
|
||||
/// Established channel with independent implicit send and receive nonces.
|
||||
@@ -294,6 +312,8 @@ fn ensure_noise_frame_len(
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum NoiseChannelError {
|
||||
#[error("environment registry authentication rejected")]
|
||||
RegistryAuthenticationRejected,
|
||||
#[error("Noise channel key generation failed: {0}")]
|
||||
KeyGeneration(String),
|
||||
#[error("invalid Noise channel public key: {0}")]
|
||||
|
||||
@@ -83,6 +83,40 @@ fn initiator_rejects_wrong_responder_key() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_rejection_must_be_authenticated() {
|
||||
for tamper in [false, true] {
|
||||
let initiator = NoiseChannelIdentity::generate().expect("initiator");
|
||||
let responder = NoiseChannelIdentity::generate().expect("responder");
|
||||
let prologue = noise_channel_prologue("env-1", "registration-1", "stream-1");
|
||||
let (handshake, request) = InitiatorHandshake::start(
|
||||
&initiator,
|
||||
&responder.public_key(),
|
||||
&prologue,
|
||||
b"authorization",
|
||||
)
|
||||
.expect("request");
|
||||
let pending = PendingResponderHandshake::read_request(&responder, &prologue, &request)
|
||||
.expect("read request");
|
||||
let mut response = pending.reject_registry_authentication().expect("rejection");
|
||||
if tamper {
|
||||
*response.last_mut().expect("encrypted payload") ^= 1;
|
||||
}
|
||||
let error = handshake
|
||||
.finish(&response)
|
||||
.err()
|
||||
.expect("must not open transport");
|
||||
if tamper {
|
||||
assert!(matches!(error, NoiseChannelError::Handshake(_)));
|
||||
} else {
|
||||
assert!(matches!(
|
||||
error,
|
||||
NoiseChannelError::RegistryAuthenticationRejected
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn responder_rejects_mismatched_prologue() {
|
||||
let initiator = NoiseChannelIdentity::generate().expect("generate initiator identity");
|
||||
|
||||
@@ -31,6 +31,7 @@ use crate::connection::JsonRpcConnectionEvent;
|
||||
use crate::connection::JsonRpcTransport;
|
||||
use crate::connection::WEBSOCKET_KEEPALIVE_INTERVAL;
|
||||
use crate::noise_channel::InitiatorHandshake;
|
||||
use crate::noise_channel::NoiseChannelError;
|
||||
use crate::noise_channel::NoiseChannelIdentity;
|
||||
use crate::noise_channel::NoiseChannelPublicKey;
|
||||
use crate::noise_channel::NoiseTransport;
|
||||
@@ -73,7 +74,7 @@ pub(crate) struct NoiseHarnessConnectionArgs {
|
||||
/// transport is ready for application messages.
|
||||
pub(crate) struct NoiseHarnessConnection {
|
||||
pub(crate) connection: JsonRpcConnection,
|
||||
pub(crate) handshake_ready: oneshot::Receiver<()>,
|
||||
pub(crate) handshake_ready: oneshot::Receiver<Result<(), ExecServerError>>,
|
||||
pub(crate) stream_id: String,
|
||||
pub(crate) handshake_started: Instant,
|
||||
}
|
||||
@@ -277,10 +278,21 @@ where
|
||||
"Noise harness handshake completed"
|
||||
);
|
||||
if let Some(handshake_ready_tx) = handshake_ready_tx.take() {
|
||||
let _ = handshake_ready_tx.send(());
|
||||
let _ = handshake_ready_tx.send(Ok(()));
|
||||
}
|
||||
break transport;
|
||||
}
|
||||
Err(NoiseChannelError::RegistryAuthenticationRejected) => {
|
||||
record_handshake_failure("registry_authentication_rejected");
|
||||
if let Some(handshake_ready_tx) = handshake_ready_tx.take() {
|
||||
let _ = handshake_ready_tx.send(Err(ExecServerError::EnvironmentRegistryAuth(
|
||||
"executor credentials rejected".to_string(),
|
||||
)));
|
||||
}
|
||||
send_disconnected(&incoming_tx, &disconnected_tx,
|
||||
"environment registry rejected executor credentials".to_string());
|
||||
return;
|
||||
}
|
||||
Err(error) => {
|
||||
record_handshake_failure("handshake_finish");
|
||||
send_disconnected(
|
||||
|
||||
@@ -495,7 +495,7 @@ where
|
||||
///
|
||||
/// Implementations must consult an authority independent of rendezvous. The
|
||||
/// exec-server invokes this after parsing the first IK message and before
|
||||
/// completing the responder handshake.
|
||||
/// opening an application transport.
|
||||
pub(crate) trait HarnessKeyValidator: Send + Sync {
|
||||
fn validate_harness_key(
|
||||
&self,
|
||||
@@ -645,7 +645,20 @@ where
|
||||
reason,
|
||||
registry_status,
|
||||
);
|
||||
send_reset(&physical_outgoing_tx, validation_result.stream_id);
|
||||
// Authenticate the rejection so the harness can stop retrying.
|
||||
if registry_status == Some(401)
|
||||
&& let Ok(response) = pending.handshake.reject_registry_authentication()
|
||||
{
|
||||
let frame = RelayMessageFrame::handshake(
|
||||
validation_result.stream_id,
|
||||
response,
|
||||
);
|
||||
let _ = physical_outgoing_tx.try_send(
|
||||
encode_relay_message_frame(&frame),
|
||||
);
|
||||
} else {
|
||||
send_reset(&physical_outgoing_tx, validation_result.stream_id);
|
||||
}
|
||||
record_failed_handshake(
|
||||
&mut failed_handshakes,
|
||||
&mut handshake_cooldown_until,
|
||||
@@ -864,8 +877,8 @@ where
|
||||
);
|
||||
let validator = validator.clone();
|
||||
|
||||
// Failed validation leaves no transport state and sends only a
|
||||
// generic reset.
|
||||
// Failed validation leaves no transport state. Preserve the
|
||||
// trusted registry status for request-scoped rejection handling.
|
||||
validation_tasks.spawn(async move {
|
||||
let result = match timeout(
|
||||
HARNESS_KEY_VALIDATION_TIMEOUT,
|
||||
|
||||
@@ -432,11 +432,8 @@ impl HarnessKeyValidator for RegistryHarnessKeyValidator {
|
||||
if !status.is_success() {
|
||||
// The request contains the short-lived authorization. Do not include
|
||||
// a response body that might echo it in logs or error chains.
|
||||
if matches!(status, StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN) {
|
||||
return Err(ExecServerError::EnvironmentRegistryAuth(format!(
|
||||
"environment registry harness key validation authentication failed ({status})"
|
||||
)));
|
||||
}
|
||||
// Preserve the trusted HTTP status: a 401 rejects this executor's
|
||||
// credentials; a 403 can reject a stale harness registration/grant.
|
||||
return Err(ExecServerError::EnvironmentRegistryHttp {
|
||||
status,
|
||||
code: None,
|
||||
|
||||
Reference in New Issue
Block a user