Add rustls fallback for local MCP HTTP requests (#38436)

## Why

Local MCP requests can fail when the platform TLS backend cannot negotiate a
protocol version with an HTTPS endpoint.

## What changed

- Retry replayable local MCP requests once with rustls after a recognized TLS
  protocol-version negotiation failure. Keep certificate, timeout, and unrelated
  connection failures on the existing error path.
- Remember successful fallback per HTTPS origin and outbound route, while keeping
  the platform TLS backend as the default for other destinations.
- Share the fallback-enabled client across local MCP resolution, CLI login, and
  OAuth discovery while preserving remote environment HTTP clients.

## Testing

Added coverage for platform-specific error detection, request replay, cached
fallback reuse and isolation, non-replayable requests, redirects, and remote MCP
client selection.

GitOrigin-RevId: 39a2d96fdb2ea0e51df14f652ba2a953d24e69a1
This commit is contained in:
Celia Chen
2026-08-13 21:08:10 +00:00
committed by copyberry
parent 93327c852a
commit b87327f4e5
10 changed files with 1204 additions and 30 deletions

View File

@@ -9,6 +9,8 @@ use http::HeaderMap;
use std::sync::Arc;
use std::time::Duration;
use codex_utils_rustls_provider::ensure_rustls_crypto_provider;
use crate::BuildCustomCaTransportError;
use crate::BuildRouteAwareHttpClientError;
use crate::ClientRouteClass;
@@ -33,6 +35,14 @@ pub struct HttpClientBuilder {
chatgpt_cloudflare_cookie_store: bool,
chatgpt_cookie_store: Option<Arc<ChatGptCookieStore>>,
request_logging: RequestLogging,
tls_backend: TlsBackend,
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
enum TlsBackend {
#[default]
TransportDefault,
Rustls,
}
impl HttpClientFactory {
@@ -85,6 +95,11 @@ impl HttpClientBuilder {
self.follow_redirects
}
pub(crate) fn with_rustls_tls(mut self) -> Self {
self.tls_backend = TlsBackend::Rustls;
self
}
/// Limits only connection establishment, not the request as a whole.
pub fn connect_timeout(mut self, timeout: Duration) -> Self {
self.connect_timeout = Some(timeout);
@@ -260,6 +275,10 @@ impl HttpClientBuilder {
fn base_reqwest_builder(self) -> reqwest::ClientBuilder {
let mut builder = reqwest::Client::builder();
if self.tls_backend == TlsBackend::Rustls {
ensure_rustls_crypto_provider();
builder = builder.use_rustls_tls();
}
if let Some(default_headers) = self.default_headers {
builder = builder.default_headers(default_headers);
}
@@ -288,6 +307,7 @@ impl Default for HttpClientBuilder {
chatgpt_cloudflare_cookie_store: false,
chatgpt_cookie_store: None,
request_logging: RequestLogging::Enabled,
tls_backend: TlsBackend::TransportDefault,
}
}
}

View File

@@ -8,6 +8,7 @@ mod outbound_proxy;
mod request;
mod route_aware_client_pool;
mod route_aware_redirect;
mod tls_backend_fallback;
mod transport;
pub use crate::chatgpt_cloudflare_cookies::with_chatgpt_cloudflare_cookie_store;

View File

@@ -31,6 +31,8 @@ use crate::route_aware_redirect::is_redirect;
use crate::route_aware_redirect::redirect_request;
use crate::route_aware_redirect::redirect_url;
use crate::route_aware_redirect::remove_sensitive_headers;
use crate::tls_backend_fallback::RustlsClientCache;
use crate::tls_backend_fallback::should_retry_with_rustls;
const MAX_CACHED_ROUTES: usize = 16;
@@ -41,6 +43,12 @@ enum CustomCaFallback {
LegacyTransportDefault,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum SelectedTlsBackend {
TransportDefault,
RustlsFallback,
}
/// Reuses transport clients by resolved route while selecting a route for every request URL.
///
/// Request creation stays on the pool so the URL used for PAC or system-proxy resolution cannot
@@ -53,6 +61,7 @@ pub struct RouteAwareClientPool {
client_builder: HttpClientBuilder,
custom_ca_fallback: CustomCaFallback,
clients: Arc<Mutex<HashMap<OutboundProxyRoute, HttpClient>>>,
rustls_clients: Option<RustlsClientCache>,
}
impl fmt::Debug for RouteAwareClientPool {
@@ -322,9 +331,20 @@ impl RouteAwareClientPool {
client_builder,
custom_ca_fallback: CustomCaFallback::Disabled,
clients: Arc::new(Mutex::new(HashMap::new())),
rustls_clients: None,
}
}
/// Retries recognized TLS protocol-negotiation failures once using rustls.
///
/// Successful fallback is remembered for each HTTPS origin and resolved outbound route.
/// Rustls clients are reused across fallback destinations that share a route, while other
/// destinations retain the existing transport-default backend.
pub fn with_tls_backend_fallback(mut self) -> Self {
self.rustls_clients = Some(RustlsClientCache::default());
self
}
/// Creates a pool with the shared defaults but without URL or response-header diagnostics.
pub fn new_without_request_logging(
http_client_factory: HttpClientFactory,
@@ -464,9 +484,7 @@ impl RouteAwareClientPool {
{
let request_method = request.method().clone();
let request_url = request.url().to_string();
let follows_redirects_manually = self.client_builder.follows_redirects()
&& self.http_client_factory.outbound_proxy_policy()
== OutboundProxyPolicy::RespectSystemProxy;
let follows_redirects_manually = self.follows_redirects_manually();
let timeout_deadline = request
.timeout()
.copied()
@@ -475,7 +493,7 @@ impl RouteAwareClientPool {
let mut previous_route = None;
loop {
let current_url = request.url().clone();
let (current_route, client) = match timeout_deadline {
let (current_route, client, selected_tls_backend) = match timeout_deadline {
Some(timeout_deadline) => tokio::time::timeout_at(
timeout_deadline,
self.client_for_url_with_resolver(current_url.as_str(), &resolve_route),
@@ -493,7 +511,7 @@ impl RouteAwareClientPool {
{
request.headers_mut().remove(PROXY_AUTHORIZATION);
}
previous_route = Some(current_route);
previous_route = Some(current_route.clone());
if let Some(timeout_deadline) = timeout_deadline {
let remaining = timeout_deadline
.checked_duration_since(tokio::time::Instant::now())
@@ -525,10 +543,24 @@ impl RouteAwareClientPool {
} {
Ok(response) => response,
Err(error) => {
if follows_redirects_manually {
client.log_error_summary(&request_method, &request_url, &error);
let result = self
.retry_with_rustls(
&current_url,
&current_route,
selected_tls_backend,
replay.as_ref(),
error,
timeout_deadline,
)
.await;
if follows_redirects_manually
&& let Err(RouteAwareRequestError::Request(error)) = &result
{
client.log_error_summary(&request_method, &request_url, error);
}
return Err(error.into());
result?
}
};
let status = response.status();
@@ -568,11 +600,75 @@ impl RouteAwareClientPool {
}
}
async fn retry_with_rustls(
&self,
current_url: &reqwest::Url,
current_route: &OutboundProxyRoute,
selected_tls_backend: SelectedTlsBackend,
replay: Option<&reqwest::Request>,
error: reqwest::Error,
timeout_deadline: Option<tokio::time::Instant>,
) -> Result<reqwest::Response, RouteAwareRequestError> {
let Some(rustls_clients) = self.rustls_clients.as_ref() else {
return Err(error.into());
};
if current_url.scheme() != "https"
|| selected_tls_backend == SelectedTlsBackend::RustlsFallback
|| !should_retry_with_rustls(&error)
{
return Err(error.into());
}
let Some(mut retry_request) = replay.and_then(reqwest::Request::try_clone) else {
return Err(error.into());
};
let fallback_client = match rustls_clients.client_for_route(current_route) {
Some(client) => client,
None => self.rustls_client_for_route(current_route)?,
};
if let Some(timeout_deadline) = timeout_deadline {
let remaining = timeout_deadline
.checked_duration_since(tokio::time::Instant::now())
.ok_or(RouteAwareRequestError::Timeout)?;
if remaining.is_zero() {
return Err(RouteAwareRequestError::Timeout);
}
*retry_request.timeout_mut() = Some(remaining);
}
let execute_retry = async {
if self.follows_redirects_manually() {
fallback_client
.execute_without_request_logging(retry_request)
.await
} else {
fallback_client.execute(retry_request).await
}
};
let response = match timeout_deadline {
Some(timeout_deadline) => tokio::time::timeout_at(timeout_deadline, execute_retry)
.await
.map_err(|_| RouteAwareRequestError::Timeout)?,
None => execute_retry.await,
}?;
rustls_clients.remember(current_url, current_route, fallback_client);
tracing::info!(
event.name = "codex.http_client.tls_backend_fallback",
"HTTP client switched to rustls after a TLS protocol negotiation failure"
);
Ok(response)
}
async fn client_for_url_with_resolver<F, Fut>(
&self,
request_url: &str,
resolve_route: F,
) -> Result<(OutboundProxyRoute, HttpClient), RouteAwareClientPoolError>
) -> Result<(OutboundProxyRoute, HttpClient, SelectedTlsBackend), RouteAwareClientPoolError>
where
F: FnOnce(String) -> Fut,
Fut: Future<Output = io::Result<OutboundProxyRoute>>,
@@ -580,20 +676,26 @@ impl RouteAwareClientPool {
let route = resolve_route(request_url.to_string())
.await
.map_err(RouteAwareClientPoolError::Resolve)?;
if let Some(rustls_clients) = self.rustls_clients.as_ref()
&& let Ok(url) = reqwest::Url::parse(request_url)
&& rustls_clients.requires_rustls(&url, &route)
&& let Some(client) = rustls_clients.client_for_route(&route)
{
return Ok((route, client, SelectedTlsBackend::RustlsFallback));
}
let clients = match self.clients.lock() {
Ok(clients) => clients,
Err(error) => panic!("route-aware client cache lock should not be poisoned: {error}"),
};
if let Some(client) = clients.get(&route) {
return Ok((route, client.clone()));
return Ok((route, client.clone(), SelectedTlsBackend::TransportDefault));
}
drop(clients);
let client_builder = match self.http_client_factory.outbound_proxy_policy() {
OutboundProxyPolicy::ReqwestDefault => self.client_builder.clone(),
OutboundProxyPolicy::RespectSystemProxy => {
self.client_builder.clone().without_redirects()
}
let client_builder = if self.follows_redirects_manually() {
self.client_builder.clone().without_redirects()
} else {
self.client_builder.clone()
};
#[expect(
deprecated,
@@ -621,7 +723,11 @@ impl RouteAwareClientPool {
Err(error) => panic!("route-aware client cache lock should not be poisoned: {error}"),
};
if let Some(existing_client) = clients.get(&route) {
return Ok((route, existing_client.clone()));
return Ok((
route,
existing_client.clone(),
SelectedTlsBackend::TransportDefault,
));
}
if clients.len() >= MAX_CACHED_ROUTES
&& let Some(route_to_evict) = clients.keys().next().cloned()
@@ -629,10 +735,34 @@ impl RouteAwareClientPool {
clients.remove(&route_to_evict);
}
clients.insert(route.clone(), client.clone());
Ok((route, client))
Ok((route, client, SelectedTlsBackend::TransportDefault))
}
fn follows_redirects_manually(&self) -> bool {
self.client_builder.follows_redirects()
&& (self.http_client_factory.outbound_proxy_policy()
== OutboundProxyPolicy::RespectSystemProxy
|| self.rustls_clients.is_some())
}
fn rustls_client_for_route(
&self,
route: &OutboundProxyRoute,
) -> Result<HttpClient, RouteAwareClientPoolError> {
let mut client_builder = self.client_builder.clone().with_rustls_tls();
if self.follows_redirects_manually() {
client_builder = client_builder.without_redirects();
}
client_builder
.build_for_resolved_route(&self.http_client_factory, self.route_class, route)
.map_err(Into::into)
}
}
#[cfg(test)]
#[path = "route_aware_client_pool_tests.rs"]
mod tests;
#[cfg(test)]
#[path = "route_aware_tls_fallback_tests.rs"]
mod tls_fallback_tests;

View File

@@ -200,6 +200,89 @@ async fn forwards_exact_urls_and_caches_clients_by_resolved_route() {
);
}
#[tokio::test]
async fn cached_tls_backend_only_changes_its_destination_and_route() {
let pool = RouteAwareClientPool::with_builder(
HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
ClientRouteClass::Other,
HttpClientBuilder::new(),
)
.with_tls_backend_fallback();
let rustls_url = "https://mcp.example.com/first";
let native_url = "https://another.example.com/first";
let proxied_url = "https://mcp.example.com/proxied";
let proxy = OutboundProxyRoute::Proxy {
url: "http://proxy.example.com".to_string(),
no_proxy: None,
};
let resolver = FakeRouteResolver::new(HashMap::from([
(rustls_url.to_string(), OutboundProxyRoute::Direct),
(native_url.to_string(), OutboundProxyRoute::Direct),
(proxied_url.to_string(), proxy),
]));
let fallback_client = HttpClientBuilder::new()
.with_rustls_tls()
.build_direct()
.expect("rustls client should build without proxy autodiscovery");
pool.rustls_clients
.as_ref()
.expect("TLS fallback cache")
.remember(
&reqwest::Url::parse(rustls_url).expect("valid rustls URL"),
&OutboundProxyRoute::Direct,
fallback_client,
);
resolve_with(&pool, &resolver, rustls_url)
.await
.expect("remembered destination should build the rustls client");
assert_eq!(pool.clients.lock().expect("client cache lock").len(), 0);
resolve_with(&pool, &resolver, native_url)
.await
.expect("another destination should retain its native TLS client");
resolve_with(&pool, &resolver, proxied_url)
.await
.expect("another proxy route should retain its native TLS client");
assert_eq!(pool.clients.lock().expect("client cache lock").len(), 2);
}
#[tokio::test]
async fn tls_fallback_pool_reselects_routes_for_each_redirect_hop() {
let (address, server) = spawn_response_server(vec![
"HTTP/1.1 302 Found\r\nLocation: /final\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
.to_string(),
"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok".to_string(),
]);
let initial_url = format!("http://{address}/start");
let final_url = format!("http://{address}/final");
let resolver = FakeRouteResolver::new(HashMap::from([
(initial_url.clone(), OutboundProxyRoute::Direct),
(final_url.clone(), OutboundProxyRoute::Direct),
]));
let pool = RouteAwareClientPool::new(
HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
ClientRouteClass::Other,
)
.with_tls_backend_fallback();
let request = reqwest::Request::new(
Method::GET,
reqwest::Url::parse(&initial_url).expect("valid initial URL"),
);
let response = pool
.send_with_resolver(request, |url| resolver.resolve(url))
.await
.expect("fallback-enabled client should follow redirects");
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(resolver.observed_urls(), vec![initial_url, final_url]);
assert_eq!(
server.join().expect("redirect server should finish").len(),
2
);
}
#[tokio::test]
async fn reqwest_default_route_preserves_transport_redirects() {
let listener = TcpListener::bind(("127.0.0.1", 0)).expect("redirect listener should bind");
@@ -595,7 +678,7 @@ async fn resolve_with(
request_url: &str,
) -> Result<HttpClient, RouteAwareClientPoolError> {
let resolver = resolver.clone();
let (_, client) = pool
let (_, client, _) = pool
.client_for_url_with_resolver(request_url, move |request_url| async move {
resolver.resolve(request_url).await
})

View File

@@ -0,0 +1,510 @@
use std::io;
use std::io::Read;
use std::io::Write;
use std::net::TcpListener;
use std::sync::Arc;
use std::sync::mpsc;
use std::thread;
use std::time::Duration;
use std::time::Instant;
use bytes::Bytes;
use futures::stream;
use pretty_assertions::assert_eq;
use rcgen::CertifiedKey;
use rcgen::generate_simple_self_signed;
use rustls_pki_types::PrivateKeyDer;
use super::ClientRouteClass;
use super::HttpClient;
use super::HttpClientFactory;
use super::Method;
use super::OutboundProxyPolicy;
use super::OutboundProxyRoute;
use super::RouteAwareClientPool;
use super::RouteAwareRequestError;
use super::SelectedTlsBackend;
use crate::tls_backend_fallback::should_retry_with_rustls;
const PROTOCOL_VERSION_TLS_ALERT: &[u8] = &[21, 3, 3, 0, 2, 2, 70];
type SuccessfulTlsFallbackServer = (String, HttpClient, mpsc::Receiver<io::Result<Vec<String>>>);
#[tokio::test]
async fn default_pool_does_not_retry_a_native_tls_protocol_failure() {
let (url, attempts, stop_server) =
spawn_protocol_version_rejection_server(/*maximum_attempts*/ 2)
.expect("TLS rejection server should start");
let pool = RouteAwareClientPool::new(
HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
ClientRouteClass::Other,
);
let request = reqwest::Request::new(
Method::POST,
reqwest::Url::parse(&url).expect("valid HTTPS URL"),
);
let error = pool
.send_with_resolver(request, |_| async { Ok(OutboundProxyRoute::Direct) })
.await
.expect_err("ordinary traffic should preserve the native TLS failure");
let _ = stop_server.send(());
assert!(error.is_connect());
assert_eq!(
attempts
.recv()
.expect("TLS server should finish")
.expect("TLS server should reject one connection"),
1
);
}
#[tokio::test]
async fn retries_a_native_tls_protocol_failure_once_with_rustls() {
let (url, attempts, stop_server) =
spawn_protocol_version_rejection_server(/*maximum_attempts*/ 2)
.expect("TLS rejection server should start");
let pool = RouteAwareClientPool::new(
HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
ClientRouteClass::Other,
)
.with_tls_backend_fallback();
let destination = reqwest::Url::parse(&url).expect("valid HTTPS URL");
let mut request = reqwest::Request::new(Method::POST, destination.clone());
*request.body_mut() = Some(Bytes::from_static(b"mcp-initialize").into());
let error = pool
.send_with_resolver(request, |_| async { Ok(OutboundProxyRoute::Direct) })
.await
.expect_err("both TLS handshakes should be rejected");
let _ = stop_server.send(());
assert!(error.is_connect());
assert_eq!(
attempts
.recv()
.expect("TLS server should finish")
.expect("TLS server should reject both connections"),
2,
"native TLS protocol failure should retry with rustls: {error:?}"
);
let rustls_clients = pool.rustls_clients.as_ref().expect("TLS fallback cache");
assert_eq!(
(
rustls_clients.requires_rustls(&destination, &OutboundProxyRoute::Direct),
rustls_clients
.client_for_route(&OutboundProxyRoute::Direct)
.is_some(),
),
(false, false)
);
}
#[tokio::test]
async fn retries_a_native_tls_failure_after_another_request_caches_rustls() {
let (url, attempts, stop_server) =
spawn_protocol_version_rejection_server(/*maximum_attempts*/ 2)
.expect("TLS rejection server should start");
let pool = RouteAwareClientPool::new(
HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
ClientRouteClass::Other,
)
.with_tls_backend_fallback();
let destination = reqwest::Url::parse(&url).expect("valid HTTPS URL");
let (route, native_client, selected_tls_backend) = pool
.client_for_url_with_resolver(destination.as_str(), |_| async {
Ok(OutboundProxyRoute::Direct)
})
.await
.expect("native TLS client should resolve");
assert_eq!(selected_tls_backend, SelectedTlsBackend::TransportDefault);
let request = reqwest::Request::new(Method::POST, destination.clone());
let replay = request.try_clone().expect("request should be replayable");
let error = native_client
.execute_without_request_logging(request)
.await
.expect_err("native TLS handshake should fail");
let rustls_client = pool
.rustls_client_for_route(&route)
.expect("rustls fallback client should build");
pool.rustls_clients
.as_ref()
.expect("TLS fallback cache")
.remember(&destination, &route, rustls_client);
let error = pool
.retry_with_rustls(
&destination,
&route,
selected_tls_backend,
Some(&replay),
error,
/*timeout_deadline*/ None,
)
.await
.expect_err("both TLS handshakes should be rejected");
let _ = stop_server.send(());
assert!(error.is_connect());
assert_eq!(
attempts
.recv()
.expect("TLS server should finish")
.expect("TLS server should reject both connections"),
2,
"an in-flight native TLS failure should retry despite a concurrent cache update"
);
}
#[tokio::test]
async fn does_not_retry_a_cached_rustls_tls_protocol_failure() {
let (url, attempts, stop_server) =
spawn_protocol_version_rejection_server(/*maximum_attempts*/ 2)
.expect("TLS rejection server should start");
let pool = RouteAwareClientPool::new(
HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
ClientRouteClass::Other,
)
.with_tls_backend_fallback();
let destination = reqwest::Url::parse(&url).expect("valid HTTPS URL");
let rustls_client = pool
.rustls_client_for_route(&OutboundProxyRoute::Direct)
.expect("rustls fallback client should build");
pool.rustls_clients
.as_ref()
.expect("TLS fallback cache")
.remember(&destination, &OutboundProxyRoute::Direct, rustls_client);
let request = reqwest::Request::new(Method::POST, destination);
let error = pool
.send_with_resolver(request, |_| async { Ok(OutboundProxyRoute::Direct) })
.await
.expect_err("cached rustls TLS handshake should fail");
let _ = stop_server.send(());
assert!(error.is_connect());
assert_eq!(
attempts
.recv()
.expect("TLS server should finish")
.expect("TLS server should reject one connection"),
1,
"a cached rustls client must not retry against itself"
);
}
#[tokio::test]
async fn successful_rustls_fallback_replays_the_request_and_reuses_the_destination() {
let (url, trusted_rustls_client, observed_requests) =
spawn_successful_tls_fallback_server().expect("TLS fallback server should start");
let pool = RouteAwareClientPool::new(
HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
ClientRouteClass::Other,
)
.with_tls_backend_fallback();
let destination = reqwest::Url::parse(&url).expect("valid HTTPS URL");
let existing_destination =
reqwest::Url::parse("https://another-mcp.example.com/mcp").expect("valid HTTPS URL");
let rustls_clients = pool.rustls_clients.as_ref().expect("TLS fallback cache");
// Another destination may already have established a trusted rustls client on this route.
// The test destination must still attempt native TLS before reusing that route-level client.
rustls_clients.remember(
&existing_destination,
&OutboundProxyRoute::Direct,
trusted_rustls_client,
);
assert!(!rustls_clients.requires_rustls(&destination, &OutboundProxyRoute::Direct));
let mut initialize_request = reqwest::Request::new(Method::POST, destination.clone());
*initialize_request.body_mut() = Some(Bytes::from_static(b"mcp-initialize").into());
let initialize_response = pool
.send_with_resolver(initialize_request, |_| async {
Ok(OutboundProxyRoute::Direct)
})
.await
.expect("native TLS protocol failure should recover with rustls");
assert_eq!(
initialize_response
.text()
.await
.expect("fallback response body should be readable"),
"ok"
);
assert!(rustls_clients.requires_rustls(&destination, &OutboundProxyRoute::Direct));
let mut tool_request = reqwest::Request::new(Method::POST, destination);
*tool_request.body_mut() = Some(Bytes::from_static(b"mcp-tool-call").into());
let tool_response = pool
.send_with_resolver(tool_request, |_| async { Ok(OutboundProxyRoute::Direct) })
.await
.expect("remembered destination should reuse rustls without another native TLS attempt");
assert_eq!(
tool_response
.text()
.await
.expect("cached rustls response body should be readable"),
"ok"
);
assert_eq!(
observed_requests
.recv_timeout(Duration::from_secs(5))
.expect("TLS fallback server should finish")
.expect("TLS fallback server should capture both requests"),
vec!["mcp-initialize".to_string(), "mcp-tool-call".to_string()]
);
}
#[tokio::test]
async fn retries_a_tls_protocol_failure_when_request_url_contains_certificate_markers() {
let (url, attempts, stop_server) =
spawn_protocol_version_rejection_server(/*maximum_attempts*/ 2)
.expect("TLS rejection server should start");
let pool = RouteAwareClientPool::new(
HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
ClientRouteClass::Other,
)
.with_tls_backend_fallback();
let mut destination = reqwest::Url::parse(&url).expect("valid HTTPS URL");
destination.set_path("/certificate/hostname/expired/revoked/mcp");
let request = reqwest::Request::new(Method::POST, destination);
let error = pool
.send_with_resolver(request, |_| async { Ok(OutboundProxyRoute::Direct) })
.await
.expect_err("both TLS handshakes should be rejected");
let _ = stop_server.send(());
assert!(error.is_connect());
assert_eq!(
attempts
.recv()
.expect("TLS server should finish")
.expect("TLS server should reject both connections"),
2,
"request URL text should not prevent TLS backend fallback: {error:?}"
);
}
#[tokio::test]
async fn does_not_retry_a_non_replayable_streaming_request() {
let (url, attempts, stop_server) =
spawn_protocol_version_rejection_server(/*maximum_attempts*/ 2)
.expect("TLS rejection server should start");
let pool = RouteAwareClientPool::new(
HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
ClientRouteClass::Other,
)
.with_tls_backend_fallback();
let mut request = reqwest::Request::new(
Method::POST,
reqwest::Url::parse(&url).expect("valid HTTPS URL"),
);
*request.body_mut() = Some(reqwest::Body::wrap_stream(stream::iter(vec![Ok::<
_,
io::Error,
>(
Bytes::from_static(b"streaming body"),
)])));
let error = pool
.send_with_resolver(request, |_| async { Ok(OutboundProxyRoute::Direct) })
.await
.expect_err("non-replayable request should preserve the native TLS failure");
let _ = stop_server.send(());
assert!(error.is_connect());
let RouteAwareRequestError::Request(ref request_error) = error else {
panic!("expected native TLS request error, got {error:?}");
};
assert!(
should_retry_with_rustls(request_error),
"native TLS protocol failure should be retryable: {request_error:?}"
);
assert_eq!(
attempts
.recv()
.expect("TLS server should finish")
.expect("TLS server should reject one connection"),
1
);
}
fn spawn_successful_tls_fallback_server() -> io::Result<SuccessfulTlsFallbackServer> {
codex_utils_rustls_provider::ensure_rustls_crypto_provider();
let CertifiedKey { cert, signing_key } =
generate_simple_self_signed(vec!["127.0.0.1".to_string()]).map_err(io::Error::other)?;
let certificate = cert.der().clone();
let private_key = PrivateKeyDer::from(signing_key);
let tls_config = Arc::new(
rustls::ServerConfig::builder()
.with_no_client_auth()
.with_single_cert(vec![certificate.clone()], private_key)
.map_err(io::Error::other)?,
);
let trusted_rustls_client = HttpClient::new(
reqwest::Client::builder()
.use_rustls_tls()
.add_root_certificate(
reqwest::Certificate::from_der(certificate.as_ref()).map_err(io::Error::other)?,
)
.no_proxy()
.redirect(reqwest::redirect::Policy::none())
.build()
.map_err(io::Error::other)?,
);
let listener = TcpListener::bind(("127.0.0.1", 0))?;
let address = listener.local_addr()?;
listener.set_nonblocking(true)?;
let (requests_tx, requests_rx) = mpsc::channel();
thread::spawn(move || {
let result = (|| -> io::Result<Vec<String>> {
let mut observed_requests = Vec::new();
for connection_index in 0..3 {
let deadline = Instant::now() + Duration::from_secs(5);
let mut stream = loop {
match listener.accept() {
Ok((stream, _)) => break stream,
Err(error) if error.kind() == io::ErrorKind::WouldBlock => {
if Instant::now() >= deadline {
return Err(io::Error::new(
io::ErrorKind::TimedOut,
"timed out waiting for the TLS fallback client",
));
}
thread::sleep(Duration::from_millis(10));
}
Err(error) => return Err(error),
}
};
stream.set_nonblocking(false)?;
stream.set_read_timeout(Some(Duration::from_secs(5)))?;
stream.set_write_timeout(Some(Duration::from_secs(5)))?;
if connection_index == 0 {
let mut client_hello = [0_u8; 2_048];
if stream.read(&mut client_hello)? == 0 {
return Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"native TLS peer closed before sending a ClientHello",
));
}
stream.write_all(PROTOCOL_VERSION_TLS_ALERT)?;
stream.flush()?;
continue;
}
let connection = rustls::ServerConnection::new(Arc::clone(&tls_config))
.map_err(io::Error::other)?;
let mut tls = rustls::StreamOwned::new(connection, stream);
let mut request = Vec::new();
let mut chunk = [0_u8; 1_024];
let header_end = loop {
let bytes_read = tls.read(&mut chunk)?;
if bytes_read == 0 {
return Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"rustls peer closed before sending HTTP headers",
));
}
request.extend_from_slice(&chunk[..bytes_read]);
if let Some(header_end) =
request.windows(4).position(|window| window == b"\r\n\r\n")
{
break header_end + 4;
}
};
let headers =
std::str::from_utf8(&request[..header_end]).map_err(io::Error::other)?;
let content_length = headers
.lines()
.find_map(|line| {
let (name, value) = line.split_once(':')?;
name.eq_ignore_ascii_case("content-length")
.then_some(value.trim())
})
.ok_or_else(|| {
io::Error::new(io::ErrorKind::InvalidData, "missing Content-Length")
})?
.parse::<usize>()
.map_err(io::Error::other)?;
while request.len() < header_end + content_length {
let bytes_read = tls.read(&mut chunk)?;
if bytes_read == 0 {
return Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"rustls peer closed before sending the HTTP body",
));
}
request.extend_from_slice(&chunk[..bytes_read]);
}
let body = std::str::from_utf8(&request[header_end..header_end + content_length])
.map_err(io::Error::other)?;
observed_requests.push(body.to_string());
tls.write_all(
b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok",
)?;
tls.flush()?;
}
Ok(observed_requests)
})();
let _ = requests_tx.send(result);
});
Ok((
format!("https://{address}/mcp"),
trusted_rustls_client,
requests_rx,
))
}
fn spawn_protocol_version_rejection_server(
maximum_attempts: usize,
) -> io::Result<(String, mpsc::Receiver<io::Result<usize>>, mpsc::Sender<()>)> {
let listener = TcpListener::bind(("127.0.0.1", 0))?;
let address = listener.local_addr()?;
listener.set_nonblocking(true)?;
let (attempts_tx, attempts_rx) = mpsc::channel();
let (stop_tx, stop_rx) = mpsc::channel();
thread::spawn(move || {
let result = (|| -> io::Result<usize> {
let mut attempts = 0;
let deadline = Instant::now() + Duration::from_secs(5);
while attempts < maximum_attempts && Instant::now() < deadline {
if stop_rx.try_recv().is_ok() {
break;
}
match listener.accept() {
Ok((mut stream, _)) => {
stream.set_nonblocking(false)?;
stream.set_read_timeout(Some(Duration::from_secs(2)))?;
let mut client_hello = [0_u8; 2_048];
if stream.read(&mut client_hello)? == 0 {
return Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"TLS peer closed before sending a ClientHello",
));
}
stream.write_all(PROTOCOL_VERSION_TLS_ALERT)?;
attempts += 1;
}
Err(error) if error.kind() == io::ErrorKind::WouldBlock => {
thread::sleep(Duration::from_millis(10));
}
Err(error) => return Err(error),
}
}
Ok(attempts)
})();
let _ = attempts_tx.send(result);
});
Ok((format!("https://{address}/mcp"), attempts_rx, stop_tx))
}

View File

@@ -0,0 +1,156 @@
//! Narrow TLS backend fallback for delegated requests that select their route per destination.
//!
//! Native TLS remains the default. A recognized connection-time protocol negotiation failure can
//! select rustls for one HTTPS origin and outbound route without changing other destinations.
use std::collections::HashMap;
use std::collections::HashSet;
use std::error::Error;
use std::sync::Arc;
use std::sync::Mutex;
use crate::HttpClient;
use crate::OutboundProxyRoute;
const MAX_CACHED_RUSTLS_DESTINATIONS: usize = 16;
// Schannel maps TLS alert 70 (protocol_version) to SEC_E_UNSUPPORTED_FUNCTION.
const SCHANNEL_PROTOCOL_VERSION_ERROR: i32 = 0x8009_0302_u32 as i32;
#[derive(Clone, Default)]
pub(crate) struct RustlsClientCache {
state: Arc<Mutex<RustlsClientCacheState>>,
}
#[derive(Default)]
struct RustlsClientCacheState {
destinations: HashSet<DestinationRoute>,
clients: HashMap<OutboundProxyRoute, HttpClient>,
}
#[derive(Clone, Hash, PartialEq, Eq)]
struct DestinationRoute {
host: String,
port: u16,
route: OutboundProxyRoute,
}
impl RustlsClientCache {
pub(crate) fn requires_rustls(&self, url: &reqwest::Url, route: &OutboundProxyRoute) -> bool {
let Some(destination) = DestinationRoute::new(url, route) else {
return false;
};
self.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.destinations
.contains(&destination)
}
pub(crate) fn client_for_route(&self, route: &OutboundProxyRoute) -> Option<HttpClient> {
self.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clients
.get(route)
.cloned()
}
pub(crate) fn remember(
&self,
url: &reqwest::Url,
route: &OutboundProxyRoute,
client: HttpClient,
) {
let Some(destination) = DestinationRoute::new(url, route) else {
return;
};
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if state.destinations.contains(&destination) {
return;
}
if state.destinations.len() >= MAX_CACHED_RUSTLS_DESTINATIONS
&& let Some(destination_to_evict) = state.destinations.iter().next().cloned()
{
state.destinations.remove(&destination_to_evict);
if !state
.destinations
.iter()
.any(|destination| destination.route == destination_to_evict.route)
{
state.clients.remove(&destination_to_evict.route);
}
}
state.clients.entry(route.clone()).or_insert(client);
state.destinations.insert(destination);
}
}
impl DestinationRoute {
fn new(url: &reqwest::Url, route: &OutboundProxyRoute) -> Option<Self> {
if url.scheme() != "https" {
return None;
}
Some(Self {
host: url.host_str()?.to_ascii_lowercase(),
port: url.port_or_known_default()?,
route: route.clone(),
})
}
}
pub(crate) fn should_retry_with_rustls(error: &reqwest::Error) -> bool {
error.is_connect() && !error.is_timeout() && error.source().is_some_and(has_retryable_tls_error)
}
fn has_retryable_tls_error(error: &(dyn Error + 'static)) -> bool {
let mut source = Some(error);
let mut recognized_negotiation_failure = false;
while let Some(error) = source {
let message = error.to_string().to_ascii_lowercase();
if [
"certificate",
"unknown issuer",
"unknown ca",
"untrusted",
"self signed",
"self-signed",
"hostname",
"expired",
"revoked",
]
.iter()
.any(|marker| message.contains(marker))
{
return false;
}
// macOS Secure Transport reports the protocol alert as "bad protocol version".
let is_macos_protocol_version_error = message.contains("bad protocol version");
// Linux OpenSSL reports the peer's "tlsv1 alert protocol version".
let is_linux_protocol_version_error = message.contains("tlsv1 alert protocol version");
// Windows Schannel may expose the protocol alert as a raw or formatted OS error.
let is_schannel_protocol_version_error = error
.downcast_ref::<std::io::Error>()
.and_then(std::io::Error::raw_os_error)
== Some(SCHANNEL_PROTOCOL_VERSION_ERROR)
|| message.contains("(os error -2146893054)")
|| message.contains("0x80090302");
if is_macos_protocol_version_error
|| is_linux_protocol_version_error
|| is_schannel_protocol_version_error
{
recognized_negotiation_failure = true;
}
source = error.source();
}
recognized_negotiation_failure
}
#[cfg(test)]
#[path = "tls_backend_fallback_tests.rs"]
mod tests;

View File

@@ -0,0 +1,216 @@
use std::io;
use pretty_assertions::assert_eq;
use super::MAX_CACHED_RUSTLS_DESTINATIONS;
use super::RustlsClientCache;
use super::SCHANNEL_PROTOCOL_VERSION_ERROR;
use super::has_retryable_tls_error;
use crate::HttpClientBuilder;
use crate::OutboundProxyRoute;
#[test]
fn recognizes_platform_specific_tls_protocol_negotiation_failures() {
let errors = [
("client error (Connect): bad protocol version", true),
("BAD PROTOCOL VERSION", true),
(
"error:0A00042E:SSL routines:ssl3_read_bytes:tlsv1 alert protocol version",
true,
),
("TLSV1 ALERT PROTOCOL VERSION", true),
(
"The function requested is not supported. (os error -2146893054)",
true,
),
("Schannel protocol error 0x80090302", true),
("SCHANNEL PROTOCOL ERROR 0X80090302", true),
("certificate validation failed: bad protocol version", false),
("bad protocol version: certificate has expired", false),
(
"certificate validation failed: tlsv1 alert protocol version",
false,
),
(
"tlsv1 alert protocol version: certificate has expired",
false,
),
(
"certificate validation failed (os error -2146893054)",
false,
),
("certificate validation failed: 0x80090302", false),
("unknown issuer", false),
("self-signed certificate", false),
("hostname mismatch", false),
("connection refused", false),
("connection reset", false),
("dns lookup failed", false),
("tls handshake failed", false),
("unsupported protocol", false),
("wrong version number", false),
("The function requested is not supported.", false),
(
"The client and server cannot communicate. (os error -2146893007)",
false,
),
("operation timed out", false),
("407 Proxy Authentication Required", false),
];
for (message, expected) in errors {
let error = io::Error::other(message);
assert_eq!(has_retryable_tls_error(&error), expected, "{message}");
}
}
#[test]
fn recognizes_schannel_protocol_version_error_codes() {
let protocol_error = io::Error::from_raw_os_error(SCHANNEL_PROTOCOL_VERSION_ERROR);
let another_schannel_error = io::Error::from_raw_os_error(/*code*/ -2_146_893_007);
assert_eq!(
(
has_retryable_tls_error(&protocol_error),
has_retryable_tls_error(&another_schannel_error),
),
(true, false)
);
}
#[test]
fn certificate_errors_in_an_error_source_never_enable_fallback() {
for message in [
"certificate verification failed: bad protocol version",
"certificate verification failed: tlsv1 alert protocol version",
"certificate verification failed (os error -2146893054)",
"certificate verification failed: 0x80090302",
] {
let error = io::Error::other(io::Error::other(message));
assert!(!has_retryable_tls_error(&error), "{message}");
}
}
#[test]
fn rustls_fallback_decisions_are_scoped_to_origin_and_outbound_route() {
let cache = RustlsClientCache::default();
let destination = reqwest::Url::parse("https://mcp.example.com/first").expect("valid URL");
let same_origin = reqwest::Url::parse("https://mcp.example.com/second").expect("valid URL");
let another_host = reqwest::Url::parse("https://another.example.com/first").expect("valid URL");
let another_port =
reqwest::Url::parse("https://mcp.example.com:8443/first").expect("valid URL");
let insecure = reqwest::Url::parse("http://mcp.example.com/first").expect("valid URL");
let direct = OutboundProxyRoute::Direct;
let proxy = OutboundProxyRoute::Proxy {
url: "http://proxy.example.com".to_string(),
no_proxy: None,
};
let client = HttpClientBuilder::new()
.with_rustls_tls()
.build_direct()
.expect("rustls client should build without proxy autodiscovery");
cache.remember(&destination, &direct, client);
assert_eq!(
[
cache.requires_rustls(&destination, &direct),
cache.requires_rustls(&same_origin, &direct),
cache.requires_rustls(&another_host, &direct),
cache.requires_rustls(&another_port, &direct),
cache.requires_rustls(&destination, &proxy),
cache.requires_rustls(&insecure, &direct),
],
[true, true, false, false, false, false]
);
}
#[test]
fn cached_rustls_clients_are_reused_for_the_same_outbound_route() {
let cache = RustlsClientCache::default();
let first_destination =
reqwest::Url::parse("https://first.example.com").expect("valid first URL");
let second_destination =
reqwest::Url::parse("https://second.example.com").expect("valid second URL");
let direct = OutboundProxyRoute::Direct;
let client = HttpClientBuilder::new()
.with_rustls_tls()
.build_direct()
.expect("rustls client should build without proxy autodiscovery");
cache.remember(&first_destination, &direct, client.clone());
cache.remember(&second_destination, &direct, client);
assert_eq!(
(
cache.requires_rustls(&first_destination, &direct),
cache.requires_rustls(&second_destination, &direct),
cache.client_for_route(&direct).is_some(),
cache
.state
.lock()
.expect("rustls client cache lock")
.clients
.len(),
),
(true, true, true, 1)
);
}
#[test]
fn cached_rustls_destinations_remain_bounded_while_sharing_a_route_client() {
let cache = RustlsClientCache::default();
let client = HttpClientBuilder::new()
.with_rustls_tls()
.build_direct()
.expect("rustls client should build without proxy autodiscovery");
for index in 0..=MAX_CACHED_RUSTLS_DESTINATIONS {
let destination =
reqwest::Url::parse(&format!("https://mcp-{index}.example.com")).expect("valid URL");
cache.remember(&destination, &OutboundProxyRoute::Direct, client.clone());
}
let state = cache.state.lock().expect("rustls client cache lock");
assert_eq!(
(state.destinations.len(), state.clients.len()),
(MAX_CACHED_RUSTLS_DESTINATIONS, 1)
);
}
#[test]
fn evicting_a_destination_removes_its_unshared_route_client() {
let cache = RustlsClientCache::default();
let client = HttpClientBuilder::new()
.with_rustls_tls()
.build_direct()
.expect("rustls client should build without proxy autodiscovery");
for index in 0..=MAX_CACHED_RUSTLS_DESTINATIONS {
let destination =
reqwest::Url::parse(&format!("https://mcp-{index}.example.com")).expect("valid URL");
let route = OutboundProxyRoute::Proxy {
url: format!("http://proxy-{index}.example.com"),
no_proxy: None,
};
cache.remember(&destination, &route, client.clone());
}
let state = cache.state.lock().expect("rustls client cache lock");
assert_eq!(
(
state.destinations.len(),
state.clients.len(),
state
.destinations
.iter()
.all(|destination| state.clients.contains_key(&destination.route)),
),
(
MAX_CACHED_RUSTLS_DESTINATIONS,
MAX_CACHED_RUSTLS_DESTINATIONS,
true,
)
);
}