Honor network policy in remote control and recover remote execution (#47410)

## What changed

- Route remote-control enrollment, pairing, client management, and WebSocket connections through the authenticated HTTP client factory. Keep policy denials distinct from authentication failures and wait for policy changes before reconnecting.
- Preserve policy denials through exec-server connection failures so later connection attempts and capability discovery can recover. Keep direct executors running through temporary policy outages while rejecting permanent policy failures.
- Build cached HTTP transports on blocking workers, serialize construction, and retain successful builds after callers time out so request deadlines cover queued construction.

## Testing

Add regression coverage for policy recovery during remote startup, direct registration, accepted WebSocket reconnection, and capability discovery; queued transport construction timeouts; and remote-control WebSocket fallback with invalid custom CA files.

GitOrigin-RevId: c81db18e83de957175bb213d165281ac701089dd
This commit is contained in:
acrognale-oai
2026-09-23 00:48:53 +00:00
committed by copyberry
parent 8f8ace78cb
commit db3cb33eca
27 changed files with 833 additions and 218 deletions
@@ -57,6 +57,7 @@ pub struct RouteAwareClientPool {
default_headers: HeaderMap,
custom_ca_fallback: CustomCaFallback,
clients: Arc<Mutex<HashMap<OutboundProxyRoute, TransportClient>>>,
client_build: Arc<tokio::sync::Mutex<()>>,
rustls_clients: Option<RustlsClientCache>,
}
@@ -77,6 +78,8 @@ pub enum RouteAwareClientPoolError {
Resolve(#[source] io::Error),
#[error(transparent)]
Build(#[from] BuildRouteAwareHttpClientError),
#[error("HTTP transport construction task failed: {0}")]
BuildTask(#[source] tokio::task::JoinError),
}
/// Error returned while building, routing, or sending a route-aware request.
@@ -138,7 +141,8 @@ impl RouteAwareRequestError {
Self::Route(RouteAwareClientPoolError::Resolve(_)) => {
Some(RouteFailureClass::ProxyResolutionUnavailable)
}
Self::Request(_)
Self::Route(RouteAwareClientPoolError::BuildTask(_))
| Self::Request(_)
| Self::Policy(_)
| Self::Build(_)
| Self::UnsupportedRedirectScheme(_)
@@ -286,6 +290,7 @@ impl RouteAwareClientPool {
default_headers,
custom_ca_fallback: CustomCaFallback::Disabled,
clients: Arc::new(Mutex::new(HashMap::new())),
client_build: Arc::default(),
rustls_clients: None,
}
}
@@ -469,57 +474,74 @@ impl RouteAwareClientPool {
{
return Ok((route, client, SelectedTlsBackend::RustlsFallback));
}
let clients = match self.clients.lock() {
Ok(clients) => clients,
Err(error) => panic!("route-aware client cache lock should not be poisoned: {error}"),
};
if let Some(client) = clients.get(&route) {
return Ok((route, client.clone(), SelectedTlsBackend::TransportDefault));
{
let clients = match self.clients.lock() {
Ok(clients) => clients,
Err(error) => {
panic!("route-aware client cache lock should not be poisoned: {error}")
}
};
if let Some(client) = clients.get(&route) {
return Ok((route, client.clone(), SelectedTlsBackend::TransportDefault));
}
}
drop(clients);
let build_permit = Arc::clone(&self.client_build).lock_owned().await;
{
let clients = self.clients.lock().unwrap_or_else(|error| {
panic!("route-aware client cache lock should not be poisoned: {error}")
});
if let Some(client) = clients.get(&route) {
return Ok((route, client.clone(), SelectedTlsBackend::TransportDefault));
}
}
let client_builder = if self.follows_redirects_manually() {
self.client_builder.clone().without_redirects()
} else {
self.client_builder.clone()
};
let client = match (
self.http_client_factory.outbound_proxy_policy(),
self.custom_ca_fallback,
) {
(_, CustomCaFallback::LegacyDirect) => {
client_builder.build_with_custom_ca_fallback(ProxyRouting::Direct)
let pool = self.clone();
let build_route = route.clone();
let client = tokio::task::spawn_blocking(move || {
// A timed-out caller must not release the slot or discard a successful build.
let _build_permit = build_permit;
let client = match (
pool.http_client_factory.outbound_proxy_policy(),
pool.custom_ca_fallback,
) {
(_, CustomCaFallback::LegacyDirect) => {
Ok(client_builder.build_with_custom_ca_fallback(ProxyRouting::Direct))
}
(OutboundProxyPolicy::ReqwestDefault, CustomCaFallback::LegacyTransportDefault) => {
Ok(
client_builder
.build_with_custom_ca_fallback(ProxyRouting::TransportDefault),
)
}
(OutboundProxyPolicy::ReqwestDefault, CustomCaFallback::Disabled)
| (OutboundProxyPolicy::RespectSystemProxy, CustomCaFallback::Disabled)
| (
OutboundProxyPolicy::RespectSystemProxy,
CustomCaFallback::LegacyTransportDefault,
) => client_builder.build_for_resolved_route(
&pool.http_client_factory,
pool.route_class,
&build_route,
),
}?;
let mut clients = pool.clients.lock().unwrap_or_else(|error| {
panic!("route-aware client cache lock should not be poisoned: {error}")
});
if clients.len() >= MAX_CACHED_ROUTES
&& let Some(route_to_evict) = clients.keys().next().cloned()
{
clients.remove(&route_to_evict);
}
(OutboundProxyPolicy::ReqwestDefault, CustomCaFallback::LegacyTransportDefault) => {
client_builder.build_with_custom_ca_fallback(ProxyRouting::TransportDefault)
}
(OutboundProxyPolicy::ReqwestDefault, CustomCaFallback::Disabled)
| (OutboundProxyPolicy::RespectSystemProxy, CustomCaFallback::Disabled)
| (OutboundProxyPolicy::RespectSystemProxy, CustomCaFallback::LegacyTransportDefault) => {
client_builder.build_for_resolved_route(
&self.http_client_factory,
self.route_class,
&route,
)?
}
};
let mut clients = match self.clients.lock() {
Ok(clients) => clients,
Err(error) => panic!("route-aware client cache lock should not be poisoned: {error}"),
};
if let Some(existing_client) = clients.get(&route) {
return Ok((
route,
existing_client.clone(),
SelectedTlsBackend::TransportDefault,
));
}
if clients.len() >= MAX_CACHED_ROUTES
&& let Some(route_to_evict) = clients.keys().next().cloned()
{
clients.remove(&route_to_evict);
}
clients.insert(route.clone(), client.clone());
clients.insert(build_route, client.clone());
Ok::<_, BuildRouteAwareHttpClientError>(client)
})
.await
.map_err(RouteAwareClientPoolError::BuildTask)??;
Ok((route, client, SelectedTlsBackend::TransportDefault))
}
@@ -12,6 +12,7 @@ use std::time::Duration;
use std::time::Instant;
use bytes::Bytes;
use futures::FutureExt;
use futures::stream;
use http::HeaderValue;
use pretty_assertions::assert_eq;
@@ -710,6 +711,75 @@ async fn request_timeout_covers_route_selection() {
assert_eq!(resolver_calls.load(Ordering::SeqCst), 1);
}
#[test]
fn managed_request_timeout_covers_queued_transport_construction() {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.max_blocking_threads(/*val*/ 1)
.build()
.unwrap();
runtime.block_on(async {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
listener.set_nonblocking(/*nonblocking*/ true).unwrap();
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
let blocker = tokio::task::spawn_blocking(move || {
started_tx.send(()).unwrap();
let _ = release_rx.recv();
});
started_rx.await.unwrap();
let controller = crate::NetworkPolicyController::default();
let policy = controller.policy();
controller.publish(policy.revision(), crate::DestinationPolicy::Unrestricted);
let pool = RouteAwareClientPool::new(
HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault).with_network_policy(policy),
ClientRouteClass::Api,
);
let url = format!("http://{}/", listener.local_addr().unwrap());
let requests = (0..8).map(|_| {
pool.get(&url)
.timeout(Duration::from_millis(/*millis*/ 200))
.send()
});
for result in futures::future::join_all(requests).await {
assert!(result.unwrap_err().is_timeout());
}
assert!(pool.client_build.try_lock().is_err());
assert!(pool.clients.lock().unwrap().is_empty());
assert_eq!(
listener.accept().unwrap_err().kind(),
io::ErrorKind::WouldBlock
);
drop(release_tx);
blocker.await.unwrap();
let build_finished =
tokio::time::timeout(Duration::from_secs(/*secs*/ 5), pool.client_build.lock())
.await
.unwrap();
assert!(
pool.clients
.lock()
.unwrap()
.contains_key(&OutboundProxyRoute::TransportDefault)
);
let (_, _, backend) = pool
.client_for_url_with_resolver(&url, |_| async {
Ok(OutboundProxyRoute::TransportDefault)
})
.now_or_never()
.unwrap()
.unwrap();
assert_eq!(backend, SelectedTlsBackend::TransportDefault);
drop(build_finished);
assert_eq!(
listener.accept().unwrap_err().kind(),
io::ErrorKind::WouldBlock
);
});
}
#[tokio::test]
async fn request_timeout_is_shared_across_redirect_hops() {
let (address, server) = spawn_response_server(vec![