mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
Honor network policy in remote control and recover remote execution (#47410)
## What changed - Route remote-control enrollment, pairing, client management, and WebSocket connections through the authenticated HTTP client factory. Keep policy denials distinct from authentication failures and wait for policy changes before reconnecting. - Preserve policy denials through exec-server connection failures so later connection attempts and capability discovery can recover. Keep direct executors running through temporary policy outages while rejecting permanent policy failures. - Build cached HTTP transports on blocking workers, serialize construction, and retain successful builds after callers time out so request deadlines cover queued construction. ## Testing Add regression coverage for policy recovery during remote startup, direct registration, accepted WebSocket reconnection, and capability discovery; queued transport construction timeouts; and remote-control WebSocket fallback with invalid custom CA files. GitOrigin-RevId: c81db18e83de957175bb213d165281ac701089dd
This commit is contained in:
@@ -57,6 +57,7 @@ pub struct RouteAwareClientPool {
|
||||
default_headers: HeaderMap,
|
||||
custom_ca_fallback: CustomCaFallback,
|
||||
clients: Arc<Mutex<HashMap<OutboundProxyRoute, TransportClient>>>,
|
||||
client_build: Arc<tokio::sync::Mutex<()>>,
|
||||
rustls_clients: Option<RustlsClientCache>,
|
||||
}
|
||||
|
||||
@@ -77,6 +78,8 @@ pub enum RouteAwareClientPoolError {
|
||||
Resolve(#[source] io::Error),
|
||||
#[error(transparent)]
|
||||
Build(#[from] BuildRouteAwareHttpClientError),
|
||||
#[error("HTTP transport construction task failed: {0}")]
|
||||
BuildTask(#[source] tokio::task::JoinError),
|
||||
}
|
||||
|
||||
/// Error returned while building, routing, or sending a route-aware request.
|
||||
@@ -138,7 +141,8 @@ impl RouteAwareRequestError {
|
||||
Self::Route(RouteAwareClientPoolError::Resolve(_)) => {
|
||||
Some(RouteFailureClass::ProxyResolutionUnavailable)
|
||||
}
|
||||
Self::Request(_)
|
||||
Self::Route(RouteAwareClientPoolError::BuildTask(_))
|
||||
| Self::Request(_)
|
||||
| Self::Policy(_)
|
||||
| Self::Build(_)
|
||||
| Self::UnsupportedRedirectScheme(_)
|
||||
@@ -286,6 +290,7 @@ impl RouteAwareClientPool {
|
||||
default_headers,
|
||||
custom_ca_fallback: CustomCaFallback::Disabled,
|
||||
clients: Arc::new(Mutex::new(HashMap::new())),
|
||||
client_build: Arc::default(),
|
||||
rustls_clients: None,
|
||||
}
|
||||
}
|
||||
@@ -469,57 +474,74 @@ impl RouteAwareClientPool {
|
||||
{
|
||||
return Ok((route, client, SelectedTlsBackend::RustlsFallback));
|
||||
}
|
||||
let clients = match self.clients.lock() {
|
||||
Ok(clients) => clients,
|
||||
Err(error) => panic!("route-aware client cache lock should not be poisoned: {error}"),
|
||||
};
|
||||
if let Some(client) = clients.get(&route) {
|
||||
return Ok((route, client.clone(), SelectedTlsBackend::TransportDefault));
|
||||
{
|
||||
let clients = match self.clients.lock() {
|
||||
Ok(clients) => clients,
|
||||
Err(error) => {
|
||||
panic!("route-aware client cache lock should not be poisoned: {error}")
|
||||
}
|
||||
};
|
||||
if let Some(client) = clients.get(&route) {
|
||||
return Ok((route, client.clone(), SelectedTlsBackend::TransportDefault));
|
||||
}
|
||||
}
|
||||
drop(clients);
|
||||
|
||||
let build_permit = Arc::clone(&self.client_build).lock_owned().await;
|
||||
{
|
||||
let clients = self.clients.lock().unwrap_or_else(|error| {
|
||||
panic!("route-aware client cache lock should not be poisoned: {error}")
|
||||
});
|
||||
if let Some(client) = clients.get(&route) {
|
||||
return Ok((route, client.clone(), SelectedTlsBackend::TransportDefault));
|
||||
}
|
||||
}
|
||||
let client_builder = if self.follows_redirects_manually() {
|
||||
self.client_builder.clone().without_redirects()
|
||||
} else {
|
||||
self.client_builder.clone()
|
||||
};
|
||||
let client = match (
|
||||
self.http_client_factory.outbound_proxy_policy(),
|
||||
self.custom_ca_fallback,
|
||||
) {
|
||||
(_, CustomCaFallback::LegacyDirect) => {
|
||||
client_builder.build_with_custom_ca_fallback(ProxyRouting::Direct)
|
||||
let pool = self.clone();
|
||||
let build_route = route.clone();
|
||||
let client = tokio::task::spawn_blocking(move || {
|
||||
// A timed-out caller must not release the slot or discard a successful build.
|
||||
let _build_permit = build_permit;
|
||||
let client = match (
|
||||
pool.http_client_factory.outbound_proxy_policy(),
|
||||
pool.custom_ca_fallback,
|
||||
) {
|
||||
(_, CustomCaFallback::LegacyDirect) => {
|
||||
Ok(client_builder.build_with_custom_ca_fallback(ProxyRouting::Direct))
|
||||
}
|
||||
(OutboundProxyPolicy::ReqwestDefault, CustomCaFallback::LegacyTransportDefault) => {
|
||||
Ok(
|
||||
client_builder
|
||||
.build_with_custom_ca_fallback(ProxyRouting::TransportDefault),
|
||||
)
|
||||
}
|
||||
(OutboundProxyPolicy::ReqwestDefault, CustomCaFallback::Disabled)
|
||||
| (OutboundProxyPolicy::RespectSystemProxy, CustomCaFallback::Disabled)
|
||||
| (
|
||||
OutboundProxyPolicy::RespectSystemProxy,
|
||||
CustomCaFallback::LegacyTransportDefault,
|
||||
) => client_builder.build_for_resolved_route(
|
||||
&pool.http_client_factory,
|
||||
pool.route_class,
|
||||
&build_route,
|
||||
),
|
||||
}?;
|
||||
let mut clients = pool.clients.lock().unwrap_or_else(|error| {
|
||||
panic!("route-aware client cache lock should not be poisoned: {error}")
|
||||
});
|
||||
if clients.len() >= MAX_CACHED_ROUTES
|
||||
&& let Some(route_to_evict) = clients.keys().next().cloned()
|
||||
{
|
||||
clients.remove(&route_to_evict);
|
||||
}
|
||||
(OutboundProxyPolicy::ReqwestDefault, CustomCaFallback::LegacyTransportDefault) => {
|
||||
client_builder.build_with_custom_ca_fallback(ProxyRouting::TransportDefault)
|
||||
}
|
||||
(OutboundProxyPolicy::ReqwestDefault, CustomCaFallback::Disabled)
|
||||
| (OutboundProxyPolicy::RespectSystemProxy, CustomCaFallback::Disabled)
|
||||
| (OutboundProxyPolicy::RespectSystemProxy, CustomCaFallback::LegacyTransportDefault) => {
|
||||
client_builder.build_for_resolved_route(
|
||||
&self.http_client_factory,
|
||||
self.route_class,
|
||||
&route,
|
||||
)?
|
||||
}
|
||||
};
|
||||
let mut clients = match self.clients.lock() {
|
||||
Ok(clients) => clients,
|
||||
Err(error) => panic!("route-aware client cache lock should not be poisoned: {error}"),
|
||||
};
|
||||
if let Some(existing_client) = clients.get(&route) {
|
||||
return Ok((
|
||||
route,
|
||||
existing_client.clone(),
|
||||
SelectedTlsBackend::TransportDefault,
|
||||
));
|
||||
}
|
||||
if clients.len() >= MAX_CACHED_ROUTES
|
||||
&& let Some(route_to_evict) = clients.keys().next().cloned()
|
||||
{
|
||||
clients.remove(&route_to_evict);
|
||||
}
|
||||
clients.insert(route.clone(), client.clone());
|
||||
clients.insert(build_route, client.clone());
|
||||
Ok::<_, BuildRouteAwareHttpClientError>(client)
|
||||
})
|
||||
.await
|
||||
.map_err(RouteAwareClientPoolError::BuildTask)??;
|
||||
Ok((route, client, SelectedTlsBackend::TransportDefault))
|
||||
}
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ use std::time::Duration;
|
||||
use std::time::Instant;
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures::FutureExt;
|
||||
use futures::stream;
|
||||
use http::HeaderValue;
|
||||
use pretty_assertions::assert_eq;
|
||||
@@ -710,6 +711,75 @@ async fn request_timeout_covers_route_selection() {
|
||||
assert_eq!(resolver_calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn managed_request_timeout_covers_queued_transport_construction() {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.max_blocking_threads(/*val*/ 1)
|
||||
.build()
|
||||
.unwrap();
|
||||
runtime.block_on(async {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
listener.set_nonblocking(/*nonblocking*/ true).unwrap();
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let blocker = tokio::task::spawn_blocking(move || {
|
||||
started_tx.send(()).unwrap();
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
started_rx.await.unwrap();
|
||||
|
||||
let controller = crate::NetworkPolicyController::default();
|
||||
let policy = controller.policy();
|
||||
controller.publish(policy.revision(), crate::DestinationPolicy::Unrestricted);
|
||||
let pool = RouteAwareClientPool::new(
|
||||
HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault).with_network_policy(policy),
|
||||
ClientRouteClass::Api,
|
||||
);
|
||||
let url = format!("http://{}/", listener.local_addr().unwrap());
|
||||
let requests = (0..8).map(|_| {
|
||||
pool.get(&url)
|
||||
.timeout(Duration::from_millis(/*millis*/ 200))
|
||||
.send()
|
||||
});
|
||||
for result in futures::future::join_all(requests).await {
|
||||
assert!(result.unwrap_err().is_timeout());
|
||||
}
|
||||
assert!(pool.client_build.try_lock().is_err());
|
||||
assert!(pool.clients.lock().unwrap().is_empty());
|
||||
assert_eq!(
|
||||
listener.accept().unwrap_err().kind(),
|
||||
io::ErrorKind::WouldBlock
|
||||
);
|
||||
drop(release_tx);
|
||||
blocker.await.unwrap();
|
||||
|
||||
let build_finished =
|
||||
tokio::time::timeout(Duration::from_secs(/*secs*/ 5), pool.client_build.lock())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
pool.clients
|
||||
.lock()
|
||||
.unwrap()
|
||||
.contains_key(&OutboundProxyRoute::TransportDefault)
|
||||
);
|
||||
let (_, _, backend) = pool
|
||||
.client_for_url_with_resolver(&url, |_| async {
|
||||
Ok(OutboundProxyRoute::TransportDefault)
|
||||
})
|
||||
.now_or_never()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(backend, SelectedTlsBackend::TransportDefault);
|
||||
drop(build_finished);
|
||||
assert_eq!(
|
||||
listener.accept().unwrap_err().kind(),
|
||||
io::ErrorKind::WouldBlock
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn request_timeout_is_shared_across_redirect_hops() {
|
||||
let (address, server) = spawn_response_server(vec![
|
||||
|
||||
Reference in New Issue
Block a user