Add opt-in tool-call attribution for upstream proxies (#61860)

## What changed

Allow executors to set `CODEX_TOOL_CALL_ATTRIBUTION_PROXY` to designate an upstream proxy that consumes and strips `x-openai-internal-tool-call-id`. When the selected upstream matches that endpoint, attach the controller's tool-call ID to plaintext HTTP requests or the outer HTTPS `CONNECT` request.

Strip caller-supplied attribution headers and send no attribution to direct destinations or other upstream proxies. Leave attribution disabled when the setting is unset.

## Testing

Add request-capture tests for HTTP and HTTPS attribution, execution-specific IDs, header stripping, split upstream selection with nonstandard origin ports, and inbound `CONNECT` tunneling.

GitOrigin-RevId: f908faf55a991bc11f2f9c7fd3100f42b4f638b9
This commit is contained in:
Hendrik Buyle
2026-11-16 19:13:33 +00:00
committed by copyberry
parent 3257e1b3eb
commit 457a191479
4 changed files with 352 additions and 14 deletions
+118 -4
View File
@@ -36,6 +36,7 @@ use crate::state::BlockedRequestArgs;
use crate::state::NetworkProxyState;
use crate::upstream::UpstreamClient;
use crate::upstream::proxy_for_connect;
use crate::upstream::upstream_proxy_connector;
use anyhow::Context as _;
use anyhow::Result;
use codex_utils_rustls_provider::ensure_rustls_crypto_provider;
@@ -61,7 +62,6 @@ use rama_http::headers::HeaderMapExt;
use rama_http::headers::Host;
use rama_http::layer::remove_header::RemoveResponseHeaderLayer;
use rama_http::matcher::MethodMatcher;
use rama_http_backend::client::proxy::layer::HttpProxyConnector;
use rama_http_backend::server::HttpServer;
use rama_http_backend::server::layer::upgrade::UpgradeLayer;
use rama_http_backend::server::layer::upgrade::Upgraded;
@@ -491,10 +491,10 @@ where
false
}
};
let proxy = if allow_upstream_proxy {
let (proxy, connect_call_id) = if allow_upstream_proxy {
proxy_for_connect(&authority, &app_state)
} else {
None
(None, None)
};
match proxy.as_ref() {
Some(proxy) => info!(
@@ -514,7 +514,8 @@ where
let req = TcpRequest::new_with_extensions(authority.clone(), extensions)
.with_protocol(Protocol::HTTPS);
let proxy_connector = HttpProxyConnector::optional(TargetCheckedTcpConnector::new(app_state));
let proxy_connector =
upstream_proxy_connector(TargetCheckedTcpConnector::new(app_state), connect_call_id);
let tls_config = TlsConnectorDataBuilder::new()
.with_alpn_protocols_http_auto()
.build();
@@ -1833,6 +1834,119 @@ mod tests {
target_task.await.expect("target task should finish");
}
#[tokio::test]
async fn inbound_connect_attributes_the_selected_upstream() {
use tokio::io::AsyncBufReadExt;
use tokio::io::BufReader;
use tokio::process::Command;
const CHILD_ENV: &str = "CODEX_TEST_INBOUND_CONNECT_ATTRIBUTION";
const TARGET: &str = "same.example:443";
const HEADER: &str = "x-openai-internal-tool-call-id";
const CALL_ID: &str = "controller-call";
const TUNNEL_BYTES: &[u8] = b"tunnel ready";
async fn read_http_head(stream: &mut BufReader<tokio::net::TcpStream>) -> String {
let mut head = String::new();
loop {
let previous_len = head.len();
assert_ne!(stream.read_line(&mut head).await.unwrap(), 0, "early EOF");
if &head[previous_len..] == "\r\n" {
return head;
}
}
}
if std::env::var_os(CHILD_ENV).is_some() {
let mut config = NetworkProxyConfig {
allow_local_binding: Some(true),
..NetworkProxyConfig::default()
};
config.set_allowed_domains(vec!["same.example".to_string()]);
let mut state = network_proxy_state_for_policy(config);
state.set_process_log_metadata(crate::NetworkProxyProcessLogMetadata {
tool_call_id: Some(CALL_ID.to_string()),
..Default::default()
});
let listener = StdTcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap();
let proxy_addr = listener.local_addr().unwrap();
let lifecycle = crate::connection_lifecycle::ConnectionLifecycle::new();
let proxy_task = tokio::spawn(run_http_proxy_with_std_listener(
Arc::new(state),
listener,
/*policy_decider*/ None,
/*environment_id*/ None,
lifecycle.guard(),
));
let mut stream =
BufReader::new(tokio::net::TcpStream::connect(proxy_addr).await.unwrap());
let connect = format!(
"CONNECT {TARGET} HTTP/1.1\r\nHost: {TARGET}\r\n{HEADER}: guest-value\r\n\r\n"
);
stream
.get_mut()
.write_all(connect.as_bytes())
.await
.unwrap();
let response = read_http_head(&mut stream).await;
assert!(response.starts_with("HTTP/1.1 200 "), "{response}");
let mut tunnel_bytes = [0; TUNNEL_BYTES.len()];
stream.read_exact(&mut tunnel_bytes).await.unwrap();
assert_eq!(&tunnel_bytes, TUNNEL_BYTES);
proxy_task.abort();
return;
}
let listener = TokioTcpListener::bind((Ipv4Addr::LOCALHOST, 0))
.await
.unwrap();
let upstream = format!("http://{}", listener.local_addr().unwrap());
let mut child = Command::new(std::env::current_exe().unwrap());
child
.args([
"--exact",
concat!(
module_path!(),
"::inbound_connect_attributes_the_selected_upstream"
)
.split_once("::")
.unwrap()
.1,
"--nocapture",
])
.env(CHILD_ENV, "1")
.env("HTTPS_PROXY", &upstream)
.env("CODEX_TOOL_CALL_ATTRIBUTION_PROXY", &upstream)
.kill_on_drop(true);
let capture = async {
let (stream, _) = listener.accept().await.unwrap();
let mut stream = BufReader::new(stream);
let head = read_http_head(&mut stream).await;
stream
.get_mut()
.write_all(b"HTTP/1.1 200 Connection Established\r\n\r\n")
.await
.unwrap();
stream.get_mut().write_all(TUNNEL_BYTES).await.unwrap();
head
};
let (output, head) = timeout(Duration::from_secs(10), async {
tokio::join!(child.output(), capture)
})
.await
.expect("inbound CONNECT should reach the configured upstream");
let output = output.unwrap();
assert!(output.status.success(), "{output:?}");
assert!(head.starts_with(&format!("CONNECT {TARGET} HTTP/1.1\r\n")));
let ids: Vec<_> = head
.lines()
.filter_map(|line| line.split_once(':'))
.filter(|(name, _)| name.eq_ignore_ascii_case(HEADER))
.map(|(_, value)| value.trim())
.collect();
assert_eq!(ids, vec![CALL_ID]);
}
#[tokio::test]
async fn http_proxy_blocks_absolute_form_https_for_hooked_host() {
let target_listener = TokioTcpListener::bind((Ipv4Addr::LOCALHOST, 0))
+77 -10
View File
@@ -11,14 +11,17 @@ use rama_core::error::OpaqueError;
use rama_core::extensions::ExtensionsMut;
use rama_core::extensions::ExtensionsRef;
use rama_core::service::BoxService;
use rama_core::service::service_fn;
use rama_http::Body;
use rama_http::HeaderName;
use rama_http::HeaderValue;
use rama_http::Request;
use rama_http::Response;
use rama_http::Version;
use rama_http::layer::version_adapter::RequestVersionAdapter;
use rama_http_backend::client::HttpClientService;
use rama_http_backend::client::HttpConnector;
use rama_http_backend::client::proxy::layer::HttpProxyConnectorLayer;
use rama_http_backend::client::proxy::layer::HttpProxyConnector;
use rama_net::address::Host;
use rama_net::address::HostWithPort;
use rama_net::address::ProxyAddress;
@@ -36,16 +39,35 @@ use tracing::warn;
#[cfg(target_os = "macos")]
use rama_unix::client::UnixConnector;
const TOOL_CALL_HEADER: HeaderName = HeaderName::from_static("x-openai-internal-tool-call-id");
#[derive(Clone)]
struct ToolCallAttribution {
endpoint: ProxyAddress,
call_id: HeaderValue,
}
impl ToolCallAttribution {
fn call_id_for(&self, proxy: &ProxyAddress) -> Option<HeaderValue> {
let http = &rama_net::Protocol::HTTP;
(self.endpoint.address == proxy.address
&& self.endpoint.protocol.as_ref().unwrap_or(http)
== proxy.protocol.as_ref().unwrap_or(http))
.then(|| self.call_id.clone())
}
}
#[derive(Clone, Default)]
struct ProxyConfig {
http: Option<ProxyAddress>,
https: Option<ProxyAddress>,
all: Option<ProxyAddress>,
proxy_private_ips_via_upstream: bool,
attribution: Option<ToolCallAttribution>,
}
impl ProxyConfig {
fn from_env(proxy_private_ips_via_upstream: bool) -> Self {
fn from_env(state: &NetworkProxyState) -> Self {
let http = read_proxy_env(&["HTTP_PROXY", "http_proxy"]);
let https = read_proxy_env(&["HTTPS_PROXY", "https_proxy"]);
let all = read_proxy_env(&["ALL_PROXY", "all_proxy"]);
@@ -53,7 +75,18 @@ impl ProxyConfig {
http,
https,
all,
proxy_private_ips_via_upstream,
proxy_private_ips_via_upstream: state.proxy_private_ips_via_upstream,
attribution: state
.process_log_metadata
.tool_call_id
.as_deref()
.and_then(|id| {
let endpoint = read_proxy_env(&["CODEX_TOOL_CALL_ATTRIBUTION_PROXY"])?;
Some(ToolCallAttribution {
endpoint,
call_id: HeaderValue::from_str(id).ok()?,
})
}),
}
}
@@ -110,9 +143,15 @@ fn read_proxy_env(keys: &[&str]) -> Option<ProxyAddress> {
pub(crate) fn proxy_for_connect(
target: &HostWithPort,
state: &NetworkProxyState,
) -> Option<ProxyAddress> {
ProxyConfig::from_env(state.proxy_private_ips_via_upstream)
.proxy_for_target(target, /*is_secure*/ true)
) -> (Option<ProxyAddress>, Option<HeaderValue>) {
let config = ProxyConfig::from_env(state);
let proxy = config.proxy_for_target(target, /*is_secure*/ true);
let call_id = config
.attribution
.as_ref()
.zip(proxy.as_ref())
.and_then(|(attribution, proxy)| attribution.call_id_for(proxy));
(proxy, call_id)
}
#[derive(Clone)]
@@ -136,7 +175,7 @@ impl UpstreamClient {
pub(crate) fn from_env_proxy(state: Arc<NetworkProxyState>) -> Self {
Self::new(
ProxyConfig::from_env(state.proxy_private_ips_via_upstream),
ProxyConfig::from_env(&state),
TargetCheckedTcpConnector::new(state),
client_root_certs(),
)
@@ -158,7 +197,7 @@ impl UpstreamClient {
tls_root_store: Arc<rustls::RootCertStore>,
) -> Self {
Self::new(
ProxyConfig::from_env(state.proxy_private_ips_via_upstream),
ProxyConfig::from_env(&state),
TargetCheckedTcpConnector::new(state),
tls_root_store,
)
@@ -178,7 +217,8 @@ impl UpstreamClient {
transport: TargetCheckedTcpConnector,
tls_root_store: Arc<rustls::RootCertStore>,
) -> Self {
let connector = build_http_connector(transport, tls_root_store);
let connector =
build_http_connector(transport, tls_root_store, proxy_config.attribution.clone());
Self {
connector,
proxy_config,
@@ -210,7 +250,14 @@ impl Service<Request<Body>> for UpstreamClient {
),
None => info!("HTTP upstream route selected (target={authority}, route=direct)"),
}
req.headers_mut().remove(TOOL_CALL_HEADER);
if let Some(proxy) = proxy {
if request_context.is_some_and(|ctx| ctx.protocol == rama_net::Protocol::HTTP)
&& let Some(attribution) = &self.proxy_config.attribution
&& let Some(id) = attribution.call_id_for(&proxy)
{
req.headers_mut().insert(TOOL_CALL_HEADER, id);
}
req.extensions_mut().insert(proxy);
}
@@ -258,16 +305,36 @@ impl Service<Request<Body>> for UpstreamClient {
}
}
pub(crate) fn upstream_proxy_connector(
transport: TargetCheckedTcpConnector,
connect_call_id: Option<HeaderValue>,
) -> HttpProxyConnector<TargetCheckedTcpConnector> {
let mut connector = HttpProxyConnector::optional(transport);
if let Some(id) = connect_call_id {
connector.set_custom_header(TOOL_CALL_HEADER, id);
}
connector
}
fn build_http_connector(
transport: TargetCheckedTcpConnector,
tls_root_store: Arc<rustls::RootCertStore>,
attribution: Option<ToolCallAttribution>,
) -> BoxService<
Request<Body>,
EstablishedClientConnection<HttpClientService<Body>, Request<Body>>,
BoxError,
> {
ensure_rustls_crypto_provider();
let proxy = HttpProxyConnectorLayer::optional().into_layer(transport);
let proxy = service_fn(move |req: Request<Body>| {
// Rama uses custom connector headers only on its outbound CONNECT.
let call_id = attribution
.as_ref()
.zip(req.extensions().get::<ProxyAddress>())
.and_then(|(attribution, proxy)| attribution.call_id_for(proxy));
let connector = upstream_proxy_connector(transport.clone(), call_id);
async move { connector.serve(req).await }
});
let client_config = rustls::ClientConfig::builder_with_protocol_versions(rustls::ALL_VERSIONS)
.with_root_certificates(tls_root_store)
.with_no_client_auth();
@@ -159,3 +159,152 @@ fn private_ip_upstream_routing_is_opt_in() {
}
}
}
async fn capture_proxy_request(
uri: String,
listener: TcpListener,
proxy_config: ProxyConfig,
) -> String {
let network = NetworkProxyConfig {
allow_local_binding: Some(true),
..NetworkProxyConfig::default()
};
let state = Arc::new(network_proxy_state_for_policy(network));
let client = UpstreamClient::new(
proxy_config,
TargetCheckedTcpConnector::new(state),
client_root_certs(),
);
let request = Request::builder()
.uri(uri)
.header(TOOL_CALL_HEADER, "from-command")
.body(Body::empty())
.unwrap();
let capture = async {
let (mut socket, _) = listener.accept().await.unwrap();
let mut bytes = Vec::new();
while !bytes.ends_with(b"\r\n\r\n") {
bytes.push(socket.read_u8().await.unwrap());
assert!(bytes.len() < 8192);
}
socket
.write_all(
b"HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\nConnection: close\r\n\r\n",
)
.await
.unwrap();
String::from_utf8(bytes).unwrap()
};
let (wire, _) = tokio::time::timeout(std::time::Duration::from_secs(5), async {
tokio::join!(capture, client.serve(request))
})
.await
.expect("request reached proxy or direct destination");
wire
}
#[tokio::test]
async fn tool_call_header_is_scoped_to_the_configured_upstream_and_execution() {
async fn exchange(scheme: &str, call_id: &'static str) -> String {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let proxy = ProxyAddress::try_from(addr.to_string()).unwrap();
let endpoint = ProxyAddress::try_from(format!("http://{addr}")).unwrap();
let config = ProxyConfig {
http: Some(proxy.clone()),
https: Some(proxy),
attribution: Some(ToolCallAttribution {
endpoint,
call_id: HeaderValue::from_static(call_id),
}),
..ProxyConfig::default()
};
capture_proxy_request(format!("{scheme}://same.example/path"), listener, config).await
}
let (http, https) = tokio::join!(exchange("http", "call-A"), exchange("https", "exec-B"));
assert!(http.starts_with("GET "));
assert!(https.starts_with("CONNECT same.example:443 "));
for (wire, expected) in [(http, "call-A"), (https, "exec-B")] {
let ids: Vec<_> = wire
.lines()
.filter_map(|line| line.split_once(": "))
.filter(|(name, _)| name.eq_ignore_ascii_case(TOOL_CALL_HEADER.as_str()))
.map(|(_, value)| value)
.collect();
assert_eq!(ids, vec![expected]);
}
}
#[tokio::test]
async fn direct_and_other_proxy_requests_strip_tool_call_headers() {
let trusted = ProxyAddress::try_from("http://other-proxy.test:8889").unwrap();
for (scheme, use_proxy, enabled) in [
("http", false, true),
("http", true, false),
("http", true, true),
("https", true, true),
] {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let proxy = ProxyAddress::try_from(format!("http://{address}")).unwrap();
let uri = if use_proxy {
format!("{scheme}://same.example/path")
} else {
format!("http://{address}/path")
};
let config = ProxyConfig {
http: use_proxy.then(|| proxy.clone()),
https: use_proxy.then_some(proxy),
attribution: enabled.then(|| ToolCallAttribution {
endpoint: trusted.clone(),
call_id: HeaderValue::from_static("call-A"),
}),
..ProxyConfig::default()
};
let wire = capture_proxy_request(uri, listener, config).await;
assert!(
!wire
.to_ascii_lowercase()
.contains(TOOL_CALL_HEADER.as_str())
);
}
}
#[tokio::test]
async fn split_upstreams_attribute_only_the_selected_hop_regardless_of_origin_port() {
for (uri, secure) in [
("http://same.example:443/path", false),
("https://same.example:80/path", true),
] {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let active =
ProxyAddress::try_from(format!("http://{}", listener.local_addr().unwrap())).unwrap();
let other = ProxyAddress::try_from("http://unused-proxy.test:8889").unwrap();
let (http, https) = if secure {
(other, active)
} else {
(active, other)
};
let config = ProxyConfig {
http: Some(http),
https: Some(https.clone()),
attribution: Some(ToolCallAttribution {
endpoint: https,
call_id: HeaderValue::from_static("exec-A"),
}),
..ProxyConfig::default()
};
let wire = capture_proxy_request(uri.to_string(), listener, config).await;
assert_eq!(
wire.to_ascii_lowercase()
.contains(TOOL_CALL_HEADER.as_str()),
secure
);
if secure {
assert!(wire.starts_with("CONNECT same.example:80 "));
assert!(wire.contains("exec-A"));
}
}
}