Add opt-in tool-call attribution for upstream proxies (#61860)

## What changed

Allow executors to set `CODEX_TOOL_CALL_ATTRIBUTION_PROXY` to designate an upstream proxy that consumes and strips `x-openai-internal-tool-call-id`. When the selected upstream matches that endpoint, attach the controller's tool-call ID to plaintext HTTP requests or the outer HTTPS `CONNECT` request.

Strip caller-supplied attribution headers and send no attribution to direct destinations or other upstream proxies. Leave attribution disabled when the setting is unset.

## Testing

Add request-capture tests for HTTP and HTTPS attribution, execution-specific IDs, header stripping, split upstream selection with nonstandard origin ports, and inbound `CONNECT` tunneling.

GitOrigin-RevId: f908faf55a991bc11f2f9c7fd3100f42b4f638b9
This commit is contained in:
Hendrik Buyle
2026-11-16 19:13:33 +00:00
committed by copyberry
parent 3257e1b3eb
commit 457a191479
4 changed files with 352 additions and 14 deletions
+8
View File
@@ -155,6 +155,14 @@ they are blocked. SOCKS5 UDP and non-HTTPS SOCKS5 TCP remain blocked in limited
Websocket clients typically tunnel `wss://` through HTTPS `CONNECT`; those CONNECT targets still go
through the same host allowlist/denylist checks.
### Tool-call attribution on upstream requests
A managed executor can set `CODEX_TOOL_CALL_ATTRIBUTION_PROXY` to the HTTP(S) URL of an upstream
proxy that consumes and strips `x-openai-internal-tool-call-id`. When that endpoint is selected,
the execution-scoped proxy attaches the controller's tool-call ID to plaintext HTTP requests or
the outer HTTPS CONNECT. Set this option in the executor process environment. Leaving it unset
disables attribution. Direct destinations and other upstream proxies receive no attribution header.
## Library API
`codex-network-proxy` can be embedded as a library with a thin API:
+118 -4
View File
@@ -36,6 +36,7 @@ use crate::state::BlockedRequestArgs;
use crate::state::NetworkProxyState;
use crate::upstream::UpstreamClient;
use crate::upstream::proxy_for_connect;
use crate::upstream::upstream_proxy_connector;
use anyhow::Context as _;
use anyhow::Result;
use codex_utils_rustls_provider::ensure_rustls_crypto_provider;
@@ -61,7 +62,6 @@ use rama_http::headers::HeaderMapExt;
use rama_http::headers::Host;
use rama_http::layer::remove_header::RemoveResponseHeaderLayer;
use rama_http::matcher::MethodMatcher;
use rama_http_backend::client::proxy::layer::HttpProxyConnector;
use rama_http_backend::server::HttpServer;
use rama_http_backend::server::layer::upgrade::UpgradeLayer;
use rama_http_backend::server::layer::upgrade::Upgraded;
@@ -491,10 +491,10 @@ where
false
}
};
let proxy = if allow_upstream_proxy {
let (proxy, connect_call_id) = if allow_upstream_proxy {
proxy_for_connect(&authority, &app_state)
} else {
None
(None, None)
};
match proxy.as_ref() {
Some(proxy) => info!(
@@ -514,7 +514,8 @@ where
let req = TcpRequest::new_with_extensions(authority.clone(), extensions)
.with_protocol(Protocol::HTTPS);
let proxy_connector = HttpProxyConnector::optional(TargetCheckedTcpConnector::new(app_state));
let proxy_connector =
upstream_proxy_connector(TargetCheckedTcpConnector::new(app_state), connect_call_id);
let tls_config = TlsConnectorDataBuilder::new()
.with_alpn_protocols_http_auto()
.build();
@@ -1833,6 +1834,119 @@ mod tests {
target_task.await.expect("target task should finish");
}
#[tokio::test]
async fn inbound_connect_attributes_the_selected_upstream() {
use tokio::io::AsyncBufReadExt;
use tokio::io::BufReader;
use tokio::process::Command;
const CHILD_ENV: &str = "CODEX_TEST_INBOUND_CONNECT_ATTRIBUTION";
const TARGET: &str = "same.example:443";
const HEADER: &str = "x-openai-internal-tool-call-id";
const CALL_ID: &str = "controller-call";
const TUNNEL_BYTES: &[u8] = b"tunnel ready";
async fn read_http_head(stream: &mut BufReader<tokio::net::TcpStream>) -> String {
let mut head = String::new();
loop {
let previous_len = head.len();
assert_ne!(stream.read_line(&mut head).await.unwrap(), 0, "early EOF");
if &head[previous_len..] == "\r\n" {
return head;
}
}
}
if std::env::var_os(CHILD_ENV).is_some() {
let mut config = NetworkProxyConfig {
allow_local_binding: Some(true),
..NetworkProxyConfig::default()
};
config.set_allowed_domains(vec!["same.example".to_string()]);
let mut state = network_proxy_state_for_policy(config);
state.set_process_log_metadata(crate::NetworkProxyProcessLogMetadata {
tool_call_id: Some(CALL_ID.to_string()),
..Default::default()
});
let listener = StdTcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap();
let proxy_addr = listener.local_addr().unwrap();
let lifecycle = crate::connection_lifecycle::ConnectionLifecycle::new();
let proxy_task = tokio::spawn(run_http_proxy_with_std_listener(
Arc::new(state),
listener,
/*policy_decider*/ None,
/*environment_id*/ None,
lifecycle.guard(),
));
let mut stream =
BufReader::new(tokio::net::TcpStream::connect(proxy_addr).await.unwrap());
let connect = format!(
"CONNECT {TARGET} HTTP/1.1\r\nHost: {TARGET}\r\n{HEADER}: guest-value\r\n\r\n"
);
stream
.get_mut()
.write_all(connect.as_bytes())
.await
.unwrap();
let response = read_http_head(&mut stream).await;
assert!(response.starts_with("HTTP/1.1 200 "), "{response}");
let mut tunnel_bytes = [0; TUNNEL_BYTES.len()];
stream.read_exact(&mut tunnel_bytes).await.unwrap();
assert_eq!(&tunnel_bytes, TUNNEL_BYTES);
proxy_task.abort();
return;
}
let listener = TokioTcpListener::bind((Ipv4Addr::LOCALHOST, 0))
.await
.unwrap();
let upstream = format!("http://{}", listener.local_addr().unwrap());
let mut child = Command::new(std::env::current_exe().unwrap());
child
.args([
"--exact",
concat!(
module_path!(),
"::inbound_connect_attributes_the_selected_upstream"
)
.split_once("::")
.unwrap()
.1,
"--nocapture",
])
.env(CHILD_ENV, "1")
.env("HTTPS_PROXY", &upstream)
.env("CODEX_TOOL_CALL_ATTRIBUTION_PROXY", &upstream)
.kill_on_drop(true);
let capture = async {
let (stream, _) = listener.accept().await.unwrap();
let mut stream = BufReader::new(stream);
let head = read_http_head(&mut stream).await;
stream
.get_mut()
.write_all(b"HTTP/1.1 200 Connection Established\r\n\r\n")
.await
.unwrap();
stream.get_mut().write_all(TUNNEL_BYTES).await.unwrap();
head
};
let (output, head) = timeout(Duration::from_secs(10), async {
tokio::join!(child.output(), capture)
})
.await
.expect("inbound CONNECT should reach the configured upstream");
let output = output.unwrap();
assert!(output.status.success(), "{output:?}");
assert!(head.starts_with(&format!("CONNECT {TARGET} HTTP/1.1\r\n")));
let ids: Vec<_> = head
.lines()
.filter_map(|line| line.split_once(':'))
.filter(|(name, _)| name.eq_ignore_ascii_case(HEADER))
.map(|(_, value)| value.trim())
.collect();
assert_eq!(ids, vec![CALL_ID]);
}
#[tokio::test]
async fn http_proxy_blocks_absolute_form_https_for_hooked_host() {
let target_listener = TokioTcpListener::bind((Ipv4Addr::LOCALHOST, 0))
+77 -10
View File
@@ -11,14 +11,17 @@ use rama_core::error::OpaqueError;
use rama_core::extensions::ExtensionsMut;
use rama_core::extensions::ExtensionsRef;
use rama_core::service::BoxService;
use rama_core::service::service_fn;
use rama_http::Body;
use rama_http::HeaderName;
use rama_http::HeaderValue;
use rama_http::Request;
use rama_http::Response;
use rama_http::Version;
use rama_http::layer::version_adapter::RequestVersionAdapter;
use rama_http_backend::client::HttpClientService;
use rama_http_backend::client::HttpConnector;
use rama_http_backend::client::proxy::layer::HttpProxyConnectorLayer;
use rama_http_backend::client::proxy::layer::HttpProxyConnector;
use rama_net::address::Host;
use rama_net::address::HostWithPort;
use rama_net::address::ProxyAddress;
@@ -36,16 +39,35 @@ use tracing::warn;
#[cfg(target_os = "macos")]
use rama_unix::client::UnixConnector;
const TOOL_CALL_HEADER: HeaderName = HeaderName::from_static("x-openai-internal-tool-call-id");
#[derive(Clone)]
struct ToolCallAttribution {
endpoint: ProxyAddress,
call_id: HeaderValue,
}
impl ToolCallAttribution {
fn call_id_for(&self, proxy: &ProxyAddress) -> Option<HeaderValue> {
let http = &rama_net::Protocol::HTTP;
(self.endpoint.address == proxy.address
&& self.endpoint.protocol.as_ref().unwrap_or(http)
== proxy.protocol.as_ref().unwrap_or(http))
.then(|| self.call_id.clone())
}
}
#[derive(Clone, Default)]
struct ProxyConfig {
http: Option<ProxyAddress>,
https: Option<ProxyAddress>,
all: Option<ProxyAddress>,
proxy_private_ips_via_upstream: bool,
attribution: Option<ToolCallAttribution>,
}
impl ProxyConfig {
fn from_env(proxy_private_ips_via_upstream: bool) -> Self {
fn from_env(state: &NetworkProxyState) -> Self {
let http = read_proxy_env(&["HTTP_PROXY", "http_proxy"]);
let https = read_proxy_env(&["HTTPS_PROXY", "https_proxy"]);
let all = read_proxy_env(&["ALL_PROXY", "all_proxy"]);
@@ -53,7 +75,18 @@ impl ProxyConfig {
http,
https,
all,
proxy_private_ips_via_upstream,
proxy_private_ips_via_upstream: state.proxy_private_ips_via_upstream,
attribution: state
.process_log_metadata
.tool_call_id
.as_deref()
.and_then(|id| {
let endpoint = read_proxy_env(&["CODEX_TOOL_CALL_ATTRIBUTION_PROXY"])?;
Some(ToolCallAttribution {
endpoint,
call_id: HeaderValue::from_str(id).ok()?,
})
}),
}
}
@@ -110,9 +143,15 @@ fn read_proxy_env(keys: &[&str]) -> Option<ProxyAddress> {
pub(crate) fn proxy_for_connect(
target: &HostWithPort,
state: &NetworkProxyState,
) -> Option<ProxyAddress> {
ProxyConfig::from_env(state.proxy_private_ips_via_upstream)
.proxy_for_target(target, /*is_secure*/ true)
) -> (Option<ProxyAddress>, Option<HeaderValue>) {
let config = ProxyConfig::from_env(state);
let proxy = config.proxy_for_target(target, /*is_secure*/ true);
let call_id = config
.attribution
.as_ref()
.zip(proxy.as_ref())
.and_then(|(attribution, proxy)| attribution.call_id_for(proxy));
(proxy, call_id)
}
#[derive(Clone)]
@@ -136,7 +175,7 @@ impl UpstreamClient {
pub(crate) fn from_env_proxy(state: Arc<NetworkProxyState>) -> Self {
Self::new(
ProxyConfig::from_env(state.proxy_private_ips_via_upstream),
ProxyConfig::from_env(&state),
TargetCheckedTcpConnector::new(state),
client_root_certs(),
)
@@ -158,7 +197,7 @@ impl UpstreamClient {
tls_root_store: Arc<rustls::RootCertStore>,
) -> Self {
Self::new(
ProxyConfig::from_env(state.proxy_private_ips_via_upstream),
ProxyConfig::from_env(&state),
TargetCheckedTcpConnector::new(state),
tls_root_store,
)
@@ -178,7 +217,8 @@ impl UpstreamClient {
transport: TargetCheckedTcpConnector,
tls_root_store: Arc<rustls::RootCertStore>,
) -> Self {
let connector = build_http_connector(transport, tls_root_store);
let connector =
build_http_connector(transport, tls_root_store, proxy_config.attribution.clone());
Self {
connector,
proxy_config,
@@ -210,7 +250,14 @@ impl Service<Request<Body>> for UpstreamClient {
),
None => info!("HTTP upstream route selected (target={authority}, route=direct)"),
}
req.headers_mut().remove(TOOL_CALL_HEADER);
if let Some(proxy) = proxy {
if request_context.is_some_and(|ctx| ctx.protocol == rama_net::Protocol::HTTP)
&& let Some(attribution) = &self.proxy_config.attribution
&& let Some(id) = attribution.call_id_for(&proxy)
{
req.headers_mut().insert(TOOL_CALL_HEADER, id);
}
req.extensions_mut().insert(proxy);
}
@@ -258,16 +305,36 @@ impl Service<Request<Body>> for UpstreamClient {
}
}
pub(crate) fn upstream_proxy_connector(
transport: TargetCheckedTcpConnector,
connect_call_id: Option<HeaderValue>,
) -> HttpProxyConnector<TargetCheckedTcpConnector> {
let mut connector = HttpProxyConnector::optional(transport);
if let Some(id) = connect_call_id {
connector.set_custom_header(TOOL_CALL_HEADER, id);
}
connector
}
fn build_http_connector(
transport: TargetCheckedTcpConnector,
tls_root_store: Arc<rustls::RootCertStore>,
attribution: Option<ToolCallAttribution>,
) -> BoxService<
Request<Body>,
EstablishedClientConnection<HttpClientService<Body>, Request<Body>>,
BoxError,
> {
ensure_rustls_crypto_provider();
let proxy = HttpProxyConnectorLayer::optional().into_layer(transport);
let proxy = service_fn(move |req: Request<Body>| {
// Rama uses custom connector headers only on its outbound CONNECT.
let call_id = attribution
.as_ref()
.zip(req.extensions().get::<ProxyAddress>())
.and_then(|(attribution, proxy)| attribution.call_id_for(proxy));
let connector = upstream_proxy_connector(transport.clone(), call_id);
async move { connector.serve(req).await }
});
let client_config = rustls::ClientConfig::builder_with_protocol_versions(rustls::ALL_VERSIONS)
.with_root_certificates(tls_root_store)
.with_no_client_auth();
@@ -159,3 +159,152 @@ fn private_ip_upstream_routing_is_opt_in() {
}
}
}
async fn capture_proxy_request(
uri: String,
listener: TcpListener,
proxy_config: ProxyConfig,
) -> String {
let network = NetworkProxyConfig {
allow_local_binding: Some(true),
..NetworkProxyConfig::default()
};
let state = Arc::new(network_proxy_state_for_policy(network));
let client = UpstreamClient::new(
proxy_config,
TargetCheckedTcpConnector::new(state),
client_root_certs(),
);
let request = Request::builder()
.uri(uri)
.header(TOOL_CALL_HEADER, "from-command")
.body(Body::empty())
.unwrap();
let capture = async {
let (mut socket, _) = listener.accept().await.unwrap();
let mut bytes = Vec::new();
while !bytes.ends_with(b"\r\n\r\n") {
bytes.push(socket.read_u8().await.unwrap());
assert!(bytes.len() < 8192);
}
socket
.write_all(
b"HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\nConnection: close\r\n\r\n",
)
.await
.unwrap();
String::from_utf8(bytes).unwrap()
};
let (wire, _) = tokio::time::timeout(std::time::Duration::from_secs(5), async {
tokio::join!(capture, client.serve(request))
})
.await
.expect("request reached proxy or direct destination");
wire
}
#[tokio::test]
async fn tool_call_header_is_scoped_to_the_configured_upstream_and_execution() {
async fn exchange(scheme: &str, call_id: &'static str) -> String {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let proxy = ProxyAddress::try_from(addr.to_string()).unwrap();
let endpoint = ProxyAddress::try_from(format!("http://{addr}")).unwrap();
let config = ProxyConfig {
http: Some(proxy.clone()),
https: Some(proxy),
attribution: Some(ToolCallAttribution {
endpoint,
call_id: HeaderValue::from_static(call_id),
}),
..ProxyConfig::default()
};
capture_proxy_request(format!("{scheme}://same.example/path"), listener, config).await
}
let (http, https) = tokio::join!(exchange("http", "call-A"), exchange("https", "exec-B"));
assert!(http.starts_with("GET "));
assert!(https.starts_with("CONNECT same.example:443 "));
for (wire, expected) in [(http, "call-A"), (https, "exec-B")] {
let ids: Vec<_> = wire
.lines()
.filter_map(|line| line.split_once(": "))
.filter(|(name, _)| name.eq_ignore_ascii_case(TOOL_CALL_HEADER.as_str()))
.map(|(_, value)| value)
.collect();
assert_eq!(ids, vec![expected]);
}
}
#[tokio::test]
async fn direct_and_other_proxy_requests_strip_tool_call_headers() {
let trusted = ProxyAddress::try_from("http://other-proxy.test:8889").unwrap();
for (scheme, use_proxy, enabled) in [
("http", false, true),
("http", true, false),
("http", true, true),
("https", true, true),
] {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let proxy = ProxyAddress::try_from(format!("http://{address}")).unwrap();
let uri = if use_proxy {
format!("{scheme}://same.example/path")
} else {
format!("http://{address}/path")
};
let config = ProxyConfig {
http: use_proxy.then(|| proxy.clone()),
https: use_proxy.then_some(proxy),
attribution: enabled.then(|| ToolCallAttribution {
endpoint: trusted.clone(),
call_id: HeaderValue::from_static("call-A"),
}),
..ProxyConfig::default()
};
let wire = capture_proxy_request(uri, listener, config).await;
assert!(
!wire
.to_ascii_lowercase()
.contains(TOOL_CALL_HEADER.as_str())
);
}
}
#[tokio::test]
async fn split_upstreams_attribute_only_the_selected_hop_regardless_of_origin_port() {
for (uri, secure) in [
("http://same.example:443/path", false),
("https://same.example:80/path", true),
] {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let active =
ProxyAddress::try_from(format!("http://{}", listener.local_addr().unwrap())).unwrap();
let other = ProxyAddress::try_from("http://unused-proxy.test:8889").unwrap();
let (http, https) = if secure {
(other, active)
} else {
(active, other)
};
let config = ProxyConfig {
http: Some(http),
https: Some(https.clone()),
attribution: Some(ToolCallAttribution {
endpoint: https,
call_id: HeaderValue::from_static("exec-A"),
}),
..ProxyConfig::default()
};
let wire = capture_proxy_request(uri.to_string(), listener, config).await;
assert_eq!(
wire.to_ascii_lowercase()
.contains(TOOL_CALL_HEADER.as_str()),
secure
);
if secure {
assert!(wire.starts_with("CONNECT same.example:80 "));
assert!(wire.contains("exec-A"));
}
}
}