mirror of
https://github.com/openai/codex.git
synced 2026-11-16 22:00:36 +00:00
Add opt-in tool-call attribution for upstream proxies (#61860)
## What changed Allow executors to set `CODEX_TOOL_CALL_ATTRIBUTION_PROXY` to designate an upstream proxy that consumes and strips `x-openai-internal-tool-call-id`. When the selected upstream matches that endpoint, attach the controller's tool-call ID to plaintext HTTP requests or the outer HTTPS `CONNECT` request. Strip caller-supplied attribution headers and send no attribution to direct destinations or other upstream proxies. Leave attribution disabled when the setting is unset. ## Testing Add request-capture tests for HTTP and HTTPS attribution, execution-specific IDs, header stripping, split upstream selection with nonstandard origin ports, and inbound `CONNECT` tunneling. GitOrigin-RevId: f908faf55a991bc11f2f9c7fd3100f42b4f638b9
This commit is contained in:
@@ -36,6 +36,7 @@ use crate::state::BlockedRequestArgs;
|
||||
use crate::state::NetworkProxyState;
|
||||
use crate::upstream::UpstreamClient;
|
||||
use crate::upstream::proxy_for_connect;
|
||||
use crate::upstream::upstream_proxy_connector;
|
||||
use anyhow::Context as _;
|
||||
use anyhow::Result;
|
||||
use codex_utils_rustls_provider::ensure_rustls_crypto_provider;
|
||||
@@ -61,7 +62,6 @@ use rama_http::headers::HeaderMapExt;
|
||||
use rama_http::headers::Host;
|
||||
use rama_http::layer::remove_header::RemoveResponseHeaderLayer;
|
||||
use rama_http::matcher::MethodMatcher;
|
||||
use rama_http_backend::client::proxy::layer::HttpProxyConnector;
|
||||
use rama_http_backend::server::HttpServer;
|
||||
use rama_http_backend::server::layer::upgrade::UpgradeLayer;
|
||||
use rama_http_backend::server::layer::upgrade::Upgraded;
|
||||
@@ -491,10 +491,10 @@ where
|
||||
false
|
||||
}
|
||||
};
|
||||
let proxy = if allow_upstream_proxy {
|
||||
let (proxy, connect_call_id) = if allow_upstream_proxy {
|
||||
proxy_for_connect(&authority, &app_state)
|
||||
} else {
|
||||
None
|
||||
(None, None)
|
||||
};
|
||||
match proxy.as_ref() {
|
||||
Some(proxy) => info!(
|
||||
@@ -514,7 +514,8 @@ where
|
||||
|
||||
let req = TcpRequest::new_with_extensions(authority.clone(), extensions)
|
||||
.with_protocol(Protocol::HTTPS);
|
||||
let proxy_connector = HttpProxyConnector::optional(TargetCheckedTcpConnector::new(app_state));
|
||||
let proxy_connector =
|
||||
upstream_proxy_connector(TargetCheckedTcpConnector::new(app_state), connect_call_id);
|
||||
let tls_config = TlsConnectorDataBuilder::new()
|
||||
.with_alpn_protocols_http_auto()
|
||||
.build();
|
||||
@@ -1833,6 +1834,119 @@ mod tests {
|
||||
target_task.await.expect("target task should finish");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn inbound_connect_attributes_the_selected_upstream() {
|
||||
use tokio::io::AsyncBufReadExt;
|
||||
use tokio::io::BufReader;
|
||||
use tokio::process::Command;
|
||||
|
||||
const CHILD_ENV: &str = "CODEX_TEST_INBOUND_CONNECT_ATTRIBUTION";
|
||||
const TARGET: &str = "same.example:443";
|
||||
const HEADER: &str = "x-openai-internal-tool-call-id";
|
||||
const CALL_ID: &str = "controller-call";
|
||||
const TUNNEL_BYTES: &[u8] = b"tunnel ready";
|
||||
|
||||
async fn read_http_head(stream: &mut BufReader<tokio::net::TcpStream>) -> String {
|
||||
let mut head = String::new();
|
||||
loop {
|
||||
let previous_len = head.len();
|
||||
assert_ne!(stream.read_line(&mut head).await.unwrap(), 0, "early EOF");
|
||||
if &head[previous_len..] == "\r\n" {
|
||||
return head;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if std::env::var_os(CHILD_ENV).is_some() {
|
||||
let mut config = NetworkProxyConfig {
|
||||
allow_local_binding: Some(true),
|
||||
..NetworkProxyConfig::default()
|
||||
};
|
||||
config.set_allowed_domains(vec!["same.example".to_string()]);
|
||||
let mut state = network_proxy_state_for_policy(config);
|
||||
state.set_process_log_metadata(crate::NetworkProxyProcessLogMetadata {
|
||||
tool_call_id: Some(CALL_ID.to_string()),
|
||||
..Default::default()
|
||||
});
|
||||
let listener = StdTcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap();
|
||||
let proxy_addr = listener.local_addr().unwrap();
|
||||
let lifecycle = crate::connection_lifecycle::ConnectionLifecycle::new();
|
||||
let proxy_task = tokio::spawn(run_http_proxy_with_std_listener(
|
||||
Arc::new(state),
|
||||
listener,
|
||||
/*policy_decider*/ None,
|
||||
/*environment_id*/ None,
|
||||
lifecycle.guard(),
|
||||
));
|
||||
let mut stream =
|
||||
BufReader::new(tokio::net::TcpStream::connect(proxy_addr).await.unwrap());
|
||||
let connect = format!(
|
||||
"CONNECT {TARGET} HTTP/1.1\r\nHost: {TARGET}\r\n{HEADER}: guest-value\r\n\r\n"
|
||||
);
|
||||
stream
|
||||
.get_mut()
|
||||
.write_all(connect.as_bytes())
|
||||
.await
|
||||
.unwrap();
|
||||
let response = read_http_head(&mut stream).await;
|
||||
assert!(response.starts_with("HTTP/1.1 200 "), "{response}");
|
||||
let mut tunnel_bytes = [0; TUNNEL_BYTES.len()];
|
||||
stream.read_exact(&mut tunnel_bytes).await.unwrap();
|
||||
assert_eq!(&tunnel_bytes, TUNNEL_BYTES);
|
||||
proxy_task.abort();
|
||||
return;
|
||||
}
|
||||
|
||||
let listener = TokioTcpListener::bind((Ipv4Addr::LOCALHOST, 0))
|
||||
.await
|
||||
.unwrap();
|
||||
let upstream = format!("http://{}", listener.local_addr().unwrap());
|
||||
let mut child = Command::new(std::env::current_exe().unwrap());
|
||||
child
|
||||
.args([
|
||||
"--exact",
|
||||
concat!(
|
||||
module_path!(),
|
||||
"::inbound_connect_attributes_the_selected_upstream"
|
||||
)
|
||||
.split_once("::")
|
||||
.unwrap()
|
||||
.1,
|
||||
"--nocapture",
|
||||
])
|
||||
.env(CHILD_ENV, "1")
|
||||
.env("HTTPS_PROXY", &upstream)
|
||||
.env("CODEX_TOOL_CALL_ATTRIBUTION_PROXY", &upstream)
|
||||
.kill_on_drop(true);
|
||||
let capture = async {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let mut stream = BufReader::new(stream);
|
||||
let head = read_http_head(&mut stream).await;
|
||||
stream
|
||||
.get_mut()
|
||||
.write_all(b"HTTP/1.1 200 Connection Established\r\n\r\n")
|
||||
.await
|
||||
.unwrap();
|
||||
stream.get_mut().write_all(TUNNEL_BYTES).await.unwrap();
|
||||
head
|
||||
};
|
||||
let (output, head) = timeout(Duration::from_secs(10), async {
|
||||
tokio::join!(child.output(), capture)
|
||||
})
|
||||
.await
|
||||
.expect("inbound CONNECT should reach the configured upstream");
|
||||
let output = output.unwrap();
|
||||
assert!(output.status.success(), "{output:?}");
|
||||
assert!(head.starts_with(&format!("CONNECT {TARGET} HTTP/1.1\r\n")));
|
||||
let ids: Vec<_> = head
|
||||
.lines()
|
||||
.filter_map(|line| line.split_once(':'))
|
||||
.filter(|(name, _)| name.eq_ignore_ascii_case(HEADER))
|
||||
.map(|(_, value)| value.trim())
|
||||
.collect();
|
||||
assert_eq!(ids, vec![CALL_ID]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_proxy_blocks_absolute_form_https_for_hooked_host() {
|
||||
let target_listener = TokioTcpListener::bind((Ipv4Addr::LOCALHOST, 0))
|
||||
|
||||
@@ -11,14 +11,17 @@ use rama_core::error::OpaqueError;
|
||||
use rama_core::extensions::ExtensionsMut;
|
||||
use rama_core::extensions::ExtensionsRef;
|
||||
use rama_core::service::BoxService;
|
||||
use rama_core::service::service_fn;
|
||||
use rama_http::Body;
|
||||
use rama_http::HeaderName;
|
||||
use rama_http::HeaderValue;
|
||||
use rama_http::Request;
|
||||
use rama_http::Response;
|
||||
use rama_http::Version;
|
||||
use rama_http::layer::version_adapter::RequestVersionAdapter;
|
||||
use rama_http_backend::client::HttpClientService;
|
||||
use rama_http_backend::client::HttpConnector;
|
||||
use rama_http_backend::client::proxy::layer::HttpProxyConnectorLayer;
|
||||
use rama_http_backend::client::proxy::layer::HttpProxyConnector;
|
||||
use rama_net::address::Host;
|
||||
use rama_net::address::HostWithPort;
|
||||
use rama_net::address::ProxyAddress;
|
||||
@@ -36,16 +39,35 @@ use tracing::warn;
|
||||
#[cfg(target_os = "macos")]
|
||||
use rama_unix::client::UnixConnector;
|
||||
|
||||
const TOOL_CALL_HEADER: HeaderName = HeaderName::from_static("x-openai-internal-tool-call-id");
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ToolCallAttribution {
|
||||
endpoint: ProxyAddress,
|
||||
call_id: HeaderValue,
|
||||
}
|
||||
|
||||
impl ToolCallAttribution {
|
||||
fn call_id_for(&self, proxy: &ProxyAddress) -> Option<HeaderValue> {
|
||||
let http = &rama_net::Protocol::HTTP;
|
||||
(self.endpoint.address == proxy.address
|
||||
&& self.endpoint.protocol.as_ref().unwrap_or(http)
|
||||
== proxy.protocol.as_ref().unwrap_or(http))
|
||||
.then(|| self.call_id.clone())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct ProxyConfig {
|
||||
http: Option<ProxyAddress>,
|
||||
https: Option<ProxyAddress>,
|
||||
all: Option<ProxyAddress>,
|
||||
proxy_private_ips_via_upstream: bool,
|
||||
attribution: Option<ToolCallAttribution>,
|
||||
}
|
||||
|
||||
impl ProxyConfig {
|
||||
fn from_env(proxy_private_ips_via_upstream: bool) -> Self {
|
||||
fn from_env(state: &NetworkProxyState) -> Self {
|
||||
let http = read_proxy_env(&["HTTP_PROXY", "http_proxy"]);
|
||||
let https = read_proxy_env(&["HTTPS_PROXY", "https_proxy"]);
|
||||
let all = read_proxy_env(&["ALL_PROXY", "all_proxy"]);
|
||||
@@ -53,7 +75,18 @@ impl ProxyConfig {
|
||||
http,
|
||||
https,
|
||||
all,
|
||||
proxy_private_ips_via_upstream,
|
||||
proxy_private_ips_via_upstream: state.proxy_private_ips_via_upstream,
|
||||
attribution: state
|
||||
.process_log_metadata
|
||||
.tool_call_id
|
||||
.as_deref()
|
||||
.and_then(|id| {
|
||||
let endpoint = read_proxy_env(&["CODEX_TOOL_CALL_ATTRIBUTION_PROXY"])?;
|
||||
Some(ToolCallAttribution {
|
||||
endpoint,
|
||||
call_id: HeaderValue::from_str(id).ok()?,
|
||||
})
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -110,9 +143,15 @@ fn read_proxy_env(keys: &[&str]) -> Option<ProxyAddress> {
|
||||
pub(crate) fn proxy_for_connect(
|
||||
target: &HostWithPort,
|
||||
state: &NetworkProxyState,
|
||||
) -> Option<ProxyAddress> {
|
||||
ProxyConfig::from_env(state.proxy_private_ips_via_upstream)
|
||||
.proxy_for_target(target, /*is_secure*/ true)
|
||||
) -> (Option<ProxyAddress>, Option<HeaderValue>) {
|
||||
let config = ProxyConfig::from_env(state);
|
||||
let proxy = config.proxy_for_target(target, /*is_secure*/ true);
|
||||
let call_id = config
|
||||
.attribution
|
||||
.as_ref()
|
||||
.zip(proxy.as_ref())
|
||||
.and_then(|(attribution, proxy)| attribution.call_id_for(proxy));
|
||||
(proxy, call_id)
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -136,7 +175,7 @@ impl UpstreamClient {
|
||||
|
||||
pub(crate) fn from_env_proxy(state: Arc<NetworkProxyState>) -> Self {
|
||||
Self::new(
|
||||
ProxyConfig::from_env(state.proxy_private_ips_via_upstream),
|
||||
ProxyConfig::from_env(&state),
|
||||
TargetCheckedTcpConnector::new(state),
|
||||
client_root_certs(),
|
||||
)
|
||||
@@ -158,7 +197,7 @@ impl UpstreamClient {
|
||||
tls_root_store: Arc<rustls::RootCertStore>,
|
||||
) -> Self {
|
||||
Self::new(
|
||||
ProxyConfig::from_env(state.proxy_private_ips_via_upstream),
|
||||
ProxyConfig::from_env(&state),
|
||||
TargetCheckedTcpConnector::new(state),
|
||||
tls_root_store,
|
||||
)
|
||||
@@ -178,7 +217,8 @@ impl UpstreamClient {
|
||||
transport: TargetCheckedTcpConnector,
|
||||
tls_root_store: Arc<rustls::RootCertStore>,
|
||||
) -> Self {
|
||||
let connector = build_http_connector(transport, tls_root_store);
|
||||
let connector =
|
||||
build_http_connector(transport, tls_root_store, proxy_config.attribution.clone());
|
||||
Self {
|
||||
connector,
|
||||
proxy_config,
|
||||
@@ -210,7 +250,14 @@ impl Service<Request<Body>> for UpstreamClient {
|
||||
),
|
||||
None => info!("HTTP upstream route selected (target={authority}, route=direct)"),
|
||||
}
|
||||
req.headers_mut().remove(TOOL_CALL_HEADER);
|
||||
if let Some(proxy) = proxy {
|
||||
if request_context.is_some_and(|ctx| ctx.protocol == rama_net::Protocol::HTTP)
|
||||
&& let Some(attribution) = &self.proxy_config.attribution
|
||||
&& let Some(id) = attribution.call_id_for(&proxy)
|
||||
{
|
||||
req.headers_mut().insert(TOOL_CALL_HEADER, id);
|
||||
}
|
||||
req.extensions_mut().insert(proxy);
|
||||
}
|
||||
|
||||
@@ -258,16 +305,36 @@ impl Service<Request<Body>> for UpstreamClient {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn upstream_proxy_connector(
|
||||
transport: TargetCheckedTcpConnector,
|
||||
connect_call_id: Option<HeaderValue>,
|
||||
) -> HttpProxyConnector<TargetCheckedTcpConnector> {
|
||||
let mut connector = HttpProxyConnector::optional(transport);
|
||||
if let Some(id) = connect_call_id {
|
||||
connector.set_custom_header(TOOL_CALL_HEADER, id);
|
||||
}
|
||||
connector
|
||||
}
|
||||
|
||||
fn build_http_connector(
|
||||
transport: TargetCheckedTcpConnector,
|
||||
tls_root_store: Arc<rustls::RootCertStore>,
|
||||
attribution: Option<ToolCallAttribution>,
|
||||
) -> BoxService<
|
||||
Request<Body>,
|
||||
EstablishedClientConnection<HttpClientService<Body>, Request<Body>>,
|
||||
BoxError,
|
||||
> {
|
||||
ensure_rustls_crypto_provider();
|
||||
let proxy = HttpProxyConnectorLayer::optional().into_layer(transport);
|
||||
let proxy = service_fn(move |req: Request<Body>| {
|
||||
// Rama uses custom connector headers only on its outbound CONNECT.
|
||||
let call_id = attribution
|
||||
.as_ref()
|
||||
.zip(req.extensions().get::<ProxyAddress>())
|
||||
.and_then(|(attribution, proxy)| attribution.call_id_for(proxy));
|
||||
let connector = upstream_proxy_connector(transport.clone(), call_id);
|
||||
async move { connector.serve(req).await }
|
||||
});
|
||||
let client_config = rustls::ClientConfig::builder_with_protocol_versions(rustls::ALL_VERSIONS)
|
||||
.with_root_certificates(tls_root_store)
|
||||
.with_no_client_auth();
|
||||
|
||||
@@ -159,3 +159,152 @@ fn private_ip_upstream_routing_is_opt_in() {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn capture_proxy_request(
|
||||
uri: String,
|
||||
listener: TcpListener,
|
||||
proxy_config: ProxyConfig,
|
||||
) -> String {
|
||||
let network = NetworkProxyConfig {
|
||||
allow_local_binding: Some(true),
|
||||
..NetworkProxyConfig::default()
|
||||
};
|
||||
let state = Arc::new(network_proxy_state_for_policy(network));
|
||||
let client = UpstreamClient::new(
|
||||
proxy_config,
|
||||
TargetCheckedTcpConnector::new(state),
|
||||
client_root_certs(),
|
||||
);
|
||||
let request = Request::builder()
|
||||
.uri(uri)
|
||||
.header(TOOL_CALL_HEADER, "from-command")
|
||||
.body(Body::empty())
|
||||
.unwrap();
|
||||
|
||||
let capture = async {
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
let mut bytes = Vec::new();
|
||||
while !bytes.ends_with(b"\r\n\r\n") {
|
||||
bytes.push(socket.read_u8().await.unwrap());
|
||||
assert!(bytes.len() < 8192);
|
||||
}
|
||||
socket
|
||||
.write_all(
|
||||
b"HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\nConnection: close\r\n\r\n",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
String::from_utf8(bytes).unwrap()
|
||||
};
|
||||
let (wire, _) = tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
tokio::join!(capture, client.serve(request))
|
||||
})
|
||||
.await
|
||||
.expect("request reached proxy or direct destination");
|
||||
wire
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tool_call_header_is_scoped_to_the_configured_upstream_and_execution() {
|
||||
async fn exchange(scheme: &str, call_id: &'static str) -> String {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let proxy = ProxyAddress::try_from(addr.to_string()).unwrap();
|
||||
let endpoint = ProxyAddress::try_from(format!("http://{addr}")).unwrap();
|
||||
let config = ProxyConfig {
|
||||
http: Some(proxy.clone()),
|
||||
https: Some(proxy),
|
||||
attribution: Some(ToolCallAttribution {
|
||||
endpoint,
|
||||
call_id: HeaderValue::from_static(call_id),
|
||||
}),
|
||||
..ProxyConfig::default()
|
||||
};
|
||||
capture_proxy_request(format!("{scheme}://same.example/path"), listener, config).await
|
||||
}
|
||||
|
||||
let (http, https) = tokio::join!(exchange("http", "call-A"), exchange("https", "exec-B"));
|
||||
assert!(http.starts_with("GET "));
|
||||
assert!(https.starts_with("CONNECT same.example:443 "));
|
||||
for (wire, expected) in [(http, "call-A"), (https, "exec-B")] {
|
||||
let ids: Vec<_> = wire
|
||||
.lines()
|
||||
.filter_map(|line| line.split_once(": "))
|
||||
.filter(|(name, _)| name.eq_ignore_ascii_case(TOOL_CALL_HEADER.as_str()))
|
||||
.map(|(_, value)| value)
|
||||
.collect();
|
||||
assert_eq!(ids, vec![expected]);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn direct_and_other_proxy_requests_strip_tool_call_headers() {
|
||||
let trusted = ProxyAddress::try_from("http://other-proxy.test:8889").unwrap();
|
||||
for (scheme, use_proxy, enabled) in [
|
||||
("http", false, true),
|
||||
("http", true, false),
|
||||
("http", true, true),
|
||||
("https", true, true),
|
||||
] {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let proxy = ProxyAddress::try_from(format!("http://{address}")).unwrap();
|
||||
let uri = if use_proxy {
|
||||
format!("{scheme}://same.example/path")
|
||||
} else {
|
||||
format!("http://{address}/path")
|
||||
};
|
||||
let config = ProxyConfig {
|
||||
http: use_proxy.then(|| proxy.clone()),
|
||||
https: use_proxy.then_some(proxy),
|
||||
attribution: enabled.then(|| ToolCallAttribution {
|
||||
endpoint: trusted.clone(),
|
||||
call_id: HeaderValue::from_static("call-A"),
|
||||
}),
|
||||
..ProxyConfig::default()
|
||||
};
|
||||
let wire = capture_proxy_request(uri, listener, config).await;
|
||||
assert!(
|
||||
!wire
|
||||
.to_ascii_lowercase()
|
||||
.contains(TOOL_CALL_HEADER.as_str())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn split_upstreams_attribute_only_the_selected_hop_regardless_of_origin_port() {
|
||||
for (uri, secure) in [
|
||||
("http://same.example:443/path", false),
|
||||
("https://same.example:80/path", true),
|
||||
] {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let active =
|
||||
ProxyAddress::try_from(format!("http://{}", listener.local_addr().unwrap())).unwrap();
|
||||
let other = ProxyAddress::try_from("http://unused-proxy.test:8889").unwrap();
|
||||
let (http, https) = if secure {
|
||||
(other, active)
|
||||
} else {
|
||||
(active, other)
|
||||
};
|
||||
let config = ProxyConfig {
|
||||
http: Some(http),
|
||||
https: Some(https.clone()),
|
||||
attribution: Some(ToolCallAttribution {
|
||||
endpoint: https,
|
||||
call_id: HeaderValue::from_static("exec-A"),
|
||||
}),
|
||||
..ProxyConfig::default()
|
||||
};
|
||||
let wire = capture_proxy_request(uri.to_string(), listener, config).await;
|
||||
assert_eq!(
|
||||
wire.to_ascii_lowercase()
|
||||
.contains(TOOL_CALL_HEADER.as_str()),
|
||||
secure
|
||||
);
|
||||
if secure {
|
||||
assert!(wire.starts_with("CONNECT same.example:80 "));
|
||||
assert!(wire.contains("exec-A"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user