http-client: extract shared HTTP transport

2026-11-05 20:10:33 +00:00 · 2026-07-06 17:42:44 -07:00
parent 7226904ed1
commit 4c995310e6
30 changed files with 187 additions and 128 deletions
@@ -2392,33 +2392,12 @@ dependencies = [
 name = "codex-client"
 version = "0.0.0"
 dependencies = [
- "bytes",
- "codex-utils-cargo-bin",
- "codex-utils-rustls-provider",
+ "codex-http-client",
 "eventsource-stream",
 "futures",
 "http 1.4.0",
- "opentelemetry",
- "opentelemetry_sdk",
- "pretty_assertions",
 "rand 0.9.3",
- "rcgen",
- "reqwest 0.12.28",
- "rustls",
- "rustls-native-certs",
- "rustls-pki-types",
- "serde",
- "serde_json",
- "sha2 0.10.9",
- "system-configuration",
- "tempfile",
- "thiserror 2.0.18",
 "tokio",
- "tracing",
- "tracing-opentelemetry",
- "tracing-subscriber",
- "windows-sys 0.52.0",
- "zstd 0.13.3",
 ]

 [[package]]
@@ -3214,6 +3193,37 @@ dependencies = [
 "uuid",
 ]

+[[package]]
+name = "codex-http-client"
+version = "0.0.0"
+dependencies = [
+ "bytes",
+ "codex-utils-cargo-bin",
+ "codex-utils-rustls-provider",
+ "futures",
+ "http 1.4.0",
+ "opentelemetry",
+ "opentelemetry_sdk",
+ "pretty_assertions",
+ "rcgen",
+ "reqwest 0.12.28",
+ "rustls",
+ "rustls-native-certs",
+ "rustls-pki-types",
+ "serde",
+ "serde_json",
+ "sha2 0.10.9",
+ "system-configuration",
+ "tempfile",
+ "thiserror 2.0.18",
+ "tokio",
+ "tracing",
+ "tracing-opentelemetry",
+ "tracing-subscriber",
+ "windows-sys 0.52.0",
+ "zstd 0.13.3",
+]
+
 [[package]]
 name = "codex-image-generation-extension"
 version = "0.0.0"
@@ -41,6 +41,7 @@ members = [
    "core-plugins",
    "core-skills",
    "hooks",
+    "http-client",
    "secrets",
    "exec",
    "file-system",
@@ -164,6 +165,7 @@ codex-cloud-tasks-mock-client = { path = "cloud-tasks-mock-client" }
 codex-code-mode = { path = "code-mode" }
 codex-code-mode-protocol = { path = "code-mode-protocol" }
 codex-home = { path = "codex-home" }
+codex-http-client = { path = "http-client" }
 codex-config = { path = "config" }
 codex-connectors = { path = "connectors" }
 codex-connectors-extension = { path = "ext/connectors" }
@@ -2,6 +2,5 @@ load("//:defs.bzl", "codex_rust_crate")

 codex_rust_crate(
    name = "codex-client",
-    compile_data = glob(["tests/fixtures/**"]),
    crate_name = "codex_client",
 )
@@ -5,47 +5,16 @@ name = "codex-client"
 version.workspace = true

 [dependencies]
-bytes = { workspace = true }
+codex-http-client = { workspace = true }
 eventsource-stream = { workspace = true }
 futures = { workspace = true }
 http = { workspace = true }
-opentelemetry = { workspace = true }
 rand = { workspace = true }
-reqwest = { workspace = true, features = ["json", "rustls-tls-native-roots", "stream"] }
-rustls = { workspace = true }
-rustls-native-certs = { workspace = true }
-rustls-pki-types = { workspace = true }
-serde = { workspace = true, features = ["derive"] }
-serde_json = { workspace = true }
-thiserror = { workspace = true }
 tokio = { workspace = true, features = ["macros", "rt", "time", "sync"] }
-tracing = { workspace = true }
-tracing-opentelemetry = { workspace = true }
-codex-utils-rustls-provider = { workspace = true }
-zstd = { workspace = true }
-
-[target.'cfg(any(target_os = "windows", target_os = "macos"))'.dependencies]
-sha2 = { workspace = true }
-
-[target.'cfg(target_os = "macos")'.dependencies]
-system-configuration = { workspace = true }
-
-[target.'cfg(target_os = "windows")'.dependencies]
-windows-sys = { version = "0.52", features = [
-    "Win32_Foundation",
-    "Win32_Networking_WinHttp",
-] }

 [lints]
 workspace = true

-[dev-dependencies]
-codex-utils-cargo-bin = { workspace = true }
-opentelemetry_sdk = { workspace = true }
-pretty_assertions = { workspace = true }
-rcgen = { workspace = true }
-tempfile = { workspace = true }
-tracing-subscriber = { workspace = true }
-
 [lib]
 doctest = false
+test = false
@@ -1,8 +1,8 @@
 # codex-client

-Generic transport layer that wraps HTTP requests, retries, and streaming primitives without any Codex/OpenAI awareness.
+Higher-level request policy layered on `codex-http-client` without any Codex/OpenAI API awareness.

- Defines `HttpTransport` and a default `ReqwestTransport` plus thin `Request`/`Response` types.
 - Provides retry utilities (`RetryPolicy`, `RetryOn`, `run_with_retry`, `backoff`) that callers plug into for unary and streaming calls.
 - Supplies the `sse_stream` helper to turn byte streams into raw SSE `data:` frames with idle timeouts and surfaced stream errors.
- Consumed by higher-level crates like `codex-api`; it stays neutral on endpoints, headers, or API-specific error shapes.
+- Defines the request telemetry callback used by higher-level clients.
+- Re-exports the low-level HTTP types temporarily so consumers can migrate to `codex-http-client` incrementally.
@@ -1,49 +1,13 @@
-mod chatgpt_cloudflare_cookies;
-mod chatgpt_hosts;
-mod custom_ca;
-mod default_client;
-mod error;
-mod outbound_proxy;
-mod request;
 mod retry;
 mod sse;
 mod telemetry;
-mod transport;

-pub use crate::chatgpt_cloudflare_cookies::with_chatgpt_cloudflare_cookie_store;
-pub use crate::chatgpt_hosts::is_allowed_chatgpt_host;
-pub use crate::custom_ca::BuildCustomCaTransportError;
-/// Test-only subprocess hook for custom CA coverage.
-///
-/// This stays public only so the `custom_ca_probe` binary target can reuse the shared helper. It
-/// is hidden from normal docs because ordinary callers should use
-/// [`build_reqwest_client_with_custom_ca`] instead.
-#[doc(hidden)]
-pub use crate::custom_ca::build_reqwest_client_for_subprocess_tests;
-pub use crate::custom_ca::build_reqwest_client_with_custom_ca;
-pub use crate::custom_ca::maybe_build_rustls_client_config_with_custom_ca;
-pub use crate::default_client::CodexHttpClient;
-pub use crate::default_client::CodexRequestBuilder;
-pub use crate::error::StreamError;
-pub use crate::error::TransportError;
-pub use crate::outbound_proxy::BuildRouteAwareHttpClientError;
-pub use crate::outbound_proxy::ClientRouteClass;
-pub use crate::outbound_proxy::OutboundProxyConfig;
-pub use crate::outbound_proxy::RouteFailureClass;
-pub use crate::outbound_proxy::build_reqwest_client_for_route;
-pub use crate::request::EncodedJsonBody;
-pub use crate::request::PreparedRequestBody;
-pub use crate::request::Request;
-pub use crate::request::RequestBody;
-pub use crate::request::RequestCompression;
-pub use crate::request::Response;
 pub use crate::retry::RetryOn;
 pub use crate::retry::RetryPolicy;
 pub use crate::retry::backoff;
 pub use crate::retry::run_with_retry;
 pub use crate::sse::sse_stream;
 pub use crate::telemetry::RequestTelemetry;
-pub use crate::transport::ByteStream;
-pub use crate::transport::HttpTransport;
-pub use crate::transport::ReqwestTransport;
-pub use crate::transport::StreamResponse;
+pub use codex_http_client::HttpClient as CodexHttpClient;
+pub use codex_http_client::RequestBuilder as CodexRequestBuilder;
+pub use codex_http_client::*;
@@ -1,5 +1,5 @@
-use crate::error::TransportError;
-use crate::request::Request;
+use codex_http_client::Request;
+use codex_http_client::TransportError;
 use rand::Rng;
 use std::future::Future;
 use std::time::Duration;
@@ -1,5 +1,5 @@
-use crate::error::StreamError;
-use crate::transport::ByteStream;
+use codex_http_client::ByteStream;
+use codex_http_client::StreamError;
 use eventsource_stream::Eventsource;
 use futures::StreamExt;
 use tokio::sync::mpsc;
@@ -1,4 +1,4 @@
-use crate::error::TransportError;
+use codex_http_client::TransportError;
 use http::StatusCode;
 use std::time::Duration;

@@ -0,0 +1,7 @@
+load("//:defs.bzl", "codex_rust_crate")
+
+codex_rust_crate(
+    name = "http-client",
+    compile_data = glob(["tests/fixtures/**"]),
+    crate_name = "codex_http_client",
+)
@@ -0,0 +1,49 @@
+[package]
+edition.workspace = true
+license.workspace = true
+name = "codex-http-client"
+version.workspace = true
+
+[dependencies]
+bytes = { workspace = true }
+codex-utils-rustls-provider = { workspace = true }
+futures = { workspace = true }
+http = { workspace = true }
+opentelemetry = { workspace = true }
+reqwest = { workspace = true, features = ["json", "rustls-tls-native-roots", "stream"] }
+rustls = { workspace = true }
+rustls-native-certs = { workspace = true }
+rustls-pki-types = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true }
+thiserror = { workspace = true }
+tokio = { workspace = true, features = ["macros", "rt", "time", "sync"] }
+tracing = { workspace = true }
+tracing-opentelemetry = { workspace = true }
+zstd = { workspace = true }
+
+[target.'cfg(any(target_os = "windows", target_os = "macos"))'.dependencies]
+sha2 = { workspace = true }
+
+[target.'cfg(target_os = "macos")'.dependencies]
+system-configuration = { workspace = true }
+
+[target.'cfg(target_os = "windows")'.dependencies]
+windows-sys = { version = "0.52", features = [
+    "Win32_Foundation",
+    "Win32_Networking_WinHttp",
+] }
+
+[lints]
+workspace = true
+
+[dev-dependencies]
+codex-utils-cargo-bin = { workspace = true }
+opentelemetry_sdk = { workspace = true }
+pretty_assertions = { workspace = true }
+rcgen = { workspace = true }
+tempfile = { workspace = true }
+tracing-subscriber = { workspace = true }
+
+[lib]
+doctest = false
@@ -0,0 +1,9 @@
+# codex-http-client
+
+Low-level HTTP transport shared by Codex crates.
+
+- Defines the request, response, streaming, and transport types used for outbound HTTP calls.
+- Owns the `reqwest` implementation, custom CA handling, and ChatGPT Cloudflare cookie policy.
+- Resolves system, PAC/WPAD, environment, and direct proxy routes for supported clients.
+
+Higher-level retry, SSE, and request telemetry policy remains in `codex-client`.
@@ -10,9 +10,9 @@
 //! - error messages guide users when CA files are invalid.
 //! - optional HTTPS probes can complete a request through the constructed client.
 //!
-//! The detailed explanation of what "hermetic" means here lives in `codex_client::custom_ca`.
+//! The detailed explanation of what "hermetic" means here lives in `codex_http_client::custom_ca`.
 //! This binary exists so the tests can exercise
-//! [`codex_client::build_reqwest_client_for_subprocess_tests`] in a separate process without
+//! [`codex_http_client::build_reqwest_client_for_subprocess_tests`] in a separate process without
 //! duplicating client-construction logic.

 use std::env;
@@ -69,11 +69,11 @@ fn build_probe_client(
    if let Some(proxy_url) = proxy_url {
        let proxy = reqwest::Proxy::https(proxy_url)
            .map_err(|error| format!("failed to configure probe proxy {proxy_url}: {error}"))?;
-        return codex_client::build_reqwest_client_with_custom_ca(builder.proxy(proxy))
+        return codex_http_client::build_reqwest_client_with_custom_ca(builder.proxy(proxy))
            .map_err(|error| error.to_string());
    }

-    codex_client::build_reqwest_client_for_subprocess_tests(builder)
+    codex_http_client::build_reqwest_client_for_subprocess_tests(builder)
        .map_err(|error| error.to_string())
 }

@@ -7,7 +7,7 @@ use reqwest::header::HeaderValue;

 use crate::chatgpt_hosts::is_allowed_chatgpt_host;

-// WARNING: this store is process-global and may be shared across auth contexts.
+// WARNING: this HTTP cookie store is process-global and may be shared across auth contexts.
 // It must only ever contain Cloudflare infrastructure cookies. Never extend this
 // store to persist ChatGPT account, session, auth, or other user-specific cookie
 // data.
@@ -1,5 +1,5 @@
 /// Returns whether `host` is one of the ChatGPT hosts Codex is allowed to treat
-/// as first-party ChatGPT traffic.
+/// as first-party ChatGPT HTTP traffic.
 pub fn is_allowed_chatgpt_host(host: &str) -> bool {
    const EXACT_HOSTS: &[&str] = &["chatgpt.com", "chat.openai.com", "chatgpt-staging.com"];
    const SUBDOMAIN_SUFFIXES: &[&str] = &[".chatgpt.com", ".chatgpt-staging.com"];
@@ -1,4 +1,4 @@
-//! Custom CA handling for Codex outbound HTTP and websocket clients.
+//! Custom CA handling shared by Codex outbound HTTP and websocket clients.
 //!
 //! Codex constructs outbound reqwest clients and secure websocket connections in a few crates, but
 //! they all need the same trust-store policy when enterprise proxies or gateways intercept TLS.
@@ -14,47 +14,47 @@ use tracing::Span;
 use tracing_opentelemetry::OpenTelemetrySpanExt;

 #[derive(Clone, Debug)]
-pub struct CodexHttpClient {
+pub struct HttpClient {
    inner: reqwest::Client,
 }

-impl CodexHttpClient {
+impl HttpClient {
    pub fn new(inner: reqwest::Client) -> Self {
        Self { inner }
    }

-    pub fn get<U>(&self, url: U) -> CodexRequestBuilder
+    pub fn get<U>(&self, url: U) -> RequestBuilder
    where
        U: IntoUrl,
    {
        self.request(Method::GET, url)
    }

-    pub fn post<U>(&self, url: U) -> CodexRequestBuilder
+    pub fn post<U>(&self, url: U) -> RequestBuilder
    where
        U: IntoUrl,
    {
        self.request(Method::POST, url)
    }

-    pub fn request<U>(&self, method: Method, url: U) -> CodexRequestBuilder
+    pub fn request<U>(&self, method: Method, url: U) -> RequestBuilder
    where
        U: IntoUrl,
    {
        let url_str = url.as_str().to_string();
-        CodexRequestBuilder::new(self.inner.request(method.clone(), url), method, url_str)
+        RequestBuilder::new(self.inner.request(method.clone(), url), method, url_str)
    }
 }

 #[must_use = "requests are not sent unless `send` is awaited"]
 #[derive(Debug)]
-pub struct CodexRequestBuilder {
+pub struct RequestBuilder {
    builder: reqwest::RequestBuilder,
    method: Method,
    url: String,
 }

-impl CodexRequestBuilder {
+impl RequestBuilder {
    fn new(builder: reqwest::RequestBuilder, method: Method, url: String) -> Self {
        Self {
            builder,
@@ -1,3 +1,5 @@
+//! Errors returned by the shared Codex HTTP transport.
+
 use http::HeaderMap;
 use http::StatusCode;
 use thiserror::Error;
@@ -0,0 +1,40 @@
+mod chatgpt_cloudflare_cookies;
+mod chatgpt_hosts;
+mod custom_ca;
+mod default_client;
+mod error;
+mod outbound_proxy;
+mod request;
+mod transport;
+
+pub use crate::chatgpt_cloudflare_cookies::with_chatgpt_cloudflare_cookie_store;
+pub use crate::chatgpt_hosts::is_allowed_chatgpt_host;
+pub use crate::custom_ca::BuildCustomCaTransportError;
+/// Test-only subprocess hook for custom CA coverage.
+///
+/// This stays public only so the `custom_ca_probe` binary target can reuse the shared helper. It
+/// is hidden from normal docs because ordinary callers should use
+/// [`build_reqwest_client_with_custom_ca`] instead.
+#[doc(hidden)]
+pub use crate::custom_ca::build_reqwest_client_for_subprocess_tests;
+pub use crate::custom_ca::build_reqwest_client_with_custom_ca;
+pub use crate::custom_ca::maybe_build_rustls_client_config_with_custom_ca;
+pub use crate::default_client::HttpClient;
+pub use crate::default_client::RequestBuilder;
+pub use crate::error::StreamError;
+pub use crate::error::TransportError;
+pub use crate::outbound_proxy::BuildRouteAwareHttpClientError;
+pub use crate::outbound_proxy::ClientRouteClass;
+pub use crate::outbound_proxy::OutboundProxyConfig;
+pub use crate::outbound_proxy::RouteFailureClass;
+pub use crate::outbound_proxy::build_reqwest_client_for_route;
+pub use crate::request::EncodedJsonBody;
+pub use crate::request::PreparedRequestBody;
+pub use crate::request::Request;
+pub use crate::request::RequestBody;
+pub use crate::request::RequestCompression;
+pub use crate::request::Response;
+pub use crate::transport::ByteStream;
+pub use crate::transport::HttpTransport;
+pub use crate::transport::ReqwestTransport;
+pub use crate::transport::StreamResponse;
@@ -1,4 +1,4 @@
-//! Conservative outbound proxy selection for resolver-aware clients.
+//! Conservative outbound proxy selection for resolver-aware HTTP clients.
 //!
 //! When enabled, platform system discovery is tried first, explicit environment
 //! proxies are the fallback, and the final fallback is a direct connection.
@@ -1,3 +1,5 @@
+//! macOS system proxy resolution through SystemConfiguration and CFNetwork.
+
 use std::ffi::c_void;
 use std::ptr;
 use std::time::Duration;
@@ -1,3 +1,5 @@
+//! Windows system proxy resolution through WinHTTP.
+
 use std::ffi::c_void;
 use std::ptr;

@@ -1,3 +1,5 @@
+//! Windows proxy parsing tests.
+
 use super::*;

 #[test]
@@ -1,3 +1,5 @@
+//! Shared outbound proxy policy tests.
+
 use super::*;
 use pretty_assertions::assert_eq;
 use std::io::Read;
@@ -1,7 +1,7 @@
 use bytes::Bytes;
+use http::HeaderMap;
+use http::HeaderValue;
 use http::Method;
-use reqwest::header::HeaderMap;
-use reqwest::header::HeaderValue;
 use serde::Serialize;
 use serde_json::Value;
 use std::time::Duration;
@@ -121,7 +121,7 @@ impl Request {
            Some(RequestBody::Json(_) | RequestBody::EncodedJson(_))
        );
        let trace_bytes = if self.compression != RequestCompression::None
-            && tracing::enabled!(target: "codex_client::transport", tracing::Level::TRACE)
+            && tracing::enabled!(target: "codex_http_client::transport", tracing::Level::TRACE)
        {
            match self.body.as_ref() {
                Some(RequestBody::Json(body)) => Some(Bytes::from(
@@ -1,5 +1,5 @@
-use crate::default_client::CodexHttpClient;
-use crate::default_client::CodexRequestBuilder;
+use crate::default_client::HttpClient;
+use crate::default_client::RequestBuilder;
 use crate::error::TransportError;
 use crate::request::Request;
 use crate::request::RequestBody;
@@ -35,17 +35,17 @@ pub trait HttpTransport: Send + Sync {

 #[derive(Clone, Debug)]
 pub struct ReqwestTransport {
-    client: CodexHttpClient,
+    client: HttpClient,
 }

 impl ReqwestTransport {
    pub fn new(client: reqwest::Client) -> Self {
        Self {
-            client: CodexHttpClient::new(client),
+            client: HttpClient::new(client),
        }
    }

-    fn build(&self, req: Request) -> Result<CodexRequestBuilder, TransportError> {
+    fn build(&self, req: Request) -> Result<RequestBuilder, TransportError> {
        let prepared = req.prepare_body_for_send().map_err(TransportError::Build)?;

        let Request {
@@ -3,7 +3,7 @@
 //!
 //! These tests intentionally run through `custom_ca_probe` and
 //! `build_reqwest_client_for_subprocess_tests` instead of calling the helper in-process. The
-//! detailed explanation of what "hermetic" means here lives in `codex_client::custom_ca`; these
+//! detailed explanation of what "hermetic" means here lives in `codex_http_client::custom_ca`; these
 //! tests add the process-level half of that contract by scrubbing inherited CA environment
 //! variables before each subprocess launch. Most assertions here cover CA file selection, PEM
 //! parsing, and user-facing errors. The HTTPS probes go further and perform real POSTs against