From 83c46d0a58950c63f1ceff9a79ccb31bc612ec07 Mon Sep 17 00:00:00 2001 From: Ax Date: Wed, 18 Nov 2026 00:00:32 +0000 Subject: [PATCH] Add audio transcription to the backend client (#62142) ## What changed Add `Client::transcribe_audio` to send a single audio clip to the `/transcribe` dictation endpoint as an authenticated multipart upload and return the transcript text. Accept WAV, MP3, M4A, WebM, and Ogg, and cap the response body at 64 KiB. ## Testing Add a mock-server test that checks the bearer token, the exact multipart body for an M4A clip, and the returned transcript text. GitOrigin-RevId: 50ab00d932aefda942c730a8b037b53d4f5003c2 --- codex-rs/backend-client/Cargo.toml | 2 +- codex-rs/backend-client/src/client.rs | 36 +++++++++++++++++ .../src/client_request_tests.rs | 40 +++++++++++++++++++ 3 files changed, 77 insertions(+), 1 deletion(-) diff --git a/codex-rs/backend-client/Cargo.toml b/codex-rs/backend-client/Cargo.toml index d739f26812..786704a253 100644 --- a/codex-rs/backend-client/Cargo.toml +++ b/codex-rs/backend-client/Cargo.toml @@ -29,10 +29,10 @@ codex-runtime-agent-instance = { workspace = true } codex-protocol = { workspace = true } codex-response-debug-context = { workspace = true } tokio = { workspace = true, features = ["time"] } +uuid = { workspace = true, features = ["v4"] } [dev-dependencies] base64 = { workspace = true } -uuid = { workspace = true } pretty_assertions = "1" tempfile = { workspace = true } tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/codex-rs/backend-client/src/client.rs b/codex-rs/backend-client/src/client.rs index 6dafe2e1af..5af484cbbc 100644 --- a/codex-rs/backend-client/src/client.rs +++ b/codex-rs/backend-client/src/client.rs @@ -509,6 +509,42 @@ impl Client { } } + /// Submits a single clip as the dictation endpoint's multipart file field. + pub async fn transcribe_audio(&self, mime: &str, bytes: &[u8]) -> Result { + let extension = match mime { + "audio/wav" => "wav", + "audio/mpeg" => "mp3", + "audio/mp4" => "m4a", + "audio/webm" => "webm", + "audio/ogg" => "ogg", + _ => anyhow::bail!("unsupported audio format"), + }; + let url = format!("{}/transcribe", self.base_url); + self.auth.validate_destination(&self.base_url, &url)?; + let boundary = uuid::Uuid::new_v4(); + let mut body = format!("--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"audio.{extension}\"\r\nContent-Type: {mime}\r\n\r\n").into_bytes(); + body.extend_from_slice(bytes); + body.extend_from_slice(format!("\r\n--{boundary}--\r\n").as_bytes()); + let response = self + .request(Method::POST, &url) + .headers(self.headers().await?) + .header( + CONTENT_TYPE, + format!("multipart/form-data; boundary={boundary}"), + ) + .body(body) + .send() + .await?; + anyhow::ensure!(response.status().is_success(), "transcription failed"); + #[derive(serde::Deserialize)] + struct Transcript { + text: String, + } + const MAX_RESPONSE_BYTES: usize = 64 * 1024; + let body = codex_http_client::bounded_response_bytes(response, MAX_RESPONSE_BYTES).await?; + Ok(serde_json::from_slice::(&body)?.text) + } + pub async fn get_rate_limits(&self) -> Result { let snapshots = self.get_rate_limits_many().await?; let preferred = snapshots diff --git a/codex-rs/backend-client/src/client_request_tests.rs b/codex-rs/backend-client/src/client_request_tests.rs index 9583b62f7e..7f0e879b99 100644 --- a/codex-rs/backend-client/src/client_request_tests.rs +++ b/codex-rs/backend-client/src/client_request_tests.rs @@ -20,6 +20,46 @@ use wiremock::matchers::path; use super::*; +#[tokio::test] +async fn transcribe_audio_posts_multipart_file_and_returns_text() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/backend-api/transcribe")) + .and(header("authorization", "Bearer test-token")) + .respond_with( + ResponseTemplate::new(200).set_body_json(serde_json::json!({"text": "Hello there."})), + ) + .expect(1) + .mount(&server) + .await; + let client = Client::new( + format!("{}/backend-api", server.uri()), + HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault), + ) + .with_bearer_token("test-token".to_string()) + .unwrap(); + + let audio = b"\0\xff\x80\r\n"; + let text = client.transcribe_audio("audio/mp4", audio).await.unwrap(); + assert_eq!(text, "Hello there."); + + let request = server.received_requests().await.unwrap().pop().unwrap(); + let boundary = request.headers[CONTENT_TYPE] + .to_str() + .unwrap() + .strip_prefix("multipart/form-data; boundary=") + .unwrap(); + let expected_body = [ + format!("--{boundary}\r\n").as_bytes(), + b"Content-Disposition: form-data; name=\"file\"; filename=\"audio.m4a\"\r\n", + b"Content-Type: audio/mp4\r\n\r\n", + audio, + format!("\r\n--{boundary}--\r\n").as_bytes(), + ] + .concat(); + assert_eq!(request.body, expected_body); +} + #[tokio::test] async fn client_preserves_supplied_factory_and_typed_policy_denial() { let controller = codex_http_client::NetworkPolicyController::default();