Add audio transcription to the backend client (#62142)

## What changed

Add `Client::transcribe_audio` to send a single audio clip to the
`/transcribe` dictation endpoint as an authenticated multipart upload
and return the transcript text. Accept WAV, MP3, M4A, WebM, and Ogg,
and cap the response body at 64 KiB.

## Testing

Add a mock-server test that checks the bearer token, the exact multipart
body for an M4A clip, and the returned transcript text.

GitOrigin-RevId: 50ab00d932aefda942c730a8b037b53d4f5003c2
This commit is contained in:
Ax
2026-11-18 00:33:23 +00:00
committed by copyberry
parent 2ae806bdf7
commit 83c46d0a58
3 changed files with 77 additions and 1 deletions
+1 -1
View File
@@ -29,10 +29,10 @@ codex-runtime-agent-instance = { workspace = true }
codex-protocol = { workspace = true }
codex-response-debug-context = { workspace = true }
tokio = { workspace = true, features = ["time"] }
uuid = { workspace = true, features = ["v4"] }
[dev-dependencies]
base64 = { workspace = true }
uuid = { workspace = true }
pretty_assertions = "1"
tempfile = { workspace = true }
tokio = { workspace = true, features = ["macros", "rt"] }
+36
View File
@@ -509,6 +509,42 @@ impl Client {
}
}
/// Submits a single clip as the dictation endpoint's multipart file field.
pub async fn transcribe_audio(&self, mime: &str, bytes: &[u8]) -> Result<String> {
let extension = match mime {
"audio/wav" => "wav",
"audio/mpeg" => "mp3",
"audio/mp4" => "m4a",
"audio/webm" => "webm",
"audio/ogg" => "ogg",
_ => anyhow::bail!("unsupported audio format"),
};
let url = format!("{}/transcribe", self.base_url);
self.auth.validate_destination(&self.base_url, &url)?;
let boundary = uuid::Uuid::new_v4();
let mut body = format!("--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"audio.{extension}\"\r\nContent-Type: {mime}\r\n\r\n").into_bytes();
body.extend_from_slice(bytes);
body.extend_from_slice(format!("\r\n--{boundary}--\r\n").as_bytes());
let response = self
.request(Method::POST, &url)
.headers(self.headers().await?)
.header(
CONTENT_TYPE,
format!("multipart/form-data; boundary={boundary}"),
)
.body(body)
.send()
.await?;
anyhow::ensure!(response.status().is_success(), "transcription failed");
#[derive(serde::Deserialize)]
struct Transcript {
text: String,
}
const MAX_RESPONSE_BYTES: usize = 64 * 1024;
let body = codex_http_client::bounded_response_bytes(response, MAX_RESPONSE_BYTES).await?;
Ok(serde_json::from_slice::<Transcript>(&body)?.text)
}
pub async fn get_rate_limits(&self) -> Result<RateLimitSnapshot> {
let snapshots = self.get_rate_limits_many().await?;
let preferred = snapshots
@@ -20,6 +20,46 @@ use wiremock::matchers::path;
use super::*;
#[tokio::test]
async fn transcribe_audio_posts_multipart_file_and_returns_text() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/backend-api/transcribe"))
.and(header("authorization", "Bearer test-token"))
.respond_with(
ResponseTemplate::new(200).set_body_json(serde_json::json!({"text": "Hello there."})),
)
.expect(1)
.mount(&server)
.await;
let client = Client::new(
format!("{}/backend-api", server.uri()),
HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
)
.with_bearer_token("test-token".to_string())
.unwrap();
let audio = b"\0\xff\x80\r\n";
let text = client.transcribe_audio("audio/mp4", audio).await.unwrap();
assert_eq!(text, "Hello there.");
let request = server.received_requests().await.unwrap().pop().unwrap();
let boundary = request.headers[CONTENT_TYPE]
.to_str()
.unwrap()
.strip_prefix("multipart/form-data; boundary=")
.unwrap();
let expected_body = [
format!("--{boundary}\r\n").as_bytes(),
b"Content-Disposition: form-data; name=\"file\"; filename=\"audio.m4a\"\r\n",
b"Content-Type: audio/mp4\r\n\r\n",
audio,
format!("\r\n--{boundary}--\r\n").as_bytes(),
]
.concat();
assert_eq!(request.body, expected_body);
}
#[tokio::test]
async fn client_preserves_supplied_factory_and_typed_policy_denial() {
let controller = codex_http_client::NetworkPolicyController::default();