mirror of
https://github.com/openai/codex.git
synced 2026-11-18 22:20:48 +00:00
Add audio transcription to the backend client (#62142)
## What changed Add `Client::transcribe_audio` to send a single audio clip to the `/transcribe` dictation endpoint as an authenticated multipart upload and return the transcript text. Accept WAV, MP3, M4A, WebM, and Ogg, and cap the response body at 64 KiB. ## Testing Add a mock-server test that checks the bearer token, the exact multipart body for an M4A clip, and the returned transcript text. GitOrigin-RevId: 50ab00d932aefda942c730a8b037b53d4f5003c2
This commit is contained in:
@@ -29,10 +29,10 @@ codex-runtime-agent-instance = { workspace = true }
|
||||
codex-protocol = { workspace = true }
|
||||
codex-response-debug-context = { workspace = true }
|
||||
tokio = { workspace = true, features = ["time"] }
|
||||
uuid = { workspace = true, features = ["v4"] }
|
||||
|
||||
[dev-dependencies]
|
||||
base64 = { workspace = true }
|
||||
uuid = { workspace = true }
|
||||
pretty_assertions = "1"
|
||||
tempfile = { workspace = true }
|
||||
tokio = { workspace = true, features = ["macros", "rt"] }
|
||||
|
||||
@@ -509,6 +509,42 @@ impl Client {
|
||||
}
|
||||
}
|
||||
|
||||
/// Submits a single clip as the dictation endpoint's multipart file field.
|
||||
pub async fn transcribe_audio(&self, mime: &str, bytes: &[u8]) -> Result<String> {
|
||||
let extension = match mime {
|
||||
"audio/wav" => "wav",
|
||||
"audio/mpeg" => "mp3",
|
||||
"audio/mp4" => "m4a",
|
||||
"audio/webm" => "webm",
|
||||
"audio/ogg" => "ogg",
|
||||
_ => anyhow::bail!("unsupported audio format"),
|
||||
};
|
||||
let url = format!("{}/transcribe", self.base_url);
|
||||
self.auth.validate_destination(&self.base_url, &url)?;
|
||||
let boundary = uuid::Uuid::new_v4();
|
||||
let mut body = format!("--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"audio.{extension}\"\r\nContent-Type: {mime}\r\n\r\n").into_bytes();
|
||||
body.extend_from_slice(bytes);
|
||||
body.extend_from_slice(format!("\r\n--{boundary}--\r\n").as_bytes());
|
||||
let response = self
|
||||
.request(Method::POST, &url)
|
||||
.headers(self.headers().await?)
|
||||
.header(
|
||||
CONTENT_TYPE,
|
||||
format!("multipart/form-data; boundary={boundary}"),
|
||||
)
|
||||
.body(body)
|
||||
.send()
|
||||
.await?;
|
||||
anyhow::ensure!(response.status().is_success(), "transcription failed");
|
||||
#[derive(serde::Deserialize)]
|
||||
struct Transcript {
|
||||
text: String,
|
||||
}
|
||||
const MAX_RESPONSE_BYTES: usize = 64 * 1024;
|
||||
let body = codex_http_client::bounded_response_bytes(response, MAX_RESPONSE_BYTES).await?;
|
||||
Ok(serde_json::from_slice::<Transcript>(&body)?.text)
|
||||
}
|
||||
|
||||
pub async fn get_rate_limits(&self) -> Result<RateLimitSnapshot> {
|
||||
let snapshots = self.get_rate_limits_many().await?;
|
||||
let preferred = snapshots
|
||||
|
||||
@@ -20,6 +20,46 @@ use wiremock::matchers::path;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn transcribe_audio_posts_multipart_file_and_returns_text() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/backend-api/transcribe"))
|
||||
.and(header("authorization", "Bearer test-token"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200).set_body_json(serde_json::json!({"text": "Hello there."})),
|
||||
)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let client = Client::new(
|
||||
format!("{}/backend-api", server.uri()),
|
||||
HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
|
||||
)
|
||||
.with_bearer_token("test-token".to_string())
|
||||
.unwrap();
|
||||
|
||||
let audio = b"\0\xff\x80\r\n";
|
||||
let text = client.transcribe_audio("audio/mp4", audio).await.unwrap();
|
||||
assert_eq!(text, "Hello there.");
|
||||
|
||||
let request = server.received_requests().await.unwrap().pop().unwrap();
|
||||
let boundary = request.headers[CONTENT_TYPE]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.strip_prefix("multipart/form-data; boundary=")
|
||||
.unwrap();
|
||||
let expected_body = [
|
||||
format!("--{boundary}\r\n").as_bytes(),
|
||||
b"Content-Disposition: form-data; name=\"file\"; filename=\"audio.m4a\"\r\n",
|
||||
b"Content-Type: audio/mp4\r\n\r\n",
|
||||
audio,
|
||||
format!("\r\n--{boundary}--\r\n").as_bytes(),
|
||||
]
|
||||
.concat();
|
||||
assert_eq!(request.body, expected_body);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn client_preserves_supplied_factory_and_typed_policy_denial() {
|
||||
let controller = codex_http_client::NetworkPolicyController::default();
|
||||
|
||||
Reference in New Issue
Block a user