openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-11-01 19:33:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use opus-rs for realtime transport

Browse Source 
- switch realtime transport and test server to opus-rs
- drop native opus bazel and cmake plumbing

Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Ahmed Ibrahim
2026-04-05 00:43:41 -07:00
co-authored by Codex
parent f9739d0178
commit 76397dbdd0
11 changed files with 118 additions and 214 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/Cargo.lock
Generated
+7 -17
View File
@@ -784,17 +784,6 @@ version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
[[package]]
name = "audiopus_sys"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62314a1546a2064e033665d658e88c620a62904be945f8147e6b16c3db9f8651"
dependencies = [
"cmake",
"log",
"pkg-config",
]
[[package]]
name = "autocfg"
version = "1.5.0"
@@ -1472,7 +1461,7 @@ dependencies = [
"futures",
"http 1.4.0",
"interceptor",
"opus",
"opus-rs",
"pretty_assertions",
"regex-lite",
"reqwest",
@@ -3407,7 +3396,7 @@ dependencies = [
"notify",
"opentelemetry",
"opentelemetry_sdk",
"opus",
"opus-rs",
"pretty_assertions",
"regex-lite",
"reqwest",
@@ -7329,12 +7318,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
[[package]]
name = "opus"
version = "0.3.1"
name = "opus-rs"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d3809943dff6fbad5f0484449ea26bdb9cb7d8efdf26ed50d3c7f227f69eb5c"
checksum = "f857fcc607bea70f9b073ad267e829080646b14c260ea5e43c48ba9fca8e7dd4"
dependencies = [
"audiopus_sys",
"num-complex",
"once_cell",
]
[[package]]
codex-rs/Cargo.toml
+1 -1
View File
@@ -256,7 +256,7 @@ notify = "8.2.0"
nucleo = { git = "https://github.com/helix-editor/nucleo.git", rev = "4253de9faabb4e5c6d81d946a5e35a90f87347ee" }
once_cell = "1.20.2"
openssl-sys = "*"
opus = "0.3.0"
opus-rs = "0.1.11"
opentelemetry = "0.31.0"
opentelemetry-appender-tracing = "0.31.0"
opentelemetry-otlp = "0.31.0"
codex-rs/codex-api/Cargo.toml
+1 -1
View File
@@ -15,7 +15,7 @@ codex-utils-rustls-provider = { workspace = true }
futures = { workspace = true }
http = { workspace = true }
interceptor = { workspace = true }
opus = { workspace = true }
opus-rs = { workspace = true }
reqwest = { workspace = true, features = ["multipart"] }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs
+76 -15
View File
@@ -21,10 +21,9 @@ use codex_utils_rustls_provider::ensure_rustls_crypto_provider;
use http::HeaderMap;
use http::HeaderValue;
use interceptor::registry::Registry;
use opus::Application;
use opus::Channels;
use opus::Decoder as OpusDecoder;
use opus::Encoder as OpusEncoder;
use opus_rs::Application;
use opus_rs::OpusDecoder;
use opus_rs::OpusEncoder;
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::Mutex as StdMutex;
@@ -607,10 +606,7 @@ async fn connect_webrtc_transport(
"realtime WebRTC session connected"
);
let opus_encoder =
OpusEncoder::new(24_000, Channels::Mono, Application::Voip).map_err(|err| {
ApiError::Stream(format!("failed to initialize realtime Opus encoder: {err}"))
})?;
let opus_encoder = new_realtime_opus_encoder(Application::Voip)?;
Ok((
RealtimeWebRtcTransport {
@@ -742,7 +738,7 @@ async fn pump_remote_audio_track(
track: Arc<TrackRemote>,
tx_message: mpsc::UnboundedSender<Result<RealtimeTransportEvent, ApiError>>,
) {
let mut decoder = match OpusDecoder::new(24_000, Channels::Mono) {
let mut decoder = match new_realtime_opus_decoder() {
Ok(decoder) => decoder,
Err(err) => {
let _ = tx_message.send(Err(ApiError::Stream(format!(
@@ -751,7 +747,7 @@ async fn pump_remote_audio_track(
return;
}
};
let mut decoded = vec![0i16; OPUS_MAX_DECODED_SAMPLES_PER_CHANNEL];
let mut decoded = vec![0.0f32; OPUS_MAX_DECODED_SAMPLES_PER_CHANNEL];
loop {
let packet = match track.read_rtp().await {
@@ -767,9 +763,18 @@ async fn pump_remote_audio_track(
continue;
}
let samples_per_channel = match decoder.decode(&packet.payload, &mut decoded, false) {
let samples_per_channel = match decoder.decode(
&packet.payload,
OPUS_MAX_DECODED_SAMPLES_PER_CHANNEL,
&mut decoded,
) {
Ok(samples_per_channel) => samples_per_channel,
Err(err) => {
warn!(
%err,
payload_len = packet.payload.len(),
"failed to decode realtime Opus packet"
);
let _ = tx_message.send(Err(ApiError::Stream(format!(
"failed to decode realtime Opus packet: {err}"
))));
@@ -782,7 +787,7 @@ async fn pump_remote_audio_track(
let mut pcm_bytes = Vec::with_capacity(samples_per_channel * 2);
for sample in &decoded[..samples_per_channel] {
pcm_bytes.extend_from_slice(&sample.to_le_bytes());
pcm_bytes.extend_from_slice(&f32_to_i16(*sample).to_le_bytes());
}
let _ = tx_message.send(Ok(RealtimeTransportEvent::AudioOut(RealtimeAudioFrame {
data: BASE64_STANDARD.encode(pcm_bytes),
@@ -862,13 +867,24 @@ impl RealtimeInputAudioEncoder {
let mut samples = Vec::new();
while self.pending_samples.len() >= REALTIME_AUDIO_PACKET_SAMPLES {
let frame: Vec<f32> = self.pending_samples[..REALTIME_AUDIO_PACKET_SAMPLES]
.iter()
.copied()
.map(i16_to_f32)
.collect();
let encoded_len = self
.opus_encoder
.encode(
&self.pending_samples[..REALTIME_AUDIO_PACKET_SAMPLES],
&frame,
REALTIME_AUDIO_PACKET_SAMPLES,
&mut self.encoded_packet,
)
.map_err(|err| {
warn!(
%err,
pending_samples = self.pending_samples.len(),
"failed to encode realtime input Opus"
);
ApiError::Stream(format!("failed to encode realtime input Opus: {err}"))
})?;
self.pending_samples.drain(..REALTIME_AUDIO_PACKET_SAMPLES);
@@ -883,6 +899,52 @@ impl RealtimeInputAudioEncoder {
}
}
fn new_realtime_opus_encoder(application: Application) -> Result<OpusEncoder, ApiError> {
let encoder = OpusEncoder::new(24_000, usize::from(REALTIME_AUDIO_CHANNELS), application)
.map_err(|err| {
warn!(
%err,
sample_rate = 24_000,
channels = REALTIME_AUDIO_CHANNELS,
"failed to initialize realtime Opus encoder"
);
ApiError::Stream(format!("failed to initialize realtime Opus encoder: {err}"))
})?;
debug!(
sample_rate = 24_000,
channels = REALTIME_AUDIO_CHANNELS,
"initialized realtime Opus encoder"
);
Ok(encoder)
}
fn new_realtime_opus_decoder() -> Result<OpusDecoder, ApiError> {
let decoder =
OpusDecoder::new(24_000, usize::from(REALTIME_AUDIO_CHANNELS)).map_err(|err| {
warn!(
%err,
sample_rate = 24_000,
channels = REALTIME_AUDIO_CHANNELS,
"failed to initialize realtime Opus decoder"
);
ApiError::Stream(format!("failed to initialize realtime Opus decoder: {err}"))
})?;
debug!(
sample_rate = 24_000,
channels = REALTIME_AUDIO_CHANNELS,
"initialized realtime Opus decoder"
);
Ok(decoder)
}
fn i16_to_f32(sample: i16) -> f32 {
(sample as f32) / (i16::MAX as f32)
}
fn f32_to_i16(sample: f32) -> i16 {
(sample.clamp(-1.0, 1.0) * i16::MAX as f32) as i16
}
#[cfg(test)]
mod tests {
use super::*;
@@ -959,8 +1021,7 @@ mod tests {
#[test]
fn input_audio_encoder_rejects_non_mono_24khz_audio() {
let mut encoder = RealtimeInputAudioEncoder {
opus_encoder: OpusEncoder::new(24_000, Channels::Mono, Application::Audio)
.expect("encoder"),
opus_encoder: new_realtime_opus_encoder(Application::Audio).expect("encoder"),
pending_samples: Vec::new(),
encoded_packet: vec![0u8; OPUS_MAX_PACKET_BYTES],
};
codex-rs/core/tests/common/Cargo.toml
+1 -1
View File
@@ -29,7 +29,7 @@ futures = { workspace = true }
notify = { workspace = true }
opentelemetry = { workspace = true }
opentelemetry_sdk = { workspace = true }
opus = { workspace = true }
opus-rs = { workspace = true }
regex-lite = { workspace = true }
serde_json = { workspace = true }
tempfile = { workspace = true }
codex-rs/core/tests/common/responses/realtime_webrtc_server.rs
+30 -8
View File
@@ -5,8 +5,7 @@ use std::time::Duration;
use base64::Engine;
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
use opus::Channels;
use opus::Decoder as OpusDecoder;
use opus_rs::OpusDecoder;
use serde_json::Value;
use tokio::io::AsyncReadExt;
use tokio::io::AsyncWriteExt;
@@ -15,6 +14,8 @@ use tokio::sync::Notify;
use tokio::sync::mpsc;
use tokio::sync::oneshot;
use tokio::time::timeout;
use tracing::debug;
use tracing::warn;
use webrtc::api::APIBuilder;
use webrtc::api::interceptor_registry::register_default_interceptors;
use webrtc::api::media_engine::MediaEngine;
@@ -290,17 +291,34 @@ async fn pump_remote_audio_track(
track: Arc<TrackRemote>,
tx_request: mpsc::UnboundedSender<Value>,
) {
let Ok(mut decoder) = OpusDecoder::new(24_000, Channels::Mono) else {
return;
let mut decoder = match OpusDecoder::new(24_000, usize::from(REALTIME_AUDIO_CHANNELS)) {
Ok(decoder) => decoder,
Err(err) => {
warn!(%err, "failed to initialize realtime Opus decoder in test server");
return;
}
};
let mut decoded = vec![0i16; REALTIME_MAX_DECODED_SAMPLES_PER_CHANNEL];
debug!("initialized realtime Opus decoder in test server");
let mut decoded = vec![0.0f32; REALTIME_MAX_DECODED_SAMPLES_PER_CHANNEL];
while let Ok((packet, _)) = track.read_rtp().await {
if packet.payload.is_empty() {
continue;
}
let Ok(samples_per_channel) = decoder.decode(&packet.payload, &mut decoded, false) else {
return;
let samples_per_channel = match decoder.decode(
&packet.payload,
REALTIME_MAX_DECODED_SAMPLES_PER_CHANNEL,
&mut decoded,
) {
Ok(samples_per_channel) => samples_per_channel,
Err(err) => {
warn!(
%err,
payload_len = packet.payload.len(),
"failed to decode realtime Opus packet in test server"
);
return;
}
};
if samples_per_channel == 0 {
continue;
@@ -308,7 +326,7 @@ async fn pump_remote_audio_track(
let mut pcm_bytes = Vec::with_capacity(samples_per_channel * 2);
for sample in &decoded[..samples_per_channel] {
pcm_bytes.extend_from_slice(&sample.to_le_bytes());
pcm_bytes.extend_from_slice(&f32_to_i16(*sample).to_le_bytes());
}
let _ = tx_request.send(serde_json::json!({
"type": "input_audio_buffer.append",
@@ -320,6 +338,10 @@ async fn pump_remote_audio_track(
}
}
fn f32_to_i16(sample: f32) -> i16 {
(sample.clamp(-1.0, 1.0) * i16::MAX as f32) as i16
}
async fn serve_scripted_requests(
connection: WebSocketConnectionConfig,
connection_index: usize,