mirror of
https://github.com/openai/codex.git
synced 2026-11-21 22:50:37 +00:00
Forward realtime assistant output through handoffs
This commit is contained in:
@@ -61,14 +61,12 @@ use tracing::warn;
|
||||
|
||||
const AUDIO_IN_QUEUE_CAPACITY: usize = 256;
|
||||
const USER_TEXT_IN_QUEUE_CAPACITY: usize = 64;
|
||||
const TERMINAL_OUTPUT_QUEUE_CAPACITY: usize = 64;
|
||||
const ASSISTANT_OUTPUT_QUEUE_CAPACITY: usize = 64;
|
||||
const OUTPUT_EVENTS_QUEUE_CAPACITY: usize = 256;
|
||||
const REALTIME_STARTUP_CONTEXT_TOKEN_BUDGET: usize = 5_300;
|
||||
const DEFAULT_REALTIME_MODEL: &str = "gpt-realtime-1.5";
|
||||
pub(crate) const REALTIME_USER_TEXT_PREFIX: &str = "[USER] ";
|
||||
pub(crate) const REALTIME_BACKEND_TEXT_PREFIX: &str = "[BACKEND] ";
|
||||
const REALTIME_V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT: &str =
|
||||
"Background agent finished. Use the preceding [BACKEND] messages as the result.";
|
||||
const DEFAULT_HANDOFF_ID: &str = "codex";
|
||||
const REALTIME_V2_STEER_ACKNOWLEDGEMENT: &str =
|
||||
"This was sent to steer the previous background agent task.";
|
||||
const REALTIME_ACTIVE_RESPONSE_ERROR_PREFIX: &str =
|
||||
@@ -98,20 +96,14 @@ enum RealtimeSessionKind {
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct RealtimeHandoffState {
|
||||
terminal_output_tx: Sender<RealtimeTerminalOutput>,
|
||||
assistant_output_tx: Sender<RealtimeAssistantOutput>,
|
||||
active_handoff: Arc<Mutex<Option<String>>>,
|
||||
session_kind: RealtimeSessionKind,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
enum RealtimeTerminalOutput {
|
||||
Direct {
|
||||
output_text: String,
|
||||
},
|
||||
Handoff {
|
||||
handoff_id: String,
|
||||
output_text: String,
|
||||
},
|
||||
struct RealtimeAssistantOutput {
|
||||
handoff_id: Option<String>,
|
||||
output_text: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
@@ -186,50 +178,37 @@ struct RealtimeInputTask {
|
||||
writer: RealtimeWebsocketWriter,
|
||||
events: RealtimeWebsocketEvents,
|
||||
user_text_rx: Receiver<String>,
|
||||
terminal_output_rx: Receiver<RealtimeTerminalOutput>,
|
||||
assistant_output_rx: Receiver<RealtimeAssistantOutput>,
|
||||
audio_rx: Receiver<RealtimeAudioFrame>,
|
||||
events_tx: Sender<RealtimeEvent>,
|
||||
handoff_state: RealtimeHandoffState,
|
||||
session_kind: RealtimeSessionKind,
|
||||
event_parser: RealtimeEventParser,
|
||||
}
|
||||
|
||||
struct RealtimeInputChannels {
|
||||
user_text_rx: Receiver<String>,
|
||||
terminal_output_rx: Receiver<RealtimeTerminalOutput>,
|
||||
assistant_output_rx: Receiver<RealtimeAssistantOutput>,
|
||||
audio_rx: Receiver<RealtimeAudioFrame>,
|
||||
}
|
||||
|
||||
impl RealtimeHandoffState {
|
||||
fn new(
|
||||
terminal_output_tx: Sender<RealtimeTerminalOutput>,
|
||||
session_kind: RealtimeSessionKind,
|
||||
) -> Self {
|
||||
fn new(assistant_output_tx: Sender<RealtimeAssistantOutput>) -> Self {
|
||||
Self {
|
||||
terminal_output_tx,
|
||||
assistant_output_tx,
|
||||
active_handoff: Arc::new(Mutex::new(None)),
|
||||
session_kind,
|
||||
}
|
||||
}
|
||||
|
||||
async fn take_terminal_output(
|
||||
&self,
|
||||
output_text: Option<String>,
|
||||
) -> Option<RealtimeTerminalOutput> {
|
||||
let handoff_id = self.active_handoff.lock().await.take();
|
||||
let output_text = output_text?;
|
||||
match handoff_id {
|
||||
Some(handoff_id) => Some(RealtimeTerminalOutput::Handoff {
|
||||
handoff_id,
|
||||
output_text: prefix_realtime_text(
|
||||
output_text,
|
||||
REALTIME_BACKEND_TEXT_PREFIX,
|
||||
self.session_kind,
|
||||
),
|
||||
}),
|
||||
None => Some(RealtimeTerminalOutput::Direct { output_text }),
|
||||
async fn assistant_output(&self, output_text: String) -> RealtimeAssistantOutput {
|
||||
RealtimeAssistantOutput {
|
||||
handoff_id: self.active_handoff.lock().await.clone(),
|
||||
output_text,
|
||||
}
|
||||
}
|
||||
|
||||
async fn finish_turn(&self) {
|
||||
self.active_handoff.lock().await.take();
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
@@ -312,16 +291,16 @@ impl RealtimeConversationManager {
|
||||
async_channel::bounded::<RealtimeAudioFrame>(AUDIO_IN_QUEUE_CAPACITY);
|
||||
let (user_text_tx, user_text_rx) =
|
||||
async_channel::bounded::<String>(USER_TEXT_IN_QUEUE_CAPACITY);
|
||||
let (terminal_output_tx, terminal_output_rx) =
|
||||
async_channel::bounded::<RealtimeTerminalOutput>(TERMINAL_OUTPUT_QUEUE_CAPACITY);
|
||||
let (assistant_output_tx, assistant_output_rx) =
|
||||
async_channel::bounded::<RealtimeAssistantOutput>(ASSISTANT_OUTPUT_QUEUE_CAPACITY);
|
||||
let (events_tx, events_rx) =
|
||||
async_channel::bounded::<RealtimeEvent>(OUTPUT_EVENTS_QUEUE_CAPACITY);
|
||||
|
||||
let realtime_active = Arc::new(AtomicBool::new(true));
|
||||
let handoff = RealtimeHandoffState::new(terminal_output_tx, session_kind);
|
||||
let handoff = RealtimeHandoffState::new(assistant_output_tx);
|
||||
let input_channels = RealtimeInputChannels {
|
||||
user_text_rx,
|
||||
terminal_output_rx,
|
||||
assistant_output_rx,
|
||||
audio_rx,
|
||||
};
|
||||
|
||||
@@ -343,7 +322,6 @@ impl RealtimeConversationManager {
|
||||
events_tx,
|
||||
handoff_state: handoff.clone(),
|
||||
session_kind,
|
||||
event_parser,
|
||||
realtime_active: Arc::clone(&realtime_active),
|
||||
});
|
||||
(task, Some(call.sdp))
|
||||
@@ -360,12 +338,11 @@ impl RealtimeConversationManager {
|
||||
writer: connection.writer(),
|
||||
events: connection.events(),
|
||||
user_text_rx: input_channels.user_text_rx,
|
||||
terminal_output_rx: input_channels.terminal_output_rx,
|
||||
assistant_output_rx: input_channels.assistant_output_rx,
|
||||
audio_rx: input_channels.audio_rx,
|
||||
events_tx,
|
||||
handoff_state: handoff.clone(),
|
||||
session_kind,
|
||||
event_parser,
|
||||
});
|
||||
(task, None)
|
||||
};
|
||||
@@ -468,7 +445,7 @@ impl RealtimeConversationManager {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn finish_turn(&self, output_text: Option<String>) -> CodexResult<()> {
|
||||
pub(crate) async fn send_assistant_output(&self, output_text: String) -> CodexResult<()> {
|
||||
let handoff = {
|
||||
let guard = self.state.lock().await;
|
||||
guard.as_ref().map(|state| state.handoff.clone())
|
||||
@@ -477,17 +454,25 @@ impl RealtimeConversationManager {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let Some(terminal_output) = handoff.take_terminal_output(output_text).await else {
|
||||
return Ok(());
|
||||
};
|
||||
handoff
|
||||
.terminal_output_tx
|
||||
.send(terminal_output)
|
||||
.assistant_output_tx
|
||||
.send(handoff.assistant_output(output_text).await)
|
||||
.await
|
||||
.map_err(|_| CodexErr::InvalidRequest("conversation is not running".to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn finish_turn(&self) {
|
||||
let handoff = {
|
||||
let guard = self.state.lock().await;
|
||||
guard.as_ref().map(|state| state.handoff.clone())
|
||||
};
|
||||
let Some(handoff) = handoff else {
|
||||
return;
|
||||
};
|
||||
handoff.finish_turn().await;
|
||||
}
|
||||
|
||||
pub(crate) async fn shutdown(&self) -> CodexResult<()> {
|
||||
let state = {
|
||||
let mut guard = self.state.lock().await;
|
||||
@@ -987,7 +972,6 @@ struct RealtimeWebrtcSidebandInputTask {
|
||||
events_tx: Sender<RealtimeEvent>,
|
||||
handoff_state: RealtimeHandoffState,
|
||||
session_kind: RealtimeSessionKind,
|
||||
event_parser: RealtimeEventParser,
|
||||
realtime_active: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
@@ -1001,7 +985,6 @@ fn spawn_webrtc_sideband_input_task(input: RealtimeWebrtcSidebandInputTask) -> J
|
||||
events_tx,
|
||||
handoff_state,
|
||||
session_kind,
|
||||
event_parser,
|
||||
realtime_active,
|
||||
} = input;
|
||||
|
||||
@@ -1040,12 +1023,11 @@ fn spawn_webrtc_sideband_input_task(input: RealtimeWebrtcSidebandInputTask) -> J
|
||||
writer: connection.writer(),
|
||||
events: connection.events(),
|
||||
user_text_rx: input_channels.user_text_rx,
|
||||
terminal_output_rx: input_channels.terminal_output_rx,
|
||||
assistant_output_rx: input_channels.assistant_output_rx,
|
||||
audio_rx: input_channels.audio_rx,
|
||||
events_tx,
|
||||
handoff_state,
|
||||
session_kind,
|
||||
event_parser,
|
||||
})
|
||||
.await;
|
||||
})
|
||||
@@ -1056,12 +1038,11 @@ async fn run_realtime_input_task(input: RealtimeInputTask) {
|
||||
writer,
|
||||
events,
|
||||
user_text_rx,
|
||||
terminal_output_rx,
|
||||
assistant_output_rx,
|
||||
audio_rx,
|
||||
events_tx,
|
||||
handoff_state,
|
||||
session_kind,
|
||||
event_parser,
|
||||
} = input;
|
||||
|
||||
let mut output_audio_state: Option<OutputAudioState> = None;
|
||||
@@ -1078,14 +1059,12 @@ async fn run_realtime_input_task(input: RealtimeInputTask) {
|
||||
)
|
||||
.await
|
||||
}
|
||||
// Terminal Codex output that should be sent back to realtime.
|
||||
terminal_output = terminal_output_rx.recv() => {
|
||||
handle_terminal_output(
|
||||
terminal_output,
|
||||
// Assistant output that should be sent back to realtime.
|
||||
assistant_output = assistant_output_rx.recv() => {
|
||||
handle_assistant_output(
|
||||
assistant_output,
|
||||
&writer,
|
||||
&events_tx,
|
||||
event_parser,
|
||||
&mut response_create_queue,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -1132,70 +1111,31 @@ async fn handle_user_text_input(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn handle_terminal_output(
|
||||
terminal_output: Result<RealtimeTerminalOutput, RecvError>,
|
||||
async fn handle_assistant_output(
|
||||
assistant_output: Result<RealtimeAssistantOutput, RecvError>,
|
||||
writer: &RealtimeWebsocketWriter,
|
||||
events_tx: &Sender<RealtimeEvent>,
|
||||
event_parser: RealtimeEventParser,
|
||||
response_create_queue: &mut RealtimeResponseCreateQueue,
|
||||
) -> anyhow::Result<()> {
|
||||
let terminal_output = terminal_output.context("terminal output channel closed")?;
|
||||
|
||||
let (result, request_response) = match terminal_output {
|
||||
RealtimeTerminalOutput::Direct { output_text } => {
|
||||
let request_response = match event_parser {
|
||||
RealtimeEventParser::V1 => false,
|
||||
RealtimeEventParser::RealtimeV2 => true,
|
||||
};
|
||||
(
|
||||
writer
|
||||
.send_conversation_developer_item_create(format!(
|
||||
"Speak the following text:\n{output_text}"
|
||||
))
|
||||
.await,
|
||||
request_response,
|
||||
)
|
||||
}
|
||||
RealtimeTerminalOutput::Handoff {
|
||||
handoff_id,
|
||||
output_text,
|
||||
} => match event_parser {
|
||||
RealtimeEventParser::V1 => (
|
||||
writer
|
||||
.send_conversation_handoff_append(Some(handoff_id), output_text)
|
||||
.await,
|
||||
false,
|
||||
let assistant_output = assistant_output.context("assistant output channel closed")?;
|
||||
if let Err(err) = writer
|
||||
.send_conversation_handoff_append(
|
||||
Some(
|
||||
assistant_output
|
||||
.handoff_id
|
||||
.unwrap_or_else(|| DEFAULT_HANDOFF_ID.to_string()),
|
||||
),
|
||||
RealtimeEventParser::RealtimeV2 => (
|
||||
if let Err(err) = writer.send_conversation_item_create(output_text).await {
|
||||
Err(err)
|
||||
} else {
|
||||
writer
|
||||
.send_conversation_function_call_output(
|
||||
handoff_id,
|
||||
REALTIME_V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT.to_string(),
|
||||
)
|
||||
.await
|
||||
},
|
||||
true,
|
||||
),
|
||||
},
|
||||
};
|
||||
if let Err(err) = result {
|
||||
assistant_output.output_text,
|
||||
)
|
||||
.await
|
||||
{
|
||||
let mapped_error = map_api_error(err);
|
||||
warn!("failed to send terminal output: {mapped_error}");
|
||||
warn!("failed to send assistant output: {mapped_error}");
|
||||
let _ = events_tx
|
||||
.send(RealtimeEvent::Error(mapped_error.to_string()))
|
||||
.await;
|
||||
return Err(mapped_error.into());
|
||||
}
|
||||
if request_response {
|
||||
response_create_queue
|
||||
.request_create(writer, events_tx, "terminal output")
|
||||
.await
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn handle_realtime_server_event(
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
use super::RealtimeAssistantOutput;
|
||||
use super::RealtimeHandoffState;
|
||||
use super::RealtimeSessionKind;
|
||||
use super::RealtimeTerminalOutput;
|
||||
use super::realtime_delegation_from_handoff;
|
||||
use super::realtime_request_headers;
|
||||
use super::realtime_text_from_handoff_request;
|
||||
@@ -127,71 +126,66 @@ fn wraps_realtime_delegation_input_with_xml_escaping_without_transcript() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn terminal_output_consumes_only_its_handoff() {
|
||||
for (session_kind, expected_output_text) in [
|
||||
(RealtimeSessionKind::V1, "finished"),
|
||||
(RealtimeSessionKind::V2, "[BACKEND] finished"),
|
||||
] {
|
||||
let (tx, _rx) = bounded(1);
|
||||
let state = RealtimeHandoffState::new(tx, session_kind);
|
||||
async fn assistant_outputs_preserve_active_handoff_until_turn_completion() {
|
||||
let (tx, _rx) = bounded(1);
|
||||
let state = RealtimeHandoffState::new(tx);
|
||||
*state.active_handoff.lock().await = Some("handoff_1".to_string());
|
||||
|
||||
*state.active_handoff.lock().await = Some("handoff_1".to_string());
|
||||
let first_output = state
|
||||
.take_terminal_output(Some("finished".to_string()))
|
||||
.await;
|
||||
*state.active_handoff.lock().await = Some("handoff_2".to_string());
|
||||
|
||||
assert_eq!(
|
||||
first_output,
|
||||
Some(RealtimeTerminalOutput::Handoff {
|
||||
handoff_id: "handoff_1".to_string(),
|
||||
output_text: expected_output_text.to_string(),
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
state.active_handoff.lock().await.clone(),
|
||||
Some("handoff_2".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
state
|
||||
.take_terminal_output(Some("finished again".to_string()))
|
||||
.await,
|
||||
Some(RealtimeTerminalOutput::Handoff {
|
||||
handoff_id: "handoff_2".to_string(),
|
||||
output_text: format!("{expected_output_text} again"),
|
||||
})
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
state.assistant_output("working".to_string()).await,
|
||||
RealtimeAssistantOutput {
|
||||
handoff_id: Some("handoff_1".to_string()),
|
||||
output_text: "working".to_string(),
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
state.active_handoff.lock().await.as_deref(),
|
||||
Some("handoff_1")
|
||||
);
|
||||
assert_eq!(
|
||||
state.assistant_output("finished".to_string()).await,
|
||||
RealtimeAssistantOutput {
|
||||
handoff_id: Some("handoff_1".to_string()),
|
||||
output_text: "finished".to_string(),
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
state.active_handoff.lock().await.as_deref(),
|
||||
Some("handoff_1")
|
||||
);
|
||||
state.finish_turn().await;
|
||||
assert_eq!(state.active_handoff.lock().await.as_deref(), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn terminal_output_without_handoff_stays_plain() {
|
||||
for session_kind in [RealtimeSessionKind::V1, RealtimeSessionKind::V2] {
|
||||
let (tx, _rx) = bounded(1);
|
||||
let state = RealtimeHandoffState::new(tx, session_kind);
|
||||
async fn assistant_output_without_handoff_has_no_active_id() {
|
||||
let (tx, _rx) = bounded(1);
|
||||
let state = RealtimeHandoffState::new(tx);
|
||||
|
||||
assert_eq!(
|
||||
state
|
||||
.take_terminal_output(Some("finished".to_string()))
|
||||
.await,
|
||||
Some(RealtimeTerminalOutput::Direct {
|
||||
output_text: "finished".to_string(),
|
||||
})
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
state.assistant_output("working".to_string()).await,
|
||||
RealtimeAssistantOutput {
|
||||
handoff_id: None,
|
||||
output_text: "working".to_string(),
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
state.assistant_output("finished".to_string()).await,
|
||||
RealtimeAssistantOutput {
|
||||
handoff_id: None,
|
||||
output_text: "finished".to_string(),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn terminal_output_without_message_still_consumes_handoff() {
|
||||
for session_kind in [RealtimeSessionKind::V1, RealtimeSessionKind::V2] {
|
||||
let (tx, _rx) = bounded(1);
|
||||
let state = RealtimeHandoffState::new(tx, session_kind);
|
||||
async fn finishing_turn_consumes_handoff() {
|
||||
let (tx, _rx) = bounded(1);
|
||||
let state = RealtimeHandoffState::new(tx);
|
||||
*state.active_handoff.lock().await = Some("handoff_1".to_string());
|
||||
|
||||
*state.active_handoff.lock().await = Some("handoff_1".to_string());
|
||||
|
||||
assert_eq!(state.take_terminal_output(/*output_text*/ None).await, None);
|
||||
assert_eq!(state.active_handoff.lock().await.clone(), None);
|
||||
}
|
||||
state.finish_turn().await;
|
||||
assert_eq!(state.active_handoff.lock().await.as_deref(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -1672,7 +1672,7 @@ impl Session {
|
||||
id: turn_context.sub_id.clone(),
|
||||
msg,
|
||||
};
|
||||
self.maybe_send_realtime_terminal_output(&legacy_source)
|
||||
self.maybe_send_realtime_assistant_output(&legacy_source)
|
||||
.await;
|
||||
self.send_event_raw(event).await;
|
||||
self.maybe_notify_parent_of_terminal_turn(turn_context, &legacy_source)
|
||||
@@ -1782,16 +1782,32 @@ impl Session {
|
||||
}
|
||||
}
|
||||
|
||||
async fn maybe_send_realtime_terminal_output(&self, msg: &EventMsg) {
|
||||
let EventMsg::TurnComplete(event) = msg else {
|
||||
return;
|
||||
async fn maybe_send_realtime_assistant_output(&self, msg: &EventMsg) {
|
||||
let result = match msg {
|
||||
EventMsg::ItemCompleted(ItemCompletedEvent {
|
||||
item: TurnItem::AgentMessage(message),
|
||||
..
|
||||
}) => {
|
||||
let output_text = message
|
||||
.content
|
||||
.iter()
|
||||
.map(|content| match content {
|
||||
codex_protocol::items::AgentMessageContent::Text { text } => text.as_str(),
|
||||
})
|
||||
.collect::<String>();
|
||||
if output_text.trim().is_empty() {
|
||||
return;
|
||||
}
|
||||
self.conversation.send_assistant_output(output_text).await
|
||||
}
|
||||
EventMsg::TurnComplete(_) | EventMsg::TurnAborted(_) => {
|
||||
self.conversation.finish_turn().await;
|
||||
return;
|
||||
}
|
||||
_ => return,
|
||||
};
|
||||
if let Err(err) = self
|
||||
.conversation
|
||||
.finish_turn(event.last_agent_message.clone())
|
||||
.await
|
||||
{
|
||||
debug!("failed to send terminal output to realtime conversation: {err}");
|
||||
if let Err(err) = result {
|
||||
debug!("failed to send assistant output to realtime conversation: {err}");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user