mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
Refresh realtime context for each model request (#47596)
## Why A realtime call can start or end while a text turn is still running. Using call activity captured at turn creation leaves subsequent model requests with stale realtime context. ## What changed Capture realtime activity and mode instructions together in a `StepContext` snapshot for each model request. Use that snapshot to build realtime instructions and persist `realtime_active` in the request's turn context. Store the conversation and mode instructions under one mutex so they are installed together when a call starts. ## Testing Extend the realtime handoff integration test to start and close a call during one text turn, verifying that successive requests include the appropriate start and end instructions. GitOrigin-RevId: 4c5c2b47b7c0c06ae28167013218dc2e3cd72c42
This commit is contained in:
@@ -165,8 +165,12 @@ impl RealtimeHandoffAdmission {
|
||||
}
|
||||
|
||||
pub(crate) struct RealtimeConversationManager {
|
||||
state: Mutex<Option<ConversationState>>,
|
||||
mode_instructions: Mutex<Option<RealtimeModeInstructions>>,
|
||||
state: Mutex<RealtimeConversationManagerState>,
|
||||
}
|
||||
|
||||
struct RealtimeConversationManagerState {
|
||||
conversation: Option<ConversationState>,
|
||||
mode_instructions: Option<RealtimeModeInstructions>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
@@ -175,6 +179,12 @@ pub(crate) struct RealtimeModeInstructions {
|
||||
pub(crate) end: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct RealtimeConversationSnapshot {
|
||||
pub(crate) active: bool,
|
||||
pub(crate) mode_instructions: Option<RealtimeModeInstructions>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum RealtimeSessionKind {
|
||||
V1,
|
||||
@@ -543,13 +553,22 @@ struct RealtimeStartOutput {
|
||||
impl RealtimeConversationManager {
|
||||
pub(crate) fn new() -> Self {
|
||||
Self {
|
||||
state: Mutex::new(None),
|
||||
mode_instructions: Mutex::new(None),
|
||||
state: Mutex::new(RealtimeConversationManagerState {
|
||||
conversation: None,
|
||||
mode_instructions: None,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn mode_instructions(&self) -> Option<RealtimeModeInstructions> {
|
||||
self.mode_instructions.lock().await.clone()
|
||||
pub(crate) async fn snapshot(&self) -> RealtimeConversationSnapshot {
|
||||
let state = self.state.lock().await;
|
||||
RealtimeConversationSnapshot {
|
||||
active: state
|
||||
.conversation
|
||||
.as_ref()
|
||||
.is_some_and(|conversation| conversation.realtime_active.load(Ordering::Relaxed)),
|
||||
mode_instructions: state.mode_instructions.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(
|
||||
@@ -560,6 +579,7 @@ impl RealtimeConversationManager {
|
||||
pub(crate) async fn running_state(&self) -> Option<()> {
|
||||
let state = self.state.lock().await;
|
||||
state
|
||||
.conversation
|
||||
.as_ref()
|
||||
.and_then(|state| state.realtime_active.load(Ordering::Relaxed).then_some(()))
|
||||
}
|
||||
@@ -567,7 +587,7 @@ impl RealtimeConversationManager {
|
||||
pub(crate) async fn is_running_v2(&self) -> bool {
|
||||
let state = self.state.lock().await;
|
||||
matches!(
|
||||
state.as_ref(),
|
||||
state.conversation.as_ref(),
|
||||
Some(state)
|
||||
if state.realtime_active.load(Ordering::Relaxed)
|
||||
&& state.session_kind == RealtimeSessionKind::V2
|
||||
@@ -581,6 +601,7 @@ impl RealtimeConversationManager {
|
||||
.state
|
||||
.lock()
|
||||
.await
|
||||
.conversation
|
||||
.as_ref()
|
||||
.map(|state| Arc::clone(&state.route_handoffs));
|
||||
if let Some(route_handoffs) = route_handoffs {
|
||||
@@ -595,18 +616,20 @@ impl RealtimeConversationManager {
|
||||
) -> CodexResult<RealtimeStartOutput> {
|
||||
let previous_state = {
|
||||
let mut guard = self.state.lock().await;
|
||||
guard.take()
|
||||
guard.conversation.take()
|
||||
};
|
||||
if let Some(state) = previous_state {
|
||||
stop_conversation_state(state, RealtimeFanoutTaskStop::Await).await;
|
||||
}
|
||||
|
||||
let output = self.start_inner(start).await?;
|
||||
*self.mode_instructions.lock().await = Some(mode_instructions);
|
||||
Ok(output)
|
||||
self.start_inner(start, mode_instructions).await
|
||||
}
|
||||
|
||||
async fn start_inner(&self, start: RealtimeStart) -> CodexResult<RealtimeStartOutput> {
|
||||
async fn start_inner(
|
||||
&self,
|
||||
start: RealtimeStart,
|
||||
mode_instructions: RealtimeModeInstructions,
|
||||
) -> CodexResult<RealtimeStartOutput> {
|
||||
let RealtimeStart {
|
||||
api_provider,
|
||||
http_client_factory,
|
||||
@@ -744,8 +767,8 @@ impl RealtimeConversationManager {
|
||||
(task, None)
|
||||
};
|
||||
|
||||
let mut guard = self.state.lock().await;
|
||||
*guard = Some(ConversationState {
|
||||
let mut state = self.state.lock().await;
|
||||
state.conversation = Some(ConversationState {
|
||||
audio_tx,
|
||||
text_tx,
|
||||
session_kind,
|
||||
@@ -756,6 +779,7 @@ impl RealtimeConversationManager {
|
||||
route_handoffs: Arc::clone(&route_handoffs),
|
||||
stop_token,
|
||||
});
|
||||
state.mode_instructions = Some(mode_instructions);
|
||||
Ok(RealtimeStartOutput {
|
||||
realtime_active,
|
||||
route_handoffs,
|
||||
@@ -773,7 +797,7 @@ impl RealtimeConversationManager {
|
||||
let mut fanout_task = Some(fanout_task);
|
||||
{
|
||||
let mut guard = self.state.lock().await;
|
||||
if let Some(state) = guard.as_mut()
|
||||
if let Some(state) = guard.conversation.as_mut()
|
||||
&& Arc::ptr_eq(&state.realtime_active, realtime_active)
|
||||
{
|
||||
state.fanout_task = fanout_task.take();
|
||||
@@ -789,8 +813,10 @@ impl RealtimeConversationManager {
|
||||
pub(crate) async fn finish_if_active(&self, realtime_active: &Arc<AtomicBool>) {
|
||||
let state = {
|
||||
let mut guard = self.state.lock().await;
|
||||
match guard.as_ref() {
|
||||
Some(state) if Arc::ptr_eq(&state.realtime_active, realtime_active) => guard.take(),
|
||||
match guard.conversation.as_ref() {
|
||||
Some(state) if Arc::ptr_eq(&state.realtime_active, realtime_active) => {
|
||||
guard.conversation.take()
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
};
|
||||
@@ -803,7 +829,10 @@ impl RealtimeConversationManager {
|
||||
pub(crate) async fn audio_in(&self, frame: RealtimeAudioFrame) -> CodexResult<()> {
|
||||
let sender = {
|
||||
let guard = self.state.lock().await;
|
||||
guard.as_ref().map(|state| state.audio_tx.clone())
|
||||
guard
|
||||
.conversation
|
||||
.as_ref()
|
||||
.map(|state| state.audio_tx.clone())
|
||||
};
|
||||
|
||||
let Some(sender) = sender else {
|
||||
@@ -828,6 +857,7 @@ impl RealtimeConversationManager {
|
||||
let sender = {
|
||||
let guard = self.state.lock().await;
|
||||
guard
|
||||
.conversation
|
||||
.as_ref()
|
||||
.map(|state| (state.text_tx.clone(), state.session_kind))
|
||||
};
|
||||
@@ -854,6 +884,7 @@ impl RealtimeConversationManager {
|
||||
.state
|
||||
.lock()
|
||||
.await
|
||||
.conversation
|
||||
.as_ref()
|
||||
.map(|state| state.handoff.clone());
|
||||
let Some(handoff) = handoff.filter(|handoff| {
|
||||
@@ -884,7 +915,7 @@ impl RealtimeConversationManager {
|
||||
) -> CodexResult<()> {
|
||||
let handoff = {
|
||||
let guard = self.state.lock().await;
|
||||
let Some(state) = guard.as_ref() else {
|
||||
let Some(state) = guard.conversation.as_ref() else {
|
||||
return Err(CodexErr::InvalidRequest(
|
||||
"conversation is not running".to_string(),
|
||||
));
|
||||
@@ -979,7 +1010,10 @@ impl RealtimeConversationManager {
|
||||
) {
|
||||
let handoff = {
|
||||
let guard = self.state.lock().await;
|
||||
guard.as_ref().map(|state| state.handoff.clone())
|
||||
guard
|
||||
.conversation
|
||||
.as_ref()
|
||||
.map(|state| state.handoff.clone())
|
||||
};
|
||||
let Some(handoff) = handoff else {
|
||||
return;
|
||||
@@ -1037,7 +1071,7 @@ impl RealtimeConversationManager {
|
||||
}
|
||||
let handoff = {
|
||||
let guard = self.state.lock().await;
|
||||
let Some(state) = guard.as_ref() else {
|
||||
let Some(state) = guard.conversation.as_ref() else {
|
||||
return Err(CodexErr::InvalidRequest(
|
||||
"conversation is not running".to_string(),
|
||||
));
|
||||
@@ -1069,7 +1103,10 @@ impl RealtimeConversationManager {
|
||||
pub(crate) async fn finish_handoff_stream_item(&self, item_id: &str) -> bool {
|
||||
let handoff = {
|
||||
let guard = self.state.lock().await;
|
||||
guard.as_ref().map(|state| state.handoff.clone())
|
||||
guard
|
||||
.conversation
|
||||
.as_ref()
|
||||
.map(|state| state.handoff.clone())
|
||||
};
|
||||
let Some(handoff) = handoff else {
|
||||
return false;
|
||||
@@ -1103,7 +1140,7 @@ impl RealtimeConversationManager {
|
||||
|
||||
let handoff = {
|
||||
let guard = self.state.lock().await;
|
||||
let Some(state) = guard.as_ref() else {
|
||||
let Some(state) = guard.conversation.as_ref() else {
|
||||
return Err(CodexErr::InvalidRequest(
|
||||
"conversation is not running".to_string(),
|
||||
));
|
||||
@@ -1124,7 +1161,10 @@ impl RealtimeConversationManager {
|
||||
pub(crate) async fn handoff_complete(&self) -> CodexResult<()> {
|
||||
let handoff = {
|
||||
let guard = self.state.lock().await;
|
||||
guard.as_ref().map(|state| state.handoff.clone())
|
||||
guard
|
||||
.conversation
|
||||
.as_ref()
|
||||
.map(|state| state.handoff.clone())
|
||||
};
|
||||
let Some(handoff) = handoff else {
|
||||
return Ok(());
|
||||
@@ -1164,7 +1204,10 @@ impl RealtimeConversationManager {
|
||||
pub(crate) async fn clear_active_handoff(&self) {
|
||||
let handoff = {
|
||||
let guard = self.state.lock().await;
|
||||
guard.as_ref().map(|state| state.handoff.clone())
|
||||
guard
|
||||
.conversation
|
||||
.as_ref()
|
||||
.map(|state| state.handoff.clone())
|
||||
};
|
||||
if let Some(handoff) = handoff {
|
||||
{
|
||||
@@ -1179,7 +1222,7 @@ impl RealtimeConversationManager {
|
||||
pub(crate) async fn shutdown(&self) -> CodexResult<()> {
|
||||
let state = {
|
||||
let mut guard = self.state.lock().await;
|
||||
guard.take()
|
||||
guard.conversation.take()
|
||||
};
|
||||
|
||||
if let Some(state) = state {
|
||||
|
||||
@@ -3947,6 +3947,7 @@ impl Session {
|
||||
) = prepared_tools??;
|
||||
turn_context.extension_data.insert(selected_plugins);
|
||||
Ok(Arc::new(StepContext {
|
||||
realtime: self.conversation.snapshot().await,
|
||||
settings,
|
||||
token_budget,
|
||||
session_telemetry,
|
||||
|
||||
@@ -5,6 +5,7 @@ use std::sync::Arc;
|
||||
use crate::agents_md::LoadedAgentsMd;
|
||||
use crate::config::TokenBudgetConfig;
|
||||
use crate::environment_selection::TurnEnvironmentSnapshot;
|
||||
use crate::realtime_conversation::RealtimeConversationSnapshot;
|
||||
use crate::session::step_settings::ResolvedStepSettings;
|
||||
use crate::session::turn_context::TurnContext;
|
||||
use crate::tools::router::ToolRouter;
|
||||
@@ -18,6 +19,8 @@ use codex_protocol::protocol::TurnContextItem;
|
||||
/// Request-scoped state that may change between model sampling requests.
|
||||
pub(crate) struct StepContext {
|
||||
pub(crate) turn: Arc<TurnContext>,
|
||||
/// Realtime call activity and instructions captured for this sampling request.
|
||||
pub(crate) realtime: RealtimeConversationSnapshot,
|
||||
/// One immutable settings version captured before request preparation.
|
||||
pub(crate) settings: Arc<ResolvedStepSettings>,
|
||||
/// Frozen turn preferences resolved against this step's captured model.
|
||||
@@ -38,9 +41,10 @@ pub(crate) struct StepContext {
|
||||
}
|
||||
|
||||
impl StepContext {
|
||||
/// Persist the summary captured for this request, even after a live settings update.
|
||||
/// Persist the context captured for this request, even after a live update.
|
||||
pub(crate) fn to_turn_context_item(&self) -> TurnContextItem {
|
||||
let mut item = self.turn.to_turn_context_item();
|
||||
item.realtime_active = Some(self.realtime.active);
|
||||
item.summary = self.settings.reasoning_summary;
|
||||
item
|
||||
}
|
||||
|
||||
@@ -26,6 +26,7 @@ use crate::environment_selection::TurnEnvironmentState;
|
||||
use crate::function_tool::FunctionCallError;
|
||||
use crate::hook_mcp_executor::CoreHookMcpExecutor;
|
||||
use crate::plugins::plugins_manager_for_config;
|
||||
use crate::realtime_conversation::RealtimeConversationSnapshot;
|
||||
use crate::session::step_context::StepContext;
|
||||
use crate::shell::default_user_shell;
|
||||
use crate::shell_snapshot::ShellSnapshot;
|
||||
@@ -275,6 +276,10 @@ impl StepContext {
|
||||
),
|
||||
settings: Arc::new(settings),
|
||||
session_telemetry: turn.session_telemetry.clone(),
|
||||
realtime: RealtimeConversationSnapshot {
|
||||
active: turn.realtime_active,
|
||||
mode_instructions: None,
|
||||
},
|
||||
turn: Arc::clone(&turn),
|
||||
environments,
|
||||
selected_capability_roots: Vec::new(),
|
||||
|
||||
@@ -304,6 +304,7 @@ pub(crate) struct NewTurnContextOptions {
|
||||
pub struct TurnContext {
|
||||
pub(crate) sub_id: String,
|
||||
pub(crate) trace_id: Option<String>,
|
||||
/// Call state at turn creation; model requests use the `StepContext` snapshot.
|
||||
pub(crate) realtime_active: bool,
|
||||
pub(crate) code_mode_available: bool,
|
||||
/// Turn-scoped configuration. Read step-specific settings such as service tier and
|
||||
|
||||
@@ -143,17 +143,19 @@ impl Session {
|
||||
.and_then(|config| config.guidance_message.as_deref())
|
||||
.filter(|_| token_budget_enabled);
|
||||
world_state.add_section(ContextWindowGuidanceState::new(guidance));
|
||||
let realtime_mode_instructions = self.conversation.mode_instructions().await;
|
||||
let realtime = &step_context.realtime;
|
||||
world_state.add_section(RealtimeState::new(
|
||||
turn_context.realtime_active,
|
||||
realtime_mode_instructions
|
||||
realtime.active,
|
||||
realtime
|
||||
.mode_instructions
|
||||
.as_ref()
|
||||
.and_then(|instructions| instructions.start.as_deref())
|
||||
.or(turn_context
|
||||
.config
|
||||
.experimental_realtime_start_instructions
|
||||
.as_deref()),
|
||||
realtime_mode_instructions
|
||||
realtime
|
||||
.mode_instructions
|
||||
.as_ref()
|
||||
.and_then(|instructions| instructions.end.as_deref()),
|
||||
));
|
||||
|
||||
Reference in New Issue
Block a user