openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-10-29 19:08:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add realtime queue and runtime control tools

Browse Source
This commit is contained in:
Peter Bakkum
2026-03-10 21:59:00 -07:00
parent 103d384781
commit 29e856b588
12 changed files with 2108 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs
+276 -45
View File
@@ -369,7 +369,7 @@ impl RealtimeWebsocketWriter {
pub async fn send_session_update(&self, instructions: String) -> Result<(), ApiError> {
self.send_json(RealtimeOutboundMessage::SessionUpdate {
session: SessionUpdateSession {
session: Box::new(SessionUpdateSession {
kind: "realtime".to_string(),
instructions,
output_modalities: vec!["audio".to_string()],
@@ -396,54 +396,161 @@ impl RealtimeWebsocketWriter {
voice: "marin".to_string(),
},
},
tools: vec![SessionTool {
kind: "function".to_string(),
name: "codex".to_string(),
description:
"Delegate a request to Codex and return the final result to the user."
.to_string(),
parameters: SessionToolParameters {
kind: "object".to_string(),
properties: BTreeMap::from([
(
"prompt".to_string(),
SessionToolProperty {
kind: "string".to_string(),
description: "The user request to delegate to Codex."
.to_string(),
},
),
(
"send_immediately".to_string(),
SessionToolProperty {
kind: "boolean".to_string(),
description: "When true, send this to Codex immediately, steering any running turn. When false or omitted, queue it for the next turn.".to_string(),
},
),
]),
required: vec!["prompt".to_string()],
tools: vec![
SessionTool {
kind: "function".to_string(),
name: "codex".to_string(),
description:
"Delegate a request to Codex and return the final result to the user."
.to_string(),
parameters: SessionToolParameters {
kind: "object".to_string(),
properties: BTreeMap::from([
(
"prompt".to_string(),
SessionToolProperty {
kind: "string".to_string(),
description: "The user request to delegate to Codex."
.to_string(),
},
),
(
"send_immediately".to_string(),
SessionToolProperty {
kind: "boolean".to_string(),
description: "When true, send this to Codex immediately, steering any running turn. When false or omitted, queue it for the next turn.".to_string(),
},
),
]),
required: vec!["prompt".to_string()],
},
},
}, SessionTool {
kind: "function".to_string(),
name: "cancel_current_operation".to_string(),
description: "Cancel the current Codex operation, equivalent to pressing Ctrl-C without exiting Codex.".to_string(),
parameters: SessionToolParameters {
kind: "object".to_string(),
properties: BTreeMap::new(),
required: Vec::new(),
SessionTool {
kind: "function".to_string(),
name: "cancel_current_operation".to_string(),
description: "Cancel the current Codex operation, equivalent to pressing Ctrl-C without exiting Codex.".to_string(),
parameters: SessionToolParameters {
kind: "object".to_string(),
properties: BTreeMap::new(),
required: Vec::new(),
},
},
}, SessionTool {
kind: "function".to_string(),
name: "turn_off_realtime_mode".to_string(),
description: "Turn off realtime voice mode, equivalent to using /realtime to stop live voice.".to_string(),
parameters: SessionToolParameters {
kind: "object".to_string(),
properties: BTreeMap::new(),
required: Vec::new(),
SessionTool {
kind: "function".to_string(),
name: "turn_off_realtime_mode".to_string(),
description: "Turn off realtime voice mode, equivalent to using /realtime to stop live voice.".to_string(),
parameters: SessionToolParameters {
kind: "object".to_string(),
properties: BTreeMap::new(),
required: Vec::new(),
},
},
}],
SessionTool {
kind: "function".to_string(),
name: "manage_message_queue".to_string(),
description: "Inspect or edit queued draft messages. Supported actions: list, replace_last, remove_last, clear. Queue editing affects queued draft messages only, not pending steers.".to_string(),
parameters: SessionToolParameters {
kind: "object".to_string(),
properties: BTreeMap::from([
(
"action".to_string(),
SessionToolProperty {
kind: "string".to_string(),
description: "Queue action to run. Use one of: list, replace_last, remove_last, clear.".to_string(),
},
),
(
"message".to_string(),
SessionToolProperty {
kind: "string".to_string(),
description: "Replacement text for action=replace_last.".to_string(),
},
),
]),
required: vec!["action".to_string()],
},
},
SessionTool {
kind: "function".to_string(),
name: "manage_runtime_settings".to_string(),
description: "Inspect or update runtime settings for future Codex turns. Call with no setting fields to list current settings, possible settings, and allowed values. Supported setting keys: model, working_directory, reasoning_effort, fast_mode, personality, collaboration_mode. Changes are not persisted to disk.".to_string(),
parameters: SessionToolParameters {
kind: "object".to_string(),
properties: BTreeMap::from([
(
"model".to_string(),
SessionToolProperty {
kind: "string".to_string(),
description: "Optional model slug to use for future Codex turns.".to_string(),
},
),
(
"working_directory".to_string(),
SessionToolProperty {
kind: "string".to_string(),
description: "Optional working directory for future Codex turns. Relative paths are resolved against the current working directory.".to_string(),
},
),
(
"reasoning_effort".to_string(),
SessionToolProperty {
kind: "string".to_string(),
description: "Optional reasoning effort. Supported values: default, minimal, low, medium, high, xhigh.".to_string(),
},
),
(
"fast_mode".to_string(),
SessionToolProperty {
kind: "boolean".to_string(),
description: "Optional Fast mode toggle for future Codex turns.".to_string(),
},
),
(
"personality".to_string(),
SessionToolProperty {
kind: "string".to_string(),
description: "Optional personality. Supported values: none, friendly, pragmatic.".to_string(),
},
),
(
"collaboration_mode".to_string(),
SessionToolProperty {
kind: "string".to_string(),
description: "Optional collaboration mode. Supported values: default, plan.".to_string(),
},
),
]),
required: Vec::new(),
},
},
SessionTool {
kind: "function".to_string(),
name: "run_tui_command".to_string(),
description: "Run a small set of built-in TUI actions. Supported commands: compact, review, plan, diff, agent. review can take an optional prompt for custom review instructions. plan can take an optional prompt to switch to Plan mode and submit work immediately.".to_string(),
parameters: SessionToolParameters {
kind: "object".to_string(),
properties: BTreeMap::from([
(
"command".to_string(),
SessionToolProperty {
kind: "string".to_string(),
description: "Built-in TUI command to run. Use one of: compact, review, plan, diff, agent.".to_string(),
},
),
(
"prompt".to_string(),
SessionToolProperty {
kind: "string".to_string(),
description: "Optional text argument for command=review or command=plan.".to_string(),
},
),
]),
required: vec!["command".to_string()],
},
},
],
tool_choice: "auto".to_string(),
},
}),
})
.await
}
@@ -552,6 +659,7 @@ impl RealtimeWebsocketEvents {
| RealtimeEvent::ResponseCancelled(_)
| RealtimeEvent::ConversationItemAdded(_)
| RealtimeEvent::ConversationItemDone { .. }
| RealtimeEvent::ToolActionRequested(_)
| RealtimeEvent::Error(_) => {}
}
}
@@ -748,6 +856,8 @@ mod tests {
use crate::endpoint::realtime_websocket::protocol::RealtimeInterruptRequested;
use crate::endpoint::realtime_websocket::protocol::RealtimeOutputAudioDelta;
use crate::endpoint::realtime_websocket::protocol::RealtimeResponseCancelled;
use crate::endpoint::realtime_websocket::protocol::RealtimeToolAction;
use crate::endpoint::realtime_websocket::protocol::RealtimeToolActionRequested;
use crate::endpoint::realtime_websocket::protocol::RealtimeTranscriptDelta;
use crate::endpoint::realtime_websocket::protocol::RealtimeTranscriptEntry;
use http::HeaderValue;
@@ -989,6 +1099,106 @@ mod tests {
);
}
#[test]
fn parse_manage_message_queue_requested_event() {
let payload = json!({
"type": "response.done",
"response": {
"output": [
{
"id": "item_queue",
"type": "function_call",
"name": "manage_message_queue",
"call_id": "queue_123",
"arguments": "{\"action\":\"list\"}"
}
]
}
})
.to_string();
assert_eq!(
parse_realtime_event(payload.as_str()),
Some(RealtimeEvent::ToolActionRequested(
RealtimeToolActionRequested {
call_id: "queue_123".to_string(),
action: RealtimeToolAction::ManageMessageQueue {
action: "list".to_string(),
message: None,
},
}
))
);
}
#[test]
fn parse_manage_runtime_settings_requested_event() {
let payload = json!({
"type": "response.done",
"response": {
"output": [
{
"id": "item_settings",
"type": "function_call",
"name": "manage_runtime_settings",
"call_id": "settings_123",
"arguments": "{\"model\":\"gpt-5\",\"working_directory\":\"src\",\"reasoning_effort\":\"low\"}"
}
]
}
})
.to_string();
assert_eq!(
parse_realtime_event(payload.as_str()),
Some(RealtimeEvent::ToolActionRequested(
RealtimeToolActionRequested {
call_id: "settings_123".to_string(),
action: RealtimeToolAction::ManageRuntimeSettings {
model: Some("gpt-5".to_string()),
working_directory: Some("src".to_string()),
reasoning_effort: Some("low".to_string()),
fast_mode: None,
personality: None,
collaboration_mode: None,
},
}
))
);
}
#[test]
fn parse_run_tui_command_requested_event() {
let payload = json!({
"type": "response.done",
"response": {
"output": [
{
"id": "item_command",
"type": "function_call",
"name": "run_tui_command",
"call_id": "command_123",
"arguments": "{\"command\":\"plan\",\"prompt\":\"make a plan\"}"
}
]
}
})
.to_string();
assert_eq!(
parse_realtime_event(payload.as_str()),
Some(RealtimeEvent::ToolActionRequested(
RealtimeToolActionRequested {
call_id: "command_123".to_string(),
action: RealtimeToolAction::RunTuiCommand {
command: "plan".to_string(),
prompt: Some("make a plan".to_string()),
},
}
))
);
}
#[test]
fn parse_input_audio_speech_started_event() {
let payload = json!({
@@ -1295,6 +1505,27 @@ mod tests {
first_json["session"]["tools"][2]["parameters"]["properties"],
json!({})
);
assert_eq!(
first_json["session"]["tools"]
.as_array()
.expect("tools array")
.iter()
.map(|tool| tool["name"].as_str().expect("tool name"))
.collect::<Vec<_>>(),
vec![
"codex",
"cancel_current_operation",
"turn_off_realtime_mode",
"manage_message_queue",
"manage_runtime_settings",
"run_tui_command",
]
);
assert_eq!(
first_json["session"]["tools"][4]["parameters"]["properties"]["working_directory"]
["type"],
Value::String("string".to_string())
);
ws.send(Message::Text(
json!({
codex-rs/codex-api/src/endpoint/realtime_websocket/protocol.rs
+179 -1
View File
@@ -6,6 +6,8 @@ pub use codex_protocol::protocol::RealtimeInputAudioSpeechStarted;
pub use codex_protocol::protocol::RealtimeInterruptRequested;
pub use codex_protocol::protocol::RealtimeOutputAudioDelta;
pub use codex_protocol::protocol::RealtimeResponseCancelled;
pub use codex_protocol::protocol::RealtimeToolAction;
pub use codex_protocol::protocol::RealtimeToolActionRequested;
pub use codex_protocol::protocol::RealtimeTranscriptDelta;
pub use codex_protocol::protocol::RealtimeTranscriptEntry;
use serde::Deserialize;
@@ -36,7 +38,7 @@ pub(super) enum RealtimeOutboundMessage {
audio_end_ms: u32,
},
#[serde(rename = "session.update")]
SessionUpdate { session: SessionUpdateSession },
SessionUpdate { session: Box<SessionUpdateSession> },
#[serde(rename = "conversation.item.create")]
ConversationItemCreate { item: ConversationItem },
}
@@ -273,6 +275,9 @@ pub(super) fn parse_realtime_event(payload: &str) -> Option<RealtimeEvent> {
if let Some(close) = parse_close_requested(&parsed) {
return Some(RealtimeEvent::CloseRequested(close));
}
if let Some(tool_action) = parse_tool_action_requested(&parsed) {
return Some(RealtimeEvent::ToolActionRequested(tool_action));
}
if let Some(cancelled) = parse_response_cancelled(&parsed) {
return Some(RealtimeEvent::ResponseCancelled(cancelled));
}
@@ -357,6 +362,179 @@ fn parse_close_requested(parsed: &Value) -> Option<RealtimeCloseRequested> {
})
}
fn parse_tool_action_requested(parsed: &Value) -> Option<RealtimeToolActionRequested> {
#[derive(Debug, Deserialize, Default)]
struct ReplaceLastQueuedMessageArguments {
#[serde(default)]
message: String,
}
#[derive(Debug, Deserialize, Default)]
struct ManageMessageQueueArguments {
#[serde(default)]
action: String,
#[serde(default)]
message: Option<String>,
}
#[derive(Debug, Deserialize, Default)]
struct UpdateRuntimeSettingsArguments {
#[serde(default)]
model: Option<String>,
#[serde(default)]
working_directory: Option<String>,
#[serde(default)]
reasoning_effort: Option<String>,
#[serde(default)]
fast_mode: Option<bool>,
#[serde(default)]
personality: Option<String>,
#[serde(default)]
collaboration_mode: Option<String>,
}
#[derive(Debug, Deserialize, Default)]
struct RunTuiCommandArguments {
#[serde(default)]
command: String,
#[serde(default)]
prompt: Option<String>,
}
let parse_call_id = |function_call: &Value| {
function_call
.get("call_id")
.and_then(Value::as_str)
.map(str::to_string)
};
if let Some(function_call) = find_function_call(parsed, "manage_message_queue") {
let arguments = function_call
.get("arguments")
.and_then(Value::as_str)
.unwrap_or_default();
let parsed_arguments =
serde_json::from_str::<ManageMessageQueueArguments>(arguments).unwrap_or_default();
return Some(RealtimeToolActionRequested {
call_id: parse_call_id(function_call)?,
action: RealtimeToolAction::ManageMessageQueue {
action: parsed_arguments.action,
message: parsed_arguments.message,
},
});
}
if let Some(function_call) = find_function_call(parsed, "list_message_queue") {
return Some(RealtimeToolActionRequested {
call_id: parse_call_id(function_call)?,
action: RealtimeToolAction::ListMessageQueue,
});
}
if let Some(function_call) = find_function_call(parsed, "replace_last_queued_message") {
let arguments = function_call
.get("arguments")
.and_then(Value::as_str)
.unwrap_or_default();
let parsed_arguments = serde_json::from_str::<ReplaceLastQueuedMessageArguments>(arguments)
.unwrap_or_else(|_| ReplaceLastQueuedMessageArguments {
message: arguments.to_string(),
});
return Some(RealtimeToolActionRequested {
call_id: parse_call_id(function_call)?,
action: RealtimeToolAction::ReplaceLastQueuedMessage {
message: parsed_arguments.message,
},
});
}
if let Some(function_call) = find_function_call(parsed, "remove_last_queued_message") {
return Some(RealtimeToolActionRequested {
call_id: parse_call_id(function_call)?,
action: RealtimeToolAction::RemoveLastQueuedMessage,
});
}
if let Some(function_call) = find_function_call(parsed, "clear_queued_messages") {
return Some(RealtimeToolActionRequested {
call_id: parse_call_id(function_call)?,
action: RealtimeToolAction::ClearQueuedMessages,
});
}
if let Some(function_call) = find_function_call(parsed, "list_runtime_settings") {
return Some(RealtimeToolActionRequested {
call_id: parse_call_id(function_call)?,
action: RealtimeToolAction::ListRuntimeSettings,
});
}
if let Some(function_call) = find_function_call(parsed, "manage_runtime_settings") {
let arguments = function_call
.get("arguments")
.and_then(Value::as_str)
.unwrap_or_default();
let parsed_arguments =
serde_json::from_str::<UpdateRuntimeSettingsArguments>(arguments).unwrap_or_default();
return Some(RealtimeToolActionRequested {
call_id: parse_call_id(function_call)?,
action: RealtimeToolAction::ManageRuntimeSettings {
model: parsed_arguments.model,
working_directory: parsed_arguments.working_directory,
reasoning_effort: parsed_arguments.reasoning_effort,
fast_mode: parsed_arguments.fast_mode,
personality: parsed_arguments.personality,
collaboration_mode: parsed_arguments.collaboration_mode,
},
});
}
if let Some(function_call) = find_function_call(parsed, "update_runtime_settings") {
let arguments = function_call
.get("arguments")
.and_then(Value::as_str)
.unwrap_or_default();
let parsed_arguments =
serde_json::from_str::<UpdateRuntimeSettingsArguments>(arguments).unwrap_or_default();
return Some(RealtimeToolActionRequested {
call_id: parse_call_id(function_call)?,
action: RealtimeToolAction::UpdateRuntimeSettings {
model: parsed_arguments.model,
working_directory: parsed_arguments.working_directory,
reasoning_effort: parsed_arguments.reasoning_effort,
fast_mode: parsed_arguments.fast_mode,
personality: parsed_arguments.personality,
collaboration_mode: parsed_arguments.collaboration_mode,
},
});
}
if let Some(function_call) = find_function_call(parsed, "run_tui_command") {
let arguments = function_call
.get("arguments")
.and_then(Value::as_str)
.unwrap_or_default();
let parsed_arguments =
serde_json::from_str::<RunTuiCommandArguments>(arguments).unwrap_or_default();
return Some(RealtimeToolActionRequested {
call_id: parse_call_id(function_call)?,
action: RealtimeToolAction::RunTuiCommand {
command: parsed_arguments.command,
prompt: parsed_arguments.prompt,
},
});
}
if let Some(function_call) = find_function_call(parsed, "compact_conversation") {
return Some(RealtimeToolActionRequested {
call_id: parse_call_id(function_call)?,
action: RealtimeToolAction::CompactConversation,
});
}
None
}
fn parse_response_cancelled(parsed: &Value) -> Option<RealtimeResponseCancelled> {
let response = parsed.get("response")?.as_object()?;
let status = response.get("status").and_then(Value::as_str)?;