openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-10-29 19:08:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into codex/find-method-to-set-codex_session_id

Browse Source
This commit is contained in:
Ilan Bigio
2025-10-24 11:10:34 -07:00
parent c0530ebc06 a6b9471548
commit 6f978a9f52
76 changed files with 5049 additions and 2257 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/Cargo.lock
Generated
+2 -14
View File
@@ -1062,12 +1062,12 @@ dependencies = [
"codex-apply-patch",
"codex-async-utils",
"codex-file-search",
"codex-mcp-client",
"codex-otel",
"codex-protocol",
"codex-rmcp-client",
"codex-utils-pty",
"codex-utils-string",
"codex-utils-tokenizer",
"core-foundation 0.9.4",
"core_test_support",
"dirs",
@@ -1250,19 +1250,6 @@ dependencies = [
"wiremock",
]
[[package]]
name = "codex-mcp-client"
version = "0.0.0"
dependencies = [
"anyhow",
"mcp-types",
"serde",
"serde_json",
"tokio",
"tracing",
"tracing-subscriber",
]
[[package]]
name = "codex-mcp-server"
version = "0.0.0"
@@ -1648,6 +1635,7 @@ dependencies = [
"anyhow",
"assert_cmd",
"codex-core",
"codex-protocol",
"notify",
"regex-lite",
"serde_json",
codex-rs/Cargo.toml
-2
View File
@@ -20,7 +20,6 @@ members = [
"git-tooling",
"linux-sandbox",
"login",
"mcp-client",
"mcp-server",
"mcp-types",
"ollama",
@@ -68,7 +67,6 @@ codex-file-search = { path = "file-search" }
codex-git-tooling = { path = "git-tooling" }
codex-linux-sandbox = { path = "linux-sandbox" }
codex-login = { path = "login" }
codex-mcp-client = { path = "mcp-client" }
codex-mcp-server = { path = "mcp-server" }
codex-ollama = { path = "ollama" }
codex-otel = { path = "otel" }
codex-rs/app-server-protocol/src/protocol.rs
+153 -15
View File
@@ -5,6 +5,7 @@ use crate::JSONRPCNotification;
use crate::JSONRPCRequest;
use crate::RequestId;
use codex_protocol::ConversationId;
use codex_protocol::account::Account;
use codex_protocol::config_types::ForcedLoginMethod;
use codex_protocol::config_types::ReasoningEffort;
use codex_protocol::config_types::ReasoningSummary;
@@ -93,6 +94,43 @@ macro_rules! client_request_definitions {
}
client_request_definitions! {
/// NEW APIs
#[serde(rename = "model/list")]
#[ts(rename = "model/list")]
ListModels {
params: ListModelsParams,
response: ListModelsResponse,
},
#[serde(rename = "account/login")]
#[ts(rename = "account/login")]
LoginAccount {
params: LoginAccountParams,
response: LoginAccountResponse,
},
#[serde(rename = "account/logout")]
#[ts(rename = "account/logout")]
LogoutAccount {
params: #[ts(type = "undefined")] #[serde(skip_serializing_if = "Option::is_none")] Option<()>,
response: LogoutAccountResponse,
},
#[serde(rename = "account/rateLimits/read")]
#[ts(rename = "account/rateLimits/read")]
GetAccountRateLimits {
params: #[ts(type = "undefined")] #[serde(skip_serializing_if = "Option::is_none")] Option<()>,
response: GetAccountRateLimitsResponse,
},
#[serde(rename = "account/read")]
#[ts(rename = "account/read")]
GetAccount {
params: #[ts(type = "undefined")] #[serde(skip_serializing_if = "Option::is_none")] Option<()>,
response: Option<Account>,
},
/// DEPRECATED APIs below
Initialize {
params: InitializeParams,
response: InitializeResponse,
@@ -106,13 +144,6 @@ client_request_definitions! {
params: ListConversationsParams,
response: ListConversationsResponse,
},
#[serde(rename = "model/list")]
#[ts(rename = "model/list")]
/// List available Codex models along with display metadata.
ListModels {
params: ListModelsParams,
response: ListModelsResponse,
},
/// Resume a recorded Codex conversation from a rollout file.
ResumeConversation {
params: ResumeConversationParams,
@@ -191,12 +222,6 @@ client_request_definitions! {
params: ExecOneOffCommandParams,
response: ExecOneOffCommandResponse,
},
#[serde(rename = "account/rateLimits/read")]
#[ts(rename = "account/rateLimits/read")]
GetAccountRateLimits {
params: #[ts(type = "undefined")] #[serde(skip_serializing_if = "Option::is_none")] Option<()>,
response: GetAccountRateLimitsResponse,
},
}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default, JsonSchema, TS)]
@@ -352,6 +377,38 @@ pub struct ListModelsResponse {
pub next_cursor: Option<String>,
}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
#[serde(tag = "type")]
#[ts(tag = "type")]
pub enum LoginAccountParams {
#[serde(rename = "apiKey")]
#[ts(rename = "apiKey")]
ApiKey {
#[serde(rename = "apiKey")]
#[ts(rename = "apiKey")]
api_key: String,
},
#[serde(rename = "chatgpt")]
#[ts(rename = "chatgpt")]
ChatGpt,
}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
pub struct LoginAccountResponse {
/// Only set if the login method is ChatGPT.
#[schemars(with = "String")]
pub login_id: Option<Uuid>,
/// URL the client should open in a browser to initiate the OAuth flow.
/// Only set if the login method is ChatGPT.
pub auth_url: Option<String>,
}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
pub struct LogoutAccountResponse {}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
pub struct ResumeConversationParams {
@@ -875,6 +932,13 @@ pub struct AuthStatusChangeNotification {
#[serde(tag = "method", content = "params", rename_all = "camelCase")]
#[strum(serialize_all = "camelCase")]
pub enum ServerNotification {
/// NEW NOTIFICATIONS
#[serde(rename = "account/rateLimits/updated")]
#[ts(rename = "account/rateLimits/updated")]
#[strum(serialize = "account/rateLimits/updated")]
AccountRateLimitsUpdated(RateLimitSnapshot),
/// DEPRECATED NOTIFICATIONS below
/// Authentication status changed
AuthStatusChange(AuthStatusChangeNotification),
@@ -888,6 +952,7 @@ pub enum ServerNotification {
impl ServerNotification {
pub fn to_params(self) -> Result<serde_json::Value, serde_json::Error> {
match self {
ServerNotification::AccountRateLimitsUpdated(params) => serde_json::to_value(params),
ServerNotification::AuthStatusChange(params) => serde_json::to_value(params),
ServerNotification::LoginChatGptComplete(params) => serde_json::to_value(params),
ServerNotification::SessionConfigured(params) => serde_json::to_value(params),
@@ -1043,16 +1108,89 @@ mod tests {
Ok(())
}
#[test]
fn serialize_account_login_api_key() -> Result<()> {
let request = ClientRequest::LoginAccount {
request_id: RequestId::Integer(2),
params: LoginAccountParams::ApiKey {
api_key: "secret".to_string(),
},
};
assert_eq!(
json!({
"method": "account/login",
"id": 2,
"params": {
"type": "apiKey",
"apiKey": "secret"
}
}),
serde_json::to_value(&request)?,
);
Ok(())
}
#[test]
fn serialize_account_login_chatgpt() -> Result<()> {
let request = ClientRequest::LoginAccount {
request_id: RequestId::Integer(3),
params: LoginAccountParams::ChatGpt,
};
assert_eq!(
json!({
"method": "account/login",
"id": 3,
"params": {
"type": "chatgpt"
}
}),
serde_json::to_value(&request)?,
);
Ok(())
}
#[test]
fn serialize_account_logout() -> Result<()> {
let request = ClientRequest::LogoutAccount {
request_id: RequestId::Integer(4),
params: None,
};
assert_eq!(
json!({
"method": "account/logout",
"id": 4,
}),
serde_json::to_value(&request)?,
);
Ok(())
}
#[test]
fn serialize_get_account() -> Result<()> {
let request = ClientRequest::GetAccount {
request_id: RequestId::Integer(5),
params: None,
};
assert_eq!(
json!({
"method": "account/read",
"id": 5,
}),
serde_json::to_value(&request)?,
);
Ok(())
}
#[test]
fn serialize_list_models() -> Result<()> {
let request = ClientRequest::ListModels {
request_id: RequestId::Integer(2),
request_id: RequestId::Integer(6),
params: ListModelsParams::default(),
};
assert_eq!(
json!({
"method": "model/list",
"id": 2,
"id": 6,
"params": {}
}),
serde_json::to_value(&request)?,
codex-rs/app-server/src/codex_message_processor.rs
+45 -26
View File
@@ -90,9 +90,8 @@ use codex_login::ShutdownHandle;
use codex_login::run_login_server;
use codex_protocol::ConversationId;
use codex_protocol::config_types::ForcedLoginMethod;
use codex_protocol::models::ContentItem;
use codex_protocol::items::TurnItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::InputMessageKind;
use codex_protocol::protocol::RateLimitSnapshot;
use codex_protocol::protocol::USER_MESSAGE_BEGIN;
use codex_protocol::user_input::UserInput as CoreInputItem;
@@ -177,6 +176,27 @@ impl CodexMessageProcessor {
ClientRequest::ListModels { request_id, params } => {
self.list_models(request_id, params).await;
}
ClientRequest::LoginAccount {
request_id,
params: _,
} => {
self.send_unimplemented_error(request_id, "account/login")
.await;
}
ClientRequest::LogoutAccount {
request_id,
params: _,
} => {
self.send_unimplemented_error(request_id, "account/logout")
.await;
}
ClientRequest::GetAccount {
request_id,
params: _,
} => {
self.send_unimplemented_error(request_id, "account/read")
.await;
}
ClientRequest::ResumeConversation { request_id, params } => {
self.handle_resume_conversation(request_id, params).await;
}
@@ -258,6 +278,15 @@ impl CodexMessageProcessor {
}
}
async fn send_unimplemented_error(&self, request_id: RequestId, method: &str) {
let error = JSONRPCErrorError {
code: INTERNAL_ERROR_CODE,
message: format!("{method} is not implemented yet"),
data: None,
};
self.outgoing.send_error(request_id, error).await;
}
async fn login_api_key(&mut self, request_id: RequestId, params: LoginApiKeyParams) {
if matches!(
self.config.forced_login_method,
@@ -940,18 +969,9 @@ impl CodexMessageProcessor {
},
))
.await;
let initial_messages = session_configured.initial_messages.map(|msgs| {
msgs.into_iter()
.filter(|event| {
// Don't send non-plain user messages (like user instructions
// or environment context) back so they don't get rendered.
if let EventMsg::UserMessage(user_message) = event {
return matches!(user_message.kind, Some(InputMessageKind::Plain));
}
true
})
.collect()
});
let initial_messages = session_configured
.initial_messages
.map(|msgs| msgs.into_iter().collect());
// Reply with conversation id + model and initial messages (when present)
let response = codex_app_server_protocol::ResumeConversationResponse {
@@ -1446,6 +1466,15 @@ async fn apply_bespoke_event_handling(
on_exec_approval_response(event_id, rx, conversation).await;
});
}
EventMsg::TokenCount(token_count_event) => {
if let Some(rate_limits) = token_count_event.rate_limits {
outgoing
.send_server_notification(ServerNotification::AccountRateLimitsUpdated(
rate_limits,
))
.await;
}
}
// If this is a TurnAborted, reply to any pending interrupt requests.
EventMsg::TurnAborted(turn_aborted_event) => {
let pending = {
@@ -1596,18 +1625,8 @@ fn extract_conversation_summary(
let preview = head
.iter()
.filter_map(|value| serde_json::from_value::<ResponseItem>(value.clone()).ok())
.find_map(|item| match item {
ResponseItem::Message { content, .. } => {
content.into_iter().find_map(|content| match content {
ContentItem::InputText { text } => {
match InputMessageKind::from(("user", &text)) {
InputMessageKind::Plain => Some(text),
_ => None,
}
}
_ => None,
})
}
.find_map(|item| match codex_core::parse_turn_item(&item) {
Some(TurnItem::UserMessage(user)) => Some(user.message()),
_ => None,
})?;
codex-rs/app-server/src/fuzzy_file_search.rs
+1
View File
@@ -46,6 +46,7 @@ pub(crate) async fn run_fuzzy_file_search(
threads,
cancel_flag,
COMPUTE_INDICES,
true,
) {
Ok(res) => Ok((root, res)),
Err(err) => Err((root, err)),
codex-rs/app-server/src/outgoing_message.rs
+32
View File
@@ -142,6 +142,8 @@ pub(crate) struct OutgoingError {
#[cfg(test)]
mod tests {
use codex_app_server_protocol::LoginChatGptCompleteNotification;
use codex_protocol::protocol::RateLimitSnapshot;
use codex_protocol::protocol::RateLimitWindow;
use pretty_assertions::assert_eq;
use serde_json::json;
use uuid::Uuid;
@@ -171,4 +173,34 @@ mod tests {
"ensure the strum macros serialize the method field correctly"
);
}
#[test]
fn verify_account_rate_limits_notification_serialization() {
let notification = ServerNotification::AccountRateLimitsUpdated(RateLimitSnapshot {
primary: Some(RateLimitWindow {
used_percent: 25.0,
window_minutes: Some(15),
resets_at: Some(123),
}),
secondary: None,
});
let jsonrpc_notification = OutgoingMessage::AppServerNotification(notification);
assert_eq!(
json!({
"method": "account/rateLimits/updated",
"params": {
"primary": {
"used_percent": 25.0,
"window_minutes": 15,
"resets_at": 123,
},
"secondary": null,
},
}),
serde_json::to_value(jsonrpc_notification)
.expect("ensure the notification serializes correctly"),
"ensure the notification serializes correctly"
);
}
}
codex-rs/app-server/tests/suite/codex_message_processor_flow.rs
-38
View File
@@ -30,7 +30,6 @@ use codex_protocol::config_types::SandboxMode;
use codex_protocol::parse_command::ParsedCommand;
use codex_protocol::protocol::Event;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::InputMessageKind;
use pretty_assertions::assert_eq;
use std::env;
use tempfile::TempDir;
@@ -528,43 +527,6 @@ async fn test_send_user_turn_updates_sandbox_and_cwd_between_turns() {
.expect("sendUserTurn 2 timeout")
.expect("sendUserTurn 2 resp");
let mut env_message: Option<String> = None;
let second_cwd_str = second_cwd.to_string_lossy().into_owned();
for _ in 0..10 {
let notification = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_notification_message("codex/event/user_message"),
)
.await
.expect("user_message timeout")
.expect("user_message notification");
let params = notification
.params
.clone()
.expect("user_message should include params");
let event: Event = serde_json::from_value(params).expect("deserialize user_message event");
if let EventMsg::UserMessage(user) = event.msg
&& matches!(user.kind, Some(InputMessageKind::EnvironmentContext))
&& user.message.contains(&second_cwd_str)
{
env_message = Some(user.message);
break;
}
}
let env_message = env_message.expect("expected environment context update");
assert!(
env_message.contains("<sandbox_mode>danger-full-access</sandbox_mode>"),
"env context should reflect new sandbox mode: {env_message}"
);
assert!(
env_message.contains("<network_access>enabled</network_access>"),
"env context should enable network access for danger-full-access policy: {env_message}"
);
assert!(
env_message.contains(&second_cwd_str),
"env context should include updated cwd: {env_message}"
);
let exec_begin_notification = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_notification_message("codex/event/exec_command_begin"),
codex-rs/apply-patch/tests/suite/mod.rs
+2
View File
@@ -1 +1,3 @@
mod cli;
#[cfg(not(target_os = "windows"))]
mod tool;
codex-rs/apply-patch/tests/suite/tool.rs
+257
View File
@@ -0,0 +1,257 @@
use assert_cmd::Command;
use pretty_assertions::assert_eq;
use std::fs;
use std::path::Path;
use tempfile::tempdir;
fn run_apply_patch_in_dir(dir: &Path, patch: &str) -> anyhow::Result<assert_cmd::assert::Assert> {
let mut cmd = Command::cargo_bin("apply_patch")?;
cmd.current_dir(dir);
Ok(cmd.arg(patch).assert())
}
fn apply_patch_command(dir: &Path) -> anyhow::Result<Command> {
let mut cmd = Command::cargo_bin("apply_patch")?;
cmd.current_dir(dir);
Ok(cmd)
}
#[test]
fn test_apply_patch_cli_applies_multiple_operations() -> anyhow::Result<()> {
let tmp = tempdir()?;
let modify_path = tmp.path().join("modify.txt");
let delete_path = tmp.path().join("delete.txt");
fs::write(&modify_path, "line1\nline2\n")?;
fs::write(&delete_path, "obsolete\n")?;
let patch = "*** Begin Patch\n*** Add File: nested/new.txt\n+created\n*** Delete File: delete.txt\n*** Update File: modify.txt\n@@\n-line2\n+changed\n*** End Patch";
run_apply_patch_in_dir(tmp.path(), patch)?.success().stdout(
"Success. Updated the following files:\nA nested/new.txt\nM modify.txt\nD delete.txt\n",
);
assert_eq!(
fs::read_to_string(tmp.path().join("nested/new.txt"))?,
"created\n"
);
assert_eq!(fs::read_to_string(&modify_path)?, "line1\nchanged\n");
assert!(!delete_path.exists());
Ok(())
}
#[test]
fn test_apply_patch_cli_applies_multiple_chunks() -> anyhow::Result<()> {
let tmp = tempdir()?;
let target_path = tmp.path().join("multi.txt");
fs::write(&target_path, "line1\nline2\nline3\nline4\n")?;
let patch = "*** Begin Patch\n*** Update File: multi.txt\n@@\n-line2\n+changed2\n@@\n-line4\n+changed4\n*** End Patch";
run_apply_patch_in_dir(tmp.path(), patch)?
.success()
.stdout("Success. Updated the following files:\nM multi.txt\n");
assert_eq!(
fs::read_to_string(&target_path)?,
"line1\nchanged2\nline3\nchanged4\n"
);
Ok(())
}
#[test]
fn test_apply_patch_cli_moves_file_to_new_directory() -> anyhow::Result<()> {
let tmp = tempdir()?;
let original_path = tmp.path().join("old/name.txt");
let new_path = tmp.path().join("renamed/dir/name.txt");
fs::create_dir_all(original_path.parent().expect("parent should exist"))?;
fs::write(&original_path, "old content\n")?;
let patch = "*** Begin Patch\n*** Update File: old/name.txt\n*** Move to: renamed/dir/name.txt\n@@\n-old content\n+new content\n*** End Patch";
run_apply_patch_in_dir(tmp.path(), patch)?
.success()
.stdout("Success. Updated the following files:\nM renamed/dir/name.txt\n");
assert!(!original_path.exists());
assert_eq!(fs::read_to_string(&new_path)?, "new content\n");
Ok(())
}
#[test]
fn test_apply_patch_cli_rejects_empty_patch() -> anyhow::Result<()> {
let tmp = tempdir()?;
apply_patch_command(tmp.path())?
.arg("*** Begin Patch\n*** End Patch")
.assert()
.failure()
.stderr("No files were modified.\n");
Ok(())
}
#[test]
fn test_apply_patch_cli_reports_missing_context() -> anyhow::Result<()> {
let tmp = tempdir()?;
let target_path = tmp.path().join("modify.txt");
fs::write(&target_path, "line1\nline2\n")?;
apply_patch_command(tmp.path())?
.arg("*** Begin Patch\n*** Update File: modify.txt\n@@\n-missing\n+changed\n*** End Patch")
.assert()
.failure()
.stderr("Failed to find expected lines in modify.txt:\nmissing\n");
assert_eq!(fs::read_to_string(&target_path)?, "line1\nline2\n");
Ok(())
}
#[test]
fn test_apply_patch_cli_rejects_missing_file_delete() -> anyhow::Result<()> {
let tmp = tempdir()?;
apply_patch_command(tmp.path())?
.arg("*** Begin Patch\n*** Delete File: missing.txt\n*** End Patch")
.assert()
.failure()
.stderr("Failed to delete file missing.txt\n");
Ok(())
}
#[test]
fn test_apply_patch_cli_rejects_empty_update_hunk() -> anyhow::Result<()> {
let tmp = tempdir()?;
apply_patch_command(tmp.path())?
.arg("*** Begin Patch\n*** Update File: foo.txt\n*** End Patch")
.assert()
.failure()
.stderr("Invalid patch hunk on line 2: Update file hunk for path 'foo.txt' is empty\n");
Ok(())
}
#[test]
fn test_apply_patch_cli_requires_existing_file_for_update() -> anyhow::Result<()> {
let tmp = tempdir()?;
apply_patch_command(tmp.path())?
.arg("*** Begin Patch\n*** Update File: missing.txt\n@@\n-old\n+new\n*** End Patch")
.assert()
.failure()
.stderr(
"Failed to read file to update missing.txt: No such file or directory (os error 2)\n",
);
Ok(())
}
#[test]
fn test_apply_patch_cli_move_overwrites_existing_destination() -> anyhow::Result<()> {
let tmp = tempdir()?;
let original_path = tmp.path().join("old/name.txt");
let destination = tmp.path().join("renamed/dir/name.txt");
fs::create_dir_all(original_path.parent().expect("parent should exist"))?;
fs::create_dir_all(destination.parent().expect("parent should exist"))?;
fs::write(&original_path, "from\n")?;
fs::write(&destination, "existing\n")?;
run_apply_patch_in_dir(
tmp.path(),
"*** Begin Patch\n*** Update File: old/name.txt\n*** Move to: renamed/dir/name.txt\n@@\n-from\n+new\n*** End Patch",
)?
.success()
.stdout("Success. Updated the following files:\nM renamed/dir/name.txt\n");
assert!(!original_path.exists());
assert_eq!(fs::read_to_string(&destination)?, "new\n");
Ok(())
}
#[test]
fn test_apply_patch_cli_add_overwrites_existing_file() -> anyhow::Result<()> {
let tmp = tempdir()?;
let path = tmp.path().join("duplicate.txt");
fs::write(&path, "old content\n")?;
run_apply_patch_in_dir(
tmp.path(),
"*** Begin Patch\n*** Add File: duplicate.txt\n+new content\n*** End Patch",
)?
.success()
.stdout("Success. Updated the following files:\nA duplicate.txt\n");
assert_eq!(fs::read_to_string(&path)?, "new content\n");
Ok(())
}
#[test]
fn test_apply_patch_cli_delete_directory_fails() -> anyhow::Result<()> {
let tmp = tempdir()?;
fs::create_dir(tmp.path().join("dir"))?;
apply_patch_command(tmp.path())?
.arg("*** Begin Patch\n*** Delete File: dir\n*** End Patch")
.assert()
.failure()
.stderr("Failed to delete file dir\n");
Ok(())
}
#[test]
fn test_apply_patch_cli_rejects_invalid_hunk_header() -> anyhow::Result<()> {
let tmp = tempdir()?;
apply_patch_command(tmp.path())?
.arg("*** Begin Patch\n*** Frobnicate File: foo\n*** End Patch")
.assert()
.failure()
.stderr("Invalid patch hunk on line 2: '*** Frobnicate File: foo' is not a valid hunk header. Valid hunk headers: '*** Add File: {path}', '*** Delete File: {path}', '*** Update File: {path}'\n");
Ok(())
}
#[test]
fn test_apply_patch_cli_updates_file_appends_trailing_newline() -> anyhow::Result<()> {
let tmp = tempdir()?;
let target_path = tmp.path().join("no_newline.txt");
fs::write(&target_path, "no newline at end")?;
run_apply_patch_in_dir(
tmp.path(),
"*** Begin Patch\n*** Update File: no_newline.txt\n@@\n-no newline at end\n+first line\n+second line\n*** End Patch",
)?
.success()
.stdout("Success. Updated the following files:\nM no_newline.txt\n");
let contents = fs::read_to_string(&target_path)?;
assert!(contents.ends_with('\n'));
assert_eq!(contents, "first line\nsecond line\n");
Ok(())
}
#[test]
fn test_apply_patch_cli_failure_after_partial_success_leaves_changes() -> anyhow::Result<()> {
let tmp = tempdir()?;
let new_file = tmp.path().join("created.txt");
apply_patch_command(tmp.path())?
.arg("*** Begin Patch\n*** Add File: created.txt\n+hello\n*** Update File: missing.txt\n@@\n-old\n+new\n*** End Patch")
.assert()
.failure()
.stdout("")
.stderr("Failed to read file to update missing.txt: No such file or directory (os error 2)\n");
assert_eq!(fs::read_to_string(&new_file)?, "hello\n");
Ok(())
}
codex-rs/core/Cargo.toml
+1 -1
View File
@@ -22,13 +22,13 @@ chrono = { workspace = true, features = ["serde"] }
codex-app-server-protocol = { workspace = true }
codex-apply-patch = { workspace = true }
codex-file-search = { workspace = true }
codex-mcp-client = { workspace = true }
codex-otel = { workspace = true, features = ["otel"] }
codex-protocol = { workspace = true }
codex-rmcp-client = { workspace = true }
codex-async-utils = { workspace = true }
codex-utils-string = { workspace = true }
codex-utils-pty = { workspace = true }
codex-utils-tokenizer = { workspace = true }
dirs = { workspace = true }
dunce = { workspace = true }
env-flags = { workspace = true }
codex-rs/core/src/chat_completions.rs
+4 -4
View File
@@ -104,10 +104,10 @@ pub(crate) async fn stream_chat_completions(
} = item
{
let mut text = String::new();
for c in items {
match c {
ReasoningItemContent::ReasoningText { text: t }
| ReasoningItemContent::Text { text: t } => text.push_str(t),
for entry in items {
match entry {
ReasoningItemContent::ReasoningText { text: segment }
| ReasoningItemContent::Text { text: segment } => text.push_str(segment),
}
}
if text.trim().is_empty() {
codex-rs/core/src/codex.rs
+193 -267
View File
@@ -1,4 +1,3 @@
use std::borrow::Cow;
use std::collections::HashMap;
use std::fmt::Debug;
use std::path::PathBuf;
@@ -7,12 +6,12 @@ use std::sync::atomic::AtomicU64;
use crate::AuthManager;
use crate::client_common::REVIEW_PROMPT;
use crate::event_mapping::map_response_item_to_event_messages;
use crate::function_tool::FunctionCallError;
use crate::mcp::auth::McpAuthStatusEntry;
use crate::parse_command::parse_command;
use crate::parse_turn_item;
use crate::response_processing::process_items;
use crate::review_format::format_review_findings_block;
use crate::state::ItemCollector;
use crate::terminal;
use crate::user_notification::UserNotifier;
use async_channel::Receiver;
@@ -20,9 +19,10 @@ use async_channel::Sender;
use codex_apply_patch::ApplyPatchAction;
use codex_protocol::ConversationId;
use codex_protocol::items::TurnItem;
use codex_protocol::items::UserMessageItem;
use codex_protocol::protocol::ConversationPathResponseEvent;
use codex_protocol::protocol::ExitedReviewModeEvent;
use codex_protocol::protocol::ItemCompletedEvent;
use codex_protocol::protocol::ItemStartedEvent;
use codex_protocol::protocol::ReviewRequest;
use codex_protocol::protocol::RolloutItem;
use codex_protocol::protocol::SessionSource;
@@ -270,7 +270,6 @@ pub(crate) struct TurnContext {
pub(crate) is_review_mode: bool,
pub(crate) final_output_json_schema: Option<Value>,
pub(crate) codex_linux_sandbox_exe: Option<PathBuf>,
pub(crate) item_collector: ItemCollector,
}
impl TurnContext {
@@ -359,7 +358,6 @@ impl Session {
provider: ModelProviderInfo,
session_configuration: &SessionConfiguration,
conversation_id: ConversationId,
tx_event: Sender<Event>,
sub_id: String,
) -> TurnContext {
let config = session_configuration.original_config_do_not_use.clone();
@@ -394,8 +392,6 @@ impl Session {
features: &config.features,
});
let item_collector = ItemCollector::new(tx_event, conversation_id, sub_id.clone());
TurnContext {
sub_id,
client,
@@ -409,7 +405,6 @@ impl Session {
is_review_mode: false,
final_output_json_schema: None,
codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(),
item_collector,
}
}
@@ -463,9 +458,6 @@ impl Session {
let mcp_fut = McpConnectionManager::new(
config.mcp_servers.clone(),
config
.features
.enabled(crate::features::Feature::RmcpClient),
config.mcp_oauth_credentials_store_mode,
);
let default_shell_fut = shell::default_user_shell();
@@ -669,7 +661,6 @@ impl Session {
session_configuration.provider.clone(),
&session_configuration,
self.conversation_id,
self.get_tx_event(),
sub_id,
);
if let Some(final_schema) = updates.final_output_json_schema {
@@ -714,6 +705,59 @@ impl Session {
}
}
async fn emit_turn_item_started(&self, turn_context: &TurnContext, item: &TurnItem) {
self.send_event(
turn_context,
EventMsg::ItemStarted(ItemStartedEvent {
thread_id: self.conversation_id,
turn_id: turn_context.sub_id.clone(),
item: item.clone(),
}),
)
.await;
}
async fn emit_turn_item_completed(
&self,
turn_context: &TurnContext,
item: TurnItem,
emit_raw_agent_reasoning: bool,
) {
self.send_event(
turn_context,
EventMsg::ItemCompleted(ItemCompletedEvent {
thread_id: self.conversation_id,
turn_id: turn_context.sub_id.clone(),
item: item.clone(),
}),
)
.await;
self.emit_turn_item_legacy_events(turn_context, &item, emit_raw_agent_reasoning)
.await;
}
async fn emit_turn_item_started_completed(
&self,
turn_context: &TurnContext,
item: TurnItem,
emit_raw_agent_reasoning: bool,
) {
self.emit_turn_item_started(turn_context, &item).await;
self.emit_turn_item_completed(turn_context, item, emit_raw_agent_reasoning)
.await;
}
async fn emit_turn_item_legacy_events(
&self,
turn_context: &TurnContext,
item: &TurnItem,
emit_raw_agent_reasoning: bool,
) {
for event in item.as_legacy_events(emit_raw_agent_reasoning) {
self.send_event(turn_context, event).await;
}
}
/// Emit an exec approval request event and await the user's decision.
///
/// The request is keyed by `sub_id`/`call_id` so matching responses are delivered
@@ -816,7 +860,7 @@ impl Session {
/// Records input items: always append to conversation history and
/// persist these response items to rollout.
async fn record_conversation_items(&self, items: &[ResponseItem]) {
pub(crate) async fn record_conversation_items(&self, items: &[ResponseItem]) {
self.record_into_history(items).await;
self.persist_rollout_response_items(items).await;
}
@@ -833,7 +877,7 @@ impl Session {
history.record_items(std::iter::once(response_item));
}
RolloutItem::Compacted(compacted) => {
let snapshot = history.contents();
let snapshot = history.get_history();
let user_messages = collect_user_messages(&snapshot);
let rebuilt = build_compacted_history(
self.build_initial_context(turn_context),
@@ -845,7 +889,7 @@ impl Session {
_ => {}
}
}
history.contents()
history.get_history()
}
/// Append ResponseItems to the in-memory conversation history only.
@@ -894,11 +938,17 @@ impl Session {
}
}
// todo (aibrahim): get rid of this method. we shouldn't deal with vec[resposne_item] and rather use ConversationHistory.
pub(crate) async fn history_snapshot(&self) -> Vec<ResponseItem> {
let state = self.state.lock().await;
let mut state = self.state.lock().await;
state.history_snapshot()
}
pub(crate) async fn clone_history(&self) -> ConversationHistory {
let state = self.state.lock().await;
state.clone_history()
}
async fn update_token_usage_info(
&self,
turn_context: &TurnContext,
@@ -950,24 +1000,22 @@ impl Session {
/// Record a user input item to conversation history and also persist a
/// corresponding UserMessage EventMsg to rollout.
async fn record_input_and_rollout_usermsg(&self, response_input: &ResponseInputItem) {
async fn record_input_and_rollout_usermsg(
&self,
turn_context: &TurnContext,
response_input: &ResponseInputItem,
) {
let response_item: ResponseItem = response_input.clone().into();
// Add to conversation history and persist response item to rollout
self.record_conversation_items(std::slice::from_ref(&response_item))
.await;
// Derive user message events and persist only UserMessage to rollout
let msgs =
map_response_item_to_event_messages(&response_item, self.show_raw_agent_reasoning());
let user_msgs: Vec<RolloutItem> = msgs
.into_iter()
.filter_map(|m| match m {
EventMsg::UserMessage(ev) => Some(RolloutItem::EventMsg(EventMsg::UserMessage(ev))),
_ => None,
})
.collect();
if !user_msgs.is_empty() {
self.persist_rollout_items(&user_msgs).await;
let turn_item = parse_turn_item(&response_item);
if let Some(item @ TurnItem::UserMessage(_)) = turn_item {
self.emit_turn_item_started_completed(turn_context, item, false)
.await;
}
}
@@ -992,16 +1040,6 @@ impl Session {
self.send_event(turn_context, event).await;
}
/// Build the full turn input by concatenating the current conversation
/// history with additional items for this turn.
pub async fn turn_input_with_history(&self, extra: Vec<ResponseItem>) -> Vec<ResponseItem> {
let history = {
let state = self.state.lock().await;
state.history_snapshot()
};
[history, extra].concat()
}
/// Returns the input if there was no task running to inject into
pub async fn inject_input(&self, input: Vec<UserInput>) -> Result<(), Vec<UserInput>> {
let mut active = self.active_turn.lock().await;
@@ -1162,19 +1200,8 @@ async fn submission_loop(sess: Arc<Session>, config: Arc<Config>, rx_sub: Receiv
{
sess.record_conversation_items(std::slice::from_ref(&env_item))
.await;
for msg in map_response_item_to_event_messages(
&env_item,
sess.show_raw_agent_reasoning(),
) {
sess.send_event(&current_context, msg).await;
}
}
current_context
.item_collector
.started_completed(TurnItem::UserMessage(UserMessageItem::new(&items)))
.await;
sess.spawn_task(Arc::clone(&current_context), items, RegularTask)
.await;
previous_context = Some(current_context);
@@ -1448,11 +1475,6 @@ async fn spawn_review_thread(
is_review_mode: true,
final_output_json_schema: None,
codex_linux_sandbox_exe: parent_turn_context.codex_linux_sandbox_exe.clone(),
item_collector: ItemCollector::new(
sess.get_tx_event(),
sess.conversation_id,
sub_id.to_string(),
),
};
// Seed the child task with the review prompt as the initial user message.
@@ -1504,13 +1526,15 @@ pub(crate) async fn run_task(
// model sees a fresh conversation without the parent session's history.
// For normal turns, continue recording to the session history as before.
let is_review_mode = turn_context.is_review_mode;
let mut review_thread_history: Vec<ResponseItem> = Vec::new();
let mut review_thread_history: ConversationHistory = ConversationHistory::new();
if is_review_mode {
// Seed review threads with environment context so the model knows the working directory.
review_thread_history.extend(sess.build_initial_context(turn_context.as_ref()));
review_thread_history.push(initial_input_for_turn.into());
review_thread_history
.record_items(sess.build_initial_context(turn_context.as_ref()).iter());
review_thread_history.record_items(std::iter::once(&initial_input_for_turn.into()));
} else {
sess.record_input_and_rollout_usermsg(&initial_input_for_turn)
sess.record_input_and_rollout_usermsg(turn_context.as_ref(), &initial_input_for_turn)
.await;
}
@@ -1543,12 +1567,12 @@ pub(crate) async fn run_task(
// represents an append-only log without duplicates.
let turn_input: Vec<ResponseItem> = if is_review_mode {
if !pending_input.is_empty() {
review_thread_history.extend(pending_input);
review_thread_history.record_items(&pending_input);
}
review_thread_history.clone()
review_thread_history.get_history()
} else {
sess.record_conversation_items(&pending_input).await;
sess.turn_input_with_history(pending_input).await
sess.history_snapshot().await
};
let turn_input_messages: Vec<String> = turn_input
@@ -1589,109 +1613,13 @@ pub(crate) async fn run_task(
let token_limit_reached = total_usage_tokens
.map(|tokens| tokens >= limit)
.unwrap_or(false);
let mut items_to_record_in_conversation_history = Vec::<ResponseItem>::new();
let mut responses = Vec::<ResponseInputItem>::new();
for processed_response_item in processed_items {
let ProcessedResponseItem { item, response } = processed_response_item;
match (&item, &response) {
(ResponseItem::Message { role, .. }, None) if role == "assistant" => {
// If the model returned a message, we need to record it.
items_to_record_in_conversation_history.push(item);
}
(
ResponseItem::LocalShellCall { .. },
Some(ResponseInputItem::FunctionCallOutput { call_id, output }),
) => {
items_to_record_in_conversation_history.push(item);
items_to_record_in_conversation_history.push(
ResponseItem::FunctionCallOutput {
call_id: call_id.clone(),
output: output.clone(),
},
);
}
(
ResponseItem::FunctionCall { .. },
Some(ResponseInputItem::FunctionCallOutput { call_id, output }),
) => {
items_to_record_in_conversation_history.push(item);
items_to_record_in_conversation_history.push(
ResponseItem::FunctionCallOutput {
call_id: call_id.clone(),
output: output.clone(),
},
);
}
(
ResponseItem::CustomToolCall { .. },
Some(ResponseInputItem::CustomToolCallOutput { call_id, output }),
) => {
items_to_record_in_conversation_history.push(item);
items_to_record_in_conversation_history.push(
ResponseItem::CustomToolCallOutput {
call_id: call_id.clone(),
output: output.clone(),
},
);
}
(
ResponseItem::FunctionCall { .. },
Some(ResponseInputItem::McpToolCallOutput { call_id, result }),
) => {
items_to_record_in_conversation_history.push(item);
let output = match result {
Ok(call_tool_result) => {
convert_call_tool_result_to_function_call_output_payload(
call_tool_result,
)
}
Err(err) => FunctionCallOutputPayload {
content: err.clone(),
success: Some(false),
},
};
items_to_record_in_conversation_history.push(
ResponseItem::FunctionCallOutput {
call_id: call_id.clone(),
output,
},
);
}
(
ResponseItem::Reasoning {
id,
summary,
content,
encrypted_content,
},
None,
) => {
items_to_record_in_conversation_history.push(ResponseItem::Reasoning {
id: id.clone(),
summary: summary.clone(),
content: content.clone(),
encrypted_content: encrypted_content.clone(),
});
}
_ => {
warn!("Unexpected response item: {item:?} with response: {response:?}");
}
};
if let Some(response) = response {
responses.push(response);
}
}
// Only attempt to take the lock if there is something to record.
if !items_to_record_in_conversation_history.is_empty() {
if is_review_mode {
review_thread_history
.extend(items_to_record_in_conversation_history.clone());
} else {
sess.record_conversation_items(&items_to_record_in_conversation_history)
.await;
}
}
let (responses, items_to_record_in_conversation_history) = process_items(
processed_items,
is_review_mode,
&mut review_thread_history,
&sess,
)
.await;
if token_limit_reached {
if auto_compact_recently_attempted {
@@ -1730,7 +1658,16 @@ pub(crate) async fn run_task(
}
continue;
}
Err(CodexErr::TurnAborted) => {
Err(CodexErr::TurnAborted {
dangling_artifacts: processed_items,
}) => {
let _ = process_items(
processed_items,
is_review_mode,
&mut review_thread_history,
&sess,
)
.await;
// Aborted turn is reported via a different event.
break;
}
@@ -1831,7 +1768,13 @@ async fn run_turn(
.await
{
Ok(output) => return Ok(output),
Err(CodexErr::TurnAborted) => return Err(CodexErr::TurnAborted),
Err(CodexErr::TurnAborted {
dangling_artifacts: processed_items,
}) => {
return Err(CodexErr::TurnAborted {
dangling_artifacts: processed_items,
});
}
Err(CodexErr::Interrupted) => return Err(CodexErr::Interrupted),
Err(CodexErr::EnvVar(var)) => return Err(CodexErr::EnvVar(var)),
Err(e @ CodexErr::Fatal(_)) => return Err(e),
@@ -1884,9 +1827,9 @@ async fn run_turn(
/// "handled" such that it produces a `ResponseInputItem` that needs to be
/// sent back to the model on the next turn.
#[derive(Debug)]
pub(crate) struct ProcessedResponseItem {
pub(crate) item: ResponseItem,
pub(crate) response: Option<ResponseInputItem>,
pub struct ProcessedResponseItem {
pub item: ResponseItem,
pub response: Option<ResponseInputItem>,
}
#[derive(Debug)]
@@ -1905,61 +1848,6 @@ async fn try_run_turn(
task_kind: TaskKind,
cancellation_token: CancellationToken,
) -> CodexResult<TurnRunResult> {
// call_ids that are part of this response.
let completed_call_ids = prompt
.input
.iter()
.filter_map(|ri| match ri {
ResponseItem::FunctionCallOutput { call_id, .. } => Some(call_id),
ResponseItem::LocalShellCall {
call_id: Some(call_id),
..
} => Some(call_id),
ResponseItem::CustomToolCallOutput { call_id, .. } => Some(call_id),
_ => None,
})
.collect::<Vec<_>>();
// call_ids that were pending but are not part of this response.
// This usually happens because the user interrupted the model before we responded to one of its tool calls
// and then the user sent a follow-up message.
let missing_calls = {
prompt
.input
.iter()
.filter_map(|ri| match ri {
ResponseItem::FunctionCall { call_id, .. } => Some(call_id),
ResponseItem::LocalShellCall {
call_id: Some(call_id),
..
} => Some(call_id),
ResponseItem::CustomToolCall { call_id, .. } => Some(call_id),
_ => None,
})
.filter_map(|call_id| {
if completed_call_ids.contains(&call_id) {
None
} else {
Some(call_id.clone())
}
})
.map(|call_id| ResponseItem::CustomToolCallOutput {
call_id,
output: "aborted".to_string(),
})
.collect::<Vec<_>>()
};
let prompt: Cow<Prompt> = if missing_calls.is_empty() {
Cow::Borrowed(prompt)
} else {
// Add the synthetic aborted missing calls to the beginning of the input to ensure all call ids have responses.
let input = [missing_calls, prompt.input.clone()].concat();
Cow::Owned(Prompt {
input,
..prompt.clone()
})
};
let rollout_item = RolloutItem::TurnContext(TurnContextItem {
cwd: turn_context.cwd.clone(),
approval_policy: turn_context.approval_policy,
@@ -1968,11 +1856,12 @@ async fn try_run_turn(
effort: turn_context.client.get_reasoning_effort(),
summary: turn_context.client.get_reasoning_summary(),
});
sess.persist_rollout_items(&[rollout_item]).await;
let mut stream = turn_context
.client
.clone()
.stream_with_task_kind(prompt.as_ref(), task_kind)
.stream_with_task_kind(prompt, task_kind)
.or_cancel(&cancellation_token)
.await??;
@@ -1989,7 +1878,15 @@ async fn try_run_turn(
// Poll the next item from the model stream. We must inspect *both* Ok and Err
// cases so that transient stream failures (e.g., dropped SSE connection before
// `response.completed`) bubble up and trigger the caller's retry logic.
let event = stream.next().or_cancel(&cancellation_token).await?;
let event = match stream.next().or_cancel(&cancellation_token).await {
Ok(event) => event,
Err(codex_async_utils::CancelErr::Cancelled) => {
let processed_items = output.try_collect().await?;
return Err(CodexErr::TurnAborted {
dangling_artifacts: processed_items,
});
}
};
let event = match event {
Some(res) => res?,
@@ -2013,7 +1910,8 @@ async fn try_run_turn(
let payload_preview = call.payload.log_payload().into_owned();
tracing::info!("ToolCall: {} {}", call.tool_name, payload_preview);
let response = tool_runtime.handle_tool_call(call);
let response =
tool_runtime.handle_tool_call(call, cancellation_token.child_token());
output.push_back(
async move {
@@ -2027,9 +1925,10 @@ async fn try_run_turn(
}
Ok(None) => {
let response = handle_non_tool_response_item(
Arc::clone(&sess),
sess.as_ref(),
Arc::clone(&turn_context),
item.clone(),
sess.show_raw_agent_reasoning(),
)
.await?;
add_completed(ProcessedResponseItem { item, response });
@@ -2094,12 +1993,7 @@ async fn try_run_turn(
} => {
sess.update_token_usage_info(turn_context.as_ref(), token_usage.as_ref())
.await;
let processed_items = output
.try_collect()
.or_cancel(&cancellation_token)
.await??;
let processed_items = output.try_collect().await?;
let unified_diff = {
let mut tracker = turn_diff_tracker.lock().await;
tracker.get_unified_diff()
@@ -2148,9 +2042,10 @@ async fn try_run_turn(
}
async fn handle_non_tool_response_item(
sess: Arc<Session>,
sess: &Session,
turn_context: Arc<TurnContext>,
item: ResponseItem,
show_raw_agent_reasoning: bool,
) -> CodexResult<Option<ResponseInputItem>> {
debug!(?item, "Output item");
@@ -2158,15 +2053,20 @@ async fn handle_non_tool_response_item(
ResponseItem::Message { .. }
| ResponseItem::Reasoning { .. }
| ResponseItem::WebSearchCall { .. } => {
let msgs = match &item {
let turn_item = match &item {
ResponseItem::Message { .. } if turn_context.is_review_mode => {
trace!("suppressing assistant Message in review mode");
Vec::new()
None
}
_ => map_response_item_to_event_messages(&item, sess.show_raw_agent_reasoning()),
_ => parse_turn_item(&item),
};
for msg in msgs {
sess.send_event(&turn_context, msg).await;
if let Some(turn_item) = turn_item {
sess.emit_turn_item_started_completed(
turn_context.as_ref(),
turn_item,
show_raw_agent_reasoning,
)
.await;
}
}
ResponseItem::FunctionCallOutput { .. } | ResponseItem::CustomToolCallOutput { .. } => {
@@ -2197,7 +2097,7 @@ pub(super) fn get_last_assistant_message_from_turn(responses: &[ResponseItem]) -
}
})
}
fn convert_call_tool_result_to_function_call_output_payload(
pub(crate) fn convert_call_tool_result_to_function_call_output_payload(
call_tool_result: &CallToolResult,
) -> FunctionCallOutputPayload {
let CallToolResult {
@@ -2344,7 +2244,11 @@ mod tests {
use crate::tools::MODEL_FORMAT_MAX_LINES;
use crate::tools::MODEL_FORMAT_TAIL_LINES;
use crate::tools::ToolRouter;
use crate::tools::handle_container_exec_with_params;
use crate::tools::context::ToolInvocation;
use crate::tools::context::ToolOutput;
use crate::tools::context::ToolPayload;
use crate::tools::handlers::ShellHandler;
use crate::tools::registry::ToolHandler;
use crate::turn_diff_tracker::TurnDiffTracker;
use codex_app_server_protocol::AuthMode;
use codex_protocol::models::ContentItem;
@@ -2653,7 +2557,6 @@ mod tests {
session_configuration.provider.clone(),
&session_configuration,
conversation_id,
tx_event.clone(),
"turn_id".to_string(),
);
@@ -2722,7 +2625,6 @@ mod tests {
session_configuration.provider.clone(),
&session_configuration,
conversation_id,
tx_event.clone(),
"turn_id".to_string(),
));
@@ -2955,7 +2857,7 @@ mod tests {
rollout_items.push(RolloutItem::ResponseItem(assistant1.clone()));
let summary1 = "summary one";
let snapshot1 = live_history.contents();
let snapshot1 = live_history.get_history();
let user_messages1 = collect_user_messages(&snapshot1);
let rebuilt1 = build_compacted_history(
session.build_initial_context(turn_context),
@@ -2988,7 +2890,7 @@ mod tests {
rollout_items.push(RolloutItem::ResponseItem(assistant2.clone()));
let summary2 = "summary two";
let snapshot2 = live_history.contents();
let snapshot2 = live_history.get_history();
let user_messages2 = collect_user_messages(&snapshot2);
let rebuilt2 = build_compacted_history(
session.build_initial_context(turn_context),
@@ -3020,7 +2922,7 @@ mod tests {
live_history.record_items(std::iter::once(&assistant3));
rollout_items.push(RolloutItem::ResponseItem(assistant3.clone()));
(rollout_items, live_history.contents())
(rollout_items, live_history.get_history())
}
#[tokio::test]
@@ -3069,15 +2971,26 @@ mod tests {
let tool_name = "shell";
let call_id = "test-call".to_string();
let resp = handle_container_exec_with_params(
tool_name,
params,
Arc::clone(&session),
Arc::clone(&turn_context),
Arc::clone(&turn_diff_tracker),
call_id,
)
.await;
let handler = ShellHandler;
let resp = handler
.handle(ToolInvocation {
session: Arc::clone(&session),
turn: Arc::clone(&turn_context),
tracker: Arc::clone(&turn_diff_tracker),
call_id,
tool_name: tool_name.to_string(),
payload: ToolPayload::Function {
arguments: serde_json::json!({
"command": params.command.clone(),
"workdir": Some(turn_context.cwd.to_string_lossy().to_string()),
"timeout_ms": params.timeout_ms,
"with_escalated_permissions": params.with_escalated_permissions,
"justification": params.justification.clone(),
})
.to_string(),
},
})
.await;
let Err(FunctionCallError::RespondToModel(output)) = resp else {
panic!("expected error result");
@@ -3096,17 +3009,30 @@ mod tests {
.expect("unique turn context Arc")
.sandbox_policy = SandboxPolicy::DangerFullAccess;
let resp2 = handle_container_exec_with_params(
tool_name,
params2,
Arc::clone(&session),
Arc::clone(&turn_context),
Arc::clone(&turn_diff_tracker),
"test-call-2".to_string(),
)
.await;
let resp2 = handler
.handle(ToolInvocation {
session: Arc::clone(&session),
turn: Arc::clone(&turn_context),
tracker: Arc::clone(&turn_diff_tracker),
call_id: "test-call-2".to_string(),
tool_name: tool_name.to_string(),
payload: ToolPayload::Function {
arguments: serde_json::json!({
"command": params2.command.clone(),
"workdir": Some(turn_context.cwd.to_string_lossy().to_string()),
"timeout_ms": params2.timeout_ms,
"with_escalated_permissions": params2.with_escalated_permissions,
"justification": params2.justification.clone(),
})
.to_string(),
},
})
.await;
let output = resp2.expect("expected Ok result");
let output = match resp2.expect("expected Ok result") {
ToolOutput::Function { content, .. } => content,
_ => panic!("unexpected tool output"),
};
#[derive(Deserialize, PartialEq, Eq, Debug)]
struct ResponseExecMetadata {
codex-rs/core/src/codex/compact.rs
+18 -26
View File
@@ -11,19 +11,20 @@ use crate::protocol::AgentMessageEvent;
use crate::protocol::CompactedItem;
use crate::protocol::ErrorEvent;
use crate::protocol::EventMsg;
use crate::protocol::InputMessageKind;
use crate::protocol::TaskStartedEvent;
use crate::protocol::TurnContextItem;
use crate::state::TaskKind;
use crate::truncate::truncate_middle;
use crate::util::backoff;
use askama::Template;
use codex_protocol::items::TurnItem;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::RolloutItem;
use codex_protocol::user_input::UserInput;
use futures::prelude::*;
use tracing::error;
pub const SUMMARIZATION_PROMPT: &str = include_str!("../../templates/compact/prompt.md");
const COMPACT_USER_MESSAGE_MAX_TOKENS: usize = 20_000;
@@ -64,9 +65,10 @@ async fn run_compact_task_inner(
input: Vec<UserInput>,
) {
let initial_input_for_turn: ResponseInputItem = ResponseInputItem::from(input);
let mut turn_input = sess
.turn_input_with_history(vec![initial_input_for_turn.clone().into()])
.await;
let mut history = sess.clone_history().await;
history.record_items(&[initial_input_for_turn.into()]);
let mut truncated_count = 0usize;
let max_retries = turn_context.client.get_provider().stream_max_retries();
@@ -83,6 +85,7 @@ async fn run_compact_task_inner(
sess.persist_rollout_items(&[rollout_item]).await;
loop {
let turn_input = history.get_history();
let prompt = Prompt {
input: turn_input.clone(),
..Default::default()
@@ -107,7 +110,11 @@ async fn run_compact_task_inner(
}
Err(e @ CodexErr::ContextWindowExceeded) => {
if turn_input.len() > 1 {
turn_input.remove(0);
// Trim from the beginning to preserve cache (prefix-based) and keep recent messages intact.
error!(
"Context window exceeded while compacting; removing oldest history item. Error: {e}"
);
history.remove_first_item();
truncated_count += 1;
retries = 0;
continue;
@@ -181,23 +188,13 @@ pub fn content_items_to_text(content: &[ContentItem]) -> Option<String> {
pub(crate) fn collect_user_messages(items: &[ResponseItem]) -> Vec<String> {
items
.iter()
.filter_map(|item| match item {
ResponseItem::Message { role, content, .. } if role == "user" => {
content_items_to_text(content)
}
.filter_map(|item| match crate::event_mapping::parse_turn_item(item) {
Some(TurnItem::UserMessage(user)) => Some(user.message()),
_ => None,
})
.filter(|text| !is_session_prefix_message(text))
.collect()
}
pub fn is_session_prefix_message(text: &str) -> bool {
matches!(
InputMessageKind::from(("user", text)),
InputMessageKind::UserInstructions | InputMessageKind::EnvironmentContext
)
}
pub(crate) fn build_compacted_history(
initial_context: Vec<ResponseItem>,
user_messages: &[String],
@@ -319,21 +316,16 @@ mod tests {
ResponseItem::Message {
id: Some("user".to_string()),
role: "user".to_string(),
content: vec![
ContentItem::InputText {
text: "first".to_string(),
},
ContentItem::OutputText {
text: "second".to_string(),
},
],
content: vec![ContentItem::InputText {
text: "first".to_string(),
}],
},
ResponseItem::Other,
];
let collected = collect_user_messages(&items);
assert_eq!(vec!["first\nsecond".to_string()], collected);
assert_eq!(vec!["first".to_string()], collected);
}
#[test]
codex-rs/core/src/conversation_history.rs
+739 -5
View File
@@ -1,4 +1,6 @@
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ResponseItem;
use tracing::error;
/// Transcript of conversation history
#[derive(Debug, Clone, Default)]
@@ -12,11 +14,6 @@ impl ConversationHistory {
Self { items: Vec::new() }
}
/// Returns a clone of the contents in the transcript.
pub(crate) fn contents(&self) -> Vec<ResponseItem> {
self.items.clone()
}
/// `items` is ordered from oldest to newest.
pub(crate) fn record_items<I>(&mut self, items: I)
where
@@ -32,9 +29,287 @@ impl ConversationHistory {
}
}
pub(crate) fn get_history(&mut self) -> Vec<ResponseItem> {
self.normalize_history();
self.contents()
}
pub(crate) fn remove_first_item(&mut self) {
if !self.items.is_empty() {
// Remove the oldest item (front of the list). Items are ordered from
// oldest → newest, so index 0 is the first entry recorded.
let removed = self.items.remove(0);
// If the removed item participates in a call/output pair, also remove
// its corresponding counterpart to keep the invariants intact without
// running a full normalization pass.
self.remove_corresponding_for(&removed);
}
}
/// This function enforces a couple of invariants on the in-memory history:
/// 1. every call (function/custom) has a corresponding output entry
/// 2. every output has a corresponding call entry
fn normalize_history(&mut self) {
// all function/tool calls must have a corresponding output
self.ensure_call_outputs_present();
// all outputs must have a corresponding function/tool call
self.remove_orphan_outputs();
}
/// Returns a clone of the contents in the transcript.
fn contents(&self) -> Vec<ResponseItem> {
self.items.clone()
}
fn ensure_call_outputs_present(&mut self) {
// Collect synthetic outputs to insert immediately after their calls.
// Store the insertion position (index of call) alongside the item so
// we can insert in reverse order and avoid index shifting.
let mut missing_outputs_to_insert: Vec<(usize, ResponseItem)> = Vec::new();
for (idx, item) in self.items.iter().enumerate() {
match item {
ResponseItem::FunctionCall { call_id, .. } => {
let has_output = self.items.iter().any(|i| match i {
ResponseItem::FunctionCallOutput {
call_id: existing, ..
} => existing == call_id,
_ => false,
});
if !has_output {
error_or_panic(format!(
"Function call output is missing for call id: {call_id}"
));
missing_outputs_to_insert.push((
idx,
ResponseItem::FunctionCallOutput {
call_id: call_id.clone(),
output: FunctionCallOutputPayload {
content: "aborted".to_string(),
success: None,
},
},
));
}
}
ResponseItem::CustomToolCall { call_id, .. } => {
let has_output = self.items.iter().any(|i| match i {
ResponseItem::CustomToolCallOutput {
call_id: existing, ..
} => existing == call_id,
_ => false,
});
if !has_output {
error_or_panic(format!(
"Custom tool call output is missing for call id: {call_id}"
));
missing_outputs_to_insert.push((
idx,
ResponseItem::CustomToolCallOutput {
call_id: call_id.clone(),
output: "aborted".to_string(),
},
));
}
}
// LocalShellCall is represented in upstream streams by a FunctionCallOutput
ResponseItem::LocalShellCall { call_id, .. } => {
if let Some(call_id) = call_id.as_ref() {
let has_output = self.items.iter().any(|i| match i {
ResponseItem::FunctionCallOutput {
call_id: existing, ..
} => existing == call_id,
_ => false,
});
if !has_output {
error_or_panic(format!(
"Local shell call output is missing for call id: {call_id}"
));
missing_outputs_to_insert.push((
idx,
ResponseItem::FunctionCallOutput {
call_id: call_id.clone(),
output: FunctionCallOutputPayload {
content: "aborted".to_string(),
success: None,
},
},
));
}
}
}
ResponseItem::Reasoning { .. }
| ResponseItem::WebSearchCall { .. }
| ResponseItem::FunctionCallOutput { .. }
| ResponseItem::CustomToolCallOutput { .. }
| ResponseItem::Other
| ResponseItem::Message { .. } => {
// nothing to do for these variants
}
}
}
if !missing_outputs_to_insert.is_empty() {
// Insert from the end to avoid shifting subsequent indices.
missing_outputs_to_insert.sort_by_key(|(i, _)| *i);
for (idx, item) in missing_outputs_to_insert.into_iter().rev() {
let insert_pos = idx + 1; // place immediately after the call
if insert_pos <= self.items.len() {
self.items.insert(insert_pos, item);
} else {
self.items.push(item);
}
}
}
}
fn remove_orphan_outputs(&mut self) {
// Work on a snapshot to avoid borrowing `self.items` while mutating it.
let snapshot = self.items.clone();
let mut orphan_output_call_ids: std::collections::HashSet<String> =
std::collections::HashSet::new();
for item in &snapshot {
match item {
ResponseItem::FunctionCallOutput { call_id, .. } => {
let has_call = snapshot.iter().any(|i| match i {
ResponseItem::FunctionCall {
call_id: existing, ..
} => existing == call_id,
ResponseItem::LocalShellCall {
call_id: Some(existing),
..
} => existing == call_id,
_ => false,
});
if !has_call {
error_or_panic(format!("Function call is missing for call id: {call_id}"));
orphan_output_call_ids.insert(call_id.clone());
}
}
ResponseItem::CustomToolCallOutput { call_id, .. } => {
let has_call = snapshot.iter().any(|i| match i {
ResponseItem::CustomToolCall {
call_id: existing, ..
} => existing == call_id,
_ => false,
});
if !has_call {
error_or_panic(format!(
"Custom tool call is missing for call id: {call_id}"
));
orphan_output_call_ids.insert(call_id.clone());
}
}
ResponseItem::FunctionCall { .. }
| ResponseItem::CustomToolCall { .. }
| ResponseItem::LocalShellCall { .. }
| ResponseItem::Reasoning { .. }
| ResponseItem::WebSearchCall { .. }
| ResponseItem::Other
| ResponseItem::Message { .. } => {
// nothing to do for these variants
}
}
}
if !orphan_output_call_ids.is_empty() {
let ids = orphan_output_call_ids;
self.items.retain(|i| match i {
ResponseItem::FunctionCallOutput { call_id, .. }
| ResponseItem::CustomToolCallOutput { call_id, .. } => !ids.contains(call_id),
_ => true,
});
}
}
pub(crate) fn replace(&mut self, items: Vec<ResponseItem>) {
self.items = items;
}
/// Removes the corresponding paired item for the provided `item`, if any.
///
/// Pairs:
/// - FunctionCall <-> FunctionCallOutput
/// - CustomToolCall <-> CustomToolCallOutput
/// - LocalShellCall(call_id: Some) <-> FunctionCallOutput
fn remove_corresponding_for(&mut self, item: &ResponseItem) {
match item {
ResponseItem::FunctionCall { call_id, .. } => {
self.remove_first_matching(|i| match i {
ResponseItem::FunctionCallOutput {
call_id: existing, ..
} => existing == call_id,
_ => false,
});
}
ResponseItem::CustomToolCall { call_id, .. } => {
self.remove_first_matching(|i| match i {
ResponseItem::CustomToolCallOutput {
call_id: existing, ..
} => existing == call_id,
_ => false,
});
}
ResponseItem::LocalShellCall {
call_id: Some(call_id),
..
} => {
self.remove_first_matching(|i| match i {
ResponseItem::FunctionCallOutput {
call_id: existing, ..
} => existing == call_id,
_ => false,
});
}
ResponseItem::FunctionCallOutput { call_id, .. } => {
self.remove_first_matching(|i| match i {
ResponseItem::FunctionCall {
call_id: existing, ..
} => existing == call_id,
ResponseItem::LocalShellCall {
call_id: Some(existing),
..
} => existing == call_id,
_ => false,
});
}
ResponseItem::CustomToolCallOutput { call_id, .. } => {
self.remove_first_matching(|i| match i {
ResponseItem::CustomToolCall {
call_id: existing, ..
} => existing == call_id,
_ => false,
});
}
_ => {}
}
}
/// Remove the first item matching the predicate.
fn remove_first_matching<F>(&mut self, predicate: F)
where
F: FnMut(&ResponseItem) -> bool,
{
if let Some(pos) = self.items.iter().position(predicate) {
self.items.remove(pos);
}
}
}
#[inline]
fn error_or_panic(message: String) {
if cfg!(debug_assertions) || env!("CARGO_PKG_VERSION").contains("alpha") {
panic!("{message}");
} else {
error!("{message}");
}
}
/// Anything that is not a system message or "reasoning" message is considered
@@ -57,6 +332,11 @@ fn is_api_message(message: &ResponseItem) -> bool {
mod tests {
use super::*;
use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::LocalShellAction;
use codex_protocol::models::LocalShellExecAction;
use codex_protocol::models::LocalShellStatus;
use pretty_assertions::assert_eq;
fn assistant_msg(text: &str) -> ResponseItem {
ResponseItem::Message {
@@ -68,6 +348,12 @@ mod tests {
}
}
fn create_history_with_items(items: Vec<ResponseItem>) -> ConversationHistory {
let mut h = ConversationHistory::new();
h.record_items(items.iter());
h
}
fn user_msg(text: &str) -> ResponseItem {
ResponseItem::Message {
id: None,
@@ -117,4 +403,452 @@ mod tests {
]
);
}
#[test]
fn remove_first_item_removes_matching_output_for_function_call() {
let items = vec![
ResponseItem::FunctionCall {
id: None,
name: "do_it".to_string(),
arguments: "{}".to_string(),
call_id: "call-1".to_string(),
},
ResponseItem::FunctionCallOutput {
call_id: "call-1".to_string(),
output: FunctionCallOutputPayload {
content: "ok".to_string(),
success: None,
},
},
];
let mut h = create_history_with_items(items);
h.remove_first_item();
assert_eq!(h.contents(), vec![]);
}
#[test]
fn remove_first_item_removes_matching_call_for_output() {
let items = vec![
ResponseItem::FunctionCallOutput {
call_id: "call-2".to_string(),
output: FunctionCallOutputPayload {
content: "ok".to_string(),
success: None,
},
},
ResponseItem::FunctionCall {
id: None,
name: "do_it".to_string(),
arguments: "{}".to_string(),
call_id: "call-2".to_string(),
},
];
let mut h = create_history_with_items(items);
h.remove_first_item();
assert_eq!(h.contents(), vec![]);
}
#[test]
fn remove_first_item_handles_local_shell_pair() {
let items = vec![
ResponseItem::LocalShellCall {
id: None,
call_id: Some("call-3".to_string()),
status: LocalShellStatus::Completed,
action: LocalShellAction::Exec(LocalShellExecAction {
command: vec!["echo".to_string(), "hi".to_string()],
timeout_ms: None,
working_directory: None,
env: None,
user: None,
}),
},
ResponseItem::FunctionCallOutput {
call_id: "call-3".to_string(),
output: FunctionCallOutputPayload {
content: "ok".to_string(),
success: None,
},
},
];
let mut h = create_history_with_items(items);
h.remove_first_item();
assert_eq!(h.contents(), vec![]);
}
#[test]
fn remove_first_item_handles_custom_tool_pair() {
let items = vec![
ResponseItem::CustomToolCall {
id: None,
status: None,
call_id: "tool-1".to_string(),
name: "my_tool".to_string(),
input: "{}".to_string(),
},
ResponseItem::CustomToolCallOutput {
call_id: "tool-1".to_string(),
output: "ok".to_string(),
},
];
let mut h = create_history_with_items(items);
h.remove_first_item();
assert_eq!(h.contents(), vec![]);
}
//TODO(aibrahim): run CI in release mode.
#[cfg(not(debug_assertions))]
#[test]
fn normalize_adds_missing_output_for_function_call() {
let items = vec![ResponseItem::FunctionCall {
id: None,
name: "do_it".to_string(),
arguments: "{}".to_string(),
call_id: "call-x".to_string(),
}];
let mut h = create_history_with_items(items);
h.normalize_history();
assert_eq!(
h.contents(),
vec![
ResponseItem::FunctionCall {
id: None,
name: "do_it".to_string(),
arguments: "{}".to_string(),
call_id: "call-x".to_string(),
},
ResponseItem::FunctionCallOutput {
call_id: "call-x".to_string(),
output: FunctionCallOutputPayload {
content: "aborted".to_string(),
success: None,
},
},
]
);
}
#[cfg(not(debug_assertions))]
#[test]
fn normalize_adds_missing_output_for_custom_tool_call() {
let items = vec![ResponseItem::CustomToolCall {
id: None,
status: None,
call_id: "tool-x".to_string(),
name: "custom".to_string(),
input: "{}".to_string(),
}];
let mut h = create_history_with_items(items);
h.normalize_history();
assert_eq!(
h.contents(),
vec![
ResponseItem::CustomToolCall {
id: None,
status: None,
call_id: "tool-x".to_string(),
name: "custom".to_string(),
input: "{}".to_string(),
},
ResponseItem::CustomToolCallOutput {
call_id: "tool-x".to_string(),
output: "aborted".to_string(),
},
]
);
}
#[cfg(not(debug_assertions))]
#[test]
fn normalize_adds_missing_output_for_local_shell_call_with_id() {
let items = vec![ResponseItem::LocalShellCall {
id: None,
call_id: Some("shell-1".to_string()),
status: LocalShellStatus::Completed,
action: LocalShellAction::Exec(LocalShellExecAction {
command: vec!["echo".to_string(), "hi".to_string()],
timeout_ms: None,
working_directory: None,
env: None,
user: None,
}),
}];
let mut h = create_history_with_items(items);
h.normalize_history();
assert_eq!(
h.contents(),
vec![
ResponseItem::LocalShellCall {
id: None,
call_id: Some("shell-1".to_string()),
status: LocalShellStatus::Completed,
action: LocalShellAction::Exec(LocalShellExecAction {
command: vec!["echo".to_string(), "hi".to_string()],
timeout_ms: None,
working_directory: None,
env: None,
user: None,
}),
},
ResponseItem::FunctionCallOutput {
call_id: "shell-1".to_string(),
output: FunctionCallOutputPayload {
content: "aborted".to_string(),
success: None,
},
},
]
);
}
#[cfg(not(debug_assertions))]
#[test]
fn normalize_removes_orphan_function_call_output() {
let items = vec![ResponseItem::FunctionCallOutput {
call_id: "orphan-1".to_string(),
output: FunctionCallOutputPayload {
content: "ok".to_string(),
success: None,
},
}];
let mut h = create_history_with_items(items);
h.normalize_history();
assert_eq!(h.contents(), vec![]);
}
#[cfg(not(debug_assertions))]
#[test]
fn normalize_removes_orphan_custom_tool_call_output() {
let items = vec![ResponseItem::CustomToolCallOutput {
call_id: "orphan-2".to_string(),
output: "ok".to_string(),
}];
let mut h = create_history_with_items(items);
h.normalize_history();
assert_eq!(h.contents(), vec![]);
}
#[cfg(not(debug_assertions))]
#[test]
fn normalize_mixed_inserts_and_removals() {
let items = vec![
// Will get an inserted output
ResponseItem::FunctionCall {
id: None,
name: "f1".to_string(),
arguments: "{}".to_string(),
call_id: "c1".to_string(),
},
// Orphan output that should be removed
ResponseItem::FunctionCallOutput {
call_id: "c2".to_string(),
output: FunctionCallOutputPayload {
content: "ok".to_string(),
success: None,
},
},
// Will get an inserted custom tool output
ResponseItem::CustomToolCall {
id: None,
status: None,
call_id: "t1".to_string(),
name: "tool".to_string(),
input: "{}".to_string(),
},
// Local shell call also gets an inserted function call output
ResponseItem::LocalShellCall {
id: None,
call_id: Some("s1".to_string()),
status: LocalShellStatus::Completed,
action: LocalShellAction::Exec(LocalShellExecAction {
command: vec!["echo".to_string()],
timeout_ms: None,
working_directory: None,
env: None,
user: None,
}),
},
];
let mut h = create_history_with_items(items);
h.normalize_history();
assert_eq!(
h.contents(),
vec![
ResponseItem::FunctionCall {
id: None,
name: "f1".to_string(),
arguments: "{}".to_string(),
call_id: "c1".to_string(),
},
ResponseItem::FunctionCallOutput {
call_id: "c1".to_string(),
output: FunctionCallOutputPayload {
content: "aborted".to_string(),
success: None,
},
},
ResponseItem::CustomToolCall {
id: None,
status: None,
call_id: "t1".to_string(),
name: "tool".to_string(),
input: "{}".to_string(),
},
ResponseItem::CustomToolCallOutput {
call_id: "t1".to_string(),
output: "aborted".to_string(),
},
ResponseItem::LocalShellCall {
id: None,
call_id: Some("s1".to_string()),
status: LocalShellStatus::Completed,
action: LocalShellAction::Exec(LocalShellExecAction {
command: vec!["echo".to_string()],
timeout_ms: None,
working_directory: None,
env: None,
user: None,
}),
},
ResponseItem::FunctionCallOutput {
call_id: "s1".to_string(),
output: FunctionCallOutputPayload {
content: "aborted".to_string(),
success: None,
},
},
]
);
}
// In debug builds we panic on normalization errors instead of silently fixing them.
#[cfg(debug_assertions)]
#[test]
#[should_panic]
fn normalize_adds_missing_output_for_function_call_panics_in_debug() {
let items = vec![ResponseItem::FunctionCall {
id: None,
name: "do_it".to_string(),
arguments: "{}".to_string(),
call_id: "call-x".to_string(),
}];
let mut h = create_history_with_items(items);
h.normalize_history();
}
#[cfg(debug_assertions)]
#[test]
#[should_panic]
fn normalize_adds_missing_output_for_custom_tool_call_panics_in_debug() {
let items = vec![ResponseItem::CustomToolCall {
id: None,
status: None,
call_id: "tool-x".to_string(),
name: "custom".to_string(),
input: "{}".to_string(),
}];
let mut h = create_history_with_items(items);
h.normalize_history();
}
#[cfg(debug_assertions)]
#[test]
#[should_panic]
fn normalize_adds_missing_output_for_local_shell_call_with_id_panics_in_debug() {
let items = vec![ResponseItem::LocalShellCall {
id: None,
call_id: Some("shell-1".to_string()),
status: LocalShellStatus::Completed,
action: LocalShellAction::Exec(LocalShellExecAction {
command: vec!["echo".to_string(), "hi".to_string()],
timeout_ms: None,
working_directory: None,
env: None,
user: None,
}),
}];
let mut h = create_history_with_items(items);
h.normalize_history();
}
#[cfg(debug_assertions)]
#[test]
#[should_panic]
fn normalize_removes_orphan_function_call_output_panics_in_debug() {
let items = vec![ResponseItem::FunctionCallOutput {
call_id: "orphan-1".to_string(),
output: FunctionCallOutputPayload {
content: "ok".to_string(),
success: None,
},
}];
let mut h = create_history_with_items(items);
h.normalize_history();
}
#[cfg(debug_assertions)]
#[test]
#[should_panic]
fn normalize_removes_orphan_custom_tool_call_output_panics_in_debug() {
let items = vec![ResponseItem::CustomToolCallOutput {
call_id: "orphan-2".to_string(),
output: "ok".to_string(),
}];
let mut h = create_history_with_items(items);
h.normalize_history();
}
#[cfg(debug_assertions)]
#[test]
#[should_panic]
fn normalize_mixed_inserts_and_removals_panics_in_debug() {
let items = vec![
ResponseItem::FunctionCall {
id: None,
name: "f1".to_string(),
arguments: "{}".to_string(),
call_id: "c1".to_string(),
},
ResponseItem::FunctionCallOutput {
call_id: "c2".to_string(),
output: FunctionCallOutputPayload {
content: "ok".to_string(),
success: None,
},
},
ResponseItem::CustomToolCall {
id: None,
status: None,
call_id: "t1".to_string(),
name: "tool".to_string(),
input: "{}".to_string(),
},
ResponseItem::LocalShellCall {
id: None,
call_id: Some("s1".to_string()),
status: LocalShellStatus::Completed,
action: LocalShellAction::Exec(LocalShellExecAction {
command: vec!["echo".to_string()],
timeout_ms: None,
working_directory: None,
env: None,
user: None,
}),
},
];
let mut h = create_history_with_items(items);
h.normalize_history();
}
}
codex-rs/core/src/conversation_manager.rs
+6 -5
View File
@@ -3,8 +3,6 @@ use crate::CodexAuth;
use crate::codex::Codex;
use crate::codex::CodexSpawnOk;
use crate::codex::INITIAL_SUBMIT_ID;
use crate::codex::compact::content_items_to_text;
use crate::codex::compact::is_session_prefix_message;
use crate::codex_conversation::CodexConversation;
use crate::config::Config;
use crate::error::CodexErr;
@@ -14,6 +12,7 @@ use crate::protocol::EventMsg;
use crate::protocol::SessionConfiguredEvent;
use crate::rollout::RolloutRecorder;
use codex_protocol::ConversationId;
use codex_protocol::items::TurnItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::InitialHistory;
use codex_protocol::protocol::RolloutItem;
@@ -182,9 +181,11 @@ fn truncate_before_nth_user_message(history: InitialHistory, n: usize) -> Initia
// Find indices of user message inputs in rollout order.
let mut user_positions: Vec<usize> = Vec::new();
for (idx, item) in items.iter().enumerate() {
if let RolloutItem::ResponseItem(ResponseItem::Message { role, content, .. }) = item
&& role == "user"
&& content_items_to_text(content).is_some_and(|text| !is_session_prefix_message(&text))
if let RolloutItem::ResponseItem(item @ ResponseItem::Message { .. }) = item
&& matches!(
crate::event_mapping::parse_turn_item(item),
Some(TurnItem::UserMessage(_))
)
{
user_positions.push(idx);
}
codex-rs/core/src/error.rs
+8 -2
View File
@@ -1,3 +1,4 @@
use crate::codex::ProcessedResponseItem;
use crate::exec::ExecToolCallOutput;
use crate::token_data::KnownPlan;
use crate::token_data::PlanType;
@@ -53,8 +54,11 @@ pub enum SandboxErr {
#[derive(Error, Debug)]
pub enum CodexErr {
// todo(aibrahim): git rid of this error carrying the dangling artifacts
#[error("turn aborted")]
TurnAborted,
TurnAborted {
dangling_artifacts: Vec<ProcessedResponseItem>,
},
/// Returned by ResponsesClient when the SSE stream disconnects or errors out **after** the HTTP
/// handshake has succeeded but **before** it finished emitting `response.completed`.
@@ -158,7 +162,9 @@ pub enum CodexErr {
impl From<CancelErr> for CodexErr {
fn from(_: CancelErr) -> Self {
CodexErr::TurnAborted
CodexErr::TurnAborted {
dangling_artifacts: Vec::new(),
}
}
}
codex-rs/core/src/event_mapping.rs
+230 -126
View File
@@ -1,139 +1,131 @@
use crate::protocol::AgentMessageEvent;
use crate::protocol::AgentReasoningEvent;
use crate::protocol::AgentReasoningRawContentEvent;
use crate::protocol::EventMsg;
use crate::protocol::InputMessageKind;
use crate::protocol::UserMessageEvent;
use crate::protocol::WebSearchEndEvent;
use codex_protocol::items::AgentMessageContent;
use codex_protocol::items::AgentMessageItem;
use codex_protocol::items::ReasoningItem;
use codex_protocol::items::TurnItem;
use codex_protocol::items::UserMessageItem;
use codex_protocol::items::WebSearchItem;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ReasoningItemContent;
use codex_protocol::models::ReasoningItemReasoningSummary;
use codex_protocol::models::ResponseItem;
use codex_protocol::models::WebSearchAction;
use codex_protocol::user_input::UserInput;
use tracing::warn;
/// Convert a `ResponseItem` into zero or more `EventMsg` values that the UI can render.
///
/// When `show_raw_agent_reasoning` is false, raw reasoning content events are omitted.
pub(crate) fn map_response_item_to_event_messages(
item: &ResponseItem,
show_raw_agent_reasoning: bool,
) -> Vec<EventMsg> {
fn is_session_prefix(text: &str) -> bool {
let trimmed = text.trim_start();
let lowered = trimmed.to_ascii_lowercase();
lowered.starts_with("<environment_context>") || lowered.starts_with("<user_instructions>")
}
fn parse_user_message(message: &[ContentItem]) -> Option<UserMessageItem> {
let mut content: Vec<UserInput> = Vec::new();
for content_item in message.iter() {
match content_item {
ContentItem::InputText { text } => {
if is_session_prefix(text) {
return None;
}
content.push(UserInput::Text { text: text.clone() });
}
ContentItem::InputImage { image_url } => {
content.push(UserInput::Image {
image_url: image_url.clone(),
});
}
ContentItem::OutputText { text } => {
if is_session_prefix(text) {
return None;
}
warn!("Output text in user message: {}", text);
}
}
}
Some(UserMessageItem::new(&content))
}
fn parse_agent_message(message: &[ContentItem]) -> AgentMessageItem {
let mut content: Vec<AgentMessageContent> = Vec::new();
for content_item in message.iter() {
match content_item {
ContentItem::OutputText { text } => {
content.push(AgentMessageContent::Text { text: text.clone() });
}
_ => {
warn!(
"Unexpected content item in agent message: {:?}",
content_item
);
}
}
}
AgentMessageItem::new(&content)
}
pub fn parse_turn_item(item: &ResponseItem) -> Option<TurnItem> {
match item {
ResponseItem::Message { role, content, .. } => {
// Do not surface system messages as user events.
if role == "system" {
return Vec::new();
}
let mut events: Vec<EventMsg> = Vec::new();
let mut message_parts: Vec<String> = Vec::new();
let mut images: Vec<String> = Vec::new();
let mut kind: Option<InputMessageKind> = None;
for content_item in content.iter() {
match content_item {
ContentItem::InputText { text } => {
if kind.is_none() {
let trimmed = text.trim_start();
kind = if trimmed.starts_with("<environment_context>") {
Some(InputMessageKind::EnvironmentContext)
} else if trimmed.starts_with("<user_instructions>") {
Some(InputMessageKind::UserInstructions)
} else {
Some(InputMessageKind::Plain)
};
}
message_parts.push(text.clone());
}
ContentItem::InputImage { image_url } => {
images.push(image_url.clone());
}
ContentItem::OutputText { text } => {
events.push(EventMsg::AgentMessage(AgentMessageEvent {
message: text.clone(),
}));
}
}
}
if !message_parts.is_empty() || !images.is_empty() {
let message = if message_parts.is_empty() {
String::new()
} else {
message_parts.join("")
};
let images = if images.is_empty() {
None
} else {
Some(images)
};
events.push(EventMsg::UserMessage(UserMessageEvent {
message,
kind,
images,
}));
}
events
}
ResponseItem::Reasoning {
summary, content, ..
} => {
let mut events = Vec::new();
for ReasoningItemReasoningSummary::SummaryText { text } in summary {
events.push(EventMsg::AgentReasoning(AgentReasoningEvent {
text: text.clone(),
}));
}
if let Some(items) = content.as_ref().filter(|_| show_raw_agent_reasoning) {
for c in items {
let text = match c {
ReasoningItemContent::ReasoningText { text }
| ReasoningItemContent::Text { text } => text,
};
events.push(EventMsg::AgentReasoningRawContent(
AgentReasoningRawContentEvent { text: text.clone() },
));
}
}
events
}
ResponseItem::WebSearchCall { id, action, .. } => match action {
WebSearchAction::Search { query } => {
let call_id = id.clone().unwrap_or_else(|| "".to_string());
vec![EventMsg::WebSearchEnd(WebSearchEndEvent {
call_id,
query: query.clone(),
})]
}
WebSearchAction::Other => Vec::new(),
ResponseItem::Message { role, content, .. } => match role.as_str() {
"user" => parse_user_message(content).map(TurnItem::UserMessage),
"assistant" => Some(TurnItem::AgentMessage(parse_agent_message(content))),
"system" => None,
_ => None,
},
// Variants that require side effects are handled by higher layers and do not emit events here.
ResponseItem::FunctionCall { .. }
| ResponseItem::FunctionCallOutput { .. }
| ResponseItem::LocalShellCall { .. }
| ResponseItem::CustomToolCall { .. }
| ResponseItem::CustomToolCallOutput { .. }
| ResponseItem::Other => Vec::new(),
ResponseItem::Reasoning {
id,
summary,
content,
..
} => {
let summary_text = summary
.iter()
.map(|entry| match entry {
ReasoningItemReasoningSummary::SummaryText { text } => text.clone(),
})
.collect();
let raw_content = content
.clone()
.unwrap_or_default()
.into_iter()
.map(|entry| match entry {
ReasoningItemContent::ReasoningText { text }
| ReasoningItemContent::Text { text } => text,
})
.collect();
Some(TurnItem::Reasoning(ReasoningItem {
id: id.clone(),
summary_text,
raw_content,
}))
}
ResponseItem::WebSearchCall {
id,
action: WebSearchAction::Search { query },
..
} => Some(TurnItem::WebSearch(WebSearchItem {
id: id.clone().unwrap_or_default(),
query: query.clone(),
})),
_ => None,
}
}
#[cfg(test)]
mod tests {
use super::map_response_item_to_event_messages;
use crate::protocol::EventMsg;
use crate::protocol::InputMessageKind;
use assert_matches::assert_matches;
use super::parse_turn_item;
use codex_protocol::items::AgentMessageContent;
use codex_protocol::items::TurnItem;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ReasoningItemContent;
use codex_protocol::models::ReasoningItemReasoningSummary;
use codex_protocol::models::ResponseItem;
use codex_protocol::models::WebSearchAction;
use codex_protocol::user_input::UserInput;
use pretty_assertions::assert_eq;
#[test]
fn maps_user_message_with_text_and_two_images() {
fn parses_user_message_with_text_and_two_images() {
let img1 = "https://example.com/one.png".to_string();
let img2 = "https://example.com/two.jpg".to_string();
@@ -153,16 +145,128 @@ mod tests {
],
};
let events = map_response_item_to_event_messages(&item, false);
assert_eq!(events.len(), 1, "expected a single user message event");
let turn_item = parse_turn_item(&item).expect("expected user message turn item");
match &events[0] {
EventMsg::UserMessage(user) => {
assert_eq!(user.message, "Hello world");
assert_matches!(user.kind, Some(InputMessageKind::Plain));
assert_eq!(user.images, Some(vec![img1, img2]));
match turn_item {
TurnItem::UserMessage(user) => {
let expected_content = vec![
UserInput::Text {
text: "Hello world".to_string(),
},
UserInput::Image { image_url: img1 },
UserInput::Image { image_url: img2 },
];
assert_eq!(user.content, expected_content);
}
other => panic!("expected UserMessage, got {other:?}"),
other => panic!("expected TurnItem::UserMessage, got {other:?}"),
}
}
#[test]
fn parses_agent_message() {
let item = ResponseItem::Message {
id: Some("msg-1".to_string()),
role: "assistant".to_string(),
content: vec![ContentItem::OutputText {
text: "Hello from Codex".to_string(),
}],
};
let turn_item = parse_turn_item(&item).expect("expected agent message turn item");
match turn_item {
TurnItem::AgentMessage(message) => {
let Some(AgentMessageContent::Text { text }) = message.content.first() else {
panic!("expected agent message text content");
};
assert_eq!(text, "Hello from Codex");
}
other => panic!("expected TurnItem::AgentMessage, got {other:?}"),
}
}
#[test]
fn parses_reasoning_summary_and_raw_content() {
let item = ResponseItem::Reasoning {
id: "reasoning_1".to_string(),
summary: vec![
ReasoningItemReasoningSummary::SummaryText {
text: "Step 1".to_string(),
},
ReasoningItemReasoningSummary::SummaryText {
text: "Step 2".to_string(),
},
],
content: Some(vec![ReasoningItemContent::ReasoningText {
text: "raw details".to_string(),
}]),
encrypted_content: None,
};
let turn_item = parse_turn_item(&item).expect("expected reasoning turn item");
match turn_item {
TurnItem::Reasoning(reasoning) => {
assert_eq!(
reasoning.summary_text,
vec!["Step 1".to_string(), "Step 2".to_string()]
);
assert_eq!(reasoning.raw_content, vec!["raw details".to_string()]);
}
other => panic!("expected TurnItem::Reasoning, got {other:?}"),
}
}
#[test]
fn parses_reasoning_including_raw_content() {
let item = ResponseItem::Reasoning {
id: "reasoning_2".to_string(),
summary: vec![ReasoningItemReasoningSummary::SummaryText {
text: "Summarized step".to_string(),
}],
content: Some(vec![
ReasoningItemContent::ReasoningText {
text: "raw step".to_string(),
},
ReasoningItemContent::Text {
text: "final thought".to_string(),
},
]),
encrypted_content: None,
};
let turn_item = parse_turn_item(&item).expect("expected reasoning turn item");
match turn_item {
TurnItem::Reasoning(reasoning) => {
assert_eq!(reasoning.summary_text, vec!["Summarized step".to_string()]);
assert_eq!(
reasoning.raw_content,
vec!["raw step".to_string(), "final thought".to_string()]
);
}
other => panic!("expected TurnItem::Reasoning, got {other:?}"),
}
}
#[test]
fn parses_web_search_call() {
let item = ResponseItem::WebSearchCall {
id: Some("ws_1".to_string()),
status: Some("completed".to_string()),
action: WebSearchAction::Search {
query: "weather".to_string(),
},
};
let turn_item = parse_turn_item(&item).expect("expected web search turn item");
match turn_item {
TurnItem::WebSearch(search) => {
assert_eq!(search.id, "ws_1");
assert_eq!(search.query, "weather");
}
other => panic!("expected TurnItem::WebSearch, got {other:?}"),
}
}
}
codex-rs/core/src/features.rs
+1 -1
View File
@@ -31,7 +31,7 @@ pub enum Feature {
UnifiedExec,
/// Use the streamable exec-command/write-stdin tool pair.
StreamableShell,
/// Use the official Rust MCP client (rmcp).
/// Enable experimental RMCP features such as OAuth login.
RmcpClient,
/// Include the freeform apply_patch tool.
ApplyPatchFreeform,
codex-rs/core/src/lib.rs
+2 -2
View File
@@ -36,6 +36,7 @@ mod mcp_tool_call;
mod message_history;
mod model_provider_info;
pub mod parse_command;
mod response_processing;
pub mod sandboxing;
pub mod token_data;
mod truncate;
@@ -98,11 +99,10 @@ pub use client_common::REVIEW_PROMPT;
pub use client_common::ResponseEvent;
pub use client_common::ResponseStream;
pub use codex::compact::content_items_to_text;
pub use codex::compact::is_session_prefix_message;
pub use codex_protocol::models::ContentItem;
pub use codex_protocol::models::LocalShellAction;
pub use codex_protocol::models::LocalShellExecAction;
pub use codex_protocol::models::LocalShellStatus;
pub use codex_protocol::models::ReasoningItemContent;
pub use codex_protocol::models::ResponseItem;
pub use event_mapping::parse_turn_item;
pub mod otel_init;
codex-rs/core/src/mcp_connection_manager.rs
+37 -150
View File
@@ -1,6 +1,6 @@
//! Connection manager for Model Context Protocol (MCP) servers.
//!
//! The [`McpConnectionManager`] owns one [`codex_mcp_client::McpClient`] per
//! The [`McpConnectionManager`] owns one [`codex_rmcp_client::RmcpClient`] per
//! configured server (keyed by the *server name*). It offers convenience
//! helpers to query the available tools across *all* servers and returns them
//! in a single aggregated map using the fully-qualified tool name
@@ -10,14 +10,12 @@ use std::collections::HashMap;
use std::collections::HashSet;
use std::env;
use std::ffi::OsString;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use anyhow::Context;
use anyhow::Result;
use anyhow::anyhow;
use codex_mcp_client::McpClient;
use codex_rmcp_client::OAuthCredentialsStoreMode;
use codex_rmcp_client::RmcpClient;
use mcp_types::ClientCapabilities;
@@ -99,134 +97,12 @@ struct ToolInfo {
}
struct ManagedClient {
client: McpClientAdapter,
client: Arc<RmcpClient>,
startup_timeout: Duration,
tool_timeout: Option<Duration>,
}
#[derive(Clone)]
enum McpClientAdapter {
Legacy(Arc<McpClient>),
Rmcp(Arc<RmcpClient>),
}
impl McpClientAdapter {
#[allow(clippy::too_many_arguments)]
async fn new_stdio_client(
use_rmcp_client: bool,
program: OsString,
args: Vec<OsString>,
env: Option<HashMap<String, String>>,
env_vars: Vec<String>,
cwd: Option<PathBuf>,
params: mcp_types::InitializeRequestParams,
startup_timeout: Duration,
) -> Result<Self> {
if use_rmcp_client {
let client =
Arc::new(RmcpClient::new_stdio_client(program, args, env, &env_vars, cwd).await?);
client.initialize(params, Some(startup_timeout)).await?;
Ok(McpClientAdapter::Rmcp(client))
} else {
let client =
Arc::new(McpClient::new_stdio_client(program, args, env, &env_vars, cwd).await?);
client.initialize(params, Some(startup_timeout)).await?;
Ok(McpClientAdapter::Legacy(client))
}
}
#[allow(clippy::too_many_arguments)]
async fn new_streamable_http_client(
server_name: String,
url: String,
bearer_token: Option<String>,
http_headers: Option<HashMap<String, String>>,
env_http_headers: Option<HashMap<String, String>>,
params: mcp_types::InitializeRequestParams,
startup_timeout: Duration,
store_mode: OAuthCredentialsStoreMode,
) -> Result<Self> {
let client = Arc::new(
RmcpClient::new_streamable_http_client(
&server_name,
&url,
bearer_token,
http_headers,
env_http_headers,
store_mode,
)
.await?,
);
client.initialize(params, Some(startup_timeout)).await?;
Ok(McpClientAdapter::Rmcp(client))
}
async fn list_tools(
&self,
params: Option<mcp_types::ListToolsRequestParams>,
timeout: Option<Duration>,
) -> Result<mcp_types::ListToolsResult> {
match self {
McpClientAdapter::Legacy(client) => client.list_tools(params, timeout).await,
McpClientAdapter::Rmcp(client) => client.list_tools(params, timeout).await,
}
}
async fn list_resources(
&self,
params: Option<mcp_types::ListResourcesRequestParams>,
timeout: Option<Duration>,
) -> Result<mcp_types::ListResourcesResult> {
match self {
McpClientAdapter::Legacy(_) => Ok(ListResourcesResult {
next_cursor: None,
resources: Vec::new(),
}),
McpClientAdapter::Rmcp(client) => client.list_resources(params, timeout).await,
}
}
async fn read_resource(
&self,
params: mcp_types::ReadResourceRequestParams,
timeout: Option<Duration>,
) -> Result<mcp_types::ReadResourceResult> {
match self {
McpClientAdapter::Legacy(_) => Err(anyhow!(
"resources/read is not supported by legacy MCP clients"
)),
McpClientAdapter::Rmcp(client) => client.read_resource(params, timeout).await,
}
}
async fn list_resource_templates(
&self,
params: Option<mcp_types::ListResourceTemplatesRequestParams>,
timeout: Option<Duration>,
) -> Result<mcp_types::ListResourceTemplatesResult> {
match self {
McpClientAdapter::Legacy(_) => Ok(ListResourceTemplatesResult {
next_cursor: None,
resource_templates: Vec::new(),
}),
McpClientAdapter::Rmcp(client) => client.list_resource_templates(params, timeout).await,
}
}
async fn call_tool(
&self,
name: String,
arguments: Option<serde_json::Value>,
timeout: Option<Duration>,
) -> Result<mcp_types::CallToolResult> {
match self {
McpClientAdapter::Legacy(client) => client.call_tool(name, arguments, timeout).await,
McpClientAdapter::Rmcp(client) => client.call_tool(name, arguments, timeout).await,
}
}
}
/// A thin wrapper around a set of running [`McpClient`] instances.
/// A thin wrapper around a set of running [`RmcpClient`] instances.
#[derive(Default)]
pub(crate) struct McpConnectionManager {
/// Server-name -> client instance.
@@ -243,7 +119,7 @@ pub(crate) struct McpConnectionManager {
}
impl McpConnectionManager {
/// Spawn a [`McpClient`] for each configured server.
/// Spawn a [`RmcpClient`] for each configured server.
///
/// * `mcp_servers` – Map loaded from the user configuration where *keys*
/// are human-readable server identifiers and *values* are the spawn
@@ -253,7 +129,6 @@ impl McpConnectionManager {
/// user should be informed about these errors.
pub async fn new(
mcp_servers: HashMap<String, McpServerConfig>,
use_rmcp_client: bool,
store_mode: OAuthCredentialsStoreMode,
) -> Result<(Self, ClientStartErrors)> {
// Early exit if no servers are configured.
@@ -316,7 +191,8 @@ impl McpConnectionManager {
protocol_version: mcp_types::MCP_SCHEMA_VERSION.to_owned(),
};
let client = match transport {
let resolved_bearer_token = resolved_bearer_token.unwrap_or_default();
let client_result = match transport {
McpServerTransportConfig::Stdio {
command,
args,
@@ -326,17 +202,18 @@ impl McpConnectionManager {
} => {
let command_os: OsString = command.into();
let args_os: Vec<OsString> = args.into_iter().map(Into::into).collect();
McpClientAdapter::new_stdio_client(
use_rmcp_client,
command_os,
args_os,
env,
env_vars,
cwd,
params,
startup_timeout,
)
.await
match RmcpClient::new_stdio_client(command_os, args_os, env, &env_vars, cwd)
.await
{
Ok(client) => {
let client = Arc::new(client);
client
.initialize(params.clone(), Some(startup_timeout))
.await
.map(|_| client)
}
Err(err) => Err(err.into()),
}
}
McpServerTransportConfig::StreamableHttp {
url,
@@ -344,22 +221,32 @@ impl McpConnectionManager {
env_http_headers,
..
} => {
McpClientAdapter::new_streamable_http_client(
server_name.clone(),
url,
resolved_bearer_token.unwrap_or_default(),
match RmcpClient::new_streamable_http_client(
&server_name,
&url,
resolved_bearer_token.clone(),
http_headers,
env_http_headers,
params,
startup_timeout,
store_mode,
)
.await
{
Ok(client) => {
let client = Arc::new(client);
client
.initialize(params.clone(), Some(startup_timeout))
.await
.map(|_| client)
}
Err(err) => Err(err),
}
}
}
.map(|c| (c, startup_timeout));
};
((server_name, tool_timeout), client)
(
(server_name, tool_timeout),
client_result.map(|client| (client, startup_timeout)),
)
});
}
codex-rs/core/src/response_processing.rs
+112
View File
@@ -0,0 +1,112 @@
use crate::codex::Session;
use crate::conversation_history::ConversationHistory;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::models::ResponseItem;
use tracing::warn;
/// Process streamed `ResponseItem`s from the model into the pair of:
/// - items we should record in conversation history; and
/// - `ResponseInputItem`s to send back to the model on the next turn.
pub(crate) async fn process_items(
processed_items: Vec<crate::codex::ProcessedResponseItem>,
is_review_mode: bool,
review_thread_history: &mut ConversationHistory,
sess: &Session,
) -> (Vec<ResponseInputItem>, Vec<ResponseItem>) {
let mut items_to_record_in_conversation_history = Vec::<ResponseItem>::new();
let mut responses = Vec::<ResponseInputItem>::new();
for processed_response_item in processed_items {
let crate::codex::ProcessedResponseItem { item, response } = processed_response_item;
match (&item, &response) {
(ResponseItem::Message { role, .. }, None) if role == "assistant" => {
// If the model returned a message, we need to record it.
items_to_record_in_conversation_history.push(item);
}
(
ResponseItem::LocalShellCall { .. },
Some(ResponseInputItem::FunctionCallOutput { call_id, output }),
) => {
items_to_record_in_conversation_history.push(item);
items_to_record_in_conversation_history.push(ResponseItem::FunctionCallOutput {
call_id: call_id.clone(),
output: output.clone(),
});
}
(
ResponseItem::FunctionCall { .. },
Some(ResponseInputItem::FunctionCallOutput { call_id, output }),
) => {
items_to_record_in_conversation_history.push(item);
items_to_record_in_conversation_history.push(ResponseItem::FunctionCallOutput {
call_id: call_id.clone(),
output: output.clone(),
});
}
(
ResponseItem::CustomToolCall { .. },
Some(ResponseInputItem::CustomToolCallOutput { call_id, output }),
) => {
items_to_record_in_conversation_history.push(item);
items_to_record_in_conversation_history.push(ResponseItem::CustomToolCallOutput {
call_id: call_id.clone(),
output: output.clone(),
});
}
(
ResponseItem::FunctionCall { .. },
Some(ResponseInputItem::McpToolCallOutput { call_id, result }),
) => {
items_to_record_in_conversation_history.push(item);
let output = match result {
Ok(call_tool_result) => {
crate::codex::convert_call_tool_result_to_function_call_output_payload(
call_tool_result,
)
}
Err(err) => FunctionCallOutputPayload {
content: err.clone(),
success: Some(false),
},
};
items_to_record_in_conversation_history.push(ResponseItem::FunctionCallOutput {
call_id: call_id.clone(),
output,
});
}
(
ResponseItem::Reasoning {
id,
summary,
content,
encrypted_content,
},
None,
) => {
items_to_record_in_conversation_history.push(ResponseItem::Reasoning {
id: id.clone(),
summary: summary.clone(),
content: content.clone(),
encrypted_content: encrypted_content.clone(),
});
}
_ => {
warn!("Unexpected response item: {item:?} with response: {response:?}");
}
};
if let Some(response) = response {
responses.push(response);
}
}
// Only attempt to take the lock if there is something to record.
if !items_to_record_in_conversation_history.is_empty() {
if is_review_mode {
review_thread_history.record_items(items_to_record_in_conversation_history.iter());
} else {
sess.record_conversation_items(&items_to_record_in_conversation_history)
.await;
}
}
(responses, items_to_record_in_conversation_history)
}
codex-rs/core/src/rollout/list.rs
+4 -3
View File
@@ -1,12 +1,11 @@
use std::cmp::Reverse;
use std::io::{self};
use std::num::NonZero;
use std::path::Path;
use std::path::PathBuf;
use codex_file_search as file_search;
use std::num::NonZero;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use time::OffsetDateTime;
use time::PrimitiveDateTime;
use time::format_description::FormatItem;
@@ -15,6 +14,7 @@ use uuid::Uuid;
use super::SESSIONS_SUBDIR;
use crate::protocol::EventMsg;
use codex_file_search as file_search;
use codex_protocol::protocol::RolloutItem;
use codex_protocol::protocol::RolloutLine;
use codex_protocol::protocol::SessionSource;
@@ -515,6 +515,7 @@ pub async fn find_conversation_path_by_id_str(
threads,
cancel,
compute_indices,
false,
)
.map_err(|e| io::Error::other(format!("file search failed: {e}")))?;
codex-rs/core/src/rollout/tests.rs
-4
View File
@@ -24,7 +24,6 @@ use codex_protocol::models::ContentItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::CompactedItem;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::InputMessageKind;
use codex_protocol::protocol::RolloutItem;
use codex_protocol::protocol::RolloutLine;
use codex_protocol::protocol::SessionMeta;
@@ -543,7 +542,6 @@ async fn test_tail_includes_last_response_items() -> Result<()> {
timestamp: ts.to_string(),
item: RolloutItem::EventMsg(EventMsg::UserMessage(UserMessageEvent {
message: "hello".into(),
kind: Some(InputMessageKind::Plain),
images: None,
})),
};
@@ -627,7 +625,6 @@ async fn test_tail_handles_short_sessions() -> Result<()> {
timestamp: ts.to_string(),
item: RolloutItem::EventMsg(EventMsg::UserMessage(UserMessageEvent {
message: "hi".into(),
kind: Some(InputMessageKind::Plain),
images: None,
})),
};
@@ -712,7 +709,6 @@ async fn test_tail_skips_trailing_non_responses() -> Result<()> {
timestamp: ts.to_string(),
item: RolloutItem::EventMsg(EventMsg::UserMessage(UserMessageEvent {
message: "hello".into(),
kind: Some(InputMessageKind::Plain),
images: None,
})),
};
codex-rs/core/src/state/item_collector.rs
-68
View File
@@ -1,68 +0,0 @@
use async_channel::Sender;
use codex_protocol::ConversationId;
use codex_protocol::items::TurnItem;
use codex_protocol::protocol::Event;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::ItemCompletedEvent;
use codex_protocol::protocol::ItemStartedEvent;
use tracing::error;
#[derive(Debug)]
pub(crate) struct ItemCollector {
thread_id: ConversationId,
turn_id: String,
tx_event: Sender<Event>,
}
impl ItemCollector {
pub fn new(
tx_event: Sender<Event>,
thread_id: ConversationId,
turn_id: String,
) -> ItemCollector {
ItemCollector {
tx_event,
thread_id,
turn_id,
}
}
pub async fn started(&self, item: TurnItem) {
let err = self
.tx_event
.send(Event {
id: self.turn_id.clone(),
msg: EventMsg::ItemStarted(ItemStartedEvent {
thread_id: self.thread_id,
turn_id: self.turn_id.clone(),
item,
}),
})
.await;
if let Err(e) = err {
error!("failed to send item started event: {e}");
}
}
pub async fn completed(&self, item: TurnItem) {
let err = self
.tx_event
.send(Event {
id: self.turn_id.clone(),
msg: EventMsg::ItemCompleted(ItemCompletedEvent {
thread_id: self.thread_id,
turn_id: self.turn_id.clone(),
item,
}),
})
.await;
if let Err(e) = err {
error!("failed to send item completed event: {e}");
}
}
pub async fn started_completed(&self, item: TurnItem) {
self.started(item.clone()).await;
self.completed(item).await;
}
}
codex-rs/core/src/state/mod.rs
-2
View File
@@ -1,9 +1,7 @@
mod item_collector;
mod service;
mod session;
mod turn;
pub(crate) use item_collector::ItemCollector;
pub(crate) use service::SessionServices;
pub(crate) use session::SessionState;
pub(crate) use turn::ActiveTurn;
codex-rs/core/src/state/session.rs
+6 -2
View File
@@ -36,8 +36,12 @@ impl SessionState {
self.history.record_items(items)
}
pub(crate) fn history_snapshot(&self) -> Vec<ResponseItem> {
self.history.contents()
pub(crate) fn history_snapshot(&mut self) -> Vec<ResponseItem> {
self.history.get_history()
}
pub(crate) fn clone_history(&self) -> ConversationHistory {
self.history.clone()
}
pub(crate) fn replace_history(&mut self, items: Vec<ResponseItem>) {
codex-rs/core/src/tools/events.rs
+97 -2
View File
@@ -1,6 +1,9 @@
use crate::codex::Session;
use crate::codex::TurnContext;
use crate::error::CodexErr;
use crate::error::SandboxErr;
use crate::exec::ExecToolCallOutput;
use crate::function_tool::FunctionCallError;
use crate::parse_command::parse_command;
use crate::protocol::EventMsg;
use crate::protocol::ExecCommandBeginEvent;
@@ -10,6 +13,7 @@ use crate::protocol::PatchApplyBeginEvent;
use crate::protocol::PatchApplyEndEvent;
use crate::protocol::TurnDiffEvent;
use crate::tools::context::SharedTurnDiffTracker;
use crate::tools::sandboxing::ToolError;
use std::collections::HashMap;
use std::path::Path;
use std::path::PathBuf;
@@ -196,12 +200,103 @@ impl ToolEmitter {
) => {
emit_patch_end(ctx, String::new(), (*message).to_string(), false).await;
}
(Self::UnifiedExec { command, cwd, .. }, _) => {
// TODO(jif) add end and failures.
(Self::UnifiedExec { command, cwd, .. }, ToolEventStage::Begin) => {
emit_exec_command_begin(ctx, &[command.to_string()], cwd.as_path()).await;
}
(Self::UnifiedExec { .. }, ToolEventStage::Success(output)) => {
emit_exec_end(
ctx,
output.stdout.text.clone(),
output.stderr.text.clone(),
output.aggregated_output.text.clone(),
output.exit_code,
output.duration,
format_exec_output_str(&output),
)
.await;
}
(
Self::UnifiedExec { .. },
ToolEventStage::Failure(ToolEventFailure::Output(output)),
) => {
emit_exec_end(
ctx,
output.stdout.text.clone(),
output.stderr.text.clone(),
output.aggregated_output.text.clone(),
output.exit_code,
output.duration,
format_exec_output_str(&output),
)
.await;
}
(
Self::UnifiedExec { .. },
ToolEventStage::Failure(ToolEventFailure::Message(message)),
) => {
emit_exec_end(
ctx,
String::new(),
(*message).to_string(),
(*message).to_string(),
-1,
Duration::ZERO,
format_exec_output(&message),
)
.await;
}
}
}
pub async fn begin(&self, ctx: ToolEventCtx<'_>) {
self.emit(ctx, ToolEventStage::Begin).await;
}
pub async fn finish(
&self,
ctx: ToolEventCtx<'_>,
out: Result<ExecToolCallOutput, ToolError>,
) -> Result<String, FunctionCallError> {
let event;
let result = match out {
Ok(output) => {
let content = super::format_exec_output_for_model(&output);
let exit_code = output.exit_code;
event = ToolEventStage::Success(output);
if exit_code == 0 {
Ok(content)
} else {
Err(FunctionCallError::RespondToModel(content))
}
}
Err(ToolError::Codex(CodexErr::Sandbox(SandboxErr::Timeout { output })))
| Err(ToolError::Codex(CodexErr::Sandbox(SandboxErr::Denied { output }))) => {
let response = super::format_exec_output_for_model(&output);
event = ToolEventStage::Failure(ToolEventFailure::Output(*output));
Err(FunctionCallError::RespondToModel(response))
}
Err(ToolError::Codex(err)) => {
let message = format!("execution error: {err:?}");
let response = super::format_exec_output(&message);
event = ToolEventStage::Failure(ToolEventFailure::Message(message));
Err(FunctionCallError::RespondToModel(response))
}
Err(ToolError::Rejected(msg)) | Err(ToolError::SandboxDenied(msg)) => {
// Normalize common rejection messages for exec tools so tests and
// users see a clear, consistent phrase.
let normalized = if msg == "rejected by user" {
"exec command rejected by user".to_string()
} else {
msg
};
let response = super::format_exec_output(&normalized);
event = ToolEventStage::Failure(ToolEventFailure::Message(normalized));
Err(FunctionCallError::RespondToModel(response))
}
};
self.emit(ctx, event).await;
result
}
}
async fn emit_exec_end(
codex-rs/core/src/tools/handlers/apply_patch.rs
+86 -26
View File
@@ -1,19 +1,24 @@
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::sync::Arc;
use crate::apply_patch;
use crate::apply_patch::InternalApplyPatchInvocation;
use crate::apply_patch::convert_apply_patch_to_protocol;
use crate::client_common::tools::FreeformTool;
use crate::client_common::tools::FreeformToolFormat;
use crate::client_common::tools::ResponsesApiTool;
use crate::client_common::tools::ToolSpec;
use crate::exec::ExecParams;
use crate::function_tool::FunctionCallError;
use crate::tools::context::ToolInvocation;
use crate::tools::context::ToolOutput;
use crate::tools::context::ToolPayload;
use crate::tools::handle_container_exec_with_params;
use crate::tools::events::ToolEmitter;
use crate::tools::events::ToolEventCtx;
use crate::tools::orchestrator::ToolOrchestrator;
use crate::tools::registry::ToolHandler;
use crate::tools::registry::ToolKind;
use crate::tools::runtimes::apply_patch::ApplyPatchRequest;
use crate::tools::runtimes::apply_patch::ApplyPatchRuntime;
use crate::tools::sandboxing::ToolCtx;
use crate::tools::spec::ApplyPatchToolArgs;
use crate::tools::spec::JsonSchema;
use async_trait::async_trait;
@@ -64,30 +69,85 @@ impl ToolHandler for ApplyPatchHandler {
}
};
let exec_params = ExecParams {
command: vec!["apply_patch".to_string(), patch_input.clone()],
cwd: turn.cwd.clone(),
timeout_ms: None,
env: HashMap::new(),
with_escalated_permissions: None,
justification: None,
arg0: None,
};
// Re-parse and verify the patch so we can compute changes and approval.
// Avoid building temporary ExecParams/command vectors; derive directly from inputs.
let cwd = turn.cwd.clone();
let command = vec!["apply_patch".to_string(), patch_input.clone()];
match codex_apply_patch::maybe_parse_apply_patch_verified(&command, &cwd) {
codex_apply_patch::MaybeApplyPatchVerified::Body(changes) => {
match apply_patch::apply_patch(session.as_ref(), turn.as_ref(), &call_id, changes)
.await
{
InternalApplyPatchInvocation::Output(item) => {
let content = item?;
Ok(ToolOutput::Function {
content,
success: Some(true),
})
}
InternalApplyPatchInvocation::DelegateToExec(apply) => {
let emitter = ToolEmitter::apply_patch(
convert_apply_patch_to_protocol(&apply.action),
!apply.user_explicitly_approved_this_action,
);
let event_ctx = ToolEventCtx::new(
session.as_ref(),
turn.as_ref(),
&call_id,
Some(&tracker),
);
emitter.begin(event_ctx).await;
let content = handle_container_exec_with_params(
tool_name.as_str(),
exec_params,
Arc::clone(&session),
Arc::clone(&turn),
Arc::clone(&tracker),
call_id.clone(),
)
.await?;
let req = ApplyPatchRequest {
patch: apply.action.patch.clone(),
cwd: apply.action.cwd.clone(),
timeout_ms: None,
user_explicitly_approved: apply.user_explicitly_approved_this_action,
codex_exe: turn.codex_linux_sandbox_exe.clone(),
};
Ok(ToolOutput::Function {
content,
success: Some(true),
})
let mut orchestrator = ToolOrchestrator::new();
let mut runtime = ApplyPatchRuntime::new();
let tool_ctx = ToolCtx {
session: session.as_ref(),
turn: turn.as_ref(),
call_id: call_id.clone(),
tool_name: tool_name.to_string(),
};
let out = orchestrator
.run(&mut runtime, &req, &tool_ctx, &turn, turn.approval_policy)
.await;
let event_ctx = ToolEventCtx::new(
session.as_ref(),
turn.as_ref(),
&call_id,
Some(&tracker),
);
let content = emitter.finish(event_ctx, out).await?;
Ok(ToolOutput::Function {
content,
success: Some(true),
})
}
}
}
codex_apply_patch::MaybeApplyPatchVerified::CorrectnessError(parse_error) => {
Err(FunctionCallError::RespondToModel(format!(
"apply_patch verification failed: {parse_error}"
)))
}
codex_apply_patch::MaybeApplyPatchVerified::ShellParseError(error) => {
tracing::trace!("Failed to parse apply_patch input, {error:?}");
Err(FunctionCallError::RespondToModel(
"apply_patch handler received invalid patch input".to_string(),
))
}
codex_apply_patch::MaybeApplyPatchVerified::NotApplyPatch => {
Err(FunctionCallError::RespondToModel(
"apply_patch handler received non-apply_patch input".to_string(),
))
}
}
}
}
codex-rs/core/src/tools/handlers/shell.rs
+147 -21
View File
@@ -2,6 +2,9 @@ use async_trait::async_trait;
use codex_protocol::models::ShellToolCallParams;
use std::sync::Arc;
use crate::apply_patch;
use crate::apply_patch::InternalApplyPatchInvocation;
use crate::apply_patch::convert_apply_patch_to_protocol;
use crate::codex::Session;
use crate::codex::TurnContext;
use crate::exec::ExecParams;
@@ -11,9 +14,16 @@ use crate::function_tool::FunctionCallError;
use crate::tools::context::ToolInvocation;
use crate::tools::context::ToolOutput;
use crate::tools::context::ToolPayload;
use crate::tools::handle_container_exec_with_params;
use crate::tools::events::ToolEmitter;
use crate::tools::events::ToolEventCtx;
use crate::tools::orchestrator::ToolOrchestrator;
use crate::tools::registry::ToolHandler;
use crate::tools::registry::ToolKind;
use crate::tools::runtimes::apply_patch::ApplyPatchRequest;
use crate::tools::runtimes::apply_patch::ApplyPatchRuntime;
use crate::tools::runtimes::shell::ShellRequest;
use crate::tools::runtimes::shell::ShellRuntime;
use crate::tools::sandboxing::ToolCtx;
pub struct ShellHandler;
@@ -73,35 +83,27 @@ impl ToolHandler for ShellHandler {
))
})?;
let exec_params = Self::to_exec_params(params, session.as_ref(), turn.as_ref());
let content = handle_container_exec_with_params(
Self::run_exec_like(
tool_name.as_str(),
exec_params,
Arc::clone(&session),
Arc::clone(&turn),
Arc::clone(&tracker),
call_id.clone(),
session,
turn,
tracker,
call_id,
)
.await?;
Ok(ToolOutput::Function {
content,
success: Some(true),
})
.await
}
ToolPayload::LocalShell { params } => {
let exec_params = Self::to_exec_params(params, session.as_ref(), turn.as_ref());
let content = handle_container_exec_with_params(
Self::run_exec_like(
tool_name.as_str(),
exec_params,
Arc::clone(&session),
Arc::clone(&turn),
Arc::clone(&tracker),
call_id.clone(),
session,
turn,
tracker,
call_id,
)
.await?;
Ok(ToolOutput::Function {
content,
success: Some(true),
})
.await
}
_ => Err(FunctionCallError::RespondToModel(format!(
"unsupported payload for shell handler: {tool_name}"
@@ -110,6 +112,130 @@ impl ToolHandler for ShellHandler {
}
}
impl ShellHandler {
async fn run_exec_like(
tool_name: &str,
exec_params: ExecParams,
session: Arc<Session>,
turn: Arc<TurnContext>,
tracker: crate::tools::context::SharedTurnDiffTracker,
call_id: String,
) -> Result<ToolOutput, FunctionCallError> {
if exec_params.with_escalated_permissions.unwrap_or(false)
&& !matches!(
turn.approval_policy,
codex_protocol::protocol::AskForApproval::OnRequest
)
{
return Err(FunctionCallError::RespondToModel(format!(
"approval policy is {policy:?}; reject command — you should not ask for escalated permissions if the approval policy is {policy:?}",
policy = turn.approval_policy
)));
}
match codex_apply_patch::maybe_parse_apply_patch_verified(
&exec_params.command,
&exec_params.cwd,
) {
codex_apply_patch::MaybeApplyPatchVerified::Body(changes) => {
match apply_patch::apply_patch(session.as_ref(), turn.as_ref(), &call_id, changes)
.await
{
InternalApplyPatchInvocation::Output(item) => {
let content = item?;
return Ok(ToolOutput::Function {
content,
success: Some(true),
});
}
InternalApplyPatchInvocation::DelegateToExec(apply) => {
let emitter = ToolEmitter::apply_patch(
convert_apply_patch_to_protocol(&apply.action),
!apply.user_explicitly_approved_this_action,
);
let event_ctx = ToolEventCtx::new(
session.as_ref(),
turn.as_ref(),
&call_id,
Some(&tracker),
);
emitter.begin(event_ctx).await;
let req = ApplyPatchRequest {
patch: apply.action.patch.clone(),
cwd: apply.action.cwd.clone(),
timeout_ms: exec_params.timeout_ms,
user_explicitly_approved: apply.user_explicitly_approved_this_action,
codex_exe: turn.codex_linux_sandbox_exe.clone(),
};
let mut orchestrator = ToolOrchestrator::new();
let mut runtime = ApplyPatchRuntime::new();
let tool_ctx = ToolCtx {
session: session.as_ref(),
turn: turn.as_ref(),
call_id: call_id.clone(),
tool_name: tool_name.to_string(),
};
let out = orchestrator
.run(&mut runtime, &req, &tool_ctx, &turn, turn.approval_policy)
.await;
let event_ctx = ToolEventCtx::new(
session.as_ref(),
turn.as_ref(),
&call_id,
Some(&tracker),
);
let content = emitter.finish(event_ctx, out).await?;
return Ok(ToolOutput::Function {
content,
success: Some(true),
});
}
}
}
codex_apply_patch::MaybeApplyPatchVerified::CorrectnessError(parse_error) => {
return Err(FunctionCallError::RespondToModel(format!(
"apply_patch verification failed: {parse_error}"
)));
}
codex_apply_patch::MaybeApplyPatchVerified::ShellParseError(error) => {
tracing::trace!("Failed to parse shell command, {error:?}");
}
codex_apply_patch::MaybeApplyPatchVerified::NotApplyPatch => {}
}
let emitter = ToolEmitter::shell(exec_params.command.clone(), exec_params.cwd.clone());
let event_ctx = ToolEventCtx::new(session.as_ref(), turn.as_ref(), &call_id, None);
emitter.begin(event_ctx).await;
let req = ShellRequest {
command: exec_params.command.clone(),
cwd: exec_params.cwd.clone(),
timeout_ms: exec_params.timeout_ms,
env: exec_params.env.clone(),
with_escalated_permissions: exec_params.with_escalated_permissions,
justification: exec_params.justification.clone(),
};
let mut orchestrator = ToolOrchestrator::new();
let mut runtime = ShellRuntime::new();
let tool_ctx = ToolCtx {
session: session.as_ref(),
turn: turn.as_ref(),
call_id: call_id.clone(),
tool_name: tool_name.to_string(),
};
let out = orchestrator
.run(&mut runtime, &req, &tool_ctx, &turn, turn.approval_policy)
.await;
let event_ctx = ToolEventCtx::new(session.as_ref(), turn.as_ref(), &call_id, None);
let content = emitter.finish(event_ctx, out).await?;
Ok(ToolOutput::Function {
content,
success: Some(true),
})
}
}
#[cfg(test)]
mod tests {
use super::*;
codex-rs/core/src/tools/handlers/unified_exec.rs
+22 -7
View File
@@ -5,6 +5,9 @@ use serde::Deserialize;
use serde::Serialize;
use crate::function_tool::FunctionCallError;
use crate::protocol::EventMsg;
use crate::protocol::ExecCommandOutputDeltaEvent;
use crate::protocol::ExecOutputStream;
use crate::tools::context::ToolInvocation;
use crate::tools::context::ToolOutput;
use crate::tools::context::ToolPayload;
@@ -87,11 +90,7 @@ impl ToolHandler for UnifiedExecHandler {
};
let manager: &UnifiedExecSessionManager = &session.services.unified_exec_manager;
let context = UnifiedExecContext {
session: &session,
turn: turn.as_ref(),
call_id: &call_id,
};
let context = UnifiedExecContext::new(session.clone(), turn.clone(), call_id.clone());
let response = match tool_name.as_str() {
"exec_command" => {
@@ -101,8 +100,12 @@ impl ToolHandler for UnifiedExecHandler {
))
})?;
let event_ctx =
ToolEventCtx::new(context.session, context.turn, context.call_id, None);
let event_ctx = ToolEventCtx::new(
context.session.as_ref(),
context.turn.as_ref(),
&context.call_id,
None,
);
let emitter =
ToolEmitter::unified_exec(args.cmd.clone(), context.turn.cwd.clone(), true);
emitter.emit(event_ctx, ToolEventStage::Begin).await;
@@ -148,6 +151,18 @@ impl ToolHandler for UnifiedExecHandler {
}
};
// Emit a delta event with the chunk of output we just produced, if any.
if !response.output.is_empty() {
let delta = ExecCommandOutputDeltaEvent {
call_id: response.event_call_id.clone(),
stream: ExecOutputStream::Stdout,
chunk: response.output.as_bytes().to_vec(),
};
session
.send_event(turn.as_ref(), EventMsg::ExecCommandOutputDelta(delta))
.await;
}
let content = serialize_response(&response).map_err(|err| {
FunctionCallError::RespondToModel(format!(
"failed to serialize unified exec output: {err:?}"
codex-rs/core/src/tools/mod.rs
+1 -206
View File
@@ -9,37 +9,11 @@ pub mod runtimes;
pub mod sandboxing;
pub mod spec;
use crate::apply_patch;
use crate::apply_patch::InternalApplyPatchInvocation;
use crate::apply_patch::convert_apply_patch_to_protocol;
use crate::codex::Session;
use crate::codex::TurnContext;
use crate::error::CodexErr;
use crate::error::SandboxErr;
use crate::exec::ExecParams;
use crate::exec::ExecToolCallOutput;
use crate::function_tool::FunctionCallError;
use crate::tools::context::SharedTurnDiffTracker;
use crate::tools::events::ToolEmitter;
use crate::tools::events::ToolEventCtx;
use crate::tools::events::ToolEventFailure;
use crate::tools::events::ToolEventStage;
use crate::tools::orchestrator::ToolOrchestrator;
use crate::tools::runtimes::apply_patch::ApplyPatchRequest;
use crate::tools::runtimes::apply_patch::ApplyPatchRuntime;
use crate::tools::runtimes::shell::ShellRequest;
use crate::tools::runtimes::shell::ShellRuntime;
use crate::tools::sandboxing::ToolCtx;
use crate::tools::sandboxing::ToolError;
use codex_apply_patch::MaybeApplyPatchVerified;
use codex_apply_patch::maybe_parse_apply_patch_verified;
use codex_protocol::protocol::AskForApproval;
use codex_utils_string::take_bytes_at_char_boundary;
use codex_utils_string::take_last_bytes_at_char_boundary;
pub use router::ToolRouter;
use serde::Serialize;
use std::sync::Arc;
use tracing::trace;
// Model-formatting limits: clients get full streams; only content sent to the model is truncated.
pub(crate) const MODEL_FORMAT_MAX_BYTES: usize = 10 * 1024; // 10 KiB
@@ -54,186 +28,6 @@ pub(crate) const TELEMETRY_PREVIEW_MAX_LINES: usize = 64; // lines
pub(crate) const TELEMETRY_PREVIEW_TRUNCATION_NOTICE: &str =
"[... telemetry preview truncated ...]";
// TODO(jif) break this down
pub(crate) async fn handle_container_exec_with_params(
tool_name: &str,
params: ExecParams,
sess: Arc<Session>,
turn_context: Arc<TurnContext>,
turn_diff_tracker: SharedTurnDiffTracker,
call_id: String,
) -> Result<String, FunctionCallError> {
let _otel_event_manager = turn_context.client.get_otel_event_manager();
if params.with_escalated_permissions.unwrap_or(false)
&& !matches!(turn_context.approval_policy, AskForApproval::OnRequest)
{
return Err(FunctionCallError::RespondToModel(format!(
"approval policy is {policy:?}; reject command — you should not ask for escalated permissions if the approval policy is {policy:?}",
policy = turn_context.approval_policy
)));
}
// check if this was a patch, and apply it if so
let apply_patch_exec = match maybe_parse_apply_patch_verified(&params.command, &params.cwd) {
MaybeApplyPatchVerified::Body(changes) => {
match apply_patch::apply_patch(sess.as_ref(), turn_context.as_ref(), &call_id, changes)
.await
{
InternalApplyPatchInvocation::Output(item) => return item,
InternalApplyPatchInvocation::DelegateToExec(apply_patch_exec) => {
Some(apply_patch_exec)
}
}
}
MaybeApplyPatchVerified::CorrectnessError(parse_error) => {
// It looks like an invocation of `apply_patch`, but we
// could not resolve it into a patch that would apply
// cleanly. Return to model for resample.
return Err(FunctionCallError::RespondToModel(format!(
"apply_patch verification failed: {parse_error}"
)));
}
MaybeApplyPatchVerified::ShellParseError(error) => {
trace!("Failed to parse shell command, {error:?}");
None
}
MaybeApplyPatchVerified::NotApplyPatch => None,
};
let (event_emitter, diff_opt) = match apply_patch_exec.as_ref() {
Some(exec) => (
ToolEmitter::apply_patch(
convert_apply_patch_to_protocol(&exec.action),
!exec.user_explicitly_approved_this_action,
),
Some(&turn_diff_tracker),
),
None => (
ToolEmitter::shell(params.command.clone(), params.cwd.clone()),
None,
),
};
let event_ctx = ToolEventCtx::new(sess.as_ref(), turn_context.as_ref(), &call_id, diff_opt);
event_emitter.emit(event_ctx, ToolEventStage::Begin).await;
// Build runtime contexts only when needed (shell/apply_patch below).
if let Some(exec) = apply_patch_exec {
// Route apply_patch execution through the new orchestrator/runtime.
let req = ApplyPatchRequest {
patch: exec.action.patch.clone(),
cwd: params.cwd.clone(),
timeout_ms: params.timeout_ms,
user_explicitly_approved: exec.user_explicitly_approved_this_action,
codex_exe: turn_context.codex_linux_sandbox_exe.clone(),
};
let mut orchestrator = ToolOrchestrator::new();
let mut runtime = ApplyPatchRuntime::new();
let tool_ctx = ToolCtx {
session: sess.as_ref(),
turn: turn_context.as_ref(),
call_id: call_id.clone(),
tool_name: tool_name.to_string(),
};
let out = orchestrator
.run(
&mut runtime,
&req,
&tool_ctx,
&turn_context,
turn_context.approval_policy,
)
.await;
handle_exec_outcome(&event_emitter, event_ctx, out).await
} else {
// Route shell execution through the new orchestrator/runtime.
let req = ShellRequest {
command: params.command.clone(),
cwd: params.cwd.clone(),
timeout_ms: params.timeout_ms,
env: params.env.clone(),
with_escalated_permissions: params.with_escalated_permissions,
justification: params.justification.clone(),
};
let mut orchestrator = ToolOrchestrator::new();
let mut runtime = ShellRuntime::new();
let tool_ctx = ToolCtx {
session: sess.as_ref(),
turn: turn_context.as_ref(),
call_id: call_id.clone(),
tool_name: tool_name.to_string(),
};
let out = orchestrator
.run(
&mut runtime,
&req,
&tool_ctx,
&turn_context,
turn_context.approval_policy,
)
.await;
handle_exec_outcome(&event_emitter, event_ctx, out).await
}
}
async fn handle_exec_outcome(
event_emitter: &ToolEmitter,
event_ctx: ToolEventCtx<'_>,
out: Result<ExecToolCallOutput, ToolError>,
) -> Result<String, FunctionCallError> {
let event;
let result = match out {
Ok(output) => {
let content = format_exec_output_for_model(&output);
let exit_code = output.exit_code;
event = ToolEventStage::Success(output);
if exit_code == 0 {
Ok(content)
} else {
Err(FunctionCallError::RespondToModel(content))
}
}
Err(ToolError::Codex(CodexErr::Sandbox(SandboxErr::Timeout { output })))
| Err(ToolError::Codex(CodexErr::Sandbox(SandboxErr::Denied { output }))) => {
let response = format_exec_output_for_model(&output);
event = ToolEventStage::Failure(ToolEventFailure::Output(*output));
Err(FunctionCallError::RespondToModel(response))
}
Err(ToolError::Codex(err)) => {
let message = format!("execution error: {err:?}");
let response = format_exec_output(&message);
event = ToolEventStage::Failure(ToolEventFailure::Message(message));
Err(FunctionCallError::RespondToModel(format_exec_output(
&response,
)))
}
Err(ToolError::Rejected(msg)) | Err(ToolError::SandboxDenied(msg)) => {
// Normalize common rejection messages for exec tools so tests and
// users see a clear, consistent phrase.
let normalized = if msg == "rejected by user" {
"exec command rejected by user".to_string()
} else {
msg
};
let response = format_exec_output(&normalized);
event = ToolEventStage::Failure(ToolEventFailure::Message(normalized));
Err(FunctionCallError::RespondToModel(format_exec_output(
&response,
)))
}
};
event_emitter.emit(event_ctx, event).await;
result
}
/// Format the combined exec output for sending back to the model.
/// Includes exit code and duration metadata; truncates large bodies safely.
pub fn format_exec_output_for_model(exec_output: &ExecToolCallOutput) -> String {
@@ -363,6 +157,7 @@ fn truncate_formatted_exec_output(content: &str, total_lines: usize) -> String {
#[cfg(test)]
mod tests {
use super::*;
use crate::function_tool::FunctionCallError;
use regex_lite::Regex;
fn truncate_function_error(err: FunctionCallError) -> FunctionCallError {
codex-rs/core/src/tools/orchestrator.rs
+2 -2
View File
@@ -98,9 +98,9 @@ impl ToolOrchestrator {
"sandbox denied and no retry".to_string(),
));
}
// Under `Never`, do not retry without sandbox; surface a concise message
// Under `Never` or `OnRequest`, do not retry without sandbox; surface a concise message
// derived from the actual output (platform-agnostic).
if matches!(approval_policy, AskForApproval::Never) {
if !tool.wants_no_sandbox_approval(approval_policy) {
let msg = build_never_denied_message_from_output(output.as_ref());
return Err(ToolError::SandboxDenied(msg));
}
codex-rs/core/src/tools/parallel.rs
+40 -8
View File
@@ -2,6 +2,7 @@ use std::sync::Arc;
use tokio::sync::RwLock;
use tokio_util::either::Either;
use tokio_util::sync::CancellationToken;
use tokio_util::task::AbortOnDropHandle;
use crate::codex::Session;
@@ -9,8 +10,10 @@ use crate::codex::TurnContext;
use crate::error::CodexErr;
use crate::function_tool::FunctionCallError;
use crate::tools::context::SharedTurnDiffTracker;
use crate::tools::context::ToolPayload;
use crate::tools::router::ToolCall;
use crate::tools::router::ToolRouter;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ResponseInputItem;
pub(crate) struct ToolCallRuntime {
@@ -40,6 +43,7 @@ impl ToolCallRuntime {
pub(crate) fn handle_tool_call(
&self,
call: ToolCall,
cancellation_token: CancellationToken,
) -> impl std::future::Future<Output = Result<ResponseInputItem, CodexErr>> {
let supports_parallel = self.router.tool_supports_parallel(&call.tool_name);
@@ -48,18 +52,24 @@ impl ToolCallRuntime {
let turn = Arc::clone(&self.turn_context);
let tracker = Arc::clone(&self.tracker);
let lock = Arc::clone(&self.parallel_execution);
let aborted_response = Self::aborted_response(&call);
let handle: AbortOnDropHandle<Result<ResponseInputItem, FunctionCallError>> =
AbortOnDropHandle::new(tokio::spawn(async move {
let _guard = if supports_parallel {
Either::Left(lock.read().await)
} else {
Either::Right(lock.write().await)
};
tokio::select! {
_ = cancellation_token.cancelled() => Ok(aborted_response),
res = async {
let _guard = if supports_parallel {
Either::Left(lock.read().await)
} else {
Either::Right(lock.write().await)
};
router
.dispatch_tool_call(session, turn, tracker, call)
.await
router
.dispatch_tool_call(session, turn, tracker, call)
.await
} => res,
}
}));
async move {
@@ -74,3 +84,25 @@ impl ToolCallRuntime {
}
}
}
impl ToolCallRuntime {
fn aborted_response(call: &ToolCall) -> ResponseInputItem {
match &call.payload {
ToolPayload::Custom { .. } => ResponseInputItem::CustomToolCallOutput {
call_id: call.call_id.clone(),
output: "aborted".to_string(),
},
ToolPayload::Mcp { .. } => ResponseInputItem::McpToolCallOutput {
call_id: call.call_id.clone(),
result: Err("aborted".to_string()),
},
_ => ResponseInputItem::FunctionCallOutput {
call_id: call.call_id.clone(),
output: FunctionCallOutputPayload {
content: "aborted".to_string(),
success: None,
},
},
}
}
}
codex-rs/core/src/tools/runtimes/apply_patch.rs
+5
View File
@@ -17,6 +17,7 @@ use crate::tools::sandboxing::ToolCtx;
use crate::tools::sandboxing::ToolError;
use crate::tools::sandboxing::ToolRuntime;
use crate::tools::sandboxing::with_cached_approval;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::ReviewDecision;
use futures::future::BoxFuture;
use std::collections::HashMap;
@@ -127,6 +128,10 @@ impl Approvable<ApplyPatchRequest> for ApplyPatchRuntime {
.await
})
}
fn wants_no_sandbox_approval(&self, policy: AskForApproval) -> bool {
!matches!(policy, AskForApproval::Never)
}
}
impl ToolRuntime<ApplyPatchRequest, ExecToolCallOutput> for ApplyPatchRuntime {
codex-rs/core/src/tools/sandboxing.rs
+5
View File
@@ -121,6 +121,11 @@ pub(crate) trait Approvable<Req> {
}
}
/// Decide we can request an approval for no-sandbox execution.
fn wants_no_sandbox_approval(&self, policy: AskForApproval) -> bool {
!matches!(policy, AskForApproval::Never | AskForApproval::OnRequest)
}
fn start_approval_async<'a>(
&'a mut self,
req: &'a Req,
codex-rs/core/src/tools/spec.rs
+158 -176
View File
@@ -1019,21 +1019,24 @@ mod tests {
}
}
fn assert_eq_tool_names(tools: &[ConfiguredToolSpec], expected_names: &[&str]) {
let tool_names = tools
.iter()
.map(|tool| tool_name(&tool.spec))
.collect::<Vec<_>>();
assert_eq!(
tool_names.len(),
expected_names.len(),
"tool_name mismatch, {tool_names:?}, {expected_names:?}",
// Avoid order-based assertions; compare via set containment instead.
fn assert_contains_tool_names(tools: &[ConfiguredToolSpec], expected_subset: &[&str]) {
use std::collections::HashSet;
let mut names = HashSet::new();
let mut duplicates = Vec::new();
for name in tools.iter().map(|t| tool_name(&t.spec)) {
if !names.insert(name) {
duplicates.push(name);
}
}
assert!(
duplicates.is_empty(),
"duplicate tool entries detected: {duplicates:?}"
);
for (name, expected_name) in tool_names.iter().zip(expected_names.iter()) {
assert_eq!(
name, expected_name,
"tool_name mismatch, {name:?}, {expected_name:?}"
for expected in expected_subset {
assert!(
names.contains(expected),
"expected tool {expected} to be present; had: {names:?}"
);
}
}
@@ -1056,8 +1059,105 @@ mod tests {
.unwrap_or_else(|| panic!("expected tool {expected_name}"))
}
fn strip_descriptions_schema(schema: &mut JsonSchema) {
match schema {
JsonSchema::Boolean { description }
| JsonSchema::String { description }
| JsonSchema::Number { description } => {
*description = None;
}
JsonSchema::Array { items, description } => {
strip_descriptions_schema(items);
*description = None;
}
JsonSchema::Object {
properties,
required: _,
additional_properties,
} => {
for v in properties.values_mut() {
strip_descriptions_schema(v);
}
if let Some(AdditionalProperties::Schema(s)) = additional_properties {
strip_descriptions_schema(s);
}
}
}
}
fn strip_descriptions_tool(spec: &mut ToolSpec) {
match spec {
ToolSpec::Function(ResponsesApiTool { parameters, .. }) => {
strip_descriptions_schema(parameters);
}
ToolSpec::Freeform(_) | ToolSpec::LocalShell {} | ToolSpec::WebSearch {} => {}
}
}
#[test]
fn test_build_specs() {
fn test_full_toolset_specs_for_gpt5_codex() {
let model_family = find_family_for_model("gpt-5-codex")
.expect("gpt-5-codex should be a valid model family");
let mut features = Features::with_defaults();
features.enable(Feature::UnifiedExec);
features.enable(Feature::WebSearchRequest);
features.enable(Feature::ViewImageTool);
let config = ToolsConfig::new(&ToolsConfigParams {
model_family: &model_family,
features: &features,
});
let (tools, _) = build_specs(&config, None).build();
// Build actual map name -> spec
use std::collections::BTreeMap;
use std::collections::HashSet;
let mut actual: BTreeMap<String, ToolSpec> = BTreeMap::new();
let mut duplicate_names = Vec::new();
for t in &tools {
let name = tool_name(&t.spec).to_string();
if actual.insert(name.clone(), t.spec.clone()).is_some() {
duplicate_names.push(name);
}
}
assert!(
duplicate_names.is_empty(),
"duplicate tool entries detected: {duplicate_names:?}"
);
// Build expected from the same helpers used by the builder.
let mut expected: BTreeMap<String, ToolSpec> = BTreeMap::new();
for spec in [
create_exec_command_tool(),
create_write_stdin_tool(),
create_shell_tool(),
create_list_mcp_resources_tool(),
create_list_mcp_resource_templates_tool(),
create_read_mcp_resource_tool(),
PLAN_TOOL.clone(),
create_apply_patch_freeform_tool(),
ToolSpec::WebSearch {},
create_view_image_tool(),
] {
expected.insert(tool_name(&spec).to_string(), spec);
}
// Exact name set match — this is the only test allowed to fail when tools change.
let actual_names: HashSet<_> = actual.keys().cloned().collect();
let expected_names: HashSet<_> = expected.keys().cloned().collect();
assert_eq!(actual_names, expected_names, "tool name set mismatch");
// Compare specs ignoring human-readable descriptions.
for name in expected.keys() {
let mut a = actual.get(name).expect("present").clone();
let mut e = expected.get(name).expect("present").clone();
strip_descriptions_tool(&mut a);
strip_descriptions_tool(&mut e);
assert_eq!(a, e, "spec mismatch for {name}");
}
}
#[test]
fn test_build_specs_contains_expected_basics() {
let model_family = find_family_for_model("codex-mini-latest")
.expect("codex-mini-latest should be a valid model family");
let mut features = Features::with_defaults();
@@ -1068,25 +1168,25 @@ mod tests {
features: &features,
});
let (tools, _) = build_specs(&config, Some(HashMap::new())).build();
let mut expected = vec!["exec_command", "write_stdin"];
if let Some(shell_tool) = shell_tool_name(&config) {
expected.push(shell_tool);
}
expected.extend([
"list_mcp_resources",
"list_mcp_resource_templates",
"read_mcp_resource",
"update_plan",
"web_search",
"view_image",
]);
assert_eq_tool_names(&tools, &expected);
let tool_names = tools.iter().map(|t| t.spec.name()).collect::<Vec<_>>();
assert_eq!(
&tool_names,
&[
"exec_command",
"write_stdin",
"local_shell",
"list_mcp_resources",
"list_mcp_resource_templates",
"read_mcp_resource",
"update_plan",
"web_search",
"view_image",
]
);
}
#[test]
fn test_build_specs_default_shell() {
fn test_build_specs_default_shell_present() {
let model_family = find_family_for_model("o3").expect("o3 should be a valid model family");
let mut features = Features::with_defaults();
features.enable(Feature::WebSearchRequest);
@@ -1097,20 +1197,12 @@ mod tests {
});
let (tools, _) = build_specs(&config, Some(HashMap::new())).build();
let mut expected = vec!["exec_command", "write_stdin"];
// Only check the shell variant and a couple of core tools.
let mut subset = vec!["exec_command", "write_stdin", "update_plan"];
if let Some(shell_tool) = shell_tool_name(&config) {
expected.push(shell_tool);
subset.push(shell_tool);
}
expected.extend([
"list_mcp_resources",
"list_mcp_resource_templates",
"read_mcp_resource",
"update_plan",
"web_search",
"view_image",
]);
assert_eq_tool_names(&tools, &expected);
assert_contains_tool_names(&tools, &subset);
}
#[test]
@@ -1165,7 +1257,7 @@ mod tests {
}
#[test]
fn test_build_specs_mcp_tools() {
fn test_build_specs_mcp_tools_converted() {
let model_family = find_family_for_model("o3").expect("o3 should be a valid model family");
let mut features = Features::with_defaults();
features.enable(Feature::UnifiedExec);
@@ -1213,22 +1305,6 @@ mod tests {
)
.build();
let mut expected = vec!["exec_command", "write_stdin"];
if let Some(shell_tool) = shell_tool_name(&config) {
expected.push(shell_tool);
}
expected.extend([
"list_mcp_resources",
"list_mcp_resource_templates",
"read_mcp_resource",
"update_plan",
"web_search",
"view_image",
"test_server/do_something_cool",
]);
assert_eq_tool_names(&tools, &expected);
let tool = find_tool(&tools, "test_server/do_something_cool");
assert_eq!(
&tool.spec,
@@ -1334,23 +1410,19 @@ mod tests {
]);
let (tools, _) = build_specs(&config, Some(tools_map)).build();
// Expect exec_command/write_stdin first, followed by MCP tools sorted by fully-qualified name.
let mut expected = vec!["exec_command", "write_stdin"];
if let Some(shell_tool) = shell_tool_name(&config) {
expected.push(shell_tool);
}
expected.extend([
"list_mcp_resources",
"list_mcp_resource_templates",
"read_mcp_resource",
"update_plan",
"view_image",
"test_server/cool",
"test_server/do",
"test_server/something",
]);
assert_eq_tool_names(&tools, &expected);
// Only assert that the MCP tools themselves are sorted by fully-qualified name.
let mcp_names: Vec<_> = tools
.iter()
.map(|t| tool_name(&t.spec).to_string())
.filter(|n| n.starts_with("test_server/"))
.collect();
let expected = vec![
"test_server/cool".to_string(),
"test_server/do".to_string(),
"test_server/something".to_string(),
];
assert_eq!(mcp_names, expected);
}
#[test]
@@ -1389,28 +1461,9 @@ mod tests {
)
.build();
let mut expected = vec!["exec_command", "write_stdin"];
let has_shell = if let Some(shell_tool) = shell_tool_name(&config) {
expected.push(shell_tool);
true
} else {
false
};
expected.extend([
"list_mcp_resources",
"list_mcp_resource_templates",
"read_mcp_resource",
"update_plan",
"apply_patch",
"web_search",
"view_image",
"dash/search",
]);
assert_eq_tool_names(&tools, &expected);
let tool = find_tool(&tools, "dash/search");
assert_eq!(
tools[if has_shell { 10 } else { 9 }].spec,
tool.spec,
ToolSpec::Function(ResponsesApiTool {
name: "dash/search".to_string(),
parameters: JsonSchema::Object {
@@ -1463,27 +1516,9 @@ mod tests {
)
.build();
let mut expected = vec!["exec_command", "write_stdin"];
let has_shell = if let Some(shell_tool) = shell_tool_name(&config) {
expected.push(shell_tool);
true
} else {
false
};
expected.extend([
"list_mcp_resources",
"list_mcp_resource_templates",
"read_mcp_resource",
"update_plan",
"apply_patch",
"web_search",
"view_image",
"dash/paginate",
]);
assert_eq_tool_names(&tools, &expected);
let tool = find_tool(&tools, "dash/paginate");
assert_eq!(
tools[if has_shell { 10 } else { 9 }].spec,
tool.spec,
ToolSpec::Function(ResponsesApiTool {
name: "dash/paginate".to_string(),
parameters: JsonSchema::Object {
@@ -1535,26 +1570,9 @@ mod tests {
)
.build();
let mut expected = vec!["exec_command", "write_stdin"];
let has_shell = if let Some(shell_tool) = shell_tool_name(&config) {
expected.push(shell_tool);
true
} else {
false
};
expected.extend([
"list_mcp_resources",
"list_mcp_resource_templates",
"read_mcp_resource",
"update_plan",
"apply_patch",
"web_search",
"view_image",
"dash/tags",
]);
assert_eq_tool_names(&tools, &expected);
let tool = find_tool(&tools, "dash/tags");
assert_eq!(
tools[if has_shell { 10 } else { 9 }].spec,
tool.spec,
ToolSpec::Function(ResponsesApiTool {
name: "dash/tags".to_string(),
parameters: JsonSchema::Object {
@@ -1608,26 +1626,9 @@ mod tests {
)
.build();
let mut expected = vec!["exec_command", "write_stdin"];
let has_shell = if let Some(shell_tool) = shell_tool_name(&config) {
expected.push(shell_tool);
true
} else {
false
};
expected.extend([
"list_mcp_resources",
"list_mcp_resource_templates",
"read_mcp_resource",
"update_plan",
"apply_patch",
"web_search",
"view_image",
"dash/value",
]);
assert_eq_tool_names(&tools, &expected);
let tool = find_tool(&tools, "dash/value");
assert_eq!(
tools[if has_shell { 10 } else { 9 }].spec,
tool.spec,
ToolSpec::Function(ResponsesApiTool {
name: "dash/value".to_string(),
parameters: JsonSchema::Object {
@@ -1718,28 +1719,9 @@ mod tests {
)
.build();
let mut expected = vec!["exec_command", "write_stdin"];
let has_shell = if let Some(shell_tool) = shell_tool_name(&config) {
expected.push(shell_tool);
true
} else {
false
};
expected.extend([
"list_mcp_resources",
"list_mcp_resource_templates",
"read_mcp_resource",
"update_plan",
"apply_patch",
"web_search",
"view_image",
"test_server/do_something_cool",
]);
assert_eq_tool_names(&tools, &expected);
let tool = find_tool(&tools, "test_server/do_something_cool");
assert_eq!(
tools[if has_shell { 10 } else { 9 }].spec,
tool.spec,
ToolSpec::Function(ResponsesApiTool {
name: "test_server/do_something_cool".to_string(),
parameters: JsonSchema::Object {
codex-rs/core/src/truncate.rs
+60 -40
View File
@@ -1,18 +1,35 @@
//! Utilities for truncating large chunks of output while preserving a prefix
//! and suffix on UTF-8 boundaries.
use codex_utils_tokenizer::Tokenizer;
/// Truncate the middle of a UTF-8 string to at most `max_bytes` bytes,
/// preserving the beginning and the end. Returns the possibly truncated
/// string and `Some(original_token_count)` (estimated at 4 bytes/token)
/// string and `Some(original_token_count)` (counted with the local tokenizer;
/// falls back to a 4-bytes-per-token estimate if the tokenizer cannot load)
/// if truncation occurred; otherwise returns the original string and `None`.
pub(crate) fn truncate_middle(s: &str, max_bytes: usize) -> (String, Option<u64>) {
if s.len() <= max_bytes {
return (s.to_string(), None);
}
let est_tokens = (s.len() as u64).div_ceil(4);
// Build a tokenizer for counting (default to o200k_base; fall back to cl100k_base).
// If both fail, fall back to a 4-bytes-per-token estimate.
let tok = Tokenizer::try_default().ok();
let token_count = |text: &str| -> u64 {
if let Some(ref t) = tok {
t.count(text) as u64
} else {
(text.len() as u64).div_ceil(4)
}
};
let total_tokens = token_count(s);
if max_bytes == 0 {
return (format!("…{est_tokens} tokens truncated…"), Some(est_tokens));
return (
format!("…{total_tokens} tokens truncated…"),
Some(total_tokens),
);
}
fn truncate_on_boundary(input: &str, max_len: usize) -> &str {
@@ -50,13 +67,17 @@ pub(crate) fn truncate_middle(s: &str, max_bytes: usize) -> (String, Option<u64>
idx
}
let mut guess_tokens = est_tokens;
// Iterate to stabilize marker length → keep budget → boundaries.
let mut guess_tokens: u64 = 1;
for _ in 0..4 {
let marker = format!("…{guess_tokens} tokens truncated…");
let marker_len = marker.len();
let keep_budget = max_bytes.saturating_sub(marker_len);
if keep_budget == 0 {
return (format!("…{est_tokens} tokens truncated…"), Some(est_tokens));
return (
format!("…{total_tokens} tokens truncated…"),
Some(total_tokens),
);
}
let left_budget = keep_budget / 2;
@@ -67,59 +88,72 @@ pub(crate) fn truncate_middle(s: &str, max_bytes: usize) -> (String, Option<u64>
suffix_start = prefix_end;
}
let kept_content_bytes = prefix_end + (s.len() - suffix_start);
let truncated_content_bytes = s.len().saturating_sub(kept_content_bytes);
let new_tokens = (truncated_content_bytes as u64).div_ceil(4);
// Tokens actually removed (middle slice) using the real tokenizer.
let removed_tokens = token_count(&s[prefix_end..suffix_start]);
if new_tokens == guess_tokens {
let mut out = String::with_capacity(marker_len + kept_content_bytes + 1);
// If the number of digits in the token count does not change the marker length,
// we can finalize output.
let final_marker = format!("…{removed_tokens} tokens truncated…");
if final_marker.len() == marker_len {
let kept_content_bytes = prefix_end + (s.len() - suffix_start);
let mut out = String::with_capacity(final_marker.len() + kept_content_bytes + 1);
out.push_str(&s[..prefix_end]);
out.push_str(&marker);
out.push_str(&final_marker);
out.push('\n');
out.push_str(&s[suffix_start..]);
return (out, Some(est_tokens));
return (out, Some(total_tokens));
}
guess_tokens = new_tokens;
guess_tokens = removed_tokens;
}
// Fallback build after iterations: compute with the last guess.
let marker = format!("…{guess_tokens} tokens truncated…");
let marker_len = marker.len();
let keep_budget = max_bytes.saturating_sub(marker_len);
if keep_budget == 0 {
return (format!("…{est_tokens} tokens truncated…"), Some(est_tokens));
return (
format!("…{total_tokens} tokens truncated…"),
Some(total_tokens),
);
}
let left_budget = keep_budget / 2;
let right_budget = keep_budget - left_budget;
let prefix_end = pick_prefix_end(s, left_budget);
let suffix_start = pick_suffix_start(s, right_budget);
let mut suffix_start = pick_suffix_start(s, right_budget);
if suffix_start < prefix_end {
suffix_start = prefix_end;
}
let mut out = String::with_capacity(marker_len + prefix_end + (s.len() - suffix_start) + 1);
out.push_str(&s[..prefix_end]);
out.push_str(&marker);
out.push('\n');
out.push_str(&s[suffix_start..]);
(out, Some(est_tokens))
(out, Some(total_tokens))
}
#[cfg(test)]
mod tests {
use super::truncate_middle;
use codex_utils_tokenizer::Tokenizer;
#[test]
fn truncate_middle_no_newlines_fallback() {
let tok = Tokenizer::try_default().expect("load tokenizer");
let s = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ*";
let max_bytes = 32;
let (out, original) = truncate_middle(s, max_bytes);
assert!(out.starts_with("abc"));
assert!(out.contains("tokens truncated"));
assert!(out.ends_with("XYZ*"));
assert_eq!(original, Some((s.len() as u64).div_ceil(4)));
assert_eq!(original, Some(tok.count(s) as u64));
}
#[test]
fn truncate_middle_prefers_newline_boundaries() {
let tok = Tokenizer::try_default().expect("load tokenizer");
let mut s = String::new();
for i in 1..=20 {
s.push_str(&format!("{i:03}\n"));
@@ -131,50 +165,36 @@ mod tests {
assert!(out.starts_with("001\n002\n003\n004\n"));
assert!(out.contains("tokens truncated"));
assert!(out.ends_with("017\n018\n019\n020\n"));
assert_eq!(tokens, Some(20));
assert_eq!(tokens, Some(tok.count(&s) as u64));
}
#[test]
fn truncate_middle_handles_utf8_content() {
let tok = Tokenizer::try_default().expect("load tokenizer");
let s = "😀😀😀😀😀😀😀😀😀😀\nsecond line with ascii text\n";
let max_bytes = 32;
let (out, tokens) = truncate_middle(s, max_bytes);
assert!(out.contains("tokens truncated"));
assert!(!out.contains('\u{fffd}'));
assert_eq!(tokens, Some((s.len() as u64).div_ceil(4)));
assert_eq!(tokens, Some(tok.count(s) as u64));
}
#[test]
fn truncate_middle_prefers_newline_boundaries_2() {
let tok = Tokenizer::try_default().expect("load tokenizer");
// Build a multi-line string of 20 numbered lines (each "NNN\n").
let mut s = String::new();
for i in 1..=20 {
s.push_str(&format!("{i:03}\n"));
}
// Total length: 20 lines * 4 bytes per line = 80 bytes.
assert_eq!(s.len(), 80);
// Choose a cap that forces truncation while leaving room for
// a few lines on each side after accounting for the marker.
let max_bytes = 64;
// Expect exact output: first 4 lines, marker, last 4 lines, and correct token estimate (80/4 = 20).
assert_eq!(
truncate_middle(&s, max_bytes),
(
r#"001
002
003
004
…12 tokens truncated…
017
018
019
020
"#
.to_string(),
Some(20)
)
);
let (out, total) = truncate_middle(&s, max_bytes);
assert!(out.starts_with("001\n002\n003\n004\n"));
assert!(out.contains("tokens truncated"));
assert!(out.ends_with("017\n018\n019\n020\n"));
assert_eq!(total, Some(tok.count(&s) as u64));
}
}
codex-rs/core/src/unified_exec/mod.rs
+31 -11
View File
@@ -22,6 +22,8 @@
//! - `session_manager.rs`: orchestration (approvals, sandboxing, reuse) and request handling.
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::AtomicI32;
use std::time::Duration;
@@ -45,10 +47,20 @@ pub(crate) const MAX_YIELD_TIME_MS: u64 = 30_000;
pub(crate) const DEFAULT_MAX_OUTPUT_TOKENS: usize = 10_000;
pub(crate) const UNIFIED_EXEC_OUTPUT_MAX_BYTES: usize = 1024 * 1024; // 1 MiB
pub(crate) struct UnifiedExecContext<'a> {
pub session: &'a Session,
pub turn: &'a TurnContext,
pub call_id: &'a str,
pub(crate) struct UnifiedExecContext {
pub session: Arc<Session>,
pub turn: Arc<TurnContext>,
pub call_id: String,
}
impl UnifiedExecContext {
pub fn new(session: Arc<Session>, turn: Arc<TurnContext>, call_id: String) -> Self {
Self {
session,
turn,
call_id,
}
}
}
#[derive(Debug)]
@@ -70,6 +82,7 @@ pub(crate) struct WriteStdinRequest<'a> {
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct UnifiedExecResponse {
pub event_call_id: String,
pub chunk_id: String,
pub wall_time: Duration,
pub output: String,
@@ -78,10 +91,20 @@ pub(crate) struct UnifiedExecResponse {
pub original_token_count: Option<usize>,
}
#[derive(Debug, Default)]
#[derive(Default)]
pub(crate) struct UnifiedExecSessionManager {
next_session_id: AtomicI32,
sessions: Mutex<HashMap<i32, session::UnifiedExecSession>>,
sessions: Mutex<HashMap<i32, SessionEntry>>,
}
struct SessionEntry {
session: session::UnifiedExecSession,
session_ref: Arc<Session>,
turn_ref: Arc<TurnContext>,
call_id: String,
command: String,
cwd: PathBuf,
started_at: tokio::time::Instant,
}
pub(crate) fn clamp_yield_time(yield_time_ms: Option<u64>) -> u64 {
@@ -163,11 +186,8 @@ mod tests {
cmd: &str,
yield_time_ms: Option<u64>,
) -> Result<UnifiedExecResponse, UnifiedExecError> {
let context = UnifiedExecContext {
session,
turn: turn.as_ref(),
call_id: "call",
};
let context =
UnifiedExecContext::new(Arc::clone(session), Arc::clone(turn), "call".to_string());
session
.services
codex-rs/core/src/unified_exec/session_manager.rs
+161 -33
View File
@@ -5,9 +5,14 @@ use tokio::sync::mpsc;
use tokio::time::Duration;
use tokio::time::Instant;
use crate::exec::ExecToolCallOutput;
use crate::exec::StreamOutput;
use crate::exec_env::CODEX_SESSION_ID_ENV_VAR;
use crate::exec_env::create_env;
use crate::sandboxing::ExecEnv;
use crate::tools::events::ToolEmitter;
use crate::tools::events::ToolEventCtx;
use crate::tools::events::ToolEventStage;
use crate::tools::orchestrator::ToolOrchestrator;
use crate::tools::runtimes::unified_exec::UnifiedExecRequest as UnifiedExecToolRequest;
use crate::tools::runtimes::unified_exec::UnifiedExecRuntime;
@@ -15,6 +20,7 @@ use crate::tools::sandboxing::ToolCtx;
use super::ExecCommandRequest;
use super::MIN_YIELD_TIME_MS;
use super::SessionEntry;
use super::UnifiedExecContext;
use super::UnifiedExecError;
use super::UnifiedExecResponse;
@@ -31,7 +37,7 @@ impl UnifiedExecSessionManager {
pub(crate) async fn exec_command(
&self,
request: ExecCommandRequest<'_>,
context: &UnifiedExecContext<'_>,
context: &UnifiedExecContext,
) -> Result<UnifiedExecResponse, UnifiedExecError> {
let shell_flag = if request.login { "-lc" } else { "-c" };
let command = vec![
@@ -60,17 +66,36 @@ impl UnifiedExecSessionManager {
let session_id = if session.has_exited() {
None
} else {
Some(self.store_session(session).await)
Some(
self.store_session(session, context, request.command, start)
.await,
)
};
Ok(UnifiedExecResponse {
let response = UnifiedExecResponse {
event_call_id: context.call_id.clone(),
chunk_id,
wall_time,
output,
session_id,
exit_code,
original_token_count,
})
};
// If the command completed during this call, emit an ExecCommandEnd via the emitter.
if response.session_id.is_none() {
let exit = response.exit_code.unwrap_or(-1);
Self::emit_exec_end_from_context(
context,
request.command.to_string(),
response.output.clone(),
exit,
response.wall_time,
)
.await;
}
Ok(response)
}
pub(crate) async fn write_stdin(
@@ -99,37 +124,60 @@ impl UnifiedExecSessionManager {
let (output, original_token_count) = truncate_output_to_tokens(&text, max_tokens);
let chunk_id = generate_chunk_id();
let (session_id, exit_code) = self.refresh_session_state(session_id).await;
let status = self.refresh_session_state(session_id).await;
let (session_id, exit_code, completion_entry, event_call_id) = match status {
SessionStatus::Alive { exit_code, call_id } => {
(Some(session_id), exit_code, None, call_id)
}
SessionStatus::Exited { exit_code, entry } => {
let call_id = entry.call_id.clone();
(None, exit_code, Some(*entry), call_id)
}
SessionStatus::Unknown => {
return Err(UnifiedExecError::UnknownSessionId { session_id });
}
};
Ok(UnifiedExecResponse {
let response = UnifiedExecResponse {
event_call_id,
chunk_id,
wall_time,
output,
session_id,
exit_code,
original_token_count,
})
}
};
async fn refresh_session_state(&self, session_id: i32) -> (Option<i32>, Option<i32>) {
let mut sessions = self.sessions.lock().await;
if !sessions.contains_key(&session_id) {
return (None, None);
if let (Some(exit), Some(entry)) = (response.exit_code, completion_entry) {
let total_duration = Instant::now().saturating_duration_since(entry.started_at);
Self::emit_exec_end_from_entry(entry, response.output.clone(), exit, total_duration)
.await;
}
let has_exited = sessions
.get(&session_id)
.map(UnifiedExecSession::has_exited)
.unwrap_or(false);
let exit_code = sessions
.get(&session_id)
.and_then(UnifiedExecSession::exit_code);
Ok(response)
}
if has_exited {
sessions.remove(&session_id);
(None, exit_code)
async fn refresh_session_state(&self, session_id: i32) -> SessionStatus {
let mut sessions = self.sessions.lock().await;
let Some(entry) = sessions.get(&session_id) else {
return SessionStatus::Unknown;
};
let exit_code = entry.session.exit_code();
if entry.session.has_exited() {
let Some(entry) = sessions.remove(&session_id) else {
return SessionStatus::Unknown;
};
SessionStatus::Exited {
exit_code,
entry: Box::new(entry),
}
} else {
(Some(session_id), exit_code)
SessionStatus::Alive {
exit_code,
call_id: entry.call_id.clone(),
}
}
}
@@ -139,9 +187,9 @@ impl UnifiedExecSessionManager {
) -> Result<(mpsc::Sender<Vec<u8>>, OutputBuffer, Arc<Notify>), UnifiedExecError> {
let sessions = self.sessions.lock().await;
let (output_buffer, output_notify, writer_tx) =
if let Some(session) = sessions.get(&session_id) {
let (buffer, notify) = session.output_handles();
(buffer, notify, session.writer_sender())
if let Some(entry) = sessions.get(&session_id) {
let (buffer, notify) = entry.session.output_handles();
(buffer, notify, entry.session.writer_sender())
} else {
return Err(UnifiedExecError::UnknownSessionId { session_id });
};
@@ -159,14 +207,82 @@ impl UnifiedExecSessionManager {
.map_err(|_| UnifiedExecError::WriteToStdin)
}
async fn store_session(&self, session: UnifiedExecSession) -> i32 {
async fn store_session(
&self,
session: UnifiedExecSession,
context: &UnifiedExecContext,
command: &str,
started_at: Instant,
) -> i32 {
let session_id = self
.next_session_id
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
self.sessions.lock().await.insert(session_id, session);
let entry = SessionEntry {
session,
session_ref: Arc::clone(&context.session),
turn_ref: Arc::clone(&context.turn),
call_id: context.call_id.clone(),
command: command.to_string(),
cwd: context.turn.cwd.clone(),
started_at,
};
self.sessions.lock().await.insert(session_id, entry);
session_id
}
async fn emit_exec_end_from_entry(
entry: SessionEntry,
aggregated_output: String,
exit_code: i32,
duration: Duration,
) {
let output = ExecToolCallOutput {
exit_code,
stdout: StreamOutput::new(aggregated_output.clone()),
stderr: StreamOutput::new(String::new()),
aggregated_output: StreamOutput::new(aggregated_output),
duration,
timed_out: false,
};
let event_ctx = ToolEventCtx::new(
entry.session_ref.as_ref(),
entry.turn_ref.as_ref(),
&entry.call_id,
None,
);
let emitter = ToolEmitter::unified_exec(entry.command, entry.cwd, true);
emitter
.emit(event_ctx, ToolEventStage::Success(output))
.await;
}
async fn emit_exec_end_from_context(
context: &UnifiedExecContext,
command: String,
aggregated_output: String,
exit_code: i32,
duration: Duration,
) {
let output = ExecToolCallOutput {
exit_code,
stdout: StreamOutput::new(aggregated_output.clone()),
stderr: StreamOutput::new(String::new()),
aggregated_output: StreamOutput::new(aggregated_output),
duration,
timed_out: false,
};
let event_ctx = ToolEventCtx::new(
context.session.as_ref(),
context.turn.as_ref(),
&context.call_id,
None,
);
let emitter = ToolEmitter::unified_exec(command, context.turn.cwd.clone(), true);
emitter
.emit(event_ctx, ToolEventStage::Success(output))
.await;
}
pub(crate) async fn open_session_with_exec_env(
&self,
env: &ExecEnv,
@@ -185,7 +301,7 @@ impl UnifiedExecSessionManager {
pub(super) async fn open_session_with_sandbox(
&self,
command: Vec<String>,
context: &UnifiedExecContext<'_>,
context: &UnifiedExecContext,
) -> Result<UnifiedExecSession, UnifiedExecError> {
let mut orchestrator = ToolOrchestrator::new();
let mut runtime = UnifiedExecRuntime::new(self);
@@ -196,9 +312,9 @@ impl UnifiedExecSessionManager {
);
let req = UnifiedExecToolRequest::new(command, context.turn.cwd.clone(), env);
let tool_ctx = ToolCtx {
session: context.session,
turn: context.turn,
call_id: context.call_id.to_string(),
session: context.session.as_ref(),
turn: context.turn.as_ref(),
call_id: context.call_id.clone(),
tool_name: "exec_command".to_string(),
};
orchestrator
@@ -206,7 +322,7 @@ impl UnifiedExecSessionManager {
&mut runtime,
&req,
&tool_ctx,
context.turn,
context.turn.as_ref(),
context.turn.approval_policy,
)
.await
@@ -257,3 +373,15 @@ impl UnifiedExecSessionManager {
collected
}
}
enum SessionStatus {
Alive {
exit_code: Option<i32>,
call_id: String,
},
Exited {
exit_code: Option<i32>,
entry: Box<SessionEntry>,
},
Unknown,
}
codex-rs/core/tests/chat_completions_payload.rs
+1 -1
View File
@@ -8,12 +8,12 @@ use codex_core::LocalShellStatus;
use codex_core::ModelClient;
use codex_core::ModelProviderInfo;
use codex_core::Prompt;
use codex_core::ReasoningItemContent;
use codex_core::ResponseItem;
use codex_core::WireApi;
use codex_core::spawn::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR;
use codex_otel::otel_event_manager::OtelEventManager;
use codex_protocol::ConversationId;
use codex_protocol::models::ReasoningItemContent;
use core_test_support::load_default_config_for_test;
use futures::StreamExt;
use serde_json::Value;
codex-rs/core/tests/chat_completions_sse.rs
+3 -2
View File
@@ -13,6 +13,7 @@ use codex_core::WireApi;
use codex_core::spawn::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR;
use codex_otel::otel_event_manager::OtelEventManager;
use codex_protocol::ConversationId;
use codex_protocol::models::ReasoningItemContent;
use core_test_support::load_default_config_for_test;
use futures::StreamExt;
use tempfile::TempDir;
@@ -143,8 +144,8 @@ fn assert_reasoning(item: &ResponseItem, expected: &str) {
let mut combined = String::new();
for part in parts {
match part {
codex_core::ReasoningItemContent::ReasoningText { text }
| codex_core::ReasoningItemContent::Text { text } => combined.push_str(text),
ReasoningItemContent::ReasoningText { text }
| ReasoningItemContent::Text { text } => combined.push_str(text),
}
}
assert_eq!(combined, expected);
codex-rs/core/tests/common/Cargo.toml
+1
View File
@@ -10,6 +10,7 @@ path = "lib.rs"
anyhow = { workspace = true }
assert_cmd = { workspace = true }
codex-core = { workspace = true }
codex-protocol = { workspace = true }
notify = { workspace = true }
regex-lite = { workspace = true }
serde_json = { workspace = true }
codex-rs/core/tests/common/responses.rs
+179
View File
@@ -35,6 +35,22 @@ impl ResponseMock {
pub fn requests(&self) -> Vec<ResponsesRequest> {
self.requests.lock().unwrap().clone()
}
/// Returns true if any captured request contains a `function_call` with the
/// provided `call_id`.
pub fn saw_function_call(&self, call_id: &str) -> bool {
self.requests()
.iter()
.any(|req| req.has_function_call(call_id))
}
/// Returns the `output` string for a matching `function_call_output` with
/// the provided `call_id`, searching across all captured requests.
pub fn function_call_output_text(&self, call_id: &str) -> Option<String> {
self.requests()
.iter()
.find_map(|req| req.function_call_output_text(call_id))
}
}
#[derive(Debug, Clone)]
@@ -70,6 +86,28 @@ impl ResponsesRequest {
.unwrap_or_else(|| panic!("function call output {call_id} item not found in request"))
}
/// Returns true if this request's `input` contains a `function_call` with
/// the specified `call_id`.
pub fn has_function_call(&self, call_id: &str) -> bool {
self.input().iter().any(|item| {
item.get("type").and_then(Value::as_str) == Some("function_call")
&& item.get("call_id").and_then(Value::as_str) == Some(call_id)
})
}
/// If present, returns the `output` string of the `function_call_output`
/// entry matching `call_id` in this request's `input`.
pub fn function_call_output_text(&self, call_id: &str) -> Option<String> {
let binding = self.input();
let item = binding.iter().find(|item| {
item.get("type").and_then(Value::as_str) == Some("function_call_output")
&& item.get("call_id").and_then(Value::as_str) == Some(call_id)
})?;
item.get("output")
.and_then(Value::as_str)
.map(str::to_string)
}
pub fn header(&self, name: &str) -> Option<String> {
self.0
.headers
@@ -97,6 +135,10 @@ impl Match for ResponseMock {
.lock()
.unwrap()
.push(ResponsesRequest(request.clone()));
// Enforce invariant checks on every request body captured by the mock.
// Panic on orphan tool outputs or calls to catch regressions early.
validate_request_body_invariants(request);
true
}
}
@@ -167,6 +209,56 @@ pub fn ev_assistant_message(id: &str, text: &str) -> Value {
})
}
pub fn ev_reasoning_item(id: &str, summary: &[&str], raw_content: &[&str]) -> Value {
let summary_entries: Vec<Value> = summary
.iter()
.map(|text| serde_json::json!({"type": "summary_text", "text": text}))
.collect();
let mut event = serde_json::json!({
"type": "response.output_item.done",
"item": {
"type": "reasoning",
"id": id,
"summary": summary_entries,
}
});
if !raw_content.is_empty() {
let content_entries: Vec<Value> = raw_content
.iter()
.map(|text| serde_json::json!({"type": "reasoning_text", "text": text}))
.collect();
event["item"]["content"] = Value::Array(content_entries);
}
event
}
pub fn ev_web_search_call_added(id: &str, status: &str, query: &str) -> Value {
serde_json::json!({
"type": "response.output_item.added",
"item": {
"type": "web_search_call",
"id": id,
"status": status,
"action": {"type": "search", "query": query}
}
})
}
pub fn ev_web_search_call_done(id: &str, status: &str, query: &str) -> Value {
serde_json::json!({
"type": "response.output_item.done",
"item": {
"type": "web_search_call",
"id": id,
"status": status,
"action": {"type": "search", "query": query}
}
})
}
pub fn ev_function_call(call_id: &str, name: &str, arguments: &str) -> Value {
serde_json::json!({
"type": "response.output_item.done",
@@ -336,3 +428,90 @@ pub async fn mount_sse_sequence(server: &MockServer, bodies: Vec<String>) -> Res
response_mock
}
/// Validate invariants on the request body sent to `/v1/responses`.
///
/// - No `function_call_output`/`custom_tool_call_output` with missing/empty `call_id`.
/// - Every `function_call_output` must match a prior `function_call` or
/// `local_shell_call` with the same `call_id` in the same `input`.
/// - Every `custom_tool_call_output` must match a prior `custom_tool_call`.
/// - Additionally, enforce symmetry: every `function_call`/`custom_tool_call`
/// in the `input` must have a matching output entry.
fn validate_request_body_invariants(request: &wiremock::Request) {
let Ok(body): Result<Value, _> = request.body_json() else {
return;
};
let Some(items) = body.get("input").and_then(Value::as_array) else {
panic!("input array not found in request");
};
use std::collections::HashSet;
fn get_call_id(item: &Value) -> Option<&str> {
item.get("call_id")
.and_then(Value::as_str)
.filter(|id| !id.is_empty())
}
fn gather_ids(items: &[Value], kind: &str) -> HashSet<String> {
items
.iter()
.filter(|item| item.get("type").and_then(Value::as_str) == Some(kind))
.filter_map(get_call_id)
.map(str::to_string)
.collect()
}
fn gather_output_ids(items: &[Value], kind: &str, missing_msg: &str) -> HashSet<String> {
items
.iter()
.filter(|item| item.get("type").and_then(Value::as_str) == Some(kind))
.map(|item| {
let Some(id) = get_call_id(item) else {
panic!("{missing_msg}");
};
id.to_string()
})
.collect()
}
let function_calls = gather_ids(items, "function_call");
let custom_tool_calls = gather_ids(items, "custom_tool_call");
let local_shell_calls = gather_ids(items, "local_shell_call");
let function_call_outputs = gather_output_ids(
items,
"function_call_output",
"orphan function_call_output with empty call_id should be dropped",
);
let custom_tool_call_outputs = gather_output_ids(
items,
"custom_tool_call_output",
"orphan custom_tool_call_output with empty call_id should be dropped",
);
for cid in &function_call_outputs {
assert!(
function_calls.contains(cid) || local_shell_calls.contains(cid),
"function_call_output without matching call in input: {cid}",
);
}
for cid in &custom_tool_call_outputs {
assert!(
custom_tool_calls.contains(cid),
"custom_tool_call_output without matching call in input: {cid}",
);
}
for cid in &function_calls {
assert!(
function_call_outputs.contains(cid),
"Function call output is missing for call id: {cid}",
);
}
for cid in &custom_tool_calls {
assert!(
custom_tool_call_outputs.contains(cid),
"Custom tool call output is missing for call id: {cid}",
);
}
}
codex-rs/core/tests/common/test_codex.rs
+152
View File
@@ -1,17 +1,30 @@
use std::mem::swap;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use anyhow::Result;
use codex_core::CodexAuth;
use codex_core::CodexConversation;
use codex_core::ConversationManager;
use codex_core::ModelProviderInfo;
use codex_core::built_in_model_providers;
use codex_core::config::Config;
use codex_core::features::Feature;
use codex_core::protocol::AskForApproval;
use codex_core::protocol::EventMsg;
use codex_core::protocol::Op;
use codex_core::protocol::SandboxPolicy;
use codex_core::protocol::SessionConfiguredEvent;
use codex_protocol::config_types::ReasoningSummary;
use codex_protocol::user_input::UserInput;
use serde_json::Value;
use tempfile::TempDir;
use wiremock::MockServer;
use crate::load_default_config_for_test;
use crate::responses::start_mock_server;
use crate::wait_for_event;
type ConfigMutator = dyn FnOnce(&mut Config) + Send;
@@ -96,6 +109,12 @@ impl TestCodexBuilder {
mutator(&mut config);
}
if config.include_apply_patch_tool {
config.features.enable(Feature::ApplyPatchFreeform);
} else {
config.features.disable(Feature::ApplyPatchFreeform);
}
Ok((config, cwd))
}
}
@@ -107,6 +126,139 @@ pub struct TestCodex {
pub session_configured: SessionConfiguredEvent,
}
impl TestCodex {
pub fn cwd_path(&self) -> &Path {
self.cwd.path()
}
pub fn workspace_path(&self, rel: impl AsRef<Path>) -> PathBuf {
self.cwd_path().join(rel)
}
pub async fn submit_turn(&self, prompt: &str) -> Result<()> {
self.submit_turn_with_policy(prompt, SandboxPolicy::DangerFullAccess)
.await
}
pub async fn submit_turn_with_policy(
&self,
prompt: &str,
sandbox_policy: SandboxPolicy,
) -> Result<()> {
let session_model = self.session_configured.model.clone();
self.codex
.submit(Op::UserTurn {
items: vec![UserInput::Text {
text: prompt.into(),
}],
final_output_json_schema: None,
cwd: self.cwd.path().to_path_buf(),
approval_policy: AskForApproval::Never,
sandbox_policy,
model: session_model,
effort: None,
summary: ReasoningSummary::Auto,
})
.await?;
wait_for_event(&self.codex, |event| {
matches!(event, EventMsg::TaskComplete(_))
})
.await;
Ok(())
}
}
pub struct TestCodexHarness {
server: MockServer,
test: TestCodex,
}
impl TestCodexHarness {
pub async fn new() -> Result<Self> {
Self::with_builder(test_codex()).await
}
pub async fn with_config(mutator: impl FnOnce(&mut Config) + Send + 'static) -> Result<Self> {
Self::with_builder(test_codex().with_config(mutator)).await
}
pub async fn with_builder(mut builder: TestCodexBuilder) -> Result<Self> {
let server = start_mock_server().await;
let test = builder.build(&server).await?;
Ok(Self { server, test })
}
pub fn server(&self) -> &MockServer {
&self.server
}
pub fn test(&self) -> &TestCodex {
&self.test
}
pub fn cwd(&self) -> &Path {
self.test.cwd_path()
}
pub fn path(&self, rel: impl AsRef<Path>) -> PathBuf {
self.test.workspace_path(rel)
}
pub async fn submit(&self, prompt: &str) -> Result<()> {
self.test.submit_turn(prompt).await
}
pub async fn submit_with_policy(
&self,
prompt: &str,
sandbox_policy: SandboxPolicy,
) -> Result<()> {
self.test
.submit_turn_with_policy(prompt, sandbox_policy)
.await
}
pub async fn request_bodies(&self) -> Vec<Value> {
self.server
.received_requests()
.await
.expect("requests")
.into_iter()
.map(|req| serde_json::from_slice(&req.body).expect("request body json"))
.collect()
}
pub async fn function_call_output_value(&self, call_id: &str) -> Value {
let bodies = self.request_bodies().await;
function_call_output(&bodies, call_id).clone()
}
pub async fn function_call_stdout(&self, call_id: &str) -> String {
self.function_call_output_value(call_id)
.await
.get("output")
.and_then(Value::as_str)
.expect("output string")
.to_string()
}
}
fn function_call_output<'a>(bodies: &'a [Value], call_id: &str) -> &'a Value {
for body in bodies {
if let Some(items) = body.get("input").and_then(Value::as_array) {
for item in items {
if item.get("type").and_then(Value::as_str) == Some("function_call_output")
&& item.get("call_id").and_then(Value::as_str) == Some(call_id)
{
return item;
}
}
}
}
panic!("function_call_output {call_id} not found");
}
pub fn test_codex() -> TestCodexBuilder {
TestCodexBuilder {
config_mutators: vec![],
codex-rs/core/tests/suite/abort_tasks.rs
+98
View File
@@ -1,3 +1,4 @@
use std::sync::Arc;
use std::time::Duration;
use codex_core::protocol::EventMsg;
@@ -5,7 +6,9 @@ use codex_core::protocol::Op;
use codex_protocol::user_input::UserInput;
use core_test_support::responses::ev_completed;
use core_test_support::responses::ev_function_call;
use core_test_support::responses::ev_response_created;
use core_test_support::responses::mount_sse_once;
use core_test_support::responses::mount_sse_sequence;
use core_test_support::responses::sse;
use core_test_support::responses::start_mock_server;
use core_test_support::test_codex::test_codex;
@@ -67,3 +70,98 @@ async fn interrupt_long_running_tool_emits_turn_aborted() {
)
.await;
}
/// After an interrupt we expect the next request to the model to include both
/// the original tool call and an `"aborted"` `function_call_output`. This test
/// exercises the follow-up flow: it sends another user turn, inspects the mock
/// responses server, and ensures the model receives the synthesized abort.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn interrupt_tool_records_history_entries() {
let command = vec![
"bash".to_string(),
"-lc".to_string(),
"sleep 60".to_string(),
];
let call_id = "call-history";
let args = json!({
"command": command,
"timeout_ms": 60_000
})
.to_string();
let first_body = sse(vec![
ev_response_created("resp-history"),
ev_function_call(call_id, "shell", &args),
ev_completed("resp-history"),
]);
let follow_up_body = sse(vec![
ev_response_created("resp-followup"),
ev_completed("resp-followup"),
]);
let server = start_mock_server().await;
let response_mock = mount_sse_sequence(&server, vec![first_body, follow_up_body]).await;
let fixture = test_codex().build(&server).await.unwrap();
let codex = Arc::clone(&fixture.codex);
let wait_timeout = Duration::from_millis(100);
codex
.submit(Op::UserInput {
items: vec![UserInput::Text {
text: "start history recording".into(),
}],
})
.await
.unwrap();
wait_for_event_with_timeout(
&codex,
|ev| matches!(ev, EventMsg::ExecCommandBegin(_)),
wait_timeout,
)
.await;
codex.submit(Op::Interrupt).await.unwrap();
wait_for_event_with_timeout(
&codex,
|ev| matches!(ev, EventMsg::TurnAborted(_)),
wait_timeout,
)
.await;
codex
.submit(Op::UserInput {
items: vec![UserInput::Text {
text: "follow up".into(),
}],
})
.await
.unwrap();
wait_for_event_with_timeout(
&codex,
|ev| matches!(ev, EventMsg::TaskComplete(_)),
wait_timeout,
)
.await;
let requests = response_mock.requests();
assert!(
requests.len() == 2,
"expected two calls to the responses API, got {}",
requests.len()
);
assert!(
response_mock.saw_function_call(call_id),
"function call not recorded in responses payload"
);
assert_eq!(
response_mock.function_call_output_text(call_id).as_deref(),
Some("aborted"),
"aborted function call output not recorded in responses payload"
);
}
codex-rs/core/tests/suite/apply_patch_cli.rs
+1052
View File
File diff suppressed because it is too large Load Diff
codex-rs/core/tests/suite/client.rs
+1 -1
View File
@@ -9,7 +9,6 @@ use codex_core::ModelClient;
use codex_core::ModelProviderInfo;
use codex_core::NewConversation;
use codex_core::Prompt;
use codex_core::ReasoningItemContent;
use codex_core::ResponseEvent;
use codex_core::ResponseItem;
use codex_core::WireApi;
@@ -21,6 +20,7 @@ use codex_core::protocol::Op;
use codex_core::protocol::SessionSource;
use codex_otel::otel_event_manager::OtelEventManager;
use codex_protocol::ConversationId;
use codex_protocol::models::ReasoningItemContent;
use codex_protocol::models::ReasoningItemReasoningSummary;
use codex_protocol::models::WebSearchAction;
use codex_protocol::user_input::UserInput;
codex-rs/core/tests/suite/fork_conversation.rs
+5 -14
View File
@@ -1,17 +1,15 @@
use codex_core::CodexAuth;
use codex_core::ContentItem;
use codex_core::ConversationManager;
use codex_core::ModelProviderInfo;
use codex_core::NewConversation;
use codex_core::ResponseItem;
use codex_core::built_in_model_providers;
use codex_core::content_items_to_text;
use codex_core::is_session_prefix_message;
use codex_core::parse_turn_item;
use codex_core::protocol::ConversationPathResponseEvent;
use codex_core::protocol::EventMsg;
use codex_core::protocol::Op;
use codex_core::protocol::RolloutItem;
use codex_core::protocol::RolloutLine;
use codex_protocol::items::TurnItem;
use codex_protocol::user_input::UserInput;
use core_test_support::load_default_config_for_test;
use core_test_support::skip_if_no_network;
@@ -115,19 +113,12 @@ async fn fork_conversation_twice_drops_to_first_message() {
let find_user_input_positions = |items: &[RolloutItem]| -> Vec<usize> {
let mut pos = Vec::new();
for (i, it) in items.iter().enumerate() {
if let RolloutItem::ResponseItem(ResponseItem::Message { role, content, .. }) = it
&& role == "user"
&& content_items_to_text(content)
.is_some_and(|text| !is_session_prefix_message(&text))
if let RolloutItem::ResponseItem(response_item) = it
&& let Some(TurnItem::UserMessage(_)) = parse_turn_item(response_item)
{
// Consider any user message as an input boundary; recorder stores both EventMsg and ResponseItem.
// We specifically look for input items, which are represented as ContentItem::InputText.
if content
.iter()
.any(|c| matches!(c, ContentItem::InputText { .. }))
{
pos.push(i);
}
pos.push(i);
}
}
pos
codex-rs/core/tests/suite/items.rs
+178 -10
View File
@@ -2,12 +2,18 @@
use anyhow::Ok;
use codex_core::protocol::EventMsg;
use codex_core::protocol::ItemCompletedEvent;
use codex_core::protocol::ItemStartedEvent;
use codex_core::protocol::Op;
use codex_protocol::items::TurnItem;
use codex_protocol::user_input::UserInput;
use core_test_support::responses;
use core_test_support::responses::ev_assistant_message;
use core_test_support::responses::ev_completed;
use core_test_support::responses::ev_reasoning_item;
use core_test_support::responses::ev_response_created;
use core_test_support::responses::ev_web_search_call_added;
use core_test_support::responses::ev_web_search_call_done;
use core_test_support::responses::mount_sse_once_match;
use core_test_support::responses::sse;
use core_test_support::responses::start_mock_server;
use core_test_support::skip_if_no_network;
@@ -26,7 +32,7 @@ async fn user_message_item_is_emitted() -> anyhow::Result<()> {
let TestCodex { codex, .. } = test_codex().build(&server).await?;
let first_response = sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]);
responses::mount_sse_once_match(&server, any(), first_response).await;
mount_sse_once_match(&server, any(), first_response).await;
codex
.submit(Op::UserInput {
@@ -36,21 +42,23 @@ async fn user_message_item_is_emitted() -> anyhow::Result<()> {
})
.await?;
let started = wait_for_event_match(&codex, |ev| match ev {
EventMsg::ItemStarted(e) => Some(e.clone()),
let started_item = wait_for_event_match(&codex, |ev| match ev {
EventMsg::ItemStarted(ItemStartedEvent {
item: TurnItem::UserMessage(item),
..
}) => Some(item.clone()),
_ => None,
})
.await;
let completed = wait_for_event_match(&codex, |ev| match ev {
EventMsg::ItemCompleted(e) => Some(e.clone()),
let completed_item = wait_for_event_match(&codex, |ev| match ev {
EventMsg::ItemCompleted(ItemCompletedEvent {
item: TurnItem::UserMessage(item),
..
}) => Some(item.clone()),
_ => None,
})
.await;
let TurnItem::UserMessage(started_item) = started.item;
let TurnItem::UserMessage(completed_item) = completed.item;
assert_eq!(started_item.id, completed_item.id);
assert_eq!(
started_item.content,
@@ -66,3 +74,163 @@ async fn user_message_item_is_emitted() -> anyhow::Result<()> {
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn assistant_message_item_is_emitted() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
let TestCodex { codex, .. } = test_codex().build(&server).await?;
let first_response = sse(vec![
ev_response_created("resp-1"),
ev_assistant_message("msg-1", "all done"),
ev_completed("resp-1"),
]);
mount_sse_once_match(&server, any(), first_response).await;
codex
.submit(Op::UserInput {
items: vec![UserInput::Text {
text: "please summarize results".into(),
}],
})
.await?;
let started = wait_for_event_match(&codex, |ev| match ev {
EventMsg::ItemStarted(ItemStartedEvent {
item: TurnItem::AgentMessage(item),
..
}) => Some(item.clone()),
_ => None,
})
.await;
let completed = wait_for_event_match(&codex, |ev| match ev {
EventMsg::ItemCompleted(ItemCompletedEvent {
item: TurnItem::AgentMessage(item),
..
}) => Some(item.clone()),
_ => None,
})
.await;
assert_eq!(started.id, completed.id);
let Some(codex_protocol::items::AgentMessageContent::Text { text }) = completed.content.first()
else {
panic!("expected agent message text content");
};
assert_eq!(text, "all done");
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn reasoning_item_is_emitted() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
let TestCodex { codex, .. } = test_codex().build(&server).await?;
let reasoning_item = ev_reasoning_item(
"reasoning-1",
&["Consider inputs", "Compute output"],
&["Detailed reasoning trace"],
);
let first_response = sse(vec![
ev_response_created("resp-1"),
reasoning_item,
ev_completed("resp-1"),
]);
mount_sse_once_match(&server, any(), first_response).await;
codex
.submit(Op::UserInput {
items: vec![UserInput::Text {
text: "explain your reasoning".into(),
}],
})
.await?;
let started = wait_for_event_match(&codex, |ev| match ev {
EventMsg::ItemStarted(ItemStartedEvent {
item: TurnItem::Reasoning(item),
..
}) => Some(item.clone()),
_ => None,
})
.await;
let completed = wait_for_event_match(&codex, |ev| match ev {
EventMsg::ItemCompleted(ItemCompletedEvent {
item: TurnItem::Reasoning(item),
..
}) => Some(item.clone()),
_ => None,
})
.await;
assert_eq!(started.id, completed.id);
assert_eq!(
completed.summary_text,
vec!["Consider inputs".to_string(), "Compute output".to_string()]
);
assert_eq!(
completed.raw_content,
vec!["Detailed reasoning trace".to_string()]
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn web_search_item_is_emitted() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
let TestCodex { codex, .. } = test_codex().build(&server).await?;
let web_search_added =
ev_web_search_call_added("web-search-1", "in_progress", "weather seattle");
let web_search_done = ev_web_search_call_done("web-search-1", "completed", "weather seattle");
let first_response = sse(vec![
ev_response_created("resp-1"),
web_search_added,
web_search_done,
ev_completed("resp-1"),
]);
mount_sse_once_match(&server, any(), first_response).await;
codex
.submit(Op::UserInput {
items: vec![UserInput::Text {
text: "find the weather".into(),
}],
})
.await?;
let started = wait_for_event_match(&codex, |ev| match ev {
EventMsg::ItemStarted(ItemStartedEvent {
item: TurnItem::WebSearch(item),
..
}) => Some(item.clone()),
_ => None,
})
.await;
let completed = wait_for_event_match(&codex, |ev| match ev {
EventMsg::ItemCompleted(ItemCompletedEvent {
item: TurnItem::WebSearch(item),
..
}) => Some(item.clone()),
_ => None,
})
.await;
assert_eq!(started.id, completed.id);
assert_eq!(completed.query, "weather seattle");
Ok(())
}
codex-rs/core/tests/suite/mod.rs
+2
View File
@@ -3,6 +3,8 @@
#[cfg(not(target_os = "windows"))]
mod abort_tasks;
#[cfg(not(target_os = "windows"))]
mod apply_patch_cli;
#[cfg(not(target_os = "windows"))]
mod approvals;
mod cli_stream;
mod client;
codex-rs/core/tests/suite/resume.rs
+57
View File
@@ -4,6 +4,7 @@ use codex_core::protocol::Op;
use codex_protocol::user_input::UserInput;
use core_test_support::responses::ev_assistant_message;
use core_test_support::responses::ev_completed;
use core_test_support::responses::ev_reasoning_item;
use core_test_support::responses::ev_response_created;
use core_test_support::responses::mount_sse_once_match;
use core_test_support::responses::sse;
@@ -62,3 +63,59 @@ async fn resume_includes_initial_messages_from_rollout_events() -> Result<()> {
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn resume_includes_initial_messages_from_reasoning_events() -> Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
let mut builder = test_codex().with_config(|config| {
config.show_raw_agent_reasoning = true;
});
let initial = builder.build(&server).await?;
let codex = Arc::clone(&initial.codex);
let home = initial.home.clone();
let rollout_path = initial.session_configured.rollout_path.clone();
let initial_sse = sse(vec![
ev_response_created("resp-initial"),
ev_reasoning_item("reason-1", &["Summarized step"], &["raw detail"]),
ev_assistant_message("msg-1", "Completed reasoning turn"),
ev_completed("resp-initial"),
]);
mount_sse_once_match(&server, any(), initial_sse).await;
codex
.submit(Op::UserInput {
items: vec![UserInput::Text {
text: "Record reasoning messages".into(),
}],
})
.await?;
wait_for_event(&codex, |event| matches!(event, EventMsg::TaskComplete(_))).await;
let resumed = builder.resume(&server, home, rollout_path).await?;
let initial_messages = resumed
.session_configured
.initial_messages
.expect("expected initial messages to be present for resumed session");
match initial_messages.as_slice() {
[
EventMsg::UserMessage(first_user),
EventMsg::TokenCount(_),
EventMsg::AgentReasoning(reasoning),
EventMsg::AgentReasoningRawContent(raw),
EventMsg::AgentMessage(assistant_message),
EventMsg::TokenCount(_),
] => {
assert_eq!(first_user.message, "Record reasoning messages");
assert_eq!(reasoning.text, "Summarized step");
assert_eq!(raw.text, "raw detail");
assert_eq!(assistant_message.message, "Completed reasoning turn");
}
other => panic!("unexpected initial messages after resume: {other:#?}"),
}
Ok(())
}
codex-rs/core/tests/suite/rollout_list_find.rs
+34 -3
View File
@@ -1,5 +1,6 @@
#![allow(clippy::unwrap_used, clippy::expect_used)]
use std::io::Write;
use std::path::Path;
use std::path::PathBuf;
use codex_core::find_conversation_path_by_id_str;
@@ -8,8 +9,8 @@ use uuid::Uuid;
/// Create sessions/YYYY/MM/DD and write a minimal rollout file containing the
/// provided conversation id in the SessionMeta line. Returns the absolute path.
fn write_minimal_rollout_with_id(codex_home: &TempDir, id: Uuid) -> PathBuf {
let sessions = codex_home.path().join("sessions/2024/01/01");
fn write_minimal_rollout_with_id(codex_home: &Path, id: Uuid) -> PathBuf {
let sessions = codex_home.join("sessions/2024/01/01");
std::fs::create_dir_all(&sessions).unwrap();
let file = sessions.join(format!("rollout-2024-01-01T00-00-00-{id}.jsonl"));
@@ -40,7 +41,7 @@ fn write_minimal_rollout_with_id(codex_home: &TempDir, id: Uuid) -> PathBuf {
async fn find_locates_rollout_file_by_id() {
let home = TempDir::new().unwrap();
let id = Uuid::new_v4();
let expected = write_minimal_rollout_with_id(&home, id);
let expected = write_minimal_rollout_with_id(home.path(), id);
let found = find_conversation_path_by_id_str(home.path(), &id.to_string())
.await
@@ -48,3 +49,33 @@ async fn find_locates_rollout_file_by_id() {
assert_eq!(found.unwrap(), expected);
}
#[tokio::test]
async fn find_handles_gitignore_covering_codex_home_directory() {
let repo = TempDir::new().unwrap();
let codex_home = repo.path().join(".codex");
std::fs::create_dir_all(&codex_home).unwrap();
std::fs::write(repo.path().join(".gitignore"), ".codex/**\n").unwrap();
let id = Uuid::new_v4();
let expected = write_minimal_rollout_with_id(&codex_home, id);
let found = find_conversation_path_by_id_str(&codex_home, &id.to_string())
.await
.unwrap();
assert_eq!(found, Some(expected));
}
#[tokio::test]
async fn find_ignores_granular_gitignore_rules() {
let home = TempDir::new().unwrap();
let id = Uuid::new_v4();
let expected = write_minimal_rollout_with_id(home.path(), id);
std::fs::write(home.path().join("sessions/.gitignore"), "*.jsonl\n").unwrap();
let found = find_conversation_path_by_id_str(home.path(), &id.to_string())
.await
.unwrap();
assert_eq!(found, Some(expected));
}
codex-rs/core/tests/suite/tools.rs
-56
View File
@@ -227,62 +227,6 @@ async fn shell_escalated_permissions_rejected_then_ok() -> Result<()> {
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn local_shell_missing_ids_maps_to_function_output_error() -> Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
let mut builder = test_codex();
let test = builder.build(&server).await?;
let local_shell_event = json!({
"type": "response.output_item.done",
"item": {
"type": "local_shell_call",
"status": "completed",
"action": {
"type": "exec",
"command": ["/bin/echo", "hi"],
}
}
});
mount_sse_once(
&server,
sse(vec![
ev_response_created("resp-1"),
local_shell_event,
ev_completed("resp-1"),
]),
)
.await;
let second_mock = mount_sse_once(
&server,
sse(vec![
ev_assistant_message("msg-1", "done"),
ev_completed("resp-2"),
]),
)
.await;
submit_turn(
&test,
"check shell output",
AskForApproval::Never,
SandboxPolicy::DangerFullAccess,
)
.await?;
let item = second_mock.single_request().function_call_output("");
assert_eq!(item.get("call_id").and_then(Value::as_str), Some(""));
assert_eq!(
item.get("output").and_then(Value::as_str),
Some("LocalShellCall without call_id or id"),
);
Ok(())
}
async fn collect_tools(use_unified_exec: bool) -> Result<Vec<String>> {
let server = start_mock_server().await;
codex-rs/core/tests/suite/unified_exec.rs
+345
View File
@@ -133,6 +133,247 @@ async fn unified_exec_emits_exec_command_begin_event() -> Result<()> {
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn unified_exec_emits_exec_command_end_event() -> Result<()> {
skip_if_no_network!(Ok(()));
skip_if_sandbox!(Ok(()));
let server = start_mock_server().await;
let mut builder = test_codex().with_config(|config| {
config.use_experimental_unified_exec_tool = true;
config.features.enable(Feature::UnifiedExec);
});
let TestCodex {
codex,
cwd,
session_configured,
..
} = builder.build(&server).await?;
let call_id = "uexec-end-event";
let args = json!({
"cmd": "/bin/echo END-EVENT".to_string(),
"yield_time_ms": 250,
});
let responses = vec![
sse(vec![
ev_response_created("resp-1"),
ev_function_call(call_id, "exec_command", &serde_json::to_string(&args)?),
ev_completed("resp-1"),
]),
sse(vec![
ev_response_created("resp-2"),
ev_assistant_message("msg-1", "finished"),
ev_completed("resp-2"),
]),
];
mount_sse_sequence(&server, responses).await;
let session_model = session_configured.model.clone();
codex
.submit(Op::UserTurn {
items: vec![UserInput::Text {
text: "emit end event".into(),
}],
final_output_json_schema: None,
cwd: cwd.path().to_path_buf(),
approval_policy: AskForApproval::Never,
sandbox_policy: SandboxPolicy::DangerFullAccess,
model: session_model,
effort: None,
summary: ReasoningSummary::Auto,
})
.await?;
let end_event = wait_for_event_match(&codex, |msg| match msg {
EventMsg::ExecCommandEnd(ev) if ev.call_id == call_id => Some(ev.clone()),
_ => None,
})
.await;
assert_eq!(end_event.exit_code, 0);
assert!(
end_event.aggregated_output.contains("END-EVENT"),
"expected aggregated output to contain marker"
);
wait_for_event(&codex, |event| matches!(event, EventMsg::TaskComplete(_))).await;
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn unified_exec_emits_output_delta_for_exec_command() -> Result<()> {
skip_if_no_network!(Ok(()));
skip_if_sandbox!(Ok(()));
let server = start_mock_server().await;
let mut builder = test_codex().with_config(|config| {
config.use_experimental_unified_exec_tool = true;
config.features.enable(Feature::UnifiedExec);
});
let TestCodex {
codex,
cwd,
session_configured,
..
} = builder.build(&server).await?;
let call_id = "uexec-delta-1";
let args = json!({
"cmd": "printf 'HELLO-UEXEC'",
"yield_time_ms": 250,
});
let responses = vec![
sse(vec![
ev_response_created("resp-1"),
ev_function_call(call_id, "exec_command", &serde_json::to_string(&args)?),
ev_completed("resp-1"),
]),
sse(vec![
ev_response_created("resp-2"),
ev_assistant_message("msg-1", "finished"),
ev_completed("resp-2"),
]),
];
mount_sse_sequence(&server, responses).await;
let session_model = session_configured.model.clone();
codex
.submit(Op::UserTurn {
items: vec![UserInput::Text {
text: "emit delta".into(),
}],
final_output_json_schema: None,
cwd: cwd.path().to_path_buf(),
approval_policy: AskForApproval::Never,
sandbox_policy: SandboxPolicy::DangerFullAccess,
model: session_model,
effort: None,
summary: ReasoningSummary::Auto,
})
.await?;
let delta = wait_for_event_match(&codex, |msg| match msg {
EventMsg::ExecCommandOutputDelta(ev) if ev.call_id == call_id => Some(ev.clone()),
_ => None,
})
.await;
let text = String::from_utf8_lossy(&delta.chunk).to_string();
assert!(
text.contains("HELLO-UEXEC"),
"delta chunk missing expected text: {text:?}"
);
wait_for_event(&codex, |event| matches!(event, EventMsg::TaskComplete(_))).await;
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn unified_exec_emits_output_delta_for_write_stdin() -> Result<()> {
skip_if_no_network!(Ok(()));
skip_if_sandbox!(Ok(()));
let server = start_mock_server().await;
let mut builder = test_codex().with_config(|config| {
config.use_experimental_unified_exec_tool = true;
config.features.enable(Feature::UnifiedExec);
});
let TestCodex {
codex,
cwd,
session_configured,
..
} = builder.build(&server).await?;
let open_call_id = "uexec-open";
let open_args = json!({
"cmd": "/bin/bash -i",
"yield_time_ms": 200,
});
let stdin_call_id = "uexec-stdin-delta";
let stdin_args = json!({
"chars": "echo WSTDIN-MARK\\n",
"session_id": 0,
"yield_time_ms": 800,
});
let responses = vec![
sse(vec![
ev_response_created("resp-1"),
ev_function_call(
open_call_id,
"exec_command",
&serde_json::to_string(&open_args)?,
),
ev_completed("resp-1"),
]),
sse(vec![
ev_response_created("resp-2"),
ev_function_call(
stdin_call_id,
"write_stdin",
&serde_json::to_string(&stdin_args)?,
),
ev_completed("resp-2"),
]),
sse(vec![
ev_response_created("resp-3"),
ev_assistant_message("msg-1", "done"),
ev_completed("resp-3"),
]),
];
mount_sse_sequence(&server, responses).await;
let session_model = session_configured.model.clone();
codex
.submit(Op::UserTurn {
items: vec![UserInput::Text {
text: "stdin delta".into(),
}],
final_output_json_schema: None,
cwd: cwd.path().to_path_buf(),
approval_policy: AskForApproval::Never,
sandbox_policy: SandboxPolicy::DangerFullAccess,
model: session_model,
effort: None,
summary: ReasoningSummary::Auto,
})
.await?;
// Expect a delta event corresponding to the write_stdin call.
let delta = wait_for_event_match(&codex, |msg| match msg {
EventMsg::ExecCommandOutputDelta(ev) if ev.call_id == open_call_id => {
let text = String::from_utf8_lossy(&ev.chunk);
if text.contains("WSTDIN-MARK") {
Some(ev.clone())
} else {
None
}
}
_ => None,
})
.await;
let text = String::from_utf8_lossy(&delta.chunk).to_string();
assert!(
text.contains("WSTDIN-MARK"),
"stdin delta chunk missing expected text: {text:?}"
);
wait_for_event(&codex, |event| matches!(event, EventMsg::TaskComplete(_))).await;
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn unified_exec_skips_begin_event_for_empty_input() -> Result<()> {
use tokio::time::Duration;
@@ -516,6 +757,110 @@ async fn write_stdin_returns_exit_metadata_and_clears_session() -> Result<()> {
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn unified_exec_emits_end_event_when_session_dies_via_stdin() -> Result<()> {
skip_if_no_network!(Ok(()));
skip_if_sandbox!(Ok(()));
let server = start_mock_server().await;
let mut builder = test_codex().with_config(|config| {
config.use_experimental_unified_exec_tool = true;
config.features.enable(Feature::UnifiedExec);
});
let TestCodex {
codex,
cwd,
session_configured,
..
} = builder.build(&server).await?;
let start_call_id = "uexec-end-on-exit-start";
let start_args = serde_json::json!({
"cmd": "/bin/cat",
"yield_time_ms": 200,
});
let echo_call_id = "uexec-end-on-exit-echo";
let echo_args = serde_json::json!({
"chars": "bye-END\n",
"session_id": 0,
"yield_time_ms": 300,
});
let exit_call_id = "uexec-end-on-exit";
let exit_args = serde_json::json!({
"chars": "\u{0004}",
"session_id": 0,
"yield_time_ms": 500,
});
let responses = vec![
sse(vec![
ev_response_created("resp-1"),
ev_function_call(
start_call_id,
"exec_command",
&serde_json::to_string(&start_args)?,
),
ev_completed("resp-1"),
]),
sse(vec![
ev_response_created("resp-2"),
ev_function_call(
echo_call_id,
"write_stdin",
&serde_json::to_string(&echo_args)?,
),
ev_completed("resp-2"),
]),
sse(vec![
ev_response_created("resp-3"),
ev_function_call(
exit_call_id,
"write_stdin",
&serde_json::to_string(&exit_args)?,
),
ev_completed("resp-3"),
]),
sse(vec![
ev_response_created("resp-4"),
ev_assistant_message("msg-1", "done"),
ev_completed("resp-4"),
]),
];
mount_sse_sequence(&server, responses).await;
let session_model = session_configured.model.clone();
codex
.submit(Op::UserTurn {
items: vec![UserInput::Text {
text: "end on exit".into(),
}],
final_output_json_schema: None,
cwd: cwd.path().to_path_buf(),
approval_policy: AskForApproval::Never,
sandbox_policy: SandboxPolicy::DangerFullAccess,
model: session_model,
effort: None,
summary: ReasoningSummary::Auto,
})
.await?;
// We expect the ExecCommandEnd event to match the initial exec_command call_id.
let end_event = wait_for_event_match(&codex, |msg| match msg {
EventMsg::ExecCommandEnd(ev) if ev.call_id == start_call_id => Some(ev.clone()),
_ => None,
})
.await;
assert_eq!(end_event.exit_code, 0);
wait_for_event(&codex, |event| matches!(event, EventMsg::TaskComplete(_))).await;
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn unified_exec_reuses_session_via_stdin() -> Result<()> {
skip_if_no_network!(Ok(()));
codex-rs/file-search/src/lib.rs
+11
View File
@@ -105,6 +105,7 @@ pub async fn run_main<T: Reporter>(
threads,
cancel_flag,
compute_indices,
true,
)?;
let match_count = matches.len();
let matches_truncated = total_match_count > match_count;
@@ -121,6 +122,7 @@ pub async fn run_main<T: Reporter>(
/// The worker threads will periodically check `cancel_flag` to see if they
/// should stop processing files.
#[allow(clippy::too_many_arguments)]
pub fn run(
pattern_text: &str,
limit: NonZero<usize>,
@@ -129,6 +131,7 @@ pub fn run(
threads: NonZero<usize>,
cancel_flag: Arc<AtomicBool>,
compute_indices: bool,
respect_gitignore: bool,
) -> anyhow::Result<FileSearchResults> {
let pattern = create_pattern(pattern_text);
// Create one BestMatchesList per worker thread so that each worker can
@@ -157,6 +160,14 @@ pub fn run(
.hidden(false)
// Don't require git to be present to apply to apply git-related ignore rules.
.require_git(false);
if !respect_gitignore {
walk_builder
.git_ignore(false)
.git_global(false)
.git_exclude(false)
.ignore(false)
.parents(false);
}
if !exclude.is_empty() {
let mut override_builder = OverrideBuilder::new(search_directory);
codex-rs/mcp-client/Cargo.toml
-23
View File
@@ -1,23 +0,0 @@
[package]
name = "codex-mcp-client"
version = { workspace = true }
edition = "2024"
[lints]
workspace = true
[dependencies]
anyhow = { workspace = true }
mcp-types = { workspace = true }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
tracing = { workspace = true, features = ["log"] }
tracing-subscriber = { workspace = true, features = ["fmt", "env-filter"] }
tokio = { workspace = true, features = [
"io-util",
"macros",
"process",
"rt-multi-thread",
"sync",
"time",
] }
codex-rs/mcp-client/src/lib.rs
-3
View File
@@ -1,3 +0,0 @@
mod mcp_client;
pub use mcp_client::McpClient;
codex-rs/mcp-client/src/main.rs
-88
View File
@@ -1,88 +0,0 @@
//! Simple command-line utility to exercise `McpClient`.
//!
//! Example usage:
//!
//! ```bash
//! cargo run -p codex-mcp-client -- `codex-mcp-server`
//! ```
//!
//! Any additional arguments after the first one are forwarded to the spawned
//! program. The utility connects, issues a `tools/list` request and prints the
//! server's response as pretty JSON.
use std::ffi::OsString;
use std::time::Duration;
use anyhow::Context;
use anyhow::Result;
use codex_mcp_client::McpClient;
use mcp_types::ClientCapabilities;
use mcp_types::Implementation;
use mcp_types::InitializeRequestParams;
use mcp_types::ListToolsRequestParams;
use mcp_types::MCP_SCHEMA_VERSION;
use tracing_subscriber::EnvFilter;
#[tokio::main]
async fn main() -> Result<()> {
let default_level = "debug";
let _ = tracing_subscriber::fmt()
// Fallback to the `default_level` log filter if the environment
// variable is not set _or_ contains an invalid value
.with_env_filter(
EnvFilter::try_from_default_env()
.or_else(|_| EnvFilter::try_new(default_level))
.unwrap_or_else(|_| EnvFilter::new(default_level)),
)
.with_writer(std::io::stderr)
.try_init();
// Collect command-line arguments excluding the program name itself.
let mut args: Vec<OsString> = std::env::args_os().skip(1).collect();
if args.is_empty() || args[0] == "--help" || args[0] == "-h" {
eprintln!("Usage: mcp-client <program> [args..]\n\nExample: mcp-client codex-mcp-server");
std::process::exit(1);
}
let original_args = args.clone();
// Spawn the subprocess and connect the client.
let program = args.remove(0);
let env = None;
let client = McpClient::new_stdio_client(program, args, env, &[], None)
.await
.with_context(|| format!("failed to spawn subprocess: {original_args:?}"))?;
let params = InitializeRequestParams {
capabilities: ClientCapabilities {
experimental: None,
roots: None,
sampling: None,
elicitation: None,
},
client_info: Implementation {
name: "codex-mcp-client".to_owned(),
version: env!("CARGO_PKG_VERSION").to_owned(),
title: Some("Codex".to_string()),
// This field is used by Codex when it is an MCP server: it should
// not be used when Codex is an MCP client.
user_agent: None,
},
protocol_version: MCP_SCHEMA_VERSION.to_owned(),
};
let timeout = Some(Duration::from_secs(10));
let response = client.initialize(params, timeout).await?;
eprintln!("initialize response: {response:?}");
// Issue `tools/list` request (no params).
let timeout = None;
let tools = client
.list_tools(None::<ListToolsRequestParams>, timeout)
.await
.context("tools/list request failed")?;
// Print the result in a human readable form.
println!("{}", serde_json::to_string_pretty(&tools)?);
Ok(())
}
codex-rs/mcp-client/src/mcp_client.rs
-509
View File
@@ -1,509 +0,0 @@
//! A minimal async client for the Model Context Protocol (MCP).
//!
//! The client is intentionally lightweight – it is only capable of:
//! 1. Spawning a subprocess that launches a conforming MCP server that
//! communicates over stdio.
//! 2. Sending MCP requests and pairing them with their corresponding
//! responses.
//! 3. Offering a convenience helper for the common `tools/list` request.
//!
//! The crate hides all JSON‐RPC framing details behind a typed API. Users
//! interact with the [`ModelContextProtocolRequest`] trait from `mcp-types` to
//! issue requests and receive strongly-typed results.
use std::collections::HashMap;
use std::ffi::OsString;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::AtomicI64;
use std::sync::atomic::Ordering;
use std::time::Duration;
use anyhow::Context;
use anyhow::Result;
use anyhow::anyhow;
use mcp_types::CallToolRequest;
use mcp_types::CallToolRequestParams;
use mcp_types::InitializeRequest;
use mcp_types::InitializeRequestParams;
use mcp_types::InitializedNotification;
use mcp_types::JSONRPC_VERSION;
use mcp_types::JSONRPCMessage;
use mcp_types::JSONRPCNotification;
use mcp_types::JSONRPCRequest;
use mcp_types::JSONRPCResponse;
use mcp_types::ListToolsRequest;
use mcp_types::ListToolsRequestParams;
use mcp_types::ListToolsResult;
use mcp_types::ModelContextProtocolNotification;
use mcp_types::ModelContextProtocolRequest;
use mcp_types::RequestId;
use serde::Serialize;
use serde::de::DeserializeOwned;
use tokio::io::AsyncBufReadExt;
use tokio::io::AsyncWriteExt;
use tokio::io::BufReader;
use tokio::process::Command;
use tokio::sync::Mutex;
use tokio::sync::mpsc;
use tokio::sync::oneshot;
use tokio::time;
use tracing::debug;
use tracing::error;
use tracing::info;
use tracing::warn;
/// Capacity of the bounded channels used for transporting messages between the
/// client API and the IO tasks.
const CHANNEL_CAPACITY: usize = 128;
/// Internal representation of a pending request sender.
type PendingSender = oneshot::Sender<JSONRPCMessage>;
/// A running MCP client instance.
pub struct McpClient {
/// Retain this child process until the client is dropped. The Tokio runtime
/// will make a "best effort" to reap the process after it exits, but it is
/// not a guarantee. See the `kill_on_drop` documentation for details.
#[allow(dead_code)]
child: tokio::process::Child,
/// Channel for sending JSON-RPC messages *to* the background writer task.
outgoing_tx: mpsc::Sender<JSONRPCMessage>,
/// Map of `request.id -> oneshot::Sender` used to dispatch responses back
/// to the originating caller.
pending: Arc<Mutex<HashMap<i64, PendingSender>>>,
/// Monotonically increasing counter used to generate request IDs.
id_counter: AtomicI64,
}
impl McpClient {
/// Spawn the given command and establish an MCP session over its STDIO.
/// Caller is responsible for sending the `initialize` request. See
/// [`initialize`](Self::initialize) for details.
pub async fn new_stdio_client(
program: OsString,
args: Vec<OsString>,
env: Option<HashMap<String, String>>,
env_vars: &[String],
cwd: Option<PathBuf>,
) -> std::io::Result<Self> {
let mut command = Command::new(program);
command
.args(args)
.env_clear()
.envs(create_env_for_mcp_server(env, env_vars))
.stdin(std::process::Stdio::piped())
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::null())
// As noted in the `kill_on_drop` documentation, the Tokio runtime makes
// a "best effort" to reap-after-exit to avoid zombie processes, but it
// is not a guarantee.
.kill_on_drop(true);
if let Some(cwd) = cwd {
command.current_dir(cwd);
}
let mut child = command.spawn()?;
let stdin = child
.stdin
.take()
.ok_or_else(|| std::io::Error::other("failed to capture child stdin"))?;
let stdout = child
.stdout
.take()
.ok_or_else(|| std::io::Error::other("failed to capture child stdout"))?;
let (outgoing_tx, mut outgoing_rx) = mpsc::channel::<JSONRPCMessage>(CHANNEL_CAPACITY);
let pending: Arc<Mutex<HashMap<i64, PendingSender>>> = Arc::new(Mutex::new(HashMap::new()));
// Spawn writer task. It listens on the `outgoing_rx` channel and
// writes messages to the child's STDIN.
let writer_handle = {
let mut stdin = stdin;
tokio::spawn(async move {
while let Some(msg) = outgoing_rx.recv().await {
match serde_json::to_string(&msg) {
Ok(json) => {
debug!("MCP message to server: {json}");
if stdin.write_all(json.as_bytes()).await.is_err() {
error!("failed to write message to child stdin");
break;
}
if stdin.write_all(b"\n").await.is_err() {
error!("failed to write newline to child stdin");
break;
}
// No explicit flush needed on a pipe; write_all is sufficient.
}
Err(e) => error!("failed to serialize JSONRPCMessage: {e}"),
}
}
})
};
// Spawn reader task. It reads line-delimited JSON from the child's
// STDOUT and dispatches responses to the pending map.
let reader_handle = {
let pending = pending.clone();
let mut lines = BufReader::new(stdout).lines();
tokio::spawn(async move {
while let Ok(Some(line)) = lines.next_line().await {
debug!("MCP message from server: {line}");
match serde_json::from_str::<JSONRPCMessage>(&line) {
Ok(JSONRPCMessage::Response(resp)) => {
Self::dispatch_response(resp, &pending).await;
}
Ok(JSONRPCMessage::Error(err)) => {
Self::dispatch_error(err, &pending).await;
}
Ok(JSONRPCMessage::Notification(JSONRPCNotification { .. })) => {
// For now we only log server-initiated notifications.
info!("<- notification: {}", line);
}
Ok(other) => {
// Batch responses and requests are currently not
// expected from the server – log and ignore.
info!("<- unhandled message: {:?}", other);
}
Err(e) => {
error!("failed to deserialize JSONRPCMessage: {e}; line = {}", line)
}
}
}
})
};
// We intentionally *detach* the tasks. They will keep running in the
// background as long as their respective resources (channels/stdin/
// stdout) are alive. Dropping `McpClient` cancels the tasks due to
// dropped resources.
let _ = (writer_handle, reader_handle);
Ok(Self {
child,
outgoing_tx,
pending,
id_counter: AtomicI64::new(1),
})
}
/// Send an arbitrary MCP request and await the typed result.
///
/// If `timeout` is `None` the call waits indefinitely. If `Some(duration)`
/// is supplied and no response is received within the given period, a
/// timeout error is returned.
pub async fn send_request<R>(
&self,
params: R::Params,
timeout: Option<Duration>,
) -> Result<R::Result>
where
R: ModelContextProtocolRequest,
R::Params: Serialize,
R::Result: DeserializeOwned,
{
// Create a new unique ID.
let id = self.id_counter.fetch_add(1, Ordering::SeqCst);
let request_id = RequestId::Integer(id);
// Serialize params -> JSON. For many request types `Params` is
// `Option<T>` and `None` should be encoded as *absence* of the field.
let params_json = serde_json::to_value(&params)?;
let params_field = if params_json.is_null() {
None
} else {
Some(params_json)
};
let jsonrpc_request = JSONRPCRequest {
id: request_id.clone(),
jsonrpc: JSONRPC_VERSION.to_string(),
method: R::METHOD.to_string(),
params: params_field,
};
let message = JSONRPCMessage::Request(jsonrpc_request);
// oneshot channel for the response.
let (tx, rx) = oneshot::channel();
// Register in pending map *before* sending the message so a race where
// the response arrives immediately cannot be lost.
{
let mut guard = self.pending.lock().await;
guard.insert(id, tx);
}
// Send to writer task.
if self.outgoing_tx.send(message).await.is_err() {
return Err(anyhow!(
"failed to send message to writer task - channel closed"
));
}
// Await the response, optionally bounded by a timeout.
let msg = match timeout {
Some(duration) => {
match time::timeout(duration, rx).await {
Ok(Ok(msg)) => msg,
Ok(Err(_)) => {
// Channel closed without a reply – remove the pending entry.
let mut guard = self.pending.lock().await;
guard.remove(&id);
return Err(anyhow!(
"response channel closed before a reply was received"
));
}
Err(_) => {
// Timed out. Remove the pending entry so we don't leak.
let mut guard = self.pending.lock().await;
guard.remove(&id);
return Err(anyhow!("request timed out"));
}
}
}
None => rx
.await
.map_err(|_| anyhow!("response channel closed before a reply was received"))?,
};
match msg {
JSONRPCMessage::Response(JSONRPCResponse { result, .. }) => {
let typed: R::Result = serde_json::from_value(result)?;
Ok(typed)
}
JSONRPCMessage::Error(err) => Err(anyhow!(format!(
"server returned JSON-RPC error: code = {}, message = {}",
err.error.code, err.error.message
))),
other => Err(anyhow!(format!(
"unexpected message variant received in reply path: {other:?}"
))),
}
}
pub async fn send_notification<N>(&self, params: N::Params) -> Result<()>
where
N: ModelContextProtocolNotification,
N::Params: Serialize,
{
// Serialize params -> JSON. For many request types `Params` is
// `Option<T>` and `None` should be encoded as *absence* of the field.
let params_json = serde_json::to_value(&params)?;
let params_field = if params_json.is_null() {
None
} else {
Some(params_json)
};
let method = N::METHOD.to_string();
let jsonrpc_notification = JSONRPCNotification {
jsonrpc: JSONRPC_VERSION.to_string(),
method: method.clone(),
params: params_field,
};
let notification = JSONRPCMessage::Notification(jsonrpc_notification);
self.outgoing_tx
.send(notification)
.await
.with_context(|| format!("failed to send notification `{method}` to writer task"))
}
/// Negotiates the initialization with the MCP server. Sends an `initialize`
/// request with the specified `initialize_params` and then the
/// `notifications/initialized` notification once the response has been
/// received. Returns the response to the `initialize` request.
pub async fn initialize(
&self,
initialize_params: InitializeRequestParams,
timeout: Option<Duration>,
) -> Result<mcp_types::InitializeResult> {
let response = self
.send_request::<InitializeRequest>(initialize_params, timeout)
.await?;
self.send_notification::<InitializedNotification>(None)
.await?;
Ok(response)
}
/// Convenience wrapper around `tools/list`.
pub async fn list_tools(
&self,
params: Option<ListToolsRequestParams>,
timeout: Option<Duration>,
) -> Result<ListToolsResult> {
self.send_request::<ListToolsRequest>(params, timeout).await
}
/// Convenience wrapper around `tools/call`.
pub async fn call_tool(
&self,
name: String,
arguments: Option<serde_json::Value>,
timeout: Option<Duration>,
) -> Result<mcp_types::CallToolResult> {
let params = CallToolRequestParams { name, arguments };
debug!("MCP tool call: {params:?}");
self.send_request::<CallToolRequest>(params, timeout).await
}
/// Internal helper: route a JSON-RPC *response* object to the pending map.
async fn dispatch_response(
resp: JSONRPCResponse,
pending: &Arc<Mutex<HashMap<i64, PendingSender>>>,
) {
let id = match resp.id {
RequestId::Integer(i) => i,
RequestId::String(_) => {
// We only ever generate integer IDs. Receiving a string here
// means we will not find a matching entry in `pending`.
error!("response with string ID - no matching pending request");
return;
}
};
let tx_opt = {
let mut guard = pending.lock().await;
guard.remove(&id)
};
if let Some(tx) = tx_opt {
// Ignore send errors – the receiver might have been dropped.
let _ = tx.send(JSONRPCMessage::Response(resp));
} else {
warn!(id, "no pending request found for response");
}
}
/// Internal helper: route a JSON-RPC *error* object to the pending map.
async fn dispatch_error(
err: mcp_types::JSONRPCError,
pending: &Arc<Mutex<HashMap<i64, PendingSender>>>,
) {
let id = match err.id {
RequestId::Integer(i) => i,
RequestId::String(_) => return, // see comment above
};
let tx_opt = {
let mut guard = pending.lock().await;
guard.remove(&id)
};
if let Some(tx) = tx_opt {
let _ = tx.send(JSONRPCMessage::Error(err));
}
}
}
impl Drop for McpClient {
fn drop(&mut self) {
// Even though we have already tagged this process with
// `kill_on_drop(true)` above, this extra check has the benefit of
// forcing the process to be reaped immediately if it has already exited
// instead of waiting for the Tokio runtime to reap it later.
let _ = self.child.try_wait();
}
}
/// Environment variables that are always included when spawning a new MCP
/// server.
#[rustfmt::skip]
#[cfg(unix)]
const DEFAULT_ENV_VARS: &[&str] = &[
// https://modelcontextprotocol.io/docs/tools/debugging#environment-variables
// states:
//
// > MCP servers inherit only a subset of environment variables automatically,
// > like `USER`, `HOME`, and `PATH`.
//
// But it does not fully enumerate the list. Empirically, when spawning a
// an MCP server via Claude Desktop on macOS, it reports the following
// environment variables:
"HOME",
"LOGNAME",
"PATH",
"SHELL",
"USER",
"__CF_USER_TEXT_ENCODING",
// Additional environment variables Codex chooses to include by default:
"LANG",
"LC_ALL",
"TERM",
"TMPDIR",
"TZ",
];
#[cfg(windows)]
const DEFAULT_ENV_VARS: &[&str] = &[
// TODO: More research is necessary to curate this list.
"PATH",
"PATHEXT",
"USERNAME",
"USERDOMAIN",
"USERPROFILE",
"TEMP",
"TMP",
];
/// `extra_env` comes from the config for an entry in `mcp_servers` in
/// `config.toml`.
fn create_env_for_mcp_server(
extra_env: Option<HashMap<String, String>>,
env_vars: &[String],
) -> HashMap<String, String> {
DEFAULT_ENV_VARS
.iter()
.copied()
.chain(env_vars.iter().map(String::as_str))
.filter_map(|var| {
std::env::var(var)
.ok()
.map(|value| (var.to_string(), value))
})
.chain(extra_env.unwrap_or_default())
.collect::<HashMap<_, _>>()
}
#[cfg(test)]
mod tests {
use super::*;
fn set_env_var(key: &str, value: &str) {
unsafe {
std::env::set_var(key, value);
}
}
fn remove_env_var(key: &str) {
unsafe {
std::env::remove_var(key);
}
}
#[test]
fn test_create_env_for_mcp_server() {
let env_var = "USER";
let env_var_existing_value = std::env::var(env_var).unwrap_or_default();
let env_var_new_value = format!("{env_var_existing_value}-extra");
let extra_env = HashMap::from([(env_var.to_owned(), env_var_new_value.clone())]);
let mcp_server_env = create_env_for_mcp_server(Some(extra_env), &[]);
assert!(mcp_server_env.contains_key("PATH"));
assert_eq!(Some(&env_var_new_value), mcp_server_env.get(env_var));
}
#[test]
fn test_create_env_for_mcp_server_includes_extra_whitelisted_vars() {
let custom_var = "CUSTOM_TEST_VAR";
let value = "value".to_string();
set_env_var(custom_var, &value);
let mcp_server_env = create_env_for_mcp_server(None, &[custom_var.to_string()]);
assert_eq!(Some(&value), mcp_server_env.get(custom_var));
remove_env_var(custom_var);
}
}
codex-rs/protocol/src/account.rs
+35
View File
@@ -0,0 +1,35 @@
use schemars::JsonSchema;
use serde::Deserialize;
use serde::Serialize;
use ts_rs::TS;
#[derive(Serialize, Deserialize, Copy, Clone, Debug, PartialEq, Eq, JsonSchema, TS, Default)]
#[serde(rename_all = "lowercase")]
#[ts(rename_all = "lowercase")]
pub enum PlanType {
#[default]
Free,
Plus,
Pro,
Team,
Business,
Enterprise,
Edu,
#[serde(other)]
Unknown,
}
#[derive(Debug, Clone, PartialEq, Deserialize, Serialize, JsonSchema, TS)]
#[serde(tag = "type")]
#[ts(tag = "type")]
pub enum Account {
ApiKey {
api_key: String,
},
#[serde(rename = "chatgpt")]
#[ts(rename = "chatgpt")]
ChatGpt {
email: Option<String>,
plan_type: PlanType,
},
}
codex-rs/protocol/src/items.rs
+126
View File
@@ -1,3 +1,9 @@
use crate::protocol::AgentMessageEvent;
use crate::protocol::AgentReasoningEvent;
use crate::protocol::AgentReasoningRawContentEvent;
use crate::protocol::EventMsg;
use crate::protocol::UserMessageEvent;
use crate::protocol::WebSearchEndEvent;
use crate::user_input::UserInput;
use schemars::JsonSchema;
use serde::Deserialize;
@@ -7,6 +13,9 @@ use ts_rs::TS;
#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema)]
pub enum TurnItem {
UserMessage(UserMessageItem),
AgentMessage(AgentMessageItem),
Reasoning(ReasoningItem),
WebSearch(WebSearchItem),
}
#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema)]
@@ -15,6 +24,31 @@ pub struct UserMessageItem {
pub content: Vec<UserInput>,
}
#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema)]
pub enum AgentMessageContent {
Text { text: String },
}
#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema)]
pub struct AgentMessageItem {
pub id: String,
pub content: Vec<AgentMessageContent>,
}
#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema)]
pub struct ReasoningItem {
pub id: String,
pub summary_text: Vec<String>,
#[serde(default)]
pub raw_content: Vec<String>,
}
#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema)]
pub struct WebSearchItem {
pub id: String,
pub query: String,
}
impl UserMessageItem {
pub fn new(content: &[UserInput]) -> Self {
Self {
@@ -22,12 +56,104 @@ impl UserMessageItem {
content: content.to_vec(),
}
}
pub fn as_legacy_event(&self) -> EventMsg {
EventMsg::UserMessage(UserMessageEvent {
message: self.message(),
images: Some(self.image_urls()),
})
}
pub fn message(&self) -> String {
self.content
.iter()
.map(|c| match c {
UserInput::Text { text } => text.clone(),
_ => String::new(),
})
.collect::<Vec<String>>()
.join("")
}
pub fn image_urls(&self) -> Vec<String> {
self.content
.iter()
.filter_map(|c| match c {
UserInput::Image { image_url } => Some(image_url.clone()),
_ => None,
})
.collect()
}
}
impl AgentMessageItem {
pub fn new(content: &[AgentMessageContent]) -> Self {
Self {
id: uuid::Uuid::new_v4().to_string(),
content: content.to_vec(),
}
}
pub fn as_legacy_events(&self) -> Vec<EventMsg> {
self.content
.iter()
.map(|c| match c {
AgentMessageContent::Text { text } => EventMsg::AgentMessage(AgentMessageEvent {
message: text.clone(),
}),
})
.collect()
}
}
impl ReasoningItem {
pub fn as_legacy_events(&self, show_raw_agent_reasoning: bool) -> Vec<EventMsg> {
let mut events = Vec::new();
for summary in &self.summary_text {
events.push(EventMsg::AgentReasoning(AgentReasoningEvent {
text: summary.clone(),
}));
}
if show_raw_agent_reasoning {
for entry in &self.raw_content {
events.push(EventMsg::AgentReasoningRawContent(
AgentReasoningRawContentEvent {
text: entry.clone(),
},
));
}
}
events
}
}
impl WebSearchItem {
pub fn as_legacy_event(&self) -> EventMsg {
EventMsg::WebSearchEnd(WebSearchEndEvent {
call_id: self.id.clone(),
query: self.query.clone(),
})
}
}
impl TurnItem {
pub fn id(&self) -> String {
match self {
TurnItem::UserMessage(item) => item.id.clone(),
TurnItem::AgentMessage(item) => item.id.clone(),
TurnItem::Reasoning(item) => item.id.clone(),
TurnItem::WebSearch(item) => item.id.clone(),
}
}
pub fn as_legacy_events(&self, show_raw_agent_reasoning: bool) -> Vec<EventMsg> {
match self {
TurnItem::UserMessage(item) => vec![item.as_legacy_event()],
TurnItem::AgentMessage(item) => item.as_legacy_events(),
TurnItem::WebSearch(item) => vec![item.as_legacy_event()],
TurnItem::Reasoning(item) => item.as_legacy_events(show_raw_agent_reasoning),
}
}
}
codex-rs/protocol/src/lib.rs
+1
View File
@@ -1,3 +1,4 @@
pub mod account;
mod conversation_id;
pub use conversation_id::ConversationId;
pub mod config_types;
codex-rs/protocol/src/protocol.rs
-56
View File
@@ -770,69 +770,13 @@ pub struct AgentMessageEvent {
pub message: String,
}
#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)]
#[serde(rename_all = "snake_case")]
pub enum InputMessageKind {
/// Plain user text (default)
Plain,
/// XML-wrapped user instructions (<user_instructions>...)
UserInstructions,
/// XML-wrapped environment context (<environment_context>...)
EnvironmentContext,
}
#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)]
pub struct UserMessageEvent {
pub message: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub kind: Option<InputMessageKind>,
#[serde(skip_serializing_if = "Option::is_none")]
pub images: Option<Vec<String>>,
}
impl<T, U> From<(T, U)> for InputMessageKind
where
T: AsRef<str>,
U: AsRef<str>,
{
fn from(value: (T, U)) -> Self {
let (_role, message) = value;
let message = message.as_ref();
let trimmed = message.trim();
if starts_with_ignore_ascii_case(trimmed, ENVIRONMENT_CONTEXT_OPEN_TAG)
&& ends_with_ignore_ascii_case(trimmed, ENVIRONMENT_CONTEXT_CLOSE_TAG)
{
InputMessageKind::EnvironmentContext
} else if starts_with_ignore_ascii_case(trimmed, USER_INSTRUCTIONS_OPEN_TAG)
&& ends_with_ignore_ascii_case(trimmed, USER_INSTRUCTIONS_CLOSE_TAG)
{
InputMessageKind::UserInstructions
} else {
InputMessageKind::Plain
}
}
}
fn starts_with_ignore_ascii_case(text: &str, prefix: &str) -> bool {
let text_bytes = text.as_bytes();
let prefix_bytes = prefix.as_bytes();
text_bytes.len() >= prefix_bytes.len()
&& text_bytes
.iter()
.zip(prefix_bytes.iter())
.all(|(a, b)| a.eq_ignore_ascii_case(b))
}
fn ends_with_ignore_ascii_case(text: &str, suffix: &str) -> bool {
let text_bytes = text.as_bytes();
let suffix_bytes = suffix.as_bytes();
text_bytes.len() >= suffix_bytes.len()
&& text_bytes[text_bytes.len() - suffix_bytes.len()..]
.iter()
.zip(suffix_bytes.iter())
.all(|(a, b)| a.eq_ignore_ascii_case(b))
}
#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)]
pub struct AgentMessageDeltaEvent {
pub delta: String,
codex-rs/tui/src/bottom_pane/chat_composer.rs
+29
View File
@@ -2069,6 +2069,35 @@ mod tests {
}
}
#[test]
fn ascii_prefix_survives_non_ascii_followup() {
use crossterm::event::KeyCode;
use crossterm::event::KeyEvent;
use crossterm::event::KeyModifiers;
let (tx, _rx) = unbounded_channel::<AppEvent>();
let sender = AppEventSender::new(tx);
let mut composer = ChatComposer::new(
true,
sender,
false,
"Ask Codex to do anything".to_string(),
false,
);
let _ = composer.handle_key_event(KeyEvent::new(KeyCode::Char('1'), KeyModifiers::NONE));
assert!(composer.is_in_paste_burst());
let _ = composer.handle_key_event(KeyEvent::new(KeyCode::Char('あ'), KeyModifiers::NONE));
let (result, _) =
composer.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE));
match result {
InputResult::Submitted(text) => assert_eq!(text, "1あ"),
_ => panic!("expected Submitted"),
}
}
#[test]
fn handle_paste_small_inserts_text() {
use crossterm::event::KeyCode;
codex-rs/tui/src/bottom_pane/paste_burst.rs
+8 -5
View File
@@ -198,12 +198,15 @@ impl PasteBurst {
/// Before applying modified/non-char input: flush buffered burst immediately.
pub fn flush_before_modified_input(&mut self) -> Option<String> {
if self.is_active() {
self.active = false;
Some(std::mem::take(&mut self.buffer))
} else {
None
if !self.is_active() {
return None;
}
self.active = false;
let mut out = std::mem::take(&mut self.buffer);
if let Some((ch, _at)) = self.pending_first_char.take() {
out.push(ch);
}
Some(out)
}
/// Clear only the timing window and any pending first-char.
codex-rs/tui/src/chatwidget.rs
+4 -14
View File
@@ -23,7 +23,6 @@ use codex_core::protocol::ExecApprovalRequestEvent;
use codex_core::protocol::ExecCommandBeginEvent;
use codex_core::protocol::ExecCommandEndEvent;
use codex_core::protocol::ExitedReviewModeEvent;
use codex_core::protocol::InputMessageKind;
use codex_core::protocol::ListCustomPromptsResponseEvent;
use codex_core::protocol::McpListToolsResponseEvent;
use codex_core::protocol::McpToolCallBeginEvent;
@@ -746,9 +745,8 @@ impl ChatWidget {
&ev.call_id,
CommandOutput {
exit_code: ev.exit_code,
stdout: ev.stdout.clone(),
stderr: ev.stderr.clone(),
formatted_output: ev.formatted_output.clone(),
aggregated_output: ev.aggregated_output.clone(),
},
ev.duration,
);
@@ -1553,17 +1551,9 @@ impl ChatWidget {
}
fn on_user_message_event(&mut self, event: UserMessageEvent) {
match event.kind {
Some(InputMessageKind::EnvironmentContext)
| Some(InputMessageKind::UserInstructions) => {
// Skip XML‑wrapped context blocks in the transcript.
}
Some(InputMessageKind::Plain) | None => {
let message = event.message.trim();
if !message.is_empty() {
self.add_to_history(history_cell::new_user_prompt(message.to_string()));
}
}
let message = event.message.trim();
if !message.is_empty() {
self.add_to_history(history_cell::new_user_prompt(message.to_string()));
}
}
codex-rs/tui/src/chatwidget/tests.rs
+14 -8
View File
@@ -23,7 +23,6 @@ use codex_core::protocol::ExecCommandBeginEvent;
use codex_core::protocol::ExecCommandEndEvent;
use codex_core::protocol::ExitedReviewModeEvent;
use codex_core::protocol::FileChange;
use codex_core::protocol::InputMessageKind;
use codex_core::protocol::Op;
use codex_core::protocol::PatchApplyBeginEvent;
use codex_core::protocol::PatchApplyEndEvent;
@@ -72,18 +71,26 @@ fn upgrade_event_payload_for_tests(mut payload: serde_json::Value) -> serde_json
&& let Some(m) = msg.as_object_mut()
{
let ty = m.get("type").and_then(|v| v.as_str()).unwrap_or("");
if ty == "exec_command_end" && !m.contains_key("formatted_output") {
if ty == "exec_command_end" {
let stdout = m.get("stdout").and_then(|v| v.as_str()).unwrap_or("");
let stderr = m.get("stderr").and_then(|v| v.as_str()).unwrap_or("");
let formatted = if stderr.is_empty() {
let aggregated = if stderr.is_empty() {
stdout.to_string()
} else {
format!("{stdout}{stderr}")
};
m.insert(
"formatted_output".to_string(),
serde_json::Value::String(formatted),
);
if !m.contains_key("formatted_output") {
m.insert(
"formatted_output".to_string(),
serde_json::Value::String(aggregated.clone()),
);
}
if !m.contains_key("aggregated_output") {
m.insert(
"aggregated_output".to_string(),
serde_json::Value::String(aggregated),
);
}
}
}
payload
@@ -104,7 +111,6 @@ fn resumed_initial_messages_render_history() {
initial_messages: Some(vec![
EventMsg::UserMessage(UserMessageEvent {
message: "hello from user".to_string(),
kind: Some(InputMessageKind::Plain),
images: None,
}),
EventMsg::AgentMessage(AgentMessageEvent {
codex-rs/tui/src/exec_cell/model.rs
+5 -5
View File
@@ -3,11 +3,12 @@ use std::time::Instant;
use codex_protocol::parse_command::ParsedCommand;
#[derive(Clone, Debug)]
#[derive(Clone, Debug, Default)]
pub(crate) struct CommandOutput {
pub(crate) exit_code: i32,
pub(crate) stdout: String,
pub(crate) stderr: String,
/// The aggregated stderr + stdout interleaved.
pub(crate) aggregated_output: String,
/// The formatted output of the command, as seen by the model.
pub(crate) formatted_output: String,
}
@@ -82,9 +83,8 @@ impl ExecCell {
call.duration = Some(elapsed);
call.output = Some(CommandOutput {
exit_code: 1,
stdout: String::new(),
stderr: String::new(),
formatted_output: String::new(),
aggregated_output: String::new(),
});
}
}
codex-rs/tui/src/exec_cell/render.rs
+2 -14
View File
@@ -28,7 +28,6 @@ use unicode_width::UnicodeWidthStr;
pub(crate) const TOOL_CALL_MAX_LINES: usize = 5;
pub(crate) struct OutputLinesParams {
pub(crate) only_err: bool,
pub(crate) include_angle_pipe: bool,
pub(crate) include_prefix: bool,
}
@@ -59,22 +58,12 @@ pub(crate) fn output_lines(
params: OutputLinesParams,
) -> OutputLines {
let OutputLinesParams {
only_err,
include_angle_pipe,
include_prefix,
} = params;
let CommandOutput {
exit_code,
stdout,
stderr,
..
aggregated_output, ..
} = match output {
Some(output) if only_err && output.exit_code == 0 => {
return OutputLines {
lines: Vec::new(),
omitted: None,
};
}
Some(output) => output,
None => {
return OutputLines {
@@ -84,7 +73,7 @@ pub(crate) fn output_lines(
}
};
let src = if *exit_code == 0 { stdout } else { stderr };
let src = aggregated_output;
let lines: Vec<&str> = src.lines().collect();
let total = lines.len();
let limit = TOOL_CALL_MAX_LINES;
@@ -398,7 +387,6 @@ impl ExecCell {
let raw_output = output_lines(
Some(output),
OutputLinesParams {
only_err: false,
include_angle_pipe: false,
include_prefix: false,
},
codex-rs/tui/src/file_search.rs
+1
View File
@@ -172,6 +172,7 @@ impl FileSearchManager {
NUM_FILE_SEARCH_THREADS,
cancellation_token.clone(),
compute_indices,
true,
)
.map(|res| res.matches)
.unwrap_or_default();
codex-rs/tui/src/history_cell.rs
+13 -107
View File
@@ -1293,12 +1293,10 @@ pub(crate) fn new_patch_apply_failure(stderr: String) -> PlainHistoryCell {
let output = output_lines(
Some(&CommandOutput {
exit_code: 1,
stdout: String::new(),
stderr,
formatted_output: String::new(),
aggregated_output: stderr,
}),
OutputLinesParams {
only_err: true,
include_angle_pipe: true,
include_prefix: true,
},
@@ -1739,16 +1737,7 @@ mod tests {
duration: None,
});
// Mark call complete so markers are ✓
cell.complete_call(
&call_id,
CommandOutput {
exit_code: 0,
stdout: String::new(),
stderr: String::new(),
formatted_output: String::new(),
},
Duration::from_millis(1),
);
cell.complete_call(&call_id, CommandOutput::default(), Duration::from_millis(1));
let lines = cell.display_lines(80);
let rendered = render_lines(&lines).join("\n");
@@ -1770,16 +1759,7 @@ mod tests {
duration: None,
});
// Call 1: Search only
cell.complete_call(
"c1",
CommandOutput {
exit_code: 0,
stdout: String::new(),
stderr: String::new(),
formatted_output: String::new(),
},
Duration::from_millis(1),
);
cell.complete_call("c1", CommandOutput::default(), Duration::from_millis(1));
// Call 2: Read A
cell = cell
.with_added_call(
@@ -1792,16 +1772,7 @@ mod tests {
}],
)
.unwrap();
cell.complete_call(
"c2",
CommandOutput {
exit_code: 0,
stdout: String::new(),
stderr: String::new(),
formatted_output: String::new(),
},
Duration::from_millis(1),
);
cell.complete_call("c2", CommandOutput::default(), Duration::from_millis(1));
// Call 3: Read B
cell = cell
.with_added_call(
@@ -1814,16 +1785,7 @@ mod tests {
}],
)
.unwrap();
cell.complete_call(
"c3",
CommandOutput {
exit_code: 0,
stdout: String::new(),
stderr: String::new(),
formatted_output: String::new(),
},
Duration::from_millis(1),
);
cell.complete_call("c3", CommandOutput::default(), Duration::from_millis(1));
let lines = cell.display_lines(80);
let rendered = render_lines(&lines).join("\n");
@@ -1856,16 +1818,7 @@ mod tests {
start_time: Some(Instant::now()),
duration: None,
});
cell.complete_call(
"c1",
CommandOutput {
exit_code: 0,
stdout: String::new(),
stderr: String::new(),
formatted_output: String::new(),
},
Duration::from_millis(1),
);
cell.complete_call("c1", CommandOutput::default(), Duration::from_millis(1));
let lines = cell.display_lines(80);
let rendered = render_lines(&lines).join("\n");
insta::assert_snapshot!(rendered);
@@ -1885,16 +1838,7 @@ mod tests {
duration: None,
});
// Mark call complete so it renders as "Ran"
cell.complete_call(
&call_id,
CommandOutput {
exit_code: 0,
stdout: String::new(),
stderr: String::new(),
formatted_output: String::new(),
},
Duration::from_millis(1),
);
cell.complete_call(&call_id, CommandOutput::default(), Duration::from_millis(1));
// Small width to force wrapping on both lines
let width: u16 = 28;
@@ -1914,16 +1858,7 @@ mod tests {
start_time: Some(Instant::now()),
duration: None,
});
cell.complete_call(
&call_id,
CommandOutput {
exit_code: 0,
stdout: String::new(),
stderr: String::new(),
formatted_output: String::new(),
},
Duration::from_millis(1),
);
cell.complete_call(&call_id, CommandOutput::default(), Duration::from_millis(1));
// Wide enough that it fits inline
let lines = cell.display_lines(80);
let rendered = render_lines(&lines).join("\n");
@@ -1942,16 +1877,7 @@ mod tests {
start_time: Some(Instant::now()),
duration: None,
});
cell.complete_call(
&call_id,
CommandOutput {
exit_code: 0,
stdout: String::new(),
stderr: String::new(),
formatted_output: String::new(),
},
Duration::from_millis(1),
);
cell.complete_call(&call_id, CommandOutput::default(), Duration::from_millis(1));
let lines = cell.display_lines(24);
let rendered = render_lines(&lines).join("\n");
insta::assert_snapshot!(rendered);
@@ -1969,16 +1895,7 @@ mod tests {
start_time: Some(Instant::now()),
duration: None,
});
cell.complete_call(
&call_id,
CommandOutput {
exit_code: 0,
stdout: String::new(),
stderr: String::new(),
formatted_output: String::new(),
},
Duration::from_millis(1),
);
cell.complete_call(&call_id, CommandOutput::default(), Duration::from_millis(1));
let lines = cell.display_lines(80);
let rendered = render_lines(&lines).join("\n");
insta::assert_snapshot!(rendered);
@@ -1997,16 +1914,7 @@ mod tests {
start_time: Some(Instant::now()),
duration: None,
});
cell.complete_call(
&call_id,
CommandOutput {
exit_code: 0,
stdout: String::new(),
stderr: String::new(),
formatted_output: String::new(),
},
Duration::from_millis(1),
);
cell.complete_call(&call_id, CommandOutput::default(), Duration::from_millis(1));
let lines = cell.display_lines(28);
let rendered = render_lines(&lines).join("\n");
insta::assert_snapshot!(rendered);
@@ -2033,9 +1941,8 @@ mod tests {
&call_id,
CommandOutput {
exit_code: 1,
stdout: String::new(),
stderr,
formatted_output: String::new(),
aggregated_output: stderr,
},
Duration::from_millis(1),
);
@@ -2077,9 +1984,8 @@ mod tests {
&call_id,
CommandOutput {
exit_code: 1,
stdout: String::new(),
stderr,
formatted_output: String::new(),
aggregated_output: stderr,
},
Duration::from_millis(5),
);
codex-rs/tui/src/pager_overlay.rs
+1 -2
View File
@@ -724,8 +724,7 @@ mod tests {
"exec-1",
CommandOutput {
exit_code: 0,
stdout: "src\nREADME.md\n".into(),
stderr: String::new(),
aggregated_output: "src\nREADME.md\n".into(),
formatted_output: "src\nREADME.md\n".into(),
},
Duration::from_millis(420),
codex-rs/tui/src/resume_picker.rs
+16 -34
View File
@@ -10,6 +10,7 @@ use codex_core::ConversationsPage;
use codex_core::Cursor;
use codex_core::INTERACTIVE_SESSION_SOURCES;
use codex_core::RolloutRecorder;
use codex_protocol::items::TurnItem;
use color_eyre::eyre::Result;
use crossterm::event::KeyCode;
use crossterm::event::KeyEvent;
@@ -30,10 +31,7 @@ use crate::text_formatting::truncate_text;
use crate::tui::FrameRequester;
use crate::tui::Tui;
use crate::tui::TuiEvent;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::InputMessageKind;
use codex_protocol::protocol::USER_MESSAGE_BEGIN;
const PAGE_SIZE: usize = 25;
const LOAD_NEAR_THRESHOLD: usize = 5;
@@ -616,37 +614,8 @@ fn extract_timestamp(value: &serde_json::Value) -> Option<DateTime<Utc>> {
fn preview_from_head(head: &[serde_json::Value]) -> Option<String> {
head.iter()
.filter_map(|value| serde_json::from_value::<ResponseItem>(value.clone()).ok())
.find_map(|item| match item {
ResponseItem::Message { content, .. } => {
// Find the actual user message (as opposed to user instructions or ide context)
let preview = content
.into_iter()
.filter_map(|content| match content {
ContentItem::InputText { text }
if matches!(
InputMessageKind::from(("user", text.as_str())),
InputMessageKind::Plain
) =>
{
// Strip ide context.
let text = match text.find(USER_MESSAGE_BEGIN) {
Some(idx) => {
text[idx + USER_MESSAGE_BEGIN.len()..].trim().to_string()
}
None => text,
};
Some(text)
}
_ => None,
})
.collect::<String>();
if preview.is_empty() {
None
} else {
Some(preview)
}
}
.find_map(|item| match codex_core::parse_turn_item(&item) {
Some(TurnItem::UserMessage(user)) => Some(user.message()),
_ => None,
})
}
@@ -999,6 +968,19 @@ mod tests {
"role": "user",
"content": [
{ "type": "input_text", "text": "<user_instructions>hi</user_instructions>" },
]
}),
json!({
"type": "message",
"role": "user",
"content": [
{ "type": "input_text", "text": "<environment_context>...</environment_context>" },
]
}),
json!({
"type": "message",
"role": "user",
"content": [
{ "type": "input_text", "text": "real question" },
{ "type": "input_image", "image_url": "ignored" }
]
codex-rs/utils/tokenizer/src/lib.rs
+6 -1
View File
@@ -55,8 +55,13 @@ impl Tokenizer {
Ok(Self { inner })
}
/// Default to `O200kBase`
pub fn try_default() -> Result<Self, TokenizerError> {
Self::new(EncodingKind::O200kBase)
}
/// Build a tokenizer using an `OpenAI` model name (maps to an encoding).
/// Falls back to the `o200k_base` encoding when the model is unknown.
/// Falls back to the `O200kBase` encoding when the model is unknown.
pub fn for_model(model: &str) -> Result<Self, TokenizerError> {
match tiktoken_rs::get_bpe_from_model(model) {
Ok(inner) => Ok(Self { inner }),
docs/agents_md.md
-1
View File
@@ -44,7 +44,6 @@ If the repository root contains `TEAM_GUIDE.md` and the `backend/` directory con
You can configure those fallbacks in `~/.codex/config.toml` (or another profile) like this:
```toml
[project]
project_doc_fallback_filenames = ["TEAM_GUIDE.md", ".agents.md"]
```
docs/config.md
+1 -5
View File
@@ -451,11 +451,7 @@ When both `enabled_tools` and `disabled_tools` are specified, Codex first restri
#### Experimental RMCP client
Codex is transitioning to the [official Rust MCP SDK](https://github.com/modelcontextprotocol/rust-sdk).
This flag enables OAuth support for streamable HTTP servers and switches STDIO servers over to the new client implementation.
Please try and report issues with the new client. To enable it, add this to the top level of your `config.toml`
This flag enables OAuth support for streamable HTTP servers.
```toml
experimental_use_rmcp_client = true