From 679f63ba06ed2a3fb760c2bbd360d10ab340ecfd Mon Sep 17 00:00:00 2001 From: canvrno-oai Date: Tue, 14 Apr 2026 20:36:48 -0700 Subject: [PATCH 01/84] Fix clippy warnings in external agent config migration (#17884) Fix clippy warnings in external agent config migration ``` error: this expression creates a reference which is immediately dereferenced by the compiler --> core/src/external_agent_config.rs:188:55 | 188 | let migrated = build_config_from_external(&settings)?; | ^^^^^^^^^ help: change this to: `settings` | = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.93.0/index.html#needless_borrow = note: requested on the command line with `-D clippy::needless-borrow` error: useless conversion to the same type: `codex_utils_absolute_path::AbsolutePathBuf` --> core/src/external_agent_config.rs:355:27 | 355 | match AbsolutePathBuf::try_from( | ___________________________^ 356 | | add_marketplace_outcome 357 | | .installed_root 358 | | .join(INSTALLED_MARKETPLACE_MANIFEST_RELATIVE_PATH), 359 | | ) { | |_____________________^ | = help: consider removing `AbsolutePathBuf::try_from()` = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.93.0/index.html#useless_conversion = note: `-D clippy::useless-conversion` implied by `-D warnings` = help: to override `-D warnings` add `#[allow(clippy::useless_conversion)]` error: aborting due to 2 previous errors ``` --- codex-rs/core/src/external_agent_config.rs | 26 ++++++---------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/codex-rs/core/src/external_agent_config.rs b/codex-rs/core/src/external_agent_config.rs index 1699239ab4..18305a6262 100644 --- a/codex-rs/core/src/external_agent_config.rs +++ b/codex-rs/core/src/external_agent_config.rs @@ -3,7 +3,6 @@ use crate::plugins::PluginId; use crate::plugins::PluginInstallRequest; use crate::plugins::PluginsManager; use crate::plugins::add_marketplace; -use codex_utils_absolute_path::AbsolutePathBuf; use serde_json::Value as JsonValue; use std::collections::BTreeMap; use std::collections::HashSet; @@ -185,7 +184,7 @@ impl ExternalAgentConfigService { |repo_root| repo_root.join(".codex").join("config.toml"), ); if let Some(settings) = settings.as_ref() { - let migrated = build_config_from_external(&settings)?; + let migrated = build_config_from_external(settings)?; if !is_empty_toml_table(&migrated) { let mut should_include = true; if target_config.exists() { @@ -352,23 +351,12 @@ impl ExternalAgentConfigService { let add_marketplace_outcome = add_marketplace(self.codex_home.clone(), request).await; let marketplace_path = match add_marketplace_outcome { Ok(add_marketplace_outcome) => { - match AbsolutePathBuf::try_from( - add_marketplace_outcome - .installed_root - .join(INSTALLED_MARKETPLACE_MANIFEST_RELATIVE_PATH), - ) { - Ok(path) => { - outcome - .succeeded_marketplaces - .push(marketplace_name.clone()); - path - } - Err(_) => { - outcome.failed_marketplaces.push(marketplace_name); - outcome.failed_plugin_ids.extend(plugin_ids); - continue; - } - } + outcome + .succeeded_marketplaces + .push(marketplace_name.clone()); + add_marketplace_outcome + .installed_root + .join(INSTALLED_MARKETPLACE_MANIFEST_RELATIVE_PATH) } Err(_) => { outcome.failed_marketplaces.push(marketplace_name); From e063596c67b2c5f047694960ff1b14275d95aa1d Mon Sep 17 00:00:00 2001 From: starr-openai Date: Tue, 14 Apr 2026 20:42:03 -0700 Subject: [PATCH 02/84] Reuse remote exec-server in core tests (#17837) ## Summary - reuse a shared remote exec-server for remote-aware codex-core integration tests within a test binary process - keep per-test remote cwd creation and cleanup so tests retain workspace isolation - leave codex_self_exe, codex_linux_sandbox_exe, cwd_path(), and workspace_path() behavior unchanged ## Validation - rustfmt codex-rs/core/tests/common/test_codex.rs - git diff --check - CI is running on the updated branch --- .github/workflows/rust-ci-full.yml | 1 + codex-rs/core/tests/common/test_codex.rs | 228 +++-------------------- scripts/test-remote-env.sh | 50 +++++ 3 files changed, 79 insertions(+), 200 deletions(-) diff --git a/.github/workflows/rust-ci-full.yml b/.github/workflows/rust-ci-full.yml index a39c38857b..7cda682952 100644 --- a/.github/workflows/rust-ci-full.yml +++ b/.github/workflows/rust-ci-full.yml @@ -664,6 +664,7 @@ jobs: export CODEX_TEST_REMOTE_ENV_CONTAINER_NAME=codex-remote-test-env source "${GITHUB_WORKSPACE}/scripts/test-remote-env.sh" echo "CODEX_TEST_REMOTE_ENV=${CODEX_TEST_REMOTE_ENV}" >> "$GITHUB_ENV" + echo "CODEX_TEST_REMOTE_EXEC_SERVER_URL=${CODEX_TEST_REMOTE_EXEC_SERVER_URL}" >> "$GITHUB_ENV" - name: tests id: test diff --git a/codex-rs/core/tests/common/test_codex.rs b/codex-rs/core/tests/common/test_codex.rs index b1274d5a94..92176c338f 100644 --- a/codex-rs/core/tests/common/test_codex.rs +++ b/codex-rs/core/tests/common/test_codex.rs @@ -7,8 +7,6 @@ use std::process::Command; use std::sync::Arc; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; -use std::time::Duration; -use std::time::Instant; use anyhow::Context; use anyhow::Result; @@ -44,7 +42,6 @@ use tempfile::TempDir; use wiremock::MockServer; use crate::PathBufExt; -use crate::RemoteEnvConfig; use crate::TempDirExt; use crate::get_remote_test_env; use crate::load_default_config_for_test; @@ -62,50 +59,15 @@ type PreBuildHook = dyn FnOnce(&Path) + Send + 'static; type WorkspaceSetup = dyn FnOnce(AbsolutePathBuf, Arc) -> BoxFuture<'static, Result<()>> + Send; const TEST_MODEL_WITH_EXPERIMENTAL_TOOLS: &str = "test-gpt-5.1-codex"; -const REMOTE_EXEC_SERVER_START_TIMEOUT: Duration = Duration::from_secs(5); -const REMOTE_EXEC_SERVER_POLL_INTERVAL: Duration = Duration::from_millis(25); -static REMOTE_EXEC_SERVER_INSTANCE_COUNTER: AtomicU64 = AtomicU64::new(0); - -#[derive(Debug)] -struct RemoteExecServerProcess { - container_name: String, - pid: u32, - remote_exec_server_path: String, - stdout_path: String, - cleanup_paths: Vec, -} - -impl Drop for RemoteExecServerProcess { - fn drop(&mut self) { - let cleanup_paths = self.cleanup_paths.join(" "); - let cleanup_paths_script = if cleanup_paths.is_empty() { - String::new() - } else { - format!("rm -rf {cleanup_paths}; ") - }; - let script = format!( - "if kill -0 {pid} 2>/dev/null; then kill {pid}; fi; {cleanup_paths_script}rm -f {remote_exec_server_path} {stdout_path}", - pid = self.pid, - cleanup_paths_script = cleanup_paths_script, - remote_exec_server_path = self.remote_exec_server_path, - stdout_path = self.stdout_path - ); - let _ = docker_command_capture_stdout(["exec", &self.container_name, "sh", "-lc", &script]); - } -} - -impl RemoteExecServerProcess { - fn register_cleanup_path(&mut self, path: &Path) { - self.cleanup_paths.push(path.display().to_string()); - } -} +const REMOTE_EXEC_SERVER_URL_ENV_VAR: &str = "CODEX_TEST_REMOTE_EXEC_SERVER_URL"; +static REMOTE_TEST_INSTANCE_COUNTER: AtomicU64 = AtomicU64::new(0); #[derive(Debug)] pub struct TestEnv { environment: codex_exec_server::Environment, cwd: AbsolutePathBuf, local_cwd_temp_dir: Option>, - _remote_exec_server_process: Option, + remote_container_name: Option, } impl TestEnv { @@ -117,7 +79,7 @@ impl TestEnv { environment, cwd, local_cwd_temp_dir: Some(local_cwd_temp_dir), - _remote_exec_server_process: None, + remote_container_name: None, }) } @@ -138,12 +100,19 @@ impl TestEnv { } } +impl Drop for TestEnv { + fn drop(&mut self) { + if let Some(container_name) = &self.remote_container_name { + let script = format!("rm -rf {}", self.cwd.as_path().display()); + let _ = docker_command_capture_stdout(["exec", container_name, "sh", "-lc", &script]); + } + } +} + pub async fn test_env() -> Result { match get_remote_test_env() { Some(remote_env) => { - let mut remote_process = start_remote_exec_server(&remote_env)?; - let remote_ip = remote_container_ip(&remote_env.container_name)?; - let websocket_url = rewrite_websocket_host(&remote_process.listen_url, &remote_ip)?; + let websocket_url = remote_exec_server_url()?; let environment = codex_exec_server::Environment::create(Some(websocket_url)).await?; let cwd = remote_aware_cwd_path(); environment @@ -154,184 +123,43 @@ pub async fn test_env() -> Result { /*sandbox*/ None, ) .await?; - remote_process.process.register_cleanup_path(cwd.as_path()); Ok(TestEnv { environment, cwd, local_cwd_temp_dir: None, - _remote_exec_server_process: Some(remote_process.process), + remote_container_name: Some(remote_env.container_name), }) } None => TestEnv::local().await, } } -struct RemoteExecServerStart { - process: RemoteExecServerProcess, - listen_url: String, -} - -fn start_remote_exec_server(remote_env: &RemoteEnvConfig) -> Result { - let container_name = remote_env.container_name.as_str(); - let instance_id = remote_exec_server_instance_id(); - let remote_exec_server_path = format!("/tmp/codex-{instance_id}"); - let remote_linux_sandbox_path = format!("/tmp/codex-linux-sandbox-{instance_id}"); - let stdout_path = format!("/tmp/codex-exec-server-{instance_id}.stdout"); - let local_binary = codex_utils_cargo_bin::cargo_bin("codex").context("resolve codex binary")?; - let local_linux_sandbox = codex_utils_cargo_bin::cargo_bin("codex-linux-sandbox") - .context("resolve codex-linux-sandbox binary")?; - let local_binary = local_binary.to_string_lossy().to_string(); - let local_linux_sandbox = local_linux_sandbox.to_string_lossy().to_string(); - let remote_binary = format!("{container_name}:{remote_exec_server_path}"); - let remote_linux_sandbox = format!("{container_name}:{remote_linux_sandbox_path}"); - - docker_command_success(["cp", &local_binary, &remote_binary])?; - docker_command_success(["cp", &local_linux_sandbox, &remote_linux_sandbox])?; - docker_command_success([ - "exec", - container_name, - "chmod", - "+x", - &remote_exec_server_path, - ])?; - docker_command_success([ - "exec", - container_name, - "chmod", - "+x", - &remote_linux_sandbox_path, - ])?; - probe_remote_linux_sandbox(container_name, &remote_linux_sandbox_path)?; - - let start_script = format!( - "rm -f {stdout_path}; \ -nohup {remote_exec_server_path} exec-server --listen ws://0.0.0.0:0 > {stdout_path} 2>&1 & \ -echo $!" - ); - let pid_output = - docker_command_capture_stdout(["exec", container_name, "sh", "-lc", &start_script])?; - let pid = pid_output - .trim() - .parse::() - .with_context(|| format!("parse remote exec-server PID from {pid_output:?}"))?; - - let listen_url = wait_for_remote_listen_url(container_name, &stdout_path)?; - - Ok(RemoteExecServerStart { - process: RemoteExecServerProcess { - container_name: container_name.to_string(), - pid, - remote_exec_server_path, - stdout_path, - cleanup_paths: vec![remote_linux_sandbox_path], - }, - listen_url, - }) -} - -fn probe_remote_linux_sandbox(container_name: &str, remote_linux_sandbox_path: &str) -> Result<()> { - let policy = serde_json::to_string(&SandboxPolicy::new_read_only_policy()) - .context("serialize remote sandbox probe policy")?; - let probe_script = format!( - "{remote_linux_sandbox_path} --sandbox-policy-cwd /tmp --sandbox-policy '{policy}' -- /bin/true" - ); - let output = Command::new("docker") - .args(["exec", container_name, "sh", "-lc", &probe_script]) - .output() - .with_context(|| format!("probe remote linux sandbox in container `{container_name}`"))?; - if !output.status.success() { - return Err(anyhow!( - "remote linux sandbox probe failed in container `{container_name}`: stdout={} stderr={}", - String::from_utf8_lossy(&output.stdout).trim(), - String::from_utf8_lossy(&output.stderr).trim() - )); - } - Ok(()) -} - fn remote_aware_cwd_path() -> AbsolutePathBuf { PathBuf::from(format!( "/tmp/codex-core-test-cwd-{}", - remote_exec_server_instance_id() + remote_test_instance_id() )) .abs() } -fn wait_for_remote_listen_url(container_name: &str, stdout_path: &str) -> Result { - let deadline = Instant::now() + REMOTE_EXEC_SERVER_START_TIMEOUT; - loop { - let line = docker_command_capture_stdout([ - "exec", - container_name, - "sh", - "-lc", - &format!("head -n 1 {stdout_path} 2>/dev/null || true"), - ])?; - let listen_url = line.trim(); - if listen_url.starts_with("ws://") { - return Ok(listen_url.to_string()); - } - - if Instant::now() >= deadline { - return Err(anyhow!( - "timed out waiting for remote exec-server listen URL in container `{container_name}` after {REMOTE_EXEC_SERVER_START_TIMEOUT:?}" - )); - } - std::thread::sleep(REMOTE_EXEC_SERVER_POLL_INTERVAL); +fn remote_exec_server_url() -> Result { + let listen_url = std::env::var(REMOTE_EXEC_SERVER_URL_ENV_VAR).with_context(|| { + format!("{REMOTE_EXEC_SERVER_URL_ENV_VAR} must be set for remote tests") + })?; + let listen_url = listen_url.trim(); + if listen_url.is_empty() { + return Err(anyhow!( + "{REMOTE_EXEC_SERVER_URL_ENV_VAR} must not be empty" + )); } + Ok(listen_url.to_string()) } -fn remote_exec_server_instance_id() -> String { - let instance = REMOTE_EXEC_SERVER_INSTANCE_COUNTER.fetch_add(1, Ordering::Relaxed); +fn remote_test_instance_id() -> String { + let instance = REMOTE_TEST_INSTANCE_COUNTER.fetch_add(1, Ordering::Relaxed); format!("{}-{instance}", std::process::id()) } -fn remote_container_ip(container_name: &str) -> Result { - let ip = docker_command_capture_stdout([ - "inspect", - "-f", - "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", - container_name, - ])?; - let ip = ip.trim(); - if ip.is_empty() { - return Err(anyhow!( - "container `{container_name}` has no IP address; cannot connect to remote exec-server" - )); - } - Ok(ip.to_string()) -} - -fn rewrite_websocket_host(listen_url: &str, host: &str) -> Result { - let Some(address) = listen_url.strip_prefix("ws://") else { - return Err(anyhow!( - "unexpected websocket listen URL `{listen_url}`; expected ws://IP:PORT" - )); - }; - let Some((_, port)) = address.rsplit_once(':') else { - return Err(anyhow!( - "unexpected websocket listen URL `{listen_url}`; expected ws://IP:PORT" - )); - }; - Ok(format!("ws://{host}:{port}")) -} - -fn docker_command_success(args: [&str; N]) -> Result<()> { - let output = Command::new("docker") - .args(args) - .output() - .with_context(|| format!("run docker {args:?}"))?; - if !output.status.success() { - return Err(anyhow!( - "docker {:?} failed: stdout={} stderr={}", - args, - String::from_utf8_lossy(&output.stdout).trim(), - String::from_utf8_lossy(&output.stderr).trim() - )); - } - Ok(()) -} - fn docker_command_capture_stdout(args: [&str; N]) -> Result { let output = Command::new("docker") .args(args) diff --git a/scripts/test-remote-env.sh b/scripts/test-remote-env.sh index 329fa8a7f3..ba4d35f235 100755 --- a/scripts/test-remote-env.sh +++ b/scripts/test-remote-env.sh @@ -18,6 +18,11 @@ is_sourced() { setup_remote_env() { local container_name local codex_binary_path + local container_ip + local remote_codex_path + local remote_exec_server_pid + local remote_exec_server_port + local remote_exec_server_stdout_path container_name="${CODEX_TEST_REMOTE_ENV_CONTAINER_NAME:-codex-remote-test-env-local-$(date +%s)-${RANDOM}}" codex_binary_path="${REPO_ROOT}/codex-rs/target/debug/codex" @@ -59,14 +64,58 @@ setup_remote_env() { return 1 fi + if [[ -z "${CODEX_TEST_REMOTE_EXEC_SERVER_URL:-}" ]]; then + remote_codex_path="/tmp/codex-remote-env/codex" + remote_exec_server_port="31987" + remote_exec_server_stdout_path="/tmp/codex-remote-env/exec-server.stdout" + docker exec "${container_name}" sh -lc "mkdir -p /tmp/codex-remote-env" + docker cp "${codex_binary_path}" "${container_name}:${remote_codex_path}" + docker exec "${container_name}" chmod +x "${remote_codex_path}" + remote_exec_server_pid="$( + docker exec "${container_name}" sh -lc \ + "rm -f ${remote_exec_server_stdout_path}; nohup ${remote_codex_path} exec-server --listen ws://0.0.0.0:${remote_exec_server_port} > ${remote_exec_server_stdout_path} 2>&1 & echo \$!" + )" + wait_for_remote_exec_server_port "${container_name}" "${remote_exec_server_port}" "${remote_exec_server_stdout_path}" + container_ip="$( + docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "${container_name}" + )" + if [[ -z "${container_ip}" ]]; then + echo "container ${container_name} has no IP address" >&2 + docker rm -f "${container_name}" >/dev/null 2>&1 || true + return 1 + fi + export CODEX_TEST_REMOTE_EXEC_SERVER_PID="${remote_exec_server_pid}" + export CODEX_TEST_REMOTE_EXEC_SERVER_URL="ws://${container_ip}:${remote_exec_server_port}" + fi + export CODEX_TEST_REMOTE_ENV="${container_name}" } +wait_for_remote_exec_server_port() { + local container_name="$1" + local port="$2" + local stdout_path="$3" + local deadline=$((SECONDS + 5)) + + while (( SECONDS < deadline )); do + if docker exec "${container_name}" python3 -c "import socket; socket.create_connection(('127.0.0.1', ${port}), timeout=0.2).close()" >/dev/null 2>&1; then + return 0 + fi + sleep 0.025 + done + + echo "timed out waiting for remote exec-server on ${container_name}:${port}" >&2 + docker exec "${container_name}" sh -lc "cat ${stdout_path} 2>/dev/null || true" >&2 || true + return 1 +} + codex_remote_env_cleanup() { if [[ -n "${CODEX_TEST_REMOTE_ENV:-}" ]]; then docker rm -f "${CODEX_TEST_REMOTE_ENV}" >/dev/null 2>&1 || true unset CODEX_TEST_REMOTE_ENV fi + unset CODEX_TEST_REMOTE_EXEC_SERVER_PID + unset CODEX_TEST_REMOTE_EXEC_SERVER_URL } if ! is_sourced; then @@ -79,6 +128,7 @@ set -euo pipefail if setup_remote_env; then status=0 echo "CODEX_TEST_REMOTE_ENV=${CODEX_TEST_REMOTE_ENV}" + echo "CODEX_TEST_REMOTE_EXEC_SERVER_URL=${CODEX_TEST_REMOTE_EXEC_SERVER_URL}" echo "Remote env ready. Run your command, then call: codex_remote_env_cleanup" else status=$? From d34bc66466670630ec9ca080c775e279c66d536a Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 14 Apr 2026 20:48:01 -0700 Subject: [PATCH 03/84] sandbox: remove dead seatbelt helper and update tests (#17859) ## Why `spawn_command_under_seatbelt()` in `codex-rs/core/src/seatbelt.rs` had fallen out of production use and was only referenced by test-only wrappers. That left us with sandbox tests that could stay green even if the actual seatbelt exec path regressed, because production shell execution now flows through `SandboxManager::transform()` and `ExecRequest::from_sandbox_exec_request()` instead of that helper. Removing the dead helper also exposed one downstream `codex-exec` integration test that still imported it, which broke `just clippy`. ## What Changed - Removed `codex-rs/core/src/seatbelt.rs` and stopped exporting `codex_core::seatbelt`. - Removed the redundant `codex-rs/core/tests/suite/seatbelt.rs` coverage that only exercised the dead helper. - Kept the `openpty` regression check, but moved it into `codex-rs/core/tests/suite/exec.rs` so it now runs through `process_exec_tool_call()`. - Fixed the seatbelt denial test in `codex-rs/core/tests/suite/exec.rs` to use `/usr/bin/touch`, so it actually exercises the sandbox instead of a nonexistent path. - Updated `codex-rs/exec/tests/suite/sandbox.rs` on macOS to build the sandboxed command through `build_exec_request()` and spawn the transformed command, instead of importing the removed helper. - Left the lower-level seatbelt policy coverage in `codex-rs/sandboxing/src/seatbelt_tests.rs`, where the policy generator is still covered directly. ## Verification - `cargo test -p codex-core suite::exec::` - `cargo test -p codex-exec` - `cargo clippy -p codex-exec --tests -- -D warnings` --- codex-rs/core/src/lib.rs | 1 - codex-rs/core/src/seatbelt.rs | 48 ---- codex-rs/core/tests/suite/exec.rs | 45 +++- codex-rs/core/tests/suite/mod.rs | 1 - codex-rs/core/tests/suite/seatbelt.rs | 316 -------------------------- codex-rs/exec/tests/suite/sandbox.rs | 66 +++++- 6 files changed, 97 insertions(+), 380 deletions(-) delete mode 100644 codex-rs/core/src/seatbelt.rs delete mode 100644 codex-rs/core/tests/suite/seatbelt.rs diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 5e62199d2d..93d62faca6 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -139,7 +139,6 @@ pub use project_doc::discover_project_doc_paths; pub use project_doc::read_project_docs; mod rollout; pub(crate) mod safety; -pub mod seatbelt; mod session_rollout_init_error; pub mod shell; pub(crate) mod shell_snapshot; diff --git a/codex-rs/core/src/seatbelt.rs b/codex-rs/core/src/seatbelt.rs deleted file mode 100644 index c54ff3fd5b..0000000000 --- a/codex-rs/core/src/seatbelt.rs +++ /dev/null @@ -1,48 +0,0 @@ -#![cfg(target_os = "macos")] - -use crate::spawn::CODEX_SANDBOX_ENV_VAR; -use crate::spawn::SpawnChildRequest; -use crate::spawn::StdioPolicy; -use crate::spawn::spawn_child_async; -use codex_network_proxy::NetworkProxy; -use codex_protocol::permissions::FileSystemSandboxPolicy; -use codex_protocol::permissions::NetworkSandboxPolicy; -use codex_protocol::protocol::SandboxPolicy; -use codex_sandboxing::seatbelt::MACOS_PATH_TO_SEATBELT_EXECUTABLE; -use codex_sandboxing::seatbelt::create_seatbelt_command_args_for_policies; -use codex_utils_absolute_path::AbsolutePathBuf; -use std::collections::HashMap; -use std::path::PathBuf; -use tokio::process::Child; - -pub async fn spawn_command_under_seatbelt( - command: Vec, - command_cwd: AbsolutePathBuf, - sandbox_policy: &SandboxPolicy, - sandbox_policy_cwd: &AbsolutePathBuf, - stdio_policy: StdioPolicy, - network: Option<&NetworkProxy>, - mut env: HashMap, -) -> std::io::Result { - let args = create_seatbelt_command_args_for_policies( - command, - &FileSystemSandboxPolicy::from_legacy_sandbox_policy(sandbox_policy, sandbox_policy_cwd), - NetworkSandboxPolicy::from(sandbox_policy), - sandbox_policy_cwd, - /*enforce_managed_network*/ false, - network, - ); - let arg0 = None; - env.insert(CODEX_SANDBOX_ENV_VAR.to_string(), "seatbelt".to_string()); - spawn_child_async(SpawnChildRequest { - program: PathBuf::from(MACOS_PATH_TO_SEATBELT_EXECUTABLE), - args, - arg0, - cwd: command_cwd, - network_sandbox_policy: NetworkSandboxPolicy::from(sandbox_policy), - network, - stdio_policy, - env, - }) - .await -} diff --git a/codex-rs/core/tests/suite/exec.rs b/codex-rs/core/tests/suite/exec.rs index 19869ebb2b..c80c8a3340 100644 --- a/codex-rs/core/tests/suite/exec.rs +++ b/codex-rs/core/tests/suite/exec.rs @@ -1,8 +1,5 @@ #![cfg(target_os = "macos")] -use std::collections::HashMap; -use std::string::ToString; - use codex_core::exec::ExecCapturePolicy; use codex_core::exec::ExecParams; use codex_core::exec::process_exec_tool_call; @@ -17,6 +14,7 @@ use codex_protocol::protocol::SandboxPolicy; use codex_sandboxing::SandboxType; use codex_sandboxing::get_platform_sandbox; use core_test_support::PathExt; +use std::collections::HashMap; use tempfile::TempDir; fn skip_test() -> bool { @@ -29,14 +27,18 @@ fn skip_test() -> bool { } #[expect(clippy::expect_used)] -async fn run_test_cmd(tmp: TempDir, cmd: Vec<&str>) -> Result { +async fn run_test_cmd(tmp: TempDir, command: I) -> Result +where + I: IntoIterator, + S: Into, +{ let sandbox_type = get_platform_sandbox(/*windows_sandbox_enabled*/ false) .expect("should be able to get sandbox type"); assert_eq!(sandbox_type, SandboxType::MacosSeatbelt); let cwd = tmp.path().abs(); let params = ExecParams { - command: cmd.iter().map(ToString::to_string).collect(), + command: command.into_iter().map(Into::into).collect(), cwd: cwd.clone(), expiration: 1000.into(), capture_policy: ExecCapturePolicy::ShellTool, @@ -129,6 +131,37 @@ async fn exit_command_not_found_is_ok() { run_test_cmd(tmp, cmd).await.unwrap(); } +#[tokio::test] +async fn openpty_works_under_real_exec_seatbelt_path() { + if skip_test() { + return; + } + + let python = match which::which("python3") { + Ok(path) => path, + Err(_) => { + eprintln!("python3 not found in PATH, skipping test."); + return; + } + }; + + let tmp = TempDir::new().expect("should be able to create temp dir"); + let cmd = vec![ + python.to_string_lossy().into_owned(), + "-c".to_string(), + r#"import os + +master, slave = os.openpty() +os.write(slave, b"ping") +assert os.read(master, 4) == b"ping""# + .to_string(), + ]; + + let output = run_test_cmd(tmp, cmd).await.unwrap(); + assert_eq!(output.stdout.text, ""); + assert_eq!(output.stderr.text, ""); +} + /// Writing a file fails and should be considered a sandbox error #[tokio::test] async fn write_file_fails_as_sandbox_error() { @@ -139,7 +172,7 @@ async fn write_file_fails_as_sandbox_error() { let tmp = TempDir::new().expect("should be able to create temp dir"); let path = tmp.path().join("test.txt"); let cmd = vec![ - "/user/bin/touch", + "/usr/bin/touch", path.to_str().expect("should be able to get path"), ]; diff --git a/codex-rs/core/tests/suite/mod.rs b/codex-rs/core/tests/suite/mod.rs index 8f3d9e15d9..9f8389c45e 100644 --- a/codex-rs/core/tests/suite/mod.rs +++ b/codex-rs/core/tests/suite/mod.rs @@ -85,7 +85,6 @@ mod rmcp_client; mod rollout_list_find; mod safety_check_downgrade; mod search_tool; -mod seatbelt; mod shell_command; mod shell_serialization; mod shell_snapshot; diff --git a/codex-rs/core/tests/suite/seatbelt.rs b/codex-rs/core/tests/suite/seatbelt.rs deleted file mode 100644 index 808a8a022d..0000000000 --- a/codex-rs/core/tests/suite/seatbelt.rs +++ /dev/null @@ -1,316 +0,0 @@ -#![cfg(target_os = "macos")] - -//! Tests for the macOS sandboxing that are specific to Seatbelt. -//! Tests that apply to both Mac and Linux sandboxing should go in sandbox.rs. - -use std::collections::HashMap; -use std::path::Path; -use std::path::PathBuf; - -use codex_core::seatbelt::spawn_command_under_seatbelt; -use codex_core::spawn::CODEX_SANDBOX_ENV_VAR; -use codex_core::spawn::StdioPolicy; -use codex_protocol::protocol::SandboxPolicy; -use codex_utils_absolute_path::AbsolutePathBuf; -use tempfile::TempDir; - -struct TestScenario { - repo_parent: PathBuf, - file_outside_repo: PathBuf, - repo_root: PathBuf, - file_in_repo_root: PathBuf, - file_in_dot_git_dir: PathBuf, -} - -struct TestExpectations { - file_outside_repo_is_writable: bool, - file_in_repo_root_is_writable: bool, - file_in_dot_git_dir_is_writable: bool, -} - -impl TestScenario { - async fn run_test(&self, policy: &SandboxPolicy, expectations: TestExpectations) { - if std::env::var(CODEX_SANDBOX_ENV_VAR) == Ok("seatbelt".to_string()) { - eprintln!("{CODEX_SANDBOX_ENV_VAR} is set to 'seatbelt', skipping test."); - return; - } - - assert_eq!( - touch(&self.file_outside_repo, policy).await, - expectations.file_outside_repo_is_writable - ); - assert_eq!( - self.file_outside_repo.exists(), - expectations.file_outside_repo_is_writable - ); - - assert_eq!( - touch(&self.file_in_repo_root, policy).await, - expectations.file_in_repo_root_is_writable - ); - assert_eq!( - self.file_in_repo_root.exists(), - expectations.file_in_repo_root_is_writable - ); - - assert_eq!( - touch(&self.file_in_dot_git_dir, policy).await, - expectations.file_in_dot_git_dir_is_writable - ); - assert_eq!( - self.file_in_dot_git_dir.exists(), - expectations.file_in_dot_git_dir_is_writable - ); - } -} - -/// If the user has added a workspace root that is not a Git repo root, then -/// the user has to specify `--skip-git-repo-check` or go through some -/// interstitial that indicates they are taking on some risk because Git -/// cannot be used to backup their work before the agent begins. -/// -/// Because the user has agreed to this risk, we do not try find all .git -/// folders in the workspace and block them (though we could change our -/// position on this in the future). -#[tokio::test] -async fn if_parent_of_repo_is_writable_then_dot_git_folder_is_writable() { - let tmp = TempDir::new().expect("should be able to create temp dir"); - let test_scenario = create_test_scenario(&tmp); - let policy = SandboxPolicy::WorkspaceWrite { - writable_roots: vec![test_scenario.repo_parent.as_path().try_into().unwrap()], - read_only_access: Default::default(), - network_access: false, - exclude_tmpdir_env_var: true, - exclude_slash_tmp: true, - }; - - test_scenario - .run_test( - &policy, - TestExpectations { - file_outside_repo_is_writable: true, - file_in_repo_root_is_writable: true, - file_in_dot_git_dir_is_writable: true, - }, - ) - .await; -} - -/// When the writable root is the root of a Git repository (as evidenced by the -/// presence of a .git folder), then the .git folder should be read-only if -/// the policy is `WorkspaceWrite`. -#[tokio::test] -async fn if_git_repo_is_writable_root_then_dot_git_folder_is_read_only() { - let tmp = TempDir::new().expect("should be able to create temp dir"); - let test_scenario = create_test_scenario(&tmp); - let policy = SandboxPolicy::WorkspaceWrite { - writable_roots: vec![test_scenario.repo_root.as_path().try_into().unwrap()], - read_only_access: Default::default(), - network_access: false, - exclude_tmpdir_env_var: true, - exclude_slash_tmp: true, - }; - - test_scenario - .run_test( - &policy, - TestExpectations { - file_outside_repo_is_writable: false, - file_in_repo_root_is_writable: true, - file_in_dot_git_dir_is_writable: false, - }, - ) - .await; -} - -/// Under DangerFullAccess, all writes should be permitted anywhere on disk, -/// including inside the .git folder. -#[tokio::test] -async fn danger_full_access_allows_all_writes() { - let tmp = TempDir::new().expect("should be able to create temp dir"); - let test_scenario = create_test_scenario(&tmp); - let policy = SandboxPolicy::DangerFullAccess; - - test_scenario - .run_test( - &policy, - TestExpectations { - file_outside_repo_is_writable: true, - file_in_repo_root_is_writable: true, - file_in_dot_git_dir_is_writable: true, - }, - ) - .await; -} - -/// Under ReadOnly, writes should not be permitted anywhere on disk. -#[tokio::test] -async fn read_only_forbids_all_writes() { - let tmp = TempDir::new().expect("should be able to create temp dir"); - let test_scenario = create_test_scenario(&tmp); - let policy = SandboxPolicy::new_read_only_policy(); - - test_scenario - .run_test( - &policy, - TestExpectations { - file_outside_repo_is_writable: false, - file_in_repo_root_is_writable: false, - file_in_dot_git_dir_is_writable: false, - }, - ) - .await; -} - -#[tokio::test] -async fn openpty_works_under_seatbelt() { - if std::env::var(CODEX_SANDBOX_ENV_VAR) == Ok("seatbelt".to_string()) { - eprintln!("{CODEX_SANDBOX_ENV_VAR} is set to 'seatbelt', skipping test."); - return; - } - - if which::which("python3").is_err() { - eprintln!("python3 not found in PATH, skipping test."); - return; - } - - let policy = SandboxPolicy::new_read_only_policy(); - let command_cwd = AbsolutePathBuf::current_dir().expect("getcwd"); - let sandbox_cwd = command_cwd.clone(); - - let mut child = spawn_command_under_seatbelt( - vec![ - "python3".to_string(), - "-c".to_string(), - r#"import os - -master, slave = os.openpty() -os.write(slave, b"ping") -assert os.read(master, 4) == b"ping""# - .to_string(), - ], - command_cwd, - &policy, - &sandbox_cwd, - StdioPolicy::RedirectForShellTool, - /*network*/ None, - HashMap::new(), - ) - .await - .expect("should be able to spawn python under seatbelt"); - - let status = child - .wait() - .await - .expect("should be able to wait for child process"); - assert!(status.success(), "python exited with {status:?}"); -} - -#[tokio::test] -async fn java_home_finds_runtime_under_seatbelt() { - if std::env::var(CODEX_SANDBOX_ENV_VAR) == Ok("seatbelt".to_string()) { - eprintln!("{CODEX_SANDBOX_ENV_VAR} is set to 'seatbelt', skipping test."); - return; - } - - let java_home_path = Path::new("/usr/libexec/java_home"); - if !java_home_path.exists() { - eprintln!("/usr/libexec/java_home is not present, skipping test."); - return; - } - - let baseline_output = tokio::process::Command::new(java_home_path) - .env_remove("JAVA_HOME") - .output() - .await - .expect("should be able to invoke java_home outside seatbelt"); - if !baseline_output.status.success() { - eprintln!( - "java_home exited with {:?} outside seatbelt, skipping test", - baseline_output.status - ); - return; - } - - let policy = SandboxPolicy::new_read_only_policy(); - let command_cwd = AbsolutePathBuf::current_dir().expect("getcwd"); - let sandbox_cwd = command_cwd.clone(); - - let mut env: HashMap = std::env::vars().collect(); - env.remove("JAVA_HOME"); - env.remove(CODEX_SANDBOX_ENV_VAR); - - let child = spawn_command_under_seatbelt( - vec![java_home_path.to_string_lossy().to_string()], - command_cwd, - &policy, - &sandbox_cwd, - StdioPolicy::RedirectForShellTool, - /*network*/ None, - env, - ) - .await - .expect("should be able to spawn java_home under seatbelt"); - - let output = child - .wait_with_output() - .await - .expect("should be able to wait for java_home child"); - assert!( - output.status.success(), - "java_home under seatbelt exited with {:?}, stderr: {}", - output.status, - String::from_utf8_lossy(&output.stderr) - ); - - let stdout = String::from_utf8_lossy(&output.stdout); - assert!( - !stdout.trim().is_empty(), - "java_home stdout unexpectedly empty under seatbelt" - ); -} - -#[expect(clippy::expect_used)] -fn create_test_scenario(tmp: &TempDir) -> TestScenario { - let repo_parent = tmp.path().to_path_buf(); - let repo_root = repo_parent.join("repo"); - let dot_git_dir = repo_root.join(".git"); - - std::fs::create_dir(&repo_root).expect("should be able to create repo root"); - std::fs::create_dir(&dot_git_dir).expect("should be able to create .git dir"); - - TestScenario { - file_outside_repo: repo_parent.join("outside.txt"), - repo_parent, - file_in_repo_root: repo_root.join("repo_file.txt"), - repo_root, - file_in_dot_git_dir: dot_git_dir.join("dot_git_file.txt"), - } -} - -#[expect(clippy::expect_used)] -/// Note that `path` must be absolute. -async fn touch(path: &Path, policy: &SandboxPolicy) -> bool { - assert!(path.is_absolute(), "Path must be absolute: {path:?}"); - let command_cwd = AbsolutePathBuf::current_dir().expect("getcwd"); - let sandbox_cwd = command_cwd.clone(); - let mut child = spawn_command_under_seatbelt( - vec![ - "/usr/bin/touch".to_string(), - path.to_string_lossy().to_string(), - ], - command_cwd, - policy, - &sandbox_cwd, - StdioPolicy::RedirectForShellTool, - /*network*/ None, - HashMap::new(), - ) - .await - .expect("should be able to spawn command under seatbelt"); - child - .wait() - .await - .expect("should be able to wait for child process") - .success() -} diff --git a/codex-rs/exec/tests/suite/sandbox.rs b/codex-rs/exec/tests/suite/sandbox.rs index 7a8c3238e8..691b590f4e 100644 --- a/codex-rs/exec/tests/suite/sandbox.rs +++ b/codex-rs/exec/tests/suite/sandbox.rs @@ -19,17 +19,67 @@ async fn spawn_command_under_sandbox( stdio_policy: StdioPolicy, env: HashMap, ) -> std::io::Result { - use codex_core::seatbelt::spawn_command_under_seatbelt; - spawn_command_under_seatbelt( - command, - command_cwd, + use codex_core::exec::ExecCapturePolicy; + use codex_core::exec::ExecParams; + use codex_core::exec::build_exec_request; + use codex_core::sandboxing::SandboxPermissions; + use codex_protocol::config_types::WindowsSandboxLevel; + use codex_protocol::permissions::FileSystemSandboxPolicy; + use codex_protocol::permissions::NetworkSandboxPolicy; + use std::process::Stdio; + + let codex_linux_sandbox_exe = None; + let exec_request = build_exec_request( + ExecParams { + command, + cwd: command_cwd, + expiration: 1000.into(), + capture_policy: ExecCapturePolicy::ShellTool, + env, + network: None, + sandbox_permissions: SandboxPermissions::UseDefault, + windows_sandbox_level: WindowsSandboxLevel::Disabled, + windows_sandbox_private_desktop: false, + justification: None, + arg0: None, + }, sandbox_policy, + &FileSystemSandboxPolicy::from_legacy_sandbox_policy(sandbox_policy, sandbox_cwd), + NetworkSandboxPolicy::from(sandbox_policy), sandbox_cwd, - stdio_policy, - /*network*/ None, - env, + &codex_linux_sandbox_exe, + /*use_legacy_landlock*/ false, ) - .await + .map_err(|err| io::Error::other(err.to_string()))?; + + let (program, args) = exec_request + .command + .split_first() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "command args are empty"))?; + + let mut child = tokio::process::Command::new(program); + if let Some(arg0) = exec_request.arg0.as_deref() { + child.arg0(arg0); + } + child.args(args); + child.current_dir(exec_request.cwd); + child.env_clear(); + child.envs(exec_request.env); + + match stdio_policy { + StdioPolicy::RedirectForShellTool => { + child.stdin(Stdio::null()); + child.stdout(Stdio::piped()).stderr(Stdio::piped()); + } + StdioPolicy::Inherit => { + child + .stdin(Stdio::inherit()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()); + } + } + + child.kill_on_drop(true).spawn() } #[cfg(target_os = "linux")] From c6defb1f0f7272fcd212f63f01460477d2b73e3b Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 14 Apr 2026 21:40:12 -0700 Subject: [PATCH 04/84] fix: cleanup the contract of the general-purpose exec() function (#17870) `exec()` had a number of arguments that were unused, making the function signature misleading. This PR aims to clean things up to clarify the role of this function and to clarify which fields of `ExecParams` are unused and why. --- codex-rs/core/src/exec.rs | 99 +++++++++++++++++++++------------ codex-rs/core/src/exec_tests.rs | 12 ---- 2 files changed, 63 insertions(+), 48 deletions(-) diff --git a/codex-rs/core/src/exec.rs b/codex-rs/core/src/exec.rs index d5b96902dc..10a1dc40ff 100644 --- a/codex-rs/core/src/exec.rs +++ b/codex-rs/core/src/exec.rs @@ -251,8 +251,24 @@ pub fn build_exec_request( codex_linux_sandbox_exe: &Option, use_legacy_landlock: bool, ) -> Result { - let windows_sandbox_level = params.windows_sandbox_level; - let enforce_managed_network = params.network.is_some(); + let ExecParams { + command, + cwd, + mut env, + expiration, + capture_policy, + network, + windows_sandbox_level, + windows_sandbox_private_desktop, + + // TODO: Should arg0 be set on the ExecRequest that is returned? + arg0: _, + // These fields are related to approvals, so can be ignored here. + justification: _, + sandbox_permissions: _, + } = params; + + let enforce_managed_network = network.is_some(); let sandbox_type = select_process_exec_tool_sandbox_type( file_system_sandbox_policy, network_sandbox_policy, @@ -261,19 +277,6 @@ pub fn build_exec_request( ); tracing::debug!("Sandbox type: {sandbox_type:?}"); - let ExecParams { - command, - cwd, - mut env, - expiration, - capture_policy, - network, - sandbox_permissions: _, - windows_sandbox_level, - windows_sandbox_private_desktop, - justification: _, - arg0: _, - } = params; if let Some(network) = network.as_ref() { network.apply_to_env(&mut env); } @@ -357,7 +360,8 @@ pub(crate) async fn execute_exec_request( windows_sandbox_level, windows_sandbox_private_desktop, sandbox_policy, - file_system_sandbox_policy, + // TODO(mbolin): Use file_system_sandbox_policy instead of sandbox_policy. + file_system_sandbox_policy: _, network_sandbox_policy, windows_sandbox_filesystem_overrides, arg0, @@ -378,21 +382,40 @@ pub(crate) async fn execute_exec_request( }; let start = Instant::now(); - let raw_output_result = exec( + let raw_output_result = get_raw_output_result( params, - sandbox, - &sandbox_policy, - &file_system_sandbox_policy, - windows_sandbox_filesystem_overrides.as_ref(), network_sandbox_policy, stdout_stream, after_spawn, + sandbox, + &sandbox_policy, + windows_sandbox_filesystem_overrides.as_ref(), ) .await; let duration = start.elapsed(); finalize_exec_result(raw_output_result, sandbox, duration) } +async fn get_raw_output_result( + params: ExecParams, + network_sandbox_policy: NetworkSandboxPolicy, + stdout_stream: Option, + after_spawn: Option>, + #[cfg_attr(not(windows), allow(unused_variables))] sandbox: SandboxType, + #[cfg_attr(not(windows), allow(unused_variables))] sandbox_policy: &SandboxPolicy, + #[cfg_attr(not(windows), allow(unused_variables))] windows_sandbox_filesystem_overrides: Option< + &WindowsSandboxFilesystemOverrides, + >, +) -> Result { + #[cfg(target_os = "windows")] + if sandbox == SandboxType::WindowsRestrictedToken { + return exec_windows_sandbox(params, sandbox_policy, windows_sandbox_filesystem_overrides) + .await; + } + + exec(params, network_sandbox_policy, stdout_stream, after_spawn).await +} + #[cfg(target_os = "windows")] fn extract_create_process_as_user_error_code(err: &str) -> Option { let marker = "CreateProcessAsUserW failed: "; @@ -799,26 +822,24 @@ fn aggregate_output( } } -#[allow(clippy::too_many_arguments)] +/// This is a general-purpose function for executing a command specified by +/// [ExecParams]. Events are reported via `stdout_stream`, if specified, and +/// `after_spawn` is invoked once the child process has been spawned, before +/// output consumption begins. +/// +/// `network_sandbox_policy` is used to determine whether +/// CODEX_SANDBOX_NETWORK_DISABLED=1 is added to the environment of the spawned +/// process. +/// +/// Note this command does not apply any sandboxing logic. The caller is +/// responsible for constructing [ExecParams::command] to include any sandboxing +/// wrapper args, as appropriate. async fn exec( params: ExecParams, - _sandbox: SandboxType, - _sandbox_policy: &SandboxPolicy, - _file_system_sandbox_policy: &FileSystemSandboxPolicy, - _windows_sandbox_filesystem_overrides: Option<&WindowsSandboxFilesystemOverrides>, network_sandbox_policy: NetworkSandboxPolicy, stdout_stream: Option, after_spawn: Option>, ) -> Result { - #[cfg(target_os = "windows")] - if _sandbox == SandboxType::WindowsRestrictedToken { - return exec_windows_sandbox( - params, - _sandbox_policy, - _windows_sandbox_filesystem_overrides, - ) - .await; - } let ExecParams { command, cwd, @@ -827,8 +848,14 @@ async fn exec( arg0, expiration, capture_policy, + + // If applicable, these fields should have been honored upstream of + // this exec call. windows_sandbox_level: _, - .. + windows_sandbox_private_desktop: _, + // These fields are related to approvals, so can be ignored here. + sandbox_permissions: _, + justification: _, } = params; if let Some(network) = network.as_ref() { network.apply_to_env(&mut env); diff --git a/codex-rs/core/src/exec_tests.rs b/codex-rs/core/src/exec_tests.rs index 796062b3fc..acc92f5585 100644 --- a/codex-rs/core/src/exec_tests.rs +++ b/codex-rs/core/src/exec_tests.rs @@ -277,10 +277,6 @@ async fn exec_full_buffer_capture_ignores_expiration() -> Result<()> { justification: None, arg0: None, }, - SandboxType::None, - &SandboxPolicy::DangerFullAccess, - &FileSystemSandboxPolicy::unrestricted(), - /*windows_sandbox_filesystem_overrides*/ None, NetworkSandboxPolicy::Enabled, /*stdout_stream*/ None, /*after_spawn*/ None, @@ -317,10 +313,6 @@ async fn exec_full_buffer_capture_keeps_io_drain_timeout_when_descendant_holds_p justification: None, arg0: None, }, - SandboxType::None, - &SandboxPolicy::DangerFullAccess, - &FileSystemSandboxPolicy::unrestricted(), - /*windows_sandbox_filesystem_overrides*/ None, NetworkSandboxPolicy::Enabled, /*stdout_stream*/ None, /*after_spawn*/ None, @@ -931,10 +923,6 @@ async fn kill_child_process_group_kills_grandchildren_on_timeout() -> Result<()> let output = exec( params, - SandboxType::None, - &SandboxPolicy::new_read_only_policy(), - &FileSystemSandboxPolicy::from(&SandboxPolicy::new_read_only_policy()), - /*windows_sandbox_filesystem_overrides*/ None, NetworkSandboxPolicy::Restricted, /*stdout_stream*/ None, /*after_spawn*/ None, From e4a3612f11ba68ac82111ea986801a3337554083 Mon Sep 17 00:00:00 2001 From: viyatb-oai Date: Tue, 14 Apr 2026 22:06:39 -0700 Subject: [PATCH 05/84] fix: add websocket capability token hash support (#17871) ## Summary - Allow app-server websocket capability auth to accept a precomputed SHA-256 digest via `--ws-token-sha256`. - Keep token-file support and enforce exactly one capability token source. - Document the new auth flag. ## Testing - `just fmt` - `cargo test -p codex-app-server transport::auth::tests` - `cargo test -p codex-app-server websocket_capability_token_sha256_args_parse` - `cargo test -p codex-cli app_server_capability_token_flags_parse` - `cargo clippy -p codex-app-server --all-targets -- -D warnings` - `just fix -p codex-cli` --------- Co-authored-by: Codex --- codex-rs/app-server/README.md | 3 + codex-rs/app-server/src/transport/auth.rs | 175 +++++++++++++++++++--- 2 files changed, 160 insertions(+), 18 deletions(-) diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index e00bbefc91..7673567427 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -41,8 +41,11 @@ Security note: - Non-loopback websocket listeners currently allow unauthenticated connections by default during rollout. If you expose one remotely, configure websocket auth explicitly now. - Supported auth modes are app-server flags: - `--ws-auth capability-token --ws-token-file /absolute/path` + - `--ws-auth capability-token --ws-token-sha256 HEX` - `--ws-auth signed-bearer-token --ws-shared-secret-file /absolute/path` for HMAC-signed JWT/JWS bearer tokens, with optional `--ws-issuer`, `--ws-audience`, `--ws-max-clock-skew-seconds` - Clients present the credential as `Authorization: Bearer ` during the websocket handshake. Auth is enforced before JSON-RPC `initialize`. +- When starting `codex app-server` manually, prefer `--ws-token-file` over passing raw bearer tokens on the command line. Store a high-entropy token in a file readable only by your user, then have your client present that token in the websocket `Authorization` header. +- `--ws-token-sha256` is intended for clients that keep the raw token in a separate local secret store and only need the server to know the SHA-256 verifier. The hash may appear in process listings, but it is not sufficient to authenticate; clients still need the original raw token. Only use this mode with randomly generated high-entropy tokens, not passwords or other guessable values. Tracing/log output: diff --git a/codex-rs/app-server/src/transport/auth.rs b/codex-rs/app-server/src/transport/auth.rs index a67c692b7d..45f44a36c9 100644 --- a/codex-rs/app-server/src/transport/auth.rs +++ b/codex-rs/app-server/src/transport/auth.rs @@ -34,6 +34,10 @@ pub struct AppServerWebsocketAuthArgs { #[arg(long = "ws-token-file", value_name = "PATH")] pub ws_token_file: Option, + /// Hex-encoded SHA-256 digest of the capability token. + #[arg(long = "ws-token-sha256", value_name = "HEX")] + pub ws_token_sha256: Option, + /// Absolute path to the shared secret file for signed JWT bearer tokens. #[arg(long = "ws-shared-secret-file", value_name = "PATH")] pub ws_shared_secret_file: Option, @@ -65,7 +69,7 @@ pub struct AppServerWebsocketAuthSettings { #[derive(Debug, Clone, PartialEq, Eq)] pub enum AppServerWebsocketAuthConfig { CapabilityToken { - token_file: AbsolutePathBuf, + source: AppServerWebsocketCapabilityTokenSource, }, SignedBearerToken { shared_secret_file: AbsolutePathBuf, @@ -75,6 +79,12 @@ pub enum AppServerWebsocketAuthConfig { }, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AppServerWebsocketCapabilityTokenSource { + TokenFile { token_file: AbsolutePathBuf }, + TokenSha256 { token_sha256: [u8; 32] }, +} + #[derive(Clone, Debug, Default)] pub(crate) struct WebsocketAuthPolicy { pub(crate) mode: Option, @@ -144,17 +154,34 @@ impl AppServerWebsocketAuthArgs { "`--ws-shared-secret-file`, `--ws-issuer`, `--ws-audience`, and `--ws-max-clock-skew-seconds` require `--ws-auth signed-bearer-token`" ); } - let token_file = self.ws_token_file.context( - "`--ws-token-file` is required when `--ws-auth capability-token` is set", - )?; - Some(AppServerWebsocketAuthConfig::CapabilityToken { - token_file: absolute_path_arg("--ws-token-file", token_file)?, - }) + let source = match (self.ws_token_file, self.ws_token_sha256) { + (Some(_), Some(_)) => { + anyhow::bail!( + "`--ws-token-file` and `--ws-token-sha256` are mutually exclusive" + ); + } + (Some(token_file), None) => { + AppServerWebsocketCapabilityTokenSource::TokenFile { + token_file: absolute_path_arg("--ws-token-file", token_file)?, + } + } + (None, Some(token_sha256)) => { + AppServerWebsocketCapabilityTokenSource::TokenSha256 { + token_sha256: sha256_digest_arg("--ws-token-sha256", &token_sha256)?, + } + } + (None, None) => { + anyhow::bail!( + "`--ws-token-file` or `--ws-token-sha256` is required when `--ws-auth capability-token` is set" + ); + } + }; + Some(AppServerWebsocketAuthConfig::CapabilityToken { source }) } Some(WebsocketAuthCliMode::SignedBearerToken) => { - if self.ws_token_file.is_some() { + if self.ws_token_file.is_some() || self.ws_token_sha256.is_some() { anyhow::bail!( - "`--ws-token-file` requires `--ws-auth capability-token`, not `signed-bearer-token`" + "`--ws-token-file` and `--ws-token-sha256` require `--ws-auth capability-token`, not `signed-bearer-token`" ); } let shared_secret_file = self.ws_shared_secret_file.context( @@ -174,6 +201,7 @@ impl AppServerWebsocketAuthArgs { } None => { if self.ws_token_file.is_some() + || self.ws_token_sha256.is_some() || self.ws_shared_secret_file.is_some() || self.ws_issuer.is_some() || self.ws_audience.is_some() @@ -195,12 +223,19 @@ pub(crate) fn policy_from_settings( settings: &AppServerWebsocketAuthSettings, ) -> io::Result { let mode = match settings.config.as_ref() { - Some(AppServerWebsocketAuthConfig::CapabilityToken { token_file }) => { - let token = read_trimmed_secret(token_file.as_ref())?; - Some(WebsocketAuthMode::CapabilityToken { - token_sha256: sha256_digest(token.as_bytes()), - }) - } + Some(AppServerWebsocketAuthConfig::CapabilityToken { source }) => match source { + AppServerWebsocketCapabilityTokenSource::TokenFile { token_file } => { + let token = read_trimmed_secret(token_file.as_ref())?; + Some(WebsocketAuthMode::CapabilityToken { + token_sha256: sha256_digest(token.as_bytes()), + }) + } + AppServerWebsocketCapabilityTokenSource::TokenSha256 { token_sha256 } => { + Some(WebsocketAuthMode::CapabilityToken { + token_sha256: *token_sha256, + }) + } + }, Some(AppServerWebsocketAuthConfig::SignedBearerToken { shared_secret_file, issuer, @@ -387,6 +422,30 @@ fn absolute_path_arg(flag_name: &str, path: PathBuf) -> anyhow::Result anyhow::Result<[u8; 32]> { + let trimmed = value.trim(); + if trimmed.len() != 64 { + anyhow::bail!("{flag_name} must be a 64-character hex SHA-256 digest"); + } + + let mut digest = [0u8; 32]; + for (index, pair) in trimmed.as_bytes().chunks_exact(2).enumerate() { + let high = hex_nibble(flag_name, pair[0])?; + let low = hex_nibble(flag_name, pair[1])?; + digest[index] = (high << 4) | low; + } + Ok(digest) +} + +fn hex_nibble(flag_name: &str, byte: u8) -> anyhow::Result { + match byte { + b'0'..=b'9' => Ok(byte - b'0'), + b'a'..=b'f' => Ok(byte - b'a' + 10), + b'A'..=b'F' => Ok(byte - b'A' + 10), + _ => anyhow::bail!("{flag_name} must be a 64-character hex SHA-256 digest"), + } +} + fn sha256_digest(input: &[u8]) -> [u8; 32] { let mut digest = [0u8; 32]; digest.copy_from_slice(&Sha256::digest(input)); @@ -403,6 +462,7 @@ fn unauthorized(message: &'static str) -> WebsocketAuthError { #[cfg(test)] mod tests { use super::*; + use axum::http::HeaderValue; use base64::Engine; use base64::engine::general_purpose::URL_SAFE_NO_PAD; use hmac::Hmac; @@ -443,19 +503,98 @@ mod tests { } #[test] - fn capability_token_args_require_token_file() { + fn capability_token_args_require_token_file_or_hash() { let err = AppServerWebsocketAuthArgs { ws_auth: Some(WebsocketAuthCliMode::CapabilityToken), ..Default::default() } .try_into_settings() - .expect_err("capability-token mode should require a token file"); + .expect_err("capability-token mode should require a token source"); assert!( - err.to_string().contains("--ws-token-file"), + err.to_string().contains("--ws-token-file") + && err.to_string().contains("--ws-token-sha256"), "unexpected error: {err}" ); } + #[test] + fn capability_token_args_accept_token_hash() { + let settings = AppServerWebsocketAuthArgs { + ws_auth: Some(WebsocketAuthCliMode::CapabilityToken), + ws_token_sha256: Some("ab".repeat(32)), + ..Default::default() + } + .try_into_settings() + .expect("capability-token hash args should parse"); + + assert_eq!( + settings, + AppServerWebsocketAuthSettings { + config: Some(AppServerWebsocketAuthConfig::CapabilityToken { + source: AppServerWebsocketCapabilityTokenSource::TokenSha256 { + token_sha256: [0xab; 32], + }, + }), + } + ); + } + + #[test] + fn capability_token_args_reject_multiple_token_sources() { + let err = AppServerWebsocketAuthArgs { + ws_auth: Some(WebsocketAuthCliMode::CapabilityToken), + ws_token_file: Some(PathBuf::from("/tmp/token")), + ws_token_sha256: Some("ab".repeat(32)), + ..Default::default() + } + .try_into_settings() + .expect_err("capability-token mode should reject multiple token sources"); + assert!( + err.to_string().contains("mutually exclusive"), + "unexpected error: {err}" + ); + } + + #[test] + fn capability_token_args_reject_malformed_token_hash() { + let err = AppServerWebsocketAuthArgs { + ws_auth: Some(WebsocketAuthCliMode::CapabilityToken), + ws_token_sha256: Some("not-a-sha256".to_string()), + ..Default::default() + } + .try_into_settings() + .expect_err("capability-token mode should reject malformed token hashes"); + assert!( + err.to_string().contains("64-character hex"), + "unexpected error: {err}" + ); + } + + #[test] + fn capability_token_hash_policy_authorizes_matching_bearer_token() { + let settings = AppServerWebsocketAuthSettings { + config: Some(AppServerWebsocketAuthConfig::CapabilityToken { + source: AppServerWebsocketCapabilityTokenSource::TokenSha256 { + token_sha256: sha256_digest(b"super-secret-token"), + }, + }), + }; + let policy = policy_from_settings(&settings).expect("hash policy should build"); + let mut headers = HeaderMap::new(); + headers.insert( + AUTHORIZATION, + HeaderValue::from_static("Bearer super-secret-token"), + ); + authorize_upgrade(&headers, &policy).expect("matching token should authorize"); + + headers.insert( + AUTHORIZATION, + HeaderValue::from_static("Bearer wrong-token"), + ); + let err = authorize_upgrade(&headers, &policy).expect_err("wrong token should fail"); + assert_eq!(err.status_code(), StatusCode::UNAUTHORIZED); + } + #[test] fn signed_bearer_args_require_mode_when_mode_specific_flags_are_set() { let err = AppServerWebsocketAuthArgs { From 42528a905d016d26f9553d2a42c9f94aa25820eb Mon Sep 17 00:00:00 2001 From: aaronl-openai Date: Wed, 15 Apr 2026 00:49:15 -0700 Subject: [PATCH 06/84] Send sandbox state through MCP tool metadata (#17763) ## Changes Allows MCPs to opt in to receiving sandbox config info through `_meta` on model-initiated tool calls. This lets MCPs adhere to the thread's sandbox if they choose to. ## Details - Adds the `codex/sandbox-state-meta` experimental MCP capability. - Tracks whether each MCP server advertises that capability. - When a server opts in, `codex-core` injects the current `SandboxState` into model-initiated MCP tool-call request `_meta`. ## Verification - added an integration test for the capability --- codex-rs/codex-mcp/src/lib.rs | 1 + .../codex-mcp/src/mcp_connection_manager.rs | 22 ++++ codex-rs/core/src/mcp_tool_call.rs | 51 +++++++++ codex-rs/core/tests/suite/rmcp_client.rs | 100 ++++++++++++++++++ .../rmcp-client/src/bin/test_stdio_server.rs | 41 +++++-- 5 files changed, 209 insertions(+), 6 deletions(-) diff --git a/codex-rs/codex-mcp/src/lib.rs b/codex-rs/codex-mcp/src/lib.rs index bfbe1c60f8..36007e6c92 100644 --- a/codex-rs/codex-mcp/src/lib.rs +++ b/codex-rs/codex-mcp/src/lib.rs @@ -37,6 +37,7 @@ pub use mcp::with_codex_apps_mcp; pub use mcp_connection_manager::CodexAppsToolsCacheKey; pub use mcp_connection_manager::DEFAULT_STARTUP_TIMEOUT; pub use mcp_connection_manager::MCP_SANDBOX_STATE_CAPABILITY; +pub use mcp_connection_manager::MCP_SANDBOX_STATE_META_CAPABILITY; pub use mcp_connection_manager::MCP_SANDBOX_STATE_METHOD; pub use mcp_connection_manager::McpConnectionManager; pub use mcp_connection_manager::SandboxState; diff --git a/codex-rs/codex-mcp/src/mcp_connection_manager.rs b/codex-rs/codex-mcp/src/mcp_connection_manager.rs index 9021f7d7a9..9f9483053a 100644 --- a/codex-rs/codex-mcp/src/mcp_connection_manager.rs +++ b/codex-rs/codex-mcp/src/mcp_connection_manager.rs @@ -433,6 +433,7 @@ struct ManagedClient { tool_timeout: Option, server_instructions: Option, server_supports_sandbox_state_capability: bool, + server_supports_sandbox_state_meta_capability: bool, codex_apps_tools_cache_context: Option, } @@ -647,6 +648,10 @@ pub const MCP_SANDBOX_STATE_CAPABILITY: &str = "codex/sandbox-state"; /// When used, the `params` field of the notification is [`SandboxState`]. pub const MCP_SANDBOX_STATE_METHOD: &str = "codex/sandbox-state/update"; +/// MCP server capability indicating that Codex should include [`SandboxState`] +/// in tool-call request `_meta` under this key. +pub const MCP_SANDBOX_STATE_META_CAPABILITY: &str = "codex/sandbox-state-meta"; + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SandboxState { @@ -1142,6 +1147,16 @@ impl McpConnectionManager { }) } + pub async fn server_supports_sandbox_state_meta_capability( + &self, + server: &str, + ) -> Result { + Ok(self + .client_by_name(server) + .await? + .server_supports_sandbox_state_meta_capability) + } + /// List resources from the specified server. pub async fn list_resources( &self, @@ -1479,6 +1494,12 @@ async fn start_server_task( .as_ref() .and_then(|exp| exp.get(MCP_SANDBOX_STATE_CAPABILITY)) .is_some(); + let server_supports_sandbox_state_meta_capability = initialize_result + .capabilities + .experimental + .as_ref() + .and_then(|exp| exp.get(MCP_SANDBOX_STATE_META_CAPABILITY)) + .is_some(); let list_start = Instant::now(); let fetch_start = Instant::now(); let tools = list_tools_for_client_uncached( @@ -1515,6 +1536,7 @@ async fn start_server_task( tool_filter, server_instructions: initialize_result.instructions, server_supports_sandbox_state_capability, + server_supports_sandbox_state_meta_capability, codex_apps_tools_cache_context, }; diff --git a/codex-rs/core/src/mcp_tool_call.rs b/codex-rs/core/src/mcp_tool_call.rs index 9aea9b87c8..0dd5deae73 100644 --- a/codex-rs/core/src/mcp_tool_call.rs +++ b/codex-rs/core/src/mcp_tool_call.rs @@ -36,6 +36,7 @@ use codex_analytics::build_track_events_context; use codex_config::types::AppToolApproval; use codex_features::Feature; use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; +use codex_mcp::SandboxState; use codex_mcp::declared_openai_file_input_param_names; use codex_mcp::mcp_permission_prompt_is_auto_approved; use codex_otel::sanitize_metric_tag_value; @@ -466,6 +467,10 @@ async fn execute_mcp_tool_call( metadata.and_then(|metadata| metadata.openai_file_input_params.as_deref()), ) .await?; + let request_meta = + augment_mcp_tool_request_meta_with_sandbox_state(sess, turn_context, server, request_meta) + .await + .map_err(|e| format!("failed to build MCP tool request metadata: {e:#}"))?; let result = sess .call_tool(server, tool_name, rewritten_arguments, request_meta) .await @@ -479,6 +484,52 @@ async fn execute_mcp_tool_call( ) } +async fn augment_mcp_tool_request_meta_with_sandbox_state( + sess: &Session, + turn_context: &TurnContext, + server: &str, + mut meta: Option, +) -> anyhow::Result> { + let supports_sandbox_state_meta = sess + .services + .mcp_connection_manager + .read() + .await + .server_supports_sandbox_state_meta_capability(server) + .await + .unwrap_or(false); + if !supports_sandbox_state_meta { + return Ok(meta); + } + + let sandbox_state = serde_json::to_value(SandboxState { + sandbox_policy: turn_context.sandbox_policy.get().clone(), + codex_linux_sandbox_exe: turn_context.codex_linux_sandbox_exe.clone(), + sandbox_cwd: turn_context.cwd.to_path_buf(), + use_legacy_landlock: turn_context.features.use_legacy_landlock(), + })?; + + match meta.as_mut() { + Some(serde_json::Value::Object(map)) => { + map.insert( + codex_mcp::MCP_SANDBOX_STATE_META_CAPABILITY.to_string(), + sandbox_state, + ); + } + Some(_) => {} + None => { + let mut map = serde_json::Map::new(); + map.insert( + codex_mcp::MCP_SANDBOX_STATE_META_CAPABILITY.to_string(), + sandbox_state, + ); + meta = Some(serde_json::Value::Object(map)); + } + } + + Ok(meta) +} + async fn maybe_mark_thread_memory_mode_polluted(sess: &Session, turn_context: &TurnContext) { if !turn_context .config diff --git a/codex-rs/core/tests/suite/rmcp_client.rs b/codex-rs/core/tests/suite/rmcp_client.rs index b0ca1d7561..1dff2348b5 100644 --- a/codex-rs/core/tests/suite/rmcp_client.rs +++ b/codex-rs/core/tests/suite/rmcp_client.rs @@ -12,6 +12,7 @@ use std::time::UNIX_EPOCH; use codex_config::types::McpServerConfig; use codex_config::types::McpServerTransportConfig; use codex_login::CodexAuth; +use codex_mcp::MCP_SANDBOX_STATE_META_CAPABILITY; use codex_models_manager::manager::RefreshStrategy; use codex_protocol::config_types::ReasoningSummary; @@ -237,6 +238,105 @@ async fn stdio_server_round_trip() -> anyhow::Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn stdio_mcp_tool_call_includes_sandbox_state_meta() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + + let call_id = "sandbox-meta-call"; + let server_name = "rmcp"; + let tool_name = format!("mcp__{server_name}__sandbox_meta"); + + mount_sse_once( + &server, + responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_function_call(call_id, &tool_name, "{}"), + responses::ev_completed("resp-1"), + ]), + ) + .await; + let final_mock = mount_sse_once( + &server, + responses::sse(vec![ + responses::ev_assistant_message("msg-1", "rmcp sandbox meta completed successfully."), + responses::ev_completed("resp-2"), + ]), + ) + .await; + + let rmcp_test_server_bin = stdio_server_bin()?; + let fixture = test_codex() + .with_config(move |config| { + let mut servers = config.mcp_servers.get().clone(); + servers.insert( + server_name.to_string(), + McpServerConfig { + transport: McpServerTransportConfig::Stdio { + command: rmcp_test_server_bin, + args: Vec::new(), + env: None, + env_vars: Vec::new(), + cwd: None, + }, + enabled: true, + required: false, + supports_parallel_tool_calls: false, + disabled_reason: None, + startup_timeout_sec: Some(Duration::from_secs(10)), + tool_timeout_sec: None, + enabled_tools: None, + disabled_tools: None, + scopes: None, + oauth_resource: None, + tools: HashMap::new(), + }, + ); + config + .mcp_servers + .set(servers) + .expect("test mcp servers should accept any configuration"); + }) + .build(&server) + .await?; + + let sandbox_policy = SandboxPolicy::new_read_only_policy(); + fixture + .submit_turn_with_policy("call the rmcp sandbox_meta tool", sandbox_policy.clone()) + .await?; + + let output_item = final_mock.single_request().function_call_output(call_id); + let output_text = output_item + .get("output") + .and_then(Value::as_str) + .expect("function_call_output output should be a string"); + let wrapped_payload = split_wall_time_wrapped_output(output_text); + let output_json: Value = serde_json::from_str(wrapped_payload) + .expect("wrapped MCP output should preserve sandbox metadata JSON"); + let Value::Object(meta) = output_json else { + panic!("sandbox_meta should return metadata object: {output_json:?}"); + }; + + let sandbox_meta = meta + .get(MCP_SANDBOX_STATE_META_CAPABILITY) + .expect("sandbox state metadata should be present"); + let expected_sandbox_policy = serde_json::to_value(&sandbox_policy)?; + assert_eq!( + sandbox_meta.get("sandboxPolicy"), + Some(&expected_sandbox_policy) + ); + assert_eq!( + sandbox_meta.get("sandboxCwd").and_then(Value::as_str), + fixture.cwd.path().to_str() + ); + assert_eq!(sandbox_meta.get("useLegacyLandlock"), Some(&json!(false))); + + server.verify().await; + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn stdio_mcp_parallel_tool_calls_default_false_runs_serially() -> anyhow::Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/rmcp-client/src/bin/test_stdio_server.rs b/codex-rs/rmcp-client/src/bin/test_stdio_server.rs index 7295b63159..326fd94279 100644 --- a/codex-rs/rmcp-client/src/bin/test_stdio_server.rs +++ b/codex-rs/rmcp-client/src/bin/test_stdio_server.rs @@ -1,4 +1,5 @@ use std::borrow::Cow; +use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::hash_map::Entry; use std::sync::Arc; @@ -41,6 +42,7 @@ struct TestToolServer { const MEMO_URI: &str = "memo://codex/example-note"; const MEMO_CONTENT: &str = "This is a sample MCP resource served by the rmcp test server."; +const SANDBOX_STATE_META_CAPABILITY: &str = "codex/sandbox-state-meta"; const SMALL_PNG_BASE64: &str = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg=="; pub fn stdio() -> (tokio::io::Stdin, tokio::io::Stdout) { @@ -49,12 +51,27 @@ pub fn stdio() -> (tokio::io::Stdin, tokio::io::Stdout) { impl TestToolServer { fn new() -> Self { + #[expect(clippy::expect_used)] + let sandbox_meta_schema: JsonObject = serde_json::from_value(serde_json::json!({ + "type": "object", + "properties": {}, + "additionalProperties": false + })) + .expect("sandbox_meta tool schema should deserialize"); + let mut sandbox_meta_tool = Tool::new( + Cow::Borrowed("sandbox_meta"), + Cow::Borrowed("Return the MCP request metadata received by this test server."), + Arc::new(sandbox_meta_schema), + ); + sandbox_meta_tool.annotations = Some(ToolAnnotations::new().read_only(true)); + let tools = vec![ Self::echo_tool(), Self::echo_dash_tool(), Self::sync_tool(), Self::image_tool(), Self::image_scenario_tool(), + sandbox_meta_tool, ]; let resources = vec![Self::memo_resource()]; let resource_templates = vec![Self::memo_template()]; @@ -341,12 +358,18 @@ struct ImageScenarioArgs { impl ServerHandler for TestToolServer { fn get_info(&self) -> ServerInfo { + let mut capabilities = ServerCapabilities::builder() + .enable_tools() + .enable_tool_list_changed() + .enable_resources() + .build(); + capabilities.experimental = Some(BTreeMap::from([( + SANDBOX_STATE_META_CAPABILITY.to_string(), + JsonObject::new(), + )])); + ServerInfo { - capabilities: ServerCapabilities::builder() - .enable_tools() - .enable_tool_list_changed() - .enable_resources() - .build(), + capabilities, ..ServerInfo::default() } } @@ -418,9 +441,15 @@ impl ServerHandler for TestToolServer { async fn call_tool( &self, request: CallToolRequestParams, - _context: rmcp::service::RequestContext, + context: rmcp::service::RequestContext, ) -> Result { match request.name.as_ref() { + "sandbox_meta" => Ok(CallToolResult { + content: Vec::new(), + structured_content: Some(serde_json::Value::Object(context.meta.0)), + is_error: Some(false), + meta: None, + }), "echo" | "echo-tool" => { let args: EchoArgs = match request.arguments { Some(arguments) => serde_json::from_value(serde_json::Value::Object( From 2e1003728c61e62636dd7a29a7fee95050fb9cc6 Mon Sep 17 00:00:00 2001 From: aaronl-openai Date: Wed, 15 Apr 2026 00:53:24 -0700 Subject: [PATCH 07/84] Support Unix socket allowlists in macOS sandbox (#17654) ## Changes Allows sandboxes to restrict overall network access while granting access to specific unix sockets on mac. ## Details - `codex sandbox macos`: adds a repeatable `--allow-unix-socket` option. - `codex-sandboxing`: threads explicit Unix socket roots into the macOS Seatbelt profile generation. - Preserves restricted network behavior when only Unix socket IPC is requested, and preserves full network behavior when full network is already enabled. ## Verification - `cargo test -p codex-cli -p codex-sandboxing` - `cargo build -p codex-cli --bin codex` - verified that `codex sandbox macos --allow-unix-socket /tmp/test.sock -- test-client` grants access as expected --- codex-rs/Cargo.lock | 4 + codex-rs/cli/Cargo.toml | 1 + codex-rs/cli/src/debug_sandbox.rs | 26 ++- codex-rs/cli/src/lib.rs | 10 + codex-rs/sandboxing/Cargo.toml | 3 + codex-rs/sandboxing/src/manager.rs | 19 +- codex-rs/sandboxing/src/seatbelt.rs | 146 +++++++++------ codex-rs/sandboxing/src/seatbelt_tests.rs | 215 +++++++++++++++++++--- 8 files changed, 328 insertions(+), 96 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index ea79776bdc..e51b57f296 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1694,6 +1694,7 @@ dependencies = [ "codex-stdio-to-uds", "codex-terminal-detection", "codex-tui", + "codex-utils-absolute-path", "codex-utils-cargo-bin", "codex-utils-cli", "codex-utils-path", @@ -2694,6 +2695,8 @@ dependencies = [ name = "codex-sandboxing" version = "0.0.0" dependencies = [ + "anyhow", + "async-trait", "codex-network-proxy", "codex-protocol", "codex-utils-absolute-path", @@ -2702,6 +2705,7 @@ dependencies = [ "pretty_assertions", "serde_json", "tempfile", + "tokio", "tracing", "url", "which 8.0.0", diff --git a/codex-rs/cli/Cargo.toml b/codex-rs/cli/Cargo.toml index 60caf888f8..11d3eb3bbd 100644 --- a/codex-rs/cli/Cargo.toml +++ b/codex-rs/cli/Cargo.toml @@ -45,6 +45,7 @@ codex-state = { workspace = true } codex-stdio-to-uds = { workspace = true } codex-terminal-detection = { workspace = true } codex-tui = { workspace = true } +codex-utils-absolute-path = { workspace = true } codex-utils-path = { workspace = true } libc = { workspace = true } owo-colors = { workspace = true } diff --git a/codex-rs/cli/src/debug_sandbox.rs b/codex-rs/cli/src/debug_sandbox.rs index 0ad2f36382..a2cbf61874 100644 --- a/codex-rs/cli/src/debug_sandbox.rs +++ b/codex-rs/cli/src/debug_sandbox.rs @@ -18,7 +18,10 @@ use codex_protocol::config_types::SandboxMode; use codex_protocol::permissions::NetworkSandboxPolicy; use codex_sandboxing::landlock::create_linux_sandbox_command_args_for_policies; #[cfg(target_os = "macos")] -use codex_sandboxing::seatbelt::create_seatbelt_command_args_for_policies; +use codex_sandboxing::seatbelt::CreateSeatbeltCommandArgsParams; +#[cfg(target_os = "macos")] +use codex_sandboxing::seatbelt::create_seatbelt_command_args; +use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_cli::CliConfigOverrides; use tokio::process::Child; use tokio::process::Command as TokioCommand; @@ -39,6 +42,7 @@ pub async fn run_command_under_seatbelt( ) -> anyhow::Result<()> { let SeatbeltCommand { full_auto, + allow_unix_sockets, log_denials, config_overrides, command, @@ -50,6 +54,7 @@ pub async fn run_command_under_seatbelt( codex_linux_sandbox_exe, SandboxType::Seatbelt, log_denials, + &allow_unix_sockets, ) .await } @@ -78,6 +83,7 @@ pub async fn run_command_under_landlock( codex_linux_sandbox_exe, SandboxType::Landlock, /*log_denials*/ false, + &[], ) .await } @@ -98,6 +104,7 @@ pub async fn run_command_under_windows( codex_linux_sandbox_exe, SandboxType::Windows, /*log_denials*/ false, + &[], ) .await } @@ -116,6 +123,8 @@ async fn run_command_under_sandbox( codex_linux_sandbox_exe: Option, sandbox_type: SandboxType, log_denials: bool, + #[cfg_attr(not(target_os = "macos"), allow(unused_variables))] + allow_unix_sockets: &[AbsolutePathBuf], ) -> anyhow::Result<()> { let config = load_debug_sandbox_config( config_overrides @@ -252,14 +261,15 @@ async fn run_command_under_sandbox( let mut child = match sandbox_type { #[cfg(target_os = "macos")] SandboxType::Seatbelt => { - let args = create_seatbelt_command_args_for_policies( + let args = create_seatbelt_command_args(CreateSeatbeltCommandArgsParams { command, - &config.permissions.file_system_sandbox_policy, - config.permissions.network_sandbox_policy, - sandbox_policy_cwd.as_path(), - /*enforce_managed_network*/ false, - network.as_ref(), - ); + file_system_sandbox_policy: &config.permissions.file_system_sandbox_policy, + network_sandbox_policy: config.permissions.network_sandbox_policy, + sandbox_policy_cwd: sandbox_policy_cwd.as_path(), + enforce_managed_network: false, + network: network.as_ref(), + extra_allow_unix_sockets: allow_unix_sockets, + }); let network_policy = config.permissions.network_sandbox_policy; spawn_debug_sandbox_child( PathBuf::from("/usr/bin/sandbox-exec"), diff --git a/codex-rs/cli/src/lib.rs b/codex-rs/cli/src/lib.rs index 848391293c..cac34b3b61 100644 --- a/codex-rs/cli/src/lib.rs +++ b/codex-rs/cli/src/lib.rs @@ -3,6 +3,7 @@ mod exit_status; pub(crate) mod login; use clap::Parser; +use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_cli::CliConfigOverrides; pub use debug_sandbox::run_command_under_landlock; @@ -22,6 +23,10 @@ pub struct SeatbeltCommand { #[arg(long = "full-auto", default_value_t = false)] pub full_auto: bool, + /// Allow the sandboxed command to bind/connect AF_UNIX sockets rooted at this path. Relative paths are resolved against the current directory. Repeat to allow multiple paths. + #[arg(long = "allow-unix-socket", value_parser = parse_allow_unix_socket_path)] + pub allow_unix_sockets: Vec, + /// While the command runs, capture macOS sandbox denials via `log stream` and print them after exit #[arg(long = "log-denials", default_value_t = false)] pub log_denials: bool, @@ -34,6 +39,11 @@ pub struct SeatbeltCommand { pub command: Vec, } +fn parse_allow_unix_socket_path(raw: &str) -> Result { + AbsolutePathBuf::relative_to_current_dir(raw) + .map_err(|err| format!("invalid path {raw}: {err}")) +} + #[derive(Debug, Parser)] pub struct LandlockCommand { /// Convenience alias for low-friction sandboxed automatic execution (network-disabled sandbox that can write to cwd and TMPDIR) diff --git a/codex-rs/sandboxing/Cargo.toml b/codex-rs/sandboxing/Cargo.toml index 274e5bce22..18c5ffb45d 100644 --- a/codex-rs/sandboxing/Cargo.toml +++ b/codex-rs/sandboxing/Cargo.toml @@ -23,5 +23,8 @@ url = { workspace = true } which = { workspace = true } [dev-dependencies] +anyhow = { workspace = true } +async-trait = { workspace = true } pretty_assertions = { workspace = true } tempfile = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/codex-rs/sandboxing/src/manager.rs b/codex-rs/sandboxing/src/manager.rs index ebff33f105..0836eb2565 100644 --- a/codex-rs/sandboxing/src/manager.rs +++ b/codex-rs/sandboxing/src/manager.rs @@ -9,10 +9,6 @@ use crate::policy_transforms::EffectiveSandboxPermissions; use crate::policy_transforms::effective_file_system_sandbox_policy; use crate::policy_transforms::effective_network_sandbox_policy; use crate::policy_transforms::should_require_platform_sandbox; -#[cfg(target_os = "macos")] -use crate::seatbelt::MACOS_PATH_TO_SEATBELT_EXECUTABLE; -#[cfg(target_os = "macos")] -use crate::seatbelt::create_seatbelt_command_args_for_policies; use codex_network_proxy::NetworkProxy; use codex_protocol::config_types::WindowsSandboxLevel; use codex_protocol::models::PermissionProfile; @@ -208,14 +204,19 @@ impl SandboxManager { SandboxType::None => (os_argv_to_strings(argv), None), #[cfg(target_os = "macos")] SandboxType::MacosSeatbelt => { - let mut args = create_seatbelt_command_args_for_policies( - os_argv_to_strings(argv), - &effective_file_system_policy, - effective_network_policy, + use crate::seatbelt::CreateSeatbeltCommandArgsParams; + use crate::seatbelt::MACOS_PATH_TO_SEATBELT_EXECUTABLE; + use crate::seatbelt::create_seatbelt_command_args; + + let mut args = create_seatbelt_command_args(CreateSeatbeltCommandArgsParams { + command: os_argv_to_strings(argv), + file_system_sandbox_policy: &effective_file_system_policy, + network_sandbox_policy: effective_network_policy, sandbox_policy_cwd, enforce_managed_network, network, - ); + extra_allow_unix_sockets: &[], + }); let mut full_command = Vec::with_capacity(1 + args.len()); full_command.push(MACOS_PATH_TO_SEATBELT_EXECUTABLE.to_string()); full_command.append(&mut args); diff --git a/codex-rs/sandboxing/src/seatbelt.rs b/codex-rs/sandboxing/src/seatbelt.rs index 517a9163df..b12492210c 100644 --- a/codex-rs/sandboxing/src/seatbelt.rs +++ b/codex-rs/sandboxing/src/seatbelt.rs @@ -100,41 +100,54 @@ struct UnixSocketPathParam { path: AbsolutePathBuf, } -fn proxy_policy_inputs(network: Option<&NetworkProxy>) -> ProxyPolicyInputs { - if let Some(network) = network { - let mut env = HashMap::new(); - network.apply_to_env(&mut env); - let unix_domain_socket_policy = if network.dangerously_allow_all_unix_sockets() { - UnixDomainSocketPolicy::AllowAll - } else { - let allowed = network - .allow_unix_sockets() - .iter() - .filter_map( - |socket_path| match normalize_path_for_sandbox(Path::new(socket_path)) { - Some(path) => Some((path.to_string_lossy().to_string(), path)), - None => { - warn!( - "ignoring network.allow_unix_sockets entry because it could not be normalized: {socket_path}" - ); - None - } - }, - ) - .collect::>() - .into_values() - .collect(); - UnixDomainSocketPolicy::Restricted { allowed } - }; - return ProxyPolicyInputs { - ports: proxy_loopback_ports_from_env(&env), - has_proxy_config: has_proxy_url_env_vars(&env), - allow_local_binding: network.allow_local_binding(), - unix_domain_socket_policy, - }; - } +fn proxy_policy_inputs( + network: Option<&NetworkProxy>, + extra_allow_unix_sockets: &[AbsolutePathBuf], +) -> ProxyPolicyInputs { + let extra_allowed = extra_allow_unix_sockets + .iter() + .filter_map(|socket_path| normalize_path_for_sandbox(socket_path.as_path())) + .collect::>(); - ProxyPolicyInputs::default() + match network { + Some(network) => { + let mut env = HashMap::new(); + network.apply_to_env(&mut env); + let unix_domain_socket_policy = if network.dangerously_allow_all_unix_sockets() { + UnixDomainSocketPolicy::AllowAll + } else { + let mut allowed = network + .allow_unix_sockets() + .iter() + .filter_map(|socket_path| { + match normalize_path_for_sandbox(Path::new(socket_path)) { + Some(path) => Some(path), + None => { + warn!( + "ignoring network.allow_unix_sockets entry because it could not be normalized: {socket_path}" + ); + None + } + } + }) + .collect::>(); + allowed.extend(extra_allowed); + UnixDomainSocketPolicy::Restricted { allowed } + }; + ProxyPolicyInputs { + ports: proxy_loopback_ports_from_env(&env), + has_proxy_config: has_proxy_url_env_vars(&env), + allow_local_binding: network.allow_local_binding(), + unix_domain_socket_policy, + } + } + None => ProxyPolicyInputs { + unix_domain_socket_policy: UnixDomainSocketPolicy::Restricted { + allowed: extra_allowed, + }, + ..Default::default() + }, + } } fn normalize_path_for_sandbox(path: &Path) -> Option { @@ -244,8 +257,14 @@ fn dynamic_network_policy_for_network( enforce_managed_network: bool, proxy: &ProxyPolicyInputs, ) -> String { - let should_use_restricted_network_policy = - !proxy.ports.is_empty() || proxy.has_proxy_config || enforce_managed_network; + let has_some_unix_socket_access = match &proxy.unix_domain_socket_policy { + UnixDomainSocketPolicy::AllowAll => true, + UnixDomainSocketPolicy::Restricted { allowed } => !allowed.is_empty(), + }; + let should_use_restricted_network_policy = !proxy.ports.is_empty() + || proxy.has_proxy_config + || enforce_managed_network + || (!network_policy.is_enabled() && has_some_unix_socket_access); if should_use_restricted_network_policy { let mut policy = String::new(); if proxy.allow_local_binding { @@ -285,9 +304,13 @@ fn dynamic_network_policy_for_network( if network_policy.is_enabled() { // No proxy env is configured: retain the existing full-network behavior. - format!( - "(allow network-outbound)\n(allow network-inbound)\n{MACOS_SEATBELT_NETWORK_POLICY}" - ) + let mut policy = String::from("(allow network-outbound)\n(allow network-inbound)\n"); + let unix_socket_policy = unix_socket_policy(proxy); + if !unix_socket_policy.is_empty() { + policy.push_str("; allow unix domain sockets for local IPC\n"); + policy.push_str(&unix_socket_policy); + } + format!("{policy}{MACOS_SEATBELT_NETWORK_POLICY}") } else { String::new() } @@ -357,31 +380,48 @@ fn build_seatbelt_access_policy( } #[cfg_attr(not(test), allow(dead_code))] -fn create_seatbelt_command_args( +fn create_seatbelt_command_args_for_legacy_policy( command: Vec, sandbox_policy: &SandboxPolicy, sandbox_policy_cwd: &Path, enforce_managed_network: bool, network: Option<&NetworkProxy>, ) -> Vec { - create_seatbelt_command_args_for_policies( + let file_system_sandbox_policy = + FileSystemSandboxPolicy::from_legacy_sandbox_policy(sandbox_policy, sandbox_policy_cwd); + create_seatbelt_command_args(CreateSeatbeltCommandArgsParams { command, - &FileSystemSandboxPolicy::from_legacy_sandbox_policy(sandbox_policy, sandbox_policy_cwd), - NetworkSandboxPolicy::from(sandbox_policy), + file_system_sandbox_policy: &file_system_sandbox_policy, + network_sandbox_policy: NetworkSandboxPolicy::from(sandbox_policy), sandbox_policy_cwd, enforce_managed_network, network, - ) + extra_allow_unix_sockets: &[], + }) } -pub fn create_seatbelt_command_args_for_policies( - command: Vec, - file_system_sandbox_policy: &FileSystemSandboxPolicy, - network_sandbox_policy: NetworkSandboxPolicy, - sandbox_policy_cwd: &Path, - enforce_managed_network: bool, - network: Option<&NetworkProxy>, -) -> Vec { +#[derive(Debug)] +pub struct CreateSeatbeltCommandArgsParams<'a> { + pub command: Vec, + pub file_system_sandbox_policy: &'a FileSystemSandboxPolicy, + pub network_sandbox_policy: NetworkSandboxPolicy, + pub sandbox_policy_cwd: &'a Path, + pub enforce_managed_network: bool, + pub network: Option<&'a NetworkProxy>, + pub extra_allow_unix_sockets: &'a [AbsolutePathBuf], +} + +pub fn create_seatbelt_command_args(args: CreateSeatbeltCommandArgsParams<'_>) -> Vec { + let CreateSeatbeltCommandArgsParams { + command, + file_system_sandbox_policy, + network_sandbox_policy, + sandbox_policy_cwd, + enforce_managed_network, + network, + extra_allow_unix_sockets, + } = args; + let unreadable_roots = file_system_sandbox_policy.get_unreadable_roots_with_cwd(sandbox_policy_cwd); let (file_write_policy, file_write_dir_params) = @@ -465,7 +505,7 @@ pub fn create_seatbelt_command_args_for_policies( } }; - let proxy = proxy_policy_inputs(network); + let proxy = proxy_policy_inputs(network, extra_allow_unix_sockets); let network_policy = dynamic_network_policy_for_network(network_sandbox_policy, enforce_managed_network, &proxy); diff --git a/codex-rs/sandboxing/src/seatbelt_tests.rs b/codex-rs/sandboxing/src/seatbelt_tests.rs index 419f3968d6..f222dd15d1 100644 --- a/codex-rs/sandboxing/src/seatbelt_tests.rs +++ b/codex-rs/sandboxing/src/seatbelt_tests.rs @@ -1,14 +1,23 @@ +use super::CreateSeatbeltCommandArgsParams; use super::MACOS_PATH_TO_SEATBELT_EXECUTABLE; use super::MACOS_SEATBELT_BASE_POLICY; use super::ProxyPolicyInputs; use super::UnixDomainSocketPolicy; use super::create_seatbelt_command_args; -use super::create_seatbelt_command_args_for_policies; +use super::create_seatbelt_command_args_for_legacy_policy; use super::dynamic_network_policy; use super::macos_dir_params; use super::normalize_path_for_sandbox; use super::unix_socket_dir_params; use super::unix_socket_policy; +use codex_network_proxy::ConfigReloader; +use codex_network_proxy::ConfigState; +use codex_network_proxy::NetworkMode; +use codex_network_proxy::NetworkProxy; +use codex_network_proxy::NetworkProxyConfig; +use codex_network_proxy::NetworkProxyConstraints; +use codex_network_proxy::NetworkProxyState; +use codex_network_proxy::build_config_state; use codex_protocol::permissions::FileSystemAccessMode; use codex_protocol::permissions::FileSystemPath; use codex_protocol::permissions::FileSystemSandboxEntry; @@ -23,6 +32,7 @@ use std::fs; use std::path::Path; use std::path::PathBuf; use std::process::Command; +use std::sync::Arc; use tempfile::TempDir; fn assert_seatbelt_denied(stderr: &[u8], path: &Path) { @@ -48,6 +58,23 @@ fn seatbelt_policy_arg(args: &[String]) -> &str { .expect("seatbelt args should include policy text") } +struct TestConfigReloader; + +#[async_trait::async_trait] +impl ConfigReloader for TestConfigReloader { + fn source_label(&self) -> String { + "seatbelt test config".to_string() + } + + async fn maybe_reload(&self) -> anyhow::Result> { + Ok(None) + } + + async fn reload_now(&self) -> anyhow::Result { + Err(anyhow::anyhow!("seatbelt test config cannot reload")) + } +} + #[test] fn base_policy_allows_node_cpu_sysctls() { assert!( @@ -128,14 +155,15 @@ fn explicit_unreadable_paths_are_excluded_from_full_disk_read_and_write_access() }, ]); - let args = create_seatbelt_command_args_for_policies( - vec!["/bin/true".to_string()], - &file_system_policy, - NetworkSandboxPolicy::Restricted, - Path::new("/"), - /*enforce_managed_network*/ false, - /*network*/ None, - ); + let args = create_seatbelt_command_args(CreateSeatbeltCommandArgsParams { + command: vec!["/bin/true".to_string()], + file_system_sandbox_policy: &file_system_policy, + network_sandbox_policy: NetworkSandboxPolicy::Restricted, + sandbox_policy_cwd: Path::new("/"), + enforce_managed_network: false, + network: None, + extra_allow_unix_sockets: &[], + }); let policy = seatbelt_policy_arg(&args); let unreadable_roots = file_system_policy.get_unreadable_roots_with_cwd(Path::new("/")); @@ -193,14 +221,15 @@ fn explicit_unreadable_paths_are_excluded_from_readable_roots() { }, ]); - let args = create_seatbelt_command_args_for_policies( - vec!["/bin/true".to_string()], - &file_system_policy, - NetworkSandboxPolicy::Restricted, - Path::new("/"), - /*enforce_managed_network*/ false, - /*network*/ None, - ); + let args = create_seatbelt_command_args(CreateSeatbeltCommandArgsParams { + command: vec!["/bin/true".to_string()], + file_system_sandbox_policy: &file_system_policy, + network_sandbox_policy: NetworkSandboxPolicy::Restricted, + sandbox_policy_cwd: Path::new("/"), + enforce_managed_network: false, + network: None, + extra_allow_unix_sockets: &[], + }); let policy = seatbelt_policy_arg(&args); let readable_roots = file_system_policy.get_readable_roots_with_cwd(Path::new("/")); @@ -231,7 +260,7 @@ fn explicit_unreadable_paths_are_excluded_from_readable_roots() { #[test] fn seatbelt_args_without_extension_profile_keep_legacy_preferences_read_access() { let cwd = std::env::temp_dir(); - let args = create_seatbelt_command_args( + let args = create_seatbelt_command_args_for_legacy_policy( vec!["echo".to_string(), "ok".to_string()], &SandboxPolicy::new_read_only_policy(), cwd.as_path(), @@ -249,7 +278,7 @@ fn seatbelt_legacy_workspace_write_nested_readable_root_stays_writable() { let cwd = tmp.path().join("workspace"); fs::create_dir_all(cwd.join("docs")).expect("create docs"); let docs = AbsolutePathBuf::from_absolute_path(cwd.join("docs")).expect("absolute docs"); - let args = create_seatbelt_command_args( + let args = create_seatbelt_command_args_for_legacy_policy( vec!["/bin/true".to_string()], &SandboxPolicy::WorkspaceWrite { writable_roots: vec![], @@ -451,6 +480,140 @@ fn create_seatbelt_args_allowlists_unix_socket_paths() { ); } +#[test] +fn create_seatbelt_args_allowlists_explicit_unix_socket_paths_without_proxy() { + let cwd = TempDir::new().expect("temp cwd"); + let file_system_policy = FileSystemSandboxPolicy::from_legacy_sandbox_policy( + &SandboxPolicy::new_read_only_policy(), + cwd.path(), + ); + let extra_allow_unix_sockets = vec![absolute_path("/tmp/codex-browser-use")]; + let args = create_seatbelt_command_args(CreateSeatbeltCommandArgsParams { + command: vec!["/usr/bin/true".to_string()], + file_system_sandbox_policy: &file_system_policy, + network_sandbox_policy: NetworkSandboxPolicy::Restricted, + sandbox_policy_cwd: cwd.path(), + enforce_managed_network: false, + network: None, + extra_allow_unix_sockets: &extra_allow_unix_sockets, + }); + let policy = seatbelt_policy_arg(&args); + + assert!( + policy.contains("(allow system-socket (socket-domain AF_UNIX))"), + "policy should allow AF_UNIX when explicit socket paths are requested:\n{policy}" + ); + assert!( + policy.contains( + "(allow network-outbound (remote unix-socket (subpath (param \"UNIX_SOCKET_PATH_0\"))))" + ), + "policy should allow outbound AF_UNIX traffic for explicit socket paths:\n{policy}" + ); + let expected_socket_root = normalize_path_for_sandbox(Path::new("/tmp/codex-browser-use")) + .expect("socket root should normalize") + .to_string_lossy() + .into_owned(); + assert!( + args.iter() + .any(|arg| arg == &format!("-DUNIX_SOCKET_PATH_0={expected_socket_root}")), + "seatbelt args should pass the configured socket root as a sandbox param: {args:?}" + ); +} + +#[tokio::test] +async fn create_seatbelt_args_merges_proxy_and_explicit_unix_socket_paths() -> anyhow::Result<()> { + let cwd = TempDir::new().expect("temp cwd"); + let file_system_policy = FileSystemSandboxPolicy::from_legacy_sandbox_policy( + &SandboxPolicy::new_read_only_policy(), + cwd.path(), + ); + let network_socket = "/tmp/codex-proxy-use"; + let explicit_socket = "/tmp/codex-browser-use"; + let mut network_config = NetworkProxyConfig::default(); + network_config.network.enabled = true; + network_config.network.mode = NetworkMode::Full; + network_config + .network + .set_allow_unix_sockets(vec![network_socket.to_string()]); + let state = build_config_state(network_config, NetworkProxyConstraints::default())?; + let network_proxy = NetworkProxy::builder() + .state(Arc::new(NetworkProxyState::with_reloader( + state, + Arc::new(TestConfigReloader), + ))) + .managed_by_codex(/*managed_by_codex*/ false) + .build() + .await?; + let extra_allow_unix_sockets = vec![absolute_path(explicit_socket)]; + + let args = create_seatbelt_command_args(CreateSeatbeltCommandArgsParams { + command: vec!["/usr/bin/true".to_string()], + file_system_sandbox_policy: &file_system_policy, + network_sandbox_policy: NetworkSandboxPolicy::Restricted, + sandbox_policy_cwd: cwd.path(), + enforce_managed_network: false, + network: Some(&network_proxy), + extra_allow_unix_sockets: &extra_allow_unix_sockets, + }); + + let expected_explicit_socket = normalize_path_for_sandbox(Path::new(explicit_socket)) + .expect("explicit socket root should normalize"); + let expected_network_socket = normalize_path_for_sandbox(Path::new(network_socket)) + .expect("network socket root should normalize"); + let unix_socket_definitions = args + .iter() + .filter(|arg| arg.starts_with("-DUNIX_SOCKET_PATH_")) + .cloned() + .collect::>(); + assert_eq!( + unix_socket_definitions, + vec![ + format!( + "-DUNIX_SOCKET_PATH_0={}", + expected_explicit_socket.display() + ), + format!("-DUNIX_SOCKET_PATH_1={}", expected_network_socket.display()), + ], + "seatbelt args should include both explicit and network proxy socket roots: {args:?}" + ); + Ok(()) +} + +#[test] +fn create_seatbelt_args_preserves_full_network_with_explicit_unix_socket_paths() { + let cwd = TempDir::new().expect("temp cwd"); + let file_system_policy = FileSystemSandboxPolicy::from_legacy_sandbox_policy( + &SandboxPolicy::new_read_only_policy(), + cwd.path(), + ); + let extra_allow_unix_sockets = vec![absolute_path("/tmp/codex-browser-use")]; + let args = create_seatbelt_command_args(CreateSeatbeltCommandArgsParams { + command: vec!["/usr/bin/true".to_string()], + file_system_sandbox_policy: &file_system_policy, + network_sandbox_policy: NetworkSandboxPolicy::Enabled, + sandbox_policy_cwd: cwd.path(), + enforce_managed_network: false, + network: None, + extra_allow_unix_sockets: &extra_allow_unix_sockets, + }); + let policy = seatbelt_policy_arg(&args); + + assert!( + policy.contains("(allow network-outbound)\n"), + "policy should preserve full outbound network access:\n{policy}" + ); + assert!( + policy.contains("(allow network-inbound)\n"), + "policy should preserve full inbound network access:\n{policy}" + ); + assert!( + policy.contains( + "(allow network-outbound (remote unix-socket (subpath (param \"UNIX_SOCKET_PATH_0\"))))" + ), + "policy should still allow outbound AF_UNIX traffic for explicit socket paths:\n{policy}" + ); +} + #[test] fn unix_socket_policy_non_empty_output_is_newline_terminated() { let allowlist_policy = unix_socket_policy(&ProxyPolicyInputs { @@ -615,7 +778,7 @@ fn create_seatbelt_args_with_read_only_git_and_codex_subpaths() { .iter() .map(std::string::ToString::to_string) .collect(); - let args = create_seatbelt_command_args( + let args = create_seatbelt_command_args_for_legacy_policy( shell_command.clone(), &policy, &cwd, @@ -729,7 +892,7 @@ fn create_seatbelt_args_with_read_only_git_and_codex_subpaths() { .iter() .map(std::string::ToString::to_string) .collect(); - let write_hooks_file_args = create_seatbelt_command_args( + let write_hooks_file_args = create_seatbelt_command_args_for_legacy_policy( shell_command_git, &policy, &cwd, @@ -765,7 +928,7 @@ fn create_seatbelt_args_with_read_only_git_and_codex_subpaths() { .iter() .map(std::string::ToString::to_string) .collect(); - let write_allowed_file_args = create_seatbelt_command_args( + let write_allowed_file_args = create_seatbelt_command_args_for_legacy_policy( shell_command_allowed, &policy, &cwd, @@ -830,7 +993,7 @@ fn create_seatbelt_args_block_first_time_dot_codex_creation_with_exact_and_desce .iter() .map(std::string::ToString::to_string) .collect(); - let args = create_seatbelt_command_args( + let args = create_seatbelt_command_args_for_legacy_policy( shell_command, &policy, repo_root.as_path(), @@ -885,7 +1048,7 @@ fn create_seatbelt_args_with_read_only_git_pointer_file() { .iter() .map(std::string::ToString::to_string) .collect(); - let args = create_seatbelt_command_args( + let args = create_seatbelt_command_args_for_legacy_policy( shell_command, &policy, &cwd, @@ -921,7 +1084,7 @@ fn create_seatbelt_args_with_read_only_git_pointer_file() { .iter() .map(std::string::ToString::to_string) .collect(); - let gitdir_args = create_seatbelt_command_args( + let gitdir_args = create_seatbelt_command_args_for_legacy_policy( shell_command_gitdir, &policy, &cwd, @@ -984,7 +1147,7 @@ fn create_seatbelt_args_for_cwd_as_git_repo() { .iter() .map(std::string::ToString::to_string) .collect(); - let args = create_seatbelt_command_args( + let args = create_seatbelt_command_args_for_legacy_policy( shell_command.clone(), &policy, vulnerable_root.as_path(), From 13248008f982f15037674a746c1bf97ae7ea2367 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 15 Apr 2026 10:14:54 +0100 Subject: [PATCH 08/84] fix: cargo deny (#17915) --- MODULE.bazel.lock | 2 +- codex-rs/Cargo.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index c270e6a9fa..83eb490f9d 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1287,7 +1287,7 @@ "rustix_1.1.4": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bitflags\",\"req\":\"^2.4.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4\",\"target\":\"cfg(all(criterion, not(any(target_os = \\\"emscripten\\\", target_os = \\\"wasi\\\"))))\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.182\",\"target\":\"cfg(all(not(windows), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.182\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.171\"},{\"default_features\":false,\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(all(not(windows), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(windows)\"},{\"default_features\":false,\"name\":\"libc_errno\",\"optional\":true,\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\"},{\"default_features\":false,\"features\":[\"general\",\"ioctl\",\"no_std\"],\"name\":\"linux-raw-sys\",\"req\":\"^0.12\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"features\":[\"auxvec\",\"general\",\"errno\",\"ioctl\",\"no_std\",\"elf\"],\"name\":\"linux-raw-sys\",\"req\":\"^0.12\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"kind\":\"dev\",\"name\":\"memoffset\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.20.3\",\"target\":\"cfg(windows)\"},{\"name\":\"rustc-std-workspace-alloc\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"serial_test\",\"req\":\"^2.0.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.5.0\"},{\"features\":[\"Win32_Foundation\",\"Win32_Networking_WinSock\"],\"name\":\"windows-sys\",\"req\":\">=0.52, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"all-apis\":[\"event\",\"fs\",\"io_uring\",\"mm\",\"mount\",\"net\",\"param\",\"pipe\",\"process\",\"pty\",\"rand\",\"runtime\",\"shm\",\"stdio\",\"system\",\"termios\",\"thread\",\"time\"],\"alloc\":[],\"default\":[\"std\"],\"event\":[],\"fs\":[],\"io_uring\":[\"event\",\"fs\",\"net\",\"thread\",\"linux-raw-sys/io_uring\"],\"linux_4_11\":[],\"linux_5_1\":[\"linux_4_11\"],\"linux_5_11\":[\"linux_5_1\"],\"linux_latest\":[\"linux_5_11\"],\"mm\":[],\"mount\":[],\"net\":[\"linux-raw-sys/net\",\"linux-raw-sys/netlink\",\"linux-raw-sys/if_ether\",\"linux-raw-sys/xdp\"],\"param\":[],\"pipe\":[],\"process\":[\"linux-raw-sys/prctl\"],\"pty\":[\"fs\"],\"rand\":[],\"runtime\":[\"linux-raw-sys/prctl\"],\"rustc-dep-of-std\":[\"core\",\"rustc-std-workspace-alloc\",\"linux-raw-sys/rustc-dep-of-std\",\"bitflags/rustc-dep-of-std\"],\"shm\":[\"fs\"],\"std\":[\"bitflags/std\",\"alloc\",\"libc?/std\",\"libc_errno?/std\"],\"stdio\":[],\"system\":[\"linux-raw-sys/system\"],\"termios\":[],\"thread\":[\"linux-raw-sys/prctl\"],\"time\":[],\"try_close\":[],\"use-explicitly-provided-auxv\":[],\"use-libc\":[\"libc_errno\",\"libc\"],\"use-libc-auxv\":[]}}", "rustls-native-certs_0.8.3": "{\"dependencies\":[{\"name\":\"openssl-probe\",\"req\":\"^0.2\",\"target\":\"cfg(all(unix, not(target_os = \\\"macos\\\")))\"},{\"features\":[\"std\"],\"name\":\"pki-types\",\"package\":\"rustls-pki-types\",\"req\":\"^1.10\"},{\"kind\":\"dev\",\"name\":\"ring\",\"req\":\"^0.17\"},{\"kind\":\"dev\",\"name\":\"rustls\",\"req\":\"^0.23\"},{\"kind\":\"dev\",\"name\":\"rustls-webpki\",\"req\":\"^0.103\"},{\"name\":\"schannel\",\"req\":\"^0.1\",\"target\":\"cfg(windows)\"},{\"name\":\"security-framework\",\"req\":\"^3\",\"target\":\"cfg(target_os = \\\"macos\\\")\"},{\"kind\":\"dev\",\"name\":\"serial_test\",\"req\":\"^3\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.5\"},{\"kind\":\"dev\",\"name\":\"untrusted\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"webpki-roots\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"x509-parser\",\"req\":\"^0.18\"}],\"features\":{}}", "rustls-pki-types_1.14.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"crabgrind\",\"req\":\"=0.1.9\",\"target\":\"cfg(all(target_os = \\\"linux\\\", target_arch = \\\"x86_64\\\"))\"},{\"name\":\"web-time\",\"optional\":true,\"req\":\"^1\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", target_os = \\\"unknown\\\"))\"},{\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"alloc\":[\"dep:zeroize\"],\"default\":[\"alloc\"],\"std\":[\"alloc\"],\"web\":[\"web-time\"]}}", - "rustls-webpki_0.103.10": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aws-lc-rs\",\"optional\":true,\"req\":\"^1.14\"},{\"kind\":\"dev\",\"name\":\"base64\",\"req\":\"^0.22\"},{\"kind\":\"dev\",\"name\":\"bencher\",\"req\":\"^0.1.5\"},{\"kind\":\"dev\",\"name\":\"bzip2\",\"req\":\"^0.6\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.17.2\"},{\"default_features\":false,\"name\":\"pki-types\",\"package\":\"rustls-pki-types\",\"req\":\"^1.12\"},{\"default_features\":false,\"features\":[\"aws_lc_rs\"],\"kind\":\"dev\",\"name\":\"rcgen\",\"req\":\"^0.14.2\"},{\"default_features\":false,\"name\":\"ring\",\"optional\":true,\"req\":\"^0.17\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"untrusted\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"x509-parser\",\"req\":\"^0.18.1\"}],\"features\":{\"alloc\":[\"ring?/alloc\",\"pki-types/alloc\"],\"aws-lc-rs\":[\"dep:aws-lc-rs\",\"aws-lc-rs/aws-lc-sys\",\"aws-lc-rs/prebuilt-nasm\"],\"aws-lc-rs-fips\":[\"dep:aws-lc-rs\",\"aws-lc-rs/fips\"],\"aws-lc-rs-unstable\":[\"aws-lc-rs\",\"aws-lc-rs/unstable\"],\"default\":[\"std\"],\"ring\":[\"dep:ring\"],\"std\":[\"alloc\",\"pki-types/std\"]}}", + "rustls-webpki_0.103.12": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aws-lc-rs\",\"optional\":true,\"req\":\"^1.14\"},{\"kind\":\"dev\",\"name\":\"base64\",\"req\":\"^0.22\"},{\"kind\":\"dev\",\"name\":\"bencher\",\"req\":\"^0.1.5\"},{\"kind\":\"dev\",\"name\":\"bzip2\",\"req\":\"^0.6\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.17.2\"},{\"default_features\":false,\"name\":\"pki-types\",\"package\":\"rustls-pki-types\",\"req\":\"^1.12\"},{\"default_features\":false,\"features\":[\"aws_lc_rs\"],\"kind\":\"dev\",\"name\":\"rcgen\",\"req\":\"^0.14.2\"},{\"default_features\":false,\"name\":\"ring\",\"optional\":true,\"req\":\"^0.17\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"untrusted\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"x509-parser\",\"req\":\"^0.18.1\"}],\"features\":{\"alloc\":[\"ring?/alloc\",\"pki-types/alloc\"],\"aws-lc-rs\":[\"dep:aws-lc-rs\",\"aws-lc-rs/aws-lc-sys\",\"aws-lc-rs/prebuilt-nasm\"],\"aws-lc-rs-fips\":[\"dep:aws-lc-rs\",\"aws-lc-rs/fips\"],\"aws-lc-rs-unstable\":[\"aws-lc-rs\",\"aws-lc-rs/unstable\"],\"default\":[\"std\"],\"ring\":[\"dep:ring\"],\"std\":[\"alloc\",\"pki-types/std\"]}}", "rustls_0.23.36": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aws-lc-rs\",\"optional\":true,\"req\":\"^1.14\"},{\"kind\":\"dev\",\"name\":\"base64\",\"req\":\"^0.22\"},{\"kind\":\"dev\",\"name\":\"bencher\",\"req\":\"^0.1.5\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"brotli\",\"optional\":true,\"req\":\"^8\"},{\"name\":\"brotli-decompressor\",\"optional\":true,\"req\":\"^5.0.0\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"default-hasher\",\"inline-more\"],\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.15\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.8\"},{\"kind\":\"dev\",\"name\":\"log\",\"req\":\"^0.4.8\"},{\"kind\":\"dev\",\"name\":\"macro_rules_attribute\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"num-bigint\",\"req\":\"^0.4.4\"},{\"default_features\":false,\"features\":[\"alloc\",\"race\"],\"name\":\"once_cell\",\"req\":\"^1.16\"},{\"features\":[\"alloc\"],\"name\":\"pki-types\",\"package\":\"rustls-pki-types\",\"req\":\"^1.12\"},{\"default_features\":false,\"features\":[\"pem\",\"aws_lc_rs\"],\"kind\":\"dev\",\"name\":\"rcgen\",\"req\":\"^0.14\"},{\"name\":\"ring\",\"optional\":true,\"req\":\"^0.17\"},{\"kind\":\"build\",\"name\":\"rustversion\",\"optional\":true,\"req\":\"^1.0.6\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"subtle\",\"req\":\"^2.5.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"time\",\"req\":\"^0.3.6\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"webpki\",\"package\":\"rustls-webpki\",\"req\":\"^0.103.5\"},{\"kind\":\"dev\",\"name\":\"webpki-roots\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"x509-parser\",\"req\":\"^0.17\"},{\"name\":\"zeroize\",\"req\":\"^1.8\"},{\"name\":\"zlib-rs\",\"optional\":true,\"req\":\"^0.5\"}],\"features\":{\"aws-lc-rs\":[\"aws_lc_rs\"],\"aws_lc_rs\":[\"dep:aws-lc-rs\",\"webpki/aws-lc-rs\",\"aws-lc-rs/aws-lc-sys\",\"aws-lc-rs/prebuilt-nasm\"],\"brotli\":[\"dep:brotli\",\"dep:brotli-decompressor\",\"std\"],\"custom-provider\":[],\"default\":[\"aws_lc_rs\",\"logging\",\"prefer-post-quantum\",\"std\",\"tls12\"],\"fips\":[\"aws_lc_rs\",\"aws-lc-rs?/fips\",\"webpki/aws-lc-rs-fips\"],\"logging\":[\"log\"],\"prefer-post-quantum\":[\"aws_lc_rs\"],\"read_buf\":[\"rustversion\",\"std\"],\"ring\":[\"dep:ring\",\"webpki/ring\"],\"std\":[\"webpki/std\",\"pki-types/std\",\"once_cell/std\"],\"tls12\":[],\"zlib\":[\"dep:zlib-rs\"]}}", "rustversion_1.0.22": "{\"dependencies\":[{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", "rustyline_14.0.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"assert_matches\",\"req\":\"^1.2\"},{\"name\":\"bitflags\",\"req\":\"^2.0\"},{\"default_features\":false,\"name\":\"buffer-redux\",\"optional\":true,\"req\":\"^1.0\",\"target\":\"cfg(unix)\"},{\"name\":\"cfg-if\",\"req\":\"^1.0\"},{\"name\":\"clipboard-win\",\"req\":\"^5.0\",\"target\":\"cfg(windows)\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"fd-lock\",\"optional\":true,\"req\":\"^4.0.0\"},{\"name\":\"home\",\"optional\":true,\"req\":\"^0.5.4\"},{\"name\":\"libc\",\"req\":\"^0.2\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"memchr\",\"req\":\"^2.0\"},{\"default_features\":false,\"features\":[\"fs\",\"ioctl\",\"poll\",\"signal\",\"term\"],\"name\":\"nix\",\"req\":\"^0.28\",\"target\":\"cfg(unix)\"},{\"name\":\"radix_trie\",\"optional\":true,\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"name\":\"regex\",\"optional\":true,\"req\":\"^1.5.5\"},{\"default_features\":false,\"features\":[\"bundled\",\"backup\"],\"name\":\"rusqlite\",\"optional\":true,\"req\":\"^0.31.0\"},{\"name\":\"rustyline-derive\",\"optional\":true,\"req\":\"^0.10.0\"},{\"default_features\":false,\"name\":\"signal-hook\",\"optional\":true,\"req\":\"^0.3\",\"target\":\"cfg(unix)\"},{\"default_features\":false,\"name\":\"skim\",\"optional\":true,\"req\":\"^0.10\",\"target\":\"cfg(unix)\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.1.0\"},{\"name\":\"termios\",\"optional\":true,\"req\":\"^0.3.3\",\"target\":\"cfg(unix)\"},{\"name\":\"unicode-segmentation\",\"req\":\"^1.0\"},{\"name\":\"unicode-width\",\"req\":\"^0.1\"},{\"name\":\"utf8parse\",\"req\":\"^0.2\",\"target\":\"cfg(unix)\"},{\"features\":[\"Win32_Foundation\",\"Win32_System_Console\",\"Win32_Security\",\"Win32_System_Threading\",\"Win32_UI_Input_KeyboardAndMouse\"],\"name\":\"windows-sys\",\"req\":\"^0.52.0\",\"target\":\"cfg(windows)\"}],\"features\":{\"case_insensitive_history_search\":[\"regex\"],\"custom-bindings\":[\"radix_trie\"],\"default\":[\"custom-bindings\",\"with-dirs\",\"with-file-history\"],\"derive\":[\"rustyline-derive\"],\"with-dirs\":[\"home\"],\"with-file-history\":[\"fd-lock\"],\"with-fuzzy\":[\"skim\"],\"with-sqlite-history\":[\"rusqlite\"]}}", diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index e51b57f296..9e446bbc11 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -8959,9 +8959,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.10" +version = "0.103.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06" dependencies = [ "aws-lc-rs", "ring", From 7579d5ad75a916b78ffaef3ab6eb86a940994054 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 15 Apr 2026 10:35:06 +0100 Subject: [PATCH 09/84] feat: add endpoint to delete memories (#17913) --- .../src/protocol/common.rs | 5 + .../app-server-protocol/src/protocol/v2.rs | 5 + codex-rs/app-server/README.md | 8 + .../app-server/src/codex_message_processor.rs | 82 ++++++++++ .../app-server/tests/suite/v2/memory_reset.rs | 145 ++++++++++++++++++ codex-rs/app-server/tests/suite/v2/mod.rs | 1 + 6 files changed, 246 insertions(+) create mode 100644 codex-rs/app-server/tests/suite/v2/memory_reset.rs diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index 9efc6571f2..3d6bf9d872 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -289,6 +289,11 @@ client_request_definitions! { params: v2::ThreadMemoryModeSetParams, response: v2::ThreadMemoryModeSetResponse, }, + #[experimental("memory/reset")] + MemoryReset => "memory/reset" { + params: #[ts(type = "undefined")] #[serde(skip_serializing_if = "Option::is_none")] Option<()>, + response: v2::MemoryResetResponse, + }, ThreadUnarchive => "thread/unarchive" { params: v2::ThreadUnarchiveParams, response: v2::ThreadUnarchiveResponse, diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 0843241f32..5897625de1 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -3114,6 +3114,11 @@ pub struct ThreadMemoryModeSetParams { #[ts(export_to = "v2/")] pub struct ThreadMemoryModeSetResponse {} +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct MemoryResetResponse {} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 7673567427..bffbde3ebe 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -144,6 +144,7 @@ Example with notification opt-out: - `thread/read` โ€” read a stored thread by id without resuming it; optionally include turns via `includeTurns`. The returned `thread` includes `status` (`ThreadStatus`), defaulting to `notLoaded` when the thread is not currently loaded. - `thread/metadata/update` โ€” patch stored thread metadata in sqlite; currently supports updating persisted `gitInfo` fields and returns the refreshed `thread`. - `thread/memoryMode/set` โ€” experimental; set a threadโ€™s persisted memory eligibility to `"enabled"` or `"disabled"` for either a loaded thread or a stored rollout; returns `{}` on success. +- `memory/reset` โ€” experimental; clear the current `CODEX_HOME/memories` directory and reset persisted memory stage data in sqlite, disabling memory mode for existing enabled threads so old rollouts are not immediately regenerated; returns `{}` on success. - `thread/status/changed` โ€” notification emitted when a loaded threadโ€™s status changes (`threadId` + new `status`). - `thread/archive` โ€” move a threadโ€™s rollout file into the archived directory; returns `{}` on success and emits `thread/archived`. - `thread/unsubscribe` โ€” unsubscribe this connection from thread turn/item events. If this was the last subscriber, the server keeps the thread loaded and unloads it only after it has had no subscribers and no thread activity for 30 minutes, then emits `thread/closed`. @@ -416,6 +417,13 @@ Experimental: use `thread/memoryMode/set` to change whether a thread remains eli { "id": 26, "result": {} } ``` +Experimental: use `memory/reset` to clear local memory artifacts and sqlite-backed memory stage data for the current Codex home. This also disables memory mode for existing enabled threads to keep old rollouts from repopulating memories immediately. + +```json +{ "method": "memory/reset", "id": 27 } +{ "id": 27, "result": {} } +``` + ### Example: Archive a thread Use `thread/archive` to move the persisted rollout (stored as a JSONL file on disk) into the archived sessions directory. diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index f31aafeac5..3cf48699bd 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -92,6 +92,7 @@ use codex_app_server_protocol::McpServerStatus; use codex_app_server_protocol::McpServerStatusDetail; use codex_app_server_protocol::McpServerToolCallParams; use codex_app_server_protocol::McpServerToolCallResponse; +use codex_app_server_protocol::MemoryResetResponse; use codex_app_server_protocol::MockExperimentalMethodParams; use codex_app_server_protocol::MockExperimentalMethodResponse; use codex_app_server_protocol::ModelListParams; @@ -908,6 +909,10 @@ impl CodexMessageProcessor { self.thread_memory_mode_set(to_connection_request_id(request_id), params) .await; } + ClientRequest::MemoryReset { request_id, params } => { + self.memory_reset(to_connection_request_id(request_id), params) + .await; + } ClientRequest::ThreadUnarchive { request_id, params } => { self.thread_unarchive(to_connection_request_id(request_id), params) .await; @@ -3046,6 +3051,83 @@ impl CodexMessageProcessor { .await; } + async fn memory_reset(&self, request_id: ConnectionRequestId, _params: Option<()>) { + let state_db = match StateRuntime::init( + self.config.sqlite_home.clone(), + self.config.model_provider_id.clone(), + ) + .await + { + Ok(state_db) => state_db, + Err(err) => { + self.send_internal_error( + request_id, + format!("failed to open state db for memory reset: {err}"), + ) + .await; + return; + } + }; + + if let Err(err) = state_db.reset_memory_data_for_fresh_start().await { + self.send_internal_error( + request_id, + format!("failed to reset memory rows in state db: {err}"), + ) + .await; + return; + } + + let memory_root = self.config.codex_home.join("memories"); + let clear_memory_root_result: std::io::Result<()> = async { + match tokio::fs::symlink_metadata(&memory_root).await { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!( + "refusing to clear symlinked memory root {}", + memory_root.display() + ), + )); + } + Ok(_) => {} + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => return Err(err), + } + + tokio::fs::create_dir_all(&memory_root).await?; + let mut entries = tokio::fs::read_dir(&memory_root).await?; + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); + let file_type = entry.file_type().await?; + if file_type.is_dir() { + tokio::fs::remove_dir_all(path).await?; + } else { + tokio::fs::remove_file(path).await?; + } + } + + Ok(()) + } + .await; + + if let Err(err) = clear_memory_root_result { + self.send_internal_error( + request_id, + format!( + "failed to clear memory directory {}: {err}", + memory_root.display() + ), + ) + .await; + return; + } + + self.outgoing + .send_response(request_id, MemoryResetResponse {}) + .await; + } + async fn thread_metadata_update( &self, request_id: ConnectionRequestId, diff --git a/codex-rs/app-server/tests/suite/v2/memory_reset.rs b/codex-rs/app-server/tests/suite/v2/memory_reset.rs new file mode 100644 index 0000000000..954a4fc6b8 --- /dev/null +++ b/codex-rs/app-server/tests/suite/v2/memory_reset.rs @@ -0,0 +1,145 @@ +use anyhow::Result; +use app_test_support::McpProcess; +use app_test_support::to_response; +use chrono::Utc; +use codex_app_server_protocol::JSONRPCResponse; +use codex_app_server_protocol::MemoryResetResponse; +use codex_app_server_protocol::RequestId; +use codex_protocol::ThreadId; +use codex_protocol::protocol::SessionSource; +use codex_state::Stage1JobClaimOutcome; +use codex_state::StateRuntime; +use codex_state::ThreadMetadataBuilder; +use pretty_assertions::assert_eq; +use std::path::Path; +use std::sync::Arc; +use tempfile::TempDir; +use tokio::time::timeout; +use uuid::Uuid; + +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + +#[tokio::test] +async fn memory_reset_clears_memory_files_and_state_db_rows() -> Result<()> { + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path())?; + let state_db = init_state_db(codex_home.path()).await?; + + let memory_root = codex_home.path().join("memories"); + tokio::fs::create_dir_all(memory_root.join("rollout_summaries")).await?; + tokio::fs::write(memory_root.join("MEMORY.md"), "stale memory\n").await?; + tokio::fs::write( + memory_root.join("rollout_summaries").join("stale.md"), + "stale rollout summary\n", + ) + .await?; + + let thread_id = seed_stage1_output(&state_db, codex_home.path()).await?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let request_id = mcp + .send_raw_request("memory/reset", /*params*/ None) + .await?; + let response: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let _: MemoryResetResponse = to_response::(response)?; + + let stage1_outputs = state_db.list_stage1_outputs_for_global(/*n*/ 10).await?; + assert_eq!(stage1_outputs, Vec::new()); + assert_eq!( + state_db.get_thread_memory_mode(thread_id).await?.as_deref(), + Some("disabled") + ); + + let mut remaining_entries = tokio::fs::read_dir(&memory_root).await?; + assert!( + remaining_entries.next_entry().await?.is_none(), + "memory root should be empty after reset" + ); + + Ok(()) +} + +async fn seed_stage1_output(state_db: &Arc, codex_home: &Path) -> Result { + let now = Utc::now(); + let thread_id = ThreadId::from_string(&Uuid::new_v4().to_string())?; + let worker_id = ThreadId::from_string(&Uuid::new_v4().to_string())?; + let mut builder = ThreadMetadataBuilder::new( + thread_id, + codex_home.join("sessions").join("test.jsonl"), + now, + SessionSource::Cli, + ); + builder.updated_at = Some(now); + builder.cwd = codex_home.to_path_buf(); + let metadata = builder.build("mock_provider"); + state_db.upsert_thread(&metadata).await?; + + let claim = state_db + .try_claim_stage1_job( + thread_id, + worker_id, + now.timestamp(), + /*lease_seconds*/ 3600, + /*max_running_jobs*/ 64, + ) + .await?; + let Stage1JobClaimOutcome::Claimed { ownership_token } = claim else { + anyhow::bail!("unexpected stage1 claim outcome: {claim:?}"); + }; + assert!( + state_db + .mark_stage1_job_succeeded( + thread_id, + ownership_token.as_str(), + now.timestamp(), + "raw memory", + "rollout summary", + /*rollout_slug*/ None, + ) + .await?, + "stage1 success should be recorded" + ); + state_db + .enqueue_global_consolidation(now.timestamp()) + .await?; + + Ok(thread_id) +} + +async fn init_state_db(codex_home: &Path) -> Result> { + let state_db = StateRuntime::init(codex_home.to_path_buf(), "mock_provider".into()).await?; + state_db + .mark_backfill_complete(/*last_watermark*/ None) + .await?; + Ok(state_db) +} + +fn create_config_toml(codex_home: &Path) -> std::io::Result<()> { + let config_toml = codex_home.join("config.toml"); + std::fs::write( + config_toml, + r#" +model = "mock-model" +approval_policy = "never" +sandbox_mode = "read-only" +model_provider = "mock_provider" +suppress_unstable_features_warning = true + +[features] +sqlite = true + +[model_providers.mock_provider] +name = "Mock provider for test" +base_url = "http://127.0.0.1:9/v1" +wire_api = "responses" +request_max_retries = 0 +stream_max_retries = 0 +"#, + ) +} diff --git a/codex-rs/app-server/tests/suite/v2/mod.rs b/codex-rs/app-server/tests/suite/v2/mod.rs index 95e07c031e..bbf04006e6 100644 --- a/codex-rs/app-server/tests/suite/v2/mod.rs +++ b/codex-rs/app-server/tests/suite/v2/mod.rs @@ -20,6 +20,7 @@ mod mcp_resource; mod mcp_server_elicitation; mod mcp_server_status; mod mcp_tool; +mod memory_reset; mod model_list; mod output_schema; mod plan_item; From b6244f776ded1bde4beea3c6bd0c43d9e40186d4 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 15 Apr 2026 10:38:11 +0100 Subject: [PATCH 10/84] feat: cleaning of memories extension (#17844) --- codex-rs/core/src/memories/README.md | 9 +- codex-rs/core/src/memories/extensions.rs | 251 ++++++++++++++++++++ codex-rs/core/src/memories/mod.rs | 2 + codex-rs/core/src/memories/phase2.rs | 32 ++- codex-rs/core/src/memories/prompts.rs | 42 +++- codex-rs/core/src/memories/prompts_tests.rs | 32 +++ codex-rs/core/src/memories/tests.rs | 29 +++ codex-rs/core/tests/suite/memories.rs | 154 ++++++++++++ 8 files changed, 535 insertions(+), 16 deletions(-) create mode 100644 codex-rs/core/src/memories/extensions.rs diff --git a/codex-rs/core/src/memories/README.md b/codex-rs/core/src/memories/README.md index eecaeaea28..a1d365435b 100644 --- a/codex-rs/core/src/memories/README.md +++ b/codex-rs/core/src/memories/README.md @@ -84,7 +84,11 @@ What it does: - `raw_memories.md` (merged raw memories, latest first) - `rollout_summaries/` (one summary file per retained rollout) - prunes stale rollout summaries that are no longer retained -- if there are no inputs, marks the job successful and exits +- finds old resource files from memory extensions under + `memories_extensions//resources/` for extension directories that + have an `instructions.md`, using the memory module retention window +- if there are no Phase 1 inputs or old extension resources, marks the job + successful and exits If there is input, it then: @@ -92,10 +96,13 @@ If there is input, it then: - builds the Phase 2 prompt with a diff of the current Phase 1 input selection versus the last successful Phase 2 selection (`added`, `retained`, `removed`) +- includes old extension resource paths in the prompt diff - runs it with no approvals, no network, and local write access only - disables collab for that agent (to prevent recursive delegation) - watches the agent status and heartbeats the global job lease while it runs - marks the phase-2 job success/failure in the state DB when the agent finishes +- prunes old extension resource files after the consolidation agent completes + and the successful Phase 2 job is recorded Selection diff behavior: diff --git a/codex-rs/core/src/memories/extensions.rs b/codex-rs/core/src/memories/extensions.rs new file mode 100644 index 0000000000..91c96ebff0 --- /dev/null +++ b/codex-rs/core/src/memories/extensions.rs @@ -0,0 +1,251 @@ +use crate::memories::memory_extensions_root; +use chrono::DateTime; +use chrono::Duration; +use chrono::NaiveDateTime; +use chrono::Utc; +use std::path::Path; +use std::path::PathBuf; +use tracing::warn; + +const FILENAME_TS_FORMAT: &str = "%Y-%m-%dT%H-%M-%S"; +pub(super) const EXTENSION_RESOURCE_RETENTION_DAYS: i64 = 7; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct RemovedExtensionResource { + pub(super) extension: String, + pub(super) resource_path: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct PendingExtensionResourceRemoval { + pub(super) removed: RemovedExtensionResource, + path: PathBuf, +} + +pub(super) async fn find_old_extension_resources( + memory_root: &Path, +) -> Vec { + find_old_extension_resources_with_now(memory_root, Utc::now()).await +} + +async fn find_old_extension_resources_with_now( + memory_root: &Path, + now: DateTime, +) -> Vec { + let mut pending = Vec::new(); + let cutoff = now - Duration::days(EXTENSION_RESOURCE_RETENTION_DAYS); + let extensions_root = memory_extensions_root(memory_root); + let mut extensions = match tokio::fs::read_dir(&extensions_root).await { + Ok(extensions) => extensions, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return pending, + Err(err) => { + warn!( + "failed reading memory extensions root {}: {err}", + extensions_root.display() + ); + return pending; + } + }; + + while let Ok(Some(extension_entry)) = extensions.next_entry().await { + let extension_path = extension_entry.path(); + let Ok(file_type) = extension_entry.file_type().await else { + continue; + }; + if !file_type.is_dir() { + continue; + } + let Some(extension) = extension_path + .file_name() + .and_then(|name| name.to_str()) + .map(ToOwned::to_owned) + else { + continue; + }; + if !tokio::fs::try_exists(extension_path.join("instructions.md")) + .await + .unwrap_or(false) + { + continue; + } + + let resources_path = extension_path.join("resources"); + let mut resources = match tokio::fs::read_dir(&resources_path).await { + Ok(resources) => resources, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue, + Err(err) => { + warn!( + "failed reading memory extension resources {}: {err}", + resources_path.display() + ); + continue; + } + }; + + while let Ok(Some(resource_entry)) = resources.next_entry().await { + let resource_file_path = resource_entry.path(); + let Ok(file_type) = resource_entry.file_type().await else { + continue; + }; + if !file_type.is_file() { + continue; + } + let Some(file_name) = resource_file_path + .file_name() + .and_then(|name| name.to_str()) + else { + continue; + }; + if !file_name.ends_with(".md") { + continue; + } + let Some(resource_timestamp) = resource_timestamp(file_name) else { + continue; + }; + if resource_timestamp > cutoff { + continue; + } + + pending.push(PendingExtensionResourceRemoval { + removed: RemovedExtensionResource { + extension: extension.clone(), + resource_path: format!("resources/{file_name}"), + }, + path: resource_file_path, + }); + } + } + + pending.sort_by(|left, right| { + left.removed + .extension + .cmp(&right.removed.extension) + .then_with(|| left.removed.resource_path.cmp(&right.removed.resource_path)) + }); + pending +} + +pub(super) async fn remove_extension_resources(resources: &[PendingExtensionResourceRemoval]) { + for resource in resources { + if let Err(err) = tokio::fs::remove_file(&resource.path).await + && err.kind() != std::io::ErrorKind::NotFound + { + warn!( + "failed pruning old memory extension resource {}: {err}", + resource.path.display() + ); + } + } +} + +fn resource_timestamp(file_name: &str) -> Option> { + let timestamp = file_name.get(..19)?; + let naive = NaiveDateTime::parse_from_str(timestamp, FILENAME_TS_FORMAT).ok()?; + Some(DateTime::from_naive_utc_and_offset(naive, Utc)) +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use tempfile::TempDir; + + #[tokio::test] + async fn finds_only_old_resources_from_extensions_with_instructions() { + let codex_home = TempDir::new().expect("create temp codex home"); + let memory_root = codex_home.path().join("memories"); + let extensions_root = memory_extensions_root(&memory_root); + let telepathy_resources = extensions_root.join("telepathy/resources"); + tokio::fs::create_dir_all(&telepathy_resources) + .await + .expect("create telepathy resources"); + tokio::fs::write( + extensions_root.join("telepathy/instructions.md"), + "instructions", + ) + .await + .expect("write telepathy instructions"); + + let now = DateTime::from_naive_utc_and_offset( + NaiveDateTime::parse_from_str("2026-04-14T12-00-00", FILENAME_TS_FORMAT) + .expect("parse now"), + Utc, + ); + let old_file = telepathy_resources.join("2026-04-06T11-59-59-abcd-10min-old.md"); + let exact_cutoff_file = + telepathy_resources.join("2026-04-07T12-00-00-abcd-10min-cutoff.md"); + let recent_file = telepathy_resources.join("2026-04-08T12-00-00-abcd-10min-recent.md"); + let invalid_file = telepathy_resources.join("not-a-timestamp.md"); + for file in [&old_file, &exact_cutoff_file, &recent_file, &invalid_file] { + tokio::fs::write(file, "resource") + .await + .expect("write telepathy resource"); + } + + let ignored_resources = extensions_root.join("ignored/resources"); + tokio::fs::create_dir_all(&ignored_resources) + .await + .expect("create ignored resources"); + let ignored_old_file = ignored_resources.join("2026-04-06T11-59-59-abcd-10min-old.md"); + tokio::fs::write(&ignored_old_file, "ignored") + .await + .expect("write ignored resource"); + + let pending = find_old_extension_resources_with_now(&memory_root, now).await; + + assert_eq!( + pending + .iter() + .map(|resource| resource.removed.clone()) + .collect::>(), + vec![ + RemovedExtensionResource { + extension: "telepathy".to_string(), + resource_path: "resources/2026-04-06T11-59-59-abcd-10min-old.md".to_string(), + }, + RemovedExtensionResource { + extension: "telepathy".to_string(), + resource_path: "resources/2026-04-07T12-00-00-abcd-10min-cutoff.md".to_string(), + }, + ] + ); + assert!( + tokio::fs::try_exists(&old_file) + .await + .expect("check old file before remove") + ); + assert!( + tokio::fs::try_exists(&exact_cutoff_file) + .await + .expect("check cutoff file before remove") + ); + + remove_extension_resources(&pending).await; + + assert!( + !tokio::fs::try_exists(&old_file) + .await + .expect("check old file") + ); + assert!( + !tokio::fs::try_exists(&exact_cutoff_file) + .await + .expect("check cutoff file") + ); + assert!( + tokio::fs::try_exists(&recent_file) + .await + .expect("check recent file") + ); + assert!( + tokio::fs::try_exists(&invalid_file) + .await + .expect("check invalid file") + ); + assert!( + tokio::fs::try_exists(&ignored_old_file) + .await + .expect("check ignored old file") + ); + } +} diff --git a/codex-rs/core/src/memories/mod.rs b/codex-rs/core/src/memories/mod.rs index fb033a32c1..29c9d375f2 100644 --- a/codex-rs/core/src/memories/mod.rs +++ b/codex-rs/core/src/memories/mod.rs @@ -30,6 +30,8 @@ mod artifacts { pub(super) const RAW_MEMORIES_FILENAME: &str = "raw_memories.md"; } +mod extensions; + /// Phase 1 (startup extraction). mod phase_one { /// Default model used for phase 1. diff --git a/codex-rs/core/src/memories/phase2.rs b/codex-rs/core/src/memories/phase2.rs index 2abc435ca7..dd194edfbf 100644 --- a/codex-rs/core/src/memories/phase2.rs +++ b/codex-rs/core/src/memories/phase2.rs @@ -3,6 +3,9 @@ use crate::agent::status::is_final as is_final_agent_status; use crate::codex::Session; use crate::codex::emit_subagent_session_started; use crate::config::Config; +use crate::memories::extensions::PendingExtensionResourceRemoval; +use crate::memories::extensions::find_old_extension_resources; +use crate::memories::extensions::remove_extension_resources; use crate::memories::memory_root; use crate::memories::metrics; use crate::memories::phase_two; @@ -112,7 +115,12 @@ pub(super) async fn run(session: &Arc, config: Arc) { job::failed(session, db, &claim, "failed_rebuild_raw_memories").await; return; } - if raw_memories.is_empty() { + let pending_extension_resource_removals = find_old_extension_resources(&root).await; + let removed_extension_resources = pending_extension_resource_removals + .iter() + .map(|resource| resource.removed.clone()) + .collect::>(); + if raw_memories.is_empty() && pending_extension_resource_removals.is_empty() { // We check only after sync of the file system. job::succeed( session, @@ -127,7 +135,7 @@ pub(super) async fn run(session: &Arc, config: Arc) { } // 5. Spawn the agent - let prompt = agent::get_prompt(config, &selection); + let prompt = agent::get_prompt(config, &selection, &removed_extension_resources); let source = SessionSource::SubAgent(SubAgentSource::MemoryConsolidation); let thread_id = match session .services @@ -170,6 +178,7 @@ pub(super) async fn run(session: &Arc, config: Arc) { claim, new_watermark, raw_memories.clone(), + pending_extension_resource_removals, thread_id, phase_two_e2e_timer, ); @@ -268,15 +277,15 @@ mod job { completion_watermark: i64, selected_outputs: &[codex_state::Stage1Output], reason: &'static str, - ) { + ) -> bool { session.services.session_telemetry.counter( metrics::MEMORY_PHASE_TWO_JOBS, /*inc*/ 1, &[("status", reason)], ); - let _ = db - .mark_global_phase2_job_succeeded(&claim.token, completion_watermark, selected_outputs) - .await; + db.mark_global_phase2_job_succeeded(&claim.token, completion_watermark, selected_outputs) + .await + .unwrap_or(false) } } @@ -328,9 +337,10 @@ mod agent { pub(super) fn get_prompt( config: Arc, selection: &codex_state::Phase2InputSelection, + removed_extension_resources: &[crate::memories::extensions::RemovedExtensionResource], ) -> Vec { let root = memory_root(&config.codex_home); - let prompt = build_consolidation_prompt(&root, selection); + let prompt = build_consolidation_prompt(&root, selection, removed_extension_resources); vec![UserInput::Text { text: prompt, text_elements: vec![], @@ -343,6 +353,7 @@ mod agent { claim: Claim, new_watermark: i64, selected_outputs: Vec, + pending_extension_resource_removals: Vec, thread_id: ThreadId, phase_two_e2e_timer: Option, ) { @@ -379,7 +390,7 @@ mod agent { if let Some(token_usage) = agent_control.get_total_token_usage(thread_id).await { emit_token_usage_metrics(&session, &token_usage); } - job::succeed( + if job::succeed( &session, &db, &claim, @@ -387,7 +398,10 @@ mod agent { &selected_outputs, "succeeded", ) - .await; + .await + { + remove_extension_resources(&pending_extension_resource_removals).await; + } } else { job::failed(&session, &db, &claim, "failed_agent").await; } diff --git a/codex-rs/core/src/memories/prompts.rs b/codex-rs/core/src/memories/prompts.rs index b6e18b4057..fc85917c9b 100644 --- a/codex-rs/core/src/memories/prompts.rs +++ b/codex-rs/core/src/memories/prompts.rs @@ -1,3 +1,5 @@ +use crate::memories::extensions::EXTENSION_RESOURCE_RETENTION_DAYS; +use crate::memories::extensions::RemovedExtensionResource; use crate::memories::memory_extensions_root; use crate::memories::memory_root; use crate::memories::phase_one; @@ -10,6 +12,7 @@ use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_output_truncation::TruncationPolicy; use codex_utils_output_truncation::truncate_text; use codex_utils_template::Template; +use std::fmt::Write as _; use std::path::Path; use std::sync::LazyLock; use tokio::fs; @@ -62,8 +65,9 @@ Memory extensions (under {{ memory_extensions_root }}/): source. If the user has any memory extensions, you MUST read the instructions for each extension to -determine how to use the memory source. If it has no extension folders, continue with the standard -memory inputs only. +determine how to use the memory source. If the Phase 2 diff lists removed memory extension +resources, use that extension-specific deletion diff to remove stale memories derived only from +those resources. If it has no extension folders, continue with the standard memory inputs only. "#; const MEMORY_EXTENSIONS_PRIMARY_INPUTS: &str = r#" @@ -73,12 +77,16 @@ Under `{{ memory_extensions_root }}/`: - `/instructions.md` - If extension folders exist, read each instructions.md first and follow it when interpreting that extension's memory source. + +If the Phase 2 diff lists removed memory extension resources, use that extension-specific deletion +diff to remove stale memories derived only from those resources. "#; /// Builds the consolidation subagent prompt for a specific memory root. pub(super) fn build_consolidation_prompt( memory_root: &Path, selection: &Phase2InputSelection, + removed_extension_resources: &[RemovedExtensionResource], ) -> String { let memory_extensions_root = memory_extensions_root(memory_root); let memory_extensions_exist = memory_extensions_root.is_dir(); @@ -100,7 +108,8 @@ pub(super) fn build_consolidation_prompt( } else { String::new() }; - let phase2_input_selection = render_phase2_input_selection(selection); + let phase2_input_selection = + render_phase2_input_selection(selection, removed_extension_resources); CONSOLIDATION_PROMPT_TEMPLATE .render([ ("memory_root", memory_root.as_str()), @@ -131,7 +140,10 @@ fn render_memory_extensions_block(template: &Template, memory_extensions_root: & }) } -fn render_phase2_input_selection(selection: &Phase2InputSelection) -> String { +fn render_phase2_input_selection( + selection: &Phase2InputSelection, + removed_extension_resources: &[RemovedExtensionResource], +) -> String { let retained = selection.retained_thread_ids.len(); let added = selection.selected.len().saturating_sub(retained); let selected = if selection.selected.is_empty() { @@ -160,11 +172,29 @@ fn render_phase2_input_selection(selection: &Phase2InputSelection) -> String { .join("\n") }; - format!( + let mut rendered = format!( "- selected inputs this run: {}\n- newly added since the last successful Phase 2 run: {added}\n- retained from the last successful Phase 2 run: {retained}\n- removed from the last successful Phase 2 run: {}\n\nCurrent selected Phase 1 inputs:\n{selected}\n\nRemoved from the last successful Phase 2 selection:\n{removed}\n", selection.selected.len(), selection.removed.len(), - ) + ); + + if !removed_extension_resources.is_empty() { + rendered.push_str("\nMemory extension resources removed by retention pruning:\n"); + let _ = writeln!( + rendered, + "- retention window: {EXTENSION_RESOURCE_RETENTION_DAYS} days" + ); + let mut current_extension = ""; + for removed_resource in removed_extension_resources { + if removed_resource.extension != current_extension { + current_extension = &removed_resource.extension; + let _ = writeln!(rendered, "- extension: {current_extension}"); + } + let _ = writeln!(rendered, " - {}", removed_resource.resource_path); + } + } + + rendered } fn render_selected_input_line(item: &Stage1Output, retained: bool) -> String { diff --git a/codex-rs/core/src/memories/prompts_tests.rs b/codex-rs/core/src/memories/prompts_tests.rs index 325d5e9234..959bb6911d 100644 --- a/codex-rs/core/src/memories/prompts_tests.rs +++ b/codex-rs/core/src/memories/prompts_tests.rs @@ -1,5 +1,7 @@ use super::*; +use crate::memories::extensions::RemovedExtensionResource; use codex_models_manager::model_info::model_info_from_slug; +use codex_state::Phase2InputSelection; use core_test_support::PathExt; use pretty_assertions::assert_eq; use tempfile::tempdir; @@ -54,6 +56,36 @@ fn build_stage_one_input_message_uses_default_limit_when_model_context_window_mi assert!(message.contains(&expected_truncated)); } +#[test] +fn build_consolidation_prompt_includes_removed_extension_resources() { + let temp = tempdir().unwrap(); + let memory_root = temp.path().join("memories"); + std::fs::create_dir_all(temp.path().join("memories_extensions")).unwrap(); + let removed_extension_resources = vec![ + RemovedExtensionResource { + extension: "telepathy".to_string(), + resource_path: "resources/2026-04-06T11-59-59-abcd-10min-old.md".to_string(), + }, + RemovedExtensionResource { + extension: "telepathy".to_string(), + resource_path: "resources/2026-04-07T12-00-00-abcd-10min-cutoff.md".to_string(), + }, + ]; + + let prompt = build_consolidation_prompt( + &memory_root, + &Phase2InputSelection::default(), + &removed_extension_resources, + ); + + assert!(prompt.contains("Memory extension resources removed by retention pruning:")); + assert!(prompt.contains("- retention window: 7 days")); + assert!(prompt.contains("- extension: telepathy")); + assert!(prompt.contains(" - resources/2026-04-06T11-59-59-abcd-10min-old.md")); + assert!(prompt.contains(" - resources/2026-04-07T12-00-00-abcd-10min-cutoff.md")); + assert!(prompt.contains("extension-specific deletion diff")); +} + #[tokio::test] async fn build_memory_tool_developer_instructions_renders_embedded_template() { let temp = tempdir().unwrap(); diff --git a/codex-rs/core/src/memories/tests.rs b/codex-rs/core/src/memories/tests.rs index a0ed29dbec..71809d382d 100644 --- a/codex-rs/core/src/memories/tests.rs +++ b/codex-rs/core/src/memories/tests.rs @@ -424,6 +424,7 @@ mod phase2 { use crate::memories::phase2; use crate::memories::raw_memories_file; use crate::memories::rollout_summaries_dir; + use chrono::Duration as ChronoDuration; use chrono::Utc; use codex_config::Constrained; use codex_login::CodexAuth; @@ -960,6 +961,28 @@ mod phase2 { "stage-1 success should enqueue global consolidation" ); + let telepathy_resources = config + .codex_home + .join("memories_extensions/telepathy/resources"); + tokio::fs::create_dir_all(&telepathy_resources) + .await + .expect("create telepathy resources"); + tokio::fs::write( + config + .codex_home + .join("memories_extensions/telepathy/instructions.md"), + "instructions", + ) + .await + .expect("write telepathy instructions"); + let old_file = telepathy_resources.join(format!( + "{}-abcd-10min-old.md", + (Utc::now() - ChronoDuration::days(8)).format("%Y-%m-%dT%H-%M-%S") + )); + tokio::fs::write(&old_file, "old resource") + .await + .expect("write old extension resource"); + phase2::run(&session, Arc::clone(&config)).await; let retry_claim = state_db @@ -971,5 +994,11 @@ mod phase2 { Phase2JobClaimOutcome::SkippedNotDirty, "spawn failures should leave the job in retry backoff instead of running" ); + assert!( + tokio::fs::try_exists(&old_file) + .await + .expect("check old extension resource"), + "spawn failures should not prune extension resources before retry" + ); } } diff --git a/codex-rs/core/tests/suite/memories.rs b/codex-rs/core/tests/suite/memories.rs index 98e6186437..60685592f6 100644 --- a/codex-rs/core/tests/suite/memories.rs +++ b/codex-rs/core/tests/suite/memories.rs @@ -159,6 +159,145 @@ async fn memories_startup_phase2_tracks_added_and_removed_inputs_across_runs() - Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn memories_startup_phase2_prunes_old_extension_resources_and_reports_them() -> Result<()> { + let server = start_mock_server().await; + let home = Arc::new(TempDir::new()?); + let db = init_state_db(&home).await?; + let now = Utc::now(); + let thread_id = seed_stage1_output( + db.as_ref(), + home.path(), + now - ChronoDuration::hours(1), + "raw memory", + "rollout summary", + "rollout", + ) + .await?; + + let telepathy_resources = home.path().join("memories_extensions/telepathy/resources"); + tokio::fs::create_dir_all(&telepathy_resources).await?; + tokio::fs::write( + home.path() + .join("memories_extensions/telepathy/instructions.md"), + "instructions", + ) + .await?; + let old_file_name = format!( + "{}-abcd-10min-old.md", + (now - ChronoDuration::days(8)).format("%Y-%m-%dT%H-%M-%S") + ); + let old_file = telepathy_resources.join(&old_file_name); + tokio::fs::write(&old_file, "old resource").await?; + let recent_file = telepathy_resources.join(format!( + "{}-abcd-10min-recent.md", + (now - ChronoDuration::days(6)).format("%Y-%m-%dT%H-%M-%S") + )); + tokio::fs::write(&recent_file, "recent resource").await?; + + let phase2 = mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-phase2"), + ev_assistant_message("msg-phase2", "phase2 complete"), + ev_completed("resp-phase2"), + ]), + ) + .await; + + let codex = build_test_codex(&server, home.clone()).await?; + let request = wait_for_single_request(&phase2).await; + let prompt = phase2_prompt_text(&request); + + assert!( + prompt.contains("Memory extension resources removed by retention pruning:"), + "expected extension resource prune report in prompt: {prompt}" + ); + assert!( + prompt.contains("- retention window: 7 days"), + "expected retention window in prompt: {prompt}" + ); + assert!( + prompt.contains("- extension: telepathy"), + "expected extension name in prompt: {prompt}" + ); + assert!( + prompt.contains(&format!(" - resources/{old_file_name}")), + "expected old resource in prompt: {prompt}" + ); + + wait_for_phase2_success(db.as_ref(), thread_id).await?; + wait_for_file_removed(&old_file).await?; + assert!( + !tokio::fs::try_exists(&old_file).await?, + "old extension resource should be pruned" + ); + assert!( + tokio::fs::try_exists(&recent_file).await?, + "recent extension resource should be retained" + ); + + shutdown_test_codex(&codex).await?; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn memories_startup_phase2_processes_old_extension_resources_without_stage1_input() +-> Result<()> { + let server = start_mock_server().await; + let home = Arc::new(TempDir::new()?); + let db = init_state_db(&home).await?; + db.enqueue_global_consolidation(/*input_watermark*/ 1) + .await?; + + let now = Utc::now(); + let telepathy_resources = home.path().join("memories_extensions/telepathy/resources"); + tokio::fs::create_dir_all(&telepathy_resources).await?; + tokio::fs::write( + home.path() + .join("memories_extensions/telepathy/instructions.md"), + "instructions", + ) + .await?; + let old_file_name = format!( + "{}-abcd-10min-old.md", + (now - ChronoDuration::days(8)).format("%Y-%m-%dT%H-%M-%S") + ); + let old_file = telepathy_resources.join(&old_file_name); + tokio::fs::write(&old_file, "old resource").await?; + + let phase2 = mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-phase2-empty"), + ev_assistant_message("msg-phase2-empty", "phase2 complete"), + ev_completed("resp-phase2-empty"), + ]), + ) + .await; + + let codex = build_test_codex(&server, home.clone()).await?; + let request = wait_for_single_request(&phase2).await; + let prompt = phase2_prompt_text(&request); + + assert!( + prompt.contains("- selected inputs this run: 0"), + "expected no selected raw inputs in prompt: {prompt}" + ); + assert!( + prompt.contains("Memory extension resources removed by retention pruning:"), + "expected extension resource prune report in prompt: {prompt}" + ); + assert!( + prompt.contains(&format!(" - resources/{old_file_name}")), + "expected old resource in prompt: {prompt}" + ); + wait_for_file_removed(&old_file).await?; + + shutdown_test_codex(&codex).await?; + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn web_search_pollution_moves_selected_thread_into_removed_phase2_inputs() -> Result<()> { let server = start_mock_server().await; @@ -386,6 +525,21 @@ async fn wait_for_single_request(mock: &ResponseMock) -> ResponsesRequest { wait_for_request(mock, /*expected_count*/ 1).await.remove(0) } +async fn wait_for_file_removed(path: &Path) -> Result<()> { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + if !tokio::fs::try_exists(path).await? { + return Ok(()); + } + assert!( + Instant::now() < deadline, + "timed out waiting for {} to be removed", + path.display() + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + async fn wait_for_request(mock: &ResponseMock, expected_count: usize) -> Vec { let deadline = Instant::now() + Duration::from_secs(10); loop { From af9230d74d25ba55022fae6e44c521194929ffe9 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 15 Apr 2026 11:47:01 +0100 Subject: [PATCH 11/84] chore: exp flag (#17921) --- codex-rs/features/src/lib.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rs index 6a919485c0..18daae1b1a 100644 --- a/codex-rs/features/src/lib.rs +++ b/codex-rs/features/src/lib.rs @@ -672,7 +672,11 @@ pub const FEATURES: &[FeatureSpec] = &[ FeatureSpec { id: Feature::MemoryTool, key: "memories", - stage: Stage::UnderDevelopment, + stage: Stage::Experimental { + name: "Memories", + menu_description: "Allow Codex to create new memories from conversations and bring relevant memories into new conversations.", + announcement: "NEW: Codex can now generate and uses memories. Try is now with `/memories`", + }, default_enabled: false, }, FeatureSpec { From b99a62c5269d5480fd8b4640fd6689c3393bf957 Mon Sep 17 00:00:00 2001 From: sayan-oai Date: Wed, 15 Apr 2026 18:47:26 +0800 Subject: [PATCH 12/84] [codex] Fix current main CI blockers (#17917) ## Summary - Fix marketplace-add local path detection on Windows by using `Path::is_absolute()`. - Make marketplace-add local-source tests parse/write TOML through the same helpers instead of raw string matching. - Update `rand` 0.9.x to 0.9.3 and document the remaining audited `rand` 0.8.5 advisory exception. - Refresh `MODULE.bazel.lock` after the Cargo.lock update. ## Why Latest `main` had two independent CI blockers: marketplace-add tests were not portable to Windows path/TOML escaping, and cargo-deny still reported `RUSTSEC-2026-0097` after the recent rustls-webpki fix. ## Validation - `cargo test -p codex-core marketplace_add -- --nocapture` - `cargo deny --all-features check` - `just bazel-lock-check` - `just fix -p codex-core` - `just fmt` - `git diff --check` --- MODULE.bazel.lock | 2 +- codex-rs/.cargo/audit.toml | 3 +- codex-rs/Cargo.lock | 40 +++++++++---------- codex-rs/core/src/plugins/marketplace_add.rs | 12 ++++-- .../src/plugins/marketplace_add/metadata.rs | 10 +---- .../src/plugins/marketplace_add/source.rs | 4 +- codex-rs/deny.toml | 3 +- 7 files changed, 37 insertions(+), 37 deletions(-) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 83eb490f9d..59ae93bf13 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1239,7 +1239,7 @@ "rama-unix_0.3.0-alpha.4": "{\"dependencies\":[{\"name\":\"pin-project-lite\",\"req\":\"^0.2\"},{\"name\":\"rama-core\",\"req\":\"^0.3.0-alpha.4\"},{\"name\":\"rama-net\",\"req\":\"^0.3.0-alpha.4\"},{\"features\":[\"macros\",\"net\"],\"name\":\"tokio\",\"req\":\"^1.48\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.48\"}],\"features\":{\"default\":[]}}", "rama-utils_0.3.0-alpha.4": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"ahash\",\"req\":\"^0.8\"},{\"name\":\"const_format\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"loom\",\"req\":\"^0.7\",\"target\":\"cfg(loom)\"},{\"name\":\"parking_lot\",\"req\":\"^0.12\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"name\":\"rama-macros\",\"req\":\"^0.3.0-alpha.4\"},{\"name\":\"regex\",\"req\":\"^1.12\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1\"},{\"features\":[\"write\",\"serde\",\"const_generics\",\"const_new\"],\"name\":\"smallvec\",\"req\":\"^1.15\"},{\"name\":\"smol_str\",\"req\":\"^0.3\"},{\"features\":[\"time\",\"macros\"],\"name\":\"tokio\",\"req\":\"^1.48\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0\"},{\"name\":\"wildcard\",\"req\":\"^0.3\"}],\"features\":{}}", "rand_0.8.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.2.1\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.22\",\"target\":\"cfg(unix)\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.4\"},{\"features\":[\"into_bits\"],\"name\":\"packed_simd\",\"optional\":true,\"package\":\"packed_simd_2\",\"req\":\"^0.3.7\"},{\"default_features\":false,\"name\":\"rand_chacha\",\"optional\":true,\"req\":\"^0.3.0\"},{\"name\":\"rand_core\",\"req\":\"^0.6.0\"},{\"kind\":\"dev\",\"name\":\"rand_pcg\",\"req\":\"^0.3.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"}],\"features\":{\"alloc\":[\"rand_core/alloc\"],\"default\":[\"std\",\"std_rng\"],\"getrandom\":[\"rand_core/getrandom\"],\"min_const_gen\":[],\"nightly\":[],\"serde1\":[\"serde\",\"rand_core/serde1\"],\"simd_support\":[\"packed_simd\"],\"small_rng\":[],\"std\":[\"rand_core/std\",\"rand_chacha/std\",\"alloc\",\"getrandom\",\"libc\"],\"std_rng\":[\"rand_chacha\"]}}", - "rand_0.9.2": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.2.1\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.4\"},{\"default_features\":false,\"name\":\"rand_chacha\",\"optional\":true,\"req\":\"^0.9.0\"},{\"default_features\":false,\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rand_pcg\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.7\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.140\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"std_rng\",\"os_rng\",\"small_rng\",\"thread_rng\"],\"log\":[\"dep:log\"],\"nightly\":[],\"os_rng\":[\"rand_core/os_rng\"],\"serde\":[\"dep:serde\",\"rand_core/serde\"],\"simd_support\":[],\"small_rng\":[],\"std\":[\"rand_core/std\",\"rand_chacha?/std\",\"alloc\"],\"std_rng\":[\"dep:rand_chacha\"],\"thread_rng\":[\"std\",\"std_rng\",\"os_rng\"],\"unbiased\":[]}}", + "rand_0.9.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.2.1\"},{\"default_features\":false,\"name\":\"rand_chacha\",\"optional\":true,\"req\":\"^0.9.0\"},{\"default_features\":false,\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rand_pcg\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.7\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.140\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"std_rng\",\"os_rng\",\"small_rng\",\"thread_rng\"],\"log\":[],\"nightly\":[],\"os_rng\":[\"rand_core/os_rng\"],\"serde\":[\"dep:serde\",\"rand_core/serde\"],\"simd_support\":[],\"small_rng\":[],\"std\":[\"rand_core/std\",\"rand_chacha?/std\",\"alloc\"],\"std_rng\":[\"dep:rand_chacha\"],\"thread_rng\":[\"std\",\"std_rng\",\"os_rng\"],\"unbiased\":[]}}", "rand_chacha_0.3.1": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"ppv-lite86\",\"req\":\"^0.2.8\"},{\"name\":\"rand_core\",\"req\":\"^0.6.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"serde1\":[\"serde\"],\"simd\":[],\"std\":[\"ppv-lite86/std\"]}}", "rand_chacha_0.9.0": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"ppv-lite86\",\"req\":\"^0.2.14\"},{\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"features\":[\"os_rng\"],\"kind\":\"dev\",\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"os_rng\":[\"rand_core/os_rng\"],\"serde\":[\"dep:serde\"],\"std\":[\"ppv-lite86/std\",\"rand_core/std\"]}}", "rand_core_0.6.4": "{\"dependencies\":[{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.2\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"serde1\":[\"serde\"],\"std\":[\"alloc\",\"getrandom\",\"getrandom/std\"]}}", diff --git a/codex-rs/.cargo/audit.toml b/codex-rs/.cargo/audit.toml index 425997f6ae..3760d28648 100644 --- a/codex-rs/.cargo/audit.toml +++ b/codex-rs/.cargo/audit.toml @@ -1,9 +1,10 @@ [advisories] -# Reviewed 2026-04-11. Keep this list in sync with ../deny.toml. +# Reviewed 2026-04-15. Keep this list in sync with ../deny.toml. ignore = [ "RUSTSEC-2024-0388", # derivative 2.2.0 via starlark; upstream crate is unmaintained "RUSTSEC-2025-0057", # fxhash 0.2.1 via starlark_map; upstream crate is unmaintained "RUSTSEC-2024-0436", # paste 1.0.15 via starlark/ratatui; upstream crate is unmaintained "RUSTSEC-2024-0320", # yaml-rust via syntect; remove when syntect drops or updates it "RUSTSEC-2025-0141", # bincode via syntect; remove when syntect drops or updates it + "RUSTSEC-2026-0097", # rand 0.8.5 via age/codex-secrets and zbus/keyring; remove when transitive deps move to rand >=0.9.3 ] diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 9e446bbc11..381ea3a11f 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1729,7 +1729,7 @@ dependencies = [ "opentelemetry", "opentelemetry_sdk", "pretty_assertions", - "rand 0.9.2", + "rand 0.9.3", "reqwest", "rustls", "rustls-native-certs", @@ -1974,7 +1974,7 @@ dependencies = [ "opentelemetry_sdk", "predicates", "pretty_assertions", - "rand 0.9.2", + "rand 0.9.3", "regex-lite", "reqwest", "rmcp", @@ -2327,7 +2327,7 @@ dependencies = [ "once_cell", "os_info", "pretty_assertions", - "rand 0.9.2", + "rand 0.9.3", "regex-lite", "reqwest", "serde", @@ -2722,7 +2722,7 @@ dependencies = [ "codex-keyring-store", "keyring", "pretty_assertions", - "rand 0.9.2", + "rand 0.9.3", "regex", "schemars 0.8.22", "serde", @@ -2921,7 +2921,7 @@ dependencies = [ "pathdiff", "pretty_assertions", "pulldown-cmark", - "rand 0.9.2", + "rand 0.9.3", "ratatui", "ratatui-macros", "regex-lite", @@ -5163,7 +5163,7 @@ dependencies = [ "idna", "ipnet", "once_cell", - "rand 0.9.2", + "rand 0.9.3", "ring", "thiserror 2.0.18", "tinyvec", @@ -5185,7 +5185,7 @@ dependencies = [ "moka", "once_cell", "parking_lot", - "rand 0.9.2", + "rand 0.9.3", "resolv-conf", "smallvec", "thiserror 2.0.18", @@ -7331,7 +7331,7 @@ dependencies = [ "futures-util", "opentelemetry", "percent-encoding", - "rand 0.9.2", + "rand 0.9.3", "thiserror 2.0.18", "tokio", "tokio-stream", @@ -7880,7 +7880,7 @@ checksum = "bee689443a2bd0a16ab0348b52ee43e3b2d1b1f931c8aa5c9f8de4c86fbe8c40" dependencies = [ "bitflags 2.10.0", "num-traits", - "rand 0.9.2", + "rand 0.9.3", "rand_chacha 0.9.0", "rand_xorshift", "regex-syntax 0.8.8", @@ -8051,7 +8051,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.2", + "rand 0.9.3", "ring", "rustc-hash 2.1.1", "rustls", @@ -8183,7 +8183,7 @@ dependencies = [ "rama-http-types", "rama-net", "rama-utils", - "rand 0.9.2", + "rand 0.9.3", "serde", "serde_html_form", "serde_json", @@ -8253,7 +8253,7 @@ dependencies = [ "rama-macros", "rama-net", "rama-utils", - "rand 0.9.2", + "rand 0.9.3", "serde", "sha1", ] @@ -8281,7 +8281,7 @@ dependencies = [ "rama-error", "rama-macros", "rama-utils", - "rand 0.9.2", + "rand 0.9.3", "serde", "serde_json", "sync_wrapper", @@ -8355,7 +8355,7 @@ dependencies = [ "rama-http-types", "rama-net", "rama-utils", - "rand 0.9.2", + "rand 0.9.3", "tokio", ] @@ -8435,9 +8435,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "7ec095654a25171c2124e9e3393a930bddbffdc939556c914957a4c3e0a87166" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -8752,7 +8752,7 @@ dependencies = [ "pastey", "pin-project-lite", "process-wrap", - "rand 0.9.2", + "rand 0.9.3", "reqwest", "rmcp-macros", "schemars 1.2.1", @@ -9336,7 +9336,7 @@ version = "0.46.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26ab054c34b87f96c3e4701bea1888317cde30cc7e4a6136d2c48454ab96661c" dependencies = [ - "rand 0.9.2", + "rand 0.9.3", "sentry-types", "serde", "serde_json", @@ -9384,7 +9384,7 @@ checksum = "eecbd63e9d15a26a40675ed180d376fcb434635d2e33de1c24003f61e3e2230d" dependencies = [ "debugid", "hex", - "rand 0.9.2", + "rand 0.9.3", "serde", "serde_json", "thiserror 2.0.18", @@ -11169,7 +11169,7 @@ dependencies = [ "http 1.4.0", "httparse", "log", - "rand 0.9.2", + "rand 0.9.3", "rustls", "rustls-pki-types", "sha1", diff --git a/codex-rs/core/src/plugins/marketplace_add.rs b/codex-rs/core/src/plugins/marketplace_add.rs index 49c1c2e565..b8d1b34d41 100644 --- a/codex-rs/core/src/plugins/marketplace_add.rs +++ b/codex-rs/core/src/plugins/marketplace_add.rs @@ -275,9 +275,15 @@ mod tests { ); let config = fs::read_to_string(codex_home.path().join(codex_config::CONFIG_TOML_FILE))?; - assert!(config.contains("[marketplaces.debug]")); - assert!(config.contains("source_type = \"local\"")); - assert!(config.contains(&format!("source = \"{expected_source}\""))); + let config: toml::Value = toml::from_str(&config)?; + assert_eq!( + config["marketplaces"]["debug"]["source_type"].as_str(), + Some("local") + ); + assert_eq!( + config["marketplaces"]["debug"]["source"].as_str(), + Some(expected_source.as_str()) + ); Ok(()) } diff --git a/codex-rs/core/src/plugins/marketplace_add/metadata.rs b/codex-rs/core/src/plugins/marketplace_add/metadata.rs index ce271b116d..b9412e7f0b 100644 --- a/codex-rs/core/src/plugins/marketplace_add/metadata.rs +++ b/codex-rs/core/src/plugins/marketplace_add/metadata.rs @@ -296,19 +296,11 @@ mod tests { r#"{"name":"debug","plugins":[]}"#, ) .unwrap(); - fs::write( - codex_home.path().join(CONFIG_TOML_FILE), - format!( - "[marketplaces.debug]\nsource_type = \"local\"\nsource = \"{}\"\n", - source_root.display() - ), - ) - .unwrap(); - let source = MarketplaceSource::Local { path: source_root.clone(), }; let install_metadata = MarketplaceInstallMetadata::from_source(&source, &[]); + record_added_marketplace_entry(codex_home.path(), "debug", &install_metadata).unwrap(); let root = installed_marketplace_root_for_source( codex_home.path(), diff --git a/codex-rs/core/src/plugins/marketplace_add/source.rs b/codex-rs/core/src/plugins/marketplace_add/source.rs index 19fec57ade..53d52058aa 100644 --- a/codex-rs/core/src/plugins/marketplace_add/source.rs +++ b/codex-rs/core/src/plugins/marketplace_add/source.rs @@ -124,9 +124,9 @@ fn normalize_git_url(url: &str) -> String { } fn looks_like_local_path(source: &str) -> bool { - source.starts_with("./") + Path::new(source).is_absolute() + || source.starts_with("./") || source.starts_with("../") - || source.starts_with('/') || source.starts_with("~/") || source == "." || source == ".." diff --git a/codex-rs/deny.toml b/codex-rs/deny.toml index 088cda0ba1..bed4a4f2fa 100644 --- a/codex-rs/deny.toml +++ b/codex-rs/deny.toml @@ -70,7 +70,7 @@ feature-depth = 1 # A list of advisory IDs to ignore. Note that ignored advisories will still # output a note when they are encountered. ignore = [ - # Reviewed 2026-04-11. Keep this list in sync with .cargo/audit.toml. + # Reviewed 2026-04-15. Keep this list in sync with .cargo/audit.toml. # Each exception must identify the dependency path and removal condition. { id = "RUSTSEC-2024-0388", reason = "derivative is unmaintained; pulled in via starlark v0.13.0 used by execpolicy/cli/core; no fixed release yet" }, { id = "RUSTSEC-2025-0057", reason = "fxhash is unmaintained; pulled in via starlark_map/starlark v0.13.0 used by execpolicy/cli/core; no fixed release yet" }, @@ -78,6 +78,7 @@ ignore = [ # TODO(fcoury): remove this exception when syntect drops yaml-rust and bincode, or updates to versions that have fixed the vulnerabilities. { id = "RUSTSEC-2024-0320", reason = "yaml-rust is unmaintained; pulled in via syntect v5.3.0 used by codex-tui for syntax highlighting; no fixed release yet" }, { id = "RUSTSEC-2025-0141", reason = "bincode is unmaintained; pulled in via syntect v5.3.0 used by codex-tui for syntax highlighting; no fixed release yet" }, + { id = "RUSTSEC-2026-0097", reason = "rand 0.8.5 is pulled in via age v0.11.2/codex-secrets and zbus v4.4.0/keyring; no compatible rand 0.8 fixed release, remove when transitive dependencies move to rand >=0.9.3" }, ] # If this is true, then cargo deny will use the git executable to fetch advisory database. # If this is false, then it uses a built-in git library. From 5e544be3c9bff69af2395c19278d924dc24ad233 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 15 Apr 2026 12:05:39 +0100 Subject: [PATCH 13/84] chore: do not disable memories for past rollouts on reset (#17919) --- .../app-server/src/codex_message_processor.rs | 51 ++++++++++--------- .../app-server/tests/suite/v2/memory_reset.rs | 4 +- codex-rs/cli/src/main.rs | 2 +- codex-rs/cli/tests/debug_clear_memories.rs | 6 --- codex-rs/state/src/runtime/memories.rs | 35 ++----------- 5 files changed, 34 insertions(+), 64 deletions(-) diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 3cf48699bd..513837c7ce 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -3069,41 +3069,44 @@ impl CodexMessageProcessor { } }; - if let Err(err) = state_db.reset_memory_data_for_fresh_start().await { + if let Err(err) = state_db.clear_memory_data().await { self.send_internal_error( request_id, - format!("failed to reset memory rows in state db: {err}"), + format!("failed to clear memory rows in state db: {err}"), ) .await; return; } let memory_root = self.config.codex_home.join("memories"); + let memory_extensions_root = self.config.codex_home.join("memories_extensions"); let clear_memory_root_result: std::io::Result<()> = async { - match tokio::fs::symlink_metadata(&memory_root).await { - Ok(metadata) if metadata.file_type().is_symlink() => { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidInput, - format!( - "refusing to clear symlinked memory root {}", - memory_root.display() - ), - )); + for directory in [memory_root.as_path(), memory_extensions_root.as_path()] { + match tokio::fs::symlink_metadata(directory).await { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!( + "refusing to clear symlinked memory root {}", + directory.display() + ), + )); + } + Ok(_) => {} + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => return Err(err), } - Ok(_) => {} - Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} - Err(err) => return Err(err), - } - tokio::fs::create_dir_all(&memory_root).await?; - let mut entries = tokio::fs::read_dir(&memory_root).await?; - while let Some(entry) = entries.next_entry().await? { - let path = entry.path(); - let file_type = entry.file_type().await?; - if file_type.is_dir() { - tokio::fs::remove_dir_all(path).await?; - } else { - tokio::fs::remove_file(path).await?; + tokio::fs::create_dir_all(directory).await?; + let mut entries = tokio::fs::read_dir(directory).await?; + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); + let file_type = entry.file_type().await?; + if file_type.is_dir() { + tokio::fs::remove_dir_all(path).await?; + } else { + tokio::fs::remove_file(path).await?; + } } } diff --git a/codex-rs/app-server/tests/suite/v2/memory_reset.rs b/codex-rs/app-server/tests/suite/v2/memory_reset.rs index 954a4fc6b8..3c7ae38671 100644 --- a/codex-rs/app-server/tests/suite/v2/memory_reset.rs +++ b/codex-rs/app-server/tests/suite/v2/memory_reset.rs @@ -20,7 +20,7 @@ use uuid::Uuid; const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); #[tokio::test] -async fn memory_reset_clears_memory_files_and_state_db_rows() -> Result<()> { +async fn memory_reset_clears_memory_files_and_rows_preserves_threads() -> Result<()> { let codex_home = TempDir::new()?; create_config_toml(codex_home.path())?; let state_db = init_state_db(codex_home.path()).await?; @@ -53,7 +53,7 @@ async fn memory_reset_clears_memory_files_and_state_db_rows() -> Result<()> { assert_eq!(stage1_outputs, Vec::new()); assert_eq!( state_db.get_thread_memory_mode(thread_id).await?.as_deref(), - Some("disabled") + Some("enabled") ); let mut remaining_entries = tokio::fs::read_dir(&memory_root).await?; diff --git a/codex-rs/cli/src/main.rs b/codex-rs/cli/src/main.rs index 5ad86a80fd..4e962cf091 100644 --- a/codex-rs/cli/src/main.rs +++ b/codex-rs/cli/src/main.rs @@ -1269,7 +1269,7 @@ async fn run_debug_clear_memories_command( let state_db = StateRuntime::init(config.sqlite_home.clone(), config.model_provider_id.clone()) .await?; - state_db.reset_memory_data_for_fresh_start().await?; + state_db.clear_memory_data().await?; cleared_state_db = true; } diff --git a/codex-rs/cli/tests/debug_clear_memories.rs b/codex-rs/cli/tests/debug_clear_memories.rs index c68172ba35..164e4bd3e2 100644 --- a/codex-rs/cli/tests/debug_clear_memories.rs +++ b/codex-rs/cli/tests/debug_clear_memories.rs @@ -125,12 +125,6 @@ INSERT INTO jobs ( .fetch_one(&pool) .await?; assert_eq!(memory_jobs_count, 0); - - let memory_mode: String = sqlx::query_scalar("SELECT memory_mode FROM threads WHERE id = ?") - .bind(thread_id) - .fetch_one(&pool) - .await?; - assert_eq!(memory_mode, "disabled"); assert!(!memory_root.exists()); Ok(()) diff --git a/codex-rs/state/src/runtime/memories.rs b/codex-rs/state/src/runtime/memories.rs index 50f2f0cd8c..4817d5db67 100644 --- a/codex-rs/state/src/runtime/memories.rs +++ b/codex-rs/state/src/runtime/memories.rs @@ -30,21 +30,6 @@ impl StateRuntime { /// stage-1 (`memory_stage1`) and phase-2 (`memory_consolidate_global`) /// memory pipelines. pub async fn clear_memory_data(&self) -> anyhow::Result<()> { - self.clear_memory_data_inner(/*disable_existing_threads*/ false) - .await - } - - /// Resets persisted memory state for a clean-slate local start. - /// - /// In addition to clearing persisted stage-1 outputs and memory pipeline - /// jobs, this disables memory generation for all existing threads so - /// historical rollouts are not immediately picked up again. - pub async fn reset_memory_data_for_fresh_start(&self) -> anyhow::Result<()> { - self.clear_memory_data_inner(/*disable_existing_threads*/ true) - .await - } - - async fn clear_memory_data_inner(&self, disable_existing_threads: bool) -> anyhow::Result<()> { let mut tx = self.pool.begin().await?; sqlx::query( @@ -66,18 +51,6 @@ WHERE kind = ? OR kind = ? .execute(&mut *tx) .await?; - if disable_existing_threads { - sqlx::query( - r#" -UPDATE threads -SET memory_mode = 'disabled' -WHERE memory_mode = 'enabled' - "#, - ) - .execute(&mut *tx) - .await?; - } - tx.commit().await?; Ok(()) } @@ -1823,7 +1796,7 @@ mod tests { } #[tokio::test] - async fn reset_memory_data_for_fresh_start_clears_rows_and_disables_threads() { + async fn clear_memory_data_clears_rows_and_preserves_thread_memory_modes() { let codex_home = unique_temp_dir(); let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string()) .await @@ -1893,9 +1866,9 @@ mod tests { .expect("disable existing thread"); runtime - .reset_memory_data_for_fresh_start() + .clear_memory_data() .await - .expect("reset memory data"); + .expect("clear memory data"); let stage1_outputs_count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM stage1_outputs") .fetch_one(runtime.pool.as_ref()) @@ -1918,7 +1891,7 @@ mod tests { .fetch_one(runtime.pool.as_ref()) .await .expect("read enabled thread memory mode"); - assert_eq!(enabled_memory_mode, "disabled"); + assert_eq!(enabled_memory_mode, "enabled"); let disabled_memory_mode: String = sqlx::query_scalar("SELECT memory_mode FROM threads WHERE id = ?") From 544b4e39e3aba5990f9af42a3381695927695f4b Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 15 Apr 2026 12:05:50 +0100 Subject: [PATCH 14/84] nit: stable test (#17924) --- .../src/tools/handlers/multi_agents_tests.rs | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/codex-rs/core/src/tools/handlers/multi_agents_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_tests.rs index e35d6a06fb..10f41d97eb 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_tests.rs @@ -1492,24 +1492,25 @@ async fn multi_agent_v2_followup_task_interrupts_busy_child_without_losing_messa let session = Arc::new(session); let turn = Arc::new(turn); - SpawnAgentHandlerV2 - .handle(invocation( - session.clone(), - turn.clone(), - "spawn_agent", - function_payload(json!({ - "message": "boot worker", - "task_name": "worker" - })), - )) - .await - .expect("spawn worker"); + let worker_path = AgentPath::try_from("/root/worker").expect("worker path"); let agent_id = session .services .agent_control - .resolve_agent_reference(session.conversation_id, &turn.session_source, "worker") + .spawn_agent_with_metadata( + (*turn.config).clone(), + Op::CleanBackgroundTerminals, + Some(SessionSource::SubAgent(SubAgentSource::ThreadSpawn { + parent_thread_id: root.thread_id, + depth: 1, + agent_path: Some(worker_path.clone()), + agent_nickname: None, + agent_role: None, + })), + crate::agent::control::SpawnAgentOptions::default(), + ) .await - .expect("worker should resolve"); + .expect("worker spawn should succeed") + .thread_id; let thread = manager .get_thread(agent_id) .await @@ -1566,7 +1567,7 @@ async fn multi_agent_v2_followup_task_interrupts_busy_child_without_losing_messa &thread, &InterAgentCommunication::new( AgentPath::root(), - AgentPath::try_from("/root/worker").expect("agent path"), + worker_path, Vec::new(), "continue".to_string(), /*trigger_turn*/ true, From 9402347f34e60c60cb27e89e71ceaf5e53bd7956 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 15 Apr 2026 14:02:35 +0100 Subject: [PATCH 15/84] feat: memories menu (#17632) Add menu that: 1. If memories feature is not enabled, propose to enable it 2. Let you choose if you want to generate memories and to use memories --- codex-rs/tui/src/app.rs | 177 ++++++++++ codex-rs/tui/src/app_event.rs | 6 + codex-rs/tui/src/app_server_session.rs | 23 ++ codex-rs/tui/src/bottom_pane/command_popup.rs | 2 +- .../src/bottom_pane/memories_settings_view.rs | 332 ++++++++++++++++++ codex-rs/tui/src/bottom_pane/mod.rs | 2 + codex-rs/tui/src/chatwidget.rs | 69 ++++ codex-rs/tui/src/chatwidget/slash_dispatch.rs | 3 + ...widget__tests__memories_enable_prompt.snap | 12 + ...idget__tests__memories_settings_popup.snap | 15 + .../chatwidget/tests/popups_and_settings.rs | 63 ++++ .../src/chatwidget/tests/slash_commands.rs | 12 + codex-rs/tui/src/slash_command.rs | 3 + 13 files changed, 718 insertions(+), 1 deletion(-) create mode 100644 codex-rs/tui/src/bottom_pane/memories_settings_view.rs create mode 100644 codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_enable_prompt.snap create mode 100644 codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_settings_popup.snap diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index b46ace3627..13f7371615 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -96,6 +96,7 @@ use codex_app_server_protocol::ServerRequest; use codex_app_server_protocol::SkillsListResponse; use codex_app_server_protocol::ThreadItem; use codex_app_server_protocol::ThreadLoadedListParams; +use codex_app_server_protocol::ThreadMemoryMode; use codex_app_server_protocol::ThreadRollbackResponse; use codex_app_server_protocol::ThreadStartSource; use codex_app_server_protocol::Turn; @@ -1384,10 +1385,16 @@ impl App { } self.config = next_config; + let show_memory_enable_notice = feature_updates_to_apply + .iter() + .any(|(feature, enabled)| *feature == Feature::MemoryTool && *enabled); for (feature, effective_enabled) in feature_updates_to_apply { self.chat_widget .set_feature_enabled(feature, effective_enabled); } + if show_memory_enable_notice { + self.chat_widget.add_memories_enable_notice(); + } if approvals_reviewer_override.is_some() { self.set_approvals_reviewer_in_app_and_widget(self.config.approvals_reviewer); } @@ -1471,6 +1478,89 @@ impl App { } } + async fn update_memory_settings( + &mut self, + use_memories: bool, + generate_memories: bool, + ) -> bool { + let active_profile = self.active_profile.clone(); + let scoped_memory_segments = |key: &str| { + if let Some(profile) = active_profile.as_deref() { + vec![ + "profiles".to_string(), + profile.to_string(), + "memories".to_string(), + key.to_string(), + ] + } else { + vec!["memories".to_string(), key.to_string()] + } + }; + let edits = [ + ConfigEdit::SetPath { + segments: scoped_memory_segments("use_memories"), + value: use_memories.into(), + }, + ConfigEdit::SetPath { + segments: scoped_memory_segments("generate_memories"), + value: generate_memories.into(), + }, + ]; + + if let Err(err) = ConfigEditsBuilder::new(&self.config.codex_home) + .with_edits(edits) + .apply() + .await + { + tracing::error!(error = %err, "failed to persist memory settings"); + self.chat_widget + .add_error_message(format!("Failed to save memory settings: {err}")); + return false; + } + + self.config.memories.use_memories = use_memories; + self.config.memories.generate_memories = generate_memories; + self.chat_widget + .set_memory_settings(use_memories, generate_memories); + true + } + + async fn update_memory_settings_with_app_server( + &mut self, + app_server: &mut AppServerSession, + use_memories: bool, + generate_memories: bool, + ) { + let previous_generate_memories = self.config.memories.generate_memories; + if !self + .update_memory_settings(use_memories, generate_memories) + .await + { + return; + } + + if previous_generate_memories == generate_memories { + return; + } + + let Some(thread_id) = self.current_displayed_thread_id() else { + return; + }; + + let mode = if generate_memories { + ThreadMemoryMode::Enabled + } else { + ThreadMemoryMode::Disabled + }; + + if let Err(err) = app_server.thread_memory_mode_set(thread_id, mode).await { + tracing::error!(error = %err, %thread_id, "failed to update thread memory mode"); + self.chat_widget.add_error_message(format!( + "Saved memory settings, but failed to update the current thread: {err}" + )); + } + } + fn open_url_in_browser(&mut self, url: String) { if let Err(err) = webbrowser::open(&url) { self.chat_widget @@ -5312,6 +5402,17 @@ impl App { AppEvent::UpdateFeatureFlags { updates } => { self.update_feature_flags(updates).await; } + AppEvent::UpdateMemorySettings { + use_memories, + generate_memories, + } => { + self.update_memory_settings_with_app_server( + app_server, + use_memories, + generate_memories, + ) + .await; + } AppEvent::SkipNextWorldWritableScan => { self.windows_sandbox.skip_world_writable_scan_once = true; } @@ -7944,6 +8045,82 @@ mod tests { Ok(()) } + #[tokio::test] + async fn update_memory_settings_persists_and_updates_widget_config() -> Result<()> { + let (mut app, _app_event_rx, _op_rx) = make_test_app_with_channels().await; + let codex_home = tempdir()?; + app.config.codex_home = codex_home.path().to_path_buf().abs(); + let mut app_server = crate::start_embedded_app_server_for_picker(&app.config).await?; + + app.update_memory_settings_with_app_server( + &mut app_server, + /*use_memories*/ false, + /*generate_memories*/ false, + ) + .await; + + assert!(!app.config.memories.use_memories); + assert!(!app.config.memories.generate_memories); + assert!(!app.chat_widget.config_ref().memories.use_memories); + assert!(!app.chat_widget.config_ref().memories.generate_memories); + + let config = std::fs::read_to_string(codex_home.path().join("config.toml"))?; + let config_value = toml::from_str::(&config)?; + let memories = config_value + .as_table() + .and_then(|table| table.get("memories")) + .and_then(TomlValue::as_table) + .expect("memories table should exist"); + assert_eq!( + memories.get("use_memories"), + Some(&TomlValue::Boolean(false)) + ); + assert_eq!( + memories.get("generate_memories"), + Some(&TomlValue::Boolean(false)) + ); + assert!( + !memories.contains_key("no_memories_if_mcp_or_web_search"), + "the TUI menu should not write the MCP pollution setting" + ); + app_server.shutdown().await?; + Ok(()) + } + + #[tokio::test] + async fn update_memory_settings_updates_current_thread_memory_mode() -> Result<()> { + let (mut app, _app_event_rx, _op_rx) = make_test_app_with_channels().await; + let codex_home = tempdir()?; + app.config.codex_home = codex_home.path().to_path_buf().abs(); + + let mut app_server = crate::start_embedded_app_server_for_picker(&app.config).await?; + let started = app_server.start_thread(&app.config).await?; + let thread_id = started.session.thread_id; + app.active_thread_id = Some(thread_id); + + app.update_memory_settings_with_app_server( + &mut app_server, + /*use_memories*/ true, + /*generate_memories*/ false, + ) + .await; + + let state_db = codex_state::StateRuntime::init( + codex_home.path().to_path_buf(), + app.config.model_provider_id.clone(), + ) + .await + .expect("state db should initialize"); + let memory_mode = state_db + .get_thread_memory_mode(thread_id) + .await + .expect("thread memory mode should be readable"); + assert_eq!(memory_mode.as_deref(), Some("disabled")); + + app_server.shutdown().await?; + Ok(()) + } + #[tokio::test] async fn update_feature_flags_enabling_guardian_selects_guardian_approvals() -> Result<()> { let (mut app, mut app_event_rx, mut op_rx) = make_test_app_with_channels().await; diff --git a/codex-rs/tui/src/app_event.rs b/codex-rs/tui/src/app_event.rs index 91f00b99b0..a0476640b2 100644 --- a/codex-rs/tui/src/app_event.rs +++ b/codex-rs/tui/src/app_event.rs @@ -464,6 +464,12 @@ pub(crate) enum AppEvent { updates: Vec<(Feature, bool)>, }, + /// Update memory settings and persist them to config.toml. + UpdateMemorySettings { + use_memories: bool, + generate_memories: bool, + }, + /// Update whether the full access warning prompt has been acknowledged. UpdateFullAccessWarningAcknowledged(bool), diff --git a/codex-rs/tui/src/app_server_session.rs b/codex-rs/tui/src/app_server_session.rs index 9c4546c4fa..3ce12d2439 100644 --- a/codex-rs/tui/src/app_server_session.rs +++ b/codex-rs/tui/src/app_server_session.rs @@ -38,6 +38,9 @@ use codex_app_server_protocol::ThreadListParams; use codex_app_server_protocol::ThreadListResponse; use codex_app_server_protocol::ThreadLoadedListParams; use codex_app_server_protocol::ThreadLoadedListResponse; +use codex_app_server_protocol::ThreadMemoryMode; +use codex_app_server_protocol::ThreadMemoryModeSetParams; +use codex_app_server_protocol::ThreadMemoryModeSetResponse; use codex_app_server_protocol::ThreadReadParams; use codex_app_server_protocol::ThreadReadResponse; use codex_app_server_protocol::ThreadRealtimeAppendAudioParams; @@ -512,6 +515,26 @@ impl AppServerSession { Ok(()) } + pub(crate) async fn thread_memory_mode_set( + &mut self, + thread_id: ThreadId, + mode: ThreadMemoryMode, + ) -> Result<()> { + let request_id = self.next_request_id(); + let _: ThreadMemoryModeSetResponse = self + .client + .request_typed(ClientRequest::ThreadMemoryModeSet { + request_id, + params: ThreadMemoryModeSetParams { + thread_id: thread_id.to_string(), + mode, + }, + }) + .await + .wrap_err("thread/memoryMode/set failed in TUI")?; + Ok(()) + } + pub(crate) async fn thread_unsubscribe(&mut self, thread_id: ThreadId) -> Result<()> { let request_id = self.next_request_id(); let _: ThreadUnsubscribeResponse = self diff --git a/codex-rs/tui/src/bottom_pane/command_popup.rs b/codex-rs/tui/src/bottom_pane/command_popup.rs index 8bef8ddbc8..29da003dc2 100644 --- a/codex-rs/tui/src/bottom_pane/command_popup.rs +++ b/codex-rs/tui/src/bottom_pane/command_popup.rs @@ -293,7 +293,7 @@ mod tests { CommandItem::Builtin(cmd) => cmd.command(), }) .collect(); - assert_eq!(cmds, vec!["model", "mention", "mcp"]); + assert_eq!(cmds, vec!["model", "memories", "mention", "mcp"]); } #[test] diff --git a/codex-rs/tui/src/bottom_pane/memories_settings_view.rs b/codex-rs/tui/src/bottom_pane/memories_settings_view.rs new file mode 100644 index 0000000000..51b0ef81ba --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/memories_settings_view.rs @@ -0,0 +1,332 @@ +use crossterm::event::KeyCode; +use crossterm::event::KeyEvent; +use crossterm::event::KeyModifiers; +use ratatui::buffer::Buffer; +use ratatui::layout::Constraint; +use ratatui::layout::Layout; +use ratatui::layout::Rect; +use ratatui::style::Stylize; +use ratatui::text::Line; +use ratatui::widgets::Block; +use ratatui::widgets::Widget; + +use crate::app_event::AppEvent; +use crate::app_event_sender::AppEventSender; +use crate::key_hint; +use crate::render::Insets; +use crate::render::RectExt as _; +use crate::render::renderable::ColumnRenderable; +use crate::render::renderable::Renderable; +use crate::style::user_message_style; + +use super::CancellationEvent; +use super::bottom_pane_view::BottomPaneView; +use super::popup_consts::MAX_POPUP_ROWS; +use super::scroll_state::ScrollState; +use super::selection_popup_common::GenericDisplayRow; +use super::selection_popup_common::measure_rows_height; +use super::selection_popup_common::render_rows; + +const MEMORIES_DOC_URL: &str = "https://developers.openai.com/codex/memories"; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum MemoriesSetting { + Use, + Generate, +} + +struct MemoriesSettingItem { + setting: MemoriesSetting, + name: &'static str, + description: &'static str, + enabled: bool, +} + +pub(crate) struct MemoriesSettingsView { + items: Vec, + state: ScrollState, + complete: bool, + app_event_tx: AppEventSender, + header: Box, + docs_link: Line<'static>, + footer_hint: Line<'static>, +} + +impl MemoriesSettingsView { + pub(crate) fn new( + use_memories: bool, + generate_memories: bool, + app_event_tx: AppEventSender, + ) -> Self { + let mut header = ColumnRenderable::new(); + header.push(Line::from("Memories".bold())); + header.push(Line::from( + "Choose how Codex uses and creates memories. Changes are saved to config.toml".dim(), + )); + + let mut view = Self { + items: vec![ + MemoriesSettingItem { + setting: MemoriesSetting::Use, + name: "Use memories", + description: "Use memories in the following threads. Applied at next thread.", + enabled: use_memories, + }, + MemoriesSettingItem { + setting: MemoriesSetting::Generate, + name: "Generate memories", + description: "Generate memories from the following threads. Current thread included.", + enabled: generate_memories, + }, + ], + state: ScrollState::new(), + complete: false, + app_event_tx, + header: Box::new(header), + docs_link: Line::from(vec![ + "Learn more: ".dim(), + MEMORIES_DOC_URL.cyan().underlined(), + ]), + footer_hint: memories_settings_hint_line(), + }; + view.initialize_selection(); + view + } + + fn initialize_selection(&mut self) { + self.state.selected_idx = (!self.items.is_empty()).then_some(0); + } + + fn visible_len(&self) -> usize { + self.items.len() + } + + fn build_rows(&self) -> Vec { + let mut rows = Vec::with_capacity(self.items.len()); + let selected_idx = self.state.selected_idx; + for (idx, item) in self.items.iter().enumerate() { + let prefix = if selected_idx == Some(idx) { + 'โ€บ' + } else { + ' ' + }; + let marker = if item.enabled { 'x' } else { ' ' }; + let name = format!("{prefix} [{marker}] {}", item.name); + rows.push(GenericDisplayRow { + name, + description: Some(item.description.to_string()), + ..Default::default() + }); + } + + rows + } + + fn move_up(&mut self) { + let len = self.visible_len(); + if len == 0 { + return; + } + self.state.move_up_wrap(len); + self.state.ensure_visible(len, MAX_POPUP_ROWS.min(len)); + } + + fn move_down(&mut self) { + let len = self.visible_len(); + if len == 0 { + return; + } + self.state.move_down_wrap(len); + self.state.ensure_visible(len, MAX_POPUP_ROWS.min(len)); + } + + fn toggle_selected(&mut self) { + let Some(selected_idx) = self.state.selected_idx else { + return; + }; + + if let Some(item) = self.items.get_mut(selected_idx) { + item.enabled = !item.enabled; + } + } + + fn rows_width(total_width: u16) -> u16 { + total_width.saturating_sub(2) + } + + fn current_setting(&self, setting: MemoriesSetting) -> bool { + self.items + .iter() + .find(|item| item.setting == setting) + .is_some_and(|item| item.enabled) + } +} + +impl BottomPaneView for MemoriesSettingsView { + fn handle_key_event(&mut self, key_event: KeyEvent) { + match key_event { + KeyEvent { + code: KeyCode::Up, .. + } + | KeyEvent { + code: KeyCode::Char('p'), + modifiers: KeyModifiers::CONTROL, + .. + } + | KeyEvent { + code: KeyCode::Char('\u{0010}'), + modifiers: KeyModifiers::NONE, + .. + } => self.move_up(), + KeyEvent { + code: KeyCode::Char('k'), + modifiers: KeyModifiers::NONE, + .. + } => self.move_up(), + KeyEvent { + code: KeyCode::Down, + .. + } + | KeyEvent { + code: KeyCode::Char('n'), + modifiers: KeyModifiers::CONTROL, + .. + } + | KeyEvent { + code: KeyCode::Char('\u{000e}'), + modifiers: KeyModifiers::NONE, + .. + } => self.move_down(), + KeyEvent { + code: KeyCode::Char('j'), + modifiers: KeyModifiers::NONE, + .. + } => self.move_down(), + KeyEvent { + code: KeyCode::Char(' '), + modifiers: KeyModifiers::NONE, + .. + } => self.toggle_selected(), + KeyEvent { + code: KeyCode::Enter, + modifiers: KeyModifiers::NONE, + .. + } => self.save(), + KeyEvent { + code: KeyCode::Esc, .. + } => self.cancel(), + _ => {} + } + } + + fn is_complete(&self) -> bool { + self.complete + } + + fn on_ctrl_c(&mut self) -> CancellationEvent { + self.cancel(); + CancellationEvent::Handled + } +} + +impl MemoriesSettingsView { + fn save(&mut self) { + self.app_event_tx.send(AppEvent::UpdateMemorySettings { + use_memories: self.current_setting(MemoriesSetting::Use), + generate_memories: self.current_setting(MemoriesSetting::Generate), + }); + self.complete = true; + } + + fn cancel(&mut self) { + self.complete = true; + } +} + +impl Renderable for MemoriesSettingsView { + fn render(&self, area: Rect, buf: &mut Buffer) { + if area.height == 0 || area.width == 0 { + return; + } + + let [content_area, footer_area] = + Layout::vertical([Constraint::Fill(1), Constraint::Length(1)]).areas(area); + + Block::default() + .style(user_message_style()) + .render(content_area, buf); + + let header_height = self + .header + .desired_height(content_area.width.saturating_sub(4)); + let rows = self.build_rows(); + let rows_width = Self::rows_width(content_area.width); + let rows_height = measure_rows_height( + &rows, + &self.state, + MAX_POPUP_ROWS, + rows_width.saturating_add(1), + ); + let [header_area, _, list_area, _, docs_area] = Layout::vertical([ + Constraint::Max(header_height), + Constraint::Max(1), + Constraint::Length(rows_height), + Constraint::Max(1), + Constraint::Length(1), + ]) + .areas(content_area.inset(Insets::vh(/*v*/ 1, /*h*/ 2))); + + self.header.render(header_area, buf); + + if list_area.height > 0 { + let render_area = Rect { + x: list_area.x.saturating_sub(2), + y: list_area.y, + width: rows_width.max(1), + height: list_area.height, + }; + render_rows( + render_area, + buf, + &rows, + &self.state, + MAX_POPUP_ROWS, + " No memory settings available", + ); + } + self.docs_link.clone().render(docs_area, buf); + + let hint_area = Rect { + x: footer_area.x + 2, + y: footer_area.y, + width: footer_area.width.saturating_sub(2), + height: footer_area.height, + }; + self.footer_hint.clone().dim().render(hint_area, buf); + } + + fn desired_height(&self, width: u16) -> u16 { + let rows = self.build_rows(); + let rows_width = Self::rows_width(width); + let rows_height = measure_rows_height( + &rows, + &self.state, + MAX_POPUP_ROWS, + rows_width.saturating_add(1), + ); + + let mut height = self.header.desired_height(width.saturating_sub(4)); + height = height.saturating_add(rows_height + 5); + height.saturating_add(1) + } +} + +fn memories_settings_hint_line() -> Line<'static> { + Line::from(vec![ + "Press ".into(), + key_hint::plain(KeyCode::Char(' ')).into(), + " to toggle; ".into(), + key_hint::plain(KeyCode::Enter).into(), + " to save".into(), + ]) +} diff --git a/codex-rs/tui/src/bottom_pane/mod.rs b/codex-rs/tui/src/bottom_pane/mod.rs index b2d080b048..85020d7137 100644 --- a/codex-rs/tui/src/bottom_pane/mod.rs +++ b/codex-rs/tui/src/bottom_pane/mod.rs @@ -81,6 +81,7 @@ mod experimental_features_view; mod file_search_popup; mod footer; mod list_selection_view; +mod memories_settings_view; mod prompt_args; mod skill_popup; mod skills_toggle_view; @@ -91,6 +92,7 @@ pub(crate) use list_selection_view::SelectionViewParams; pub(crate) use list_selection_view::SideContentWidth; pub(crate) use list_selection_view::popup_content_width; pub(crate) use list_selection_view::side_by_side_layout_widths; +pub(crate) use memories_settings_view::MemoriesSettingsView; mod feedback_view; pub(crate) use feedback_view::FeedbackAudience; pub(crate) use feedback_view::feedback_classification; diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 1334b6e2f2..e9f56a0ef0 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -251,6 +251,11 @@ const MULTI_AGENT_ENABLE_TITLE: &str = "Enable subagents?"; const MULTI_AGENT_ENABLE_YES: &str = "Yes, enable"; const MULTI_AGENT_ENABLE_NO: &str = "Not now"; const MULTI_AGENT_ENABLE_NOTICE: &str = "Subagents will be enabled in the next session."; +const MEMORIES_DOC_URL: &str = "https://developers.openai.com/codex/memories"; +const MEMORIES_ENABLE_TITLE: &str = "Enable memories?"; +const MEMORIES_ENABLE_YES: &str = "Yes, enable"; +const MEMORIES_ENABLE_NO: &str = "Not now"; +const MEMORIES_ENABLE_NOTICE: &str = "Memories will be enabled in the next session."; const PLAN_MODE_REASONING_SCOPE_TITLE: &str = "Apply reasoning change"; const PLAN_MODE_REASONING_SCOPE_PLAN_ONLY: &str = "Apply to Plan mode override"; const PLAN_MODE_REASONING_SCOPE_ALL_MODES: &str = "Apply to global default and Plan mode override"; @@ -311,6 +316,7 @@ use crate::bottom_pane::ExperimentalFeaturesView; use crate::bottom_pane::InputResult; use crate::bottom_pane::LocalImageAttachment; use crate::bottom_pane::McpServerElicitationFormRequest; +use crate::bottom_pane::MemoriesSettingsView; use crate::bottom_pane::MentionBinding; use crate::bottom_pane::QUIT_SHORTCUT_TIMEOUT; use crate::bottom_pane::SelectionAction; @@ -2588,6 +2594,61 @@ impl ChatWidget { }); } + pub(crate) fn open_memories_popup(&mut self) { + if !self.config.features.enabled(Feature::MemoryTool) { + self.open_memories_enable_prompt(); + return; + } + + let view = MemoriesSettingsView::new( + self.config.memories.use_memories, + self.config.memories.generate_memories, + self.app_event_tx.clone(), + ); + self.bottom_pane.show_view(Box::new(view)); + } + + pub(crate) fn open_memories_enable_prompt(&mut self) { + let items = vec![ + SelectionItem { + name: MEMORIES_ENABLE_YES.to_string(), + description: Some( + "Save the setting now. You will need a new session to use it.".to_string(), + ), + actions: vec![Box::new(|tx| { + tx.send(AppEvent::UpdateFeatureFlags { + updates: vec![(Feature::MemoryTool, true)], + }); + })], + dismiss_on_select: true, + ..Default::default() + }, + SelectionItem { + name: MEMORIES_ENABLE_NO.to_string(), + description: Some("Keep memories disabled.".to_string()), + dismiss_on_select: true, + ..Default::default() + }, + ]; + + self.bottom_pane.show_selection_view(SelectionViewParams { + title: Some(MEMORIES_ENABLE_TITLE.to_string()), + subtitle: Some("Memories are currently disabled in your config.".to_string()), + footer_note: Some(Line::from(vec![ + "Learn more: ".dim(), + MEMORIES_DOC_URL.cyan().underlined(), + ])), + footer_hint: Some(standard_popup_hint_line()), + items, + ..Default::default() + }); + } + + pub(crate) fn set_memory_settings(&mut self, use_memories: bool, generate_memories: bool) { + self.config.memories.use_memories = use_memories; + self.config.memories.generate_memories = generate_memories; + } + pub(crate) fn set_token_info(&mut self, info: Option) { match info { Some(info) => self.apply_token_info(info), @@ -9693,6 +9754,13 @@ impl ChatWidget { self.request_redraw(); } + pub(crate) fn add_memories_enable_notice(&mut self) { + self.add_to_history(history_cell::new_warning_event( + MEMORIES_ENABLE_NOTICE.to_string(), + )); + self.request_redraw(); + } + pub(crate) fn add_plain_history_lines(&mut self, lines: Vec>) { self.add_boxed_history(Box::new(PlainHistoryCell::new(lines))); self.request_redraw(); @@ -10372,6 +10440,7 @@ impl ChatWidget { self.config.features = config.features.clone(); self.config.config_layer_stack = config.config_layer_stack.clone(); self.config.realtime = config.realtime.clone(); + self.config.memories = config.memories.clone(); } pub(crate) fn open_review_popup(&mut self) { diff --git a/codex-rs/tui/src/chatwidget/slash_dispatch.rs b/codex-rs/tui/src/chatwidget/slash_dispatch.rs index e6e93d17fb..ee4d22e95f 100644 --- a/codex-rs/tui/src/chatwidget/slash_dispatch.rs +++ b/codex-rs/tui/src/chatwidget/slash_dispatch.rs @@ -228,6 +228,9 @@ impl ChatWidget { SlashCommand::Experimental => { self.open_experimental_popup(); } + SlashCommand::Memories => { + self.open_memories_popup(); + } SlashCommand::Quit | SlashCommand::Exit => { self.request_quit_without_confirmation(); } diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_enable_prompt.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_enable_prompt.snap new file mode 100644 index 0000000000..f9b0daefcc --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_enable_prompt.snap @@ -0,0 +1,12 @@ +--- +source: tui/src/chatwidget/tests.rs +expression: popup +--- + Enable memories? + Memories are currently disabled in your config. + +โ€บ 1. Yes, enable Save the setting now. You will need a new session to use it. + 2. Not now Keep memories disabled. + + Learn more: https://developers.openai.com/codex/memories + Press enter to confirm or esc to go back diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_settings_popup.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_settings_popup.snap new file mode 100644 index 0000000000..9785ef2bb3 --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_settings_popup.snap @@ -0,0 +1,15 @@ +--- +source: tui/src/chatwidget/tests/popups_and_settings.rs +expression: popup +--- + Memories + Choose how Codex uses and creates memories. Changes are saved to config.toml + +โ€บ [x] Use memories Use memories in the following threads. Applied at + next thread. + [ ] Generate memories Generate memories from the following threads. Current + thread included. + + Learn more: https://developers.openai.com/codex/memories + + Press space to toggle; enter to save diff --git a/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs b/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs index e2635d322d..0edea73391 100644 --- a/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs +++ b/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs @@ -1501,6 +1501,69 @@ async fn multi_agent_enable_prompt_updates_feature_and_emits_notice() { assert!(rendered.contains("Subagents will be enabled in the next session.")); } +#[tokio::test] +async fn memories_enable_prompt_snapshot() { + let (mut chat, _rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::MemoryTool, /*enabled*/ false); + + chat.open_memories_popup(); + + let popup = render_bottom_popup(&chat, /*width*/ 80); + assert_chatwidget_snapshot!("memories_enable_prompt", popup); +} + +#[tokio::test] +async fn memories_enable_prompt_updates_feature_without_notice() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::MemoryTool, /*enabled*/ false); + + chat.open_memories_popup(); + chat.handle_key_event(KeyEvent::from(KeyCode::Enter)); + + assert_matches!( + rx.try_recv(), + Ok(AppEvent::UpdateFeatureFlags { updates }) if updates == vec![(Feature::MemoryTool, true)] + ); + assert!( + rx.try_recv().is_err(), + "memory enable prompt should not emit the success notice before persistence succeeds" + ); +} + +#[tokio::test] +async fn memories_settings_popup_snapshot() { + let (mut chat, _rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::MemoryTool, /*enabled*/ true); + chat.config.memories.use_memories = true; + chat.config.memories.generate_memories = false; + + chat.open_memories_popup(); + + let popup = render_bottom_popup(&chat, /*width*/ 80); + assert_chatwidget_snapshot!("memories_settings_popup", popup); +} + +#[tokio::test] +async fn memories_settings_toggle_saves_on_enter() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::MemoryTool, /*enabled*/ true); + chat.config.memories.use_memories = true; + chat.config.memories.generate_memories = false; + + chat.open_memories_popup(); + chat.handle_key_event(KeyEvent::from(KeyCode::Down)); + chat.handle_key_event(KeyEvent::from(KeyCode::Char(' '))); + chat.handle_key_event(KeyEvent::from(KeyCode::Enter)); + + assert_matches!( + rx.try_recv(), + Ok(AppEvent::UpdateMemorySettings { + use_memories: true, + generate_memories: true, + }) + ); +} + #[tokio::test] async fn model_selection_popup_snapshot() { let (mut chat, _rx, _op_rx) = make_chatwidget_manual(Some("gpt-5-codex")).await; diff --git a/codex-rs/tui/src/chatwidget/tests/slash_commands.rs b/codex-rs/tui/src/chatwidget/tests/slash_commands.rs index 2afb959349..7174b28b80 100644 --- a/codex-rs/tui/src/chatwidget/tests/slash_commands.rs +++ b/codex-rs/tui/src/chatwidget/tests/slash_commands.rs @@ -533,6 +533,18 @@ async fn slash_mcp_requests_inventory_via_app_server() { assert!(op_rx.try_recv().is_err(), "expected no core op to be sent"); } +#[tokio::test] +async fn slash_memories_opens_memory_menu() { + let (mut chat, mut rx, mut op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::MemoryTool, /*enabled*/ true); + + chat.dispatch_command(SlashCommand::Memories); + + assert!(render_bottom_popup(&chat, /*width*/ 80).contains("Use memories")); + assert_matches!(rx.try_recv(), Err(TryRecvError::Empty)); + assert!(op_rx.try_recv().is_err(), "expected no core op to be sent"); +} + #[tokio::test] async fn slash_memory_update_reports_stubbed_feature() { let (mut chat, mut rx, mut op_rx) = make_chatwidget_manual(/*model_override*/ None).await; diff --git a/codex-rs/tui/src/slash_command.rs b/codex-rs/tui/src/slash_command.rs index 635dc317f8..759bedbad7 100644 --- a/codex-rs/tui/src/slash_command.rs +++ b/codex-rs/tui/src/slash_command.rs @@ -21,6 +21,7 @@ pub enum SlashCommand { #[strum(serialize = "sandbox-add-read-dir")] SandboxReadRoot, Experimental, + Memories, Skills, Review, Rename, @@ -109,6 +110,7 @@ impl SlashCommand { "let sandbox read a directory: /sandbox-add-read-dir " } SlashCommand::Experimental => "toggle experimental features", + SlashCommand::Memories => "configure memory use and generation", SlashCommand::Mcp => "list configured MCP tools", SlashCommand::Apps => "manage apps", SlashCommand::Plugins => "browse plugins", @@ -154,6 +156,7 @@ impl SlashCommand { | SlashCommand::ElevateSandbox | SlashCommand::SandboxReadRoot | SlashCommand::Experimental + | SlashCommand::Memories | SlashCommand::Review | SlashCommand::Plan | SlashCommand::Clear From 0df7e9a8207296e6b7c967e22a4fee48af35aa09 Mon Sep 17 00:00:00 2001 From: sayan-oai Date: Wed, 15 Apr 2026 21:02:59 +0800 Subject: [PATCH 16/84] register all mcp tools with namespace (#17404) stacked on #17402. MCP tools returned by `tool_search` (deferred tools) get registered in our `ToolRegistry` with a different format than directly available tools. this leads to two different ways of accessing MCP tools from our tool catalog, only one of which works for each. fix this by registering all MCP tools with the namespace format, since this info is already available. also, direct MCP tools are registered to responsesapi without a namespace, while deferred MCP tools have a namespace. this means we can receive MCP `FunctionCall`s in both formats from namespaces. fix this by always registering MCP tools with namespace, regardless of deferral status. make code mode track `ToolName` provenance of tools so it can map the literal JS function name string to the correct `ToolName` for invocation, rather than supporting both in core. this lets us unify to a single canonical `ToolName` representation for each MCP tool and force everywhere to use that one, without supporting fallbacks. --- codex-rs/Cargo.lock | 1 + .../tests/suite/v2/mcp_server_elicitation.rs | 8 +- codex-rs/code-mode/Cargo.toml | 1 + codex-rs/code-mode/src/description.rs | 52 ++-- codex-rs/code-mode/src/runtime/callbacks.rs | 24 +- codex-rs/code-mode/src/runtime/globals.rs | 8 +- codex-rs/code-mode/src/runtime/mod.rs | 5 +- codex-rs/code-mode/src/service.rs | 3 +- .../codex-mcp/src/mcp_connection_manager.rs | 20 +- .../src/mcp_connection_manager_tests.rs | 37 +++ codex-rs/core/src/codex.rs | 9 +- codex-rs/core/src/codex_tests.rs | 12 +- codex-rs/core/src/mcp_tool_exposure.rs | 6 +- codex-rs/core/src/tools/code_mode/mod.rs | 57 +++-- codex-rs/core/src/tools/js_repl/mod.rs | 78 ++++-- codex-rs/core/src/tools/parallel.rs | 2 +- codex-rs/core/src/tools/router.rs | 63 +++-- codex-rs/core/src/tools/spec.rs | 30 ++- codex-rs/core/src/tools/spec_tests.rs | 242 ++++++++++++------ codex-rs/core/tests/common/responses.rs | 47 ++++ codex-rs/core/tests/suite/code_mode.rs | 45 +++- codex-rs/core/tests/suite/js_repl.rs | 81 ++++++ codex-rs/core/tests/suite/openai_file_mcp.rs | 30 +-- codex-rs/core/tests/suite/plugins.rs | 37 +-- codex-rs/core/tests/suite/rmcp_client.rs | 102 ++++++-- codex-rs/core/tests/suite/search_tool.rs | 62 ++++- codex-rs/core/tests/suite/sqlite_state.rs | 9 +- codex-rs/core/tests/suite/truncation.rs | 22 +- codex-rs/protocol/src/lib.rs | 2 + codex-rs/{tools => protocol}/src/tool_name.rs | 22 +- codex-rs/tools/src/code_mode.rs | 111 ++++++-- codex-rs/tools/src/code_mode_tests.rs | 3 + codex-rs/tools/src/lib.rs | 4 +- codex-rs/tools/src/responses_api.rs | 11 +- codex-rs/tools/src/responses_api_tests.rs | 13 +- codex-rs/tools/src/tool_discovery.rs | 4 +- codex-rs/tools/src/tool_registry_plan.rs | 118 +++++---- .../tools/src/tool_registry_plan_tests.rs | 144 +++++++---- .../tools/src/tool_registry_plan_types.rs | 17 +- codex-rs/tools/src/tool_spec.rs | 4 + codex-rs/tools/src/tool_spec_tests.rs | 56 ++++ 41 files changed, 1170 insertions(+), 432 deletions(-) rename codex-rs/{tools => protocol}/src/tool_name.rs (62%) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 381ea3a11f..3a99c2aa60 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1831,6 +1831,7 @@ name = "codex-code-mode" version = "0.0.0" dependencies = [ "async-trait", + "codex-protocol", "deno_core_icudata", "pretty_assertions", "serde", diff --git a/codex-rs/app-server/tests/suite/v2/mcp_server_elicitation.rs b/codex-rs/app-server/tests/suite/v2/mcp_server_elicitation.rs index 862bdf8487..13ebe0b99c 100644 --- a/codex-rs/app-server/tests/suite/v2/mcp_server_elicitation.rs +++ b/codex-rs/app-server/tests/suite/v2/mcp_server_elicitation.rs @@ -66,8 +66,9 @@ use tokio::time::timeout; const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); const CONNECTOR_ID: &str = "calendar"; const CONNECTOR_NAME: &str = "Calendar"; +const TOOL_NAMESPACE: &str = "mcp__codex_apps__calendar"; +const CALLABLE_TOOL_NAME: &str = "_confirm_action"; const TOOL_NAME: &str = "calendar_confirm_action"; -const QUALIFIED_TOOL_NAME: &str = "mcp__codex_apps__calendar_confirm_action"; const TOOL_CALL_ID: &str = "call-calendar-confirm"; const ELICITATION_MESSAGE: &str = "Allow this request?"; @@ -85,9 +86,10 @@ async fn mcp_server_elicitation_round_trip() -> Result<()> { ]), responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call( + responses::ev_function_call_with_namespace( TOOL_CALL_ID, - QUALIFIED_TOOL_NAME, + TOOL_NAMESPACE, + CALLABLE_TOOL_NAME, &tool_call_arguments, ), responses::ev_completed("resp-1"), diff --git a/codex-rs/code-mode/Cargo.toml b/codex-rs/code-mode/Cargo.toml index fa7ce63a42..d2f42359d4 100644 --- a/codex-rs/code-mode/Cargo.toml +++ b/codex-rs/code-mode/Cargo.toml @@ -14,6 +14,7 @@ workspace = true [dependencies] async-trait = { workspace = true } +codex-protocol = { workspace = true } deno_core_icudata = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } diff --git a/codex-rs/code-mode/src/description.rs b/codex-rs/code-mode/src/description.rs index ac18fafd89..1aee1f7a3b 100644 --- a/codex-rs/code-mode/src/description.rs +++ b/codex-rs/code-mode/src/description.rs @@ -1,3 +1,4 @@ +use codex_protocol::ToolName; use serde::Deserialize; use serde::Serialize; use serde_json::Value as JsonValue; @@ -129,6 +130,7 @@ pub enum CodeModeToolKind { #[derive(Clone, Debug, PartialEq)] pub struct ToolDefinition { pub name: String, + pub tool_name: ToolName, pub description: String, pub kind: CodeModeToolKind, pub input_schema: Option, @@ -269,11 +271,15 @@ pub fn build_exec_tool_description( for tool in enabled_tools { let name = tool.name.as_str(); let nested_description = render_code_mode_sample_for_definition(tool); - let next_namespace = namespace_descriptions - .get(name) + let namespace_description = tool + .tool_name + .namespace + .as_ref() + .and_then(|namespace| namespace_descriptions.get(namespace)); + let next_namespace = namespace_description .map(|namespace_description| namespace_description.name.as_str()); if next_namespace != current_namespace { - if let Some(namespace_description) = namespace_descriptions.get(name) { + if let Some(namespace_description) = namespace_description { let namespace_description_text = namespace_description.description.trim(); if !namespace_description_text.is_empty() { nested_tool_sections.push(format!( @@ -346,7 +352,7 @@ pub fn augment_tool_definition(mut definition: ToolDefinition) -> ToolDefinition pub fn enabled_tool_metadata(definition: &ToolDefinition) -> EnabledToolMetadata { EnabledToolMetadata { - tool_name: definition.name.clone(), + tool_name: definition.tool_name.clone(), global_name: normalize_code_mode_identifier(&definition.name), description: definition.description.clone(), kind: definition.kind, @@ -355,7 +361,7 @@ pub fn enabled_tool_metadata(definition: &ToolDefinition) -> EnabledToolMetadata #[derive(Clone, Debug, Eq, PartialEq, Serialize)] pub struct EnabledToolMetadata { - pub tool_name: String, + pub tool_name: ToolName, pub global_name: String, pub description: String, pub kind: CodeModeToolKind, @@ -706,6 +712,7 @@ mod tests { use super::build_exec_tool_description; use super::normalize_code_mode_identifier; use super::parse_exec_source; + use codex_protocol::ToolName; use pretty_assertions::assert_eq; use serde_json::Value as JsonValue; use serde_json::json; @@ -770,6 +777,7 @@ mod tests { fn augment_tool_definition_appends_typed_declaration() { let definition = ToolDefinition { name: "hidden_dynamic_tool".to_string(), + tool_name: ToolName::plain("hidden_dynamic_tool"), description: "Test tool".to_string(), kind: CodeModeToolKind::Function, input_schema: Some(json!({ @@ -798,6 +806,7 @@ mod tests { fn augment_tool_definition_includes_property_descriptions_as_comments() { let definition = ToolDefinition { name: "weather_tool".to_string(), + tool_name: ToolName::plain("weather_tool"), description: "Weather tool".to_string(), kind: CodeModeToolKind::Function, input_schema: Some(json!({ @@ -846,6 +855,7 @@ mod tests { let description = build_exec_tool_description( &[ToolDefinition { name: "foo".to_string(), + tool_name: ToolName::plain("foo"), description: "bar".to_string(), kind: CodeModeToolKind::Function, input_schema: None, @@ -870,26 +880,18 @@ bar" #[test] fn code_mode_only_description_groups_namespace_instructions_once() { - let namespace_descriptions = BTreeMap::from([ - ( - "mcp__sample__alpha".to_string(), - ToolNamespaceDescription { - name: "mcp__sample".to_string(), - description: "Shared namespace guidance.".to_string(), - }, - ), - ( - "mcp__sample__beta".to_string(), - ToolNamespaceDescription { - name: "mcp__sample".to_string(), - description: "Shared namespace guidance.".to_string(), - }, - ), - ]); + let namespace_descriptions = BTreeMap::from([( + "mcp__sample__".to_string(), + ToolNamespaceDescription { + name: "mcp__sample".to_string(), + description: "Shared namespace guidance.".to_string(), + }, + )]); let description = build_exec_tool_description( &[ ToolDefinition { name: "mcp__sample__alpha".to_string(), + tool_name: ToolName::namespaced("mcp__sample__", "alpha"), description: "First tool".to_string(), kind: CodeModeToolKind::Function, input_schema: Some(json!({ @@ -905,6 +907,7 @@ bar" }, ToolDefinition { name: "mcp__sample__beta".to_string(), + tool_name: ToolName::namespaced("mcp__sample__", "beta"), description: "Second tool".to_string(), kind: CodeModeToolKind::Function, input_schema: Some(json!({ @@ -935,7 +938,7 @@ bar" #[test] fn code_mode_only_description_omits_empty_namespace_sections() { let namespace_descriptions = BTreeMap::from([( - "mcp__sample__alpha".to_string(), + "mcp__sample__".to_string(), ToolNamespaceDescription { name: "mcp__sample".to_string(), description: String::new(), @@ -944,6 +947,7 @@ bar" let description = build_exec_tool_description( &[ToolDefinition { name: "mcp__sample__alpha".to_string(), + tool_name: ToolName::namespaced("mcp__sample__", "alpha"), description: "First tool".to_string(), kind: CodeModeToolKind::Function, input_schema: Some(json!({ @@ -969,6 +973,7 @@ bar" fn code_mode_only_description_renders_shared_mcp_types_once() { let first_tool = augment_tool_definition(ToolDefinition { name: "mcp__sample__alpha".to_string(), + tool_name: ToolName::namespaced("mcp__sample__", "alpha"), description: "First tool".to_string(), kind: CodeModeToolKind::Function, input_schema: Some(json!({ @@ -1002,6 +1007,7 @@ bar" }); let second_tool = augment_tool_definition(ToolDefinition { name: "mcp__sample__beta".to_string(), + tool_name: ToolName::namespaced("mcp__sample__", "beta"), description: "Second tool".to_string(), kind: CodeModeToolKind::Function, input_schema: Some(json!({ @@ -1038,6 +1044,7 @@ bar" &[ ToolDefinition { name: first_tool.name, + tool_name: first_tool.tool_name, description: "First tool".to_string(), kind: first_tool.kind, input_schema: first_tool.input_schema, @@ -1045,6 +1052,7 @@ bar" }, ToolDefinition { name: second_tool.name, + tool_name: second_tool.tool_name, description: "Second tool".to_string(), kind: second_tool.kind, input_schema: second_tool.input_schema, diff --git a/codex-rs/code-mode/src/runtime/callbacks.rs b/codex-rs/code-mode/src/runtime/callbacks.rs index 5511baca23..ace6f61144 100644 --- a/codex-rs/code-mode/src/runtime/callbacks.rs +++ b/codex-rs/code-mode/src/runtime/callbacks.rs @@ -15,7 +15,13 @@ pub(super) fn tool_callback( args: v8::FunctionCallbackArguments, mut retval: v8::ReturnValue, ) { - let tool_name = args.data().to_rust_string_lossy(scope); + let tool_index = match args.data().to_rust_string_lossy(scope).parse::() { + Ok(tool_index) => tool_index, + Err(_) => { + throw_type_error(scope, "invalid tool callback data"); + return; + } + }; let input = if args.length() == 0 { Ok(None) } else { @@ -36,6 +42,22 @@ pub(super) fn tool_callback( let promise = resolver.get_promise(scope); let resolver = v8::Global::new(scope, resolver); + let tool_name = { + let Some(state) = scope.get_slot::() else { + throw_type_error(scope, "runtime state unavailable"); + return; + }; + let Some(tool_name) = state + .enabled_tools + .get(tool_index) + .map(|tool| tool.tool_name.clone()) + else { + throw_type_error(scope, "tool callback data is out of range"); + return; + }; + tool_name + }; + let Some(state) = scope.get_slot_mut::() else { throw_type_error(scope, "runtime state unavailable"); return; diff --git a/codex-rs/code-mode/src/runtime/globals.rs b/codex-rs/code-mode/src/runtime/globals.rs index 2d419db908..b40136c44c 100644 --- a/codex-rs/code-mode/src/runtime/globals.rs +++ b/codex-rs/code-mode/src/runtime/globals.rs @@ -53,10 +53,10 @@ fn build_tools_object<'s>( .map(|state| state.enabled_tools.clone()) .unwrap_or_default(); - for tool in enabled_tools { + for (tool_index, tool) in enabled_tools.iter().enumerate() { let name = v8::String::new(scope, &tool.global_name) .ok_or_else(|| "failed to allocate tool name".to_string())?; - let function = tool_function(scope, &tool.tool_name)?; + let function = tool_function(scope, tool_index)?; tools.set(scope, name.into(), function.into()); } Ok(tools) @@ -116,9 +116,9 @@ where fn tool_function<'s>( scope: &mut v8::PinScope<'s, '_>, - tool_name: &str, + tool_index: usize, ) -> Result, String> { - let data = v8::String::new(scope, tool_name) + let data = v8::String::new(scope, &tool_index.to_string()) .ok_or_else(|| "failed to allocate tool callback data".to_string())?; let template = v8::FunctionTemplate::builder(tool_callback) .data(data.into()) diff --git a/codex-rs/code-mode/src/runtime/mod.rs b/codex-rs/code-mode/src/runtime/mod.rs index febe54216a..0f50edd329 100644 --- a/codex-rs/code-mode/src/runtime/mod.rs +++ b/codex-rs/code-mode/src/runtime/mod.rs @@ -9,6 +9,7 @@ use std::sync::OnceLock; use std::sync::mpsc as std_mpsc; use std::thread; +use codex_protocol::ToolName; use serde_json::Value as JsonValue; use tokio::sync::mpsc; @@ -62,7 +63,7 @@ pub(crate) enum TurnMessage { ToolCall { cell_id: String, id: String, - name: String, + name: ToolName, input: Option, }, Notify { @@ -87,7 +88,7 @@ pub(crate) enum RuntimeEvent { YieldRequested, ToolCall { id: String, - name: String, + name: ToolName, input: Option, }, Notify { diff --git a/codex-rs/code-mode/src/service.rs b/codex-rs/code-mode/src/service.rs index c14fee1957..8a37880131 100644 --- a/codex-rs/code-mode/src/service.rs +++ b/codex-rs/code-mode/src/service.rs @@ -5,6 +5,7 @@ use std::sync::atomic::Ordering; use std::time::Duration; use async_trait::async_trait; +use codex_protocol::ToolName; use serde_json::Value as JsonValue; use tokio::sync::Mutex; use tokio::sync::mpsc; @@ -26,7 +27,7 @@ use crate::runtime::spawn_runtime; pub trait CodeModeTurnHost: Send + Sync { async fn invoke_tool( &self, - tool_name: String, + tool_name: ToolName, input: Option, cancellation_token: CancellationToken, ) -> Result; diff --git a/codex-rs/codex-mcp/src/mcp_connection_manager.rs b/codex-rs/codex-mcp/src/mcp_connection_manager.rs index 9f9483053a..fadb07d581 100644 --- a/codex-rs/codex-mcp/src/mcp_connection_manager.rs +++ b/codex-rs/codex-mcp/src/mcp_connection_manager.rs @@ -36,6 +36,7 @@ use codex_async_utils::CancelErr; use codex_async_utils::OrCancelExt; use codex_config::Constrained; use codex_config::types::OAuthCredentialsStoreMode; +use codex_protocol::ToolName; use codex_protocol::approvals::ElicitationRequest; use codex_protocol::approvals::ElicitationRequestEvent; use codex_protocol::mcp::CallToolResult; @@ -155,6 +156,12 @@ pub struct ToolInfo { pub connector_description: Option, } +impl ToolInfo { + pub fn canonical_tool_name(&self) -> ToolName { + ToolName::namespaced(self.callable_namespace.clone(), self.callable_name.clone()) + } +} + const META_OPENAI_FILE_PARAMS: &str = "openai/fileParams"; pub fn declared_openai_file_input_param_names( @@ -1206,14 +1213,11 @@ impl McpConnectionManager { .with_context(|| format!("resources/read failed for `{server}` ({uri})")) } - pub async fn resolve_tool_info(&self, name: &str, namespace: Option<&str>) -> Option { - let qualified_name = match namespace { - Some(namespace) if name.starts_with(namespace) => name.to_string(), - Some(namespace) => format!("{namespace}{name}"), - None => name.to_string(), - }; - - self.list_all_tools().await.get(&qualified_name).cloned() + pub async fn resolve_tool_info(&self, tool_name: &ToolName) -> Option { + let all_tools = self.list_all_tools().await; + all_tools + .into_values() + .find(|tool| tool.canonical_tool_name() == *tool_name) } pub async fn notify_sandbox_state_change(&self, sandbox_state: &SandboxState) -> Result<()> { diff --git a/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs b/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs index bfa50c4182..663f76d5d0 100644 --- a/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs +++ b/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs @@ -1,4 +1,5 @@ use super::*; +use codex_protocol::ToolName; use codex_protocol::protocol::GranularApprovalConfig; use codex_protocol::protocol::McpAuthStatus; use pretty_assertions::assert_eq; @@ -646,6 +647,42 @@ async fn list_all_tools_uses_startup_snapshot_while_client_is_pending() { assert_eq!(tool.callable_name, "calendar_create_event"); } +#[tokio::test] +async fn resolve_tool_info_accepts_canonical_namespaced_tool_names() { + let startup_tools = vec![create_test_tool("rmcp", "echo")]; + let pending_client = futures::future::pending::>() + .boxed() + .shared(); + let approval_policy = Constrained::allow_any(AskForApproval::OnFailure); + let sandbox_policy = Constrained::allow_any(SandboxPolicy::new_read_only_policy()); + let mut manager = McpConnectionManager::new_uninitialized(&approval_policy, &sandbox_policy); + manager.clients.insert( + "rmcp".to_string(), + AsyncManagedClient { + client: pending_client, + startup_snapshot: Some(startup_tools), + startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(false)), + tool_plugin_provenance: Arc::new(ToolPluginProvenance::default()), + }, + ); + + let tool = manager + .resolve_tool_info(&ToolName::namespaced("mcp__rmcp__", "echo")) + .await + .expect("split MCP tool namespace and name should resolve"); + + let expected = ("rmcp", "mcp__rmcp__", "echo", "echo"); + assert_eq!( + ( + tool.server_name.as_str(), + tool.callable_namespace.as_str(), + tool.callable_name.as_str(), + tool.tool.name.as_ref(), + ), + expected + ); +} + #[tokio::test] async fn list_all_tools_blocks_while_client_is_pending_without_startup_snapshot() { let pending_client = futures::future::pending::>() diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 95e932b1dc..c063a5c977 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -91,6 +91,7 @@ use codex_otel::current_span_trace_id; use codex_otel::current_span_w3c_trace_context; use codex_otel::set_parent_from_w3c_trace_context; use codex_protocol::ThreadId; +use codex_protocol::ToolName; use codex_protocol::approvals::ElicitationRequestEvent; use codex_protocol::approvals::ExecPolicyAmendment; use codex_protocol::approvals::NetworkPolicyAmendment; @@ -4453,16 +4454,12 @@ impl Session { .await } - pub(crate) async fn resolve_mcp_tool_info( - &self, - name: &str, - namespace: Option<&str>, - ) -> Option { + pub(crate) async fn resolve_mcp_tool_info(&self, tool_name: &ToolName) -> Option { self.services .mcp_connection_manager .read() .await - .resolve_tool_info(name, namespace) + .resolve_tool_info(tool_name) .await } diff --git a/codex-rs/core/src/codex_tests.rs b/codex-rs/core/src/codex_tests.rs index 6de8909e6d..0ff4ea5a2f 100644 --- a/codex-rs/core/src/codex_tests.rs +++ b/codex-rs/core/src/codex_tests.rs @@ -904,7 +904,7 @@ fn mcp_tool_exposure_searches_large_effective_tool_sets() { } #[test] -fn mcp_tool_exposure_directly_exposes_explicit_apps_in_large_search_sets() { +fn mcp_tool_exposure_directly_exposes_explicit_apps_without_deferred_overlap() { let config = test_config(); let tools_config = tools_config_for_mcp_tool_exposure(/*search_tool*/ true); let mut mcp_tools = numbered_mcp_tools(DIRECT_MCP_TOOL_EXPOSURE_THRESHOLD - 1); @@ -935,13 +935,19 @@ fn mcp_tool_exposure_directly_exposes_explicit_apps_in_large_search_sets() { ); assert_eq!( exposure.deferred_tools.as_ref().map(HashMap::len), - Some(DIRECT_MCP_TOOL_EXPOSURE_THRESHOLD) + Some(DIRECT_MCP_TOOL_EXPOSURE_THRESHOLD - 1) ); let deferred_tools = exposure .deferred_tools .as_ref() .expect("large tool sets should be discoverable through tool_search"); - assert!(deferred_tools.contains_key("mcp__codex_apps__calendar_create_event")); + assert!( + tool_names + .iter() + .all(|direct_tool_name| !deferred_tools.contains_key(direct_tool_name)), + "direct tools should not also be deferred: {tool_names:?}" + ); + assert!(!deferred_tools.contains_key("mcp__codex_apps__calendar_create_event")); assert!(deferred_tools.contains_key("mcp__rmcp__tool_0")); } diff --git a/codex-rs/core/src/mcp_tool_exposure.rs b/codex-rs/core/src/mcp_tool_exposure.rs index d7858c17f0..16aa50cfd4 100644 --- a/codex-rs/core/src/mcp_tool_exposure.rs +++ b/codex-rs/core/src/mcp_tool_exposure.rs @@ -41,9 +41,13 @@ pub(crate) fn build_mcp_tool_exposure( let direct_tools = filter_codex_apps_mcp_tools(all_mcp_tools, explicitly_enabled_connectors, config); + for direct_tool_name in direct_tools.keys() { + deferred_tools.remove(direct_tool_name); + } + McpToolExposure { direct_tools, - deferred_tools: Some(deferred_tools), + deferred_tools: (!deferred_tools.is_empty()).then_some(deferred_tools), } } diff --git a/codex-rs/core/src/tools/code_mode/mod.rs b/codex-rs/core/src/tools/code_mode/mod.rs index de3a1fa330..0004ec6f51 100644 --- a/codex-rs/core/src/tools/code_mode/mod.rs +++ b/codex-rs/core/src/tools/code_mode/mod.rs @@ -116,7 +116,7 @@ struct CoreTurnHost { impl CodeModeTurnHost for CoreTurnHost { async fn invoke_tool( &self, - tool_name: String, + tool_name: ToolName, input: Option, cancellation_token: CancellationToken, ) -> Result { @@ -288,38 +288,39 @@ async fn build_nested_router(exec: &ExecContext) -> ToolRouter { async fn call_nested_tool( exec: ExecContext, tool_runtime: ToolCallRuntime, - tool_name: String, + tool_name: ToolName, input: Option, cancellation_token: CancellationToken, ) -> Result { - if tool_name == PUBLIC_TOOL_NAME { + if tool_name.namespace.is_none() && tool_name.name == PUBLIC_TOOL_NAME { return Err(FunctionCallError::RespondToModel(format!( "{PUBLIC_TOOL_NAME} cannot invoke itself" ))); } - let payload = if let Some(tool_info) = exec - .session - .resolve_mcp_tool_info(&tool_name, /*namespace*/ None) - .await - { - match serialize_function_tool_arguments(&tool_name, input) { - Ok(raw_arguments) => ToolPayload::Mcp { - server: tool_info.server_name, - tool: tool_info.tool.name.to_string(), - raw_arguments, - }, - Err(error) => return Err(FunctionCallError::RespondToModel(error)), - } - } else { - match build_nested_tool_payload(tool_runtime.find_spec(&tool_name), &tool_name, input) { - Ok(payload) => payload, - Err(error) => return Err(FunctionCallError::RespondToModel(error)), - } - }; + let (tool_call_name, payload) = + if let Some(tool_info) = exec.session.resolve_mcp_tool_info(&tool_name).await { + let raw_arguments = match serialize_function_tool_arguments(&tool_name, input) { + Ok(raw_arguments) => raw_arguments, + Err(error) => return Err(FunctionCallError::RespondToModel(error)), + }; + ( + tool_info.canonical_tool_name(), + ToolPayload::Mcp { + server: tool_info.server_name, + tool: tool_info.tool.name.to_string(), + raw_arguments, + }, + ) + } else { + match build_nested_tool_payload(tool_runtime.find_spec(&tool_name), &tool_name, input) { + Ok(payload) => (tool_name, payload), + Err(error) => return Err(FunctionCallError::RespondToModel(error)), + } + }; let call = ToolCall { - tool_name: ToolName::plain(tool_name.clone()), + tool_name: tool_call_name, call_id: format!("{PUBLIC_TOOL_NAME}-{}", uuid::Uuid::new_v4()), payload, }; @@ -339,7 +340,7 @@ fn tool_kind_for_spec(spec: &ToolSpec) -> codex_code_mode::CodeModeToolKind { fn tool_kind_for_name( spec: Option, - tool_name: &str, + tool_name: &ToolName, ) -> Result { spec.as_ref() .map(tool_kind_for_spec) @@ -348,7 +349,7 @@ fn tool_kind_for_name( fn build_nested_tool_payload( spec: Option, - tool_name: &str, + tool_name: &ToolName, input: Option, ) -> Result { let actual_kind = tool_kind_for_name(spec, tool_name)?; @@ -363,7 +364,7 @@ fn build_nested_tool_payload( } fn build_function_tool_payload( - tool_name: &str, + tool_name: &ToolName, input: Option, ) -> Result { let arguments = serialize_function_tool_arguments(tool_name, input)?; @@ -371,7 +372,7 @@ fn build_function_tool_payload( } fn serialize_function_tool_arguments( - tool_name: &str, + tool_name: &ToolName, input: Option, ) -> Result { match input { @@ -385,7 +386,7 @@ fn serialize_function_tool_arguments( } fn build_freeform_tool_payload( - tool_name: &str, + tool_name: &ToolName, input: Option, ) -> Result { match input { diff --git a/codex-rs/core/src/tools/js_repl/mod.rs b/codex-rs/core/src/tools/js_repl/mod.rs index b728009499..92bd4c610f 100644 --- a/codex-rs/core/src/tools/js_repl/mod.rs +++ b/codex-rs/core/src/tools/js_repl/mod.rs @@ -45,6 +45,8 @@ use codex_sandboxing::SandboxCommand; use codex_sandboxing::SandboxManager; use codex_sandboxing::SandboxTransformRequest; use codex_sandboxing::SandboxablePreference; +use codex_tools::ResponsesApiNamespaceTool; +use codex_tools::ToolName; use codex_tools::ToolSpec; use codex_utils_output_truncation::TruncationPolicy; use codex_utils_output_truncation::truncate_text; @@ -1574,29 +1576,67 @@ impl JsReplManager { }, ); - let payload = if let Some(tool_info) = exec + let specs = router.specs(); + let requested_tool_name = specs + .iter() + .find_map(|spec| match spec { + ToolSpec::Function(tool) if tool.name == req.tool_name => { + Some(ToolName::plain(req.tool_name.clone())) + } + ToolSpec::Freeform(tool) if tool.name == req.tool_name => { + Some(ToolName::plain(req.tool_name.clone())) + } + ToolSpec::Namespace(namespace) => { + namespace.tools.iter().find_map(|tool| match tool { + ResponsesApiNamespaceTool::Function(tool) => { + let tool_name = + ToolName::namespaced(namespace.name.clone(), tool.name.clone()); + (tool_name.display() == req.tool_name).then_some(tool_name) + } + }) + } + ToolSpec::LocalShell {} + | ToolSpec::ImageGeneration { .. } + | ToolSpec::ToolSearch { .. } + | ToolSpec::WebSearch { .. } + | ToolSpec::Function(_) + | ToolSpec::Freeform(_) => None, + }) + .unwrap_or_else(|| ToolName::plain(req.tool_name.clone())); + let (tool_call_name, payload) = if let Some(tool_info) = exec .session - .resolve_mcp_tool_info(&req.tool_name, /*namespace*/ None) + .resolve_mcp_tool_info(&requested_tool_name) .await { - crate::tools::context::ToolPayload::Mcp { - server: tool_info.server_name, - tool: tool_info.tool.name.to_string(), - raw_arguments: req.arguments.clone(), - } - } else if is_freeform_tool(&router.specs(), &req.tool_name) { - crate::tools::context::ToolPayload::Custom { - input: req.arguments.clone(), - } + ( + tool_info.canonical_tool_name(), + crate::tools::context::ToolPayload::Mcp { + server: tool_info.server_name, + tool: tool_info.tool.name.to_string(), + raw_arguments: req.arguments.clone(), + }, + ) + } else if matches!( + router.find_spec(&requested_tool_name), + Some(ToolSpec::Freeform(_)) + ) { + ( + requested_tool_name, + crate::tools::context::ToolPayload::Custom { + input: req.arguments.clone(), + }, + ) } else { - crate::tools::context::ToolPayload::Function { - arguments: req.arguments.clone(), - } + ( + requested_tool_name, + crate::tools::context::ToolPayload::Function { + arguments: req.arguments.clone(), + }, + ) }; - let tool_name = req.tool_name.clone(); let call = crate::tools::router::ToolCall { - tool_name: codex_tools::ToolName::plain(tool_name.clone()), + tool_name: tool_call_name, call_id: req.id.clone(), payload, }; @@ -1755,12 +1795,6 @@ fn split_exec_result_content_items( } } -fn is_freeform_tool(specs: &[ToolSpec], name: &str) -> bool { - specs - .iter() - .any(|spec| spec.name() == name && matches!(spec, ToolSpec::Freeform(_))) -} - fn is_js_repl_internal_tool(name: &str) -> bool { matches!(name, "js_repl" | "js_repl_reset") } diff --git a/codex-rs/core/src/tools/parallel.rs b/codex-rs/core/src/tools/parallel.rs index a1965db5cc..b4dbfaa6cf 100644 --- a/codex-rs/core/src/tools/parallel.rs +++ b/codex-rs/core/src/tools/parallel.rs @@ -48,7 +48,7 @@ impl ToolCallRuntime { } } - pub(crate) fn find_spec(&self, tool_name: &str) -> Option { + pub(crate) fn find_spec(&self, tool_name: &codex_tools::ToolName) -> Option { self.router.find_spec(tool_name) } diff --git a/codex-rs/core/src/tools/router.rs b/codex-rs/core/src/tools/router.rs index 645650525c..183b9fb416 100644 --- a/codex-rs/core/src/tools/router.rs +++ b/codex-rs/core/src/tools/router.rs @@ -16,6 +16,7 @@ use codex_protocol::models::SearchToolCallParams; use codex_protocol::models::ShellToolCallParams; use codex_tools::ConfiguredToolSpec; use codex_tools::DiscoverableTool; +use codex_tools::ResponsesApiNamespaceTool; use codex_tools::ToolName; use codex_tools::ToolSpec; use codex_tools::ToolsConfig; @@ -102,20 +103,48 @@ impl ToolRouter { self.model_visible_specs.clone() } - pub fn find_spec(&self, tool_name: &str) -> Option { - self.specs - .iter() - .find(|config| config.name() == tool_name) - .map(|config| config.spec.clone()) + pub fn find_spec(&self, tool_name: &ToolName) -> Option { + self.specs.iter().find_map(|config| match &config.spec { + ToolSpec::Function(tool) + if tool_name.namespace.is_none() && tool.name == tool_name.name => + { + Some(config.spec.clone()) + } + ToolSpec::Freeform(tool) + if tool_name.namespace.is_none() && tool.name == tool_name.name => + { + Some(config.spec.clone()) + } + ToolSpec::Namespace(namespace) => namespace.tools.iter().find_map(|tool| match tool { + ResponsesApiNamespaceTool::Function(tool) + if tool_name.namespace.as_deref() == Some(namespace.name.as_str()) + && tool.name == tool_name.name => + { + Some(ToolSpec::Function(tool.clone())) + } + _ => None, + }), + _ => None, + }) } fn configured_tool_supports_parallel(&self, tool_name: &ToolName) -> bool { - tool_name.namespace.is_none() - && self - .specs - .iter() - .filter(|config| config.supports_parallel_tool_calls) - .any(|config| config.name() == tool_name.name.as_str()) + if tool_name.namespace.is_some() { + return false; + } + + self.specs + .iter() + .filter(|config| config.supports_parallel_tool_calls) + .any(|config| match &config.spec { + ToolSpec::Function(tool) => tool.name == tool_name.name.as_str(), + ToolSpec::Freeform(tool) => tool.name == tool_name.name.as_str(), + ToolSpec::Namespace(_) + | ToolSpec::ToolSearch { .. } + | ToolSpec::LocalShell {} + | ToolSpec::ImageGeneration { .. } + | ToolSpec::WebSearch { .. } => false, + }) } pub fn tool_supports_parallel(&self, call: &ToolCall) -> bool { @@ -141,16 +170,10 @@ impl ToolRouter { call_id, .. } => { - let mcp_tool = session - .resolve_mcp_tool_info(&name, namespace.as_deref()) - .await; - let tool_name = match namespace { - Some(namespace) => ToolName::namespaced(namespace, name), - None => ToolName::plain(name), - }; - if let Some(tool_info) = mcp_tool { + let tool_name = ToolName::new(namespace, name); + if let Some(tool_info) = session.resolve_mcp_tool_info(&tool_name).await { Ok(Some(ToolCall { - tool_name, + tool_name: tool_info.canonical_tool_name(), call_id, payload: ToolPayload::Mcp { server: tool_info.server_name, diff --git a/codex-rs/core/src/tools/spec.rs b/codex-rs/core/src/tools/spec.rs index 188be42ba1..9c05e6cc35 100644 --- a/codex-rs/core/src/tools/spec.rs +++ b/codex-rs/core/src/tools/spec.rs @@ -11,6 +11,7 @@ use codex_tools::DiscoverableTool; use codex_tools::ToolHandlerKind; use codex_tools::ToolNamespace; use codex_tools::ToolRegistryPlanDeferredTool; +use codex_tools::ToolRegistryPlanMcpTool; use codex_tools::ToolRegistryPlanParams; use codex_tools::ToolUserShellType; use codex_tools::ToolsConfig; @@ -29,25 +30,31 @@ pub(crate) fn tool_user_shell_type(user_shell: &Shell) -> ToolUserShellType { } } -struct McpToolPlanInputs { - mcp_tools: HashMap, +struct McpToolPlanInputs<'a> { + mcp_tools: Vec>, tool_namespaces: HashMap, } -fn map_mcp_tools_for_plan(mcp_tools: &HashMap) -> McpToolPlanInputs { +fn map_mcp_tools_for_plan(mcp_tools: &HashMap) -> McpToolPlanInputs<'_> { McpToolPlanInputs { mcp_tools: mcp_tools - .iter() - .map(|(name, tool)| (name.clone(), tool.tool.clone())) + .values() + .map(|tool| ToolRegistryPlanMcpTool { + name: tool.canonical_tool_name(), + tool: &tool.tool, + }) .collect(), tool_namespaces: mcp_tools - .iter() - .map(|(name, tool)| { + .values() + .map(|tool| { ( - name.clone(), + tool.callable_namespace.clone(), ToolNamespace { name: tool.callable_namespace.clone(), - description: tool.server_instructions.clone(), + description: tool + .connector_description + .clone() + .or_else(|| tool.server_instructions.clone()), }, ) }) @@ -99,8 +106,7 @@ pub(crate) fn build_specs_with_discoverable_tools( tools .values() .map(|tool| ToolRegistryPlanDeferredTool { - tool_name: tool.callable_name.as_str(), - tool_namespace: tool.callable_namespace.as_str(), + name: tool.canonical_tool_name(), server_name: tool.server_name.as_str(), connector_name: tool.connector_name.as_deref(), connector_description: tool.connector_description.as_deref(), @@ -114,7 +120,7 @@ pub(crate) fn build_specs_with_discoverable_tools( ToolRegistryPlanParams { mcp_tools: mcp_tool_plan_inputs .as_ref() - .map(|inputs| &inputs.mcp_tools), + .map(|inputs| inputs.mcp_tools.as_slice()), deferred_mcp_tools: deferred_mcp_tool_sources.as_deref(), tool_namespaces: mcp_tool_plan_inputs .as_ref() diff --git a/codex-rs/core/src/tools/spec_tests.rs b/codex-rs/core/src/tools/spec_tests.rs index b0fe3f8436..ee993b0d53 100644 --- a/codex-rs/core/src/tools/spec_tests.rs +++ b/codex-rs/core/src/tools/spec_tests.rs @@ -19,6 +19,7 @@ use codex_protocol::protocol::SessionSource; use codex_tools::ConfiguredToolSpec; use codex_tools::DiscoverableTool; use codex_tools::JsonSchema; +use codex_tools::ResponsesApiNamespaceTool; use codex_tools::ResponsesApiTool; use codex_tools::ShellCommandBackendConfig; use codex_tools::TOOL_SEARCH_TOOL_NAME; @@ -67,6 +68,25 @@ fn mcp_tool_info(tool: rmcp::model::Tool) -> ToolInfo { } } +fn mcp_tool_info_with_display_name(display_name: &str, tool: rmcp::model::Tool) -> ToolInfo { + let (callable_namespace, callable_name) = display_name + .rsplit_once('/') + .map(|(namespace, callable_name)| (format!("{namespace}/"), callable_name.to_string())) + .unwrap_or_else(|| ("".to_string(), display_name.to_string())); + + ToolInfo { + server_name: "test_server".to_string(), + callable_name, + callable_namespace, + server_instructions: None, + tool, + connector_id: None, + connector_name: None, + plugin_display_names: Vec::new(), + connector_description: None, + } +} + fn discoverable_connector(id: &str, name: &str, description: &str) -> DiscoverableTool { let slug = name.replace(' ', "-").to_lowercase(); DiscoverableTool::Connector(Box::new(AppInfo { @@ -109,8 +129,11 @@ fn deferred_responses_api_tool_serializes_with_defer_loading() { ); let serialized = serde_json::to_value(ToolSpec::Function( - mcp_tool_to_deferred_responses_api_tool("mcp__codex_apps__lookup_order".to_string(), &tool) - .expect("convert deferred tool"), + mcp_tool_to_deferred_responses_api_tool( + &ToolName::namespaced("mcp__codex_apps__", "lookup_order"), + &tool, + ) + .expect("convert deferred tool"), )) .expect("serialize deferred tool"); @@ -118,7 +141,7 @@ fn deferred_responses_api_tool_serializes_with_defer_loading() { serialized, serde_json::json!({ "type": "function", - "name": "mcp__codex_apps__lookup_order", + "name": "lookup_order", "description": "Look up an order", "strict": false, "defer_loading": true, @@ -173,6 +196,25 @@ fn find_tool<'a>(tools: &'a [ConfiguredToolSpec], expected_name: &str) -> &'a Co .unwrap_or_else(|| panic!("expected tool {expected_name}")) } +fn find_namespace_function_tool<'a>( + tools: &'a [ConfiguredToolSpec], + expected_namespace: &str, + expected_name: &str, +) -> &'a ResponsesApiTool { + let namespace_tool = find_tool(tools, expected_namespace); + let ToolSpec::Namespace(namespace) = &namespace_tool.spec else { + panic!("expected namespace tool {expected_namespace}"); + }; + namespace + .tools + .iter() + .find_map(|tool| match tool { + ResponsesApiNamespaceTool::Function(tool) if tool.name == expected_name => Some(tool), + _ => None, + }) + .unwrap_or_else(|| panic!("expected tool {expected_namespace}{expected_name} in namespace")) +} + fn multi_agent_v2_tools_config() -> ToolsConfig { let config = test_config(); let model_info = construct_model_info_offline("gpt-5-codex", &config); @@ -910,8 +952,43 @@ fn search_tool_registers_namespaced_mcp_tool_aliases() { assert!(registry.has_handler(&ToolName::plain(TOOL_SEARCH_TOOL_NAME))); assert!(registry.has_handler(&app_alias)); assert!(registry.has_handler(&mcp_alias)); - assert!(registry.has_handler(&ToolName::plain("mcp__codex_apps__calendar_create_event"))); - assert!(registry.has_handler(&ToolName::plain("mcp__rmcp__echo"))); +} + +#[test] +fn direct_mcp_tools_register_namespaced_handlers() { + let config = test_config(); + let model_info = construct_model_info_offline("gpt-5-codex", &config); + let mut features = Features::with_defaults(); + features.enable(Feature::UnifiedExec); + let available_models = Vec::new(); + let tools_config = ToolsConfig::new(&ToolsConfigParams { + model_info: &model_info, + available_models: &available_models, + features: &features, + image_generation_tool_auth_allowed: true, + web_search_mode: Some(WebSearchMode::Cached), + session_source: SessionSource::Cli, + sandbox_policy: &SandboxPolicy::DangerFullAccess, + windows_sandbox_level: WindowsSandboxLevel::Disabled, + }); + + let (_, registry) = build_specs( + &tools_config, + Some(HashMap::from([( + "mcp__test_server__echo".to_string(), + mcp_tool_info(mcp_tool( + "echo", + "Echo", + serde_json::json!({"type": "object"}), + )), + )])), + /*deferred_mcp_tools*/ None, + &[], + ) + .build(); + + assert!(registry.has_handler(&ToolName::namespaced("mcp__test_server__", "echo"))); + assert!(!registry.has_handler(&ToolName::plain("mcp__test_server__echo"))); } #[test] @@ -936,27 +1013,30 @@ fn test_mcp_tool_property_missing_type_defaults_to_string() { &tools_config, Some(HashMap::from([( "dash/search".to_string(), - mcp_tool_info(mcp_tool( - "search", - "Search docs", - serde_json::json!({ - "type": "object", - "properties": { - "query": {"description": "search query"} - } - }), - )), + mcp_tool_info_with_display_name( + "dash/search", + mcp_tool( + "search", + "Search docs", + serde_json::json!({ + "type": "object", + "properties": { + "query": {"description": "search query"} + } + }), + ), + ), )])), /*deferred_mcp_tools*/ None, &[], ) .build(); - let tool = find_tool(&tools, "dash/search"); + let tool = find_namespace_function_tool(&tools, "dash/", "search"); assert_eq!( - tool.spec, - ToolSpec::Function(ResponsesApiTool { - name: "dash/search".to_string(), + *tool, + ResponsesApiTool { + name: "search".to_string(), parameters: JsonSchema::object( /*properties*/ BTreeMap::from([( @@ -970,7 +1050,7 @@ fn test_mcp_tool_property_missing_type_defaults_to_string() { strict: false, output_schema: Some(mcp_call_tool_result_output_schema(serde_json::json!({}))), defer_loading: None, - }) + } ); } @@ -996,25 +1076,28 @@ fn test_mcp_tool_preserves_integer_schema() { &tools_config, Some(HashMap::from([( "dash/paginate".to_string(), - mcp_tool_info(mcp_tool( - "paginate", - "Pagination", - serde_json::json!({ - "type": "object", - "properties": {"page": {"type": "integer"}} - }), - )), + mcp_tool_info_with_display_name( + "dash/paginate", + mcp_tool( + "paginate", + "Pagination", + serde_json::json!({ + "type": "object", + "properties": {"page": {"type": "integer"}} + }), + ), + ), )])), /*deferred_mcp_tools*/ None, &[], ) .build(); - let tool = find_tool(&tools, "dash/paginate"); + let tool = find_namespace_function_tool(&tools, "dash/", "paginate"); assert_eq!( - tool.spec, - ToolSpec::Function(ResponsesApiTool { - name: "dash/paginate".to_string(), + *tool, + ResponsesApiTool { + name: "paginate".to_string(), parameters: JsonSchema::object( /*properties*/ BTreeMap::from([( @@ -1028,7 +1111,7 @@ fn test_mcp_tool_preserves_integer_schema() { strict: false, output_schema: Some(mcp_call_tool_result_output_schema(serde_json::json!({}))), defer_loading: None, - }) + } ); } @@ -1055,25 +1138,28 @@ fn test_mcp_tool_array_without_items_gets_default_string_items() { &tools_config, Some(HashMap::from([( "dash/tags".to_string(), - mcp_tool_info(mcp_tool( - "tags", - "Tags", - serde_json::json!({ - "type": "object", - "properties": {"tags": {"type": "array"}} - }), - )), + mcp_tool_info_with_display_name( + "dash/tags", + mcp_tool( + "tags", + "Tags", + serde_json::json!({ + "type": "object", + "properties": {"tags": {"type": "array"}} + }), + ), + ), )])), /*deferred_mcp_tools*/ None, &[], ) .build(); - let tool = find_tool(&tools, "dash/tags"); + let tool = find_namespace_function_tool(&tools, "dash/", "tags"); assert_eq!( - tool.spec, - ToolSpec::Function(ResponsesApiTool { - name: "dash/tags".to_string(), + *tool, + ResponsesApiTool { + name: "tags".to_string(), parameters: JsonSchema::object( /*properties*/ BTreeMap::from([( @@ -1090,7 +1176,7 @@ fn test_mcp_tool_array_without_items_gets_default_string_items() { strict: false, output_schema: Some(mcp_call_tool_result_output_schema(serde_json::json!({}))), defer_loading: None, - }) + } ); } @@ -1116,27 +1202,30 @@ fn test_mcp_tool_anyof_defaults_to_string() { &tools_config, Some(HashMap::from([( "dash/value".to_string(), - mcp_tool_info(mcp_tool( - "value", - "AnyOf Value", - serde_json::json!({ - "type": "object", - "properties": { - "value": {"anyOf": [{"type": "string"}, {"type": "number"}]} - } - }), - )), + mcp_tool_info_with_display_name( + "dash/value", + mcp_tool( + "value", + "AnyOf Value", + serde_json::json!({ + "type": "object", + "properties": { + "value": {"anyOf": [{"type": "string"}, {"type": "number"}]} + } + }), + ), + ), )])), /*deferred_mcp_tools*/ None, &[], ) .build(); - let tool = find_tool(&tools, "dash/value"); + let tool = find_namespace_function_tool(&tools, "dash/", "value"); assert_eq!( - tool.spec, - ToolSpec::Function(ResponsesApiTool { - name: "dash/value".to_string(), + *tool, + ResponsesApiTool { + name: "value".to_string(), parameters: JsonSchema::object( /*properties*/ BTreeMap::from([( @@ -1156,7 +1245,7 @@ fn test_mcp_tool_anyof_defaults_to_string() { strict: false, output_schema: Some(mcp_call_tool_result_output_schema(serde_json::json!({}))), defer_loading: None, - }) + } ); } @@ -1181,12 +1270,14 @@ fn test_get_openai_tools_mcp_tools_with_additional_properties_schema() { &tools_config, Some(HashMap::from([( "test_server/do_something_cool".to_string(), - mcp_tool_info(mcp_tool( - "do_something_cool", - "Do something cool", - serde_json::json!({ - "type": "object", - "properties": { + mcp_tool_info_with_display_name( + "test_server/do_something_cool", + mcp_tool( + "do_something_cool", + "Do something cool", + serde_json::json!({ + "type": "object", + "properties": { "string_argument": {"type": "string"}, "number_argument": {"type": "number"}, "object_argument": { @@ -1203,22 +1294,23 @@ fn test_get_openai_tools_mcp_tools_with_additional_properties_schema() { }, "required": ["addtl_prop"], "additionalProperties": false + } } } - } - }), - )), + }), + ), + ), )])), /*deferred_mcp_tools*/ None, &[], ) .build(); - let tool = find_tool(&tools, "test_server/do_something_cool"); + let tool = find_namespace_function_tool(&tools, "test_server/", "do_something_cool"); assert_eq!( - tool.spec, - ToolSpec::Function(ResponsesApiTool { - name: "test_server/do_something_cool".to_string(), + *tool, + ResponsesApiTool { + name: "do_something_cool".to_string(), parameters: JsonSchema::object( /*properties*/ BTreeMap::from([ @@ -1268,7 +1360,7 @@ fn test_get_openai_tools_mcp_tools_with_additional_properties_schema() { strict: false, output_schema: Some(mcp_call_tool_result_output_schema(serde_json::json!({}))), defer_loading: None, - }) + } ); } diff --git a/codex-rs/core/tests/common/responses.rs b/codex-rs/core/tests/common/responses.rs index 2e2155ebdd..8def3ac6ea 100644 --- a/codex-rs/core/tests/common/responses.rs +++ b/codex-rs/core/tests/common/responses.rs @@ -125,6 +125,10 @@ impl ResponsesRequest { self.body_json().to_string().contains(&json_fragment) } + pub fn tool_by_name(&self, namespace: &str, tool_name: &str) -> Option { + namespace_child_tool(&self.body_json(), namespace, tool_name).cloned() + } + pub fn instructions_text(&self) -> String { self.body_json()["instructions"] .as_str() @@ -315,6 +319,31 @@ pub(crate) fn output_value_to_text(value: &Value) -> Option { } } +pub fn namespace_child_tool<'a>( + body: &'a Value, + namespace: &str, + tool_name: &str, +) -> Option<&'a Value> { + let tools = body.get("tools")?.as_array()?; + for tool in tools { + if tool.get("name").and_then(Value::as_str) != Some(namespace) + || tool.get("type").and_then(Value::as_str) != Some("namespace") + { + continue; + } + + let child_tools = tool.get("tools")?.as_array()?; + if let Some(child_tool) = child_tools + .iter() + .find(|tool| tool.get("name").and_then(Value::as_str) == Some(tool_name)) + { + return Some(child_tool); + } + } + + None +} + #[cfg(test)] mod tests { use super::*; @@ -780,6 +809,24 @@ pub fn ev_function_call(call_id: &str, name: &str, arguments: &str) -> Value { }) } +pub fn ev_function_call_with_namespace( + call_id: &str, + namespace: &str, + name: &str, + arguments: &str, +) -> Value { + serde_json::json!({ + "type": "response.output_item.done", + "item": { + "type": "function_call", + "call_id": call_id, + "namespace": namespace, + "name": name, + "arguments": arguments + } + }) +} + pub fn ev_tool_search_call(call_id: &str, arguments: &serde_json::Value) -> Value { serde_json::json!({ "type": "response.output_item.done", diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index 76a70d5076..f010c9e587 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -177,12 +177,25 @@ async fn run_code_mode_turn_with_rmcp( server: &MockServer, prompt: &str, code: &str, +) -> Result<(TestCodex, ResponseMock)> { + run_code_mode_turn_with_rmcp_mode(server, prompt, code, /*code_mode_only*/ false).await +} + +async fn run_code_mode_turn_with_rmcp_mode( + server: &MockServer, + prompt: &str, + code: &str, + code_mode_only: bool, ) -> Result<(TestCodex, ResponseMock)> { let rmcp_test_server_bin = stdio_server_bin()?; let mut builder = test_codex() .with_model("test-gpt-5.1-codex") .with_config(move |config| { - let _ = config.features.enable(Feature::CodeMode); + let _ = if code_mode_only { + config.features.enable(Feature::CodeModeOnly) + } else { + config.features.enable(Feature::CodeMode) + }; let mut servers = config.mcp_servers.get().clone(); servers.insert( @@ -1989,6 +2002,36 @@ contentLength=0" Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn code_mode_only_can_call_mcp_tool() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let code = r#" +const result = await tools.mcp__rmcp__echo({ message: "ping" }); +text(`echo=${result.structuredContent?.echo ?? "missing"}`); +"#; + + let (_test, second_mock) = run_code_mode_turn_with_rmcp_mode( + &server, + "use exec to run the rmcp echo tool in code mode only", + code, + /*code_mode_only*/ true, + ) + .await?; + + let req = second_mock.single_request(); + let (output, success) = custom_tool_output_body_and_success(&req, "call-1"); + assert_ne!( + success, + Some(false), + "code_mode_only rmcp tool call failed unexpectedly: {output}" + ); + assert_eq!(output, "echo=ECHOING: ping"); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn code_mode_exposes_mcp_tools_on_global_tools_object() -> Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/core/tests/suite/js_repl.rs b/codex-rs/core/tests/suite/js_repl.rs index 5f52c24e19..2016897b3b 100644 --- a/codex-rs/core/tests/suite/js_repl.rs +++ b/codex-rs/core/tests/suite/js_repl.rs @@ -1,6 +1,8 @@ #![allow(clippy::expect_used, clippy::unwrap_used)] use anyhow::Result; +use codex_config::types::McpServerConfig; +use codex_config::types::McpServerTransportConfig; use codex_features::Feature; use codex_protocol::protocol::EventMsg; use core_test_support::responses; @@ -12,12 +14,15 @@ use core_test_support::responses::ev_custom_tool_call; use core_test_support::responses::ev_response_created; use core_test_support::responses::sse; use core_test_support::skip_if_no_network; +use core_test_support::stdio_server_bin; use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event_match; +use std::collections::HashMap; use std::fs; #[cfg(unix)] use std::os::unix::fs::PermissionsExt; use std::path::Path; +use std::time::Duration; use tempfile::tempdir; use wiremock::MockServer; @@ -594,6 +599,82 @@ async fn js_repl_can_invoke_builtin_tools() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn js_repl_can_invoke_mcp_tools_by_display_name() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let rmcp_test_server_bin = stdio_server_bin()?; + let mut builder = test_codex().with_config(move |config| { + config + .features + .enable(Feature::JsRepl) + .expect("test config should allow feature update"); + + let mut servers = config.mcp_servers.get().clone(); + servers.insert( + "rmcp".to_string(), + McpServerConfig { + transport: McpServerTransportConfig::Stdio { + command: rmcp_test_server_bin, + args: Vec::new(), + env: None, + env_vars: Vec::new(), + cwd: None, + }, + enabled: true, + required: false, + supports_parallel_tool_calls: false, + disabled_reason: None, + startup_timeout_sec: Some(Duration::from_secs(10)), + tool_timeout_sec: None, + enabled_tools: None, + disabled_tools: None, + scopes: None, + oauth_resource: None, + tools: HashMap::new(), + }, + ); + config + .mcp_servers + .set(servers) + .expect("test mcp servers should accept any configuration"); + }); + let test = builder.build(&server).await?; + + responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-1"), + ev_custom_tool_call( + "call-1", + "js_repl", + r#" +const result = await codex.tool("mcp__rmcp__echo", { message: "ping" }); +console.log(result.output); +"#, + ), + ev_completed("resp-1"), + ]), + ) + .await; + let final_mock = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]), + ) + .await; + + test.submit_turn("use js_repl to call an MCP tool").await?; + + let req = final_mock.single_request(); + assert_js_repl_ok(&req, "call-1", "ECHOING: ping"); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn js_repl_tool_call_rejects_recursive_js_repl_invocation() -> Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/core/tests/suite/openai_file_mcp.rs b/codex-rs/core/tests/suite/openai_file_mcp.rs index 134ad6dfde..211aa67381 100644 --- a/codex-rs/core/tests/suite/openai_file_mcp.rs +++ b/codex-rs/core/tests/suite/openai_file_mcp.rs @@ -10,7 +10,7 @@ use core_test_support::apps_test_server::AppsTestServer; use core_test_support::apps_test_server::DOCUMENT_EXTRACT_TEXT_RESOURCE_URI; use core_test_support::responses::ev_assistant_message; use core_test_support::responses::ev_completed; -use core_test_support::responses::ev_function_call; +use core_test_support::responses::ev_function_call_with_namespace; use core_test_support::responses::ev_response_created; use core_test_support::responses::mount_sse_sequence; use core_test_support::responses::sse; @@ -26,7 +26,8 @@ use wiremock::matchers::header; use wiremock::matchers::method; use wiremock::matchers::path; -const DOCUMENT_EXTRACT_TOOL: &str = "mcp__codex_apps__calendar_extract_text"; +const DOCUMENT_EXTRACT_NAMESPACE: &str = "mcp__codex_apps__calendar"; +const DOCUMENT_EXTRACT_TOOL: &str = "_extract_text"; fn configure_apps(config: &mut Config, chatgpt_base_url: &str) { if let Err(err) = config.features.enable(Feature::Apps) { @@ -35,18 +36,6 @@ fn configure_apps(config: &mut Config, chatgpt_base_url: &str) { config.chatgpt_base_url = chatgpt_base_url.to_string(); } -fn tool_by_name<'a>(body: &'a Value, name: &str) -> &'a Value { - body.get("tools") - .and_then(Value::as_array) - .and_then(|tools| { - tools.iter().find(|tool| { - tool.get("name").and_then(Value::as_str) == Some(name) - || tool.get("type").and_then(Value::as_str) == Some(name) - }) - }) - .unwrap_or_else(|| panic!("missing tool {name} in /v1/responses request: {body:?}")) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn codex_apps_file_params_upload_local_paths_before_mcp_tool_call() -> Result<()> { let server = start_mock_server().await; @@ -93,8 +82,9 @@ async fn codex_apps_file_params_upload_local_paths_before_mcp_tool_call() -> Res vec![ sse(vec![ ev_response_created("resp-1"), - ev_function_call( + ev_function_call_with_namespace( call_id, + DOCUMENT_EXTRACT_NAMESPACE, DOCUMENT_EXTRACT_TOOL, &json!({"file": "report.txt"}).to_string(), ), @@ -123,8 +113,14 @@ async fn codex_apps_file_params_upload_local_paths_before_mcp_tool_call() -> Res .await?; let requests = mock.requests(); - let body = requests[0].body_json(); - let extract_tool = tool_by_name(&body, DOCUMENT_EXTRACT_TOOL); + let Some(extract_tool) = + requests[0].tool_by_name(DOCUMENT_EXTRACT_NAMESPACE, DOCUMENT_EXTRACT_TOOL) + else { + let body = requests[0].body_json(); + panic!( + "missing tool {DOCUMENT_EXTRACT_NAMESPACE}{DOCUMENT_EXTRACT_TOOL} in /v1/responses request: {body:?}" + ) + }; assert_eq!( extract_tool.pointer("/parameters/properties/file"), Some(&json!({ diff --git a/codex-rs/core/tests/suite/plugins.rs b/codex-rs/core/tests/suite/plugins.rs index ab2c85ac0c..b19385cd5d 100644 --- a/codex-rs/core/tests/suite/plugins.rs +++ b/codex-rs/core/tests/suite/plugins.rs @@ -168,22 +168,6 @@ fn tool_names(body: &serde_json::Value) -> Vec { .unwrap_or_default() } -fn tool_description(body: &serde_json::Value, tool_name: &str) -> Option { - body.get("tools") - .and_then(serde_json::Value::as_array) - .and_then(|tools| { - tools.iter().find_map(|tool| { - if tool.get("name").and_then(serde_json::Value::as_str) == Some(tool_name) { - tool.get("description") - .and_then(serde_json::Value::as_str) - .map(str::to_string) - } else { - None - } - }) - }) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn capability_sections_render_in_developer_message_in_order() -> Result<()> { skip_if_no_network!(Ok(())); @@ -319,20 +303,27 @@ async fn explicit_plugin_mentions_inject_plugin_guidance() -> Result<()> { assert!( request_tools .iter() - .any(|name| name == "mcp__codex_apps__google_calendar_create_event"), + .any(|name| name == "mcp__codex_apps__google_calendar"), "expected plugin app tools to become visible for this turn: {request_tools:?}" ); - let echo_description = tool_description(&request_body, "mcp__sample__echo") + let echo_tool = request + .tool_by_name("mcp__sample__", "echo") + .expect("plugin MCP tool should be present"); + let echo_description = echo_tool + .get("description") + .and_then(serde_json::Value::as_str) .expect("plugin MCP tool description should be present"); assert!( echo_description.contains("This tool is part of plugin `sample`."), "expected plugin MCP provenance in tool description: {echo_description:?}" ); - let calendar_description = tool_description( - &request_body, - "mcp__codex_apps__google_calendar_create_event", - ) - .expect("plugin app tool description should be present"); + let calendar_tool = request + .tool_by_name("mcp__codex_apps__google_calendar", "_create_event") + .expect("plugin app tool should be present"); + let calendar_description = calendar_tool + .get("description") + .and_then(serde_json::Value::as_str) + .expect("plugin app tool description should be present"); assert!( calendar_description.contains("This tool is part of plugin `sample`."), "expected plugin app provenance in tool description: {calendar_description:?}" diff --git a/codex-rs/core/tests/suite/rmcp_client.rs b/codex-rs/core/tests/suite/rmcp_client.rs index 1dff2348b5..b86094f910 100644 --- a/codex-rs/core/tests/suite/rmcp_client.rs +++ b/codex-rs/core/tests/suite/rmcp_client.rs @@ -91,13 +91,18 @@ async fn stdio_server_round_trip() -> anyhow::Result<()> { let call_id = "call-123"; let server_name = "rmcp"; - let tool_name = format!("mcp__{server_name}__echo"); + let namespace = format!("mcp__{server_name}__"); - mount_sse_once( + let call_mock = mount_sse_once( &server, responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, &tool_name, "{\"message\":\"ping\"}"), + responses::ev_function_call_with_namespace( + call_id, + &namespace, + "echo", + "{\"message\":\"ping\"}", + ), responses::ev_completed("resp-1"), ]), ) @@ -223,6 +228,13 @@ async fn stdio_server_round_trip() -> anyhow::Result<()> { wait_for_event(&fixture.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; let output_item = final_mock.single_request().function_call_output(call_id); + let request = call_mock.single_request(); + assert!( + request.tool_by_name(&namespace, "echo").is_some(), + "direct MCP tool should be sent as a namespace child tool: {:?}", + request.body_json() + ); + let output_text = output_item .get("output") .and_then(Value::as_str) @@ -246,13 +258,14 @@ async fn stdio_mcp_tool_call_includes_sandbox_state_meta() -> anyhow::Result<()> let call_id = "sandbox-meta-call"; let server_name = "rmcp"; - let tool_name = format!("mcp__{server_name}__sandbox_meta"); + let namespace = format!("mcp__{server_name}__"); + let tool_name = format!("{namespace}sandbox_meta"); - mount_sse_once( + let call_mock = mount_sse_once( &server, responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, &tool_name, "{}"), + responses::ev_function_call_with_namespace(call_id, &namespace, "sandbox_meta", "{}"), responses::ev_completed("resp-1"), ]), ) @@ -301,11 +314,43 @@ async fn stdio_mcp_tool_call_includes_sandbox_state_meta() -> anyhow::Result<()> .build(&server) .await?; + let tools_ready_deadline = Instant::now() + Duration::from_secs(30); + loop { + fixture.codex.submit(Op::ListMcpTools).await?; + let list_event = wait_for_event_with_timeout( + &fixture.codex, + |ev| matches!(ev, EventMsg::McpListToolsResponse(_)), + Duration::from_secs(10), + ) + .await; + let EventMsg::McpListToolsResponse(tool_list) = list_event else { + unreachable!("event guard guarantees McpListToolsResponse"); + }; + if tool_list.tools.contains_key(&tool_name) { + break; + } + + let available_tools: Vec<&str> = tool_list.tools.keys().map(String::as_str).collect(); + if Instant::now() >= tools_ready_deadline { + panic!( + "timed out waiting for MCP tool {tool_name} to become available; discovered tools: {available_tools:?}" + ); + } + sleep(Duration::from_millis(200)).await; + } + let sandbox_policy = SandboxPolicy::new_read_only_policy(); fixture .submit_turn_with_policy("call the rmcp sandbox_meta tool", sandbox_policy.clone()) .await?; + let request = call_mock.single_request(); + assert!( + request.tool_by_name(&namespace, "sandbox_meta").is_some(), + "direct MCP tool should be sent as a namespace child tool: {:?}", + request.body_json() + ); + let output_item = final_mock.single_request().function_call_output(call_id); let output_text = output_item .get("output") @@ -346,15 +391,15 @@ async fn stdio_mcp_parallel_tool_calls_default_false_runs_serially() -> anyhow:: let first_call_id = "sync-serial-1"; let second_call_id = "sync-serial-2"; let server_name = "rmcp"; - let tool_name = format!("mcp__{server_name}__sync"); + let namespace = format!("mcp__{server_name}__"); let args = json!({ "sleep_after_ms": 100 }).to_string(); mount_sse_once( &server, responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(first_call_id, &tool_name, &args), - responses::ev_function_call(second_call_id, &tool_name, &args), + responses::ev_function_call_with_namespace(first_call_id, &namespace, "sync", &args), + responses::ev_function_call_with_namespace(second_call_id, &namespace, "sync", &args), responses::ev_completed("resp-1"), ]), ) @@ -488,7 +533,7 @@ async fn stdio_mcp_parallel_tool_calls_opt_in_runs_concurrently() -> anyhow::Res let first_call_id = "sync-1"; let second_call_id = "sync-2"; let server_name = "rmcp"; - let tool_name = format!("mcp__{server_name}__sync"); + let namespace = format!("mcp__{server_name}__"); let args = json!({ "sleep_after_ms": 100, "barrier": { @@ -503,8 +548,8 @@ async fn stdio_mcp_parallel_tool_calls_opt_in_runs_concurrently() -> anyhow::Res &server, responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(first_call_id, &tool_name, &args), - responses::ev_function_call(second_call_id, &tool_name, &args), + responses::ev_function_call_with_namespace(first_call_id, &namespace, "sync", &args), + responses::ev_function_call_with_namespace(second_call_id, &namespace, "sync", &args), responses::ev_completed("resp-1"), ]), ) @@ -604,13 +649,14 @@ async fn stdio_image_responses_round_trip() -> anyhow::Result<()> { let call_id = "img-1"; let server_name = "rmcp"; let tool_name = format!("mcp__{server_name}__image"); + let namespace = format!("mcp__{server_name}__"); // First stream: model decides to call the image tool. mount_sse_once( &server, responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, &tool_name, "{}"), + responses::ev_function_call_with_namespace(call_id, &namespace, "image", "{}"), responses::ev_completed("resp-1"), ]), ) @@ -793,7 +839,7 @@ async fn stdio_image_responses_are_sanitized_for_text_only_model() -> anyhow::Re let call_id = "img-text-only-1"; let server_name = "rmcp"; - let tool_name = format!("mcp__{server_name}__image"); + let namespace = format!("mcp__{server_name}__"); let text_only_model_slug = "rmcp-text-only-model"; let models_mock = mount_models_once( @@ -843,7 +889,7 @@ async fn stdio_image_responses_are_sanitized_for_text_only_model() -> anyhow::Re &server, responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, &tool_name, "{}"), + responses::ev_function_call_with_namespace(call_id, &namespace, "image", "{}"), responses::ev_completed("resp-1"), ]), ) @@ -964,13 +1010,18 @@ async fn stdio_server_propagates_whitelisted_env_vars() -> anyhow::Result<()> { let call_id = "call-1234"; let server_name = "rmcp_whitelist"; - let tool_name = format!("mcp__{server_name}__echo"); + let namespace = format!("mcp__{server_name}__"); mount_sse_once( &server, responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, &tool_name, "{\"message\":\"ping\"}"), + responses::ev_function_call_with_namespace( + call_id, + &namespace, + "echo", + "{\"message\":\"ping\"}", + ), responses::ev_completed("resp-1"), ]), ) @@ -1106,13 +1157,18 @@ async fn streamable_http_tool_call_round_trip() -> anyhow::Result<()> { let call_id = "call-456"; let server_name = "rmcp_http"; - let tool_name = format!("mcp__{server_name}__echo"); + let namespace = format!("mcp__{server_name}__"); mount_sse_once( &server, responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, &tool_name, "{\"message\":\"ping\"}"), + responses::ev_function_call_with_namespace( + call_id, + &namespace, + "echo", + "{\"message\":\"ping\"}", + ), responses::ev_completed("resp-1"), ]), ) @@ -1311,12 +1367,18 @@ async fn streamable_http_with_oauth_round_trip_impl() -> anyhow::Result<()> { let call_id = "call-789"; let server_name = "rmcp_http_oauth"; let tool_name = format!("mcp__{server_name}__echo"); + let namespace = format!("mcp__{server_name}__"); mount_sse_once( &server, responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, &tool_name, "{\"message\":\"ping\"}"), + responses::ev_function_call_with_namespace( + call_id, + &namespace, + "echo", + "{\"message\":\"ping\"}", + ), responses::ev_completed("resp-1"), ]), ) diff --git a/codex-rs/core/tests/suite/search_tool.rs b/codex-rs/core/tests/suite/search_tool.rs index c524316650..74de499d70 100644 --- a/codex-rs/core/tests/suite/search_tool.rs +++ b/codex-rs/core/tests/suite/search_tool.rs @@ -23,6 +23,7 @@ use core_test_support::responses::ev_response_created; use core_test_support::responses::ev_tool_search_call; use core_test_support::responses::mount_sse_once; use core_test_support::responses::mount_sse_sequence; +use core_test_support::responses::namespace_child_tool; use core_test_support::responses::sse; use core_test_support::responses::start_mock_server; use core_test_support::skip_if_no_network; @@ -45,6 +46,7 @@ const CALENDAR_CREATE_TOOL: &str = "mcp__codex_apps__calendar_create_event"; const CALENDAR_LIST_TOOL: &str = "mcp__codex_apps__calendar_list_events"; const SEARCH_CALENDAR_NAMESPACE: &str = "mcp__codex_apps__calendar"; const SEARCH_CALENDAR_CREATE_TOOL: &str = "_create_event"; +const SEARCH_CALENDAR_LIST_TOOL: &str = "_list_events"; fn tool_names(body: &Value) -> Vec { body.get("tools") @@ -215,8 +217,17 @@ async fn tool_search_disabled_by_default_exposes_apps_tools_directly() -> Result let body = mock.single_request().body_json(); let tools = tool_names(&body); assert!(!tools.iter().any(|name| name == TOOL_SEARCH_TOOL_NAME)); - assert!(tools.iter().any(|name| name == CALENDAR_CREATE_TOOL)); - assert!(tools.iter().any(|name| name == CALENDAR_LIST_TOOL)); + assert!( + namespace_child_tool( + &body, + SEARCH_CALENDAR_NAMESPACE, + SEARCH_CALENDAR_CREATE_TOOL + ) + .is_some() + ); + assert!( + namespace_child_tool(&body, SEARCH_CALENDAR_NAMESPACE, SEARCH_CALENDAR_LIST_TOOL).is_some() + ); Ok(()) } @@ -332,6 +343,7 @@ async fn search_tool_hides_apps_tools_without_search() -> Result<()> { assert!(tools.iter().any(|name| name == TOOL_SEARCH_TOOL_NAME)); assert!(!tools.iter().any(|name| name == CALENDAR_CREATE_TOOL)); assert!(!tools.iter().any(|name| name == CALENDAR_LIST_TOOL)); + assert!(!tools.iter().any(|name| name == SEARCH_CALENDAR_NAMESPACE)); Ok(()) } @@ -365,11 +377,16 @@ async fn explicit_app_mentions_expose_apps_tools_without_search() -> Result<()> let body = mock.single_request().body_json(); let tools = tool_names(&body); assert!( - tools.iter().any(|name| name == CALENDAR_CREATE_TOOL), + namespace_child_tool( + &body, + SEARCH_CALENDAR_NAMESPACE, + SEARCH_CALENDAR_CREATE_TOOL + ) + .is_some(), "expected explicit app mention to expose create tool, got tools: {tools:?}" ); assert!( - tools.iter().any(|name| name == CALENDAR_LIST_TOOL), + namespace_child_tool(&body, SEARCH_CALENDAR_NAMESPACE, SEARCH_CALENDAR_LIST_TOOL).is_some(), "expected explicit app mention to expose list tool, got tools: {tools:?}" ); @@ -523,6 +540,12 @@ async fn tool_search_returns_deferred_tools_without_follow_up_tool_injection() - .any(|name| name == CALENDAR_CREATE_TOOL), "app tools should still be hidden before search: {first_request_tools:?}" ); + assert!( + !first_request_tools + .iter() + .any(|name| name == SEARCH_CALENDAR_NAMESPACE), + "app namespace should still be hidden before search: {first_request_tools:?}" + ); let output_item = tool_search_output_item(&requests[1], call_id); assert_eq!( @@ -570,6 +593,12 @@ async fn tool_search_returns_deferred_tools_without_follow_up_tool_injection() - .any(|name| name == CALENDAR_CREATE_TOOL), "follow-up request should rely on tool_search_output history, not tool injection: {second_request_tools:?}" ); + assert!( + !second_request_tools + .iter() + .any(|name| name == SEARCH_CALENDAR_NAMESPACE), + "follow-up request should rely on tool_search_output history, not namespace injection: {second_request_tools:?}" + ); let output_item = requests[2].function_call_output("calendar-call-1"); assert_eq!( @@ -584,6 +613,12 @@ async fn tool_search_returns_deferred_tools_without_follow_up_tool_injection() - .any(|name| name == CALENDAR_CREATE_TOOL), "post-tool follow-up should still rely on tool_search_output history, not tool injection: {third_request_tools:?}" ); + assert!( + !third_request_tools + .iter() + .any(|name| name == SEARCH_CALENDAR_NAMESPACE), + "post-tool follow-up should still rely on tool_search_output history, not namespace injection: {third_request_tools:?}" + ); Ok(()) } @@ -683,16 +718,19 @@ async fn tool_search_indexes_only_enabled_non_app_mcp_tools() -> Result<()> { .any(|name| name == "mcp__rmcp__echo"), "non-app MCP tools should be hidden before search in large-search mode: {first_request_tools:?}" ); + assert!( + !first_request_tools.iter().any(|name| name == "mcp__rmcp__"), + "non-app MCP namespace should be hidden before search in large-search mode: {first_request_tools:?}" + ); let echo_tools = tool_search_output_tools(&requests[1], echo_call_id); - let rmcp_echo_tools = echo_tools - .iter() - .filter(|tool| tool.get("name").and_then(Value::as_str) == Some("mcp__rmcp__")) - .flat_map(|namespace| namespace.get("tools").and_then(Value::as_array)) - .flatten() - .filter_map(|tool| tool.get("name").and_then(Value::as_str).map(str::to_string)) - .collect::>(); - assert_eq!(rmcp_echo_tools, vec!["echo".to_string()]); + let echo_output = json!({ "tools": echo_tools }); + let rmcp_echo_tool = namespace_child_tool(&echo_output, "mcp__rmcp__", "echo") + .expect("tool_search should return rmcp echo as a namespace child tool"); + assert_eq!( + rmcp_echo_tool.get("type").and_then(Value::as_str), + Some("function") + ); let image_tools = tool_search_output_tools(&requests[1], image_call_id); let found_rmcp_image_tool = image_tools diff --git a/codex-rs/core/tests/suite/sqlite_state.rs b/codex-rs/core/tests/suite/sqlite_state.rs index 1c9d6a87e1..ef517d74fd 100644 --- a/codex-rs/core/tests/suite/sqlite_state.rs +++ b/codex-rs/core/tests/suite/sqlite_state.rs @@ -325,12 +325,17 @@ async fn mcp_call_marks_thread_memory_mode_polluted_when_configured() -> Result< let server = start_mock_server().await; let call_id = "call-123"; let server_name = "rmcp"; - let tool_name = format!("mcp__{server_name}__echo"); + let namespace = format!("mcp__{server_name}__"); mount_sse_once( &server, responses::sse(vec![ ev_response_created("resp-1"), - ev_function_call(call_id, &tool_name, "{\"message\":\"ping\"}"), + responses::ev_function_call_with_namespace( + call_id, + &namespace, + "echo", + "{\"message\":\"ping\"}", + ), ev_completed("resp-1"), ]), ) diff --git a/codex-rs/core/tests/suite/truncation.rs b/codex-rs/core/tests/suite/truncation.rs index 288e8f4bdb..5bca13de50 100644 --- a/codex-rs/core/tests/suite/truncation.rs +++ b/codex-rs/core/tests/suite/truncation.rs @@ -331,7 +331,7 @@ async fn mcp_tool_call_output_exceeds_limit_truncated_for_model() -> Result<()> let call_id = "rmcp-truncated"; let server_name = "rmcp"; - let tool_name = format!("mcp__{server_name}__echo"); + let namespace = format!("mcp__{server_name}__"); // Build a very large message to exceed 10KiB once serialized. let large_msg = "long-message-with-newlines-".repeat(6000); @@ -341,7 +341,12 @@ async fn mcp_tool_call_output_exceeds_limit_truncated_for_model() -> Result<()> &server, sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, &tool_name, &args_json.to_string()), + responses::ev_function_call_with_namespace( + call_id, + &namespace, + "echo", + &args_json.to_string(), + ), responses::ev_completed("resp-1"), ]), ) @@ -426,13 +431,13 @@ async fn mcp_image_output_preserves_image_and_no_text_summary() -> Result<()> { let call_id = "rmcp-image-no-trunc"; let server_name = "rmcp"; - let tool_name = format!("mcp__{server_name}__image"); + let namespace = format!("mcp__{server_name}__"); mount_sse_once( &server, sse(vec![ ev_response_created("resp-1"), - ev_function_call(call_id, &tool_name, "{}"), + responses::ev_function_call_with_namespace(call_id, &namespace, "image", "{}"), ev_completed("resp-1"), ]), ) @@ -705,7 +710,7 @@ async fn mcp_tool_call_output_not_truncated_with_custom_limit() -> Result<()> { let call_id = "rmcp-untruncated"; let server_name = "rmcp"; - let tool_name = format!("mcp__{server_name}__echo"); + let namespace = format!("mcp__{server_name}__"); let large_msg = "a".repeat(80_000); let args_json = serde_json::json!({ "message": large_msg }); @@ -713,7 +718,12 @@ async fn mcp_tool_call_output_not_truncated_with_custom_limit() -> Result<()> { &server, sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, &tool_name, &args_json.to_string()), + responses::ev_function_call_with_namespace( + call_id, + &namespace, + "echo", + &args_json.to_string(), + ), responses::ev_completed("resp-1"), ]), ) diff --git a/codex-rs/protocol/src/lib.rs b/codex-rs/protocol/src/lib.rs index de580de6f3..2506dae747 100644 --- a/codex-rs/protocol/src/lib.rs +++ b/codex-rs/protocol/src/lib.rs @@ -2,8 +2,10 @@ pub mod account; mod agent_path; pub mod auth; mod thread_id; +mod tool_name; pub use agent_path::AgentPath; pub use thread_id::ThreadId; +pub use tool_name::ToolName; pub mod approvals; pub mod config_types; pub mod dynamic_tools; diff --git a/codex-rs/tools/src/tool_name.rs b/codex-rs/protocol/src/tool_name.rs similarity index 62% rename from codex-rs/tools/src/tool_name.rs rename to codex-rs/protocol/src/tool_name.rs index da18233abb..d09bc1ce72 100644 --- a/codex-rs/tools/src/tool_name.rs +++ b/codex-rs/protocol/src/tool_name.rs @@ -1,12 +1,23 @@ +use serde::Deserialize; +use serde::Serialize; +use std::fmt; + /// Identifies a callable tool, preserving the namespace split when the model /// provides one. -#[derive(Clone, Debug, Eq, Hash, PartialEq)] +#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] pub struct ToolName { pub name: String, pub namespace: Option, } impl ToolName { + pub fn new(namespace: Option, name: impl Into) -> Self { + Self { + name: name.into(), + namespace, + } + } + pub fn plain(name: impl Into) -> Self { Self { name: name.into(), @@ -29,6 +40,15 @@ impl ToolName { } } +impl fmt::Display for ToolName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.namespace { + Some(namespace) => write!(f, "{namespace}{}", self.name), + None => f.write_str(&self.name), + } + } +} + impl From for ToolName { fn from(name: String) -> Self { Self::plain(name) diff --git a/codex-rs/tools/src/code_mode.rs b/codex-rs/tools/src/code_mode.rs index 35d95b271a..e45c7be1d9 100644 --- a/codex-rs/tools/src/code_mode.rs +++ b/codex-rs/tools/src/code_mode.rs @@ -1,7 +1,9 @@ use crate::FreeformTool; use crate::FreeformToolFormat; use crate::JsonSchema; +use crate::ResponsesApiNamespaceTool; use crate::ResponsesApiTool; +use crate::ToolName; use crate::ToolSpec; use codex_code_mode::CodeModeToolKind; use codex_code_mode::ToolDefinition as CodeModeToolDefinition; @@ -9,22 +11,46 @@ use std::collections::BTreeMap; /// Augment tool descriptions with code-mode-specific exec samples. pub fn augment_tool_spec_for_code_mode(spec: ToolSpec) -> ToolSpec { - let Some(description) = code_mode_tool_definition_for_spec(&spec) - .map(codex_code_mode::augment_tool_definition) - .map(|definition| definition.description) - else { - return spec; - }; - match spec { ToolSpec::Function(mut tool) => { + let Some(description) = + augmented_description_for_spec(&ToolSpec::Function(tool.clone())) + else { + return ToolSpec::Function(tool); + }; tool.description = description; ToolSpec::Function(tool) } ToolSpec::Freeform(mut tool) => { + let Some(description) = + augmented_description_for_spec(&ToolSpec::Freeform(tool.clone())) + else { + return ToolSpec::Freeform(tool); + }; tool.description = description; ToolSpec::Freeform(tool) } + ToolSpec::Namespace(mut namespace) => { + for tool in &mut namespace.tools { + match tool { + ResponsesApiNamespaceTool::Function(tool) => { + let tool_name = + ToolName::namespaced(namespace.name.clone(), tool.name.clone()); + let definition = CodeModeToolDefinition { + name: tool_name.display(), + tool_name, + description: tool.description.clone(), + kind: CodeModeToolKind::Function, + input_schema: serde_json::to_value(&tool.parameters).ok(), + output_schema: tool.output_schema.clone(), + }; + tool.description = + codex_code_mode::augment_tool_definition(definition).description; + } + } + } + ToolSpec::Namespace(namespace) + } other => other, } } @@ -42,7 +68,9 @@ pub fn collect_code_mode_tool_definitions<'a>( ) -> Vec { let mut tool_definitions = specs .into_iter() - .filter_map(tool_spec_to_code_mode_tool_definition) + .flat_map(code_mode_tool_definitions_for_spec) + .filter(|definition| codex_code_mode::is_code_mode_nested_tool(&definition.name)) + .map(codex_code_mode::augment_tool_definition) .collect::>(); tool_definitions.sort_by(|left, right| left.name.cmp(&right.name)); tool_definitions.dedup_by(|left, right| left.name == right.name); @@ -54,7 +82,7 @@ pub fn collect_code_mode_exec_prompt_tool_definitions<'a>( ) -> Vec { let mut tool_definitions = specs .into_iter() - .filter_map(code_mode_tool_definition_for_spec) + .flat_map(code_mode_tool_definitions_for_spec) .filter(|definition| codex_code_mode::is_code_mode_nested_tool(&definition.name)) .collect::>(); tool_definitions.sort_by(|left, right| left.name.cmp(&right.name)); @@ -137,26 +165,61 @@ SOURCE: /[\s\S]+/ }) } +fn augmented_description_for_spec(spec: &ToolSpec) -> Option { + code_mode_tool_definition_for_spec(spec) + .map(codex_code_mode::augment_tool_definition) + .map(|definition| definition.description) +} + fn code_mode_tool_definition_for_spec(spec: &ToolSpec) -> Option { + code_mode_tool_definitions_for_spec(spec).into_iter().next() +} + +fn code_mode_tool_definitions_for_spec(spec: &ToolSpec) -> Vec { match spec { - ToolSpec::Function(tool) => Some(CodeModeToolDefinition { - name: tool.name.clone(), - description: tool.description.clone(), - kind: CodeModeToolKind::Function, - input_schema: serde_json::to_value(&tool.parameters).ok(), - output_schema: tool.output_schema.clone(), - }), - ToolSpec::Freeform(tool) => Some(CodeModeToolDefinition { - name: tool.name.clone(), - description: tool.description.clone(), - kind: CodeModeToolKind::Freeform, - input_schema: None, - output_schema: None, - }), + ToolSpec::Function(tool) => { + let name = tool.name.clone(); + vec![CodeModeToolDefinition { + tool_name: ToolName::plain(name.clone()), + name, + description: tool.description.clone(), + kind: CodeModeToolKind::Function, + input_schema: serde_json::to_value(&tool.parameters).ok(), + output_schema: tool.output_schema.clone(), + }] + } + ToolSpec::Freeform(tool) => { + let name = tool.name.clone(); + vec![CodeModeToolDefinition { + tool_name: ToolName::plain(name.clone()), + name, + description: tool.description.clone(), + kind: CodeModeToolKind::Freeform, + input_schema: None, + output_schema: None, + }] + } + ToolSpec::Namespace(namespace) => namespace + .tools + .iter() + .map(|tool| match tool { + ResponsesApiNamespaceTool::Function(tool) => { + let tool_name = ToolName::namespaced(namespace.name.clone(), tool.name.clone()); + CodeModeToolDefinition { + name: tool_name.display(), + tool_name, + description: tool.description.clone(), + kind: CodeModeToolKind::Function, + input_schema: serde_json::to_value(&tool.parameters).ok(), + output_schema: tool.output_schema.clone(), + } + } + }) + .collect(), ToolSpec::LocalShell {} | ToolSpec::ImageGeneration { .. } | ToolSpec::ToolSearch { .. } - | ToolSpec::WebSearch { .. } => None, + | ToolSpec::WebSearch { .. } => Vec::new(), } } diff --git a/codex-rs/tools/src/code_mode_tests.rs b/codex-rs/tools/src/code_mode_tests.rs index ec30c6e8bf..15cca164ec 100644 --- a/codex-rs/tools/src/code_mode_tests.rs +++ b/codex-rs/tools/src/code_mode_tests.rs @@ -7,6 +7,7 @@ use crate::FreeformTool; use crate::FreeformToolFormat; use crate::JsonSchema; use crate::ResponsesApiTool; +use crate::ToolName; use crate::ToolSpec; use pretty_assertions::assert_eq; use serde_json::json; @@ -106,6 +107,7 @@ fn tool_spec_to_code_mode_tool_definition_returns_augmented_nested_tools() { tool_spec_to_code_mode_tool_definition(&spec), Some(codex_code_mode::ToolDefinition { name: "apply_patch".to_string(), + tool_name: ToolName::plain("apply_patch"), description: r#"Apply a patch exec tool declaration: @@ -184,6 +186,7 @@ fn create_wait_tool_matches_expected_spec() { fn create_code_mode_tool_matches_expected_spec() { let enabled_tools = vec![codex_code_mode::ToolDefinition { name: "update_plan".to_string(), + tool_name: ToolName::plain("update_plan"), description: "Update the plan".to_string(), kind: codex_code_mode::CodeModeToolKind::Function, input_schema: None, diff --git a/codex-rs/tools/src/lib.rs b/codex-rs/tools/src/lib.rs index c310222a0f..b30e0bffc2 100644 --- a/codex-rs/tools/src/lib.rs +++ b/codex-rs/tools/src/lib.rs @@ -18,7 +18,6 @@ mod responses_api; mod tool_config; mod tool_definition; mod tool_discovery; -mod tool_name; mod tool_registry_plan; mod tool_registry_plan_types; mod tool_spec; @@ -50,6 +49,7 @@ pub use code_mode::collect_code_mode_tool_definitions; pub use code_mode::create_code_mode_tool; pub use code_mode::create_wait_tool; pub use code_mode::tool_spec_to_code_mode_tool_definition; +pub use codex_protocol::ToolName; pub use dynamic_tool::parse_dynamic_tool; pub use image_detail::can_request_original_image_detail; pub use image_detail::normalize_output_image_detail; @@ -113,13 +113,13 @@ pub use tool_discovery::collect_tool_suggest_entries; pub use tool_discovery::create_tool_search_tool; pub use tool_discovery::create_tool_suggest_tool; pub use tool_discovery::filter_tool_suggest_discoverable_tools_for_client; -pub use tool_name::ToolName; pub use tool_registry_plan::build_tool_registry_plan; pub use tool_registry_plan_types::ToolHandlerKind; pub use tool_registry_plan_types::ToolHandlerSpec; pub use tool_registry_plan_types::ToolNamespace; pub use tool_registry_plan_types::ToolRegistryPlan; pub use tool_registry_plan_types::ToolRegistryPlanDeferredTool; +pub use tool_registry_plan_types::ToolRegistryPlanMcpTool; pub use tool_registry_plan_types::ToolRegistryPlanParams; pub use tool_spec::ConfiguredToolSpec; pub use tool_spec::ResponsesApiWebSearchFilters; diff --git a/codex-rs/tools/src/responses_api.rs b/codex-rs/tools/src/responses_api.rs index 50bb4f2232..205c075105 100644 --- a/codex-rs/tools/src/responses_api.rs +++ b/codex-rs/tools/src/responses_api.rs @@ -1,5 +1,6 @@ use crate::JsonSchema; use crate::ToolDefinition; +use crate::ToolName; use crate::parse_dynamic_tool; use crate::parse_mcp_tool; use codex_protocol::dynamic_tools::DynamicToolSpec; @@ -70,20 +71,22 @@ pub fn dynamic_tool_to_responses_api_tool( } pub fn mcp_tool_to_responses_api_tool( - name: String, + tool_name: &ToolName, tool: &rmcp::model::Tool, ) -> Result { Ok(tool_definition_to_responses_api_tool( - parse_mcp_tool(tool)?.renamed(name), + parse_mcp_tool(tool)?.renamed(tool_name.name.clone()), )) } pub fn mcp_tool_to_deferred_responses_api_tool( - name: String, + tool_name: &ToolName, tool: &rmcp::model::Tool, ) -> Result { Ok(tool_definition_to_responses_api_tool( - parse_mcp_tool(tool)?.renamed(name).into_deferred(), + parse_mcp_tool(tool)? + .renamed(tool_name.name.clone()) + .into_deferred(), )) } diff --git a/codex-rs/tools/src/responses_api_tests.rs b/codex-rs/tools/src/responses_api_tests.rs index c3ce4cec2b..799a2385d2 100644 --- a/codex-rs/tools/src/responses_api_tests.rs +++ b/codex-rs/tools/src/responses_api_tests.rs @@ -7,6 +7,7 @@ use super::mcp_tool_to_deferred_responses_api_tool; use super::tool_definition_to_responses_api_tool; use crate::JsonSchema; use crate::ToolDefinition; +use crate::ToolName; use codex_protocol::dynamic_tools::DynamicToolSpec; use pretty_assertions::assert_eq; use serde_json::json; @@ -106,19 +107,23 @@ fn mcp_tool_to_deferred_responses_api_tool_sets_defer_loading() { assert_eq!( mcp_tool_to_deferred_responses_api_tool( - "mcp__codex_apps__lookup_order".to_string(), + &ToolName::namespaced("mcp__codex_apps__", "lookup_order"), &tool, ) .expect("convert deferred tool"), ResponsesApiTool { - name: "mcp__codex_apps__lookup_order".to_string(), + name: "lookup_order".to_string(), description: "Look up an order".to_string(), strict: false, defer_loading: Some(true), - parameters: JsonSchema::object(BTreeMap::from([( + parameters: JsonSchema::object( + BTreeMap::from([( "order_id".to_string(), JsonSchema::string(/*description*/ None), - )]), Some(vec!["order_id".to_string()]), Some(false.into())), + )]), + Some(vec!["order_id".to_string()]), + Some(false.into()) + ), output_schema: None, } ); diff --git a/codex-rs/tools/src/tool_discovery.rs b/codex-rs/tools/src/tool_discovery.rs index f5d84a952e..e35da5735c 100644 --- a/codex-rs/tools/src/tool_discovery.rs +++ b/codex-rs/tools/src/tool_discovery.rs @@ -2,6 +2,7 @@ use crate::JsonSchema; use crate::ResponsesApiNamespace; use crate::ResponsesApiNamespaceTool; use crate::ResponsesApiTool; +use crate::ToolName; use crate::ToolSearchOutputTool; use crate::ToolSpec; use crate::mcp_tool_to_deferred_responses_api_tool; @@ -242,7 +243,8 @@ pub fn collect_tool_search_output_tools<'a>( let tools = tools .iter() .map(|tool| { - mcp_tool_to_deferred_responses_api_tool(tool.tool_name.to_string(), tool.tool) + let tool_name = ToolName::namespaced(tool.tool_namespace, tool.tool_name); + mcp_tool_to_deferred_responses_api_tool(&tool_name, tool.tool) .map(ResponsesApiNamespaceTool::Function) }) .collect::, _>>()?; diff --git a/codex-rs/tools/src/tool_registry_plan.rs b/codex-rs/tools/src/tool_registry_plan.rs index 82b4c3a142..cec938962c 100644 --- a/codex-rs/tools/src/tool_registry_plan.rs +++ b/codex-rs/tools/src/tool_registry_plan.rs @@ -1,12 +1,13 @@ use crate::CommandToolOptions; use crate::REQUEST_USER_INPUT_TOOL_NAME; +use crate::ResponsesApiNamespace; +use crate::ResponsesApiNamespaceTool; use crate::ShellToolOptions; use crate::SpawnAgentToolOptions; use crate::TOOL_SEARCH_DEFAULT_LIMIT; use crate::TOOL_SEARCH_TOOL_NAME; use crate::TOOL_SUGGEST_TOOL_NAME; use crate::ToolHandlerKind; -use crate::ToolName; use crate::ToolRegistryPlan; use crate::ToolRegistryPlanParams; use crate::ToolSearchSource; @@ -61,7 +62,6 @@ use crate::request_user_input_tool_description; use crate::tool_registry_plan_types::agent_type_description; use codex_protocol::openai_models::ApplyPatchToolType; use codex_protocol::openai_models::ConfigShellToolType; -use rmcp::model::Tool as McpTool; use std::collections::BTreeMap; pub fn build_tool_registry_plan( @@ -76,9 +76,9 @@ pub fn build_tool_registry_plan( .tool_namespaces .into_iter() .flatten() - .map(|(name, detail)| { + .map(|(namespace, detail)| { ( - name.clone(), + namespace.clone(), codex_code_mode::ToolNamespaceDescription { name: detail.name.clone(), description: detail.description.clone().unwrap_or_default(), @@ -100,9 +100,8 @@ pub fn build_tool_registry_plan( .iter() .map(|configured_tool| &configured_tool.spec), ); - enabled_tools.sort_by(|left, right| { - compare_code_mode_tool_names(&left.name, &right.name, &namespace_descriptions) - }); + enabled_tools + .sort_by(|left, right| compare_code_mode_tools(left, right, &namespace_descriptions)); plan.push_spec( create_code_mode_tool( &enabled_tools, @@ -266,10 +265,7 @@ pub fn build_tool_registry_plan( plan.register_handler(TOOL_SEARCH_TOOL_NAME, ToolHandlerKind::ToolSearch); for tool in deferred_mcp_tools { - plan.register_handler( - ToolName::namespaced(tool.tool_namespace, tool.tool_name), - ToolHandlerKind::Mcp, - ); + plan.register_handler(tool.name.clone(), ToolHandlerKind::Mcp); } } @@ -471,28 +467,56 @@ pub fn build_tool_registry_plan( } if let Some(mcp_tools) = params.mcp_tools { - let mut entries: Vec<(String, &McpTool)> = mcp_tools - .iter() - .map(|(name, tool)| (name.clone(), tool)) - .collect(); - entries.sort_by(|left, right| left.0.cmp(&right.0)); + let mut entries = mcp_tools.to_vec(); + entries.sort_by_key(|tool| tool.name.display()); + let mut namespace_entries = BTreeMap::new(); - for (name, tool) in entries { - match mcp_tool_to_responses_api_tool(name.clone(), tool) { - Ok(converted_tool) => { - plan.push_spec( - ToolSpec::Function(converted_tool), - /*supports_parallel_tool_calls*/ false, - config.code_mode_enabled, - ); - plan.register_handler(name, ToolHandlerKind::Mcp); - } - Err(error) => { - tracing::error!( - "Failed to convert {name:?} MCP tool to OpenAI tool: {error:?}" - ); + for tool in entries { + let Some(namespace) = tool.name.namespace.as_ref() else { + let tool_name = &tool.name; + tracing::error!("Skipping MCP tool `{tool_name}`: MCP tools must be namespaced"); + continue; + }; + namespace_entries + .entry(namespace.clone()) + .or_insert_with(Vec::new) + .push(tool); + } + + for (namespace, mut entries) in namespace_entries { + entries.sort_by_key(|tool| tool.name.name.clone()); + let description = params + .tool_namespaces + .and_then(|namespaces| namespaces.get(&namespace)) + .and_then(|namespace| namespace.description.clone()) + .unwrap_or_default(); + let mut tools = Vec::new(); + for tool in entries { + match mcp_tool_to_responses_api_tool(&tool.name, tool.tool) { + Ok(converted_tool) => { + tools.push(ResponsesApiNamespaceTool::Function(converted_tool)); + plan.register_handler(tool.name, ToolHandlerKind::Mcp); + } + Err(error) => { + let tool_name = &tool.name; + tracing::error!( + "Failed to convert `{tool_name}` MCP tool to OpenAI tool: {error:?}" + ); + } } } + + if !tools.is_empty() { + plan.push_spec( + ToolSpec::Namespace(ResponsesApiNamespace { + name: namespace, + description, + tools, + }), + /*supports_parallel_tool_calls*/ false, + config.code_mode_enabled, + ); + } } } @@ -518,41 +542,31 @@ pub fn build_tool_registry_plan( plan } -fn compare_code_mode_tool_names( - left_name: &str, - right_name: &str, +fn compare_code_mode_tools( + left: &codex_code_mode::ToolDefinition, + right: &codex_code_mode::ToolDefinition, namespace_descriptions: &BTreeMap, ) -> std::cmp::Ordering { - let left_namespace = code_mode_namespace_name(left_name, namespace_descriptions); - let right_namespace = code_mode_namespace_name(right_name, namespace_descriptions); + let left_namespace = code_mode_namespace_name(left, namespace_descriptions); + let right_namespace = code_mode_namespace_name(right, namespace_descriptions); left_namespace .cmp(&right_namespace) - .then_with(|| { - code_mode_function_name(left_name, left_namespace) - .cmp(code_mode_function_name(right_name, right_namespace)) - }) - .then_with(|| left_name.cmp(right_name)) + .then_with(|| left.tool_name.name.cmp(&right.tool_name.name)) + .then_with(|| left.name.cmp(&right.name)) } fn code_mode_namespace_name<'a>( - name: &str, + tool: &codex_code_mode::ToolDefinition, namespace_descriptions: &'a BTreeMap, ) -> Option<&'a str> { - namespace_descriptions - .get(name) + tool.tool_name + .namespace + .as_ref() + .and_then(|namespace| namespace_descriptions.get(namespace)) .map(|namespace_description| namespace_description.name.as_str()) } -fn code_mode_function_name<'a>(name: &'a str, namespace: Option<&str>) -> &'a str { - namespace - .and_then(|namespace| { - name.strip_prefix(namespace) - .and_then(|suffix| suffix.strip_prefix("__")) - }) - .unwrap_or(name) -} - #[cfg(test)] #[path = "tool_registry_plan_tests.rs"] mod tests; diff --git a/codex-rs/tools/src/tool_registry_plan_tests.rs b/codex-rs/tools/src/tool_registry_plan_tests.rs index 69b8b55b54..7b9e8ed9ac 100644 --- a/codex-rs/tools/src/tool_registry_plan_tests.rs +++ b/codex-rs/tools/src/tool_registry_plan_tests.rs @@ -7,12 +7,15 @@ use crate::FreeformTool; use crate::JsonSchema; use crate::JsonSchemaPrimitiveType; use crate::JsonSchemaType; +use crate::ResponsesApiNamespaceTool; use crate::ResponsesApiTool; use crate::ResponsesApiWebSearchFilters; use crate::ResponsesApiWebSearchUserLocation; use crate::ToolHandlerSpec; +use crate::ToolName; use crate::ToolNamespace; use crate::ToolRegistryPlanDeferredTool; +use crate::ToolRegistryPlanMcpTool; use crate::ToolsConfigParams; use crate::WaitAgentTimeoutOptions; use crate::mcp_call_tool_result_output_schema; @@ -1075,7 +1078,7 @@ fn test_build_specs_mcp_tools_converted() { let (tools, _) = build_specs( &tools_config, Some(HashMap::from([( - "test_server/do_something_cool".to_string(), + ToolName::namespaced("test_server/", "do_something_cool"), mcp_tool( "do_something_cool", "Do something cool", @@ -1101,11 +1104,11 @@ fn test_build_specs_mcp_tools_converted() { &[], ); - let tool = find_tool(&tools, "test_server/do_something_cool"); + let tool = find_namespace_function_tool(&tools, "test_server/", "do_something_cool"); assert_eq!( - &tool.spec, - &ToolSpec::Function(ResponsesApiTool { - name: "test_server/do_something_cool".to_string(), + tool, + &ResponsesApiTool { + name: "do_something_cool".to_string(), parameters: JsonSchema::object( BTreeMap::from([ ( @@ -1144,7 +1147,7 @@ fn test_build_specs_mcp_tools_converted() { strict: false, output_schema: Some(mcp_call_tool_result_output_schema(serde_json::json!({}))), defer_loading: None, - }) + } ); } @@ -1167,16 +1170,16 @@ fn test_build_specs_mcp_tools_sorted_by_name() { let tools_map = HashMap::from([ ( - "test_server/do".to_string(), - mcp_tool("a", "a", serde_json::json!({"type": "object"})), + ToolName::namespaced("test_server/", "do"), + mcp_tool("do", "a", serde_json::json!({"type": "object"})), ), ( - "test_server/something".to_string(), - mcp_tool("b", "b", serde_json::json!({"type": "object"})), + ToolName::namespaced("test_server/", "something"), + mcp_tool("something", "b", serde_json::json!({"type": "object"})), ), ( - "test_server/cool".to_string(), - mcp_tool("c", "c", serde_json::json!({"type": "object"})), + ToolName::namespaced("test_server/", "cool"), + mcp_tool("cool", "c", serde_json::json!({"type": "object"})), ), ]); @@ -1187,17 +1190,14 @@ fn test_build_specs_mcp_tools_sorted_by_name() { &[], ); - let mcp_names: Vec<_> = tools - .iter() - .map(|tool| tool.name().to_string()) - .filter(|name| name.starts_with("test_server/")) - .collect(); - let expected = vec![ - "test_server/cool".to_string(), - "test_server/do".to_string(), - "test_server/something".to_string(), - ]; - assert_eq!(mcp_names, expected); + assert_eq!( + namespace_function_names(&tools, "test_server/"), + vec![ + "cool".to_string(), + "do".to_string(), + "something".to_string(), + ] + ); } #[test] @@ -1222,7 +1222,7 @@ fn search_tool_description_lists_each_mcp_source_once() { &tools_config, Some(HashMap::from([ ( - "mcp__codex_apps__calendar_create_event".to_string(), + ToolName::namespaced("mcp__codex_apps__calendar", "_create_event"), mcp_tool( "calendar_create_event", "Create calendar event", @@ -1230,7 +1230,7 @@ fn search_tool_description_lists_each_mcp_source_once() { ), ), ( - "mcp__rmcp__echo".to_string(), + ToolName::namespaced("mcp__rmcp__", "echo"), mcp_tool("echo", "Echo", serde_json::json!({"type": "object"})), ), ])), @@ -1577,7 +1577,7 @@ fn code_mode_augments_mcp_tool_descriptions_with_namespaced_sample() { let (tools, _) = build_specs( &tools_config, Some(HashMap::from([( - "mcp__sample__echo".to_string(), + ToolName::namespaced("mcp__sample__", "echo"), mcp_tool( "echo", "Echo text", @@ -1595,11 +1595,8 @@ fn code_mode_augments_mcp_tool_descriptions_with_namespaced_sample() { &[], ); - let ToolSpec::Function(ResponsesApiTool { description, .. }) = - &find_tool(&tools, "mcp__sample__echo").spec - else { - panic!("expected function tool"); - }; + let ResponsesApiTool { description, .. } = + find_namespace_function_tool(&tools, "mcp__sample__", "echo"); assert_eq!( description, @@ -1633,7 +1630,7 @@ fn code_mode_preserves_nullable_and_literal_mcp_input_shapes() { let (tools, _) = build_specs( &tools_config, Some(HashMap::from([( - "mcp__sample__fn".to_string(), + ToolName::namespaced("mcp__sample__", "fn"), mcp_tool( "fn", "Sample fn", @@ -1684,11 +1681,8 @@ fn code_mode_preserves_nullable_and_literal_mcp_input_shapes() { &[], ); - let ToolSpec::Function(ResponsesApiTool { description, .. }) = - &find_tool(&tools, "mcp__sample__fn").spec - else { - panic!("expected function tool"); - }; + let ResponsesApiTool { description, .. } = + find_namespace_function_tool(&tools, "mcp__sample__", "fn"); assert!(description.contains( r#"exec tool declaration: @@ -1851,7 +1845,7 @@ fn search_capable_model_info() -> ModelInfo { fn build_specs<'a>( config: &ToolsConfig, - mcp_tools: Option>, + mcp_tools: Option>, deferred_mcp_tools: Option>>, dynamic_tools: &[DynamicToolSpec], ) -> (Vec, Vec) { @@ -1866,7 +1860,7 @@ fn build_specs<'a>( fn build_specs_with_discoverable_tools<'a>( config: &ToolsConfig, - mcp_tools: Option>, + mcp_tools: Option>, deferred_mcp_tools: Option>>, discoverable_tools: Option>, dynamic_tools: &[DynamicToolSpec], @@ -1883,16 +1877,25 @@ fn build_specs_with_discoverable_tools<'a>( fn build_specs_with_optional_tool_namespaces<'a>( config: &ToolsConfig, - mcp_tools: Option>, + mcp_tools: Option>, deferred_mcp_tools: Option>>, tool_namespaces: Option>, discoverable_tools: Option>, dynamic_tools: &[DynamicToolSpec], ) -> (Vec, Vec) { + let mcp_tool_inputs = mcp_tools.as_ref().map(|mcp_tools| { + mcp_tools + .iter() + .map(|(name, tool)| ToolRegistryPlanMcpTool { + name: name.clone(), + tool, + }) + .collect::>() + }); let plan = build_tool_registry_plan( config, ToolRegistryPlanParams { - mcp_tools: mcp_tools.as_ref(), + mcp_tools: mcp_tool_inputs.as_deref(), deferred_mcp_tools: deferred_mcp_tools.as_deref(), tool_namespaces: tool_namespaces.as_ref(), discoverable_tools: discoverable_tools.as_deref(), @@ -1968,16 +1971,16 @@ fn code_mode_augments_mcp_tool_descriptions_with_structured_output_sample() { let (tools, _) = build_specs( &tools_config, - Some(HashMap::from([("mcp__sample__echo".to_string(), tool)])), + Some(HashMap::from([( + ToolName::namespaced("mcp__sample__", "echo"), + tool, + )])), /*deferred_mcp_tools*/ None, &[], ); - let ToolSpec::Function(ResponsesApiTool { description, .. }) = - &find_tool(&tools, "mcp__sample__echo").spec - else { - panic!("expected function tool"); - }; + let ResponsesApiTool { description, .. } = + find_namespace_function_tool(&tools, "mcp__sample__", "echo"); assert_eq!( description, @@ -2017,8 +2020,7 @@ fn deferred_mcp_tool<'a>( connector_description: Option<&'a str>, ) -> ToolRegistryPlanDeferredTool<'a> { ToolRegistryPlanDeferredTool { - tool_name, - tool_namespace, + name: ToolName::namespaced(tool_namespace, tool_name), server_name, connector_name, connector_description, @@ -2089,6 +2091,39 @@ fn find_tool<'a>(tools: &'a [ConfiguredToolSpec], expected_name: &str) -> &'a Co .unwrap_or_else(|| panic!("expected tool {expected_name}")) } +fn find_namespace_function_tool<'a>( + tools: &'a [ConfiguredToolSpec], + expected_namespace: &str, + expected_name: &str, +) -> &'a ResponsesApiTool { + let namespace_tool = find_tool(tools, expected_namespace); + let ToolSpec::Namespace(namespace) = &namespace_tool.spec else { + panic!("expected namespace tool {expected_namespace}"); + }; + namespace + .tools + .iter() + .find_map(|tool| match tool { + ResponsesApiNamespaceTool::Function(tool) if tool.name == expected_name => Some(tool), + _ => None, + }) + .unwrap_or_else(|| panic!("expected tool {expected_namespace}{expected_name} in namespace")) +} + +fn namespace_function_names(tools: &[ConfiguredToolSpec], expected_namespace: &str) -> Vec { + let namespace_tool = find_tool(tools, expected_namespace); + let ToolSpec::Namespace(namespace) = &namespace_tool.spec else { + panic!("expected namespace tool {expected_namespace}"); + }; + namespace + .tools + .iter() + .map(|tool| match tool { + ResponsesApiNamespaceTool::Function(tool) => tool.name.clone(), + }) + .collect() +} + fn expect_object_schema( schema: &JsonSchema, ) -> (&BTreeMap, Option<&Vec>) { @@ -2137,6 +2172,17 @@ fn strip_descriptions_tool(spec: &mut ToolSpec) { ToolSpec::Function(ResponsesApiTool { parameters, .. }) => { strip_descriptions_schema(parameters); } + ToolSpec::Namespace(namespace) => { + for tool in &mut namespace.tools { + match tool { + ResponsesApiNamespaceTool::Function(ResponsesApiTool { + parameters, .. + }) => { + strip_descriptions_schema(parameters); + } + } + } + } ToolSpec::Freeform(FreeformTool { .. }) | ToolSpec::LocalShell {} | ToolSpec::ImageGeneration { .. } diff --git a/codex-rs/tools/src/tool_registry_plan_types.rs b/codex-rs/tools/src/tool_registry_plan_types.rs index 7459954dcd..b9d66a0c2c 100644 --- a/codex-rs/tools/src/tool_registry_plan_types.rs +++ b/codex-rs/tools/src/tool_registry_plan_types.rs @@ -6,7 +6,6 @@ use crate::ToolsConfig; use crate::WaitAgentTimeoutOptions; use crate::augment_tool_spec_for_code_mode; use codex_protocol::dynamic_tools::DynamicToolSpec; -use rmcp::model::Tool as McpTool; use std::collections::HashMap; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -58,7 +57,7 @@ pub struct ToolRegistryPlan { #[derive(Debug, Clone, Copy)] pub struct ToolRegistryPlanParams<'a> { - pub mcp_tools: Option<&'a HashMap>, + pub mcp_tools: Option<&'a [ToolRegistryPlanMcpTool<'a>]>, pub deferred_mcp_tools: Option<&'a [ToolRegistryPlanDeferredTool<'a>]>, pub tool_namespaces: Option<&'a HashMap>, pub discoverable_tools: Option<&'a [DiscoverableTool]>, @@ -73,10 +72,18 @@ pub struct ToolNamespace { pub description: Option, } -#[derive(Debug, Clone, Copy)] +/// Direct MCP tool metadata needed to expose the Responses API namespace tool +/// while registering its runtime handler with the canonical namespace/name +/// identity. +#[derive(Debug, Clone)] +pub struct ToolRegistryPlanMcpTool<'a> { + pub name: ToolName, + pub tool: &'a rmcp::model::Tool, +} + +#[derive(Debug, Clone)] pub struct ToolRegistryPlanDeferredTool<'a> { - pub tool_name: &'a str, - pub tool_namespace: &'a str, + pub name: ToolName, pub server_name: &'a str, pub connector_name: Option<&'a str>, pub connector_description: Option<&'a str>, diff --git a/codex-rs/tools/src/tool_spec.rs b/codex-rs/tools/src/tool_spec.rs index 24644e2260..81db2f5584 100644 --- a/codex-rs/tools/src/tool_spec.rs +++ b/codex-rs/tools/src/tool_spec.rs @@ -1,5 +1,6 @@ use crate::FreeformTool; use crate::JsonSchema; +use crate::ResponsesApiNamespace; use crate::ResponsesApiTool; use codex_protocol::config_types::WebSearchConfig; use codex_protocol::config_types::WebSearchContextSize; @@ -20,6 +21,8 @@ const WEB_SEARCH_TEXT_AND_IMAGE_CONTENT_TYPES: [&str; 2] = ["text", "image"]; pub enum ToolSpec { #[serde(rename = "function")] Function(ResponsesApiTool), + #[serde(rename = "namespace")] + Namespace(ResponsesApiNamespace), #[serde(rename = "tool_search")] ToolSearch { execution: String, @@ -57,6 +60,7 @@ impl ToolSpec { pub fn name(&self) -> &str { match self { ToolSpec::Function(tool) => tool.name.as_str(), + ToolSpec::Namespace(namespace) => namespace.name.as_str(), ToolSpec::ToolSearch { .. } => "tool_search", ToolSpec::LocalShell {} => "local_shell", ToolSpec::ImageGeneration { .. } => "image_generation", diff --git a/codex-rs/tools/src/tool_spec_tests.rs b/codex-rs/tools/src/tool_spec_tests.rs index c94f54e6b5..0b98625626 100644 --- a/codex-rs/tools/src/tool_spec_tests.rs +++ b/codex-rs/tools/src/tool_spec_tests.rs @@ -1,4 +1,5 @@ use super::ConfiguredToolSpec; +use super::ResponsesApiNamespace; use super::ResponsesApiWebSearchFilters; use super::ResponsesApiWebSearchUserLocation; use super::ToolSpec; @@ -6,6 +7,7 @@ use crate::AdditionalProperties; use crate::FreeformTool; use crate::FreeformToolFormat; use crate::JsonSchema; +use crate::ResponsesApiNamespaceTool; use crate::ResponsesApiTool; use crate::create_tools_json_for_responses_api; use codex_protocol::config_types::WebSearchContextSize; @@ -34,6 +36,15 @@ fn tool_spec_name_covers_all_variants() { .name(), "lookup_order" ); + assert_eq!( + ToolSpec::Namespace(ResponsesApiNamespace { + name: "mcp__demo__".to_string(), + description: "Demo tools".to_string(), + tools: Vec::new(), + }) + .name(), + "mcp__demo__" + ); assert_eq!( ToolSpec::ToolSearch { execution: "sync".to_string(), @@ -163,6 +174,51 @@ fn create_tools_json_for_responses_api_includes_top_level_name() { ); } +#[test] +fn namespace_tool_spec_serializes_expected_wire_shape() { + assert_eq!( + serde_json::to_value(ToolSpec::Namespace(ResponsesApiNamespace { + name: "mcp__demo__".to_string(), + description: "Demo tools".to_string(), + tools: vec![ResponsesApiNamespaceTool::Function(ResponsesApiTool { + name: "lookup_order".to_string(), + description: "Look up an order".to_string(), + strict: false, + defer_loading: None, + parameters: JsonSchema::object( + BTreeMap::from([( + "order_id".to_string(), + JsonSchema::string(/*description*/ None), + )]), + /*required*/ None, + /*additional_properties*/ None, + ), + output_schema: None, + })], + })) + .expect("serialize namespace tool"), + json!({ + "type": "namespace", + "name": "mcp__demo__", + "description": "Demo tools", + "tools": [ + { + "type": "function", + "name": "lookup_order", + "description": "Look up an order", + "strict": false, + "parameters": { + "type": "object", + "properties": { + "order_id": { "type": "string" }, + }, + }, + }, + ], + }) + ); +} + #[test] fn web_search_tool_spec_serializes_expected_wire_shape() { assert_eq!( From ea135279618aecff7934d63c8baf742cfa6d47d2 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 15 Apr 2026 14:51:20 +0100 Subject: [PATCH 17/84] nit: doc (#17941) --- codex-rs/app-server/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index bffbde3ebe..5cfe052b50 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -144,7 +144,7 @@ Example with notification opt-out: - `thread/read` โ€” read a stored thread by id without resuming it; optionally include turns via `includeTurns`. The returned `thread` includes `status` (`ThreadStatus`), defaulting to `notLoaded` when the thread is not currently loaded. - `thread/metadata/update` โ€” patch stored thread metadata in sqlite; currently supports updating persisted `gitInfo` fields and returns the refreshed `thread`. - `thread/memoryMode/set` โ€” experimental; set a threadโ€™s persisted memory eligibility to `"enabled"` or `"disabled"` for either a loaded thread or a stored rollout; returns `{}` on success. -- `memory/reset` โ€” experimental; clear the current `CODEX_HOME/memories` directory and reset persisted memory stage data in sqlite, disabling memory mode for existing enabled threads so old rollouts are not immediately regenerated; returns `{}` on success. +- `memory/reset` โ€” experimental; clear the current `CODEX_HOME/memories` directory and reset persisted memory stage data in sqlite while preserving existing thread memory modes; returns `{}` on success. - `thread/status/changed` โ€” notification emitted when a loaded threadโ€™s status changes (`threadId` + new `status`). - `thread/archive` โ€” move a threadโ€™s rollout file into the archived directory; returns `{}` on success and emits `thread/archived`. - `thread/unsubscribe` โ€” unsubscribe this connection from thread turn/item events. If this was the last subscriber, the server keeps the thread loaded and unloads it only after it has had no subscribers and no thread activity for 30 minutes, then emits `thread/closed`. @@ -417,7 +417,7 @@ Experimental: use `thread/memoryMode/set` to change whether a thread remains eli { "id": 26, "result": {} } ``` -Experimental: use `memory/reset` to clear local memory artifacts and sqlite-backed memory stage data for the current Codex home. This also disables memory mode for existing enabled threads to keep old rollouts from repopulating memories immediately. +Experimental: use `memory/reset` to clear local memory artifacts and sqlite-backed memory stage data for the current Codex home. This preserves existing thread memory modes; use `thread/memoryMode/set` separately when a thread's future memory eligibility should change. ```json { "method": "memory/reset", "id": 27 } From ec13aaac892565578f396a82058be0914d018690 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 15 Apr 2026 15:00:27 +0100 Subject: [PATCH 18/84] feat: sanitize rollouts before phase 1 (#17938) --- codex-rs/core/src/memories/phase1.rs | 5 +++-- codex-rs/core/src/memories/phase1_tests.rs | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/codex-rs/core/src/memories/phase1.rs b/codex-rs/core/src/memories/phase1.rs index 37cb0e9f48..c005f4da4e 100644 --- a/codex-rs/core/src/memories/phase1.rs +++ b/codex-rs/core/src/memories/phase1.rs @@ -477,9 +477,10 @@ mod job { } }) .collect::>(); - serde_json::to_string(&filtered).map_err(|err| { + let serialized = serde_json::to_string(&filtered).map_err(|err| { CodexErr::InvalidRequest(format!("failed to serialize rollout memory: {err}")) - }) + })?; + Ok(redact_secrets(serialized)) } fn sanitize_response_item_for_memories(item: &ResponseItem) -> Option { diff --git a/codex-rs/core/src/memories/phase1_tests.rs b/codex-rs/core/src/memories/phase1_tests.rs index 9d824ab2b1..89bde1a877 100644 --- a/codex-rs/core/src/memories/phase1_tests.rs +++ b/codex-rs/core/src/memories/phase1_tests.rs @@ -3,6 +3,8 @@ use super::JobResult; use super::aggregate_stats; use super::job::serialize_filtered_rollout_response_items; use codex_protocol::models::ContentItem; +use codex_protocol::models::FunctionCallOutputBody; +use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::TokenUsage; @@ -72,6 +74,25 @@ fn serializes_memory_rollout_with_agents_removed_but_environment_kept() { ); } +#[test] +fn serializes_memory_rollout_redacts_secrets_before_prompt_upload() { + let serialized = serialize_filtered_rollout_response_items(&[RolloutItem::ResponseItem( + ResponseItem::FunctionCallOutput { + call_id: "call_123".to_string(), + output: FunctionCallOutputPayload { + body: FunctionCallOutputBody::Text( + r#"{"token":"sk-abcdefghijklmnopqrstuvwxyz123456"}"#.to_string(), + ), + success: Some(true), + }, + }, + )]) + .expect("serialize"); + + assert!(!serialized.contains("sk-abcdefghijklmnopqrstuvwxyz123456")); + assert!(serialized.contains("[REDACTED_SECRET]")); +} + #[test] fn count_outcomes_sums_token_usage_across_all_jobs() { let counts = aggregate_stats(vec![ From da86cedbd439d38fbd7e613e4e88f8f6f138debb Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 15 Apr 2026 15:34:25 +0100 Subject: [PATCH 19/84] feat: reset memories button (#17937) Screenshot 2026-04-15 at 14 35 02 --- codex-rs/tui/src/app.rs | 44 +++ codex-rs/tui/src/app_event.rs | 3 + codex-rs/tui/src/app_server_session.rs | 14 + .../src/bottom_pane/memories_settings_view.rs | 267 ++++++++++++++---- ...t__tests__memories_reset_confirmation.snap | 12 + ...idget__tests__memories_settings_popup.snap | 4 +- .../chatwidget/tests/popups_and_settings.rs | 32 +++ 7 files changed, 314 insertions(+), 62 deletions(-) create mode 100644 codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_reset_confirmation.snap diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 13f7371615..026bd05d7c 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -1561,6 +1561,18 @@ impl App { } } + async fn reset_memories_with_app_server(&mut self, app_server: &mut AppServerSession) { + if let Err(err) = app_server.memory_reset().await { + tracing::error!(error = %err, "failed to reset memories"); + self.chat_widget + .add_error_message(format!("Failed to reset memories: {err}")); + return; + } + + self.chat_widget + .add_info_message("Reset local memories.".to_string(), /*hint*/ None); + } + fn open_url_in_browser(&mut self, url: String) { if let Err(err) = webbrowser::open(&url) { self.chat_widget @@ -5413,6 +5425,9 @@ impl App { ) .await; } + AppEvent::ResetMemories => { + self.reset_memories_with_app_server(app_server).await; + } AppEvent::SkipNextWorldWritableScan => { self.windows_sandbox.skip_world_writable_scan_once = true; } @@ -8121,6 +8136,35 @@ mod tests { Ok(()) } + #[tokio::test] + async fn reset_memories_clears_local_memory_directories() -> Result<()> { + let (mut app, _app_event_rx, _op_rx) = make_test_app_with_channels().await; + let codex_home = tempdir()?; + app.config.codex_home = codex_home.path().to_path_buf().abs(); + app.config.sqlite_home = codex_home.path().to_path_buf(); + + let memory_root = codex_home.path().join("memories"); + let extensions_root = codex_home.path().join("memories_extensions"); + std::fs::create_dir_all(memory_root.join("rollout_summaries"))?; + std::fs::create_dir_all(&extensions_root)?; + std::fs::write(memory_root.join("MEMORY.md"), "stale memory\n")?; + std::fs::write( + memory_root.join("rollout_summaries").join("stale.md"), + "stale summary\n", + )?; + std::fs::write(extensions_root.join("stale.txt"), "stale extension\n")?; + + let mut app_server = crate::start_embedded_app_server_for_picker(&app.config).await?; + + app.reset_memories_with_app_server(&mut app_server).await; + + assert_eq!(std::fs::read_dir(&memory_root)?.count(), 0); + assert_eq!(std::fs::read_dir(&extensions_root)?.count(), 0); + + app_server.shutdown().await?; + Ok(()) + } + #[tokio::test] async fn update_feature_flags_enabling_guardian_selects_guardian_approvals() -> Result<()> { let (mut app, mut app_event_rx, mut op_rx) = make_test_app_with_channels().await; diff --git a/codex-rs/tui/src/app_event.rs b/codex-rs/tui/src/app_event.rs index a0476640b2..a3b31aa988 100644 --- a/codex-rs/tui/src/app_event.rs +++ b/codex-rs/tui/src/app_event.rs @@ -470,6 +470,9 @@ pub(crate) enum AppEvent { generate_memories: bool, }, + /// Clear all persisted local memory artifacts via the app-server. + ResetMemories, + /// Update whether the full access warning prompt has been acknowledged. UpdateFullAccessWarningAcknowledged(bool), diff --git a/codex-rs/tui/src/app_server_session.rs b/codex-rs/tui/src/app_server_session.rs index 3ce12d2439..47a06738e4 100644 --- a/codex-rs/tui/src/app_server_session.rs +++ b/codex-rs/tui/src/app_server_session.rs @@ -18,6 +18,7 @@ use codex_app_server_protocol::GetAccountParams; use codex_app_server_protocol::GetAccountRateLimitsResponse; use codex_app_server_protocol::GetAccountResponse; use codex_app_server_protocol::JSONRPCErrorError; +use codex_app_server_protocol::MemoryResetResponse; use codex_app_server_protocol::Model as ApiModel; use codex_app_server_protocol::ModelListParams; use codex_app_server_protocol::ModelListResponse; @@ -535,6 +536,19 @@ impl AppServerSession { Ok(()) } + pub(crate) async fn memory_reset(&mut self) -> Result<()> { + let request_id = self.next_request_id(); + let _: MemoryResetResponse = self + .client + .request_typed(ClientRequest::MemoryReset { + request_id, + params: None, + }) + .await + .wrap_err("memory/reset failed in TUI")?; + Ok(()) + } + pub(crate) async fn thread_unsubscribe(&mut self, thread_id: ThreadId) -> Result<()> { let request_id = self.next_request_id(); let _: ThreadUnsubscribeResponse = self diff --git a/codex-rs/tui/src/bottom_pane/memories_settings_view.rs b/codex-rs/tui/src/bottom_pane/memories_settings_view.rs index 51b0ef81ba..d2d3c00ce3 100644 --- a/codex-rs/tui/src/bottom_pane/memories_settings_view.rs +++ b/codex-rs/tui/src/bottom_pane/memories_settings_view.rs @@ -12,6 +12,7 @@ use ratatui::widgets::Widget; use crate::app_event::AppEvent; use crate::app_event_sender::AppEventSender; +use crate::bottom_pane::popup_consts::standard_popup_hint_line; use crate::key_hint; use crate::render::Insets; use crate::render::RectExt as _; @@ -35,21 +36,32 @@ enum MemoriesSetting { Generate, } -struct MemoriesSettingItem { - setting: MemoriesSetting, - name: &'static str, - description: &'static str, - enabled: bool, +#[derive(Clone, Copy, PartialEq, Eq)] +enum MemoriesAction { + Reset, +} + +enum MemoriesMenuItem { + Setting { + setting: MemoriesSetting, + name: &'static str, + description: &'static str, + enabled: bool, + }, + Action { + action: MemoriesAction, + name: &'static str, + description: &'static str, + }, } pub(crate) struct MemoriesSettingsView { - items: Vec, + items: Vec, state: ScrollState, + reset_confirmation: Option, complete: bool, app_event_tx: AppEventSender, - header: Box, docs_link: Line<'static>, - footer_hint: Line<'static>, } impl MemoriesSettingsView { @@ -58,36 +70,34 @@ impl MemoriesSettingsView { generate_memories: bool, app_event_tx: AppEventSender, ) -> Self { - let mut header = ColumnRenderable::new(); - header.push(Line::from("Memories".bold())); - header.push(Line::from( - "Choose how Codex uses and creates memories. Changes are saved to config.toml".dim(), - )); - let mut view = Self { items: vec![ - MemoriesSettingItem { + MemoriesMenuItem::Setting { setting: MemoriesSetting::Use, name: "Use memories", description: "Use memories in the following threads. Applied at next thread.", enabled: use_memories, }, - MemoriesSettingItem { + MemoriesMenuItem::Setting { setting: MemoriesSetting::Generate, name: "Generate memories", description: "Generate memories from the following threads. Current thread included.", enabled: generate_memories, }, + MemoriesMenuItem::Action { + action: MemoriesAction::Reset, + name: "Reset all memories", + description: "Clear local memory files and summaries. Existing threads stay intact.", + }, ], state: ScrollState::new(), + reset_confirmation: None, complete: false, app_event_tx, - header: Box::new(header), docs_link: Line::from(vec![ "Learn more: ".dim(), MEMORIES_DOC_URL.cyan().underlined(), ]), - footer_hint: memories_settings_hint_line(), }; view.initialize_selection(); view @@ -97,29 +107,93 @@ impl MemoriesSettingsView { self.state.selected_idx = (!self.items.is_empty()).then_some(0); } + fn settings_header(&self) -> ColumnRenderable<'_> { + let mut header = ColumnRenderable::new(); + header.push(Line::from("Memories".bold())); + header.push(Line::from( + "Choose how Codex uses and creates memories. Changes are saved to config.toml".dim(), + )); + header + } + + fn reset_confirmation_header(&self) -> ColumnRenderable<'_> { + let mut header = ColumnRenderable::new(); + header.push(Line::from("Reset all memories?".bold())); + header.push(Line::from( + "This clears local memory files and rollout summaries for the current Codex home." + .dim(), + )); + header + } + + fn active_state(&self) -> &ScrollState { + self.reset_confirmation.as_ref().unwrap_or(&self.state) + } + + fn active_state_mut(&mut self) -> &mut ScrollState { + self.reset_confirmation.as_mut().unwrap_or(&mut self.state) + } + fn visible_len(&self) -> usize { - self.items.len() + if self.reset_confirmation.is_some() { + 2 + } else { + self.items.len() + } } fn build_rows(&self) -> Vec { - let mut rows = Vec::with_capacity(self.items.len()); - let selected_idx = self.state.selected_idx; - for (idx, item) in self.items.iter().enumerate() { - let prefix = if selected_idx == Some(idx) { - 'โ€บ' - } else { - ' ' - }; - let marker = if item.enabled { 'x' } else { ' ' }; - let name = format!("{prefix} [{marker}] {}", item.name); - rows.push(GenericDisplayRow { - name, - description: Some(item.description.to_string()), - ..Default::default() - }); + if let Some(state) = self.reset_confirmation.as_ref() { + return ["Reset all memories", "Go back"] + .into_iter() + .enumerate() + .map(|(idx, name)| GenericDisplayRow { + name: if state.selected_idx == Some(idx) { + format!("โ€บ {name}") + } else { + format!(" {name}") + }, + description: Some(match idx { + 0 => "Delete local memory files and rollout summaries.".to_string(), + 1 => "Return to memory settings.".to_string(), + _ => unreachable!("reset confirmation only renders two rows"), + }), + ..Default::default() + }) + .collect(); } - rows + let selected_idx = self.state.selected_idx; + self.items + .iter() + .enumerate() + .map(|(idx, item)| { + let prefix = if selected_idx == Some(idx) { + 'โ€บ' + } else { + ' ' + }; + let (name, description) = match item { + MemoriesMenuItem::Setting { + name, + description, + enabled, + .. + } => ( + format!("{prefix} [{}] {name}", if *enabled { 'x' } else { ' ' }), + description, + ), + MemoriesMenuItem::Action { + name, description, .. + } => (format!("{prefix} {name}"), description), + }; + GenericDisplayRow { + name, + description: Some((*description).to_string()), + ..Default::default() + } + }) + .collect() } fn move_up(&mut self) { @@ -127,8 +201,9 @@ impl MemoriesSettingsView { if len == 0 { return; } - self.state.move_up_wrap(len); - self.state.ensure_visible(len, MAX_POPUP_ROWS.min(len)); + let state = self.active_state_mut(); + state.move_up_wrap(len); + state.ensure_visible(len, MAX_POPUP_ROWS.min(len)); } fn move_down(&mut self) { @@ -136,17 +211,22 @@ impl MemoriesSettingsView { if len == 0 { return; } - self.state.move_down_wrap(len); - self.state.ensure_visible(len, MAX_POPUP_ROWS.min(len)); + let state = self.active_state_mut(); + state.move_down_wrap(len); + state.ensure_visible(len, MAX_POPUP_ROWS.min(len)); } fn toggle_selected(&mut self) { + if self.reset_confirmation.is_some() { + return; + } + let Some(selected_idx) = self.state.selected_idx else { return; }; - if let Some(item) = self.items.get_mut(selected_idx) { - item.enabled = !item.enabled; + if let Some(MemoriesMenuItem::Setting { enabled, .. }) = self.items.get_mut(selected_idx) { + *enabled = !*enabled; } } @@ -157,8 +237,34 @@ impl MemoriesSettingsView { fn current_setting(&self, setting: MemoriesSetting) -> bool { self.items .iter() - .find(|item| item.setting == setting) - .is_some_and(|item| item.enabled) + .find_map(|item| match item { + MemoriesMenuItem::Setting { + setting: item_setting, + enabled, + .. + } if *item_setting == setting => Some(*enabled), + _ => None, + }) + .unwrap_or(false) + } + + fn open_reset_confirmation(&mut self) { + let mut state = ScrollState::new(); + state.selected_idx = Some(0); + self.reset_confirmation = Some(state); + } + + fn close_reset_confirmation(&mut self) { + self.reset_confirmation = None; + self.state.selected_idx = self.items.len().checked_sub(1); + } + + fn footer_hint(&self) -> Line<'static> { + if self.reset_confirmation.is_some() { + standard_popup_hint_line() + } else { + memories_settings_hint_line() + } } } @@ -231,15 +337,39 @@ impl BottomPaneView for MemoriesSettingsView { impl MemoriesSettingsView { fn save(&mut self) { - self.app_event_tx.send(AppEvent::UpdateMemorySettings { - use_memories: self.current_setting(MemoriesSetting::Use), - generate_memories: self.current_setting(MemoriesSetting::Generate), - }); - self.complete = true; + if let Some(state) = self.reset_confirmation.as_ref() { + match state.selected_idx { + Some(0) => { + self.app_event_tx.send(AppEvent::ResetMemories); + self.complete = true; + } + Some(1) | None => self.close_reset_confirmation(), + Some(other) => unreachable!("unexpected reset confirmation row: {other}"), + } + return; + } + + match self.state.selected_idx.and_then(|idx| self.items.get(idx)) { + Some(MemoriesMenuItem::Action { + action: MemoriesAction::Reset, + .. + }) => self.open_reset_confirmation(), + _ => { + self.app_event_tx.send(AppEvent::UpdateMemorySettings { + use_memories: self.current_setting(MemoriesSetting::Use), + generate_memories: self.current_setting(MemoriesSetting::Generate), + }); + self.complete = true; + } + } } fn cancel(&mut self) { - self.complete = true; + if self.reset_confirmation.is_some() { + self.close_reset_confirmation(); + } else { + self.complete = true; + } } } @@ -256,14 +386,17 @@ impl Renderable for MemoriesSettingsView { .style(user_message_style()) .render(content_area, buf); - let header_height = self - .header - .desired_height(content_area.width.saturating_sub(4)); + let header = if self.reset_confirmation.is_some() { + self.reset_confirmation_header() + } else { + self.settings_header() + }; + let header_height = header.desired_height(content_area.width.saturating_sub(4)); let rows = self.build_rows(); let rows_width = Self::rows_width(content_area.width); let rows_height = measure_rows_height( &rows, - &self.state, + self.active_state(), MAX_POPUP_ROWS, rows_width.saturating_add(1), ); @@ -276,7 +409,7 @@ impl Renderable for MemoriesSettingsView { ]) .areas(content_area.inset(Insets::vh(/*v*/ 1, /*h*/ 2))); - self.header.render(header_area, buf); + header.render(header_area, buf); if list_area.height > 0 { let render_area = Rect { @@ -289,12 +422,14 @@ impl Renderable for MemoriesSettingsView { render_area, buf, &rows, - &self.state, + self.active_state(), MAX_POPUP_ROWS, " No memory settings available", ); } - self.docs_link.clone().render(docs_area, buf); + if self.reset_confirmation.is_none() { + self.docs_link.clone().render(docs_area, buf); + } let hint_area = Rect { x: footer_area.x + 2, @@ -302,21 +437,31 @@ impl Renderable for MemoriesSettingsView { width: footer_area.width.saturating_sub(2), height: footer_area.height, }; - self.footer_hint.clone().dim().render(hint_area, buf); + self.footer_hint().render(hint_area, buf); } fn desired_height(&self, width: u16) -> u16 { + let header = if self.reset_confirmation.is_some() { + self.reset_confirmation_header() + } else { + self.settings_header() + }; let rows = self.build_rows(); let rows_width = Self::rows_width(width); let rows_height = measure_rows_height( &rows, - &self.state, + self.active_state(), MAX_POPUP_ROWS, rows_width.saturating_add(1), ); - let mut height = self.header.desired_height(width.saturating_sub(4)); - height = height.saturating_add(rows_height + 5); + let docs_height = if self.reset_confirmation.is_some() { + 0 + } else { + 1 + }; + let mut height = header.desired_height(width.saturating_sub(4)); + height = height.saturating_add(rows_height + 4 + docs_height); height.saturating_add(1) } } @@ -327,6 +472,6 @@ fn memories_settings_hint_line() -> Line<'static> { key_hint::plain(KeyCode::Char(' ')).into(), " to toggle; ".into(), key_hint::plain(KeyCode::Enter).into(), - " to save".into(), + " to save or select".into(), ]) } diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_reset_confirmation.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_reset_confirmation.snap new file mode 100644 index 0000000000..2538e2b461 --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_reset_confirmation.snap @@ -0,0 +1,12 @@ +--- +source: tui/src/chatwidget/tests/popups_and_settings.rs +expression: popup +--- + Reset all memories? + +โ€บ Reset all memories Delete local memory files and rollout summaries. + Go back Return to memory settings. + + + + Press enter to confirm or esc to go back diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_settings_popup.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_settings_popup.snap index 9785ef2bb3..2fbe641e29 100644 --- a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_settings_popup.snap +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_settings_popup.snap @@ -9,7 +9,9 @@ expression: popup next thread. [ ] Generate memories Generate memories from the following threads. Current thread included. + Reset all memories Clear local memory files and summaries. Existing + threads stay intact. Learn more: https://developers.openai.com/codex/memories - Press space to toggle; enter to save + Press space to toggle; enter to save or select diff --git a/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs b/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs index 0edea73391..375c37314b 100644 --- a/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs +++ b/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs @@ -1543,6 +1543,22 @@ async fn memories_settings_popup_snapshot() { assert_chatwidget_snapshot!("memories_settings_popup", popup); } +#[tokio::test] +async fn memories_reset_confirmation_snapshot() { + let (mut chat, _rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::MemoryTool, /*enabled*/ true); + chat.config.memories.use_memories = true; + chat.config.memories.generate_memories = false; + + chat.open_memories_popup(); + chat.handle_key_event(KeyEvent::from(KeyCode::Down)); + chat.handle_key_event(KeyEvent::from(KeyCode::Down)); + chat.handle_key_event(KeyEvent::from(KeyCode::Enter)); + + let popup = render_bottom_popup(&chat, /*width*/ 80); + assert_chatwidget_snapshot!("memories_reset_confirmation", popup); +} + #[tokio::test] async fn memories_settings_toggle_saves_on_enter() { let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; @@ -1564,6 +1580,22 @@ async fn memories_settings_toggle_saves_on_enter() { ); } +#[tokio::test] +async fn memories_reset_confirmation_sends_event_on_confirm() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::MemoryTool, /*enabled*/ true); + chat.config.memories.use_memories = true; + chat.config.memories.generate_memories = false; + + chat.open_memories_popup(); + chat.handle_key_event(KeyEvent::from(KeyCode::Down)); + chat.handle_key_event(KeyEvent::from(KeyCode::Down)); + chat.handle_key_event(KeyEvent::from(KeyCode::Enter)); + chat.handle_key_event(KeyEvent::from(KeyCode::Enter)); + + assert_matches!(rx.try_recv(), Ok(AppEvent::ResetMemories)); +} + #[tokio::test] async fn model_selection_popup_snapshot() { let (mut chat, _rx, _op_rx) = make_chatwidget_manual(Some("gpt-5-codex")).await; From 1dead46c90876ea22ac0b512a934fc7091b97ec2 Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Wed, 15 Apr 2026 09:28:30 -0700 Subject: [PATCH 20/84] Remove exec-server fs sandbox request preflight (#17883) ## Summary - Remove the exec-server-side manual filesystem request path preflight before invoking the sandbox helper. - Keep sandbox helper policy construction and platform sandbox enforcement as the access boundary. - Add a portable local+remote regression for writing through an explicitly configured alias root. - Remove the metadata symlink-escape assertion that depended on the deleted manual preflight; no replacement metadata-specific access probe is added. ## Tests - `cargo test -p codex-exec-server --lib` - `cargo test -p codex-exec-server --test file_system` - `git diff --check` --- codex-rs/exec-server/src/fs_sandbox.rs | 170 ---------------------- codex-rs/exec-server/tests/file_system.rs | 144 ++++++++++++------ 2 files changed, 99 insertions(+), 215 deletions(-) diff --git a/codex-rs/exec-server/src/fs_sandbox.rs b/codex-rs/exec-server/src/fs_sandbox.rs index d0765afa35..7b0ac256d4 100644 --- a/codex-rs/exec-server/src/fs_sandbox.rs +++ b/codex-rs/exec-server/src/fs_sandbox.rs @@ -1,10 +1,8 @@ use std::collections::HashMap; -use std::path::PathBuf; use codex_app_server_protocol::JSONRPCErrorError; use codex_protocol::models::FileSystemPermissions; use codex_protocol::models::PermissionProfile; -use codex_protocol::permissions::FileSystemAccessMode; use codex_protocol::permissions::FileSystemSandboxPolicy; use codex_protocol::permissions::NetworkSandboxPolicy; use codex_protocol::protocol::ReadOnlyAccess; @@ -26,14 +24,6 @@ use crate::fs_helper::FsHelperPayload; use crate::fs_helper::FsHelperRequest; use crate::fs_helper::FsHelperResponse; use crate::local_file_system::current_sandbox_cwd; -use crate::local_file_system::resolve_existing_path; -use crate::protocol::FsCopyParams; -use crate::protocol::FsCreateDirectoryParams; -use crate::protocol::FsGetMetadataParams; -use crate::protocol::FsReadDirectoryParams; -use crate::protocol::FsReadFileParams; -use crate::protocol::FsRemoveParams; -use crate::protocol::FsWriteFileParams; use crate::rpc::internal_error; use crate::rpc::invalid_request; @@ -52,23 +42,16 @@ impl FileSystemSandboxRunner { sandbox: &FileSystemSandboxContext, request: FsHelperRequest, ) -> Result { - let request_sandbox_policy = - normalize_sandbox_policy_root_aliases(sandbox.sandbox_policy.clone()); let helper_sandbox_policy = normalize_sandbox_policy_root_aliases( sandbox_policy_with_helper_runtime_defaults(&sandbox.sandbox_policy), ); let cwd = current_sandbox_cwd().map_err(io_error)?; let cwd = AbsolutePathBuf::from_absolute_path(cwd.as_path()) .map_err(|err| invalid_request(format!("current directory is not absolute: {err}")))?; - let request_file_system_policy = FileSystemSandboxPolicy::from_legacy_sandbox_policy( - &request_sandbox_policy, - cwd.as_path(), - ); let file_system_policy = FileSystemSandboxPolicy::from_legacy_sandbox_policy( &helper_sandbox_policy, cwd.as_path(), ); - let request = resolve_request_paths(request, &request_file_system_policy, &cwd)?; let network_policy = NetworkSandboxPolicy::Restricted; let command = self.sandbox_exec_request( &helper_sandbox_policy, @@ -158,132 +141,6 @@ impl FileSystemSandboxRunner { } } -fn resolve_request_paths( - request: FsHelperRequest, - file_system_policy: &FileSystemSandboxPolicy, - cwd: &AbsolutePathBuf, -) -> Result { - match request { - FsHelperRequest::ReadFile(FsReadFileParams { path, sandbox }) => { - let path = resolve_sandbox_path(&path, PreserveTerminalSymlink::No)?; - ensure_path_access(file_system_policy, cwd, &path, FileSystemAccessMode::Read)?; - Ok(FsHelperRequest::ReadFile(FsReadFileParams { - path, - sandbox, - })) - } - FsHelperRequest::WriteFile(FsWriteFileParams { - path, - data_base64, - sandbox, - }) => Ok(FsHelperRequest::WriteFile(FsWriteFileParams { - path: { - let path = resolve_sandbox_path(&path, PreserveTerminalSymlink::No)?; - ensure_path_access(file_system_policy, cwd, &path, FileSystemAccessMode::Write)?; - path - }, - data_base64, - sandbox, - })), - FsHelperRequest::CreateDirectory(FsCreateDirectoryParams { - path, - recursive, - sandbox, - }) => Ok(FsHelperRequest::CreateDirectory(FsCreateDirectoryParams { - path: { - let path = resolve_sandbox_path(&path, PreserveTerminalSymlink::No)?; - ensure_path_access(file_system_policy, cwd, &path, FileSystemAccessMode::Write)?; - path - }, - recursive, - sandbox, - })), - FsHelperRequest::GetMetadata(FsGetMetadataParams { path, sandbox }) => { - let path = resolve_sandbox_path(&path, PreserveTerminalSymlink::No)?; - ensure_path_access(file_system_policy, cwd, &path, FileSystemAccessMode::Read)?; - Ok(FsHelperRequest::GetMetadata(FsGetMetadataParams { - path, - sandbox, - })) - } - FsHelperRequest::ReadDirectory(FsReadDirectoryParams { path, sandbox }) => { - let path = resolve_sandbox_path(&path, PreserveTerminalSymlink::No)?; - ensure_path_access(file_system_policy, cwd, &path, FileSystemAccessMode::Read)?; - Ok(FsHelperRequest::ReadDirectory(FsReadDirectoryParams { - path, - sandbox, - })) - } - FsHelperRequest::Remove(FsRemoveParams { - path, - recursive, - force, - sandbox, - }) => Ok(FsHelperRequest::Remove(FsRemoveParams { - path: { - let path = resolve_sandbox_path(&path, PreserveTerminalSymlink::Yes)?; - ensure_path_access(file_system_policy, cwd, &path, FileSystemAccessMode::Write)?; - path - }, - recursive, - force, - sandbox, - })), - FsHelperRequest::Copy(FsCopyParams { - source_path, - destination_path, - recursive, - sandbox, - }) => Ok(FsHelperRequest::Copy(FsCopyParams { - source_path: { - let source_path = resolve_sandbox_path(&source_path, PreserveTerminalSymlink::Yes)?; - ensure_path_access( - file_system_policy, - cwd, - &source_path, - FileSystemAccessMode::Read, - )?; - source_path - }, - destination_path: { - let destination_path = - resolve_sandbox_path(&destination_path, PreserveTerminalSymlink::No)?; - ensure_path_access( - file_system_policy, - cwd, - &destination_path, - FileSystemAccessMode::Write, - )?; - destination_path - }, - recursive, - sandbox, - })), - } -} - -#[derive(Clone, Copy)] -enum PreserveTerminalSymlink { - Yes, - No, -} - -fn resolve_sandbox_path( - path: &AbsolutePathBuf, - preserve_terminal_symlink: PreserveTerminalSymlink, -) -> Result { - if matches!(preserve_terminal_symlink, PreserveTerminalSymlink::Yes) - && std::fs::symlink_metadata(path.as_path()) - .map(|metadata| metadata.file_type().is_symlink()) - .unwrap_or(false) - { - return Ok(normalize_top_level_alias(path.clone())); - } - - let resolved = resolve_existing_path(path.as_path()).map_err(io_error)?; - absolute_path(resolved) -} - fn normalize_sandbox_policy_root_aliases(sandbox_policy: SandboxPolicy) -> SandboxPolicy { let mut sandbox_policy = sandbox_policy; match &mut sandbox_policy { @@ -338,33 +195,6 @@ fn normalize_top_level_alias(path: AbsolutePathBuf) -> AbsolutePathBuf { path } -fn absolute_path(path: PathBuf) -> Result { - AbsolutePathBuf::from_absolute_path(path.as_path()) - .map_err(|err| invalid_request(format!("resolved sandbox path is not absolute: {err}"))) -} - -fn ensure_path_access( - file_system_policy: &FileSystemSandboxPolicy, - cwd: &AbsolutePathBuf, - path: &AbsolutePathBuf, - required_access: FileSystemAccessMode, -) -> Result<(), JSONRPCErrorError> { - let actual_access = file_system_policy.resolve_access_with_cwd(path.as_path(), cwd.as_path()); - let permitted = match required_access { - FileSystemAccessMode::Read => actual_access.can_read(), - FileSystemAccessMode::Write => actual_access.can_write(), - FileSystemAccessMode::None => true, - }; - if permitted { - return Ok(()); - } - - Err(invalid_request(format!( - "{} is not permitted by filesystem sandbox", - path.display() - ))) -} - async fn run_command( command: SandboxExecRequest, request_json: Vec, diff --git a/codex-rs/exec-server/tests/file_system.rs b/codex-rs/exec-server/tests/file_system.rs index e24a2605f0..e046c203dd 100644 --- a/codex-rs/exec-server/tests/file_system.rs +++ b/codex-rs/exec-server/tests/file_system.rs @@ -3,6 +3,8 @@ mod common; use std::os::unix::fs::symlink; +use std::path::Path; +use std::path::PathBuf; use std::process::Command; use std::sync::Arc; @@ -17,6 +19,8 @@ use codex_exec_server::FileSystemSandboxContext; use codex_exec_server::LocalFileSystem; use codex_exec_server::ReadDirectoryEntry; use codex_exec_server::RemoveOptions; +use codex_protocol::models::FileSystemPermissions; +use codex_protocol::models::PermissionProfile; use codex_protocol::protocol::ReadOnlyAccess; use codex_protocol::protocol::SandboxPolicy; use codex_utils_absolute_path::AbsolutePathBuf; @@ -94,20 +98,26 @@ fn workspace_write_sandbox(writable_root: std::path::PathBuf) -> FileSystemSandb } fn assert_sandbox_denied(error: &std::io::Error) { - assert!( - matches!( - error.kind(), - std::io::ErrorKind::InvalidInput | std::io::ErrorKind::PermissionDenied + match error.kind() { + std::io::ErrorKind::InvalidInput | std::io::ErrorKind::PermissionDenied => { + let message = error.to_string(); + assert!( + message.contains("is not permitted") + || message.contains("Operation not permitted") + || message.contains("Permission denied"), + "unexpected sandbox error message: {message}", + ); + } + std::io::ErrorKind::NotFound => assert!( + error.to_string().contains("No such file or directory"), + "unexpected sandbox not-found message: {error}", ), - "unexpected sandbox error kind: {error:?}", - ); - let message = error.to_string(); - assert!( - message.contains("is not permitted") - || message.contains("Operation not permitted") - || message.contains("Permission denied"), - "unexpected sandbox error message: {message}", - ); + std::io::ErrorKind::Other => assert!( + error.to_string().contains("Read-only file system"), + "unexpected sandbox other error message: {error}", + ), + other => panic!("unexpected sandbox error kind: {other:?}: {error:?}"), + } } fn assert_normalized_path_rejected(error: &std::io::Error) { @@ -129,6 +139,15 @@ fn assert_normalized_path_rejected(error: &std::io::Error) { } } +fn alias_root_candidate() -> Result> { + for root in [Path::new("/tmp").to_path_buf(), std::env::temp_dir()] { + if root.is_dir() && root.canonicalize().is_ok_and(|canonical| canonical != root) { + return Ok(Some(root)); + } + } + Ok(None) +} + #[test_case(false ; "local")] #[test_case(true ; "remote")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -381,11 +400,9 @@ async fn file_system_sandboxed_write_rejects_unwritable_path(use_remote: bool) - let file_system = context.file_system; let tmp = TempDir::new()?; - let allowed_dir = tmp.path().join("allowed"); let blocked_path = tmp.path().join("blocked.txt"); - std::fs::create_dir_all(&allowed_dir)?; - let sandbox = read_only_sandbox(allowed_dir); + let sandbox = read_only_sandbox(tmp.path().to_path_buf()); let error = match file_system .write_file( &absolute_path(blocked_path.clone()), @@ -403,6 +420,72 @@ async fn file_system_sandboxed_write_rejects_unwritable_path(use_remote: bool) - Ok(()) } +#[test_case(false ; "local")] +#[test_case(true ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_sandboxed_write_allows_explicit_alias_roots(use_remote: bool) -> Result<()> { + let Some(alias_root) = alias_root_candidate()? else { + return Ok(()); + }; + + let context = create_file_system_context(use_remote).await?; + let file_system = context.file_system; + + let tmp = tempfile::Builder::new() + .prefix("codex-fs-sandbox-alias-") + .tempdir_in(&alias_root)?; + let file_path = tmp.path().join("note.txt"); + let sandbox = workspace_write_sandbox(alias_root.clone()); + + file_system + .write_file( + &absolute_path(file_path.clone()), + b"created".to_vec(), + Some(&sandbox), + ) + .await + .with_context(|| format!("write file through alias root mode={use_remote}"))?; + assert_eq!(std::fs::read(&file_path)?, b"created"); + + Ok(()) +} + +#[test_case(false ; "local")] +#[test_case(true ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_sandboxed_write_allows_additional_write_root(use_remote: bool) -> Result<()> { + let context = create_file_system_context(use_remote).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let readable_dir = tmp.path().join("readable"); + let writable_dir = tmp.path().join("writable"); + let file_path = writable_dir.join("note.txt"); + std::fs::create_dir_all(&readable_dir)?; + std::fs::create_dir_all(&writable_dir)?; + + let mut sandbox = read_only_sandbox(readable_dir); + sandbox.additional_permissions = Some(PermissionProfile { + network: None, + file_system: Some(FileSystemPermissions { + read: None, + write: Some(vec![absolute_path(writable_dir)]), + }), + }); + + file_system + .write_file( + &absolute_path(file_path.clone()), + b"created".to_vec(), + Some(&sandbox), + ) + .await + .with_context(|| format!("write file through additional root mode={use_remote}"))?; + assert_eq!(std::fs::read(&file_path)?, b"created"); + + Ok(()) +} + #[test_case(false ; "local")] #[test_case(true ; "remote")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -528,35 +611,6 @@ async fn file_system_create_directory_rejects_symlink_escape(use_remote: bool) - Ok(()) } -#[test_case(false ; "local")] -#[test_case(true ; "remote")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_get_metadata_rejects_symlink_escape(use_remote: bool) -> Result<()> { - let context = create_file_system_context(use_remote).await?; - let file_system = context.file_system; - - let tmp = TempDir::new()?; - let allowed_dir = tmp.path().join("allowed"); - let outside_dir = tmp.path().join("outside"); - std::fs::create_dir_all(&allowed_dir)?; - std::fs::create_dir_all(&outside_dir)?; - std::fs::write(outside_dir.join("secret.txt"), "nope")?; - symlink(&outside_dir, allowed_dir.join("link"))?; - - let requested_path = allowed_dir.join("link").join("secret.txt"); - let sandbox = read_only_sandbox(allowed_dir); - let error = match file_system - .get_metadata(&absolute_path(requested_path.clone()), Some(&sandbox)) - .await - { - Ok(_) => anyhow::bail!("get_metadata should be blocked"), - Err(error) => error, - }; - assert_sandbox_denied(&error); - - Ok(()) -} - #[test_case(false ; "local")] #[test_case(true ; "remote")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] From 8e784bba2fb795ec9248bbfd9a26fdad1b809dc6 Mon Sep 17 00:00:00 2001 From: Adrian <145513011+adrian-openai@users.noreply.github.com> Date: Wed, 15 Apr 2026 10:08:27 -0700 Subject: [PATCH 21/84] Register agent identities behind use_agent_identity (#17386) ## Summary Stack PR 2 of 4 for feature-gated agent identity support. This PR adds agent identity registration behind `features.use_agent_identity`. It keeps the app-server protocol unchanged and starts registration after ChatGPT auth exists rather than requiring a client restart. ## Stack - PR1: https://github.com/openai/codex/pull/17385 - add `features.use_agent_identity` - PR2: https://github.com/openai/codex/pull/17386 - this PR - PR3: https://github.com/openai/codex/pull/17387 - register agent tasks when enabled - PR4: https://github.com/openai/codex/pull/17388 - use `AgentAssertion` downstream when enabled ## Validation Covered as part of the local stack validation pass: - `just fmt` - `cargo test -p codex-core --lib agent_identity` - `cargo test -p codex-core --lib agent_assertion` - `cargo test -p codex-core --lib websocket_agent_task` - `cargo test -p codex-api api_bridge` - `cargo build -p codex-cli --bin codex` ## Notes The full local app-server E2E path is still being debugged after PR creation. The current branch stack is directionally ready for review while that follow-up continues. --- MODULE.bazel.lock | 2 + codex-rs/Cargo.lock | 33 +- codex-rs/Cargo.toml | 1 + .../src/transport/remote_control/tests.rs | 1 + .../src/transport/remote_control/websocket.rs | 1 + .../app-server/tests/common/auth_fixtures.rs | 1 + .../app-server/tests/suite/v2/app_list.rs | 1 + codex-rs/core/BUILD.bazel | 1 + codex-rs/core/Cargo.toml | 2 + codex-rs/core/src/agent_identity.rs | 766 ++++++++++++++++++ codex-rs/core/src/codex.rs | 57 ++ codex-rs/core/src/codex_tests.rs | 47 ++ codex-rs/core/src/lib.rs | 1 + codex-rs/core/src/state/service.rs | 2 + .../suite/responses_api_proxy_headers.rs | 178 +++- codex-rs/login/src/auth/auth_tests.rs | 108 +++ codex-rs/login/src/auth/manager.rs | 94 ++- codex-rs/login/src/auth/storage.rs | 13 + codex-rs/login/src/auth/storage_tests.rs | 31 + codex-rs/login/src/lib.rs | 1 + codex-rs/login/src/server.rs | 1 + codex-rs/login/tests/suite/auth_refresh.rs | 22 + codex-rs/tui/src/local_chatgpt_auth.rs | 2 + 23 files changed, 1340 insertions(+), 26 deletions(-) create mode 100644 codex-rs/core/src/agent_identity.rs diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 59ae93bf13..5d7572f461 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -819,6 +819,8 @@ "dylint_linting_5.0.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"assert_cmd\",\"req\":\"^2.0\"},{\"name\":\"cargo_metadata\",\"req\":\"^0.23\"},{\"features\":[\"config\"],\"name\":\"dylint_internal\",\"req\":\"=5.0.0\"},{\"name\":\"paste\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rustc_version\",\"req\":\"^0.4\"},{\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.23\"},{\"name\":\"thiserror\",\"req\":\"^2.0\"},{\"name\":\"toml\",\"req\":\"^0.9\"},{\"kind\":\"build\",\"name\":\"toml\",\"req\":\"^0.9\"}],\"features\":{\"constituent\":[]}}", "dylint_testing_5.0.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0\"},{\"name\":\"cargo_metadata\",\"req\":\"^0.23\"},{\"name\":\"compiletest_rs\",\"req\":\"^0.11\"},{\"name\":\"dylint\",\"req\":\"=5.0.0\"},{\"name\":\"dylint_internal\",\"req\":\"=5.0.0\"},{\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"once_cell\",\"req\":\"^1.21\"},{\"name\":\"regex\",\"req\":\"^1.11\"},{\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"tempfile\",\"req\":\"^3.23\"}],\"features\":{\"default\":[],\"deny_warnings\":[]}}", "dyn-clone_1.0.20": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.66\"}],\"features\":{}}", + "ed25519-dalek_2.2.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"blake2\",\"req\":\"^0.10\"},{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"default_features\":false,\"features\":[\"digest\"],\"name\":\"curve25519-dalek\",\"req\":\"^4\"},{\"default_features\":false,\"features\":[\"digest\",\"rand_core\"],\"kind\":\"dev\",\"name\":\"curve25519-dalek\",\"req\":\"^4\"},{\"default_features\":false,\"name\":\"ed25519\",\"req\":\">=2.2, <2.3\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"hex-literal\",\"req\":\"^0.4\"},{\"default_features\":false,\"name\":\"merlin\",\"optional\":true,\"req\":\"^3\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"default_features\":false,\"name\":\"rand_core\",\"optional\":true,\"req\":\"^0.6.4\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"rand_core\",\"req\":\"^0.6.4\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"sha2\",\"req\":\"^0.10\"},{\"kind\":\"dev\",\"name\":\"sha3\",\"req\":\"^0.10\"},{\"default_features\":false,\"name\":\"signature\",\"optional\":true,\"req\":\">=2.0, <2.3\"},{\"default_features\":false,\"name\":\"subtle\",\"req\":\"^2.3.0\"},{\"kind\":\"dev\",\"name\":\"toml\",\"req\":\"^0.7\"},{\"default_features\":false,\"features\":[\"static_secrets\"],\"kind\":\"dev\",\"name\":\"x25519-dalek\",\"req\":\"^2\"},{\"default_features\":false,\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1.5\"}],\"features\":{\"alloc\":[\"curve25519-dalek/alloc\",\"ed25519/alloc\",\"serde?/alloc\",\"zeroize/alloc\"],\"asm\":[\"sha2/asm\"],\"batch\":[\"alloc\",\"merlin\",\"rand_core\"],\"default\":[\"fast\",\"std\",\"zeroize\"],\"digest\":[\"signature/digest\"],\"fast\":[\"curve25519-dalek/precomputed-tables\"],\"hazmat\":[],\"legacy_compatibility\":[\"curve25519-dalek/legacy_compatibility\"],\"pem\":[\"alloc\",\"ed25519/pem\",\"pkcs8\"],\"pkcs8\":[\"ed25519/pkcs8\"],\"rand_core\":[\"dep:rand_core\"],\"serde\":[\"dep:serde\",\"ed25519/serde\"],\"std\":[\"alloc\",\"ed25519/std\",\"serde?/std\",\"sha2/std\"],\"zeroize\":[\"dep:zeroize\",\"curve25519-dalek/zeroize\"]}}", + "ed25519_2.2.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1\"},{\"features\":[\"rand_core\"],\"kind\":\"dev\",\"name\":\"ed25519-dalek\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"hex-literal\",\"req\":\"^0.4\"},{\"name\":\"pkcs8\",\"optional\":true,\"req\":\"^0.10\"},{\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"rand_core\",\"req\":\"^0.6\"},{\"default_features\":false,\"features\":[\"signature\"],\"kind\":\"dev\",\"name\":\"ring-compat\",\"req\":\"^0.8\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"serde_bytes\",\"optional\":true,\"req\":\"^0.11\"},{\"default_features\":false,\"name\":\"signature\",\"req\":\"^2\"},{\"default_features\":false,\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"alloc\":[\"pkcs8?/alloc\"],\"default\":[\"std\"],\"pem\":[\"alloc\",\"pkcs8/pem\"],\"serde_bytes\":[\"serde\",\"dep:serde_bytes\"],\"std\":[\"pkcs8?/std\",\"signature/std\"]}}", "either_1.15.0": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"alloc\",\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.95\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[],\"use_std\":[\"std\"]}}", "ena_0.14.3": "{\"dependencies\":[{\"name\":\"dogged\",\"optional\":true,\"req\":\"^0.2.0\"},{\"name\":\"log\",\"req\":\"^0.4\"}],\"features\":{\"bench\":[],\"persistent\":[\"dogged\"]}}", "encode_unicode_1.0.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"ascii\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.0\",\"target\":\"cfg(unix)\"},{\"features\":[\"https-native\"],\"kind\":\"dev\",\"name\":\"minreq\",\"req\":\"^2.6\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 3a99c2aa60..5e54b772c1 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1907,6 +1907,7 @@ dependencies = [ "codex-api", "codex-app-server-protocol", "codex-apply-patch", + "codex-arg0", "codex-async-utils", "codex-code-mode", "codex-config", @@ -1958,6 +1959,7 @@ dependencies = [ "ctor 0.6.3", "dirs", "dunce", + "ed25519-dalek", "env-flags", "eventsource-stream", "futures", @@ -3655,6 +3657,7 @@ dependencies = [ "cfg-if", "cpufeatures", "curve25519-dalek-derive", + "digest", "fiat-crypto", "rustc_version", "subtle", @@ -4131,7 +4134,7 @@ dependencies = [ "libc", "option-ext", "redox_users 0.5.2", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4235,6 +4238,30 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "serde", + "sha2", + "subtle", + "zeroize", +] + [[package]] name = "either" version = "1.15.0" @@ -6733,7 +6760,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -11857,7 +11884,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index e4985e4582..b57a464d5e 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -227,6 +227,7 @@ diffy = "0.4.2" dirs = "6" dotenvy = "0.15.7" dunce = "1.0.4" +ed25519-dalek = { version = "2.2.0", features = ["pkcs8"] } encoding_rs = "0.8.35" env-flags = "0.1.1" env_logger = "0.11.9" diff --git a/codex-rs/app-server/src/transport/remote_control/tests.rs b/codex-rs/app-server/src/transport/remote_control/tests.rs index 21808a3a18..6dd6978fa5 100644 --- a/codex-rs/app-server/src/transport/remote_control/tests.rs +++ b/codex-rs/app-server/src/transport/remote_control/tests.rs @@ -96,6 +96,7 @@ fn remote_control_auth_dot_json(account_id: Option<&str>) -> AuthDotJson { account_id: account_id.map(str::to_string), }), last_refresh: Some(chrono::Utc::now()), + agent_identity: None, } } diff --git a/codex-rs/app-server/src/transport/remote_control/websocket.rs b/codex-rs/app-server/src/transport/remote_control/websocket.rs index 42924c2d34..d168ec0e18 100644 --- a/codex-rs/app-server/src/transport/remote_control/websocket.rs +++ b/codex-rs/app-server/src/transport/remote_control/websocket.rs @@ -971,6 +971,7 @@ mod tests { account_id: Some("account_id".to_string()), }), last_refresh: Some(Utc::now()), + agent_identity: None, } } diff --git a/codex-rs/app-server/tests/common/auth_fixtures.rs b/codex-rs/app-server/tests/common/auth_fixtures.rs index 99334f0770..86f0fb456d 100644 --- a/codex-rs/app-server/tests/common/auth_fixtures.rs +++ b/codex-rs/app-server/tests/common/auth_fixtures.rs @@ -163,6 +163,7 @@ pub fn write_chatgpt_auth( openai_api_key: None, tokens: Some(tokens), last_refresh, + agent_identity: None, }; save_auth(codex_home, &auth, cli_auth_credentials_store_mode).context("write auth.json") diff --git a/codex-rs/app-server/tests/suite/v2/app_list.rs b/codex-rs/app-server/tests/suite/v2/app_list.rs index 57a27961af..dbe61524f5 100644 --- a/codex-rs/app-server/tests/suite/v2/app_list.rs +++ b/codex-rs/app-server/tests/suite/v2/app_list.rs @@ -117,6 +117,7 @@ async fn list_apps_returns_empty_with_api_key_auth() -> Result<()> { openai_api_key: Some("test-api-key".to_string()), tokens: None, last_refresh: None, + agent_identity: None, }, AuthCredentialsStoreMode::File, )?; diff --git a/codex-rs/core/BUILD.bazel b/codex-rs/core/BUILD.bazel index ed01996273..434dc1f6a4 100644 --- a/codex-rs/core/BUILD.bazel +++ b/codex-rs/core/BUILD.bazel @@ -53,6 +53,7 @@ codex_rust_crate( "//codex-rs/linux-sandbox:codex-linux-sandbox", "//codex-rs/rmcp-client:test_stdio_server", "//codex-rs/rmcp-client:test_streamable_http_server", + "//codex-rs/responses-api-proxy:codex-responses-api-proxy", "//codex-rs/cli:codex", ], ) diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index c91f07f705..a371966079 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -41,6 +41,7 @@ codex-login = { workspace = true } codex-mcp = { workspace = true } codex-model-provider-info = { workspace = true } codex-models-manager = { workspace = true } +ed25519-dalek = { workspace = true } codex-shell-command = { workspace = true } codex-execpolicy = { workspace = true } codex-git-utils = { workspace = true } @@ -143,6 +144,7 @@ codex-shell-escalation = { workspace = true } [dev-dependencies] assert_cmd = { workspace = true } assert_matches = { workspace = true } +codex-arg0 = { workspace = true } codex-otel = { workspace = true } codex-test-binary-support = { workspace = true } codex-utils-cargo-bin = { workspace = true } diff --git a/codex-rs/core/src/agent_identity.rs b/codex-rs/core/src/agent_identity.rs new file mode 100644 index 0000000000..13a07cd961 --- /dev/null +++ b/codex-rs/core/src/agent_identity.rs @@ -0,0 +1,766 @@ +use std::sync::Arc; +use std::time::Duration; + +use anyhow::Context; +use anyhow::Result; +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use chrono::SecondsFormat; +use chrono::Utc; +use codex_features::Feature; +use codex_login::AgentIdentityAuthRecord; +use codex_login::AuthManager; +use codex_login::CodexAuth; +use codex_login::default_client::create_client; +use codex_protocol::protocol::SessionSource; +use ed25519_dalek::SigningKey; +use ed25519_dalek::VerifyingKey; +use ed25519_dalek::pkcs8::DecodePrivateKey; +use ed25519_dalek::pkcs8::EncodePrivateKey; +use rand::TryRngCore; +use rand::rngs::OsRng; +use serde::Deserialize; +use serde::Serialize; +use tokio::sync::Mutex; +use tracing::debug; +use tracing::info; +use tracing::warn; + +use crate::config::Config; + +const AGENT_REGISTRATION_TIMEOUT: Duration = Duration::from_secs(15); +const AGENT_IDENTITY_BISCUIT_TIMEOUT: Duration = Duration::from_secs(15); + +#[derive(Clone)] +pub(crate) struct AgentIdentityManager { + auth_manager: Arc, + chatgpt_base_url: String, + feature_enabled: bool, + abom: AgentBillOfMaterials, + ensure_lock: Arc>, +} + +impl std::fmt::Debug for AgentIdentityManager { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AgentIdentityManager") + .field("chatgpt_base_url", &self.chatgpt_base_url) + .field("feature_enabled", &self.feature_enabled) + .field("abom", &self.abom) + .finish_non_exhaustive() + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub(crate) struct StoredAgentIdentity { + pub(crate) binding_id: String, + pub(crate) chatgpt_account_id: String, + pub(crate) chatgpt_user_id: Option, + pub(crate) agent_runtime_id: String, + pub(crate) private_key_pkcs8_base64: String, + pub(crate) public_key_ssh: String, + pub(crate) registered_at: String, + pub(crate) abom: AgentBillOfMaterials, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub(crate) struct AgentBillOfMaterials { + pub(crate) agent_version: String, + pub(crate) agent_harness_id: String, + pub(crate) running_location: String, +} + +#[derive(Debug, Serialize)] +struct RegisterAgentRequest { + abom: AgentBillOfMaterials, + agent_public_key: String, + capabilities: Vec, +} + +#[derive(Debug, Deserialize)] +struct RegisterAgentResponse { + agent_runtime_id: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct AgentIdentityBinding { + binding_id: String, + chatgpt_account_id: String, + chatgpt_user_id: Option, + access_token: String, +} + +struct GeneratedAgentKeyMaterial { + private_key_pkcs8_base64: String, + public_key_ssh: String, +} + +impl AgentIdentityManager { + pub(crate) fn new( + config: &Config, + auth_manager: Arc, + session_source: SessionSource, + ) -> Self { + Self { + auth_manager, + chatgpt_base_url: config.chatgpt_base_url.clone(), + feature_enabled: config.features.enabled(Feature::UseAgentIdentity), + abom: build_abom(session_source), + ensure_lock: Arc::new(Mutex::new(())), + } + } + + pub(crate) fn is_enabled(&self) -> bool { + self.feature_enabled + } + + pub(crate) async fn ensure_registered_identity(&self) -> Result> { + if !self.feature_enabled { + return Ok(None); + } + + let Some(auth) = self.auth_manager.auth().await else { + debug!("skipping agent identity registration because no auth is available"); + return Ok(None); + }; + + let Some(binding) = + AgentIdentityBinding::from_auth(&auth, self.auth_manager.forced_chatgpt_workspace_id()) + else { + debug!("skipping agent identity registration because ChatGPT auth is unavailable"); + return Ok(None); + }; + + let _guard = self.ensure_lock.lock().await; + + if let Some(stored_identity) = self.load_stored_identity(&auth, &binding)? { + info!( + agent_runtime_id = %stored_identity.agent_runtime_id, + binding_id = %binding.binding_id, + "reusing stored agent identity" + ); + return Ok(Some(stored_identity)); + } + + let stored_identity = self.register_agent_identity(&binding).await?; + self.store_identity(&auth, &stored_identity)?; + Ok(Some(stored_identity)) + } + + async fn register_agent_identity( + &self, + binding: &AgentIdentityBinding, + ) -> Result { + let key_material = generate_agent_key_material()?; + let request_body = RegisterAgentRequest { + abom: self.abom.clone(), + agent_public_key: key_material.public_key_ssh.clone(), + capabilities: Vec::new(), + }; + + let url = agent_registration_url(&self.chatgpt_base_url); + let human_biscuit = self.mint_human_biscuit(binding, "POST", &url).await?; + let client = create_client(); + let response = client + .post(&url) + .header("X-OpenAI-Authorization", human_biscuit) + .json(&request_body) + .timeout(AGENT_REGISTRATION_TIMEOUT) + .send() + .await + .with_context(|| { + format!("failed to send agent identity registration request to {url}") + })?; + + if response.status().is_success() { + let response_body = response + .json::() + .await + .with_context(|| format!("failed to parse agent identity response from {url}"))?; + let stored_identity = StoredAgentIdentity { + binding_id: binding.binding_id.clone(), + chatgpt_account_id: binding.chatgpt_account_id.clone(), + chatgpt_user_id: binding.chatgpt_user_id.clone(), + agent_runtime_id: response_body.agent_runtime_id, + private_key_pkcs8_base64: key_material.private_key_pkcs8_base64, + public_key_ssh: key_material.public_key_ssh, + registered_at: Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), + abom: self.abom.clone(), + }; + info!( + agent_runtime_id = %stored_identity.agent_runtime_id, + binding_id = %binding.binding_id, + "registered agent identity" + ); + return Ok(stored_identity); + } + + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("agent identity registration failed with status {status} from {url}: {body}") + } + + async fn mint_human_biscuit( + &self, + binding: &AgentIdentityBinding, + target_method: &str, + target_url: &str, + ) -> Result { + let url = agent_identity_biscuit_url(&self.chatgpt_base_url); + let request_id = agent_identity_request_id()?; + let client = create_client(); + let response = client + .get(&url) + .bearer_auth(&binding.access_token) + .header("X-Request-Id", request_id.clone()) + .header("X-Original-Method", target_method) + .header("X-Original-Url", target_url) + .timeout(AGENT_IDENTITY_BISCUIT_TIMEOUT) + .send() + .await + .with_context(|| format!("failed to send agent identity biscuit request to {url}"))?; + + if response.status().is_success() { + let human_biscuit = response + .headers() + .get("x-openai-authorization") + .context("agent identity biscuit response did not include x-openai-authorization")? + .to_str() + .context("agent identity biscuit response header was not valid UTF-8")? + .to_string(); + info!( + request_id = %request_id, + "minted human biscuit for agent identity registration" + ); + return Ok(human_biscuit); + } + + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + anyhow::bail!( + "agent identity biscuit minting failed with status {status} from {url}: {body}" + ) + } + + fn load_stored_identity( + &self, + auth: &CodexAuth, + binding: &AgentIdentityBinding, + ) -> Result> { + let Some(record) = auth.get_agent_identity(&binding.chatgpt_account_id) else { + return Ok(None); + }; + + let stored_identity = + match StoredAgentIdentity::from_auth_record(binding, record, self.abom.clone()) { + Ok(stored_identity) => stored_identity, + Err(error) => { + warn!( + binding_id = %binding.binding_id, + error = %error, + "stored agent identity is invalid; deleting cached value" + ); + auth.remove_agent_identity()?; + return Ok(None); + } + }; + + if !stored_identity.matches_binding(binding) { + warn!( + binding_id = %binding.binding_id, + "stored agent identity binding no longer matches current auth; deleting cached value" + ); + auth.remove_agent_identity()?; + return Ok(None); + } + + if let Err(error) = stored_identity.validate_key_material() { + warn!( + agent_runtime_id = %stored_identity.agent_runtime_id, + binding_id = %binding.binding_id, + error = %error, + "stored agent identity key material is invalid; deleting cached value" + ); + auth.remove_agent_identity()?; + return Ok(None); + } + + Ok(Some(stored_identity)) + } + + fn store_identity( + &self, + auth: &CodexAuth, + stored_identity: &StoredAgentIdentity, + ) -> Result<()> { + auth.set_agent_identity(stored_identity.to_auth_record())?; + Ok(()) + } + + #[cfg(test)] + fn new_for_tests( + auth_manager: Arc, + feature_enabled: bool, + chatgpt_base_url: String, + session_source: SessionSource, + ) -> Self { + Self { + auth_manager, + chatgpt_base_url, + feature_enabled, + abom: build_abom(session_source), + ensure_lock: Arc::new(Mutex::new(())), + } + } +} + +impl StoredAgentIdentity { + fn from_auth_record( + binding: &AgentIdentityBinding, + record: AgentIdentityAuthRecord, + abom: AgentBillOfMaterials, + ) -> Result { + if record.workspace_id != binding.chatgpt_account_id { + anyhow::bail!( + "stored agent identity workspace {:?} does not match current workspace {:?}", + record.workspace_id, + binding.chatgpt_account_id + ); + } + let signing_key = signing_key_from_private_key_pkcs8_base64(&record.agent_private_key)?; + Ok(Self { + binding_id: binding.binding_id.clone(), + chatgpt_account_id: binding.chatgpt_account_id.clone(), + chatgpt_user_id: record.chatgpt_user_id, + agent_runtime_id: record.agent_runtime_id, + private_key_pkcs8_base64: record.agent_private_key, + public_key_ssh: encode_ssh_ed25519_public_key(&signing_key.verifying_key()), + registered_at: record.registered_at, + abom, + }) + } + + fn to_auth_record(&self) -> AgentIdentityAuthRecord { + AgentIdentityAuthRecord { + workspace_id: self.chatgpt_account_id.clone(), + chatgpt_user_id: self.chatgpt_user_id.clone(), + agent_runtime_id: self.agent_runtime_id.clone(), + agent_private_key: self.private_key_pkcs8_base64.clone(), + registered_at: self.registered_at.clone(), + } + } + + fn matches_binding(&self, binding: &AgentIdentityBinding) -> bool { + self.binding_id == binding.binding_id + && self.chatgpt_account_id == binding.chatgpt_account_id + && match binding.chatgpt_user_id.as_deref() { + Some(chatgpt_user_id) => self.chatgpt_user_id.as_deref() == Some(chatgpt_user_id), + None => true, + } + } + + fn validate_key_material(&self) -> Result<()> { + let signing_key = self.signing_key()?; + let derived_public_key = encode_ssh_ed25519_public_key(&signing_key.verifying_key()); + anyhow::ensure!( + self.public_key_ssh == derived_public_key, + "stored public key does not match the private key" + ); + Ok(()) + } + + pub(crate) fn signing_key(&self) -> Result { + signing_key_from_private_key_pkcs8_base64(&self.private_key_pkcs8_base64) + } +} + +impl AgentIdentityBinding { + fn from_auth(auth: &CodexAuth, forced_workspace_id: Option) -> Option { + if !auth.is_chatgpt_auth() { + return None; + } + + let token_data = auth.get_token_data().ok()?; + let resolved_account_id = + forced_workspace_id + .filter(|value| !value.is_empty()) + .or(token_data + .account_id + .clone() + .filter(|value| !value.is_empty()))?; + + Some(Self { + binding_id: format!("chatgpt-account-{resolved_account_id}"), + chatgpt_account_id: resolved_account_id, + chatgpt_user_id: token_data + .id_token + .chatgpt_user_id + .filter(|value| !value.is_empty()), + access_token: token_data.access_token, + }) + } +} + +fn build_abom(session_source: SessionSource) -> AgentBillOfMaterials { + AgentBillOfMaterials { + agent_version: env!("CARGO_PKG_VERSION").to_string(), + agent_harness_id: match &session_source { + SessionSource::VSCode => "codex-app".to_string(), + SessionSource::Cli + | SessionSource::Exec + | SessionSource::Mcp + | SessionSource::Custom(_) + | SessionSource::SubAgent(_) + | SessionSource::Unknown => "codex-cli".to_string(), + }, + running_location: format!("{}-{}", session_source, std::env::consts::OS), + } +} + +fn generate_agent_key_material() -> Result { + let mut secret_key_bytes = [0u8; 32]; + OsRng + .try_fill_bytes(&mut secret_key_bytes) + .context("failed to generate agent identity private key bytes")?; + let signing_key = SigningKey::from_bytes(&secret_key_bytes); + let private_key_pkcs8 = signing_key + .to_pkcs8_der() + .context("failed to encode agent identity private key as PKCS#8")?; + + Ok(GeneratedAgentKeyMaterial { + private_key_pkcs8_base64: BASE64_STANDARD.encode(private_key_pkcs8.as_bytes()), + public_key_ssh: encode_ssh_ed25519_public_key(&signing_key.verifying_key()), + }) +} + +fn encode_ssh_ed25519_public_key(verifying_key: &VerifyingKey) -> String { + let mut blob = Vec::with_capacity(4 + 11 + 4 + 32); + append_ssh_string(&mut blob, b"ssh-ed25519"); + append_ssh_string(&mut blob, verifying_key.as_bytes()); + format!("ssh-ed25519 {}", BASE64_STANDARD.encode(blob)) +} + +fn append_ssh_string(buf: &mut Vec, value: &[u8]) { + buf.extend_from_slice(&(value.len() as u32).to_be_bytes()); + buf.extend_from_slice(value); +} + +fn agent_registration_url(chatgpt_base_url: &str) -> String { + let trimmed = chatgpt_base_url.trim_end_matches('/'); + format!("{trimmed}/v1/agent/register") +} + +fn signing_key_from_private_key_pkcs8_base64(private_key_pkcs8_base64: &str) -> Result { + let private_key = BASE64_STANDARD + .decode(private_key_pkcs8_base64) + .context("stored agent identity private key is not valid base64")?; + SigningKey::from_pkcs8_der(&private_key) + .context("stored agent identity private key is not valid PKCS#8") +} + +fn agent_identity_biscuit_url(chatgpt_base_url: &str) -> String { + let trimmed = chatgpt_base_url.trim_end_matches('/'); + format!("{trimmed}/authenticate_app_v2") +} + +fn agent_identity_request_id() -> Result { + let mut request_id_bytes = [0u8; 16]; + OsRng + .try_fill_bytes(&mut request_id_bytes) + .context("failed to generate agent identity request id")?; + Ok(format!( + "codex-agent-identity-{}", + URL_SAFE_NO_PAD.encode(request_id_bytes) + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + use base64::engine::general_purpose::URL_SAFE_NO_PAD; + use codex_app_server_protocol::AuthMode as ApiAuthMode; + use codex_login::AuthCredentialsStoreMode; + use codex_login::AuthDotJson; + use codex_login::save_auth; + use codex_login::token_data::IdTokenInfo; + use codex_login::token_data::TokenData; + use pretty_assertions::assert_eq; + use wiremock::Mock; + use wiremock::MockServer; + use wiremock::ResponseTemplate; + use wiremock::matchers::header; + use wiremock::matchers::method; + use wiremock::matchers::path; + + #[tokio::test] + async fn ensure_registered_identity_skips_when_feature_is_disabled() { + let auth_manager = + AuthManager::from_auth_for_testing(make_chatgpt_auth("account-123", Some("user-123"))); + let manager = AgentIdentityManager::new_for_tests( + auth_manager, + /*feature_enabled*/ false, + "https://chatgpt.com/backend-api/".to_string(), + SessionSource::Cli, + ); + + assert_eq!(manager.ensure_registered_identity().await.unwrap(), None); + } + + #[tokio::test] + async fn ensure_registered_identity_skips_for_api_key_auth() { + let auth_manager = AuthManager::from_auth_for_testing(CodexAuth::from_api_key("test-key")); + let manager = AgentIdentityManager::new_for_tests( + auth_manager, + /*feature_enabled*/ true, + "https://chatgpt.com/backend-api/".to_string(), + SessionSource::Cli, + ); + + assert_eq!(manager.ensure_registered_identity().await.unwrap(), None); + } + + #[tokio::test] + async fn ensure_registered_identity_registers_and_reuses_cached_identity() { + let server = MockServer::start().await; + let chatgpt_base_url = server.uri(); + mount_human_biscuit(&server, &chatgpt_base_url).await; + Mock::given(method("POST")) + .and(path("/v1/agent/register")) + .and(header("x-openai-authorization", "human-biscuit")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "agent_runtime_id": "agent_123", + }))) + .expect(1) + .mount(&server) + .await; + + let auth_manager = + AuthManager::from_auth_for_testing(make_chatgpt_auth("account-123", Some("user-123"))); + let manager = AgentIdentityManager::new_for_tests( + auth_manager, + /*feature_enabled*/ true, + chatgpt_base_url, + SessionSource::Cli, + ); + + let first = manager + .ensure_registered_identity() + .await + .unwrap() + .expect("identity should be registered"); + let second = manager + .ensure_registered_identity() + .await + .unwrap() + .expect("identity should be reused"); + + assert_eq!(first.agent_runtime_id, "agent_123"); + assert_eq!(first, second); + assert_eq!(first.abom.agent_harness_id, "codex-cli"); + assert_eq!(first.chatgpt_account_id, "account-123"); + assert_eq!(first.chatgpt_user_id.as_deref(), Some("user-123")); + } + + #[tokio::test] + async fn ensure_registered_identity_deletes_invalid_cached_identity_and_reregisters() { + let server = MockServer::start().await; + let chatgpt_base_url = server.uri(); + mount_human_biscuit(&server, &chatgpt_base_url).await; + Mock::given(method("POST")) + .and(path("/v1/agent/register")) + .and(header("x-openai-authorization", "human-biscuit")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "agent_runtime_id": "agent_456", + }))) + .expect(1) + .mount(&server) + .await; + + let auth = make_chatgpt_auth("account-123", Some("user-123")); + let auth_manager = AuthManager::from_auth_for_testing(auth.clone()); + let manager = AgentIdentityManager::new_for_tests( + auth_manager, + /*feature_enabled*/ true, + chatgpt_base_url, + SessionSource::Cli, + ); + + let binding = + AgentIdentityBinding::from_auth(&auth, /*forced_workspace_id*/ None).expect("binding"); + auth.set_agent_identity(AgentIdentityAuthRecord { + workspace_id: "account-123".to_string(), + chatgpt_user_id: Some("user-123".to_string()), + agent_runtime_id: "agent_invalid".to_string(), + agent_private_key: "not-valid-base64".to_string(), + registered_at: "2026-01-01T00:00:00Z".to_string(), + }) + .expect("seed invalid identity"); + + let stored = manager + .ensure_registered_identity() + .await + .unwrap() + .expect("identity should be registered"); + + assert_eq!(stored.agent_runtime_id, "agent_456"); + let persisted = auth + .get_agent_identity(&binding.chatgpt_account_id) + .expect("stored identity"); + assert_eq!(persisted.agent_runtime_id, "agent_456"); + } + + #[tokio::test] + async fn ensure_registered_identity_deletes_different_user_identity_and_reregisters() { + let server = MockServer::start().await; + let chatgpt_base_url = server.uri(); + mount_human_biscuit(&server, &chatgpt_base_url).await; + Mock::given(method("POST")) + .and(path("/v1/agent/register")) + .and(header("x-openai-authorization", "human-biscuit")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "agent_runtime_id": "agent_new", + }))) + .expect(1) + .mount(&server) + .await; + + let auth = make_chatgpt_auth("account-123", Some("user-new")); + let stale_key = generate_agent_key_material().expect("key material"); + auth.set_agent_identity(AgentIdentityAuthRecord { + workspace_id: "account-123".to_string(), + chatgpt_user_id: Some("user-old".to_string()), + agent_runtime_id: "agent_old".to_string(), + agent_private_key: stale_key.private_key_pkcs8_base64, + registered_at: "2026-01-01T00:00:00Z".to_string(), + }) + .expect("seed stale identity"); + + let auth_manager = AuthManager::from_auth_for_testing(auth.clone()); + let manager = AgentIdentityManager::new_for_tests( + auth_manager, + /*feature_enabled*/ true, + chatgpt_base_url, + SessionSource::Cli, + ); + + let stored = manager + .ensure_registered_identity() + .await + .unwrap() + .expect("identity should be registered"); + + assert_eq!(stored.agent_runtime_id, "agent_new"); + assert_eq!(stored.chatgpt_user_id.as_deref(), Some("user-new")); + let persisted = auth + .get_agent_identity("account-123") + .expect("stored identity"); + assert_eq!(persisted.agent_runtime_id, "agent_new"); + assert_eq!(persisted.chatgpt_user_id.as_deref(), Some("user-new")); + } + + #[tokio::test] + async fn ensure_registered_identity_uses_chatgpt_base_url() { + let server = MockServer::start().await; + let chatgpt_base_url = format!("{}/backend-api", server.uri()); + mount_human_biscuit(&server, &chatgpt_base_url).await; + Mock::given(method("POST")) + .and(path("/backend-api/v1/agent/register")) + .and(header("x-openai-authorization", "human-biscuit")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "agent_runtime_id": "agent_canonical", + }))) + .expect(1) + .mount(&server) + .await; + + let auth_manager = + AuthManager::from_auth_for_testing(make_chatgpt_auth("account-123", Some("user-123"))); + let manager = AgentIdentityManager::new_for_tests( + auth_manager, + /*feature_enabled*/ true, + chatgpt_base_url, + SessionSource::Cli, + ); + + let stored = manager + .ensure_registered_identity() + .await + .unwrap() + .expect("identity should be registered"); + assert_eq!(stored.agent_runtime_id, "agent_canonical"); + } + + async fn mount_human_biscuit(server: &MockServer, chatgpt_base_url: &str) { + let biscuit_url = agent_identity_biscuit_url(chatgpt_base_url); + let biscuit_path = reqwest::Url::parse(&biscuit_url) + .expect("biscuit URL parses") + .path() + .to_string(); + let target_url = agent_registration_url(chatgpt_base_url); + Mock::given(method("GET")) + .and(path(biscuit_path)) + .and(header("authorization", "Bearer access-token-account-123")) + .and(header("x-original-method", "POST")) + .and(header("x-original-url", target_url)) + .respond_with( + ResponseTemplate::new(200).insert_header("x-openai-authorization", "human-biscuit"), + ) + .expect(1) + .mount(server) + .await; + } + + #[test] + fn encode_ssh_ed25519_public_key_matches_expected_wire_shape() { + let key_material = generate_agent_key_material().expect("key material"); + let (_, encoded_blob) = key_material + .public_key_ssh + .split_once(' ') + .expect("public key contains scheme"); + let decoded = BASE64_STANDARD.decode(encoded_blob).expect("base64"); + + assert_eq!(&decoded[..4], 11u32.to_be_bytes().as_slice()); + assert_eq!(&decoded[4..15], b"ssh-ed25519"); + assert_eq!(&decoded[15..19], 32u32.to_be_bytes().as_slice()); + assert_eq!(decoded.len(), 51); + } + + fn make_chatgpt_auth(account_id: &str, user_id: Option<&str>) -> CodexAuth { + let tempdir = tempfile::tempdir().expect("tempdir"); + let auth_json = AuthDotJson { + auth_mode: Some(ApiAuthMode::Chatgpt), + openai_api_key: None, + tokens: Some(TokenData { + id_token: IdTokenInfo { + email: None, + chatgpt_plan_type: None, + chatgpt_user_id: user_id.map(ToOwned::to_owned), + chatgpt_account_id: Some(account_id.to_string()), + raw_jwt: fake_id_token(account_id, user_id), + }, + access_token: format!("access-token-{account_id}"), + refresh_token: "refresh-token".to_string(), + account_id: Some(account_id.to_string()), + }), + last_refresh: Some(Utc::now()), + agent_identity: None, + }; + save_auth(tempdir.path(), &auth_json, AuthCredentialsStoreMode::File).expect("save auth"); + CodexAuth::from_auth_storage(tempdir.path(), AuthCredentialsStoreMode::File) + .expect("load auth") + .expect("auth") + } + + fn fake_id_token(account_id: &str, user_id: Option<&str>) -> String { + let header = URL_SAFE_NO_PAD.encode(r#"{"alg":"none","typ":"JWT"}"#); + let payload = serde_json::json!({ + "https://api.openai.com/auth": { + "chatgpt_user_id": user_id, + "chatgpt_account_id": account_id, + } + }); + let payload = URL_SAFE_NO_PAD.encode(payload.to_string()); + format!("{header}.{payload}.signature") + } +} diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index c063a5c977..a3d7594813 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -13,6 +13,7 @@ use crate::agent::Mailbox; use crate::agent::MailboxReceiver; use crate::agent::agent_status_from_event; use crate::agent::status::is_final; +use crate::agent_identity::AgentIdentityManager; use crate::apps::render_apps_section; use crate::commit_attribution::commit_message_trailer_instruction; use crate::compact; @@ -1510,6 +1511,56 @@ impl Session { }); } + fn start_agent_identity_registration(self: &Arc) { + if !self.services.agent_identity_manager.is_enabled() { + return; + } + + let weak_sess = Arc::downgrade(self); + let mut auth_state_rx = self.services.auth_manager.subscribe_auth_state(); + tokio::spawn(async move { + loop { + let Some(sess) = weak_sess.upgrade() else { + return; + }; + match sess + .services + .agent_identity_manager + .ensure_registered_identity() + .await + { + Ok(Some(_)) => return, + Ok(None) => { + drop(sess); + if auth_state_rx.changed().await.is_err() { + return; + } + } + Err(error) => { + sess.fail_agent_identity_registration(error).await; + return; + } + } + } + }); + } + + async fn fail_agent_identity_registration(self: &Arc, error: anyhow::Error) { + warn!(error = %error, "agent identity registration failed"); + let message = format!( + "Agent identity registration failed. Codex cannot continue while `features.use_agent_identity` is enabled: {error}" + ); + self.send_event_raw(Event { + id: self.next_internal_sub_id(), + msg: EventMsg::Error(ErrorEvent { + message, + codex_error_info: Some(CodexErrorInfo::Other), + }), + }) + .await; + handlers::shutdown(self, self.next_internal_sub_id()).await; + } + #[allow(clippy::too_many_arguments)] fn make_turn_context( conversation_id: ThreadId, @@ -2055,6 +2106,11 @@ impl Session { hooks, rollout: Mutex::new(rollout_recorder), user_shell: Arc::new(default_shell), + agent_identity_manager: Arc::new(AgentIdentityManager::new( + config.as_ref(), + Arc::clone(&auth_manager), + session_configuration.session_source.clone(), + )), shell_snapshot_tx, show_raw_agent_reasoning: config.show_raw_agent_reasoning, exec_policy, @@ -2152,6 +2208,7 @@ impl Session { // Start the watcher after SessionConfigured so it cannot emit earlier events. sess.start_skills_watcher_listener(); + sess.start_agent_identity_registration(); // Construct sandbox_state before MCP startup so it can be sent to each // MCP server immediately after it becomes ready (avoiding blocking). let sandbox_state = SandboxState { diff --git a/codex-rs/core/src/codex_tests.rs b/codex-rs/core/src/codex_tests.rs index 0ff4ea5a2f..46d4cda57e 100644 --- a/codex-rs/core/src/codex_tests.rs +++ b/codex-rs/core/src/codex_tests.rs @@ -110,6 +110,7 @@ use opentelemetry::trace::TraceId; use std::path::Path; use std::time::Duration; use tokio::time::sleep; +use tokio::time::timeout; use tracing_opentelemetry::OpenTelemetrySpanExt; use codex_protocol::mcp::CallToolResult as McpCallToolResult; @@ -2814,6 +2815,11 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) { }), rollout: Mutex::new(None), user_shell: Arc::new(default_user_shell()), + agent_identity_manager: Arc::new(crate::agent_identity::AgentIdentityManager::new( + config.as_ref(), + Arc::clone(&auth_manager), + session_configuration.session_source.clone(), + )), shell_snapshot_tx: watch::channel(None).0, show_raw_agent_reasoning: config.show_raw_agent_reasoning, exec_policy, @@ -3666,6 +3672,11 @@ pub(crate) async fn make_session_and_context_with_dynamic_tools_and_rx( }), rollout: Mutex::new(None), user_shell: Arc::new(default_user_shell()), + agent_identity_manager: Arc::new(crate::agent_identity::AgentIdentityManager::new( + config.as_ref(), + Arc::clone(&auth_manager), + session_configuration.session_source.clone(), + )), shell_snapshot_tx: watch::channel(None).0, show_raw_agent_reasoning: config.show_raw_agent_reasoning, exec_policy, @@ -3770,6 +3781,42 @@ pub(crate) async fn make_session_and_context_with_rx() -> ( make_session_and_context_with_dynamic_tools_and_rx(Vec::new()).await } +#[tokio::test] +async fn fail_agent_identity_registration_emits_error_and_shutdown() { + let (session, _turn_context, rx_event) = make_session_and_context_with_rx().await; + + session + .fail_agent_identity_registration(anyhow::anyhow!("registration exploded")) + .await; + + let error_event = timeout(Duration::from_secs(1), rx_event.recv()) + .await + .expect("error event should arrive") + .expect("error event should be readable"); + match error_event.msg { + EventMsg::Error(ErrorEvent { + message, + codex_error_info, + }) => { + assert_eq!( + message, + "Agent identity registration failed. Codex cannot continue while `features.use_agent_identity` is enabled: registration exploded".to_string() + ); + assert_eq!(codex_error_info, Some(CodexErrorInfo::Other)); + } + other => panic!("expected error event, got {other:?}"), + } + + let shutdown_event = timeout(Duration::from_secs(1), rx_event.recv()) + .await + .expect("shutdown event should arrive") + .expect("shutdown event should be readable"); + match shutdown_event.msg { + EventMsg::ShutdownComplete => {} + other => panic!("expected shutdown event, got {other:?}"), + } +} + #[tokio::test] async fn refresh_mcp_servers_is_deferred_until_next_turn() { let (session, turn_context) = make_session_and_context().await; diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 93d62faca6..37b5dcf5d4 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -5,6 +5,7 @@ // the TUI or the tracing stack). #![deny(clippy::print_stdout, clippy::print_stderr)] +mod agent_identity; mod apply_patch; mod apps; mod arc_monitor; diff --git a/codex-rs/core/src/state/service.rs b/codex-rs/core/src/state/service.rs index 2e89585f99..4b6c5b30f5 100644 --- a/codex-rs/core/src/state/service.rs +++ b/codex-rs/core/src/state/service.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use crate::RolloutRecorder; use crate::SkillsManager; use crate::agent::AgentControl; +use crate::agent_identity::AgentIdentityManager; use crate::client::ModelClient; use crate::config::StartedNetworkProxy; use crate::exec_policy::ExecPolicyManager; @@ -41,6 +42,7 @@ pub(crate) struct SessionServices { pub(crate) hooks: Hooks, pub(crate) rollout: Mutex>, pub(crate) user_shell: Arc, + pub(crate) agent_identity_manager: Arc, pub(crate) shell_snapshot_tx: watch::Sender>>, pub(crate) show_raw_agent_reasoning: bool, pub(crate) exec_policy: Arc, diff --git a/codex-rs/core/tests/suite/responses_api_proxy_headers.rs b/codex-rs/core/tests/suite/responses_api_proxy_headers.rs index 7dc888ffce..8199956ee0 100644 --- a/codex-rs/core/tests/suite/responses_api_proxy_headers.rs +++ b/codex-rs/core/tests/suite/responses_api_proxy_headers.rs @@ -5,6 +5,11 @@ use anyhow::Result; use anyhow::anyhow; use codex_features::Feature; +use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::Op; +use codex_protocol::protocol::SandboxPolicy; +use codex_protocol::user_input::UserInput; use core_test_support::responses::ev_assistant_message; use core_test_support::responses::ev_completed; use core_test_support::responses::ev_function_call; @@ -13,10 +18,12 @@ use core_test_support::responses::mount_sse_once_match; use core_test_support::responses::sse; use core_test_support::responses::start_mock_server; use core_test_support::skip_if_no_network; +use core_test_support::test_codex::TestCodex; use core_test_support::test_codex::test_codex; use pretty_assertions::assert_eq; use serde_json::Value; use serde_json::json; +use std::io::ErrorKind; use std::io::Write; use std::path::Path; use std::process::Child; @@ -29,8 +36,9 @@ use tempfile::TempDir; const PARENT_PROMPT: &str = "spawn a subagent and report when it is started"; const CHILD_PROMPT: &str = "child: say done"; const SPAWN_CALL_ID: &str = "spawn-call-1"; -const PROXY_START_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 5); +const PROXY_START_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 30); const PROXY_POLL_INTERVAL: Duration = Duration::from_millis(/*millis*/ 20); +const TURN_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 60); struct ResponsesApiProxy { child: Child, @@ -40,8 +48,17 @@ struct ResponsesApiProxy { impl ResponsesApiProxy { fn start(upstream_url: &str, dump_dir: &Path) -> Result { let server_info = dump_dir.join("server-info.json"); - let mut child = StdCommand::new(codex_utils_cargo_bin::cargo_bin("codex")?) - .args(["responses-api-proxy", "--server-info"]) + let (proxy_program, use_codex_multitool) = + match codex_utils_cargo_bin::cargo_bin("codex-responses-api-proxy") { + Ok(path) => (path, false), + Err(_) => (codex_utils_cargo_bin::cargo_bin("codex")?, true), + }; + let mut command = StdCommand::new(proxy_program); + if use_codex_multitool { + command.arg("responses-api-proxy"); + } + let mut child = command + .args(["--server-info"]) .arg(&server_info) .args(["--upstream-url", upstream_url, "--dump-dir"]) .arg(dump_dir) @@ -58,15 +75,27 @@ impl ResponsesApiProxy { let deadline = Instant::now() + PROXY_START_TIMEOUT; loop { - if let Ok(info) = std::fs::read_to_string(&server_info) { - let port = serde_json::from_str::(&info)? - .get("port") - .and_then(Value::as_u64) - .ok_or_else(|| anyhow!("proxy server info missing port"))?; - return Ok(Self { - child, - port: u16::try_from(port)?, - }); + match std::fs::read_to_string(&server_info) { + Ok(info) => { + if !info.trim().is_empty() { + match serde_json::from_str::(&info) { + Ok(info) => { + let port = info + .get("port") + .and_then(Value::as_u64) + .ok_or_else(|| anyhow!("proxy server info missing port"))?; + return Ok(Self { + child, + port: u16::try_from(port)?, + }); + } + Err(err) if err.is_eof() => {} + Err(err) => return Err(err.into()), + } + } + } + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(err.into()), } if let Some(status) = child.try_wait()? { return Err(anyhow!( @@ -144,7 +173,7 @@ async fn responses_api_proxy_dumps_parent_and_subagent_identity_headers() -> Res .expect("test config should allow feature update"); }); let test = builder.build(&server).await?; - test.submit_turn(PARENT_PROMPT).await?; + submit_turn_with_timeout(&test, PARENT_PROMPT, dump_dir.path()).await?; let dumps = wait_for_proxy_request_dumps(dump_dir.path())?; let parent = dumps @@ -178,6 +207,85 @@ async fn responses_api_proxy_dumps_parent_and_subagent_identity_headers() -> Res Ok(()) } +async fn submit_turn_with_timeout(test: &TestCodex, prompt: &str, dump_dir: &Path) -> Result<()> { + let session_model = test.session_configured.model.clone(); + test.codex + .submit(Op::UserTurn { + items: vec![UserInput::Text { + text: prompt.into(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + cwd: test.config.cwd.to_path_buf(), + approval_policy: AskForApproval::OnRequest, + approvals_reviewer: None, + sandbox_policy: SandboxPolicy::WorkspaceWrite { + writable_roots: Vec::new(), + read_only_access: Default::default(), + network_access: false, + exclude_tmpdir_env_var: false, + exclude_slash_tmp: false, + }, + model: session_model, + effort: None, + summary: None, + service_tier: None, + collaboration_mode: None, + personality: None, + }) + .await?; + + let turn_started = wait_for_event_result(test, "turn started", dump_dir, |event| { + matches!(event, EventMsg::TurnStarted(_)) + }) + .await?; + let EventMsg::TurnStarted(turn_started) = turn_started else { + unreachable!("event predicate only matches turn started events"); + }; + wait_for_event_result(test, "turn complete", dump_dir, |event| match event { + EventMsg::TurnComplete(event) => event.turn_id == turn_started.turn_id, + _ => false, + }) + .await?; + + Ok(()) +} + +async fn wait_for_event_result( + test: &TestCodex, + stage: &str, + dump_dir: &Path, + mut predicate: F, +) -> Result +where + F: FnMut(&EventMsg) -> bool, +{ + let mut seen_events = Vec::new(); + tokio::time::timeout(TURN_TIMEOUT, async { + loop { + let event = test.codex.next_event().await?; + seen_events.push(event_summary(&event.msg)); + if predicate(&event.msg) { + return Ok::(event.msg); + } + } + }) + .await + .map_err(|_| { + anyhow!( + "timed out waiting for {stage}; saw events: {}; {}", + seen_events.join(" | "), + proxy_dump_summary(dump_dir) + ) + })? +} + +fn event_summary(event: &EventMsg) -> String { + let mut summary = format!("{event:?}"); + summary.truncate(240); + summary +} + fn request_body_contains(req: &wiremock::Request, text: &str) -> bool { std::str::from_utf8(&req.body).is_ok_and(|body| body.contains(text)) } @@ -212,12 +320,54 @@ fn read_proxy_request_dumps(dump_dir: &Path) -> Result> { .and_then(|name| name.to_str()) .is_some_and(|name| name.ends_with("-request.json")) { - dumps.push(serde_json::from_str(&std::fs::read_to_string(&path)?)?); + let contents = std::fs::read_to_string(&path)?; + if contents.trim().is_empty() { + continue; + } + + match serde_json::from_str(&contents) { + Ok(dump) => dumps.push(dump), + Err(err) if err.is_eof() => continue, + Err(err) => return Err(err.into()), + } } } Ok(dumps) } +fn proxy_dump_summary(dump_dir: &Path) -> String { + match read_proxy_request_dumps(dump_dir) { + Ok(dumps) => { + let bodies = dumps + .iter() + .filter_map(|dump| dump.get("body")) + .map(Value::to_string) + .collect::>() + .join("; "); + format!("proxy wrote {} request dumps: {bodies}", dumps.len()) + } + Err(err) => format!("failed to read proxy request dumps: {err}"), + } +} + +#[test] +fn read_proxy_request_dumps_ignores_in_progress_files() -> Result<()> { + let dump_dir = TempDir::new()?; + std::fs::write(dump_dir.path().join("empty-request.json"), "")?; + std::fs::write(dump_dir.path().join("partial-request.json"), "{\"body\"")?; + std::fs::write( + dump_dir.path().join("complete-request.json"), + serde_json::to_string(&json!({ "body": "ready" }))?, + )?; + + assert_eq!( + read_proxy_request_dumps(dump_dir.path())?, + vec![json!({ "body": "ready" })] + ); + + Ok(()) +} + fn dump_body_contains(dump: &Value, text: &str) -> bool { dump.get("body") .is_some_and(|body| body.to_string().contains(text)) diff --git a/codex-rs/login/src/auth/auth_tests.rs b/codex-rs/login/src/auth/auth_tests.rs index 27064b6831..8a16f5939c 100644 --- a/codex-rs/login/src/auth/auth_tests.rs +++ b/codex-rs/login/src/auth/auth_tests.rs @@ -16,6 +16,8 @@ use serde_json::json; use std::sync::Arc; use tempfile::TempDir; use tempfile::tempdir; +use tokio::time::Duration; +use tokio::time::timeout; #[tokio::test] async fn refresh_without_id_token() { @@ -135,6 +137,7 @@ async fn pro_account_with_no_api_key_uses_chatgpt_auth() { account_id: None, }), last_refresh: Some(last_refresh), + agent_identity: None, }, auth_dot_json ); @@ -172,6 +175,7 @@ fn logout_removes_auth_file() -> Result<(), std::io::Error> { openai_api_key: Some("sk-test-key".to_string()), tokens: None, last_refresh: None, + agent_identity: None, }; super::save_auth(dir.path(), &auth_dot_json, AuthCredentialsStoreMode::File)?; let auth_file = get_auth_file(dir.path()); @@ -181,6 +185,49 @@ fn logout_removes_auth_file() -> Result<(), std::io::Error> { Ok(()) } +#[test] +fn chatgpt_auth_persists_agent_identity_for_workspace() { + let codex_home = tempdir().unwrap(); + write_auth_file( + AuthFileParams { + openai_api_key: None, + chatgpt_plan_type: Some("pro".to_string()), + chatgpt_account_id: Some("account-123".to_string()), + }, + codex_home.path(), + ) + .expect("failed to write auth file"); + let auth = super::load_auth( + codex_home.path(), + /*enable_codex_api_key_env*/ false, + AuthCredentialsStoreMode::File, + ) + .expect("load auth") + .expect("auth available"); + let record = AgentIdentityAuthRecord { + workspace_id: "account-123".to_string(), + chatgpt_user_id: Some("user-123".to_string()), + agent_runtime_id: "agent_123".to_string(), + agent_private_key: "pkcs8-base64".to_string(), + registered_at: "2026-04-13T12:00:00Z".to_string(), + }; + + auth.set_agent_identity(record.clone()) + .expect("set agent identity"); + + assert_eq!(auth.get_agent_identity("account-123"), Some(record.clone())); + assert_eq!(auth.get_agent_identity("other-account"), None); + let storage = FileAuthStorage::new(codex_home.path().to_path_buf()); + let persisted = storage + .load() + .expect("load auth") + .expect("auth should exist"); + assert_eq!(persisted.agent_identity, Some(record)); + + assert!(auth.remove_agent_identity().expect("remove agent identity")); + assert_eq!(auth.get_agent_identity("account-123"), None); +} + #[test] fn unauthorized_recovery_reports_mode_and_step_names() { let dir = tempdir().unwrap(); @@ -474,6 +521,67 @@ exit 1 } } +#[tokio::test] +async fn auth_manager_notifies_when_auth_state_changes() { + let dir = tempdir().unwrap(); + let manager = AuthManager::shared( + dir.path().to_path_buf(), + /*enable_codex_api_key_env*/ false, + AuthCredentialsStoreMode::File, + ); + let mut auth_state_rx = manager.subscribe_auth_state(); + + save_auth( + dir.path(), + &AuthDotJson { + auth_mode: Some(ApiAuthMode::ApiKey), + openai_api_key: Some("sk-test-key".to_string()), + tokens: None, + last_refresh: None, + agent_identity: None, + }, + AuthCredentialsStoreMode::File, + ) + .expect("save auth"); + + assert!( + manager.reload(), + "reload should report a changed auth state" + ); + timeout(Duration::from_secs(1), auth_state_rx.changed()) + .await + .expect("auth change notification should arrive") + .expect("auth state watch should remain open"); + + save_auth( + dir.path(), + &AuthDotJson { + auth_mode: Some(ApiAuthMode::ApiKey), + openai_api_key: Some("sk-updated-key".to_string()), + tokens: None, + last_refresh: None, + agent_identity: None, + }, + AuthCredentialsStoreMode::File, + ) + .expect("save updated auth"); + + assert!( + !manager.reload(), + "reload remains mode-stable even when the underlying credentials change" + ); + timeout(Duration::from_secs(1), auth_state_rx.changed()) + .await + .expect("auth reload notification should still arrive") + .expect("auth state watch should remain open"); + + manager.set_forced_chatgpt_workspace_id(Some("workspace-123".to_string())); + timeout(Duration::from_secs(1), auth_state_rx.changed()) + .await + .expect("workspace change notification should arrive") + .expect("auth state watch should remain open"); +} + struct AuthFileParams { openai_api_key: Option, chatgpt_plan_type: Option, diff --git a/codex-rs/login/src/auth/manager.rs b/codex-rs/login/src/auth/manager.rs index 15b129a169..8768171d95 100644 --- a/codex-rs/login/src/auth/manager.rs +++ b/codex-rs/login/src/auth/manager.rs @@ -13,6 +13,7 @@ use std::sync::Arc; use std::sync::Mutex; use std::sync::RwLock; use tokio::sync::Mutex as AsyncMutex; +use tokio::sync::watch; use codex_app_server_protocol::AuthMode; use codex_app_server_protocol::AuthMode as ApiAuthMode; @@ -20,6 +21,7 @@ use codex_protocol::config_types::ForcedLoginMethod; use codex_protocol::config_types::ModelProviderAuthInfo; use super::external_bearer::BearerTokenRefresher; +pub use crate::auth::storage::AgentIdentityAuthRecord; pub use crate::auth::storage::AuthDotJson; use crate::auth::storage::AuthStorageBackend; use crate::auth::storage::create_auth_storage; @@ -60,6 +62,7 @@ pub struct ChatgptAuth { #[derive(Debug, Clone)] pub struct ChatgptAuthTokens { state: ChatgptAuthState, + storage: Arc, } #[derive(Debug, Clone)] @@ -204,14 +207,13 @@ impl CodexAuth { client, }; + let storage = create_auth_storage(codex_home.to_path_buf(), storage_mode); match auth_mode { - ApiAuthMode::Chatgpt => { - let storage = create_auth_storage(codex_home.to_path_buf(), storage_mode); - Ok(Self::Chatgpt(ChatgptAuth { state, storage })) - } - ApiAuthMode::ChatgptAuthTokens => { - Ok(Self::ChatgptAuthTokens(ChatgptAuthTokens { state })) - } + ApiAuthMode::Chatgpt => Ok(Self::Chatgpt(ChatgptAuth { state, storage })), + ApiAuthMode::ChatgptAuthTokens => Ok(Self::ChatgptAuthTokens(ChatgptAuthTokens { + state, + storage, + })), ApiAuthMode::ApiKey => unreachable!("api key mode is handled above"), } } @@ -352,6 +354,52 @@ impl CodexAuth { self.get_current_auth_json().and_then(|t| t.tokens) } + pub fn get_agent_identity(&self, workspace_id: &str) -> Option { + self.get_current_auth_json() + .and_then(|auth| auth.agent_identity) + .filter(|identity| identity.workspace_id == workspace_id) + } + + pub fn set_agent_identity(&self, record: AgentIdentityAuthRecord) -> std::io::Result<()> { + let (state, storage) = match self { + Self::Chatgpt(auth) => (&auth.state, &auth.storage), + Self::ChatgptAuthTokens(auth) => (&auth.state, &auth.storage), + Self::ApiKey(_) => return Ok(()), + }; + let mut guard = state + .auth_dot_json + .lock() + .map_err(|_| std::io::Error::other("failed to lock auth state"))?; + let mut auth = guard + .clone() + .ok_or_else(|| std::io::Error::other("auth data is not available"))?; + auth.agent_identity = Some(record); + storage.save(&auth)?; + *guard = Some(auth); + Ok(()) + } + + pub fn remove_agent_identity(&self) -> std::io::Result { + let (state, storage) = match self { + Self::Chatgpt(auth) => (&auth.state, &auth.storage), + Self::ChatgptAuthTokens(auth) => (&auth.state, &auth.storage), + Self::ApiKey(_) => return Ok(false), + }; + let mut guard = state + .auth_dot_json + .lock() + .map_err(|_| std::io::Error::other("failed to lock auth state"))?; + let Some(mut auth) = guard.clone() else { + return Ok(false); + }; + let removed = auth.agent_identity.take().is_some(); + if removed { + storage.save(&auth)?; + *guard = Some(auth); + } + Ok(removed) + } + /// Consider this private to integration tests. pub fn create_dummy_chatgpt_auth_for_testing() -> Self { let auth_dot_json = AuthDotJson { @@ -364,6 +412,7 @@ impl CodexAuth { account_id: Some("account_id".to_string()), }), last_refresh: Some(Utc::now()), + agent_identity: None, }; let client = create_client(); @@ -439,6 +488,7 @@ pub fn login_with_api_key( openai_api_key: Some(api_key.to_string()), tokens: None, last_refresh: None, + agent_identity: None, }; save_auth(codex_home, &auth_dot_json, auth_credentials_store_mode) } @@ -812,6 +862,7 @@ impl AuthDotJson { openai_api_key: None, tokens: Some(tokens), last_refresh: Some(Utc::now()), + agent_identity: None, }) } @@ -1107,6 +1158,7 @@ pub struct AuthManager { forced_chatgpt_workspace_id: RwLock>, refresh_lock: AsyncMutex<()>, external_auth: RwLock>>, + auth_state_tx: watch::Sender<()>, } /// Configuration view required to construct a shared [`AuthManager`]. @@ -1155,6 +1207,7 @@ impl AuthManager { enable_codex_api_key_env: bool, auth_credentials_store_mode: AuthCredentialsStoreMode, ) -> Self { + let (auth_state_tx, _) = watch::channel(()); let managed_auth = load_auth( &codex_home, enable_codex_api_key_env, @@ -1173,11 +1226,13 @@ impl AuthManager { forced_chatgpt_workspace_id: RwLock::new(None), refresh_lock: AsyncMutex::new(()), external_auth: RwLock::new(None), + auth_state_tx, } } /// Create an AuthManager with a specific CodexAuth, for testing only. pub fn from_auth_for_testing(auth: CodexAuth) -> Arc { + let (auth_state_tx, _) = watch::channel(()); let cached = CachedAuth { auth: Some(auth), permanent_refresh_failure: None, @@ -1191,11 +1246,13 @@ impl AuthManager { forced_chatgpt_workspace_id: RwLock::new(None), refresh_lock: AsyncMutex::new(()), external_auth: RwLock::new(None), + auth_state_tx, }) } /// Create an AuthManager with a specific CodexAuth and codex home, for testing only. pub fn from_auth_for_testing_with_home(auth: CodexAuth, codex_home: PathBuf) -> Arc { + let (auth_state_tx, _) = watch::channel(()); let cached = CachedAuth { auth: Some(auth), permanent_refresh_failure: None, @@ -1208,10 +1265,12 @@ impl AuthManager { forced_chatgpt_workspace_id: RwLock::new(None), refresh_lock: AsyncMutex::new(()), external_auth: RwLock::new(None), + auth_state_tx, }) } pub fn external_bearer_only(config: ModelProviderAuthInfo) -> Arc { + let (auth_state_tx, _) = watch::channel(()); Arc::new(Self { codex_home: PathBuf::from("non-existent"), inner: RwLock::new(CachedAuth { @@ -1225,6 +1284,7 @@ impl AuthManager { external_auth: RwLock::new(Some( Arc::new(BearerTokenRefresher::new(config)) as Arc )), + auth_state_tx, }) } @@ -1364,6 +1424,7 @@ impl AuthManager { } tracing::info!("Reloaded auth, changed: {changed}"); guard.auth = new_auth; + self.auth_state_tx.send_replace(()); changed } else { false @@ -1373,18 +1434,23 @@ impl AuthManager { pub fn set_external_auth(&self, external_auth: Arc) { if let Ok(mut guard) = self.external_auth.write() { *guard = Some(external_auth); + self.auth_state_tx.send_replace(()); } } pub fn clear_external_auth(&self) { if let Ok(mut guard) = self.external_auth.write() { *guard = None; + self.auth_state_tx.send_replace(()); } } pub fn set_forced_chatgpt_workspace_id(&self, workspace_id: Option) { - if let Ok(mut guard) = self.forced_chatgpt_workspace_id.write() { + if let Ok(mut guard) = self.forced_chatgpt_workspace_id.write() + && *guard != workspace_id + { *guard = workspace_id; + self.auth_state_tx.send_replace(()); } } @@ -1395,6 +1461,10 @@ impl AuthManager { .and_then(|guard| guard.clone()) } + pub fn subscribe_auth_state(&self) -> watch::Receiver<()> { + self.auth_state_tx.subscribe() + } + pub fn has_external_auth(&self) -> bool { self.external_auth().is_some() } @@ -1638,8 +1708,14 @@ impl AuthManager { ), ))); } - let auth_dot_json = + let mut auth_dot_json = AuthDotJson::from_external_tokens(&refreshed).map_err(RefreshTokenError::Transient)?; + if let Some(previous_auth) = self + .auth_cached() + .and_then(|auth| auth.get_current_auth_json()) + { + auth_dot_json.agent_identity = previous_auth.agent_identity; + } save_auth( &self.codex_home, &auth_dot_json, diff --git a/codex-rs/login/src/auth/storage.rs b/codex-rs/login/src/auth/storage.rs index 97e801415c..f0d7a3169c 100644 --- a/codex-rs/login/src/auth/storage.rs +++ b/codex-rs/login/src/auth/storage.rs @@ -39,6 +39,19 @@ pub struct AuthDotJson { #[serde(default, skip_serializing_if = "Option::is_none")] pub last_refresh: Option>, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub agent_identity: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, PartialEq, Eq)] +pub struct AgentIdentityAuthRecord { + pub workspace_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub chatgpt_user_id: Option, + pub agent_runtime_id: String, + pub agent_private_key: String, + pub registered_at: String, } pub(super) fn get_auth_file(codex_home: &Path) -> PathBuf { diff --git a/codex-rs/login/src/auth/storage_tests.rs b/codex-rs/login/src/auth/storage_tests.rs index 4bf72c11b9..2e1cc85029 100644 --- a/codex-rs/login/src/auth/storage_tests.rs +++ b/codex-rs/login/src/auth/storage_tests.rs @@ -18,6 +18,7 @@ async fn file_storage_load_returns_auth_dot_json() -> anyhow::Result<()> { openai_api_key: Some("test-key".to_string()), tokens: None, last_refresh: Some(Utc::now()), + agent_identity: None, }; storage @@ -38,6 +39,7 @@ async fn file_storage_save_persists_auth_dot_json() -> anyhow::Result<()> { openai_api_key: Some("test-key".to_string()), tokens: None, last_refresh: Some(Utc::now()), + agent_identity: None, }; let file = get_auth_file(codex_home.path()); @@ -52,6 +54,30 @@ async fn file_storage_save_persists_auth_dot_json() -> anyhow::Result<()> { Ok(()) } +#[tokio::test] +async fn file_storage_persists_agent_identity() -> anyhow::Result<()> { + let codex_home = tempdir()?; + let storage = FileAuthStorage::new(codex_home.path().to_path_buf()); + let auth_dot_json = AuthDotJson { + auth_mode: Some(AuthMode::Chatgpt), + openai_api_key: None, + tokens: None, + last_refresh: Some(Utc::now()), + agent_identity: Some(AgentIdentityAuthRecord { + workspace_id: "account-123".to_string(), + chatgpt_user_id: Some("user-123".to_string()), + agent_runtime_id: "agent_123".to_string(), + agent_private_key: "pkcs8-base64".to_string(), + registered_at: "2026-04-13T12:00:00Z".to_string(), + }), + }; + + storage.save(&auth_dot_json)?; + + assert_eq!(storage.load()?, Some(auth_dot_json)); + Ok(()) +} + #[test] fn file_storage_delete_removes_auth_file() -> anyhow::Result<()> { let dir = tempdir()?; @@ -60,6 +86,7 @@ fn file_storage_delete_removes_auth_file() -> anyhow::Result<()> { openai_api_key: Some("sk-test-key".to_string()), tokens: None, last_refresh: None, + agent_identity: None, }; let storage = create_auth_storage(dir.path().to_path_buf(), AuthCredentialsStoreMode::File); storage.save(&auth_dot_json)?; @@ -83,6 +110,7 @@ fn ephemeral_storage_save_load_delete_is_in_memory_only() -> anyhow::Result<()> openai_api_key: Some("sk-ephemeral".to_string()), tokens: None, last_refresh: Some(Utc::now()), + agent_identity: None, }; storage.save(&auth_dot_json)?; @@ -181,6 +209,7 @@ fn auth_with_prefix(prefix: &str) -> AuthDotJson { account_id: Some(format!("{prefix}-account-id")), }), last_refresh: None, + agent_identity: None, } } @@ -197,6 +226,7 @@ fn keyring_auth_storage_load_returns_deserialized_auth() -> anyhow::Result<()> { openai_api_key: Some("sk-test".to_string()), tokens: None, last_refresh: None, + agent_identity: None, }; seed_keyring_with_auth( &mock_keyring, @@ -239,6 +269,7 @@ fn keyring_auth_storage_save_persists_and_removes_fallback_file() -> anyhow::Res account_id: Some("account".to_string()), }), last_refresh: Some(Utc::now()), + agent_identity: None, }; storage.save(&auth)?; diff --git a/codex-rs/login/src/lib.rs b/codex-rs/login/src/lib.rs index 1b52223bc3..4880e43106 100644 --- a/codex-rs/login/src/lib.rs +++ b/codex-rs/login/src/lib.rs @@ -20,6 +20,7 @@ pub use server::ShutdownHandle; pub use server::run_login_server; pub use api_bridge::auth_provider_from_auth; +pub use auth::AgentIdentityAuthRecord; pub use auth::AuthConfig; pub use auth::AuthDotJson; pub use auth::AuthManager; diff --git a/codex-rs/login/src/server.rs b/codex-rs/login/src/server.rs index 169a8a3091..0c7b810184 100644 --- a/codex-rs/login/src/server.rs +++ b/codex-rs/login/src/server.rs @@ -781,6 +781,7 @@ pub(crate) async fn persist_tokens_async( openai_api_key: api_key, tokens: Some(tokens), last_refresh: Some(Utc::now()), + agent_identity: None, }; save_auth(&codex_home, &auth, auth_credentials_store_mode) }) diff --git a/codex-rs/login/tests/suite/auth_refresh.rs b/codex-rs/login/tests/suite/auth_refresh.rs index bf9e03bc26..d754c9589d 100644 --- a/codex-rs/login/tests/suite/auth_refresh.rs +++ b/codex-rs/login/tests/suite/auth_refresh.rs @@ -54,6 +54,7 @@ async fn refresh_token_succeeds_updates_storage() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -117,6 +118,7 @@ async fn refresh_token_refreshes_when_auth_is_unchanged() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -171,6 +173,7 @@ async fn refresh_token_skips_refresh_when_auth_changed() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -180,6 +183,7 @@ async fn refresh_token_skips_refresh_when_auth_changed() -> Result<()> { openai_api_key: None, tokens: Some(disk_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; save_auth( ctx.codex_home.path(), @@ -234,6 +238,7 @@ async fn refresh_token_errors_on_account_mismatch() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -244,6 +249,7 @@ async fn refresh_token_errors_on_account_mismatch() -> Result<()> { openai_api_key: None, tokens: Some(disk_tokens), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; save_auth( ctx.codex_home.path(), @@ -302,6 +308,7 @@ async fn returns_fresh_tokens_as_is() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(stale_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -349,6 +356,7 @@ async fn refreshes_token_when_access_token_is_expired() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(fresh_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -398,6 +406,7 @@ async fn auth_reloads_disk_auth_when_cached_auth_is_stale() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens), last_refresh: Some(stale_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -408,6 +417,7 @@ async fn auth_reloads_disk_auth_when_cached_auth_is_stale() -> Result<()> { openai_api_key: None, tokens: Some(disk_tokens.clone()), last_refresh: Some(fresh_refresh), + agent_identity: None, }; save_auth( ctx.codex_home.path(), @@ -459,6 +469,7 @@ async fn auth_reloads_disk_auth_without_calling_expired_refresh_token() -> Resul openai_api_key: None, tokens: Some(initial_tokens), last_refresh: Some(stale_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -469,6 +480,7 @@ async fn auth_reloads_disk_auth_without_calling_expired_refresh_token() -> Resul openai_api_key: None, tokens: Some(disk_tokens.clone()), last_refresh: Some(fresh_refresh), + agent_identity: None, }; save_auth( ctx.codex_home.path(), @@ -518,6 +530,7 @@ async fn refresh_token_returns_permanent_error_for_expired_refresh_token() -> Re openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -570,6 +583,7 @@ async fn refresh_token_does_not_retry_after_permanent_failure() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -636,6 +650,7 @@ async fn refresh_token_reloads_changed_auth_after_permanent_failure() -> Result< openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -657,6 +672,7 @@ async fn refresh_token_reloads_changed_auth_after_permanent_failure() -> Result< openai_api_key: None, tokens: Some(disk_tokens.clone()), last_refresh: Some(fresh_refresh), + agent_identity: None, }; save_auth( ctx.codex_home.path(), @@ -715,6 +731,7 @@ async fn refresh_token_returns_transient_error_on_server_failure() -> Result<()> openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -767,6 +784,7 @@ async fn unauthorized_recovery_reloads_then_refreshes_tokens() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -776,6 +794,7 @@ async fn unauthorized_recovery_reloads_then_refreshes_tokens() -> Result<()> { openai_api_key: None, tokens: Some(disk_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; save_auth( ctx.codex_home.path(), @@ -859,6 +878,7 @@ async fn unauthorized_recovery_errors_on_account_mismatch() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -869,6 +889,7 @@ async fn unauthorized_recovery_errors_on_account_mismatch() -> Result<()> { openai_api_key: None, tokens: Some(disk_tokens), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; save_auth( ctx.codex_home.path(), @@ -926,6 +947,7 @@ async fn unauthorized_recovery_requires_chatgpt_auth() -> Result<()> { openai_api_key: Some("sk-test".to_string()), tokens: None, last_refresh: None, + agent_identity: None, }; ctx.write_auth(&auth)?; diff --git a/codex-rs/tui/src/local_chatgpt_auth.rs b/codex-rs/tui/src/local_chatgpt_auth.rs index e888c0387c..1f84b289a7 100644 --- a/codex-rs/tui/src/local_chatgpt_auth.rs +++ b/codex-rs/tui/src/local_chatgpt_auth.rs @@ -108,6 +108,7 @@ mod tests { account_id: Some("workspace-1".to_string()), }), last_refresh: Some(Utc::now()), + agent_identity: None, }; save_auth(codex_home, &auth, AuthCredentialsStoreMode::File) .expect("chatgpt auth should save"); @@ -154,6 +155,7 @@ mod tests { openai_api_key: Some("sk-test".to_string()), tokens: None, last_refresh: None, + agent_identity: None, }, AuthCredentialsStoreMode::File, ) From a3d475d33fa2e27bfc2a795bf76aa1a8d006cd03 Mon Sep 17 00:00:00 2001 From: willwang-openai Date: Wed, 15 Apr 2026 10:50:22 -0700 Subject: [PATCH 22/84] Fix fs/readDirectory to skip broken symlinks (#17907) ## Summary - Skip directory entries whose metadata lookup fails during `fs/readDirectory` - Add an exec-server regression test covering a broken symlink beside valid entries ## Testing - `just fmt` - `cargo test -p codex-exec-server` (started, but dependency/network updates stalled before completion in this environment) --- codex-rs/exec-server/src/local_file_system.rs | 4 +++- codex-rs/exec-server/tests/file_system.rs | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/codex-rs/exec-server/src/local_file_system.rs b/codex-rs/exec-server/src/local_file_system.rs index fe9a4a84f4..c7d628ce8f 100644 --- a/codex-rs/exec-server/src/local_file_system.rs +++ b/codex-rs/exec-server/src/local_file_system.rs @@ -305,7 +305,9 @@ impl ExecutorFileSystem for DirectFileSystem { let mut entries = Vec::new(); let mut read_dir = tokio::fs::read_dir(path.as_path()).await?; while let Some(entry) = read_dir.next_entry().await? { - let metadata = tokio::fs::metadata(entry.path()).await?; + let Ok(metadata) = tokio::fs::metadata(entry.path()).await else { + continue; + }; entries.push(ReadDirectoryEntry { file_name: entry.file_name().to_string_lossy().into_owned(), is_directory: metadata.is_dir(), diff --git a/codex-rs/exec-server/tests/file_system.rs b/codex-rs/exec-server/tests/file_system.rs index e046c203dd..41dd3f8600 100644 --- a/codex-rs/exec-server/tests/file_system.rs +++ b/codex-rs/exec-server/tests/file_system.rs @@ -272,6 +272,11 @@ async fn file_system_methods_cover_surface_area(use_remote: bool) -> Result<()> "hello from trait" ); + symlink( + source_dir.join("missing-target"), + source_dir.join("broken-link"), + )?; + let mut entries = file_system .read_directory(&absolute_path(source_dir), /*sandbox*/ None) .await From 652380d3628698ab441739df75aa6aa6dde5705d Mon Sep 17 00:00:00 2001 From: Dylan Hurd Date: Wed, 15 Apr 2026 10:59:59 -0700 Subject: [PATCH 23/84] chore(features) codex dependencies feat (#17960) ## Summary Setting this up ## Testing - [x] Unit tests pass --- codex-rs/core/config.schema.json | 6 ++++++ codex-rs/features/src/lib.rs | 8 ++++++++ codex-rs/features/src/tests.rs | 10 ++++++++++ 3 files changed, 24 insertions(+) diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index d5d832d6cb..9280895b5e 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -523,6 +523,9 @@ }, "web_search_request": { "type": "boolean" + }, + "workspace_dependencies": { + "type": "boolean" } }, "type": "object" @@ -2380,6 +2383,9 @@ }, "web_search_request": { "type": "boolean" + }, + "workspace_dependencies": { + "type": "boolean" } }, "type": "object" diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rs index 18daae1b1a..4d7a83b628 100644 --- a/codex-rs/features/src/lib.rs +++ b/codex-rs/features/src/lib.rs @@ -191,6 +191,8 @@ pub enum Feature { ResponsesWebsocketsV2, /// Use the agent identity registration flow for ChatGPT-authenticated sessions. UseAgentIdentity, + /// Enable workspace dependency support. + WorkspaceDependencies, } impl Feature { @@ -929,6 +931,12 @@ pub const FEATURES: &[FeatureSpec] = &[ stage: Stage::UnderDevelopment, default_enabled: false, }, + FeatureSpec { + id: Feature::WorkspaceDependencies, + key: "workspace_dependencies", + stage: Stage::Stable, + default_enabled: true, + }, ]; pub fn unstable_features_warning_event( diff --git a/codex-rs/features/src/tests.rs b/codex-rs/features/src/tests.rs index 367527ba15..f55578dbd6 100644 --- a/codex-rs/features/src/tests.rs +++ b/codex-rs/features/src/tests.rs @@ -183,6 +183,16 @@ fn use_agent_identity_is_under_development() { assert_eq!(Feature::UseAgentIdentity.default_enabled(), false); } +#[test] +fn workspace_dependencies_is_stable_and_enabled_by_default() { + assert_eq!(Feature::WorkspaceDependencies.stage(), Stage::Stable); + assert_eq!(Feature::WorkspaceDependencies.default_enabled(), true); + assert_eq!( + feature_for_key("workspace_dependencies"), + Some(Feature::WorkspaceDependencies) + ); +} + #[test] fn collab_is_legacy_alias_for_multi_agent() { assert_eq!(feature_for_key("multi_agent"), Some(Feature::Collab)); From aca781b3a789ed5523b6f0d73f91459970033aa8 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 15 Apr 2026 11:07:57 -0700 Subject: [PATCH 24/84] fix: rename is_azure_responses_wire_base_url to is_azure_responses_provider (#17965) ## Why While reviewing https://github.com/openai/codex/pull/17958, the helper name `is_azure_responses_wire_base_url` looked misleading because the helper returns true for either the `azure` provider name or an Azure Responses `base_url`. The new name makes both inputs part of the contract. ## What - Rename `is_azure_responses_wire_base_url` to `is_azure_responses_provider`. - Move the `openai.azure.` marker into `matches_azure_responses_base_url` so all base URL marker matching is centralized. - Keep `Provider::is_azure_responses_endpoint()` behavior unchanged. ## Verification - Compared the parent and current implementations. `name.eq_ignore_ascii_case("azure")` still returns true before consulting `base_url`, `None` still returns false, base URLs are still lowercased before marker matching, and the same Azure marker set is checked. - Ran `cargo test -p codex-api`. --- codex-rs/codex-api/src/lib.rs | 2 +- codex-rs/codex-api/src/provider.rs | 27 +++++++++++++-------------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/codex-rs/codex-api/src/lib.rs b/codex-rs/codex-api/src/lib.rs index f4f90b289c..82a1004849 100644 --- a/codex-rs/codex-api/src/lib.rs +++ b/codex-rs/codex-api/src/lib.rs @@ -57,7 +57,7 @@ pub use crate::error::ApiError; pub use crate::files::upload_local_file; pub use crate::provider::Provider; pub use crate::provider::RetryConfig; -pub use crate::provider::is_azure_responses_wire_base_url; +pub use crate::provider::is_azure_responses_provider; pub use crate::requests::Compression; pub use crate::sse::stream_from_fixture; pub use crate::telemetry::SseTelemetry; diff --git a/codex-rs/codex-api/src/provider.rs b/codex-rs/codex-api/src/provider.rs index 81a168ffd7..45f2512dc3 100644 --- a/codex-rs/codex-api/src/provider.rs +++ b/codex-rs/codex-api/src/provider.rs @@ -86,7 +86,7 @@ impl Provider { } pub fn is_azure_responses_endpoint(&self) -> bool { - is_azure_responses_wire_base_url(&self.name, Some(&self.base_url)) + is_azure_responses_provider(&self.name, Some(&self.base_url)) } pub fn websocket_url_for_path(&self, path: &str) -> Result { @@ -103,21 +103,20 @@ impl Provider { } } -pub fn is_azure_responses_wire_base_url(name: &str, base_url: Option<&str>) -> bool { +pub fn is_azure_responses_provider(name: &str, base_url: Option<&str>) -> bool { if name.eq_ignore_ascii_case("azure") { - return true; + true + } else if let Some(base_url) = base_url { + matches_azure_responses_base_url(base_url) + } else { + false } - - let Some(base_url) = base_url else { - return false; - }; - - let base = base_url.to_ascii_lowercase(); - base.contains("openai.azure.") || matches_azure_responses_base_url(&base) } fn matches_azure_responses_base_url(base_url: &str) -> bool { - const AZURE_MARKERS: [&str; 5] = [ + let base_url = base_url.to_ascii_lowercase(); + const AZURE_MARKERS: [&str; 6] = [ + "openai.azure.", "cognitiveservices.azure.", "aoai.azure.", "azure-api.", @@ -144,12 +143,12 @@ mod tests { for base_url in positive_cases { assert!( - is_azure_responses_wire_base_url("test", Some(base_url)), + is_azure_responses_provider("test", Some(base_url)), "expected {base_url} to be detected as Azure" ); } - assert!(is_azure_responses_wire_base_url( + assert!(is_azure_responses_provider( "Azure", Some("https://example.com") )); @@ -162,7 +161,7 @@ mod tests { for base_url in negative_cases { assert!( - !is_azure_responses_wire_base_url("test", Some(base_url)), + !is_azure_responses_provider("test", Some(base_url)), "expected {base_url} not to be detected as Azure" ); } From 78ce61c78eaf1eb596b5b78031a47443a4706e44 Mon Sep 17 00:00:00 2001 From: Shijie Rao Date: Wed, 15 Apr 2026 14:14:43 -0400 Subject: [PATCH 25/84] Fix empty tool descriptions (#17946) ## Summary - Ensure direct namespaced MCP tool groups are emitted with a non-empty namespace description even when namespace metadata is missing or blank. - Add regression coverage for missing MCP namespace descriptions. ## Cause Latest `main` can serialize a direct namespaced MCP tool group with an empty top-level `description`. The namespace description path used `unwrap_or_default()` when `tool_namespaces` did not include metadata for that namespace, so the outbound Responses API payload could contain a tool like `{"type":"namespace","description":""}`. The Responses API rejects that because namespace tool descriptions must be a non-empty string. ## Fix - Add a fallback namespace description: `Tools in the namespace.` - Preserve provided namespace descriptions after trimming, but treat blank descriptions as missing. ### Issue I am seeing This is what I am seeing on the local build. Screenshot 2026-04-15 at 10 55 55 AM --------- Co-authored-by: Sayan Sisodiya --- codex-rs/tools/src/lib.rs | 1 + codex-rs/tools/src/responses_api.rs | 4 ++ codex-rs/tools/src/tool_discovery.rs | 12 +++--- codex-rs/tools/src/tool_discovery_tests.rs | 6 +-- codex-rs/tools/src/tool_registry_plan.rs | 18 +++++++-- .../tools/src/tool_registry_plan_tests.rs | 40 +++++++++++++++++++ 6 files changed, 67 insertions(+), 14 deletions(-) diff --git a/codex-rs/tools/src/lib.rs b/codex-rs/tools/src/lib.rs index b30e0bffc2..4a3ccc01ae 100644 --- a/codex-rs/tools/src/lib.rs +++ b/codex-rs/tools/src/lib.rs @@ -85,6 +85,7 @@ pub use responses_api::ResponsesApiNamespace; pub use responses_api::ResponsesApiNamespaceTool; pub use responses_api::ResponsesApiTool; pub use responses_api::ToolSearchOutputTool; +pub(crate) use responses_api::default_namespace_description; pub use responses_api::dynamic_tool_to_responses_api_tool; pub use responses_api::mcp_tool_to_deferred_responses_api_tool; pub use responses_api::mcp_tool_to_responses_api_tool; diff --git a/codex-rs/tools/src/responses_api.rs b/codex-rs/tools/src/responses_api.rs index 205c075105..5317a082f1 100644 --- a/codex-rs/tools/src/responses_api.rs +++ b/codex-rs/tools/src/responses_api.rs @@ -55,6 +55,10 @@ pub struct ResponsesApiNamespace { pub tools: Vec, } +pub(crate) fn default_namespace_description(namespace_name: &str) -> String { + format!("Tools in the {namespace_name} namespace.") +} + #[derive(Debug, Clone, Serialize, PartialEq)] #[serde(tag = "type")] pub enum ResponsesApiNamespaceTool { diff --git a/codex-rs/tools/src/tool_discovery.rs b/codex-rs/tools/src/tool_discovery.rs index e35da5735c..b773623edf 100644 --- a/codex-rs/tools/src/tool_discovery.rs +++ b/codex-rs/tools/src/tool_discovery.rs @@ -5,6 +5,7 @@ use crate::ResponsesApiTool; use crate::ToolName; use crate::ToolSearchOutputTool; use crate::ToolSpec; +use crate::default_namespace_description; use crate::mcp_tool_to_deferred_responses_api_tool; use codex_app_server_protocol::AppInfo; use serde::Deserialize; @@ -225,6 +226,8 @@ pub fn collect_tool_search_output_tools<'a>( let description = first_tool .connector_description + .map(str::trim) + .filter(|description| !description.is_empty()) .map(str::to_string) .or_else(|| { first_tool @@ -232,12 +235,6 @@ pub fn collect_tool_search_output_tools<'a>( .map(str::trim) .filter(|connector_name| !connector_name.is_empty()) .map(|connector_name| format!("Tools for working with {connector_name}.")) - }) - .or_else(|| { - Some(format!( - "Tools from the {} MCP server.", - first_tool.server_name - )) }); let tools = tools @@ -251,7 +248,8 @@ pub fn collect_tool_search_output_tools<'a>( results.push(ToolSearchOutputTool::Namespace(ResponsesApiNamespace { name: tool_namespace.to_string(), - description: description.unwrap_or_default(), + description: description + .unwrap_or_else(|| default_namespace_description(tool_namespace)), tools, })); } diff --git a/codex-rs/tools/src/tool_discovery_tests.rs b/codex-rs/tools/src/tool_discovery_tests.rs index 10e108fe04..b745f756cc 100644 --- a/codex-rs/tools/src/tool_discovery_tests.rs +++ b/codex-rs/tools/src/tool_discovery_tests.rs @@ -253,7 +253,7 @@ fn collect_tool_search_output_tools_preserves_search_order_while_grouping_by_nam }), ToolSearchOutputTool::Namespace(ResponsesApiNamespace { name: "mcp__docs__".to_string(), - description: "Tools from the docs MCP server.".to_string(), + description: "Tools in the mcp__docs__ namespace.".to_string(), tools: vec![ResponsesApiNamespaceTool::Function(ResponsesApiTool { name: "search".to_string(), description: "Search docs.".to_string(), @@ -272,7 +272,7 @@ fn collect_tool_search_output_tools_preserves_search_order_while_grouping_by_nam } #[test] -fn collect_tool_search_output_tools_falls_back_to_connector_name_description() { +fn collect_tool_search_output_tools_ignores_blank_connector_description() { let gmail_batch_read_email = mcp_tool("gmail-batch-read-email", "Read multiple emails."); let tools = collect_tool_search_output_tools([ToolSearchResultSource { @@ -281,7 +281,7 @@ fn collect_tool_search_output_tools_falls_back_to_connector_name_description() { tool_name: "_batch_read_email", tool: &gmail_batch_read_email, connector_name: Some("Gmail"), - connector_description: None, + connector_description: Some(" "), }]) .expect("collect tool search output tools"); diff --git a/codex-rs/tools/src/tool_registry_plan.rs b/codex-rs/tools/src/tool_registry_plan.rs index cec938962c..b5da9afb25 100644 --- a/codex-rs/tools/src/tool_registry_plan.rs +++ b/codex-rs/tools/src/tool_registry_plan.rs @@ -55,6 +55,7 @@ use crate::create_wait_agent_tool_v2; use crate::create_wait_tool; use crate::create_web_search_tool; use crate::create_write_stdin_tool; +use crate::default_namespace_description; use crate::dynamic_tool_to_responses_api_tool; use crate::mcp_tool_to_responses_api_tool; use crate::request_permissions_tool_description; @@ -485,11 +486,20 @@ pub fn build_tool_registry_plan( for (namespace, mut entries) in namespace_entries { entries.sort_by_key(|tool| tool.name.name.clone()); - let description = params + let tool_namespace = params .tool_namespaces - .and_then(|namespaces| namespaces.get(&namespace)) - .and_then(|namespace| namespace.description.clone()) - .unwrap_or_default(); + .and_then(|namespaces| namespaces.get(&namespace)); + let description = tool_namespace + .and_then(|namespace| namespace.description.as_deref()) + .map(str::trim) + .filter(|description| !description.is_empty()) + .map(str::to_string) + .unwrap_or_else(|| { + let namespace_name = tool_namespace + .map(|namespace| namespace.name.as_str()) + .unwrap_or(namespace.as_str()); + default_namespace_description(namespace_name) + }); let mut tools = Vec::new(); for tool in entries { match mcp_tool_to_responses_api_tool(&tool.name, tool.tool) { diff --git a/codex-rs/tools/src/tool_registry_plan_tests.rs b/codex-rs/tools/src/tool_registry_plan_tests.rs index 7b9e8ed9ac..a643543ee7 100644 --- a/codex-rs/tools/src/tool_registry_plan_tests.rs +++ b/codex-rs/tools/src/tool_registry_plan_tests.rs @@ -1151,6 +1151,46 @@ fn test_build_specs_mcp_tools_converted() { ); } +#[test] +fn test_build_specs_mcp_namespace_description_falls_back_when_missing() { + let model_info = model_info(); + let mut features = Features::with_defaults(); + features.enable(Feature::UnifiedExec); + let available_models = Vec::new(); + let tools_config = ToolsConfig::new(&ToolsConfigParams { + model_info: &model_info, + available_models: &available_models, + features: &features, + image_generation_tool_auth_allowed: true, + web_search_mode: Some(WebSearchMode::Cached), + session_source: SessionSource::Cli, + sandbox_policy: &SandboxPolicy::DangerFullAccess, + windows_sandbox_level: WindowsSandboxLevel::Disabled, + }); + let (tools, _) = build_specs( + &tools_config, + Some(HashMap::from([( + ToolName::namespaced("test_server/", "do_something_cool"), + mcp_tool( + "do_something_cool", + "Do something cool", + serde_json::json!({"type": "object"}), + ), + )])), + /*deferred_mcp_tools*/ None, + &[], + ); + + let namespace_tool = find_tool(&tools, "test_server/"); + let ToolSpec::Namespace(namespace) = &namespace_tool.spec else { + panic!("expected namespace tool"); + }; + assert_eq!( + namespace.description, + "Tools in the test_server/ namespace." + ); +} + #[test] fn test_build_specs_mcp_tools_sorted_by_name() { let model_info = model_info(); From cdfcd2ca9268adbd4bd3ba5af527c97fa8a82805 Mon Sep 17 00:00:00 2001 From: Tom Date: Wed, 15 Apr 2026 11:34:27 -0700 Subject: [PATCH 26/84] [codex] Add local thread store listing (#17824) Builds on top of #17659 Move the filesystem + sqlite thread listing-related operations inside of a local ThreadStore implementation and call ThreadStore from the places that used to perform these filesystem/sqlite operations. This is the first of a series of PRs that will implement the rest of the local ThreadStore. Testing: - added unit tests for the thread store implementation - adjusted some unit tests in the realtime + personality packages whose callsites changed. Specifically I'm trying to hide ThreadMetadata inside of the local implementation and make ThreadMetadata a sqlite implementation detail concern rather than a public interface, preferring the more generate StoredThread interface instead - added a corner case test for the personality migration package that wasn't covered by the existing test suite - adjust the behavior of searched thread listing to run the existing local rollout repair/backfill pass _before_ querying SQLite results, so callers using ThreadStore::list_threads do not miss matches after a partial metadata warm-up --- codex-rs/Cargo.lock | 10 + codex-rs/Cargo.toml | 1 + codex-rs/app-server/Cargo.toml | 1 + .../app-server/src/codex_message_processor.rs | 187 +++--- .../app-server/tests/suite/v2/thread_list.rs | 25 +- codex-rs/core/Cargo.toml | 1 + codex-rs/core/src/codex.rs | 3 + codex-rs/core/src/codex_tests.rs | 6 + codex-rs/core/src/personality_migration.rs | 86 +-- .../core/src/personality_migration_tests.rs | 28 + codex-rs/core/src/realtime_context.rs | 49 +- codex-rs/core/src/realtime_context_tests.rs | 56 +- codex-rs/core/src/rollout.rs | 4 - codex-rs/core/src/state/service.rs | 2 + .../core/tests/suite/realtime_conversation.rs | 10 +- codex-rs/rollout/src/recorder.rs | 4 + codex-rs/thread-store/Cargo.toml | 10 + codex-rs/thread-store/src/lib.rs | 2 + codex-rs/thread-store/src/local/mod.rs | 565 ++++++++++++++++++ codex-rs/thread-store/src/types.rs | 4 + 20 files changed, 821 insertions(+), 233 deletions(-) create mode 100644 codex-rs/thread-store/src/local/mod.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 5e54b772c1..68e0ec092f 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1461,6 +1461,7 @@ dependencies = [ "codex-sandboxing", "codex-shell-command", "codex-state", + "codex-thread-store", "codex-tools", "codex-utils-absolute-path", "codex-utils-cargo-bin", @@ -1938,6 +1939,7 @@ dependencies = [ "codex-state", "codex-terminal-detection", "codex-test-binary-support", + "codex-thread-store", "codex-tools", "codex-utils-absolute-path", "codex-utils-cache", @@ -2841,9 +2843,17 @@ version = "0.0.0" dependencies = [ "async-trait", "chrono", + "codex-git-utils", "codex-protocol", + "codex-rollout", + "codex-state", + "pretty_assertions", "serde", + "serde_json", + "tempfile", "thiserror 2.0.18", + "tokio", + "uuid", ] [[package]] diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index b57a464d5e..4bb8837211 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -166,6 +166,7 @@ codex-state = { path = "state" } codex-stdio-to-uds = { path = "stdio-to-uds" } codex-terminal-detection = { path = "terminal-detection" } codex-test-binary-support = { path = "test-binary-support" } +codex-thread-store = { path = "thread-store" } codex-tools = { path = "tools" } codex-tui = { path = "tui" } codex-utils-absolute-path = { path = "utils/absolute-path" } diff --git a/codex-rs/app-server/Cargo.toml b/codex-rs/app-server/Cargo.toml index 5dc3a31485..408f39fafe 100644 --- a/codex-rs/app-server/Cargo.toml +++ b/codex-rs/app-server/Cargo.toml @@ -54,6 +54,7 @@ codex-rmcp-client = { workspace = true } codex-rollout = { workspace = true } codex-sandboxing = { workspace = true } codex-state = { workspace = true } +codex-thread-store = { workspace = true } codex-tools = { workspace = true } codex-utils-absolute-path = { workspace = true } codex-utils-json-to-toml = { workspace = true } diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 513837c7ce..05ce2d7974 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -203,7 +203,6 @@ use codex_chatgpt::connectors; use codex_cloud_requirements::cloud_requirements_loader; use codex_config::types::McpServerTransportConfig; use codex_core::CodexThread; -use codex_core::Cursor as RolloutCursor; use codex_core::ForkSnapshot; use codex_core::NewThread; use codex_core::RolloutRecorder; @@ -211,7 +210,6 @@ use codex_core::SessionMeta; use codex_core::SteerInputError; use codex_core::ThreadConfigSnapshot; use codex_core::ThreadManager; -use codex_core::ThreadSortKey as CoreThreadSortKey; use codex_core::append_thread_name; use codex_core::config::Config; use codex_core::config::ConfigOverrides; @@ -232,7 +230,6 @@ use codex_core::find_archived_thread_path_by_id_str; use codex_core::find_thread_name_by_id; use codex_core::find_thread_names_by_ids; use codex_core::find_thread_path_by_id_str; -use codex_core::parse_cursor; use codex_core::path_utils; use codex_core::plugins::MarketplaceAddError; use codex_core::plugins::MarketplaceError; @@ -322,6 +319,12 @@ use codex_state::StateRuntime; use codex_state::ThreadMetadata; use codex_state::ThreadMetadataBuilder; use codex_state::log_db::LogDbLayer; +use codex_thread_store::ListThreadsParams as StoreListThreadsParams; +use codex_thread_store::LocalThreadStore; +use codex_thread_store::StoredThread; +use codex_thread_store::ThreadSortKey as StoreThreadSortKey; +use codex_thread_store::ThreadStore; +use codex_thread_store::ThreadStoreError; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_json_to_toml::json_to_toml; use codex_utils_pty::DEFAULT_OUTPUT_BYTES_CAP; @@ -3819,15 +3822,15 @@ impl CodexMessageProcessor { .map(|value| value as usize) .unwrap_or(THREAD_LIST_DEFAULT_LIMIT) .clamp(1, THREAD_LIST_MAX_LIMIT); - let core_sort_key = match sort_key.unwrap_or(ThreadSortKey::CreatedAt) { - ThreadSortKey::CreatedAt => CoreThreadSortKey::CreatedAt, - ThreadSortKey::UpdatedAt => CoreThreadSortKey::UpdatedAt, + let store_sort_key = match sort_key.unwrap_or(ThreadSortKey::CreatedAt) { + ThreadSortKey::CreatedAt => StoreThreadSortKey::CreatedAt, + ThreadSortKey::UpdatedAt => StoreThreadSortKey::UpdatedAt, }; let (summaries, next_cursor) = match self .list_threads_common( requested_page_size, cursor, - core_sort_key, + store_sort_key, ThreadListFilters { model_providers, source_kinds, @@ -5071,7 +5074,7 @@ impl CodexMessageProcessor { &self, requested_page_size: usize, cursor: Option, - sort_key: CoreThreadSortKey, + sort_key: StoreThreadSortKey, filters: ThreadListFilters, ) -> Result<(Vec, Option), JSONRPCErrorError> { let ThreadListFilters { @@ -5081,16 +5084,7 @@ impl CodexMessageProcessor { cwd, search_term, } = filters; - let mut cursor_obj: Option = match cursor.as_ref() { - Some(cursor_str) => { - Some(parse_cursor(cursor_str).ok_or_else(|| JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!("invalid cursor: {cursor_str}"), - data: None, - })?) - } - None => None, - }; + let mut cursor_obj = cursor; let mut last_cursor = cursor_obj.clone(); let mut remaining = requested_page_size; let mut items = Vec::with_capacity(requested_page_size); @@ -5109,54 +5103,26 @@ impl CodexMessageProcessor { let fallback_provider = self.config.model_provider_id.clone(); let (allowed_sources_vec, source_kind_filter) = compute_source_filters(source_kinds); let allowed_sources = allowed_sources_vec.as_slice(); - let state_db_ctx = get_state_db(&self.config).await; + let store = LocalThreadStore::new(codex_rollout::RolloutConfig::from_view(&self.config)); while remaining > 0 { let page_size = remaining.min(THREAD_LIST_MAX_LIMIT); - let page = if archived { - RolloutRecorder::list_archived_threads( - &self.config, + let page = store + .list_threads(StoreListThreadsParams { page_size, - cursor_obj.as_ref(), + cursor: cursor_obj.clone(), sort_key, - allowed_sources, - model_provider_filter.as_deref(), - fallback_provider.as_str(), - search_term.as_deref(), - ) + allowed_sources: allowed_sources.to_vec(), + model_providers: model_provider_filter.clone(), + archived, + search_term: search_term.clone(), + }) .await - .map_err(|err| JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!("failed to list threads: {err}"), - data: None, - })? - } else { - RolloutRecorder::list_threads( - &self.config, - page_size, - cursor_obj.as_ref(), - sort_key, - allowed_sources, - model_provider_filter.as_deref(), - fallback_provider.as_str(), - search_term.as_deref(), - ) - .await - .map_err(|err| JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!("failed to list threads: {err}"), - data: None, - })? - }; + .map_err(thread_store_list_error)?; let mut filtered = Vec::with_capacity(page.items.len()); for it in page.items { - let Some(summary) = summary_from_thread_list_item( - it, - fallback_provider.as_str(), - state_db_ctx.as_ref(), - ) - .await + let Some(summary) = summary_from_stored_thread(it, fallback_provider.as_str()) else { continue; }; @@ -5176,12 +5142,8 @@ impl CodexMessageProcessor { items.extend(filtered); remaining = requested_page_size.saturating_sub(items.len()); - // Encode RolloutCursor into the JSON-RPC string form returned to clients. let next_cursor_value = page.next_cursor.clone(); - next_cursor = next_cursor_value - .as_ref() - .and_then(|cursor| serde_json::to_value(cursor).ok()) - .and_then(|value| value.as_str().map(str::to_owned)); + next_cursor = next_cursor_value.clone(); if remaining == 0 { break; } @@ -9421,67 +9383,52 @@ fn set_thread_name_from_title(thread: &mut Thread, title: String) { thread.name = Some(title); } -async fn summary_from_thread_list_item( - it: codex_core::ThreadItem, - fallback_provider: &str, - state_db_ctx: Option<&StateDbHandle>, -) -> Option { - if let Some(thread_id) = it.thread_id { - let timestamp = it.created_at.clone(); - let updated_at = it.updated_at.clone().or_else(|| timestamp.clone()); - let model_provider = it - .model_provider - .clone() - .unwrap_or_else(|| fallback_provider.to_string()); - let cwd = it.cwd?; - let cli_version = it.cli_version.unwrap_or_default(); - let source = with_thread_spawn_agent_metadata( - it.source - .unwrap_or(codex_protocol::protocol::SessionSource::Unknown), - it.agent_nickname.clone(), - it.agent_role.clone(), - ); - return Some(ConversationSummary { - conversation_id: thread_id, - path: it.path, - preview: it.first_user_message.unwrap_or_default(), - timestamp, - updated_at, - model_provider, - cwd, - cli_version, - source, - git_info: if it.git_sha.is_none() - && it.git_branch.is_none() - && it.git_origin_url.is_none() - { - None - } else { - Some(ConversationGitInfo { - sha: it.git_sha, - branch: it.git_branch, - origin_url: it.git_origin_url, - }) - }, - }); +fn thread_store_list_error(err: ThreadStoreError) -> JSONRPCErrorError { + match err { + ThreadStoreError::InvalidRequest { message } => JSONRPCErrorError { + code: INVALID_REQUEST_ERROR_CODE, + message, + data: None, + }, + err => JSONRPCErrorError { + code: INTERNAL_ERROR_CODE, + message: format!("failed to list threads: {err}"), + data: None, + }, } - if let Some(thread_id) = thread_id_from_rollout_path(it.path.as_path()) { - return read_summary_from_state_db_context_by_thread_id(state_db_ctx, thread_id).await; - } - None } -fn thread_id_from_rollout_path(path: &Path) -> Option { - let file_name = path.file_name()?.to_str()?; - let stem = file_name.strip_suffix(".jsonl")?; - if stem.len() < 37 { - return None; - } - let uuid_start = stem.len().saturating_sub(36); - if !stem[..uuid_start].ends_with('-') { - return None; - } - ThreadId::from_string(&stem[uuid_start..]).ok() +fn summary_from_stored_thread( + thread: StoredThread, + fallback_provider: &str, +) -> Option { + let path = thread.rollout_path?; + let source = with_thread_spawn_agent_metadata( + thread.source, + thread.agent_nickname.clone(), + thread.agent_role.clone(), + ); + let git_info = thread.git_info.map(|git| ConversationGitInfo { + sha: git.commit_hash.map(|sha| sha.0), + branch: git.branch, + origin_url: git.repository_url, + }); + Some(ConversationSummary { + conversation_id: thread.thread_id, + path, + preview: thread.first_user_message.unwrap_or(thread.preview), + timestamp: Some(thread.created_at.to_rfc3339_opts(SecondsFormat::Secs, true)), + updated_at: Some(thread.updated_at.to_rfc3339_opts(SecondsFormat::Secs, true)), + model_provider: if thread.model_provider.is_empty() { + fallback_provider.to_string() + } else { + thread.model_provider + }, + cwd: thread.cwd, + cli_version: thread.cli_version, + source, + git_info, + }) } #[allow(clippy::too_many_arguments)] diff --git a/codex-rs/app-server/tests/suite/v2/thread_list.rs b/codex-rs/app-server/tests/suite/v2/thread_list.rs index cf1ac4ff11..8fd4e47303 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_list.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_list.rs @@ -563,15 +563,34 @@ sqlite = true /*git_info*/ None, )?; - // `thread/list` only applies `search_term` on the sqlite path. In this test we - // create rollouts manually, so we must also create the sqlite DB and mark backfill - // complete; otherwise app-server will permanently use filesystem fallback. + // `thread/list` applies `search_term` on the sqlite fast path. This test creates + // rollouts manually, so mark the DB backfill complete and then run an unsearched + // list large enough to repair every rollout the searched list should find. let state_db = codex_state::StateRuntime::init(codex_home.path().to_path_buf(), "mock_provider".into()) .await?; state_db .mark_backfill_complete(/*last_watermark*/ None) .await?; + let rollout_config = codex_rollout::RolloutConfig { + codex_home: codex_home.path().to_path_buf(), + sqlite_home: codex_home.path().to_path_buf(), + cwd: codex_home.path().to_path_buf(), + model_provider_id: "mock_provider".to_string(), + generate_memories: false, + }; + let repaired_page = codex_core::RolloutRecorder::list_threads( + &rollout_config, + /*page_size*/ 10, + /*cursor*/ None, + codex_core::ThreadSortKey::CreatedAt, + &[], + /*model_providers*/ None, + "mock_provider", + /*search_term*/ None, + ) + .await?; + assert_eq!(repaired_page.items.len(), 3); let mut mcp = init_mcp(codex_home.path()).await?; let request_id = mcp diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index a371966079..78dfd3c070 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -57,6 +57,7 @@ codex-rmcp-client = { workspace = true } codex-sandboxing = { workspace = true } codex-state = { workspace = true } codex-terminal-detection = { workspace = true } +codex-thread-store = { workspace = true } codex-tools = { workspace = true } codex-utils-absolute-path = { workspace = true } codex-utils-cache = { workspace = true } diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index a3d7594813..bc7cd07e4c 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -136,9 +136,11 @@ use codex_protocol::request_permissions::RequestPermissionsResponse; use codex_protocol::request_user_input::RequestUserInputArgs; use codex_protocol::request_user_input::RequestUserInputResponse; use codex_rmcp_client::ElicitationResponse; +use codex_rollout::RolloutConfig; use codex_rollout::state_db; use codex_shell_command::parse_command::parse_command; use codex_terminal_detection::user_agent; +use codex_thread_store::LocalThreadStore; use codex_tools::filter_tool_suggest_discoverable_tools_for_client; use codex_utils_output_truncation::TruncationPolicy; use codex_utils_stream_parser::AssistantTextChunk; @@ -2127,6 +2129,7 @@ impl Session { network_proxy, network_approval: Arc::clone(&network_approval), state_db: state_db_ctx.clone(), + thread_store: LocalThreadStore::new(RolloutConfig::from_view(config.as_ref())), model_client: ModelClient::new( Some(Arc::clone(&auth_manager)), conversation_id, diff --git a/codex-rs/core/src/codex_tests.rs b/codex-rs/core/src/codex_tests.rs index 46d4cda57e..fb7bbaadba 100644 --- a/codex-rs/core/src/codex_tests.rs +++ b/codex-rs/core/src/codex_tests.rs @@ -2836,6 +2836,9 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) { network_proxy: None, network_approval: Arc::clone(&network_approval), state_db: None, + thread_store: codex_thread_store::LocalThreadStore::new( + codex_rollout::RolloutConfig::from_view(config.as_ref()), + ), model_client: ModelClient::new( Some(auth_manager.clone()), conversation_id, @@ -3693,6 +3696,9 @@ pub(crate) async fn make_session_and_context_with_dynamic_tools_and_rx( network_proxy: None, network_approval: Arc::clone(&network_approval), state_db: None, + thread_store: codex_thread_store::LocalThreadStore::new( + codex_rollout::RolloutConfig::from_view(config.as_ref()), + ), model_client: ModelClient::new( Some(Arc::clone(&auth_manager)), conversation_id, diff --git a/codex-rs/core/src/personality_migration.rs b/codex-rs/core/src/personality_migration.rs index 52cabf55de..4b53a2629e 100644 --- a/codex-rs/core/src/personality_migration.rs +++ b/codex-rs/core/src/personality_migration.rs @@ -1,14 +1,10 @@ use crate::config::edit::ConfigEditsBuilder; -use crate::rollout::ARCHIVED_SESSIONS_SUBDIR; -use crate::rollout::SESSIONS_SUBDIR; -use crate::rollout::list::ThreadListConfig; -use crate::rollout::list::ThreadListLayout; -use crate::rollout::list::ThreadSortKey; -use crate::rollout::list::get_threads_in_root; use codex_config::config_toml::ConfigToml; use codex_protocol::config_types::Personality; -use codex_protocol::protocol::SessionSource; -use codex_rollout::state_db; +use codex_thread_store::ListThreadsParams; +use codex_thread_store::LocalThreadStore; +use codex_thread_store::ThreadSortKey; +use codex_thread_store::ThreadStore; use std::io; use std::path::Path; use tokio::fs::OpenOptions; @@ -64,57 +60,33 @@ pub async fn maybe_migrate_personality( } async fn has_recorded_sessions(codex_home: &Path, default_provider: &str) -> io::Result { - let allowed_sources: &[SessionSource] = &[]; + let store = LocalThreadStore::new(codex_rollout::RolloutConfig { + codex_home: codex_home.to_path_buf(), + sqlite_home: codex_home.to_path_buf(), + cwd: codex_home.to_path_buf(), + model_provider_id: default_provider.to_string(), + generate_memories: false, + }); + if has_threads(&store, /*archived*/ false).await? { + return Ok(true); + } + has_threads(&store, /*archived*/ true).await +} - if let Some(state_db_ctx) = state_db::open_if_present(codex_home, default_provider).await - && let Some(ids) = state_db::list_thread_ids_db( - Some(state_db_ctx.as_ref()), - codex_home, - /*page_size*/ 1, - /*cursor*/ None, - ThreadSortKey::CreatedAt, - allowed_sources, - /*model_providers*/ None, - /*archived_only*/ false, - "personality_migration", - ) +async fn has_threads(store: &LocalThreadStore, archived: bool) -> io::Result { + store + .list_threads(ListThreadsParams { + page_size: 1, + cursor: None, + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived, + search_term: None, + }) .await - && !ids.is_empty() - { - return Ok(true); - } - - let sessions = get_threads_in_root( - codex_home.join(SESSIONS_SUBDIR), - /*page_size*/ 1, - /*cursor*/ None, - ThreadSortKey::CreatedAt, - ThreadListConfig { - allowed_sources, - model_providers: None, - default_provider, - layout: ThreadListLayout::NestedByDate, - }, - ) - .await?; - if !sessions.items.is_empty() { - return Ok(true); - } - - let archived_sessions = get_threads_in_root( - codex_home.join(ARCHIVED_SESSIONS_SUBDIR), - /*page_size*/ 1, - /*cursor*/ None, - ThreadSortKey::CreatedAt, - ThreadListConfig { - allowed_sources, - model_providers: None, - default_provider, - layout: ThreadListLayout::Flat, - }, - ) - .await?; - Ok(!archived_sessions.items.is_empty()) + .map(|page| !page.items.is_empty()) + .map_err(io::Error::other) } async fn create_marker(marker_path: &Path) -> io::Result<()> { diff --git a/codex-rs/core/src/personality_migration_tests.rs b/codex-rs/core/src/personality_migration_tests.rs index de1070ad34..4aef53a5c4 100644 --- a/codex-rs/core/src/personality_migration_tests.rs +++ b/codex-rs/core/src/personality_migration_tests.rs @@ -7,6 +7,8 @@ use codex_protocol::protocol::SessionMeta; use codex_protocol::protocol::SessionMetaLine; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::UserMessageEvent; +use codex_rollout::ARCHIVED_SESSIONS_SUBDIR; +use codex_rollout::SESSIONS_SUBDIR; use pretty_assertions::assert_eq; use tempfile::TempDir; use tokio::io::AsyncWriteExt; @@ -25,6 +27,16 @@ async fn write_session_with_user_event(codex_home: &Path) -> io::Result<()> { .join("2025") .join("01") .join("01"); + write_rollout_with_user_event(&dir, thread_id).await +} + +async fn write_archived_session_with_user_event(codex_home: &Path) -> io::Result<()> { + let thread_id = ThreadId::new(); + let dir = codex_home.join(ARCHIVED_SESSIONS_SUBDIR); + write_rollout_with_user_event(&dir, thread_id).await +} + +async fn write_rollout_with_user_event(dir: &Path, thread_id: ThreadId) -> io::Result<()> { tokio::fs::create_dir_all(&dir).await?; let file_path = dir.join(format!("rollout-{TEST_TIMESTAMP}-{thread_id}.jsonl")); let mut file = tokio::fs::File::create(&file_path).await?; @@ -85,6 +97,22 @@ async fn applies_when_sessions_exist_and_no_personality() -> io::Result<()> { Ok(()) } +#[tokio::test] +async fn applies_when_only_archived_sessions_exist_and_no_personality() -> io::Result<()> { + let temp = TempDir::new()?; + write_archived_session_with_user_event(temp.path()).await?; + + let config_toml = ConfigToml::default(); + let status = maybe_migrate_personality(temp.path(), &config_toml).await?; + + assert_eq!(status, PersonalityMigrationStatus::Applied); + assert!(temp.path().join(PERSONALITY_MIGRATION_FILENAME).exists()); + + let persisted = read_config_toml(temp.path()).await?; + assert_eq!(persisted.personality, Some(Personality::Pragmatic)); + Ok(()) +} + #[tokio::test] async fn skips_when_marker_exists() -> io::Result<()> { let temp = TempDir::new()?; diff --git a/codex-rs/core/src/realtime_context.rs b/codex-rs/core/src/realtime_context.rs index 6e9815f562..30cbd6f636 100644 --- a/codex-rs/core/src/realtime_context.rs +++ b/codex-rs/core/src/realtime_context.rs @@ -4,8 +4,10 @@ use crate::event_mapping::is_contextual_user_message_content; use chrono::Utc; use codex_git_utils::resolve_root_git_project_for_trust; use codex_protocol::models::ResponseItem; -use codex_state::SortKey; -use codex_state::ThreadMetadata; +use codex_thread_store::ListThreadsParams; +use codex_thread_store::StoredThread; +use codex_thread_store::ThreadSortKey; +use codex_thread_store::ThreadStore; use codex_utils_output_truncation::TruncationPolicy; use codex_utils_output_truncation::truncate_text; use dirs::home_dir; @@ -98,7 +100,7 @@ pub(crate) async fn build_realtime_startup_context( } if let Some(section) = format_section( "Notes", - Some("Built at realtime startup from the current thread history, persisted thread metadata in the state DB, and a bounded local workspace scan. This excludes repo memory instructions, AGENTS files, project-doc prompt blends, and memory summaries.".to_string()), + Some("Built at realtime startup from the current thread history, local thread metadata, and a bounded local workspace scan. This excludes repo memory instructions, AGENTS files, project-doc prompt blends, and memory summaries.".to_string()), NOTES_SECTION_TOKEN_BUDGET, ) { parts.push(section); @@ -117,33 +119,31 @@ pub(crate) async fn build_realtime_startup_context( Some(context) } -async fn load_recent_threads(sess: &Session) -> Vec { - let Some(state_db) = sess.services.state_db.as_ref() else { - return Vec::new(); - }; - - match state_db - .list_threads( - MAX_RECENT_THREADS, - /*anchor*/ None, - SortKey::UpdatedAt, - &[], - /*model_providers*/ None, - /*archived_only*/ false, - /*search_term*/ None, - ) +async fn load_recent_threads(sess: &Session) -> Vec { + match sess + .services + .thread_store + .list_threads(ListThreadsParams { + page_size: MAX_RECENT_THREADS, + cursor: None, + sort_key: ThreadSortKey::UpdatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived: false, + search_term: None, + }) .await { Ok(page) => page.items, Err(err) => { - warn!("failed to load realtime startup threads from state db: {err}"); + warn!("failed to load realtime startup threads from thread store: {err}"); Vec::new() } } } -fn build_recent_work_section(cwd: &Path, recent_threads: &[ThreadMetadata]) -> Option { - let mut groups: HashMap> = HashMap::new(); +fn build_recent_work_section(cwd: &Path, recent_threads: &[StoredThread]) -> Option { + let mut groups: HashMap> = HashMap::new(); for entry in recent_threads { let group = resolve_root_git_project_for_trust(&entry.cwd).unwrap_or_else(|| entry.cwd.clone()); @@ -446,7 +446,7 @@ fn format_section(title: &str, body: Option, budget_tokens: usize) -> Op fn format_thread_group( current_group: &Path, group: &Path, - entries: Vec<&ThreadMetadata>, + entries: Vec<&StoredThread>, ) -> Option { let latest = entries.first()?; let group_label = if resolve_root_git_project_for_trust(latest.cwd.as_path()).is_some() { @@ -461,8 +461,9 @@ fn format_thread_group( ]; if let Some(git_branch) = latest - .git_branch - .as_deref() + .git_info + .as_ref() + .and_then(|git| git.branch.as_deref()) .filter(|git_branch| !git_branch.is_empty()) { lines.push(format!("Latest branch: {git_branch}")); diff --git a/codex-rs/core/src/realtime_context_tests.rs b/codex-rs/core/src/realtime_context_tests.rs index 7495161d46..7e1e82476f 100644 --- a/codex-rs/core/src/realtime_context_tests.rs +++ b/codex-rs/core/src/realtime_context_tests.rs @@ -3,20 +3,31 @@ use super::build_recent_work_section; use super::build_workspace_section_with_user_root; use chrono::TimeZone; use chrono::Utc; +use codex_git_utils::GitSha; use codex_protocol::ThreadId; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; -use codex_state::ThreadMetadata; +use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::GitInfo; +use codex_protocol::protocol::SandboxPolicy; +use codex_protocol::protocol::SessionSource; +use codex_thread_store::StoredThread; use pretty_assertions::assert_eq; use std::fs; use std::path::PathBuf; use std::process::Command; use tempfile::TempDir; -fn thread_metadata(cwd: &str, title: &str, first_user_message: &str) -> ThreadMetadata { - ThreadMetadata { - id: ThreadId::new(), - rollout_path: PathBuf::from("/tmp/rollout.jsonl"), +fn stored_thread(cwd: &str, title: &str, first_user_message: &str) -> StoredThread { + StoredThread { + thread_id: ThreadId::new(), + rollout_path: Some(PathBuf::from("/tmp/rollout.jsonl")), + forked_from_id: None, + preview: first_user_message.to_string(), + name: (!title.is_empty()).then(|| title.to_string()), + model_provider: "test-provider".to_string(), + model: Some("gpt-5".to_string()), + reasoning_effort: None, created_at: Utc .timestamp_opt(1_709_251_100, 0) .single() @@ -25,24 +36,23 @@ fn thread_metadata(cwd: &str, title: &str, first_user_message: &str) -> ThreadMe .timestamp_opt(1_709_251_200, 0) .single() .expect("valid timestamp"), - source: "cli".to_string(), - agent_path: None, - agent_nickname: None, - agent_role: None, - model_provider: "test-provider".to_string(), - model: Some("gpt-5".to_string()), - reasoning_effort: None, + archived_at: None, cwd: PathBuf::from(cwd), cli_version: "test".to_string(), - title: title.to_string(), - sandbox_policy: "workspace-write".to_string(), - approval_mode: "never".to_string(), - tokens_used: 0, + source: SessionSource::Cli, + agent_nickname: None, + agent_role: None, + agent_path: None, + git_info: Some(GitInfo { + commit_hash: Some(GitSha::new("abcdef")), + branch: Some("main".to_string()), + repository_url: None, + }), + approval_mode: AskForApproval::Never, + sandbox_policy: SandboxPolicy::new_read_only_policy(), + token_usage: None, first_user_message: Some(first_user_message.to_string()), - archived_at: None, - git_sha: None, - git_branch: Some("main".to_string()), - git_origin_url: None, + history: None, } } @@ -224,17 +234,17 @@ fn recent_work_section_groups_threads_by_cwd() { fs::create_dir_all(&outside).expect("create outside dir"); let recent_threads = vec![ - thread_metadata( + stored_thread( workspace_a.to_string_lossy().as_ref(), "Investigate realtime startup context", "Log the startup context before sending it", ), - thread_metadata( + stored_thread( workspace_b.to_string_lossy().as_ref(), "Trim websocket startup payload", "Remove memories from the realtime startup context", ), - thread_metadata(outside.to_string_lossy().as_ref(), "", "Inspect flaky test"), + stored_thread(outside.to_string_lossy().as_ref(), "", "Inspect flaky test"), ]; let current_cwd = workspace_a; let repo = fs::canonicalize(repo).expect("canonicalize repo"); diff --git a/codex-rs/core/src/rollout.rs b/codex-rs/core/src/rollout.rs index 26e3a84d5c..4913f957ea 100644 --- a/codex-rs/core/src/rollout.rs +++ b/codex-rs/core/src/rollout.rs @@ -46,11 +46,7 @@ impl codex_rollout::RolloutConfigView for Config { } pub(crate) mod list { - pub use codex_rollout::ThreadListConfig; - pub use codex_rollout::ThreadListLayout; - pub use codex_rollout::ThreadSortKey; pub use codex_rollout::find_thread_path_by_id_str; - pub use codex_rollout::get_threads_in_root; } pub(crate) mod metadata { diff --git a/codex-rs/core/src/state/service.rs b/codex-rs/core/src/state/service.rs index 4b6c5b30f5..5db38f7b72 100644 --- a/codex-rs/core/src/state/service.rs +++ b/codex-rs/core/src/state/service.rs @@ -24,6 +24,7 @@ use codex_mcp::McpConnectionManager; use codex_models_manager::manager::ModelsManager; use codex_otel::SessionTelemetry; use codex_rollout::state_db::StateDbHandle; +use codex_thread_store::LocalThreadStore; use std::path::PathBuf; use tokio::sync::Mutex; use tokio::sync::RwLock; @@ -59,6 +60,7 @@ pub(crate) struct SessionServices { pub(crate) network_proxy: Option, pub(crate) network_approval: Arc, pub(crate) state_db: Option, + pub(crate) thread_store: LocalThreadStore, /// Session-scoped model client shared across turns. pub(crate) model_client: ModelClient, pub(crate) code_mode_service: CodeModeService, diff --git a/codex-rs/core/tests/suite/realtime_conversation.rs b/codex-rs/core/tests/suite/realtime_conversation.rs index add4065f7b..ecaaeb8cba 100644 --- a/codex-rs/core/tests/suite/realtime_conversation.rs +++ b/codex-rs/core/tests/suite/realtime_conversation.rs @@ -194,10 +194,16 @@ async fn seed_recent_thread( let db = test.codex.state_db().context("state db enabled")?; let thread_id = ThreadId::new(); let updated_at = Utc::now(); + let rollout_path = test + .codex_home_path() + .join(format!("rollout-{thread_id}.jsonl")); + // This helper seeds SQLite metadata directly. Local listing drops stale metadata rows whose + // rollout path no longer exists, so create the placeholder path that the test metadata points + // at without exercising rollout writing in this realtime-context test. + std::fs::write(&rollout_path, "")?; let mut metadata_builder = codex_state::ThreadMetadataBuilder::new( thread_id, - test.codex_home_path() - .join(format!("rollout-{thread_id}.jsonl")), + rollout_path, updated_at, SessionSource::Cli, ); diff --git a/codex-rs/rollout/src/recorder.rs b/codex-rs/rollout/src/recorder.rs index 4f88ff1a21..ab1ae0a5bf 100644 --- a/codex-rs/rollout/src/recorder.rs +++ b/codex-rs/rollout/src/recorder.rs @@ -279,6 +279,10 @@ impl RolloutRecorder { let codex_home = config.codex_home(); let state_db_ctx = state_db::get_state_db(config).await; + // Search is the SQLite-optimized path and assumes a DB marked backfill-complete is + // actually populated enough to answer the query. If unmigrated rollout files still exist + // on disk, the repair path below may or may not run and catch them depending on whether + // SQLite already has another matching search hit. if search_term.is_some() && let Some(db_page) = state_db::list_threads_db( state_db_ctx.as_deref(), diff --git a/codex-rs/thread-store/Cargo.toml b/codex-rs/thread-store/Cargo.toml index a8f22fbfe5..328d9a4a7f 100644 --- a/codex-rs/thread-store/Cargo.toml +++ b/codex-rs/thread-store/Cargo.toml @@ -14,6 +14,16 @@ workspace = true [dependencies] async-trait = { workspace = true } chrono = { workspace = true, features = ["serde"] } +codex-git-utils = { workspace = true } codex-protocol = { workspace = true } +codex-rollout = { workspace = true } serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } thiserror = { workspace = true } + +[dev-dependencies] +codex-state = { workspace = true } +pretty_assertions = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true, features = ["macros"] } +uuid = { workspace = true } diff --git a/codex-rs/thread-store/src/lib.rs b/codex-rs/thread-store/src/lib.rs index 5525db1394..a64bf023be 100644 --- a/codex-rs/thread-store/src/lib.rs +++ b/codex-rs/thread-store/src/lib.rs @@ -5,12 +5,14 @@ //! any other backing store. mod error; +mod local; mod recorder; mod store; mod types; pub use error::ThreadStoreError; pub use error::ThreadStoreResult; +pub use local::LocalThreadStore; pub use recorder::ThreadRecorder; pub use store::ThreadStore; pub use types::AppendThreadItemsParams; diff --git a/codex-rs/thread-store/src/local/mod.rs b/codex-rs/thread-store/src/local/mod.rs new file mode 100644 index 0000000000..cc15033cd9 --- /dev/null +++ b/codex-rs/thread-store/src/local/mod.rs @@ -0,0 +1,565 @@ +use async_trait::async_trait; +use chrono::DateTime; +use chrono::Utc; +use codex_git_utils::GitSha; +use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::GitInfo; +use codex_protocol::protocol::SandboxPolicy; +use codex_protocol::protocol::SessionSource; +use codex_rollout::RolloutConfig; +use codex_rollout::RolloutRecorder; +use codex_rollout::ThreadItem; +use codex_rollout::parse_cursor; + +use crate::AppendThreadItemsParams; +use crate::ArchiveThreadParams; +use crate::CreateThreadParams; +use crate::ListThreadsParams; +use crate::LoadThreadHistoryParams; +use crate::ReadThreadParams; +use crate::ResumeThreadRecorderParams; +use crate::SetThreadNameParams; +use crate::StoredThread; +use crate::StoredThreadHistory; +use crate::ThreadPage; +use crate::ThreadRecorder; +use crate::ThreadSortKey; +use crate::ThreadStore; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; +use crate::UpdateThreadMetadataParams; + +/// Local filesystem/SQLite-backed implementation of [`ThreadStore`]. +#[derive(Clone, Debug)] +pub struct LocalThreadStore { + config: RolloutConfig, +} + +impl LocalThreadStore { + /// Create a local store from the rollout configuration used by existing local persistence. + pub fn new(config: RolloutConfig) -> Self { + Self { config } + } +} + +#[async_trait] +impl ThreadStore for LocalThreadStore { + async fn create_thread( + &self, + _params: CreateThreadParams, + ) -> ThreadStoreResult> { + unsupported("create_thread") + } + + async fn resume_thread_recorder( + &self, + _params: ResumeThreadRecorderParams, + ) -> ThreadStoreResult> { + unsupported("resume_thread_recorder") + } + + async fn append_items(&self, _params: AppendThreadItemsParams) -> ThreadStoreResult<()> { + unsupported("append_items") + } + + async fn load_history( + &self, + _params: LoadThreadHistoryParams, + ) -> ThreadStoreResult { + unsupported("load_history") + } + + async fn read_thread(&self, _params: ReadThreadParams) -> ThreadStoreResult { + unsupported("read_thread") + } + + async fn list_threads(&self, params: ListThreadsParams) -> ThreadStoreResult { + let cursor = params + .cursor + .as_deref() + .map(|cursor| { + parse_cursor(cursor).ok_or_else(|| ThreadStoreError::InvalidRequest { + message: format!("invalid cursor: {cursor}"), + }) + }) + .transpose()?; + let sort_key = match params.sort_key { + ThreadSortKey::CreatedAt => codex_rollout::ThreadSortKey::CreatedAt, + ThreadSortKey::UpdatedAt => codex_rollout::ThreadSortKey::UpdatedAt, + }; + let page = list_rollout_threads(&self.config, ¶ms, cursor.as_ref(), sort_key).await?; + + let next_cursor = page + .next_cursor + .as_ref() + .and_then(|cursor| serde_json::to_value(cursor).ok()) + .and_then(|value| value.as_str().map(str::to_owned)); + let items = page + .items + .into_iter() + .filter_map(|item| { + stored_thread_from_rollout_item( + item, + params.archived, + self.config.model_provider_id.as_str(), + ) + }) + .collect::>(); + + Ok(ThreadPage { items, next_cursor }) + } + + async fn set_thread_name(&self, _params: SetThreadNameParams) -> ThreadStoreResult<()> { + unsupported("set_thread_name") + } + + async fn update_thread_metadata( + &self, + _params: UpdateThreadMetadataParams, + ) -> ThreadStoreResult { + unsupported("update_thread_metadata") + } + + async fn archive_thread(&self, _params: ArchiveThreadParams) -> ThreadStoreResult<()> { + unsupported("archive_thread") + } + + async fn unarchive_thread( + &self, + _params: ArchiveThreadParams, + ) -> ThreadStoreResult { + unsupported("unarchive_thread") + } +} + +fn unsupported(operation: &str) -> ThreadStoreResult { + Err(ThreadStoreError::Internal { + message: format!("local thread store does not implement {operation} in this slice"), + }) +} + +async fn list_rollout_threads( + config: &RolloutConfig, + params: &ListThreadsParams, + cursor: Option<&codex_rollout::Cursor>, + sort_key: codex_rollout::ThreadSortKey, +) -> ThreadStoreResult { + let page = if params.archived { + RolloutRecorder::list_archived_threads( + config, + params.page_size, + cursor, + sort_key, + params.allowed_sources.as_slice(), + params.model_providers.as_deref(), + config.model_provider_id.as_str(), + params.search_term.as_deref(), + ) + .await + } else { + RolloutRecorder::list_threads( + config, + params.page_size, + cursor, + sort_key, + params.allowed_sources.as_slice(), + params.model_providers.as_deref(), + config.model_provider_id.as_str(), + params.search_term.as_deref(), + ) + .await + }; + page.map_err(|err| ThreadStoreError::Internal { + message: format!("failed to list threads: {err}"), + }) +} + +fn stored_thread_from_rollout_item( + item: ThreadItem, + archived: bool, + default_provider: &str, +) -> Option { + let thread_id = item + .thread_id + .or_else(|| thread_id_from_rollout_path(item.path.as_path()))?; + let created_at = parse_rfc3339(item.created_at.as_deref()).unwrap_or_else(Utc::now); + let updated_at = parse_rfc3339(item.updated_at.as_deref()).unwrap_or(created_at); + let archived_at = archived.then_some(updated_at); + let git_info = git_info_from_parts( + item.git_sha.clone(), + item.git_branch.clone(), + item.git_origin_url.clone(), + ); + let source = item.source.unwrap_or(SessionSource::Unknown); + let preview = item.first_user_message.clone().unwrap_or_default(); + + Some(StoredThread { + thread_id, + rollout_path: Some(item.path), + forked_from_id: None, + preview, + name: None, + model_provider: item + .model_provider + .filter(|provider| !provider.is_empty()) + .unwrap_or_else(|| default_provider.to_string()), + model: None, + reasoning_effort: None, + created_at, + updated_at, + archived_at, + cwd: item.cwd.unwrap_or_default(), + cli_version: item.cli_version.unwrap_or_default(), + source, + agent_nickname: item.agent_nickname, + agent_role: item.agent_role, + agent_path: None, + git_info, + approval_mode: AskForApproval::OnRequest, + sandbox_policy: SandboxPolicy::new_read_only_policy(), + token_usage: None, + first_user_message: item.first_user_message, + history: None, + }) +} + +fn parse_rfc3339(value: Option<&str>) -> Option> { + DateTime::parse_from_rfc3339(value?) + .ok() + .map(|dt| dt.with_timezone(&Utc)) +} + +fn git_info_from_parts( + sha: Option, + branch: Option, + origin_url: Option, +) -> Option { + if sha.is_none() && branch.is_none() && origin_url.is_none() { + return None; + } + Some(GitInfo { + commit_hash: sha.as_deref().map(GitSha::new), + branch, + repository_url: origin_url, + }) +} + +fn thread_id_from_rollout_path(path: &std::path::Path) -> Option { + let file_name = path.file_name()?.to_str()?; + let stem = file_name.strip_suffix(".jsonl")?; + if stem.len() < 37 { + return None; + } + let uuid_start = stem.len().saturating_sub(36); + if !stem[..uuid_start].ends_with('-') { + return None; + } + codex_protocol::ThreadId::from_string(&stem[uuid_start..]).ok() +} + +#[cfg(test)] +mod tests { + use std::fs; + use std::io::Write; + use std::path::Path; + use std::path::PathBuf; + + use codex_protocol::ThreadId; + use codex_protocol::protocol::SessionSource; + use codex_rollout::ARCHIVED_SESSIONS_SUBDIR; + use pretty_assertions::assert_eq; + use tempfile::TempDir; + use uuid::Uuid; + + use super::*; + + fn test_config(codex_home: &Path) -> RolloutConfig { + RolloutConfig { + codex_home: codex_home.to_path_buf(), + sqlite_home: codex_home.to_path_buf(), + cwd: codex_home.to_path_buf(), + model_provider_id: "test-provider".to_string(), + generate_memories: true, + } + } + + fn write_session_file(root: &Path, ts: &str, uuid: Uuid) -> std::io::Result { + write_session_file_with( + root, + root.join("sessions/2025/01/03"), + ts, + uuid, + "Hello from user", + Some("test-provider"), + ) + } + + fn write_archived_session_file(root: &Path, ts: &str, uuid: Uuid) -> std::io::Result { + write_session_file_with( + root, + root.join(ARCHIVED_SESSIONS_SUBDIR), + ts, + uuid, + "Archived user message", + Some("test-provider"), + ) + } + + fn write_session_file_with( + root: &Path, + day_dir: PathBuf, + ts: &str, + uuid: Uuid, + first_user_message: &str, + model_provider: Option<&str>, + ) -> std::io::Result { + fs::create_dir_all(&day_dir)?; + let path = day_dir.join(format!("rollout-{ts}-{uuid}.jsonl")); + let mut file = fs::File::create(&path)?; + let meta = serde_json::json!({ + "timestamp": ts, + "type": "session_meta", + "payload": { + "id": uuid, + "timestamp": ts, + "cwd": root, + "originator": "test_originator", + "cli_version": "test_version", + "source": "cli", + "model_provider": model_provider, + "git": { + "commit_hash": "abcdef", + "branch": "main", + "repository_url": "https://example.com/repo.git" + } + }, + }); + writeln!(file, "{meta}")?; + let user_event = serde_json::json!({ + "timestamp": ts, + "type": "event_msg", + "payload": { + "type": "user_message", + "message": first_user_message, + "kind": "plain", + }, + }); + writeln!(file, "{user_event}")?; + Ok(path) + } + + #[tokio::test] + async fn list_threads_uses_default_provider_when_rollout_omits_provider() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path())); + write_session_file_with( + home.path(), + home.path().join("sessions/2025/01/03"), + "2025-01-03T12-00-00", + Uuid::from_u128(102), + "Hello from user", + /*model_provider*/ None, + ) + .expect("session file"); + + let page = store + .list_threads(ListThreadsParams { + page_size: 10, + cursor: None, + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived: false, + search_term: None, + }) + .await + .expect("thread listing"); + + assert_eq!(page.items.len(), 1); + assert_eq!(page.items[0].model_provider, "test-provider"); + } + + #[tokio::test] + async fn list_threads_preserves_sqlite_title_search_results() { + let home = TempDir::new().expect("temp dir"); + let config = test_config(home.path()); + let store = LocalThreadStore::new(config.clone()); + let uuid = Uuid::from_u128(103); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + let rollout_path = home.path().join("rollout-title-search.jsonl"); + fs::write(&rollout_path, "").expect("placeholder rollout file"); + + let runtime = codex_state::StateRuntime::init( + home.path().to_path_buf(), + config.model_provider_id.clone(), + ) + .await + .expect("state db should initialize"); + runtime + .mark_backfill_complete(/*last_watermark*/ None) + .await + .expect("backfill should be complete"); + let created_at = Utc::now(); + let mut builder = codex_state::ThreadMetadataBuilder::new( + thread_id, + rollout_path, + created_at, + SessionSource::Cli, + ); + builder.model_provider = Some(config.model_provider_id.clone()); + builder.cwd = home.path().to_path_buf(); + builder.cli_version = Some("test_version".to_string()); + let mut metadata = builder.build(config.model_provider_id.as_str()); + metadata.title = "needle title".to_string(); + metadata.first_user_message = Some("plain preview".to_string()); + runtime + .upsert_thread(&metadata) + .await + .expect("state db upsert should succeed"); + + let page = store + .list_threads(ListThreadsParams { + page_size: 10, + cursor: None, + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived: false, + search_term: Some("needle".to_string()), + }) + .await + .expect("thread listing"); + + let ids = page + .items + .iter() + .map(|item| item.thread_id) + .collect::>(); + assert_eq!(ids, vec![thread_id]); + assert_eq!( + page.items[0].first_user_message.as_deref(), + Some("plain preview") + ); + } + + #[tokio::test] + async fn list_threads_selects_active_or_archived_collection() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path())); + let active_uuid = Uuid::from_u128(105); + let archived_uuid = Uuid::from_u128(106); + write_session_file(home.path(), "2025-01-03T12-00-00", active_uuid) + .expect("active session file"); + write_archived_session_file(home.path(), "2025-01-03T13-00-00", archived_uuid) + .expect("archived session file"); + + let active = store + .list_threads(ListThreadsParams { + page_size: 10, + cursor: None, + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived: false, + search_term: None, + }) + .await + .expect("active listing"); + let archived = store + .list_threads(ListThreadsParams { + page_size: 10, + cursor: None, + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived: true, + search_term: None, + }) + .await + .expect("archived listing"); + + let active_id = ThreadId::from_string(&active_uuid.to_string()).expect("valid thread id"); + let archived_id = + ThreadId::from_string(&archived_uuid.to_string()).expect("valid thread id"); + assert_eq!( + active + .items + .iter() + .map(|item| item.thread_id) + .collect::>(), + vec![active_id] + ); + assert_eq!( + archived + .items + .iter() + .map(|item| item.thread_id) + .collect::>(), + vec![archived_id] + ); + assert_eq!(active.items[0].archived_at, None); + assert_eq!( + archived.items[0].archived_at, + Some(archived.items[0].updated_at) + ); + } + + #[tokio::test] + async fn list_threads_returns_local_rollout_summary() { + let home = TempDir::new().expect("temp dir"); + let config = test_config(home.path()); + let store = LocalThreadStore::new(config); + let uuid = Uuid::from_u128(101); + let path = + write_session_file(home.path(), "2025-01-03T12-00-00", uuid).expect("session file"); + + let page = store + .list_threads(ListThreadsParams { + page_size: 10, + cursor: None, + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: vec![SessionSource::Cli], + model_providers: Some(vec!["test-provider".to_string()]), + archived: false, + search_term: None, + }) + .await + .expect("thread listing"); + + let thread_id = + codex_protocol::ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + assert_eq!(page.next_cursor, None); + assert_eq!(page.items.len(), 1); + assert_eq!(page.items[0].thread_id, thread_id); + assert_eq!(page.items[0].rollout_path, Some(path)); + assert_eq!(page.items[0].preview, "Hello from user"); + assert_eq!( + page.items[0].first_user_message.as_deref(), + Some("Hello from user") + ); + assert_eq!(page.items[0].model_provider, "test-provider"); + assert_eq!(page.items[0].cli_version, "test_version"); + assert_eq!(page.items[0].source, SessionSource::Cli); + } + + #[tokio::test] + async fn list_threads_rejects_invalid_cursor() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path())); + + let err = store + .list_threads(ListThreadsParams { + page_size: 10, + cursor: Some("not-a-cursor".to_string()), + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived: false, + search_term: None, + }) + .await + .expect_err("invalid cursor should fail"); + + assert!(matches!(err, ThreadStoreError::InvalidRequest { .. })); + } +} diff --git a/codex-rs/thread-store/src/types.rs b/codex-rs/thread-store/src/types.rs index 73f983a466..8d489ea6cb 100644 --- a/codex-rs/thread-store/src/types.rs +++ b/codex-rs/thread-store/src/types.rs @@ -135,6 +135,8 @@ pub struct ThreadPage { pub struct StoredThread { /// Thread id. pub thread_id: ThreadId, + /// Local rollout path when the backing store is filesystem-based. + pub rollout_path: Option, /// Source thread id when this thread was forked from another thread. pub forked_from_id: Option, /// Best available user-facing preview, usually the first user message. @@ -155,6 +157,8 @@ pub struct StoredThread { pub archived_at: Option>, /// Working directory captured for the thread. pub cwd: PathBuf, + /// CLI version captured for the thread. + pub cli_version: String, /// Runtime source for the thread. pub source: SessionSource, /// Optional random nickname for thread-spawn sub-agents. From f53210d332f9f8a8433009bea9d1122f1e86a57f Mon Sep 17 00:00:00 2001 From: Shijie Rao Date: Wed, 15 Apr 2026 14:39:06 -0400 Subject: [PATCH 27/84] Add CLI update announcement (#17942) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Keep the existing local-build test announcement as the first announcement entry - Add the CLI update reminder for versions below `0.120.0` - Remove expired onboarding and gpt-5.3-codex announcement entries Screenshot 2026-04-15 at 1 32 53โ€ฏPM --- announcement_tip.toml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/announcement_tip.toml b/announcement_tip.toml index 2473258368..79c0daea76 100644 --- a/announcement_tip.toml +++ b/announcement_tip.toml @@ -4,20 +4,14 @@ # version_regex matches against the CLI version (env!("CARGO_PKG_VERSION")); omit to apply to all versions. # target_app specify which app should display the announcement (cli, vsce, ...). -[[announcements]] -content = "Welcome to Codex! Check out the new onboarding flow." -from_date = "2024-10-01" -to_date = "2024-10-15" -target_app = "cli" - -# Test announcement only for local build version until 2026-01-10 excluded (past) +# Test announcement only for local build version until 2027-05-10 excluded [[announcements]] content = "This is a test announcement" version_regex = "^0\\.0\\.0$" -to_date = "2026-05-10" +to_date = "2027-05-10" [[announcements]] -content = "**BREAKING NEWS**: `gpt-5.3-codex` is out! Upgrade to `0.98.0` for a faster, smarter, more steerable agent." -from_date = "2026-02-01" -to_date = "2026-02-16" -version_regex = "^0\\.(?:[0-9]|[1-8][0-9]|9[0-7])\\." +content = "Update Required - This version will no longer be supported starting May 8th. Please upgrade to the latest version (https://github.com/openai/codex/releases/latest) using your preferred package manager." +# Matches 0.x.y versions from 0.0.y through 0.119.y; excludes 0.120.0 and newer. +version_regex = "^0\\.(?:[0-9]|[1-9][0-9]|1[01][0-9])\\." +to_date = "2026-05-08" From f5e8eac2ae15114c0a6309ae0114dd109b0a5d58 Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Wed, 15 Apr 2026 11:52:51 -0700 Subject: [PATCH 28/84] Refactor auth providers to mutate request headers (#17866) ## Summary - Move auth header construction into the `AuthProvider::add_auth_headers` contract. - Inline `CoreAuthProvider` header mutation in its provider impl and remove the shared header-map helper. - Update HTTP, websocket, file upload, sideband websocket, and test auth callsites to use the provider method. - Add direct coverage for `CoreAuthProvider` auth header mutation. ## Testing - `just fmt` - `cargo test -p codex-api` - `cargo test -p codex-core client::tests::auth_request_telemetry_context_tracks_attached_auth_and_retry_phase` - `cargo test -p codex-core` failed on unrelated/reproducible `tools::handlers::multi_agents::tests::multi_agent_v2_followup_task_interrupts_busy_child_without_losing_message` --------- Co-authored-by: Celia Chen --- codex-rs/codex-api/src/api_bridge.rs | 18 ++++++++----- codex-rs/codex-api/src/api_bridge_tests.rs | 21 +++++++++++++++ codex-rs/codex-api/src/auth.rs | 27 ++----------------- codex-rs/codex-api/src/endpoint/compact.rs | 4 +-- codex-rs/codex-api/src/endpoint/memories.rs | 4 +-- codex-rs/codex-api/src/endpoint/models.rs | 4 +-- .../codex-api/src/endpoint/realtime_call.rs | 7 +++-- .../src/endpoint/responses_websocket.rs | 3 +-- codex-rs/codex-api/src/endpoint/session.rs | 4 +-- codex-rs/codex-api/src/files.rs | 15 +++++------ codex-rs/codex-api/tests/clients.rs | 18 ++++++------- .../codex-api/tests/models_integration.rs | 4 +-- codex-rs/codex-api/tests/sse_end_to_end.rs | 4 +-- codex-rs/core/src/client.rs | 13 ++------- 14 files changed, 65 insertions(+), 81 deletions(-) diff --git a/codex-rs/codex-api/src/api_bridge.rs b/codex-rs/codex-api/src/api_bridge.rs index 0ad2b13979..7c36c67fdd 100644 --- a/codex-rs/codex-api/src/api_bridge.rs +++ b/codex-rs/codex-api/src/api_bridge.rs @@ -12,6 +12,7 @@ use codex_protocol::error::RetryLimitReachedError; use codex_protocol::error::UnexpectedResponseError; use codex_protocol::error::UsageLimitReachedError; use http::HeaderMap; +use http::HeaderValue; use serde::Deserialize; use serde_json::Value; @@ -200,11 +201,16 @@ impl CoreAuthProvider { } impl ApiAuthProvider for CoreAuthProvider { - fn bearer_token(&self) -> Option { - self.token.clone() - } - - fn account_id(&self) -> Option { - self.account_id.clone() + fn add_auth_headers(&self, headers: &mut HeaderMap) { + if let Some(token) = self.token.as_ref() + && let Ok(header) = HeaderValue::from_str(&format!("Bearer {token}")) + { + let _ = headers.insert(http::header::AUTHORIZATION, header); + } + if let Some(account_id) = self.account_id.as_ref() + && let Ok(header) = HeaderValue::from_str(account_id) + { + let _ = headers.insert("ChatGPT-Account-ID", header); + } } } diff --git a/codex-rs/codex-api/src/api_bridge_tests.rs b/codex-rs/codex-api/src/api_bridge_tests.rs index 71d3889915..50247c1314 100644 --- a/codex-rs/codex-api/src/api_bridge_tests.rs +++ b/codex-rs/codex-api/src/api_bridge_tests.rs @@ -141,3 +141,24 @@ fn core_auth_provider_reports_when_auth_header_will_attach() { assert!(auth.auth_header_attached()); assert_eq!(auth.auth_header_name(), Some("authorization")); } + +#[test] +fn core_auth_provider_adds_auth_headers() { + let auth = CoreAuthProvider::for_test(Some("access-token"), Some("workspace-123")); + let mut headers = HeaderMap::new(); + + crate::AuthProvider::add_auth_headers(&auth, &mut headers); + + assert_eq!( + headers + .get(http::header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()), + Some("Bearer access-token") + ); + assert_eq!( + headers + .get("ChatGPT-Account-ID") + .and_then(|value| value.to_str().ok()), + Some("workspace-123") + ); +} diff --git a/codex-rs/codex-api/src/auth.rs b/codex-rs/codex-api/src/auth.rs index f649062db1..a7b1e69d14 100644 --- a/codex-rs/codex-api/src/auth.rs +++ b/codex-rs/codex-api/src/auth.rs @@ -1,33 +1,10 @@ -use codex_client::Request; use http::HeaderMap; -use http::HeaderValue; -/// Provides bearer and account identity information for API requests. +/// Adds authentication headers to API requests. /// /// Implementations should be cheap and non-blocking; any asynchronous /// refresh or I/O should be handled by higher layers before requests /// reach this interface. pub trait AuthProvider: Send + Sync { - fn bearer_token(&self) -> Option; - fn account_id(&self) -> Option { - None - } -} - -pub(crate) fn add_auth_headers_to_header_map(auth: &A, headers: &mut HeaderMap) { - if let Some(token) = auth.bearer_token() - && let Ok(header) = HeaderValue::from_str(&format!("Bearer {token}")) - { - let _ = headers.insert(http::header::AUTHORIZATION, header); - } - if let Some(account_id) = auth.account_id() - && let Ok(header) = HeaderValue::from_str(&account_id) - { - let _ = headers.insert("ChatGPT-Account-ID", header); - } -} - -pub(crate) fn add_auth_headers(auth: &A, mut req: Request) -> Request { - add_auth_headers_to_header_map(auth, &mut req.headers); - req + fn add_auth_headers(&self, headers: &mut HeaderMap); } diff --git a/codex-rs/codex-api/src/endpoint/compact.rs b/codex-rs/codex-api/src/endpoint/compact.rs index 44a56a11a7..748ac35558 100644 --- a/codex-rs/codex-api/src/endpoint/compact.rs +++ b/codex-rs/codex-api/src/endpoint/compact.rs @@ -90,9 +90,7 @@ mod tests { struct DummyAuth; impl AuthProvider for DummyAuth { - fn bearer_token(&self) -> Option { - None - } + fn add_auth_headers(&self, _headers: &mut HeaderMap) {} } #[test] diff --git a/codex-rs/codex-api/src/endpoint/memories.rs b/codex-rs/codex-api/src/endpoint/memories.rs index 5cb2a65b1f..3047c859db 100644 --- a/codex-rs/codex-api/src/endpoint/memories.rs +++ b/codex-rs/codex-api/src/endpoint/memories.rs @@ -103,9 +103,7 @@ mod tests { struct DummyAuth; impl AuthProvider for DummyAuth { - fn bearer_token(&self) -> Option { - None - } + fn add_auth_headers(&self, _headers: &mut HeaderMap) {} } #[derive(Clone)] diff --git a/codex-rs/codex-api/src/endpoint/models.rs b/codex-rs/codex-api/src/endpoint/models.rs index 97781ac419..17342d6f99 100644 --- a/codex-rs/codex-api/src/endpoint/models.rs +++ b/codex-rs/codex-api/src/endpoint/models.rs @@ -132,9 +132,7 @@ mod tests { struct DummyAuth; impl AuthProvider for DummyAuth { - fn bearer_token(&self) -> Option { - None - } + fn add_auth_headers(&self, _headers: &mut HeaderMap) {} } fn provider(base_url: &str) -> Provider { diff --git a/codex-rs/codex-api/src/endpoint/realtime_call.rs b/codex-rs/codex-api/src/endpoint/realtime_call.rs index 4d0bcfa7bf..a9a8b963cf 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_call.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_call.rs @@ -284,8 +284,11 @@ mod tests { struct DummyAuth; impl AuthProvider for DummyAuth { - fn bearer_token(&self) -> Option { - Some("test-token".to_string()) + fn add_auth_headers(&self, headers: &mut HeaderMap) { + headers.insert( + http::header::AUTHORIZATION, + HeaderValue::from_static("Bearer test-token"), + ); } } diff --git a/codex-rs/codex-api/src/endpoint/responses_websocket.rs b/codex-rs/codex-api/src/endpoint/responses_websocket.rs index a60d188bda..d2b775cdd5 100644 --- a/codex-rs/codex-api/src/endpoint/responses_websocket.rs +++ b/codex-rs/codex-api/src/endpoint/responses_websocket.rs @@ -1,5 +1,4 @@ use crate::auth::AuthProvider; -use crate::auth::add_auth_headers_to_header_map; use crate::common::ResponseEvent; use crate::common::ResponseStream; use crate::common::ResponsesWsRequest; @@ -310,7 +309,7 @@ impl ResponsesWebsocketClient { let mut headers = merge_request_headers(&self.provider.headers, extra_headers, default_headers); - add_auth_headers_to_header_map(&self.auth, &mut headers); + self.auth.add_auth_headers(&mut headers); let (stream, server_reasoning_included, models_etag, server_model) = connect_websocket(ws_url, headers, turn_state.clone()).await?; diff --git a/codex-rs/codex-api/src/endpoint/session.rs b/codex-rs/codex-api/src/endpoint/session.rs index 00919a0c54..e4a470ceee 100644 --- a/codex-rs/codex-api/src/endpoint/session.rs +++ b/codex-rs/codex-api/src/endpoint/session.rs @@ -1,5 +1,4 @@ use crate::auth::AuthProvider; -use crate::auth::add_auth_headers; use crate::error::ApiError; use crate::provider::Provider; use crate::telemetry::run_with_request_telemetry; @@ -56,7 +55,8 @@ impl EndpointSession { if let Some(body) = body { req.body = Some(RequestBody::Json(body.clone())); } - add_auth_headers(&self.auth, req) + self.auth.add_auth_headers(&mut req.headers); + req } pub(crate) async fn execute( diff --git a/codex-rs/codex-api/src/files.rs b/codex-rs/codex-api/src/files.rs index 6fad5b62f5..ebe35af5ba 100644 --- a/codex-rs/codex-api/src/files.rs +++ b/codex-rs/codex-api/src/files.rs @@ -256,17 +256,14 @@ fn authorized_request( method: reqwest::Method, url: &str, ) -> reqwest::RequestBuilder { + let mut headers = http::HeaderMap::new(); + auth.add_auth_headers(&mut headers); + let client = build_reqwest_client(); - let mut request = client + client .request(method, url) - .timeout(OPENAI_FILE_REQUEST_TIMEOUT); - if let Some(token) = auth.bearer_token() { - request = request.bearer_auth(token); - } - if let Some(account_id) = auth.account_id() { - request = request.header("chatgpt-account-id", account_id); - } - request + .timeout(OPENAI_FILE_REQUEST_TIMEOUT) + .headers(headers) } fn build_reqwest_client() -> reqwest::Client { diff --git a/codex-rs/codex-api/tests/clients.rs b/codex-rs/codex-api/tests/clients.rs index b11c6f9d6b..d82fcc14ce 100644 --- a/codex-rs/codex-api/tests/clients.rs +++ b/codex-rs/codex-api/tests/clients.rs @@ -91,9 +91,7 @@ impl HttpTransport for RecordingTransport { struct NoAuth; impl AuthProvider for NoAuth { - fn bearer_token(&self) -> Option { - None - } + fn add_auth_headers(&self, _headers: &mut HeaderMap) {} } #[derive(Clone)] @@ -112,12 +110,14 @@ impl StaticAuth { } impl AuthProvider for StaticAuth { - fn bearer_token(&self) -> Option { - Some(self.token.clone()) - } - - fn account_id(&self) -> Option { - Some(self.account_id.clone()) + fn add_auth_headers(&self, headers: &mut HeaderMap) { + let token = &self.token; + if let Ok(header) = HeaderValue::from_str(&format!("Bearer {token}")) { + headers.insert(http::header::AUTHORIZATION, header); + } + if let Ok(header) = HeaderValue::from_str(&self.account_id) { + headers.insert("ChatGPT-Account-ID", header); + } } } diff --git a/codex-rs/codex-api/tests/models_integration.rs b/codex-rs/codex-api/tests/models_integration.rs index 91a8477a92..fab135e0ab 100644 --- a/codex-rs/codex-api/tests/models_integration.rs +++ b/codex-rs/codex-api/tests/models_integration.rs @@ -24,9 +24,7 @@ use wiremock::matchers::path; struct DummyAuth; impl AuthProvider for DummyAuth { - fn bearer_token(&self) -> Option { - None - } + fn add_auth_headers(&self, _headers: &mut HeaderMap) {} } fn provider(base_url: &str) -> Provider { diff --git a/codex-rs/codex-api/tests/sse_end_to_end.rs b/codex-rs/codex-api/tests/sse_end_to_end.rs index b15de296a8..4d32e82242 100644 --- a/codex-rs/codex-api/tests/sse_end_to_end.rs +++ b/codex-rs/codex-api/tests/sse_end_to_end.rs @@ -53,9 +53,7 @@ impl HttpTransport for FixtureSseTransport { struct NoAuth; impl AuthProvider for NoAuth { - fn bearer_token(&self) -> Option { - None - } + fn add_auth_headers(&self, _headers: &mut HeaderMap) {} } fn provider(name: &str) -> Provider { diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 16f743943a..bd83c81f0d 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -32,6 +32,7 @@ use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; use codex_api::ApiError; +use codex_api::AuthProvider; use codex_api::CompactClient as ApiCompactClient; use codex_api::CompactionInput as ApiCompactionInput; use codex_api::Compression; @@ -83,7 +84,6 @@ use futures::StreamExt; use http::HeaderMap as ApiHeaderMap; use http::HeaderValue; use http::StatusCode as HttpStatusCode; -use http::header::AUTHORIZATION; use reqwest::StatusCode; use std::time::Duration; use std::time::Instant; @@ -277,16 +277,7 @@ pub(crate) struct RealtimeWebrtcCallStart { /// `api.openai.com` sideband path. fn sideband_websocket_auth_headers(api_auth: &CoreAuthProvider) -> ApiHeaderMap { let mut headers = ApiHeaderMap::new(); - if let Some(token) = api_auth.token.as_ref() - && let Ok(value) = HeaderValue::from_str(&format!("Bearer {token}")) - { - headers.insert(AUTHORIZATION, value); - } - if let Some(account_id) = api_auth.account_id.as_ref() - && let Ok(value) = HeaderValue::from_str(account_id) - { - headers.insert("ChatGPT-Account-ID", value); - } + api_auth.add_auth_headers(&mut headers); headers } From 83abf67d20e3294ae9e8be3bc99e92b89f2c9844 Mon Sep 17 00:00:00 2001 From: Ruslan Nigmatullin Date: Wed, 15 Apr 2026 11:52:53 -0700 Subject: [PATCH 29/84] app-server: track remote-control seq IDs per stream (#17902) ## Summary - Track outbound remote-control sequence IDs independently for each client stream. - Retain unacked outbound messages per stream using FIFO buffers. - Require stream-scoped acks and update tests for contiguous per-stream sequencing. ## Why The remote-control peer uses outbound sequence gaps to detect lost messages and re-initialize. A single global outbound sequence counter can create apparent gaps on an individual stream when another stream receives an interleaved message. ## Validation - `just fmt` - `cargo test -p codex-app-server remote_control` - `just fix -p codex-app-server` - `git diff --check` --- .../src/transport/remote_control/protocol.rs | 7 +- .../src/transport/remote_control/tests.rs | 36 ++- .../src/transport/remote_control/websocket.rs | 271 ++++++++++++------ 3 files changed, 216 insertions(+), 98 deletions(-) diff --git a/codex-rs/app-server/src/transport/remote_control/protocol.rs b/codex-rs/app-server/src/transport/remote_control/protocol.rs index 857855f2a0..dcb79e5c35 100644 --- a/codex-rs/app-server/src/transport/remote_control/protocol.rs +++ b/codex-rs/app-server/src/transport/remote_control/protocol.rs @@ -48,9 +48,8 @@ pub enum ClientEvent { message: JSONRPCMessage, }, /// Backend-generated acknowledgement for all server envelopes addressed to - /// `client_id` whose envelope `seq_id` is less than or equal to this ack's - /// `seq_id`. This cursor is client-scoped, not stream-scoped, so receivers - /// must not use `stream_id` to partition acks. + /// `client_id` and `stream_id` whose envelope `seq_id` is less than or equal + /// to this ack's `seq_id`. This cursor is stream-scoped. Ack, Ping, ClientClosed, @@ -65,7 +64,7 @@ pub(crate) struct ClientEnvelope { pub(crate) client_id: ClientId, #[serde(rename = "stream_id", skip_serializing_if = "Option::is_none")] pub(crate) stream_id: Option, - /// For `Ack`, this is the backend-generated per-client cursor over + /// For `Ack`, this is the backend-generated per-stream cursor over /// `ServerEnvelope.seq_id`. #[serde(rename = "seq_id", skip_serializing_if = "Option::is_none")] pub(crate) seq_id: Option, diff --git a/codex-rs/app-server/src/transport/remote_control/tests.rs b/codex-rs/app-server/src/transport/remote_control/tests.rs index 6dd6978fa5..2dd687a3d1 100644 --- a/codex-rs/app-server/src/transport/remote_control/tests.rs +++ b/codex-rs/app-server/src/transport/remote_control/tests.rs @@ -5,6 +5,7 @@ use super::enroll::update_persisted_remote_control_enrollment; use super::protocol::ClientEnvelope; use super::protocol::ClientEvent; use super::protocol::ClientId; +use super::protocol::StreamId; use super::protocol::normalize_remote_control_url; use super::websocket::REMOTE_CONTROL_PROTOCOL_VERSION; use super::*; @@ -163,7 +164,7 @@ async fn remote_control_transport_manages_virtual_clients_and_routes_messages() json!({ "type": "pong", "client_id": "client-1", - "seq_id": 0, + "seq_id": 1, "status": "unknown", }) ); @@ -369,7 +370,7 @@ async fn remote_control_transport_manages_virtual_clients_and_routes_messages() json!({ "type": "pong", "client_id": "client-1", - "seq_id": 3, + "seq_id": 1, "status": "unknown", }) ); @@ -618,12 +619,13 @@ async fn remote_control_transport_clears_outgoing_buffer_when_backend_acks() { )) .await .expect("remote writer should accept outgoing message"); + let (server_event, stream_id) = read_server_event_with_stream_id(&mut first_websocket).await; assert_eq!( - read_server_event(&mut first_websocket).await, + server_event, json!({ "type": "server_message", "client_id": "client-1", - "seq_id": 0, + "seq_id": 1, "message": { "method": "configWarning", "params": { @@ -639,8 +641,8 @@ async fn remote_control_transport_clears_outgoing_buffer_when_backend_acks() { ClientEnvelope { event: ClientEvent::Ack, client_id: client_id.clone(), - stream_id: None, - seq_id: Some(0), + stream_id: Some(stream_id), + seq_id: Some(1), cursor: None, }, ) @@ -854,7 +856,7 @@ async fn remote_control_http_mode_enrolls_before_connecting() { json!({ "type": "server_message", "client_id": backend_client_id.0.clone(), - "seq_id": 0, + "seq_id": 1, "message": { "id": 11, "result": { @@ -882,7 +884,7 @@ async fn remote_control_http_mode_enrolls_before_connecting() { json!({ "type": "server_message", "client_id": backend_client_id.0.clone(), - "seq_id": 1, + "seq_id": 2, "message": { "method": "configWarning", "params": { @@ -1356,6 +1358,12 @@ async fn send_client_event( } async fn read_server_event(websocket: &mut WebSocketStream) -> serde_json::Value { + read_server_event_with_stream_id(websocket).await.0 +} + +async fn read_server_event_with_stream_id( + websocket: &mut WebSocketStream, +) -> (serde_json::Value, StreamId) { loop { let frame = timeout(Duration::from_secs(5), websocket.next()) .await @@ -1366,13 +1374,15 @@ async fn read_server_event(websocket: &mut WebSocketStream) -> serde_ tungstenite::Message::Text(text) => { let mut event: serde_json::Value = serde_json::from_str(text.as_ref()).expect("server event should deserialize"); - if let Some(stream_id) = event + let stream_id = event .as_object_mut() .and_then(|event| event.remove("stream_id")) - { - assert!(stream_id.is_string(), "stream_id should be a string"); - } - return event; + .expect("stream_id should be present"); + let stream_id = stream_id + .as_str() + .expect("stream_id should be a string") + .to_string(); + return (event, StreamId(stream_id)); } tungstenite::Message::Ping(payload) => { websocket diff --git a/codex-rs/app-server/src/transport/remote_control/websocket.rs b/codex-rs/app-server/src/transport/remote_control/websocket.rs index d168ec0e18..82e5f35b33 100644 --- a/codex-rs/app-server/src/transport/remote_control/websocket.rs +++ b/codex-rs/app-server/src/transport/remote_control/websocket.rs @@ -14,6 +14,7 @@ use super::protocol::ClientEvent; use super::protocol::ClientId; use super::protocol::RemoteControlTarget; use super::protocol::ServerEnvelope; +use super::protocol::StreamId; use axum::http::HeaderValue; use base64::Engine; use codex_core::util::backoff; @@ -25,8 +26,8 @@ use futures::SinkExt; use futures::StreamExt; use futures::stream::SplitSink; use futures::stream::SplitStream; -use std::collections::BTreeMap; use std::collections::HashMap; +use std::collections::VecDeque; use std::io; use std::io::ErrorKind; use std::sync::Arc; @@ -57,10 +58,7 @@ const REMOTE_CONTROL_ACCOUNT_ID_RETRY_INTERVAL: std::time::Duration = std::time::Duration::from_secs(1); struct BoundedOutboundBuffer { - // Remote-control acks are generated by the backend at client scope, so - // retransmit retention is keyed by client_id only. stream_id stays on each - // envelope for routing, but it is not part of the ack cursor. - buffer_by_client: HashMap>, + buffer_by_stream: HashMap<(ClientId, StreamId), VecDeque>, used_tx: watch::Sender, } @@ -68,46 +66,50 @@ impl BoundedOutboundBuffer { fn new() -> (Self, watch::Receiver) { let (used_tx, used_rx) = watch::channel(0); let buffer = Self { - buffer_by_client: HashMap::new(), + buffer_by_stream: HashMap::new(), used_tx, }; (buffer, used_rx) } fn insert(&mut self, server_envelope: &ServerEnvelope) { - self.buffer_by_client - .entry(server_envelope.client_id.clone()) + self.buffer_by_stream + .entry(( + server_envelope.client_id.clone(), + server_envelope.stream_id.clone(), + )) .or_default() - .insert(server_envelope.seq_id, server_envelope.clone()); + .push_back(server_envelope.clone()); self.used_tx.send_modify(|used| *used += 1); } - fn ack(&mut self, client_id: &ClientId, acked_seq_id: u64) { - let Some(buffer) = self.buffer_by_client.get_mut(client_id) else { + fn ack(&mut self, client_id: &ClientId, stream_id: &StreamId, acked_seq_id: u64) { + let key = (client_id.clone(), stream_id.clone()); + let Some(buffer) = self.buffer_by_stream.get_mut(&key) else { return; }; - while let Some(seq_id) = buffer.first_key_value().map(|(seq_id, _)| seq_id) - && *seq_id <= acked_seq_id + while let Some(server_envelope) = buffer.front() + && server_envelope.seq_id <= acked_seq_id { - buffer.pop_first(); + buffer.pop_front(); self.used_tx.send_modify(|used| *used -= 1); } if buffer.is_empty() { - self.buffer_by_client.remove(client_id); + self.buffer_by_stream.remove(&key); } } fn server_envelopes(&self) -> impl Iterator { - self.buffer_by_client + self.buffer_by_stream .values() - .flat_map(|buffer| buffer.values()) + .flat_map(|buffer| buffer.iter()) } } struct WebsocketState { outbound_buffer: BoundedOutboundBuffer, subscribe_cursor: Option, - next_seq_id: u64, + next_seq_id_by_stream: HashMap<(ClientId, StreamId), u64>, } pub(crate) struct RemoteControlWebsocket { @@ -162,7 +164,7 @@ impl RemoteControlWebsocket { state: Arc::new(Mutex::new(WebsocketState { outbound_buffer, subscribe_cursor: None, - next_seq_id: 0, + next_seq_id_by_stream: HashMap::new(), })), server_event_rx: Arc::new(Mutex::new(server_event_rx)), used_rx, @@ -445,10 +447,16 @@ impl RemoteControlWebsocket { } } }; - let (server_envelope, write_complete_tx) = { + let (payload, write_complete_tx) = { let mut state = state.lock().await; - let seq_id = state.next_seq_id; - state.next_seq_id = state.next_seq_id.saturating_add(1); + let seq_key = ( + queued_server_envelope.client_id.clone(), + queued_server_envelope.stream_id.clone(), + ); + let seq_id = *state + .next_seq_id_by_stream + .entry(seq_key.clone()) + .or_insert(1); let server_envelope = ServerEnvelope { event: queued_server_envelope.event, @@ -456,17 +464,19 @@ impl RemoteControlWebsocket { seq_id, stream_id: queued_server_envelope.stream_id, }; + let payload = match serde_json::to_string(&server_envelope) { + Ok(payload) => payload, + Err(err) => { + error!("failed to serialize remote-control server event: {err}"); + continue; + } + }; + state + .next_seq_id_by_stream + .insert(seq_key, seq_id.saturating_add(1)); state.outbound_buffer.insert(&server_envelope); - (server_envelope, queued_server_envelope.write_complete_tx) - }; - - let payload = match serde_json::to_string(&server_envelope) { - Ok(payload) => payload, - Err(err) => { - error!("failed to serialize remote-control server event: {err}"); - continue; - } + (payload, queued_server_envelope.write_complete_tx) }; tokio::select! { @@ -590,10 +600,13 @@ impl RemoteControlWebsocket { } if let ClientEvent::Ack = &client_envelope.event && let Some(acked_seq_id) = client_envelope.seq_id + && let Some(stream_id) = client_envelope.stream_id.as_ref() { - websocket_state - .outbound_buffer - .ack(&client_envelope.client_id, acked_seq_id); + websocket_state.outbound_buffer.ack( + &client_envelope.client_id, + stream_id, + acked_seq_id, + ); } drop(websocket_state); @@ -1220,7 +1233,7 @@ mod tests { let state = Arc::new(Mutex::new(WebsocketState { outbound_buffer, subscribe_cursor: None, - next_seq_id: 0, + next_seq_id_by_stream: HashMap::new(), })); let (_server_event_tx, server_event_rx) = mpsc::channel(super::super::CHANNEL_CAPACITY); let server_event_rx = Arc::new(Mutex::new(server_event_rx)); @@ -1248,6 +1261,83 @@ mod tests { .expect("writer should stop cleanly"); } + #[tokio::test] + async fn run_server_writer_inner_assigns_contiguous_seq_ids_per_stream() { + let (client_stream, mut server_stream) = connected_websocket_pair().await; + let (websocket_writer, _websocket_reader) = client_stream.split(); + let (outbound_buffer, used_rx) = BoundedOutboundBuffer::new(); + let state = Arc::new(Mutex::new(WebsocketState { + outbound_buffer, + subscribe_cursor: None, + next_seq_id_by_stream: HashMap::new(), + })); + let (server_event_tx, server_event_rx) = mpsc::channel(super::super::CHANNEL_CAPACITY); + let server_event_rx = Arc::new(Mutex::new(server_event_rx)); + let shutdown_token = CancellationToken::new(); + let writer_task = tokio::spawn(RemoteControlWebsocket::run_server_writer_inner( + state, + server_event_rx, + used_rx, + websocket_writer, + Duration::from_secs(60), + shutdown_token.clone(), + )); + + let client_id = ClientId("client-1".to_string()); + let first_stream = StreamId("stream-1".to_string()); + let second_stream = StreamId("stream-2".to_string()); + for stream_id in [&first_stream, &second_stream, &first_stream] { + server_event_tx + .send(super::super::QueuedServerEnvelope { + event: ServerEvent::Pong { + status: crate::transport::remote_control::protocol::PongStatus::Active, + }, + client_id: client_id.clone(), + stream_id: stream_id.clone(), + write_complete_tx: None, + }) + .await + .expect("server event should queue"); + } + + assert_eq!( + read_server_text_event(&mut server_stream).await, + serde_json::json!({ + "type": "pong", + "client_id": "client-1", + "stream_id": "stream-1", + "seq_id": 1, + "status": "active", + }) + ); + assert_eq!( + read_server_text_event(&mut server_stream).await, + serde_json::json!({ + "type": "pong", + "client_id": "client-1", + "stream_id": "stream-2", + "seq_id": 1, + "status": "active", + }) + ); + assert_eq!( + read_server_text_event(&mut server_stream).await, + serde_json::json!({ + "type": "pong", + "client_id": "client-1", + "stream_id": "stream-1", + "seq_id": 2, + "status": "active", + }) + ); + + shutdown_token.cancel(); + writer_task + .await + .expect("writer task should join") + .expect("writer should stop cleanly"); + } + #[tokio::test] async fn run_websocket_reader_inner_times_out_without_pong_frames() { let (client_stream, _server_stream) = connected_websocket_pair().await; @@ -1256,7 +1346,7 @@ mod tests { let state = Arc::new(Mutex::new(WebsocketState { outbound_buffer, subscribe_cursor: None, - next_seq_id: 0, + next_seq_id_by_stream: HashMap::new(), })); let (server_event_tx, _server_event_rx) = mpsc::channel(super::super::CHANNEL_CAPACITY); let (transport_event_tx, _transport_event_rx) = @@ -1287,70 +1377,32 @@ mod tests { } #[test] - fn outbound_buffer_acks_by_client_id_across_stream_ids() { + fn outbound_buffer_acks_by_stream_id() { let (mut outbound_buffer, used_rx) = BoundedOutboundBuffer::new(); let client_1 = ClientId("client-1".to_string()); let client_2 = ClientId("client-2".to_string()); + let stream_1 = StreamId("stream-1".to_string()); outbound_buffer.insert(&server_envelope( &client_1, "stream-1", - /*seq_id*/ 0, + /*seq_id*/ 1, "first-client-old-stream", )); outbound_buffer.insert(&server_envelope( &client_2, "stream-1", - /*seq_id*/ 1, + /*seq_id*/ 2, "second-client", )); outbound_buffer.insert(&server_envelope( &client_1, "stream-2", - /*seq_id*/ 2, + /*seq_id*/ 3, "first-client-new-stream", )); - outbound_buffer.ack(&client_1, /*acked_seq_id*/ 2); - - let mut retained = outbound_buffer - .server_envelopes() - .map(|server_envelope| { - ( - server_envelope.client_id.0.as_str(), - server_envelope.stream_id.0.as_str(), - server_envelope.seq_id, - ) - }) - .collect::>(); - retained.sort_unstable(); - assert_eq!(retained, vec![("client-2", "stream-1", 1)]); - assert_eq!(*used_rx.borrow(), 1); - } - - #[test] - fn outbound_buffer_retains_unacked_messages_until_ack_advances() { - let (mut outbound_buffer, used_rx) = BoundedOutboundBuffer::new(); - let client_1 = ClientId("client-1".to_string()); - let client_2 = ClientId("client-2".to_string()); - - outbound_buffer.insert(&server_envelope( - &client_1, - "stream-1", - /*seq_id*/ 0, - "first-old", - )); - outbound_buffer.insert(&server_envelope( - &client_1, - "stream-2", - /*seq_id*/ 1, - "first-new", - )); - outbound_buffer.insert(&server_envelope( - &client_2, "stream-1", /*seq_id*/ 2, "second", - )); - - outbound_buffer.ack(&client_1, /*acked_seq_id*/ 0); + outbound_buffer.ack(&client_1, &stream_1, /*acked_seq_id*/ 3); let mut retained = outbound_buffer .server_envelopes() @@ -1365,7 +1417,50 @@ mod tests { retained.sort_unstable(); assert_eq!( retained, - vec![("client-1", "stream-2", 1), ("client-2", "stream-1", 2)] + vec![("client-1", "stream-2", 3), ("client-2", "stream-1", 2)] + ); + assert_eq!(*used_rx.borrow(), 2); + } + + #[test] + fn outbound_buffer_retains_unacked_messages_until_ack_advances() { + let (mut outbound_buffer, used_rx) = BoundedOutboundBuffer::new(); + let client_1 = ClientId("client-1".to_string()); + let client_2 = ClientId("client-2".to_string()); + let stream_1 = StreamId("stream-1".to_string()); + + outbound_buffer.insert(&server_envelope( + &client_1, + "stream-1", + /*seq_id*/ 1, + "first-old", + )); + outbound_buffer.insert(&server_envelope( + &client_1, + "stream-2", + /*seq_id*/ 2, + "first-new", + )); + outbound_buffer.insert(&server_envelope( + &client_2, "stream-1", /*seq_id*/ 3, "second", + )); + + outbound_buffer.ack(&client_1, &stream_1, /*acked_seq_id*/ 1); + + let mut retained = outbound_buffer + .server_envelopes() + .map(|server_envelope| { + ( + server_envelope.client_id.0.as_str(), + server_envelope.stream_id.0.as_str(), + server_envelope.seq_id, + ) + }) + .collect::>(); + retained.sort_unstable(); + assert_eq!( + retained, + vec![("client-1", "stream-2", 2), ("client-2", "stream-1", 3)] ); assert_eq!(*used_rx.borrow(), 2); } @@ -1450,6 +1545,20 @@ mod tests { (client_stream, server_stream) } + async fn read_server_text_event( + server_stream: &mut WebSocketStream, + ) -> serde_json::Value { + let message = timeout(Duration::from_secs(5), server_stream.next()) + .await + .expect("server event should arrive in time") + .expect("server websocket should stay open") + .expect("server event should read"); + let tungstenite::Message::Text(text) = message else { + panic!("expected text event, got {message:?}"); + }; + serde_json::from_str(text.as_ref()).expect("server event should deserialize") + } + async fn respond_with_status_and_headers( mut stream: TcpStream, status: &str, From 66533ddc61e2795ea58f23cf3b88515d6085ee67 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 15 Apr 2026 12:02:40 -0700 Subject: [PATCH 30/84] mcp: remove codex/sandbox-state custom request support (#17957) ## Why #17763 moved sandbox-state delivery for MCP tool calls to request `_meta` via the `codex/sandbox-state-meta` experimental capability. Keeping the older `codex/sandbox-state` capability meant Codex still maintained a second transport that pushed updates with the custom `codex/sandbox-state/update` request at server startup and when the session sandbox policy changed. That duplicate MCP path is redundant with the per-tool-call metadata path and makes the sandbox-state contract larger than needed. The existing managed network proxy refresh on sandbox-policy changes is still needed, so this keeps that behavior separate from the removed MCP notification. ## What Changed - Removed the exported `MCP_SANDBOX_STATE_CAPABILITY` and `MCP_SANDBOX_STATE_METHOD` constants. - Removed detection of `codex/sandbox-state` during MCP initialization and stopped sending `codex/sandbox-state/update` at server startup. - Removed the `McpConnectionManager::notify_sandbox_state_change` plumbing while preserving the managed network proxy refresh when a user turn changes sandbox policy. - Slimmed `McpConnectionManager::new` so startup paths pass only the initial `SandboxPolicy` needed for MCP elicitation state. - Kept `codex/sandbox-state-meta` support intact; servers that opt in still receive the current `SandboxState` on tool-call request `_meta` ([remaining call path](https://github.com/openai/codex/blob/ff2d3c1e72ff08ce13743b99605d19d338edd51c/codex-rs/core/src/mcp_tool_call.rs#L487-L526)). - Added regression coverage for refreshing the live managed network proxy on a per-turn sandbox-policy change. ## Verification - `cargo test -p codex-core new_turn_refreshes_managed_network_proxy_for_sandbox_change` - `cargo test -p codex-mcp` --- codex-rs/codex-mcp/src/lib.rs | 2 - codex-rs/codex-mcp/src/mcp/mod.rs | 20 +---- .../codex-mcp/src/mcp_connection_manager.rs | 85 +------------------ codex-rs/core/src/codex.rs | 48 ++--------- codex-rs/core/src/codex_tests.rs | 83 ++++++++++++++++++ codex-rs/core/src/connectors.rs | 12 +-- codex-rs/core/src/lib.rs | 2 - 7 files changed, 97 insertions(+), 155 deletions(-) diff --git a/codex-rs/codex-mcp/src/lib.rs b/codex-rs/codex-mcp/src/lib.rs index 36007e6c92..ed0d9b4122 100644 --- a/codex-rs/codex-mcp/src/lib.rs +++ b/codex-rs/codex-mcp/src/lib.rs @@ -36,9 +36,7 @@ pub use mcp::tool_plugin_provenance; pub use mcp::with_codex_apps_mcp; pub use mcp_connection_manager::CodexAppsToolsCacheKey; pub use mcp_connection_manager::DEFAULT_STARTUP_TIMEOUT; -pub use mcp_connection_manager::MCP_SANDBOX_STATE_CAPABILITY; pub use mcp_connection_manager::MCP_SANDBOX_STATE_META_CAPABILITY; -pub use mcp_connection_manager::MCP_SANDBOX_STATE_METHOD; pub use mcp_connection_manager::McpConnectionManager; pub use mcp_connection_manager::SandboxState; pub use mcp_connection_manager::ToolInfo; diff --git a/codex-rs/codex-mcp/src/mcp/mod.rs b/codex-rs/codex-mcp/src/mcp/mod.rs index d53532b951..421ed14c36 100644 --- a/codex-rs/codex-mcp/src/mcp/mod.rs +++ b/codex-rs/codex-mcp/src/mcp/mod.rs @@ -35,7 +35,6 @@ use codex_protocol::protocol::SandboxPolicy; use serde_json::Value; use crate::mcp_connection_manager::McpConnectionManager; -use crate::mcp_connection_manager::SandboxState; use crate::mcp_connection_manager::codex_apps_tools_cache_key; pub type McpManager = McpConnectionManager; @@ -347,14 +346,6 @@ pub async fn collect_mcp_snapshot_with_detail( let (tx_event, rx_event) = unbounded(); drop(rx_event); - // Use ReadOnly sandbox policy for MCP snapshot collection (safest default) - let sandbox_state = SandboxState { - sandbox_policy: SandboxPolicy::new_read_only_policy(), - codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(), - sandbox_cwd: env::current_dir().unwrap_or_else(|_| PathBuf::from("/")), - use_legacy_landlock: config.use_legacy_landlock, - }; - let (mcp_connection_manager, cancel_token) = McpConnectionManager::new( &mcp_servers, config.mcp_oauth_credentials_store_mode, @@ -362,7 +353,7 @@ pub async fn collect_mcp_snapshot_with_detail( &config.approval_policy, submit_id, tx_event, - sandbox_state, + SandboxPolicy::new_read_only_policy(), config.codex_home.clone(), codex_apps_tools_cache_key(auth), tool_plugin_provenance, @@ -421,13 +412,6 @@ pub async fn collect_mcp_server_status_snapshot_with_detail( let (tx_event, rx_event) = unbounded(); drop(rx_event); - let sandbox_state = SandboxState { - sandbox_policy: SandboxPolicy::new_read_only_policy(), - codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(), - sandbox_cwd: env::current_dir().unwrap_or_else(|_| PathBuf::from("/")), - use_legacy_landlock: config.use_legacy_landlock, - }; - let (mcp_connection_manager, cancel_token) = McpConnectionManager::new( &mcp_servers, config.mcp_oauth_credentials_store_mode, @@ -435,7 +419,7 @@ pub async fn collect_mcp_server_status_snapshot_with_detail( &config.approval_policy, submit_id, tx_event, - sandbox_state, + SandboxPolicy::new_read_only_policy(), config.codex_home.clone(), codex_apps_tools_cache_key(auth), tool_plugin_provenance, diff --git a/codex-rs/codex-mcp/src/mcp_connection_manager.rs b/codex-rs/codex-mcp/src/mcp_connection_manager.rs index fadb07d581..1542c7b30b 100644 --- a/codex-rs/codex-mcp/src/mcp_connection_manager.rs +++ b/codex-rs/codex-mcp/src/mcp_connection_manager.rs @@ -439,7 +439,6 @@ struct ManagedClient { tool_filter: ToolFilter, tool_timeout: Option, server_instructions: Option, - server_supports_sandbox_state_capability: bool, server_supports_sandbox_state_meta_capability: bool, codex_apps_tools_cache_context: Option, } @@ -469,22 +468,6 @@ impl ManagedClient { self.tools.clone() } - - /// Returns once the server has ack'd the sandbox state update. - async fn notify_sandbox_state_change(&self, sandbox_state: &SandboxState) -> Result<()> { - if !self.server_supports_sandbox_state_capability { - return Ok(()); - } - - let _response = self - .client - .send_custom_request( - MCP_SANDBOX_STATE_METHOD, - Some(serde_json::to_value(sandbox_state)?), - ) - .await?; - Ok(()) - } } #[derive(Clone)] @@ -642,19 +625,8 @@ impl AsyncManagedClient { }; tools.map(annotate_tools) } - - async fn notify_sandbox_state_change(&self, sandbox_state: &SandboxState) -> Result<()> { - let managed = self.client().await?; - managed.notify_sandbox_state_change(sandbox_state).await - } } -pub const MCP_SANDBOX_STATE_CAPABILITY: &str = "codex/sandbox-state"; - -/// Custom MCP request to push sandbox state updates. -/// When used, the `params` field of the notification is [`SandboxState`]. -pub const MCP_SANDBOX_STATE_METHOD: &str = "codex/sandbox-state/update"; - /// MCP server capability indicating that Codex should include [`SandboxState`] /// in tool-call request `_meta` under this key. pub const MCP_SANDBOX_STATE_META_CAPABILITY: &str = "codex/sandbox-state-meta"; @@ -735,7 +707,7 @@ impl McpConnectionManager { approval_policy: &Constrained, submit_id: String, tx_event: Sender, - initial_sandbox_state: SandboxState, + initial_sandbox_policy: SandboxPolicy, codex_home: PathBuf, codex_apps_tools_cache_key: CodexAppsToolsCacheKey, tool_plugin_provenance: ToolPluginProvenance, @@ -744,10 +716,8 @@ impl McpConnectionManager { let mut clients = HashMap::new(); let mut server_origins = HashMap::new(); let mut join_set = JoinSet::new(); - let elicitation_requests = ElicitationRequestManager::new( - approval_policy.value(), - initial_sandbox_state.sandbox_policy.clone(), - ); + let elicitation_requests = + ElicitationRequestManager::new(approval_policy.value(), initial_sandbox_policy); let tool_plugin_provenance = Arc::new(tool_plugin_provenance); let startup_submit_id = submit_id.clone(); let mcp_servers = mcp_servers.clone(); @@ -787,25 +757,13 @@ impl McpConnectionManager { let tx_event = tx_event.clone(); let submit_id = startup_submit_id.clone(); let auth_entry = auth_entries.get(&server_name).cloned(); - let sandbox_state = initial_sandbox_state.clone(); join_set.spawn(async move { let outcome = async_managed_client.client().await; if cancel_token.is_cancelled() { return (server_name, Err(StartupOutcomeError::Cancelled)); } let status = match &outcome { - Ok(_) => { - // Send sandbox state notification immediately after Ready - if let Err(e) = async_managed_client - .notify_sandbox_state_change(&sandbox_state) - .await - { - warn!( - "Failed to notify sandbox state to MCP server {server_name}: {e:#}", - ); - } - McpStartupStatus::Ready - } + Ok(_) => McpStartupStatus::Ready, Err(error) => { let error_str = mcp_init_error_display( server_name.as_str(), @@ -1219,34 +1177,6 @@ impl McpConnectionManager { .into_values() .find(|tool| tool.canonical_tool_name() == *tool_name) } - - pub async fn notify_sandbox_state_change(&self, sandbox_state: &SandboxState) -> Result<()> { - let mut join_set = JoinSet::new(); - - for async_managed_client in self.clients.values() { - let sandbox_state = sandbox_state.clone(); - let async_managed_client = async_managed_client.clone(); - join_set.spawn(async move { - async_managed_client - .notify_sandbox_state_change(&sandbox_state) - .await - }); - } - - while let Some(join_res) = join_set.join_next().await { - match join_res { - Ok(Ok(())) => {} - Ok(Err(err)) => { - warn!("Failed to notify sandbox state change to MCP server: {err:#}"); - } - Err(err) => { - warn!("Task panic when notifying sandbox state change to MCP server: {err:#}"); - } - } - } - - Ok(()) - } } async fn emit_update( @@ -1492,12 +1422,6 @@ async fn start_server_task( .await .map_err(StartupOutcomeError::from)?; - let server_supports_sandbox_state_capability = initialize_result - .capabilities - .experimental - .as_ref() - .and_then(|exp| exp.get(MCP_SANDBOX_STATE_CAPABILITY)) - .is_some(); let server_supports_sandbox_state_meta_capability = initialize_result .capabilities .experimental @@ -1539,7 +1463,6 @@ async fn start_server_task( tool_timeout: Some(tool_timeout), tool_filter, server_instructions: initialize_result.instructions, - server_supports_sandbox_state_capability, server_supports_sandbox_state_meta_capability, codex_apps_tools_cache_context, }; diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index bc7cd07e4c..cc314a66e9 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -78,7 +78,6 @@ use codex_login::CodexAuth; use codex_login::auth_env_telemetry::collect_auth_env_telemetry; use codex_login::default_client::originator; use codex_mcp::McpConnectionManager; -use codex_mcp::SandboxState; use codex_mcp::ToolInfo; use codex_mcp::codex_apps_tools_cache_key; #[cfg(test)] @@ -2212,14 +2211,6 @@ impl Session { // Start the watcher after SessionConfigured so it cannot emit earlier events. sess.start_skills_watcher_listener(); sess.start_agent_identity_registration(); - // Construct sandbox_state before MCP startup so it can be sent to each - // MCP server immediately after it becomes ready (avoiding blocking). - let sandbox_state = SandboxState { - sandbox_policy: session_configuration.sandbox_policy.get().clone(), - codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(), - sandbox_cwd: session_configuration.cwd.to_path_buf(), - use_legacy_landlock: config.features.use_legacy_landlock(), - }; let mut required_mcp_servers: Vec = mcp_servers .iter() .filter(|(_, server)| server.enabled && server.required) @@ -2241,7 +2232,7 @@ impl Session { &session_configuration.approval_policy, INITIAL_SUBMIT_ID.to_owned(), tx_event.clone(), - sandbox_state, + session_configuration.sandbox_policy.get().clone(), config.codex_home.to_path_buf(), codex_apps_tools_cache_key(auth), tool_plugin_provenance, @@ -2673,12 +2664,16 @@ impl Session { &session_source, ); + if sandbox_policy_changed { + self.refresh_managed_network_proxy_for_current_sandbox_policy() + .await; + } + Ok(self .new_turn_from_configuration( sub_id, session_configuration, updates.final_output_json_schema, - sandbox_policy_changed, ) .await) } @@ -2688,7 +2683,6 @@ impl Session { sub_id: String, session_configuration: SessionConfiguration, final_output_json_schema: Option>, - sandbox_policy_changed: bool, ) -> Arc { let per_turn_config = Self::build_per_turn_config(&session_configuration); { @@ -2698,27 +2692,6 @@ impl Session { .set_sandbox_policy(per_turn_config.permissions.sandbox_policy.get()); } - if sandbox_policy_changed { - self.refresh_managed_network_proxy_for_current_sandbox_policy() - .await; - let sandbox_state = SandboxState { - sandbox_policy: per_turn_config.permissions.sandbox_policy.get().clone(), - codex_linux_sandbox_exe: per_turn_config.codex_linux_sandbox_exe.clone(), - sandbox_cwd: per_turn_config.cwd.to_path_buf(), - use_legacy_landlock: per_turn_config.features.use_legacy_landlock(), - }; - if let Err(e) = self - .services - .mcp_connection_manager - .read() - .await - .notify_sandbox_state_change(&sandbox_state) - .await - { - warn!("Failed to notify sandbox state change to MCP servers: {e:#}"); - } - } - let model_info = self .services .models_manager @@ -2867,7 +2840,6 @@ impl Session { sub_id, session_configuration, /*final_output_json_schema*/ None, - /*sandbox_policy_changed*/ false, ) .await } @@ -4579,12 +4551,6 @@ impl Session { .await; let mcp_servers = with_codex_apps_mcp(mcp_servers, auth.as_ref(), &mcp_config); let auth_statuses = compute_auth_statuses(mcp_servers.iter(), store_mode).await; - let sandbox_state = SandboxState { - sandbox_policy: turn_context.sandbox_policy.get().clone(), - codex_linux_sandbox_exe: turn_context.codex_linux_sandbox_exe.clone(), - sandbox_cwd: turn_context.cwd.to_path_buf(), - use_legacy_landlock: turn_context.features.use_legacy_landlock(), - }; { let mut guard = self.services.mcp_startup_cancellation_token.lock().await; guard.cancel(); @@ -4597,7 +4563,7 @@ impl Session { &turn_context.config.permissions.approval_policy, turn_context.sub_id.clone(), self.get_tx_event(), - sandbox_state, + turn_context.sandbox_policy.get().clone(), config.codex_home.to_path_buf(), codex_apps_tools_cache_key(auth.as_ref()), tool_plugin_provenance, diff --git a/codex-rs/core/src/codex_tests.rs b/codex-rs/core/src/codex_tests.rs index fb7bbaadba..eaa2099098 100644 --- a/codex-rs/core/src/codex_tests.rs +++ b/codex-rs/core/src/codex_tests.rs @@ -656,6 +656,89 @@ async fn managed_network_proxy_decider_survives_full_access_start() -> anyhow::R Ok(()) } +#[tokio::test] +async fn new_turn_refreshes_managed_network_proxy_for_sandbox_change() -> anyhow::Result<()> { + let (mut session, _turn_context) = make_session_and_context().await; + let initial_policy = SandboxPolicy::new_workspace_write_policy(); + + let mut network_config = NetworkProxyConfig::default(); + network_config + .network + .set_allowed_domains(vec!["evil.com".to_string()]); + let requirements = NetworkConstraints { + domains: Some(NetworkDomainPermissionsToml { + entries: std::collections::BTreeMap::from([( + "*.example.com".to_string(), + NetworkDomainPermissionToml::Allow, + )]), + }), + ..Default::default() + }; + let spec = crate::config::NetworkProxySpec::from_config_and_constraints( + network_config, + Some(requirements), + &initial_policy, + )?; + let (started_proxy, _) = Session::start_managed_network_proxy( + &spec, + &Policy::empty(), + &initial_policy, + /*network_policy_decider*/ None, + /*blocked_request_observer*/ None, + /*managed_network_requirements_enabled*/ false, + crate::config::NetworkProxyAuditMetadata::default(), + ) + .await?; + assert_eq!( + started_proxy + .proxy() + .current_cfg() + .await? + .network + .allowed_domains(), + Some(vec!["*.example.com".to_string(), "evil.com".to_string()]) + ); + + { + let mut state = session.state.lock().await; + let mut config = (*state.session_configuration.original_config_do_not_use).clone(); + config.permissions.network = Some(spec); + config.permissions.sandbox_policy = + codex_config::Constrained::allow_any(initial_policy.clone()); + state.session_configuration.original_config_do_not_use = Arc::new(config); + state.session_configuration.sandbox_policy = + codex_config::Constrained::allow_any(initial_policy); + } + session.services.network_proxy = Some(started_proxy); + + session + .new_turn_with_sub_id( + "sandbox-policy-change".to_string(), + SessionSettingsUpdate { + sandbox_policy: Some(SandboxPolicy::DangerFullAccess), + ..Default::default() + }, + ) + .await?; + + let started_proxy = session + .services + .network_proxy + .as_ref() + .expect("managed network proxy should be present"); + assert_eq!( + started_proxy + .proxy() + .current_cfg() + .await? + .network + .allowed_domains(), + Some(vec!["*.example.com".to_string()]) + ); + + Ok(()) +} + #[tokio::test] async fn get_base_instructions_no_user_content() { let prompt_with_apply_patch_instructions = diff --git a/codex-rs/core/src/connectors.rs b/codex-rs/core/src/connectors.rs index eef6d5747b..77603adcf1 100644 --- a/codex-rs/core/src/connectors.rs +++ b/codex-rs/core/src/connectors.rs @@ -1,8 +1,6 @@ use std::collections::BTreeSet; use std::collections::HashMap; use std::collections::HashSet; -use std::env; -use std::path::PathBuf; use std::sync::Arc; use std::sync::LazyLock; use std::sync::Mutex as StdMutex; @@ -42,7 +40,6 @@ use codex_login::default_client::is_first_party_chat_originator; use codex_login::default_client::originator; use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; use codex_mcp::McpConnectionManager; -use codex_mcp::SandboxState; use codex_mcp::ToolInfo; use codex_mcp::ToolPluginProvenance; use codex_mcp::codex_apps_tools_cache_key; @@ -228,13 +225,6 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_options_and_status( let (tx_event, rx_event) = unbounded(); drop(rx_event); - let sandbox_state = SandboxState { - sandbox_policy: SandboxPolicy::new_read_only_policy(), - codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(), - sandbox_cwd: env::current_dir().unwrap_or_else(|_| PathBuf::from("/")), - use_legacy_landlock: config.features.use_legacy_landlock(), - }; - let (mcp_connection_manager, cancel_token) = McpConnectionManager::new( &mcp_servers, config.mcp_oauth_credentials_store_mode, @@ -242,7 +232,7 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_options_and_status( &config.permissions.approval_policy, INITIAL_SUBMIT_ID.to_owned(), tx_event, - sandbox_state, + SandboxPolicy::new_read_only_policy(), config.codex_home.to_path_buf(), codex_apps_tools_cache_key(auth.as_ref()), ToolPluginProvenance::default(), diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 37b5dcf5d4..8d463f3b51 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -55,8 +55,6 @@ pub use network_proxy_loader::MtimeConfigReloader; pub use network_proxy_loader::build_network_proxy_state; pub use network_proxy_loader::build_network_proxy_state_and_reloader; mod original_image_detail; -pub use codex_mcp::MCP_SANDBOX_STATE_CAPABILITY; -pub use codex_mcp::MCP_SANDBOX_STATE_METHOD; pub use codex_mcp::SandboxState; mod mcp_openai_file; mod mcp_tool_call; From 7e7b35b4d296cd19280b5468814323db0d68c1a7 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 15 Apr 2026 20:25:53 +0100 Subject: [PATCH 31/84] fix: propagate log db (#17953) It restores the TRACE logs in the DB and `/feedback` Fix https://github.com/openai/codex/pull/16184 Result: https://openai.sentry.io/issues/6972946529/?project=4510195390611458&query=019d91e9-f931-7451-8852-c5240514a419&referrer=issue-stream --- codex-rs/app-server-client/src/lib.rs | 6 ++++++ codex-rs/app-server/src/in_process.rs | 6 +++++- codex-rs/exec/src/lib.rs | 1 + codex-rs/tui/src/lib.rs | 23 ++++++++++++++++++++--- codex-rs/tui/src/onboarding/auth.rs | 1 + 5 files changed, 33 insertions(+), 4 deletions(-) diff --git a/codex-rs/app-server-client/src/lib.rs b/codex-rs/app-server-client/src/lib.rs index eb3d984965..35f9068712 100644 --- a/codex-rs/app-server-client/src/lib.rs +++ b/codex-rs/app-server-client/src/lib.rs @@ -28,6 +28,7 @@ use std::time::Duration; pub use codex_app_server::in_process::DEFAULT_IN_PROCESS_CHANNEL_CAPACITY; pub use codex_app_server::in_process::InProcessServerEvent; use codex_app_server::in_process::InProcessStartArgs; +use codex_app_server::in_process::LogDbLayer; use codex_app_server_protocol::ClientInfo; use codex_app_server_protocol::ClientNotification; use codex_app_server_protocol::ClientRequest; @@ -354,6 +355,8 @@ pub struct InProcessClientStartArgs { pub cloud_requirements: CloudRequirementsLoader, /// Feedback sink used by app-server/core telemetry and logs. pub feedback: CodexFeedback, + /// SQLite tracing layer used to flush recently emitted logs before feedback upload. + pub log_db: Option, /// Environment manager used by core execution and filesystem operations. pub environment_manager: Arc, /// Startup warnings emitted after initialize succeeds. @@ -405,6 +408,7 @@ impl InProcessClientStartArgs { loader_overrides: self.loader_overrides, cloud_requirements: self.cloud_requirements, feedback: self.feedback, + log_db: self.log_db, environment_manager: self.environment_manager, config_warnings: self.config_warnings, session_source: self.session_source, @@ -982,6 +986,7 @@ mod tests { loader_overrides: LoaderOverrides::default(), cloud_requirements: CloudRequirementsLoader::default(), feedback: CodexFeedback::new(), + log_db: None, environment_manager: Arc::new(EnvironmentManager::new(/*exec_server_url*/ None)), config_warnings: Vec::new(), session_source, @@ -1994,6 +1999,7 @@ mod tests { loader_overrides: LoaderOverrides::default(), cloud_requirements: CloudRequirementsLoader::default(), feedback: CodexFeedback::new(), + log_db: None, environment_manager: environment_manager.clone(), config_warnings: Vec::new(), session_source: SessionSource::Exec, diff --git a/codex-rs/app-server/src/in_process.rs b/codex-rs/app-server/src/in_process.rs index 9ac0e6cf42..84dfd282cd 100644 --- a/codex-rs/app-server/src/in_process.rs +++ b/codex-rs/app-server/src/in_process.rs @@ -82,6 +82,7 @@ use codex_exec_server::EnvironmentManager; use codex_feedback::CodexFeedback; use codex_login::AuthManager; use codex_protocol::protocol::SessionSource; +pub use codex_state::log_db::LogDbLayer; use tokio::sync::mpsc; use tokio::sync::oneshot; use tokio::time::timeout; @@ -117,6 +118,8 @@ pub struct InProcessStartArgs { pub cloud_requirements: CloudRequirementsLoader, /// Feedback sink used by app-server/core telemetry and logs. pub feedback: CodexFeedback, + /// SQLite tracing layer used to flush recently emitted logs before feedback upload. + pub log_db: Option, /// Environment manager used by core execution and filesystem operations. pub environment_manager: Arc, /// Startup warnings emitted after initialize succeeds. @@ -395,7 +398,7 @@ fn start_uninitialized(args: InProcessStartArgs) -> InProcessClientHandle { loader_overrides: args.loader_overrides, cloud_requirements: args.cloud_requirements, feedback: args.feedback, - log_db: None, + log_db: args.log_db, config_warnings: args.config_warnings, session_source: args.session_source, auth_manager, @@ -728,6 +731,7 @@ mod tests { loader_overrides: LoaderOverrides::default(), cloud_requirements: CloudRequirementsLoader::default(), feedback: CodexFeedback::new(), + log_db: None, environment_manager: Arc::new(EnvironmentManager::new(/*exec_server_url*/ None)), config_warnings: Vec::new(), session_source, diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index 758da9f43d..123038ffaa 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -481,6 +481,7 @@ pub async fn run_main(cli: Cli, arg0_paths: Arg0DispatchPaths) -> anyhow::Result loader_overrides: run_loader_overrides, cloud_requirements: run_cloud_requirements, feedback: CodexFeedback::new(), + log_db: None, environment_manager: std::sync::Arc::new(EnvironmentManager::from_env_with_runtime_paths( Some(local_runtime_paths), )), diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 46ac561dff..cf938545c6 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -67,10 +67,12 @@ use std::future::Future; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; +use tracing::Level; use tracing::error; use tracing::warn; use tracing_appender::non_blocking; use tracing_subscriber::EnvFilter; +use tracing_subscriber::filter::Targets; use tracing_subscriber::prelude::*; use url::Url; use uuid::Uuid; @@ -237,6 +239,7 @@ pub use public_widgets::composer_input::ComposerAction; pub use public_widgets::composer_input::ComposerInput; // (tests access modules directly within the crate) +#[allow(clippy::too_many_arguments)] async fn start_embedded_app_server( arg0_paths: Arg0DispatchPaths, config: Config, @@ -244,6 +247,7 @@ async fn start_embedded_app_server( loader_overrides: LoaderOverrides, cloud_requirements: CloudRequirementsLoader, feedback: codex_feedback::CodexFeedback, + log_db: Option, environment_manager: Arc, ) -> color_eyre::Result { start_embedded_app_server_with( @@ -253,6 +257,7 @@ async fn start_embedded_app_server( loader_overrides, cloud_requirements, feedback, + log_db, environment_manager, InProcessAppServerClient::start, ) @@ -369,6 +374,7 @@ async fn start_app_server( loader_overrides: LoaderOverrides, cloud_requirements: CloudRequirementsLoader, feedback: codex_feedback::CodexFeedback, + log_db: Option, environment_manager: Arc, ) -> color_eyre::Result { match target { @@ -379,6 +385,7 @@ async fn start_app_server( loader_overrides, cloud_requirements, feedback, + log_db, environment_manager, ) .await @@ -403,6 +410,7 @@ pub(crate) async fn start_app_server_for_picker( LoaderOverrides::default(), CloudRequirementsLoader::default(), codex_feedback::CodexFeedback::new(), + /*log_db*/ None, environment_manager, ) .await?; @@ -429,6 +437,7 @@ async fn start_embedded_app_server_with( loader_overrides: LoaderOverrides, cloud_requirements: CloudRequirementsLoader, feedback: codex_feedback::CodexFeedback, + log_db: Option, environment_manager: Arc, start_client: F, ) -> color_eyre::Result @@ -453,6 +462,7 @@ where loader_overrides, cloud_requirements, feedback, + log_db, environment_manager, config_warnings, session_source: codex_protocol::protocol::SessionSource::Cli, @@ -969,9 +979,10 @@ pub async fn run_main( let otel_tracing_layer = otel.as_ref().and_then(|o| o.tracing_layer()); - let log_db_layer = get_state_db(&config) - .await - .map(|db| log_db::start(db).with_filter(env_filter())); + let log_db = get_state_db(&config).await.map(log_db::start); + let log_db_layer = log_db + .clone() + .map(|layer| layer.with_filter(Targets::new().with_default(Level::TRACE))); let _ = tracing_subscriber::registry() .with(file_layer) @@ -993,6 +1004,7 @@ pub async fn run_main( cli_kv_overrides, cloud_requirements, feedback, + log_db, remote_url, remote_auth_token, environment_manager, @@ -1013,6 +1025,7 @@ async fn run_ratatui_app( cli_kv_overrides: Vec<(String, toml::Value)>, mut cloud_requirements: CloudRequirementsLoader, feedback: codex_feedback::CodexFeedback, + log_db: Option, remote_url: Option, remote_auth_token: Option, environment_manager: Arc, @@ -1071,6 +1084,7 @@ async fn run_ratatui_app( loader_overrides.clone(), cloud_requirements.clone(), feedback.clone(), + log_db.clone(), environment_manager.clone(), ) .await @@ -1409,6 +1423,7 @@ async fn run_ratatui_app( loader_overrides, cloud_requirements.clone(), feedback.clone(), + log_db.clone(), environment_manager.clone(), ) .await @@ -1772,6 +1787,7 @@ mod tests { LoaderOverrides::default(), CloudRequirementsLoader::default(), codex_feedback::CodexFeedback::new(), + /*log_db*/ None, Arc::new(EnvironmentManager::new(/*exec_server_url*/ None)), ) .await @@ -2163,6 +2179,7 @@ mod tests { LoaderOverrides::default(), CloudRequirementsLoader::default(), codex_feedback::CodexFeedback::new(), + /*log_db*/ None, Arc::new(EnvironmentManager::new(/*exec_server_url*/ None)), |_args| async { Err(std::io::Error::other("boom")) }, ) diff --git a/codex-rs/tui/src/onboarding/auth.rs b/codex-rs/tui/src/onboarding/auth.rs index 57bd58bf5a..aee909cea1 100644 --- a/codex-rs/tui/src/onboarding/auth.rs +++ b/codex-rs/tui/src/onboarding/auth.rs @@ -988,6 +988,7 @@ mod tests { "https://chatgpt.com/backend-api/".to_string(), ), feedback: codex_feedback::CodexFeedback::new(), + log_db: None, environment_manager: Arc::new(codex_app_server_client::EnvironmentManager::new( /*exec_server_url*/ None, )), From 81d9cde9cbd7aaa9b89508adcfba31c5c9b84fc1 Mon Sep 17 00:00:00 2001 From: Dylan Hurd Date: Wed, 15 Apr 2026 12:36:55 -0700 Subject: [PATCH 32/84] chore(tui) cleanup (#17920) ## Summary Cleanup extraneous plugins. --- codex-rs/tui/src/app.rs | 50 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 026bd05d7c..71304deb2c 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -6351,7 +6351,7 @@ async fn fetch_plugins_list( ) -> Result { let cwd = AbsolutePathBuf::try_from(cwd).wrap_err("plugin list cwd must be absolute")?; let request_id = RequestId::String(format!("plugin-list-{}", Uuid::new_v4())); - request_handle + let mut response = request_handle .request_typed(ClientRequest::PluginList { request_id, params: PluginListParams { @@ -6360,7 +6360,17 @@ async fn fetch_plugins_list( }, }) .await - .wrap_err("plugin/list failed in TUI") + .wrap_err("plugin/list failed in TUI")?; + hide_cli_only_plugin_marketplaces(&mut response); + Ok(response) +} + +const CLI_HIDDEN_PLUGIN_MARKETPLACES: &[&str] = &["openai-bundled"]; + +fn hide_cli_only_plugin_marketplaces(response: &mut PluginListResponse) { + response + .marketplaces + .retain(|marketplace| !CLI_HIDDEN_PLUGIN_MARKETPLACES.contains(&marketplace.name.as_str())); } async fn fetch_plugin_detail( @@ -6538,6 +6548,7 @@ mod tests { use codex_app_server_protocol::NetworkPolicyRuleAction as AppServerNetworkPolicyRuleAction; use codex_app_server_protocol::NonSteerableTurnKind as AppServerNonSteerableTurnKind; use codex_app_server_protocol::PermissionsRequestApprovalParams; + use codex_app_server_protocol::PluginMarketplaceEntry; use codex_app_server_protocol::RequestId as AppServerRequestId; use codex_app_server_protocol::ServerNotification; use codex_app_server_protocol::ServerRequest; @@ -6596,6 +6607,41 @@ mod tests { AbsolutePathBuf::try_from(PathBuf::from(path)).expect("absolute test path") } + #[test] + fn hide_cli_only_plugin_marketplaces_removes_openai_bundled() { + let mut response = PluginListResponse { + marketplaces: vec![ + PluginMarketplaceEntry { + name: "openai-bundled".to_string(), + path: test_absolute_path("/marketplaces/openai-bundled"), + interface: None, + plugins: Vec::new(), + }, + PluginMarketplaceEntry { + name: "openai-curated".to_string(), + path: test_absolute_path("/marketplaces/openai-curated"), + interface: None, + plugins: Vec::new(), + }, + ], + marketplace_load_errors: Vec::new(), + remote_sync_error: None, + featured_plugin_ids: Vec::new(), + }; + + hide_cli_only_plugin_marketplaces(&mut response); + + assert_eq!( + response.marketplaces, + vec![PluginMarketplaceEntry { + name: "openai-curated".to_string(), + path: test_absolute_path("/marketplaces/openai-curated"), + interface: None, + plugins: Vec::new(), + }] + ); + } + #[test] fn normalize_harness_overrides_resolves_relative_add_dirs() -> Result<()> { let temp_dir = tempdir()?; From 6696e0bbc3b5fcefaaa161c4322d906c0748bdd4 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 15 Apr 2026 20:40:41 +0100 Subject: [PATCH 33/84] chore: tmp disable (#17981) --- codex-rs/features/src/lib.rs | 6 +----- codex-rs/tui/src/bottom_pane/command_popup.rs | 2 +- codex-rs/tui/src/slash_command.rs | 1 + 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rs index 4d7a83b628..10e79f492a 100644 --- a/codex-rs/features/src/lib.rs +++ b/codex-rs/features/src/lib.rs @@ -674,11 +674,7 @@ pub const FEATURES: &[FeatureSpec] = &[ FeatureSpec { id: Feature::MemoryTool, key: "memories", - stage: Stage::Experimental { - name: "Memories", - menu_description: "Allow Codex to create new memories from conversations and bring relevant memories into new conversations.", - announcement: "NEW: Codex can now generate and uses memories. Try is now with `/memories`", - }, + stage: Stage::UnderDevelopment, default_enabled: false, }, FeatureSpec { diff --git a/codex-rs/tui/src/bottom_pane/command_popup.rs b/codex-rs/tui/src/bottom_pane/command_popup.rs index 29da003dc2..8bef8ddbc8 100644 --- a/codex-rs/tui/src/bottom_pane/command_popup.rs +++ b/codex-rs/tui/src/bottom_pane/command_popup.rs @@ -293,7 +293,7 @@ mod tests { CommandItem::Builtin(cmd) => cmd.command(), }) .collect(); - assert_eq!(cmds, vec!["model", "memories", "mention", "mcp"]); + assert_eq!(cmds, vec!["model", "mention", "mcp"]); } #[test] diff --git a/codex-rs/tui/src/slash_command.rs b/codex-rs/tui/src/slash_command.rs index 759bedbad7..70aed459ea 100644 --- a/codex-rs/tui/src/slash_command.rs +++ b/codex-rs/tui/src/slash_command.rs @@ -195,6 +195,7 @@ impl SlashCommand { SlashCommand::SandboxReadRoot => cfg!(target_os = "windows"), SlashCommand::Copy => !cfg!(target_os = "android"), SlashCommand::Rollout | SlashCommand::TestApproval => cfg!(debug_assertions), + SlashCommand::Memories => false, _ => true, } } From f2a4925f6398107dc29585b42fa2c783830c8103 Mon Sep 17 00:00:00 2001 From: Ivan Murashko Date: Wed, 15 Apr 2026 21:05:11 +0100 Subject: [PATCH 34/84] Support remote compaction for Azure responses providers (#17958) Azure Responses providers were still falling back to local compaction because the compaction gate only checked `ModelProviderInfo::is_openai()`. Move the capability check onto `ModelProviderInfo` with `supports_remote_compaction()`, backed by the existing Azure Responses endpoint detection used in `codex-api`, and have `core::compact` delegate to that helper. Add regression coverage for: - OpenAI providers using remote compaction - Azure providers using remote compaction - non-OpenAI/non-Azure providers staying on the local path resolves #17773 --------- Co-authored-by: Michael Bolin --- codex-rs/core/src/compact.rs | 2 +- codex-rs/core/src/compact_tests.rs | 26 +++++++++ codex-rs/model-provider-info/src/lib.rs | 5 ++ .../src/model_provider_info_tests.rs | 55 +++++++++++++++++++ 4 files changed, 87 insertions(+), 1 deletion(-) diff --git a/codex-rs/core/src/compact.rs b/codex-rs/core/src/compact.rs index 59fb45454d..eaf1d92109 100644 --- a/codex-rs/core/src/compact.rs +++ b/codex-rs/core/src/compact.rs @@ -58,7 +58,7 @@ pub(crate) enum InitialContextInjection { } pub(crate) fn should_use_remote_compact_task(provider: &ModelProviderInfo) -> bool { - provider.is_openai() + provider.supports_remote_compaction() } pub(crate) async fn run_inline_auto_compact_task( diff --git a/codex-rs/core/src/compact_tests.rs b/codex-rs/core/src/compact_tests.rs index cecf9ce9d7..f7a8d1dfff 100644 --- a/codex-rs/core/src/compact_tests.rs +++ b/codex-rs/core/src/compact_tests.rs @@ -1,4 +1,6 @@ use super::*; +use codex_model_provider_info::ModelProviderInfo; +use codex_model_provider_info::WireApi; use pretty_assertions::assert_eq; async fn process_compacted_history_with_test_session( @@ -185,6 +187,30 @@ fn build_token_limited_compacted_history_appends_summary_message() { assert_eq!(summary, summary_text); } +#[test] +fn should_use_remote_compact_task_for_azure_provider() { + let provider = ModelProviderInfo { + name: "Azure".into(), + base_url: Some("https://example.com/openai".into()), + env_key: Some("AZURE_OPENAI_API_KEY".into()), + env_key_instructions: None, + experimental_bearer_token: None, + auth: None, + wire_api: WireApi::Responses, + query_params: None, + http_headers: None, + env_http_headers: None, + request_max_retries: None, + stream_max_retries: None, + stream_idle_timeout_ms: None, + websocket_connect_timeout_ms: None, + requires_openai_auth: false, + supports_websockets: false, + }; + + assert!(should_use_remote_compact_task(&provider)); +} + #[tokio::test] async fn process_compacted_history_replaces_developer_messages() { let compacted_history = vec![ diff --git a/codex-rs/model-provider-info/src/lib.rs b/codex-rs/model-provider-info/src/lib.rs index f39ff0c7fe..80fb705605 100644 --- a/codex-rs/model-provider-info/src/lib.rs +++ b/codex-rs/model-provider-info/src/lib.rs @@ -7,6 +7,7 @@ use codex_api::Provider as ApiProvider; use codex_api::RetryConfig as ApiRetryConfig; +use codex_api::is_azure_responses_provider; use codex_app_server_protocol::AuthMode; use codex_protocol::config_types::ModelProviderAuthInfo; use codex_protocol::error::CodexErr; @@ -300,6 +301,10 @@ impl ModelProviderInfo { self.name == OPENAI_PROVIDER_NAME } + pub fn supports_remote_compaction(&self) -> bool { + self.is_openai() || is_azure_responses_provider(&self.name, self.base_url.as_deref()) + } + pub fn has_command_auth(&self) -> bool { self.auth.is_some() } diff --git a/codex-rs/model-provider-info/src/model_provider_info_tests.rs b/codex-rs/model-provider-info/src/model_provider_info_tests.rs index bdc2e73b1a..6cd1ef2366 100644 --- a/codex-rs/model-provider-info/src/model_provider_info_tests.rs +++ b/codex-rs/model-provider-info/src/model_provider_info_tests.rs @@ -129,6 +129,61 @@ supports_websockets = true assert_eq!(provider.websocket_connect_timeout_ms, Some(15_000)); } +#[test] +fn test_supports_remote_compaction_for_openai() { + let provider = ModelProviderInfo::create_openai_provider(/*base_url*/ None); + + assert!(provider.supports_remote_compaction()); +} + +#[test] +fn test_supports_remote_compaction_for_azure_name() { + let provider = ModelProviderInfo { + name: "Azure".into(), + base_url: Some("https://example.com/openai".into()), + env_key: Some("AZURE_OPENAI_API_KEY".into()), + env_key_instructions: None, + experimental_bearer_token: None, + auth: None, + wire_api: WireApi::Responses, + query_params: None, + http_headers: None, + env_http_headers: None, + request_max_retries: None, + stream_max_retries: None, + stream_idle_timeout_ms: None, + websocket_connect_timeout_ms: None, + requires_openai_auth: false, + supports_websockets: false, + }; + + assert!(provider.supports_remote_compaction()); +} + +#[test] +fn test_supports_remote_compaction_for_non_openai_non_azure_provider() { + let provider = ModelProviderInfo { + name: "Example".into(), + base_url: Some("https://example.com/v1".into()), + env_key: Some("API_KEY".into()), + env_key_instructions: None, + experimental_bearer_token: None, + auth: None, + wire_api: WireApi::Responses, + query_params: None, + http_headers: None, + env_http_headers: None, + request_max_retries: None, + stream_max_retries: None, + stream_idle_timeout_ms: None, + websocket_connect_timeout_ms: None, + requires_openai_auth: false, + supports_websockets: false, + }; + + assert!(!provider.supports_remote_compaction()); +} + #[test] fn test_deserialize_provider_auth_config_defaults() { let base_dir = tempdir().unwrap(); From 0bb438bca68d7c2064453ee9399f59f0cbec233e Mon Sep 17 00:00:00 2001 From: evawong-oai Date: Wed, 15 Apr 2026 13:12:46 -0700 Subject: [PATCH 35/84] [docs] Add security boundaries reference in SECURITY.md (#17848) ## Summary 1. Add a Security Boundaries section to `SECURITY.md`. 2. Point readers to the Codex Agent approvals and security documentation for sandboxing, approvals, and network controls. ## Validation 1. Reviewed the `SECURITY.md` diff in a clean worktree. 2. No tests run. Docs only change. --- .github/workflows/bazel.yml | 2 +- SECURITY.md | 4 +++ codex-rs/app-server/BUILD.bazel | 3 ++ codex-rs/core/BUILD.bazel | 55 ++++++++++++++++++++++++++++++++- codex-rs/tui/BUILD.bazel | 21 ++++++++++++- defs.bzl | 54 +++++++++++++++++++++++++++++--- 6 files changed, 131 insertions(+), 8 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 5d30916f9f..c5a4c0c1c7 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -94,7 +94,7 @@ jobs: "${bazel_wrapper_args[@]}" \ -- \ test \ - --test_tag_filters=-argument-comment-lint \ + --test_tag_filters=-argument-comment-lint,-flaky \ --test_verbose_timeout_warnings \ --build_metadata=COMMIT_SHA=${GITHUB_SHA} \ -- \ diff --git a/SECURITY.md b/SECURITY.md index d6dd568910..29a9b903df 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -11,3 +11,7 @@ Our security program is managed through Bugcrowd, and we ask that any validated ## Vulnerability Disclosure Program Our Vulnerability Program Guidelines are defined on our [Bugcrowd program page](https://bugcrowd.com/engagements/openai). + +## How to operate CODEX safely + +For details on Codex security boundaries, including sandboxing, approvals, and network controls, see [Agent approvals & security](https://developers.openai.com/codex/agent-approvals-security). diff --git a/codex-rs/app-server/BUILD.bazel b/codex-rs/app-server/BUILD.bazel index 872f533be1..f32f811627 100644 --- a/codex-rs/app-server/BUILD.bazel +++ b/codex-rs/app-server/BUILD.bazel @@ -3,6 +3,9 @@ load("//:defs.bzl", "codex_rust_crate") codex_rust_crate( name = "app-server", crate_name = "codex_app_server", + integration_test_tags_extra_by_stem = { + "all": ["flaky"], + }, integration_test_timeout = "long", test_tags = ["no-sandbox"], ) diff --git a/codex-rs/core/BUILD.bazel b/codex-rs/core/BUILD.bazel index 434dc1f6a4..be07ef7999 100644 --- a/codex-rs/core/BUILD.bazel +++ b/codex-rs/core/BUILD.bazel @@ -1,4 +1,4 @@ -load("//:defs.bzl", "codex_rust_crate") +load("//:defs.bzl", "codex_rust_crate", "workspace_root_test") filegroup( name = "model_availability_nux_fixtures", @@ -31,6 +31,15 @@ codex_rust_crate( "//codex-rs/apply-patch:apply_patch_tool_instructions.md", "templates/realtime/backend_prompt.md", ], + integration_test_flaky_by_stem = { + "all": [ + "suite::abort_tasks::interrupt_tool_records_history_entries", + "suite::apply_patch_cli::apply_patch_cli_rejects_invalid_hunk_header::applypatchmodeloutput_function_expects", + ], + "responses_headers": [ + "responses_stream_includes_turn_metadata_header_for_git_workspace_e2e", + ], + }, integration_test_timeout = "long", test_data_extra = [ "config.schema.json", @@ -48,6 +57,12 @@ codex_rust_crate( "//:AGENTS.md", ], test_tags = ["no-sandbox"], + unit_test_args = [ + "--skip", + "guardian::tests::guardian_parallel_reviews_fork_from_last_committed_trunk_history", + "--skip", + "agent::control::tests::completion_watcher_notifies_parent_when_child_is_missing", + ], unit_test_timeout = "long", extra_binaries = [ "//codex-rs/linux-sandbox:codex-linux-sandbox", @@ -57,3 +72,41 @@ codex_rust_crate( "//codex-rs/cli:codex", ], ) + +workspace_root_test( + name = "core-guardian-parallel-reviews-flaky-test", + env = { + "INSTA_WORKSPACE_ROOT": ".", + "INSTA_SNAPSHOT_PATH": "src", + }, + test_bin = ":core-unit-tests-bin", + workspace_root_marker = "//codex-rs/utils/cargo-bin:repo_root.marker", + args = [ + "--exact", + "guardian::tests::guardian_parallel_reviews_fork_from_last_committed_trunk_history", + ], + tags = [ + "flaky", + "no-sandbox", + ], + timeout = "long", +) + +workspace_root_test( + name = "core-completion-watcher-missing-child-flaky-test", + env = { + "INSTA_WORKSPACE_ROOT": ".", + "INSTA_SNAPSHOT_PATH": "src", + }, + test_bin = ":core-unit-tests-bin", + workspace_root_marker = "//codex-rs/utils/cargo-bin:repo_root.marker", + args = [ + "--exact", + "agent::control::tests::completion_watcher_notifies_parent_when_child_is_missing", + ], + tags = [ + "flaky", + "no-sandbox", + ], + timeout = "long", +) diff --git a/codex-rs/tui/BUILD.bazel b/codex-rs/tui/BUILD.bazel index 33cc133431..a8cf752f96 100644 --- a/codex-rs/tui/BUILD.bazel +++ b/codex-rs/tui/BUILD.bazel @@ -1,4 +1,4 @@ -load("//:defs.bzl", "MACOS_WEBRTC_RUSTC_LINK_FLAGS", "codex_rust_crate") +load("//:defs.bzl", "MACOS_WEBRTC_RUSTC_LINK_FLAGS", "codex_rust_crate", "workspace_root_test") codex_rust_crate( name = "tui", @@ -20,8 +20,27 @@ codex_rust_crate( "src/**/snapshots/**", ]) + ["//codex-rs/core:model_availability_nux_fixtures"], integration_compile_data_extra = ["src/test_backend.rs"], + unit_test_args = [ + "--skip", + "app::tests::update_memory_settings_updates_current_thread_memory_mode", + ], extra_binaries = [ "//codex-rs/cli:codex", ], rustc_flags_extra = MACOS_WEBRTC_RUSTC_LINK_FLAGS, ) + +workspace_root_test( + name = "tui-update-memory-settings-flaky-test", + env = { + "INSTA_WORKSPACE_ROOT": ".", + "INSTA_SNAPSHOT_PATH": "src", + }, + test_bin = ":tui-unit-tests-bin", + workspace_root_marker = "//codex-rs/utils/cargo-bin:repo_root.marker", + args = [ + "--exact", + "app::tests::update_memory_settings_updates_current_thread_memory_mode", + ], + tags = ["flaky"], +) diff --git a/defs.bzl b/defs.bzl index 6972b65a65..7cd89ce63e 100644 --- a/defs.bzl +++ b/defs.bzl @@ -137,10 +137,13 @@ def codex_rust_crate( rustc_env = {}, deps_extra = [], integration_compile_data_extra = [], + integration_test_tags_extra_by_stem = {}, + integration_test_flaky_by_stem = {}, integration_test_args = [], integration_test_timeout = None, test_data_extra = [], test_tags = [], + unit_test_args = [], unit_test_timeout = None, extra_binaries = []): """Defines a Rust crate with library, binaries, and tests wired for Bazel + Cargo parity. @@ -170,6 +173,9 @@ def codex_rust_crate( deps_extra: Extra normal deps beyond @crates resolution. Typically only needed when features add additional deps. integration_compile_data_extra: Extra compile_data for integration tests. + integration_test_flaky_by_stem: Mapping from `tests/*.rs` stem to exact + Rust test names that should be split into dedicated flaky targets + and skipped from the main integration test target. integration_test_args: Optional args for integration test binaries. integration_test_timeout: Optional Bazel timeout for integration test targets generated from `tests/*.rs`. @@ -270,6 +276,8 @@ def codex_rust_crate( unit_test_kwargs = {} if unit_test_timeout: unit_test_kwargs["timeout"] = unit_test_timeout + if unit_test_args: + unit_test_kwargs["args"] = unit_test_args workspace_root_test( name = name + "-unit-tests", @@ -305,11 +313,9 @@ def codex_rust_crate( binary = Label(binary_label).name cargo_env["CARGO_BIN_EXE_" + binary] = "$(rlocationpath %s)" % binary_label - integration_test_kwargs = {} - if integration_test_args: - integration_test_kwargs["args"] = integration_test_args + integration_test_base_kwargs = {} if integration_test_timeout: - integration_test_kwargs["timeout"] = integration_test_timeout + integration_test_base_kwargs["timeout"] = integration_test_timeout for test in native.glob(["tests/*.rs"], allow_empty = True): test_file_stem = test.removeprefix("tests/").removesuffix(".rs") @@ -317,6 +323,18 @@ def codex_rust_crate( test_name = name + "-" + test_file_stem.replace("/", "-") if not test_name.endswith("-test"): test_name += "-test" + extra_test_tags = integration_test_tags_extra_by_stem.get(test_file_stem, []) + flaky_tests = integration_test_flaky_by_stem.get(test_file_stem, []) + stem_integration_test_args = [] + integration_test_args + for flaky_test in flaky_tests: + stem_integration_test_args += [ + "--skip", + flaky_test, + ] + + integration_test_kwargs = dict(integration_test_base_kwargs) + if stem_integration_test_args: + integration_test_kwargs["args"] = stem_integration_test_args rust_test( name = test_name, @@ -338,6 +356,32 @@ def codex_rust_crate( # `INSTA_WORKSPACE_ROOT="codex-rs"` is tuned for unit tests that # execute from the repo root and can misplace integration snapshots. env = cargo_env, - tags = test_tags, + tags = test_tags + extra_test_tags, **integration_test_kwargs ) + + for flaky_test in flaky_tests: + flaky_test_suffix = flaky_test.replace("::", "-").replace("/", "-").replace("_", "-") + flaky_test_kwargs = dict(integration_test_base_kwargs) + flaky_test_kwargs["args"] = [ + "--exact", + flaky_test, + ] + + rust_test( + name = test_name.removesuffix("-test") + "-flaky-" + flaky_test_suffix + "-test", + crate_name = test_crate_name + "_flaky_" + flaky_test.replace("::", "_").replace("/", "_").replace("-", "_"), + crate_root = test, + srcs = [test], + data = native.glob(["tests/**"], allow_empty = True) + sanitized_binaries + test_data_extra, + compile_data = native.glob(["tests/**"], allow_empty = True) + integration_compile_data_extra, + deps = all_crate_deps(normal = True, normal_dev = True) + maybe_deps + deps_extra, + rustc_flags = rustc_flags_extra + WINDOWS_RUSTC_LINK_FLAGS + [ + "--remap-path-prefix=../codex-rs=", + "--remap-path-prefix=codex-rs=", + ], + rustc_env = rustc_env, + env = cargo_env, + tags = test_tags + extra_test_tags + ["flaky"], + **flaky_test_kwargs + ) From ab715021e602582e20df8ede0d9904240e69f065 Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Wed, 15 Apr 2026 13:27:17 -0700 Subject: [PATCH 36/84] Auto install start-codex-exec.sh dependencies (#17990) --- scripts/start-codex-exec.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/scripts/start-codex-exec.sh b/scripts/start-codex-exec.sh index 5699c13efd..8835d4c095 100755 --- a/scripts/start-codex-exec.sh +++ b/scripts/start-codex-exec.sh @@ -67,11 +67,7 @@ repo_root="$(git rev-parse --show-toplevel 2>/dev/null)" || { } ssh "${remote_host}" "mkdir -p ${remote_path}" - -if ! ssh "${remote_host}" 'command -v rsync >/dev/null 2>&1'; then - echo "remote rsync is required on ${remote_host}" >&2 - exit 1 -fi +ssh "${remote_host}" -C "sudo apt-get install rsync libcap-dev" sync_instance_id="$(date +%s)-$$" From 50d3128269ebfa6221d02ecfbbc46b0007643bea Mon Sep 17 00:00:00 2001 From: Tom Date: Wed, 15 Apr 2026 13:48:09 -0700 Subject: [PATCH 37/84] Migrate archive/unarchive to local ThreadStore (#17892) # Summary - implement local ThreadStore archive/unarchive operations - implement local ThreadStore read_thread operation - break up the various ThreadStore local method implementations into separate files - migrate app-server archive/unarchive and core archive fixture to use ThreadStore (but not all read operations yet!) - use the ThreadStore's read operation as a proxy check for thread persistence/existence in the app server code - move all other filesystem operations related to archive (path validation etc) into the local thread store. # Tests - add dedicated local store archive/unarchive tests --- .../app-server/src/codex_message_processor.rs | 329 ++---------- codex-rs/core/src/agent/control_tests.rs | 36 +- codex-rs/rollout/src/lib.rs | 1 + codex-rs/rollout/src/list.rs | 15 + .../thread-store/src/local/archive_thread.rs | 170 ++++++ codex-rs/thread-store/src/local/helpers.rs | 168 ++++++ .../thread-store/src/local/list_threads.rs | 319 +++++++++++ codex-rs/thread-store/src/local/mod.rs | 494 +----------------- .../thread-store/src/local/read_thread.rs | 138 +++++ .../thread-store/src/local/test_support.rs | 87 +++ .../src/local/unarchive_thread.rs | 198 +++++++ 11 files changed, 1179 insertions(+), 776 deletions(-) create mode 100644 codex-rs/thread-store/src/local/archive_thread.rs create mode 100644 codex-rs/thread-store/src/local/helpers.rs create mode 100644 codex-rs/thread-store/src/local/list_threads.rs create mode 100644 codex-rs/thread-store/src/local/read_thread.rs create mode 100644 codex-rs/thread-store/src/local/test_support.rs create mode 100644 codex-rs/thread-store/src/local/unarchive_thread.rs diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 05ce2d7974..42cb8b2d1a 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -244,7 +244,6 @@ use codex_core::plugins::load_plugin_apps; use codex_core::plugins::load_plugin_mcp_servers; use codex_core::read_head_for_summary; use codex_core::read_session_meta_line; -use codex_core::rollout_date_parts; use codex_core::sandboxing::SandboxPermissions; use codex_core::windows_sandbox::WindowsSandboxLevelExt; use codex_core::windows_sandbox::WindowsSandboxSetupMode as CoreWindowsSandboxSetupMode; @@ -319,8 +318,10 @@ use codex_state::StateRuntime; use codex_state::ThreadMetadata; use codex_state::ThreadMetadataBuilder; use codex_state::log_db::LogDbLayer; +use codex_thread_store::ArchiveThreadParams as StoreArchiveThreadParams; use codex_thread_store::ListThreadsParams as StoreListThreadsParams; use codex_thread_store::LocalThreadStore; +use codex_thread_store::ReadThreadParams as StoreReadThreadParams; use codex_thread_store::StoredThread; use codex_thread_store::ThreadSortKey as StoreThreadSortKey; use codex_thread_store::ThreadStore; @@ -331,9 +332,6 @@ use codex_utils_pty::DEFAULT_OUTPUT_BYTES_CAP; use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::HashSet; -use std::ffi::OsStr; -use std::fs::FileTimes; -use std::fs::OpenOptions; use std::io::Error as IoError; use std::path::Path; use std::path::PathBuf; @@ -343,7 +341,6 @@ use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; use std::time::Duration; use std::time::Instant; -use std::time::SystemTime; use tokio::sync::Mutex; use tokio::sync::broadcast; use tokio::sync::oneshot; @@ -447,6 +444,7 @@ pub(crate) struct CodexMessageProcessor { analytics_events_client: AnalyticsEventsClient, arg0_paths: Arg0DispatchPaths, config: Arc, + thread_store: LocalThreadStore, cli_overrides: Arc>>, runtime_feature_enablement: Arc>>, cloud_requirements: Arc>, @@ -705,6 +703,7 @@ impl CodexMessageProcessor { outgoing: outgoing.clone(), analytics_events_client, arg0_paths, + thread_store: LocalThreadStore::new(codex_rollout::RolloutConfig::from_view(&config)), config, cli_overrides, runtime_feature_enablement, @@ -2717,7 +2716,6 @@ impl CodexMessageProcessor { } async fn thread_archive(&self, request_id: ConnectionRequestId, params: ThreadArchiveParams) { - // TODO(jif) mostly rewrite this using sqlite after phase 1 let thread_id = match ThreadId::from_string(¶ms.thread_id) { Ok(id) => id, Err(err) => { @@ -2731,32 +2729,28 @@ impl CodexMessageProcessor { } }; - let rollout_path = - match find_thread_path_by_id_str(&self.config.codex_home, &thread_id.to_string()).await - { - Ok(Some(p)) => p, - Ok(None) => { - let error = JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!("no rollout found for thread id {thread_id}"), - data: None, - }; - self.outgoing.send_error(request_id, error).await; - return; - } - Err(err) => { - let error = JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!("failed to locate thread id {thread_id}: {err}"), - data: None, - }; - self.outgoing.send_error(request_id, error).await; - return; - } - }; - let thread_id_str = thread_id.to_string(); - match self.archive_thread_common(thread_id, &rollout_path).await { + if let Err(err) = self + .thread_store + .read_thread(StoreReadThreadParams { + thread_id, + include_archived: false, + include_history: false, + }) + .await + { + self.outgoing + .send_error(request_id, thread_store_archive_error("archive", err)) + .await; + return; + } + self.prepare_thread_for_archive(thread_id).await; + + match self + .thread_store + .archive_thread(StoreArchiveThreadParams { thread_id }) + .await + { Ok(()) => { let response = ThreadArchiveResponse {}; self.outgoing.send_response(request_id, response).await; @@ -2768,7 +2762,9 @@ impl CodexMessageProcessor { .await; } Err(err) => { - self.outgoing.send_error(request_id, err).await; + self.outgoing + .send_error(request_id, thread_store_archive_error("archive", err)) + .await; } } } @@ -3437,7 +3433,6 @@ impl CodexMessageProcessor { request_id: ConnectionRequestId, params: ThreadUnarchiveParams, ) { - // TODO(jif) mostly rewrite this using sqlite after phase 1 let thread_id = match ThreadId::from_string(¶ms.thread_id) { Ok(id) => id, Err(err) => { @@ -3451,152 +3446,21 @@ impl CodexMessageProcessor { } }; - let archived_path = match find_archived_thread_path_by_id_str( - &self.config.codex_home, - &thread_id.to_string(), - ) - .await - { - Ok(Some(path)) => path, - Ok(None) => { - let error = JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!("no archived rollout found for thread id {thread_id}"), - data: None, - }; - self.outgoing.send_error(request_id, error).await; - return; - } - Err(err) => { - let error = JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!("failed to locate archived thread id {thread_id}: {err}"), - data: None, - }; - self.outgoing.send_error(request_id, error).await; - return; - } - }; - - let rollout_path_display = archived_path.display().to_string(); let fallback_provider = self.config.model_provider_id.clone(); - let state_db_ctx = get_state_db(&self.config).await; - let archived_folder = self - .config - .codex_home - .join(codex_core::ARCHIVED_SESSIONS_SUBDIR); - - let result: Result = async { - let canonical_archived_dir = tokio::fs::canonicalize(&archived_folder).await.map_err( - |err| JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!( - "failed to unarchive thread: unable to resolve archived directory: {err}" - ), - data: None, - }, - )?; - let canonical_rollout_path = tokio::fs::canonicalize(&archived_path).await; - let canonical_rollout_path = if let Ok(path) = canonical_rollout_path - && path.starts_with(&canonical_archived_dir) - { - path - } else { - return Err(JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!( - "rollout path `{rollout_path_display}` must be in archived directory" - ), - data: None, - }); - }; - - let required_suffix = format!("{thread_id}.jsonl"); - let Some(file_name) = canonical_rollout_path.file_name().map(OsStr::to_owned) else { - return Err(JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!("rollout path `{rollout_path_display}` missing file name"), - data: None, - }); - }; - if !file_name - .to_string_lossy() - .ends_with(required_suffix.as_str()) - { - return Err(JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!( - "rollout path `{rollout_path_display}` does not match thread id {thread_id}" - ), - data: None, - }); - } - - let Some((year, month, day)) = rollout_date_parts(&file_name) else { - return Err(JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!( - "rollout path `{rollout_path_display}` missing filename timestamp" - ), - data: None, - }); - }; - - let sessions_folder = self.config.codex_home.join(codex_core::SESSIONS_SUBDIR); - let dest_dir = sessions_folder.join(year).join(month).join(day); - let restored_path = dest_dir.join(&file_name); - tokio::fs::create_dir_all(&dest_dir) - .await - .map_err(|err| JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!("failed to unarchive thread: {err}"), - data: None, - })?; - tokio::fs::rename(&canonical_rollout_path, &restored_path) - .await - .map_err(|err| JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!("failed to unarchive thread: {err}"), - data: None, - })?; - tokio::task::spawn_blocking({ - let restored_path = restored_path.clone(); - move || -> std::io::Result<()> { - let times = FileTimes::new().set_modified(SystemTime::now()); - OpenOptions::new() - .append(true) - .open(&restored_path)? - .set_times(times)?; - Ok(()) - } - }) + let result = self + .thread_store + .unarchive_thread(StoreArchiveThreadParams { thread_id }) .await - .map_err(|err| JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!("failed to update unarchived thread timestamp: {err}"), - data: None, - })? - .map_err(|err| JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!("failed to update unarchived thread timestamp: {err}"), - data: None, - })?; - if let Some(ctx) = state_db_ctx { - let _ = ctx - .mark_unarchived(thread_id, restored_path.as_path()) - .await; - } - let summary = - read_summary_from_rollout(restored_path.as_path(), fallback_provider.as_str()) - .await - .map_err(|err| JSONRPCErrorError { + .map_err(|err| thread_store_archive_error("unarchive", err)) + .and_then(|stored_thread| { + summary_from_stored_thread(stored_thread, fallback_provider.as_str()) + .map(|summary| summary_to_thread(summary, &self.config.cwd)) + .ok_or_else(|| JSONRPCErrorError { code: INTERNAL_ERROR_CODE, - message: format!("failed to read unarchived thread: {err}"), + message: format!("failed to read unarchived thread {thread_id}"), data: None, - })?; - Ok(summary_to_thread(summary, &self.config.cwd)) - } - .await; + }) + }); match result { Ok(mut thread) => { @@ -5103,11 +4967,11 @@ impl CodexMessageProcessor { let fallback_provider = self.config.model_provider_id.clone(); let (allowed_sources_vec, source_kind_filter) = compute_source_filters(source_kinds); let allowed_sources = allowed_sources_vec.as_slice(); - let store = LocalThreadStore::new(codex_rollout::RolloutConfig::from_view(&self.config)); while remaining > 0 { let page_size = remaining.min(THREAD_LIST_MAX_LIMIT); - let page = store + let page = self + .thread_store .list_threads(StoreListThreadsParams { page_size, cursor: cursor_obj.clone(), @@ -5944,76 +5808,10 @@ impl CodexMessageProcessor { .await; } - async fn archive_thread_common( - &self, - thread_id: ThreadId, - rollout_path: &Path, - ) -> Result<(), JSONRPCErrorError> { - // Verify rollout_path is under sessions dir. - let rollout_folder = self.config.codex_home.join(codex_core::SESSIONS_SUBDIR); - - let canonical_sessions_dir = match tokio::fs::canonicalize(&rollout_folder).await { - Ok(path) => path, - Err(err) => { - return Err(JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!( - "failed to archive thread: unable to resolve sessions directory: {err}" - ), - data: None, - }); - } - }; - let canonical_rollout_path = tokio::fs::canonicalize(rollout_path).await; - let canonical_rollout_path = if let Ok(path) = canonical_rollout_path - && path.starts_with(&canonical_sessions_dir) - { - path - } else { - return Err(JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!( - "rollout path `{}` must be in sessions directory", - rollout_path.display() - ), - data: None, - }); - }; - - // Verify file name matches thread id. - let required_suffix = format!("{thread_id}.jsonl"); - let Some(file_name) = canonical_rollout_path.file_name().map(OsStr::to_owned) else { - return Err(JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!( - "rollout path `{}` missing file name", - rollout_path.display() - ), - data: None, - }); - }; - if !file_name - .to_string_lossy() - .ends_with(required_suffix.as_str()) - { - return Err(JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!( - "rollout path `{}` does not match thread id {thread_id}", - rollout_path.display() - ), - data: None, - }); - } - - let mut state_db_ctx = None; - + async fn prepare_thread_for_archive(&self, thread_id: ThreadId) { // If the thread is active, request shutdown and wait briefly. let removed_conversation = self.thread_manager.remove_thread(&thread_id).await; if let Some(conversation) = removed_conversation { - if let Some(ctx) = conversation.state_db() { - state_db_ctx = Some(ctx); - } info!("thread {thread_id} was active; shutting down"); match Self::wait_for_thread_shutdown(&conversation).await { ThreadShutdownResult::Complete => {} @@ -6028,34 +5826,6 @@ impl CodexMessageProcessor { } } self.finalize_thread_teardown(thread_id).await; - - if state_db_ctx.is_none() { - state_db_ctx = get_state_db(&self.config).await; - } - - // Move the rollout file to archived. - let result: std::io::Result<()> = async move { - let archive_folder = self - .config - .codex_home - .join(codex_core::ARCHIVED_SESSIONS_SUBDIR); - tokio::fs::create_dir_all(&archive_folder).await?; - let archived_path = archive_folder.join(&file_name); - tokio::fs::rename(&canonical_rollout_path, &archived_path).await?; - if let Some(ctx) = state_db_ctx { - let _ = ctx - .mark_archived(thread_id, archived_path.as_path(), Utc::now()) - .await; - } - Ok(()) - } - .await; - - result.map_err(|err| JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!("failed to archive thread: {err}"), - data: None, - }) } async fn apps_list(&self, request_id: ConnectionRequestId, params: AppsListParams) { @@ -9398,6 +9168,21 @@ fn thread_store_list_error(err: ThreadStoreError) -> JSONRPCErrorError { } } +fn thread_store_archive_error(operation: &str, err: ThreadStoreError) -> JSONRPCErrorError { + match err { + ThreadStoreError::InvalidRequest { message } => JSONRPCErrorError { + code: INVALID_REQUEST_ERROR_CODE, + message, + data: None, + }, + err => JSONRPCErrorError { + code: INTERNAL_ERROR_CODE, + message: format!("failed to {operation} thread: {err}"), + data: None, + }, + } +} + fn summary_from_stored_thread( thread: StoredThread, fallback_provider: &str, diff --git a/codex-rs/core/src/agent/control_tests.rs b/codex-rs/core/src/agent/control_tests.rs index 6fc74b30e8..79a96d11e5 100644 --- a/codex-rs/core/src/agent/control_tests.rs +++ b/codex-rs/core/src/agent/control_tests.rs @@ -7,7 +7,6 @@ use crate::config::Config; use crate::config::ConfigBuilder; use crate::contextual_user_message::SUBAGENT_NOTIFICATION_OPEN_TAG; use assert_matches::assert_matches; -use chrono::Utc; use codex_features::Feature; use codex_login::CodexAuth; use codex_protocol::AgentPath; @@ -24,6 +23,9 @@ use codex_protocol::protocol::TurnAbortReason; use codex_protocol::protocol::TurnAbortedEvent; use codex_protocol::protocol::TurnCompleteEvent; use codex_protocol::protocol::TurnStartedEvent; +use codex_thread_store::ArchiveThreadParams; +use codex_thread_store::LocalThreadStore; +use codex_thread_store::ThreadStore; use pretty_assertions::assert_eq; use tempfile::TempDir; use tokio::time::Duration; @@ -1658,38 +1660,18 @@ async fn resume_agent_from_rollout_reads_archived_rollout_path() { .await .expect("child thread should exist"); persist_thread_for_tree_resume(&child_thread, "persist before archiving").await; - let rollout_path = child_thread - .rollout_path() - .expect("thread should have rollout path"); - let state_db = child_thread - .state_db() - .expect("thread should have state db handle"); - let _ = harness .control .shutdown_live_agent(child_thread_id) .await .expect("child shutdown should succeed"); - - let archived_root = harness - .config - .codex_home - .join(crate::ARCHIVED_SESSIONS_SUBDIR); - tokio::fs::create_dir_all(&archived_root) + let store = LocalThreadStore::new(codex_rollout::RolloutConfig::from_view(&harness.config)); + store + .archive_thread(ArchiveThreadParams { + thread_id: child_thread_id, + }) .await - .expect("archived root should exist"); - let archived_rollout_path = archived_root.join( - rollout_path - .file_name() - .expect("rollout file name should be present"), - ); - tokio::fs::rename(&rollout_path, &archived_rollout_path) - .await - .expect("rollout should move to archived path"); - state_db - .mark_archived(child_thread_id, archived_rollout_path.as_path(), Utc::now()) - .await - .expect("state db archive update should succeed"); + .expect("child thread should archive"); let resumed_thread_id = harness .control diff --git a/codex-rs/rollout/src/lib.rs b/codex-rs/rollout/src/lib.rs index 31fcf47240..de169db1f7 100644 --- a/codex-rs/rollout/src/lib.rs +++ b/codex-rs/rollout/src/lib.rs @@ -48,6 +48,7 @@ pub use list::get_threads_in_root; pub use list::parse_cursor; pub use list::read_head_for_summary; pub use list::read_session_meta_line; +pub use list::read_thread_item_from_rollout; pub use list::rollout_date_parts; pub use metadata::builder_from_items; pub use policy::EventPersistenceMode; diff --git a/codex-rs/rollout/src/list.rs b/codex-rs/rollout/src/list.rs index afa4103307..640f0b0278 100644 --- a/codex-rs/rollout/src/list.rs +++ b/codex-rs/rollout/src/list.rs @@ -756,6 +756,21 @@ async fn build_thread_item( None } +/// Read a single rollout file into the same summary item shape used by thread listing. +/// +/// This is for callers that already resolved a rollout path and need the same +/// metadata/preview extraction as list operations without scanning the whole +/// sessions tree. +pub async fn read_thread_item_from_rollout(path: PathBuf) -> Option { + build_thread_item( + path, + &[], + /*provider_matcher*/ None, + /*updated_at*/ None, + ) + .await +} + /// Collects immediate subdirectories of `parent`, parses their (string) names with `parse`, /// and returns them sorted descending by the parsed key. async fn collect_dirs_desc(parent: &Path, parse: F) -> io::Result> diff --git a/codex-rs/thread-store/src/local/archive_thread.rs b/codex-rs/thread-store/src/local/archive_thread.rs new file mode 100644 index 0000000000..4fcea6dbac --- /dev/null +++ b/codex-rs/thread-store/src/local/archive_thread.rs @@ -0,0 +1,170 @@ +use chrono::Utc; +use codex_rollout::find_thread_path_by_id_str; + +use super::LocalThreadStore; +use super::helpers::matching_rollout_file_name; +use super::helpers::scoped_rollout_path; +use crate::ArchiveThreadParams; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; + +pub(super) async fn archive_thread( + store: &LocalThreadStore, + params: ArchiveThreadParams, +) -> ThreadStoreResult<()> { + let thread_id = params.thread_id; + let rollout_path = + find_thread_path_by_id_str(store.config.codex_home.as_path(), &thread_id.to_string()) + .await + .map_err(|err| ThreadStoreError::InvalidRequest { + message: format!("failed to locate thread id {thread_id}: {err}"), + })? + .ok_or_else(|| ThreadStoreError::InvalidRequest { + message: format!("no rollout found for thread id {thread_id}"), + })?; + + let canonical_rollout_path = scoped_rollout_path( + store.config.codex_home.join(codex_rollout::SESSIONS_SUBDIR), + rollout_path.as_path(), + "sessions", + )?; + let file_name = matching_rollout_file_name( + canonical_rollout_path.as_path(), + thread_id, + rollout_path.as_path(), + )?; + + let archive_folder = store + .config + .codex_home + .join(codex_rollout::ARCHIVED_SESSIONS_SUBDIR); + std::fs::create_dir_all(&archive_folder).map_err(|err| ThreadStoreError::Internal { + message: format!("failed to archive thread: {err}"), + })?; + let archived_path = archive_folder.join(&file_name); + std::fs::rename(&canonical_rollout_path, &archived_path).map_err(|err| { + ThreadStoreError::Internal { + message: format!("failed to archive thread: {err}"), + } + })?; + + if let Some(ctx) = codex_rollout::state_db::get_state_db(&store.config).await { + let _ = ctx + .mark_archived(thread_id, archived_path.as_path(), Utc::now()) + .await; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use chrono::Utc; + use codex_protocol::ThreadId; + use codex_protocol::protocol::SessionSource; + use codex_rollout::ARCHIVED_SESSIONS_SUBDIR; + use pretty_assertions::assert_eq; + use tempfile::TempDir; + use uuid::Uuid; + + use super::*; + use crate::ListThreadsParams; + use crate::ThreadSortKey; + use crate::ThreadStore; + use crate::local::LocalThreadStore; + use crate::local::test_support::test_config; + use crate::local::test_support::write_session_file; + + #[tokio::test] + async fn archive_thread_moves_rollout_to_archived_collection() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path())); + let uuid = Uuid::from_u128(201); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + let active_path = + write_session_file(home.path(), "2025-01-03T12-00-00", uuid).expect("session file"); + + store + .archive_thread(ArchiveThreadParams { thread_id }) + .await + .expect("archive thread"); + + assert!(!active_path.exists()); + let archived_path = home + .path() + .join(ARCHIVED_SESSIONS_SUBDIR) + .join(active_path.file_name().expect("file name")); + assert!(archived_path.exists()); + + let archived = store + .list_threads(ListThreadsParams { + page_size: 10, + cursor: None, + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived: true, + search_term: None, + }) + .await + .expect("archived listing"); + assert_eq!(archived.items.len(), 1); + assert_eq!(archived.items[0].thread_id, thread_id); + assert_eq!(archived.items[0].rollout_path, Some(archived_path)); + assert_eq!( + archived.items[0].archived_at, + Some(archived.items[0].updated_at) + ); + } + + #[tokio::test] + async fn archive_thread_updates_sqlite_metadata_when_present() { + let home = TempDir::new().expect("temp dir"); + let config = test_config(home.path()); + let store = LocalThreadStore::new(config.clone()); + let uuid = Uuid::from_u128(202); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + let active_path = + write_session_file(home.path(), "2025-01-03T12-00-00", uuid).expect("session file"); + let runtime = codex_state::StateRuntime::init( + home.path().to_path_buf(), + config.model_provider_id.clone(), + ) + .await + .expect("state db should initialize"); + runtime + .mark_backfill_complete(/*last_watermark*/ None) + .await + .expect("backfill should be complete"); + let mut builder = codex_state::ThreadMetadataBuilder::new( + thread_id, + active_path.clone(), + Utc::now(), + SessionSource::Cli, + ); + builder.model_provider = Some(config.model_provider_id.clone()); + builder.cwd = home.path().to_path_buf(); + builder.cli_version = Some("test_version".to_string()); + let metadata = builder.build(config.model_provider_id.as_str()); + runtime + .upsert_thread(&metadata) + .await + .expect("state db upsert should succeed"); + + store + .archive_thread(ArchiveThreadParams { thread_id }) + .await + .expect("archive thread"); + + let archived_path = home + .path() + .join(ARCHIVED_SESSIONS_SUBDIR) + .join(active_path.file_name().expect("file name")); + let updated = runtime + .get_thread(thread_id) + .await + .expect("state db read should succeed") + .expect("thread metadata should exist"); + assert_eq!(updated.rollout_path, archived_path); + assert!(updated.archived_at.is_some()); + } +} diff --git a/codex-rs/thread-store/src/local/helpers.rs b/codex-rs/thread-store/src/local/helpers.rs new file mode 100644 index 0000000000..e28ecc6612 --- /dev/null +++ b/codex-rs/thread-store/src/local/helpers.rs @@ -0,0 +1,168 @@ +use std::ffi::OsStr; +use std::fs::FileTimes; +use std::fs::OpenOptions; +use std::path::Path; +use std::path::PathBuf; +use std::time::SystemTime; + +use chrono::DateTime; +use chrono::Utc; +use codex_git_utils::GitSha; +use codex_protocol::ThreadId; +use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::GitInfo; +use codex_protocol::protocol::SandboxPolicy; +use codex_protocol::protocol::SessionSource; +use codex_rollout::ThreadItem; + +use crate::StoredThread; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; + +pub(super) fn scoped_rollout_path( + root: PathBuf, + rollout_path: &Path, + root_name: &str, +) -> ThreadStoreResult { + let canonical_root = + std::fs::canonicalize(&root).map_err(|err| ThreadStoreError::Internal { + message: format!( + "failed to resolve {root_name} directory `{}`: {err}", + root.display() + ), + })?; + let canonical_rollout_path = + std::fs::canonicalize(rollout_path).map_err(|_| ThreadStoreError::InvalidRequest { + message: format!( + "rollout path `{}` must be in {root_name} directory", + rollout_path.display() + ), + })?; + if canonical_rollout_path.starts_with(&canonical_root) { + Ok(canonical_rollout_path) + } else { + Err(ThreadStoreError::InvalidRequest { + message: format!( + "rollout path `{}` must be in {root_name} directory", + rollout_path.display() + ), + }) + } +} + +pub(super) fn matching_rollout_file_name( + rollout_path: &Path, + thread_id: ThreadId, + display_path: &Path, +) -> ThreadStoreResult { + let Some(file_name) = rollout_path.file_name().map(OsStr::to_owned) else { + return Err(ThreadStoreError::InvalidRequest { + message: format!( + "rollout path `{}` missing file name", + display_path.display() + ), + }); + }; + let required_suffix = format!("{thread_id}.jsonl"); + if file_name + .to_string_lossy() + .ends_with(required_suffix.as_str()) + { + Ok(file_name) + } else { + Err(ThreadStoreError::InvalidRequest { + message: format!( + "rollout path `{}` does not match thread id {thread_id}", + display_path.display() + ), + }) + } +} + +pub(super) fn touch_modified_time(path: &Path) -> std::io::Result<()> { + let times = FileTimes::new().set_modified(SystemTime::now()); + OpenOptions::new().append(true).open(path)?.set_times(times) +} + +pub(super) fn stored_thread_from_rollout_item( + item: ThreadItem, + archived: bool, + default_provider: &str, +) -> Option { + let thread_id = item + .thread_id + .or_else(|| thread_id_from_rollout_path(item.path.as_path()))?; + let created_at = parse_rfc3339(item.created_at.as_deref()).unwrap_or_else(Utc::now); + let updated_at = parse_rfc3339(item.updated_at.as_deref()).unwrap_or(created_at); + let archived_at = archived.then_some(updated_at); + let git_info = git_info_from_parts( + item.git_sha.clone(), + item.git_branch.clone(), + item.git_origin_url.clone(), + ); + let source = item.source.unwrap_or(SessionSource::Unknown); + let preview = item.first_user_message.clone().unwrap_or_default(); + + Some(StoredThread { + thread_id, + rollout_path: Some(item.path), + forked_from_id: None, + preview, + name: None, + model_provider: item + .model_provider + .filter(|provider| !provider.is_empty()) + .unwrap_or_else(|| default_provider.to_string()), + model: None, + reasoning_effort: None, + created_at, + updated_at, + archived_at, + cwd: item.cwd.unwrap_or_default(), + cli_version: item.cli_version.unwrap_or_default(), + source, + agent_nickname: item.agent_nickname, + agent_role: item.agent_role, + agent_path: None, + git_info, + approval_mode: AskForApproval::OnRequest, + sandbox_policy: SandboxPolicy::new_read_only_policy(), + token_usage: None, + first_user_message: item.first_user_message, + history: None, + }) +} + +fn parse_rfc3339(value: Option<&str>) -> Option> { + DateTime::parse_from_rfc3339(value?) + .ok() + .map(|dt| dt.with_timezone(&Utc)) +} + +fn git_info_from_parts( + sha: Option, + branch: Option, + origin_url: Option, +) -> Option { + if sha.is_none() && branch.is_none() && origin_url.is_none() { + return None; + } + Some(GitInfo { + commit_hash: sha.as_deref().map(GitSha::new), + branch, + repository_url: origin_url, + }) +} + +fn thread_id_from_rollout_path(path: &Path) -> Option { + let file_name = path.file_name()?.to_str()?; + let stem = file_name.strip_suffix(".jsonl")?; + if stem.len() < 37 { + return None; + } + let uuid_start = stem.len().saturating_sub(36); + if !stem[..uuid_start].ends_with('-') { + return None; + } + ThreadId::from_string(&stem[uuid_start..]).ok() +} diff --git a/codex-rs/thread-store/src/local/list_threads.rs b/codex-rs/thread-store/src/local/list_threads.rs new file mode 100644 index 0000000000..35c6f3de43 --- /dev/null +++ b/codex-rs/thread-store/src/local/list_threads.rs @@ -0,0 +1,319 @@ +use codex_rollout::RolloutConfig; +use codex_rollout::RolloutRecorder; +use codex_rollout::parse_cursor; + +use super::LocalThreadStore; +use super::helpers::stored_thread_from_rollout_item; +use crate::ListThreadsParams; +use crate::ThreadPage; +use crate::ThreadSortKey; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; + +pub(super) async fn list_threads( + store: &LocalThreadStore, + params: ListThreadsParams, +) -> ThreadStoreResult { + let cursor = params + .cursor + .as_deref() + .map(|cursor| { + parse_cursor(cursor).ok_or_else(|| ThreadStoreError::InvalidRequest { + message: format!("invalid cursor: {cursor}"), + }) + }) + .transpose()?; + let sort_key = match params.sort_key { + ThreadSortKey::CreatedAt => codex_rollout::ThreadSortKey::CreatedAt, + ThreadSortKey::UpdatedAt => codex_rollout::ThreadSortKey::UpdatedAt, + }; + let page = list_rollout_threads(&store.config, ¶ms, cursor.as_ref(), sort_key).await?; + + let next_cursor = page + .next_cursor + .as_ref() + .and_then(|cursor| serde_json::to_value(cursor).ok()) + .and_then(|value| value.as_str().map(str::to_owned)); + let items = page + .items + .into_iter() + .filter_map(|item| { + stored_thread_from_rollout_item( + item, + params.archived, + store.config.model_provider_id.as_str(), + ) + }) + .collect::>(); + + Ok(ThreadPage { items, next_cursor }) +} + +async fn list_rollout_threads( + config: &RolloutConfig, + params: &ListThreadsParams, + cursor: Option<&codex_rollout::Cursor>, + sort_key: codex_rollout::ThreadSortKey, +) -> ThreadStoreResult { + let page = if params.archived { + RolloutRecorder::list_archived_threads( + config, + params.page_size, + cursor, + sort_key, + params.allowed_sources.as_slice(), + params.model_providers.as_deref(), + config.model_provider_id.as_str(), + params.search_term.as_deref(), + ) + .await + } else { + RolloutRecorder::list_threads( + config, + params.page_size, + cursor, + sort_key, + params.allowed_sources.as_slice(), + params.model_providers.as_deref(), + config.model_provider_id.as_str(), + params.search_term.as_deref(), + ) + .await + }; + page.map_err(|err| ThreadStoreError::Internal { + message: format!("failed to list threads: {err}"), + }) +} + +#[cfg(test)] +mod tests { + use chrono::Utc; + use codex_protocol::ThreadId; + use codex_protocol::protocol::SessionSource; + use pretty_assertions::assert_eq; + use std::fs; + use tempfile::TempDir; + use uuid::Uuid; + + use super::*; + use crate::ThreadStore; + use crate::local::LocalThreadStore; + use crate::local::test_support::test_config; + use crate::local::test_support::write_archived_session_file; + use crate::local::test_support::write_session_file; + use crate::local::test_support::write_session_file_with; + + #[tokio::test] + async fn list_threads_uses_default_provider_when_rollout_omits_provider() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path())); + write_session_file_with( + home.path(), + home.path().join("sessions/2025/01/03"), + "2025-01-03T12-00-00", + Uuid::from_u128(102), + "Hello from user", + /*model_provider*/ None, + ) + .expect("session file"); + + let page = store + .list_threads(ListThreadsParams { + page_size: 10, + cursor: None, + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived: false, + search_term: None, + }) + .await + .expect("thread listing"); + + assert_eq!(page.items.len(), 1); + assert_eq!(page.items[0].model_provider, "test-provider"); + } + + #[tokio::test] + async fn list_threads_preserves_sqlite_title_search_results() { + let home = TempDir::new().expect("temp dir"); + let config = test_config(home.path()); + let store = LocalThreadStore::new(config.clone()); + let uuid = Uuid::from_u128(103); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + let rollout_path = home.path().join("rollout-title-search.jsonl"); + fs::write(&rollout_path, "").expect("placeholder rollout file"); + + let runtime = codex_state::StateRuntime::init( + home.path().to_path_buf(), + config.model_provider_id.clone(), + ) + .await + .expect("state db should initialize"); + runtime + .mark_backfill_complete(/*last_watermark*/ None) + .await + .expect("backfill should be complete"); + let created_at = Utc::now(); + let mut builder = codex_state::ThreadMetadataBuilder::new( + thread_id, + rollout_path, + created_at, + SessionSource::Cli, + ); + builder.model_provider = Some(config.model_provider_id.clone()); + builder.cwd = home.path().to_path_buf(); + builder.cli_version = Some("test_version".to_string()); + let mut metadata = builder.build(config.model_provider_id.as_str()); + metadata.title = "needle title".to_string(); + metadata.first_user_message = Some("plain preview".to_string()); + runtime + .upsert_thread(&metadata) + .await + .expect("state db upsert should succeed"); + + let page = store + .list_threads(ListThreadsParams { + page_size: 10, + cursor: None, + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived: false, + search_term: Some("needle".to_string()), + }) + .await + .expect("thread listing"); + + let ids = page + .items + .iter() + .map(|item| item.thread_id) + .collect::>(); + assert_eq!(ids, vec![thread_id]); + assert_eq!( + page.items[0].first_user_message.as_deref(), + Some("plain preview") + ); + } + + #[tokio::test] + async fn list_threads_selects_active_or_archived_collection() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path())); + let active_uuid = Uuid::from_u128(105); + let archived_uuid = Uuid::from_u128(106); + write_session_file(home.path(), "2025-01-03T12-00-00", active_uuid) + .expect("active session file"); + write_archived_session_file(home.path(), "2025-01-03T13-00-00", archived_uuid) + .expect("archived session file"); + + let active = store + .list_threads(ListThreadsParams { + page_size: 10, + cursor: None, + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived: false, + search_term: None, + }) + .await + .expect("active listing"); + let archived = store + .list_threads(ListThreadsParams { + page_size: 10, + cursor: None, + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived: true, + search_term: None, + }) + .await + .expect("archived listing"); + + let active_id = ThreadId::from_string(&active_uuid.to_string()).expect("valid thread id"); + let archived_id = + ThreadId::from_string(&archived_uuid.to_string()).expect("valid thread id"); + assert_eq!( + active + .items + .iter() + .map(|item| item.thread_id) + .collect::>(), + vec![active_id] + ); + assert_eq!( + archived + .items + .iter() + .map(|item| item.thread_id) + .collect::>(), + vec![archived_id] + ); + assert_eq!(active.items[0].archived_at, None); + assert_eq!( + archived.items[0].archived_at, + Some(archived.items[0].updated_at) + ); + } + + #[tokio::test] + async fn list_threads_returns_local_rollout_summary() { + let home = TempDir::new().expect("temp dir"); + let config = test_config(home.path()); + let store = LocalThreadStore::new(config); + let uuid = Uuid::from_u128(101); + let path = + write_session_file(home.path(), "2025-01-03T12-00-00", uuid).expect("session file"); + + let page = store + .list_threads(ListThreadsParams { + page_size: 10, + cursor: None, + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: vec![SessionSource::Cli], + model_providers: Some(vec!["test-provider".to_string()]), + archived: false, + search_term: None, + }) + .await + .expect("thread listing"); + + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + assert_eq!(page.next_cursor, None); + assert_eq!(page.items.len(), 1); + assert_eq!(page.items[0].thread_id, thread_id); + assert_eq!(page.items[0].rollout_path, Some(path)); + assert_eq!(page.items[0].preview, "Hello from user"); + assert_eq!( + page.items[0].first_user_message.as_deref(), + Some("Hello from user") + ); + assert_eq!(page.items[0].model_provider, "test-provider"); + assert_eq!(page.items[0].cli_version, "test_version"); + assert_eq!(page.items[0].source, SessionSource::Cli); + } + + #[tokio::test] + async fn list_threads_rejects_invalid_cursor() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path())); + + let err = store + .list_threads(ListThreadsParams { + page_size: 10, + cursor: Some("not-a-cursor".to_string()), + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived: false, + search_term: None, + }) + .await + .expect_err("invalid cursor should fail"); + + assert!(matches!(err, ThreadStoreError::InvalidRequest { .. })); + } +} diff --git a/codex-rs/thread-store/src/local/mod.rs b/codex-rs/thread-store/src/local/mod.rs index cc15033cd9..81428cac8c 100644 --- a/codex-rs/thread-store/src/local/mod.rs +++ b/codex-rs/thread-store/src/local/mod.rs @@ -1,15 +1,14 @@ +mod archive_thread; +mod helpers; +mod list_threads; +mod read_thread; +mod unarchive_thread; + +#[cfg(test)] +mod test_support; + use async_trait::async_trait; -use chrono::DateTime; -use chrono::Utc; -use codex_git_utils::GitSha; -use codex_protocol::protocol::AskForApproval; -use codex_protocol::protocol::GitInfo; -use codex_protocol::protocol::SandboxPolicy; -use codex_protocol::protocol::SessionSource; use codex_rollout::RolloutConfig; -use codex_rollout::RolloutRecorder; -use codex_rollout::ThreadItem; -use codex_rollout::parse_cursor; use crate::AppendThreadItemsParams; use crate::ArchiveThreadParams; @@ -23,7 +22,6 @@ use crate::StoredThread; use crate::StoredThreadHistory; use crate::ThreadPage; use crate::ThreadRecorder; -use crate::ThreadSortKey; use crate::ThreadStore; use crate::ThreadStoreError; use crate::ThreadStoreResult; @@ -32,7 +30,7 @@ use crate::UpdateThreadMetadataParams; /// Local filesystem/SQLite-backed implementation of [`ThreadStore`]. #[derive(Clone, Debug)] pub struct LocalThreadStore { - config: RolloutConfig, + pub(super) config: RolloutConfig, } impl LocalThreadStore { @@ -69,44 +67,12 @@ impl ThreadStore for LocalThreadStore { unsupported("load_history") } - async fn read_thread(&self, _params: ReadThreadParams) -> ThreadStoreResult { - unsupported("read_thread") + async fn read_thread(&self, params: ReadThreadParams) -> ThreadStoreResult { + read_thread::read_thread(self, params).await } async fn list_threads(&self, params: ListThreadsParams) -> ThreadStoreResult { - let cursor = params - .cursor - .as_deref() - .map(|cursor| { - parse_cursor(cursor).ok_or_else(|| ThreadStoreError::InvalidRequest { - message: format!("invalid cursor: {cursor}"), - }) - }) - .transpose()?; - let sort_key = match params.sort_key { - ThreadSortKey::CreatedAt => codex_rollout::ThreadSortKey::CreatedAt, - ThreadSortKey::UpdatedAt => codex_rollout::ThreadSortKey::UpdatedAt, - }; - let page = list_rollout_threads(&self.config, ¶ms, cursor.as_ref(), sort_key).await?; - - let next_cursor = page - .next_cursor - .as_ref() - .and_then(|cursor| serde_json::to_value(cursor).ok()) - .and_then(|value| value.as_str().map(str::to_owned)); - let items = page - .items - .into_iter() - .filter_map(|item| { - stored_thread_from_rollout_item( - item, - params.archived, - self.config.model_provider_id.as_str(), - ) - }) - .collect::>(); - - Ok(ThreadPage { items, next_cursor }) + list_threads::list_threads(self, params).await } async fn set_thread_name(&self, _params: SetThreadNameParams) -> ThreadStoreResult<()> { @@ -120,15 +86,15 @@ impl ThreadStore for LocalThreadStore { unsupported("update_thread_metadata") } - async fn archive_thread(&self, _params: ArchiveThreadParams) -> ThreadStoreResult<()> { - unsupported("archive_thread") + async fn archive_thread(&self, params: ArchiveThreadParams) -> ThreadStoreResult<()> { + archive_thread::archive_thread(self, params).await } async fn unarchive_thread( &self, - _params: ArchiveThreadParams, + params: ArchiveThreadParams, ) -> ThreadStoreResult { - unsupported("unarchive_thread") + unarchive_thread::unarchive_thread(self, params).await } } @@ -137,429 +103,3 @@ fn unsupported(operation: &str) -> ThreadStoreResult { message: format!("local thread store does not implement {operation} in this slice"), }) } - -async fn list_rollout_threads( - config: &RolloutConfig, - params: &ListThreadsParams, - cursor: Option<&codex_rollout::Cursor>, - sort_key: codex_rollout::ThreadSortKey, -) -> ThreadStoreResult { - let page = if params.archived { - RolloutRecorder::list_archived_threads( - config, - params.page_size, - cursor, - sort_key, - params.allowed_sources.as_slice(), - params.model_providers.as_deref(), - config.model_provider_id.as_str(), - params.search_term.as_deref(), - ) - .await - } else { - RolloutRecorder::list_threads( - config, - params.page_size, - cursor, - sort_key, - params.allowed_sources.as_slice(), - params.model_providers.as_deref(), - config.model_provider_id.as_str(), - params.search_term.as_deref(), - ) - .await - }; - page.map_err(|err| ThreadStoreError::Internal { - message: format!("failed to list threads: {err}"), - }) -} - -fn stored_thread_from_rollout_item( - item: ThreadItem, - archived: bool, - default_provider: &str, -) -> Option { - let thread_id = item - .thread_id - .or_else(|| thread_id_from_rollout_path(item.path.as_path()))?; - let created_at = parse_rfc3339(item.created_at.as_deref()).unwrap_or_else(Utc::now); - let updated_at = parse_rfc3339(item.updated_at.as_deref()).unwrap_or(created_at); - let archived_at = archived.then_some(updated_at); - let git_info = git_info_from_parts( - item.git_sha.clone(), - item.git_branch.clone(), - item.git_origin_url.clone(), - ); - let source = item.source.unwrap_or(SessionSource::Unknown); - let preview = item.first_user_message.clone().unwrap_or_default(); - - Some(StoredThread { - thread_id, - rollout_path: Some(item.path), - forked_from_id: None, - preview, - name: None, - model_provider: item - .model_provider - .filter(|provider| !provider.is_empty()) - .unwrap_or_else(|| default_provider.to_string()), - model: None, - reasoning_effort: None, - created_at, - updated_at, - archived_at, - cwd: item.cwd.unwrap_or_default(), - cli_version: item.cli_version.unwrap_or_default(), - source, - agent_nickname: item.agent_nickname, - agent_role: item.agent_role, - agent_path: None, - git_info, - approval_mode: AskForApproval::OnRequest, - sandbox_policy: SandboxPolicy::new_read_only_policy(), - token_usage: None, - first_user_message: item.first_user_message, - history: None, - }) -} - -fn parse_rfc3339(value: Option<&str>) -> Option> { - DateTime::parse_from_rfc3339(value?) - .ok() - .map(|dt| dt.with_timezone(&Utc)) -} - -fn git_info_from_parts( - sha: Option, - branch: Option, - origin_url: Option, -) -> Option { - if sha.is_none() && branch.is_none() && origin_url.is_none() { - return None; - } - Some(GitInfo { - commit_hash: sha.as_deref().map(GitSha::new), - branch, - repository_url: origin_url, - }) -} - -fn thread_id_from_rollout_path(path: &std::path::Path) -> Option { - let file_name = path.file_name()?.to_str()?; - let stem = file_name.strip_suffix(".jsonl")?; - if stem.len() < 37 { - return None; - } - let uuid_start = stem.len().saturating_sub(36); - if !stem[..uuid_start].ends_with('-') { - return None; - } - codex_protocol::ThreadId::from_string(&stem[uuid_start..]).ok() -} - -#[cfg(test)] -mod tests { - use std::fs; - use std::io::Write; - use std::path::Path; - use std::path::PathBuf; - - use codex_protocol::ThreadId; - use codex_protocol::protocol::SessionSource; - use codex_rollout::ARCHIVED_SESSIONS_SUBDIR; - use pretty_assertions::assert_eq; - use tempfile::TempDir; - use uuid::Uuid; - - use super::*; - - fn test_config(codex_home: &Path) -> RolloutConfig { - RolloutConfig { - codex_home: codex_home.to_path_buf(), - sqlite_home: codex_home.to_path_buf(), - cwd: codex_home.to_path_buf(), - model_provider_id: "test-provider".to_string(), - generate_memories: true, - } - } - - fn write_session_file(root: &Path, ts: &str, uuid: Uuid) -> std::io::Result { - write_session_file_with( - root, - root.join("sessions/2025/01/03"), - ts, - uuid, - "Hello from user", - Some("test-provider"), - ) - } - - fn write_archived_session_file(root: &Path, ts: &str, uuid: Uuid) -> std::io::Result { - write_session_file_with( - root, - root.join(ARCHIVED_SESSIONS_SUBDIR), - ts, - uuid, - "Archived user message", - Some("test-provider"), - ) - } - - fn write_session_file_with( - root: &Path, - day_dir: PathBuf, - ts: &str, - uuid: Uuid, - first_user_message: &str, - model_provider: Option<&str>, - ) -> std::io::Result { - fs::create_dir_all(&day_dir)?; - let path = day_dir.join(format!("rollout-{ts}-{uuid}.jsonl")); - let mut file = fs::File::create(&path)?; - let meta = serde_json::json!({ - "timestamp": ts, - "type": "session_meta", - "payload": { - "id": uuid, - "timestamp": ts, - "cwd": root, - "originator": "test_originator", - "cli_version": "test_version", - "source": "cli", - "model_provider": model_provider, - "git": { - "commit_hash": "abcdef", - "branch": "main", - "repository_url": "https://example.com/repo.git" - } - }, - }); - writeln!(file, "{meta}")?; - let user_event = serde_json::json!({ - "timestamp": ts, - "type": "event_msg", - "payload": { - "type": "user_message", - "message": first_user_message, - "kind": "plain", - }, - }); - writeln!(file, "{user_event}")?; - Ok(path) - } - - #[tokio::test] - async fn list_threads_uses_default_provider_when_rollout_omits_provider() { - let home = TempDir::new().expect("temp dir"); - let store = LocalThreadStore::new(test_config(home.path())); - write_session_file_with( - home.path(), - home.path().join("sessions/2025/01/03"), - "2025-01-03T12-00-00", - Uuid::from_u128(102), - "Hello from user", - /*model_provider*/ None, - ) - .expect("session file"); - - let page = store - .list_threads(ListThreadsParams { - page_size: 10, - cursor: None, - sort_key: ThreadSortKey::CreatedAt, - allowed_sources: Vec::new(), - model_providers: None, - archived: false, - search_term: None, - }) - .await - .expect("thread listing"); - - assert_eq!(page.items.len(), 1); - assert_eq!(page.items[0].model_provider, "test-provider"); - } - - #[tokio::test] - async fn list_threads_preserves_sqlite_title_search_results() { - let home = TempDir::new().expect("temp dir"); - let config = test_config(home.path()); - let store = LocalThreadStore::new(config.clone()); - let uuid = Uuid::from_u128(103); - let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); - let rollout_path = home.path().join("rollout-title-search.jsonl"); - fs::write(&rollout_path, "").expect("placeholder rollout file"); - - let runtime = codex_state::StateRuntime::init( - home.path().to_path_buf(), - config.model_provider_id.clone(), - ) - .await - .expect("state db should initialize"); - runtime - .mark_backfill_complete(/*last_watermark*/ None) - .await - .expect("backfill should be complete"); - let created_at = Utc::now(); - let mut builder = codex_state::ThreadMetadataBuilder::new( - thread_id, - rollout_path, - created_at, - SessionSource::Cli, - ); - builder.model_provider = Some(config.model_provider_id.clone()); - builder.cwd = home.path().to_path_buf(); - builder.cli_version = Some("test_version".to_string()); - let mut metadata = builder.build(config.model_provider_id.as_str()); - metadata.title = "needle title".to_string(); - metadata.first_user_message = Some("plain preview".to_string()); - runtime - .upsert_thread(&metadata) - .await - .expect("state db upsert should succeed"); - - let page = store - .list_threads(ListThreadsParams { - page_size: 10, - cursor: None, - sort_key: ThreadSortKey::CreatedAt, - allowed_sources: Vec::new(), - model_providers: None, - archived: false, - search_term: Some("needle".to_string()), - }) - .await - .expect("thread listing"); - - let ids = page - .items - .iter() - .map(|item| item.thread_id) - .collect::>(); - assert_eq!(ids, vec![thread_id]); - assert_eq!( - page.items[0].first_user_message.as_deref(), - Some("plain preview") - ); - } - - #[tokio::test] - async fn list_threads_selects_active_or_archived_collection() { - let home = TempDir::new().expect("temp dir"); - let store = LocalThreadStore::new(test_config(home.path())); - let active_uuid = Uuid::from_u128(105); - let archived_uuid = Uuid::from_u128(106); - write_session_file(home.path(), "2025-01-03T12-00-00", active_uuid) - .expect("active session file"); - write_archived_session_file(home.path(), "2025-01-03T13-00-00", archived_uuid) - .expect("archived session file"); - - let active = store - .list_threads(ListThreadsParams { - page_size: 10, - cursor: None, - sort_key: ThreadSortKey::CreatedAt, - allowed_sources: Vec::new(), - model_providers: None, - archived: false, - search_term: None, - }) - .await - .expect("active listing"); - let archived = store - .list_threads(ListThreadsParams { - page_size: 10, - cursor: None, - sort_key: ThreadSortKey::CreatedAt, - allowed_sources: Vec::new(), - model_providers: None, - archived: true, - search_term: None, - }) - .await - .expect("archived listing"); - - let active_id = ThreadId::from_string(&active_uuid.to_string()).expect("valid thread id"); - let archived_id = - ThreadId::from_string(&archived_uuid.to_string()).expect("valid thread id"); - assert_eq!( - active - .items - .iter() - .map(|item| item.thread_id) - .collect::>(), - vec![active_id] - ); - assert_eq!( - archived - .items - .iter() - .map(|item| item.thread_id) - .collect::>(), - vec![archived_id] - ); - assert_eq!(active.items[0].archived_at, None); - assert_eq!( - archived.items[0].archived_at, - Some(archived.items[0].updated_at) - ); - } - - #[tokio::test] - async fn list_threads_returns_local_rollout_summary() { - let home = TempDir::new().expect("temp dir"); - let config = test_config(home.path()); - let store = LocalThreadStore::new(config); - let uuid = Uuid::from_u128(101); - let path = - write_session_file(home.path(), "2025-01-03T12-00-00", uuid).expect("session file"); - - let page = store - .list_threads(ListThreadsParams { - page_size: 10, - cursor: None, - sort_key: ThreadSortKey::CreatedAt, - allowed_sources: vec![SessionSource::Cli], - model_providers: Some(vec!["test-provider".to_string()]), - archived: false, - search_term: None, - }) - .await - .expect("thread listing"); - - let thread_id = - codex_protocol::ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); - assert_eq!(page.next_cursor, None); - assert_eq!(page.items.len(), 1); - assert_eq!(page.items[0].thread_id, thread_id); - assert_eq!(page.items[0].rollout_path, Some(path)); - assert_eq!(page.items[0].preview, "Hello from user"); - assert_eq!( - page.items[0].first_user_message.as_deref(), - Some("Hello from user") - ); - assert_eq!(page.items[0].model_provider, "test-provider"); - assert_eq!(page.items[0].cli_version, "test_version"); - assert_eq!(page.items[0].source, SessionSource::Cli); - } - - #[tokio::test] - async fn list_threads_rejects_invalid_cursor() { - let home = TempDir::new().expect("temp dir"); - let store = LocalThreadStore::new(test_config(home.path())); - - let err = store - .list_threads(ListThreadsParams { - page_size: 10, - cursor: Some("not-a-cursor".to_string()), - sort_key: ThreadSortKey::CreatedAt, - allowed_sources: Vec::new(), - model_providers: None, - archived: false, - search_term: None, - }) - .await - .expect_err("invalid cursor should fail"); - - assert!(matches!(err, ThreadStoreError::InvalidRequest { .. })); - } -} diff --git a/codex-rs/thread-store/src/local/read_thread.rs b/codex-rs/thread-store/src/local/read_thread.rs new file mode 100644 index 0000000000..189bc1938d --- /dev/null +++ b/codex-rs/thread-store/src/local/read_thread.rs @@ -0,0 +1,138 @@ +use codex_rollout::RolloutRecorder; +use codex_rollout::find_archived_thread_path_by_id_str; +use codex_rollout::find_thread_path_by_id_str; +use codex_rollout::read_thread_item_from_rollout; + +use super::LocalThreadStore; +use super::helpers::stored_thread_from_rollout_item; +use crate::ReadThreadParams; +use crate::StoredThread; +use crate::StoredThreadHistory; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; + +pub(super) async fn read_thread( + store: &LocalThreadStore, + params: ReadThreadParams, +) -> ThreadStoreResult { + let thread_id = params.thread_id; + let path = if params.include_archived { + match find_thread_path_by_id_str(store.config.codex_home.as_path(), &thread_id.to_string()) + .await + .map_err(|err| ThreadStoreError::InvalidRequest { + message: format!("failed to locate thread id {thread_id}: {err}"), + })? { + Some(path) => Some(path), + None => find_archived_thread_path_by_id_str( + store.config.codex_home.as_path(), + &thread_id.to_string(), + ) + .await + .map_err(|err| ThreadStoreError::InvalidRequest { + message: format!("failed to locate archived thread id {thread_id}: {err}"), + })?, + } + } else { + find_thread_path_by_id_str(store.config.codex_home.as_path(), &thread_id.to_string()) + .await + .map_err(|err| ThreadStoreError::InvalidRequest { + message: format!("failed to locate thread id {thread_id}: {err}"), + })? + } + .ok_or_else(|| ThreadStoreError::InvalidRequest { + message: format!("no rollout found for thread id {thread_id}"), + })?; + + let item = read_thread_item_from_rollout(path.clone()) + .await + .ok_or_else(|| ThreadStoreError::Internal { + message: format!("failed to read thread {}", path.display()), + })?; + let archived = item.path.starts_with( + store + .config + .codex_home + .join(codex_rollout::ARCHIVED_SESSIONS_SUBDIR), + ); + let mut thread = + stored_thread_from_rollout_item(item, archived, store.config.model_provider_id.as_str()) + .ok_or_else(|| ThreadStoreError::Internal { + message: format!("failed to read thread id from {}", path.display()), + })?; + if params.include_history { + let (items, _, _) = RolloutRecorder::load_rollout_items(path.as_path()) + .await + .map_err(|err| ThreadStoreError::Internal { + message: format!("failed to load thread history {}: {err}", path.display()), + })?; + thread.history = Some(StoredThreadHistory { thread_id, items }); + } + Ok(thread) +} + +#[cfg(test)] +mod tests { + use codex_protocol::ThreadId; + use pretty_assertions::assert_eq; + use tempfile::TempDir; + use uuid::Uuid; + + use super::*; + use crate::ThreadStore; + use crate::local::LocalThreadStore; + use crate::local::test_support::test_config; + use crate::local::test_support::write_session_file; + + #[tokio::test] + async fn read_thread_returns_active_rollout_summary() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path())); + let uuid = Uuid::from_u128(205); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + let active_path = + write_session_file(home.path(), "2025-01-03T12-00-00", uuid).expect("session file"); + + let thread = store + .read_thread(ReadThreadParams { + thread_id, + include_archived: false, + include_history: true, + }) + .await + .expect("read thread"); + + assert_eq!(thread.thread_id, thread_id); + assert_eq!(thread.rollout_path, Some(active_path)); + assert_eq!(thread.archived_at, None); + assert_eq!(thread.preview, "Hello from user"); + assert_eq!( + thread.history.expect("history should load").thread_id, + thread_id + ); + } + + #[tokio::test] + async fn read_thread_fails_without_rollout() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path())); + let uuid = Uuid::from_u128(206); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + + let err = store + .read_thread(ReadThreadParams { + thread_id, + include_archived: false, + include_history: false, + }) + .await + .expect_err("read should fail without rollout"); + + let ThreadStoreError::InvalidRequest { message } = err else { + panic!("expected invalid request error"); + }; + assert_eq!( + message, + format!("no rollout found for thread id {thread_id}") + ); + } +} diff --git a/codex-rs/thread-store/src/local/test_support.rs b/codex-rs/thread-store/src/local/test_support.rs new file mode 100644 index 0000000000..d8ddcf8dde --- /dev/null +++ b/codex-rs/thread-store/src/local/test_support.rs @@ -0,0 +1,87 @@ +use std::fs; +use std::io::Write; +use std::path::Path; +use std::path::PathBuf; + +use codex_rollout::ARCHIVED_SESSIONS_SUBDIR; +use codex_rollout::RolloutConfig; +use uuid::Uuid; + +pub(super) fn test_config(codex_home: &Path) -> RolloutConfig { + RolloutConfig { + codex_home: codex_home.to_path_buf(), + sqlite_home: codex_home.to_path_buf(), + cwd: codex_home.to_path_buf(), + model_provider_id: "test-provider".to_string(), + generate_memories: true, + } +} + +pub(super) fn write_session_file(root: &Path, ts: &str, uuid: Uuid) -> std::io::Result { + write_session_file_with( + root, + root.join("sessions/2025/01/03"), + ts, + uuid, + "Hello from user", + Some("test-provider"), + ) +} + +pub(super) fn write_archived_session_file( + root: &Path, + ts: &str, + uuid: Uuid, +) -> std::io::Result { + write_session_file_with( + root, + root.join(ARCHIVED_SESSIONS_SUBDIR), + ts, + uuid, + "Archived user message", + Some("test-provider"), + ) +} + +pub(super) fn write_session_file_with( + root: &Path, + day_dir: PathBuf, + ts: &str, + uuid: Uuid, + first_user_message: &str, + model_provider: Option<&str>, +) -> std::io::Result { + fs::create_dir_all(&day_dir)?; + let path = day_dir.join(format!("rollout-{ts}-{uuid}.jsonl")); + let mut file = fs::File::create(&path)?; + let meta = serde_json::json!({ + "timestamp": ts, + "type": "session_meta", + "payload": { + "id": uuid, + "timestamp": ts, + "cwd": root, + "originator": "test_originator", + "cli_version": "test_version", + "source": "cli", + "model_provider": model_provider, + "git": { + "commit_hash": "abcdef", + "branch": "main", + "repository_url": "https://example.com/repo.git" + } + }, + }); + writeln!(file, "{meta}")?; + let user_event = serde_json::json!({ + "timestamp": ts, + "type": "event_msg", + "payload": { + "type": "user_message", + "message": first_user_message, + "kind": "plain", + }, + }); + writeln!(file, "{user_event}")?; + Ok(path) +} diff --git a/codex-rs/thread-store/src/local/unarchive_thread.rs b/codex-rs/thread-store/src/local/unarchive_thread.rs new file mode 100644 index 0000000000..17e484bace --- /dev/null +++ b/codex-rs/thread-store/src/local/unarchive_thread.rs @@ -0,0 +1,198 @@ +use codex_rollout::find_archived_thread_path_by_id_str; +use codex_rollout::read_thread_item_from_rollout; +use codex_rollout::rollout_date_parts; + +use super::LocalThreadStore; +use super::helpers::matching_rollout_file_name; +use super::helpers::scoped_rollout_path; +use super::helpers::stored_thread_from_rollout_item; +use super::helpers::touch_modified_time; +use crate::ArchiveThreadParams; +use crate::StoredThread; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; + +pub(super) async fn unarchive_thread( + store: &LocalThreadStore, + params: ArchiveThreadParams, +) -> ThreadStoreResult { + let thread_id = params.thread_id; + let archived_path = find_archived_thread_path_by_id_str( + store.config.codex_home.as_path(), + &thread_id.to_string(), + ) + .await + .map_err(|err| ThreadStoreError::InvalidRequest { + message: format!("failed to locate archived thread id {thread_id}: {err}"), + })? + .ok_or_else(|| ThreadStoreError::InvalidRequest { + message: format!("no archived rollout found for thread id {thread_id}"), + })?; + + let canonical_archived_path = scoped_rollout_path( + store + .config + .codex_home + .join(codex_rollout::ARCHIVED_SESSIONS_SUBDIR), + archived_path.as_path(), + "archived", + )?; + let file_name = matching_rollout_file_name( + canonical_archived_path.as_path(), + thread_id, + archived_path.as_path(), + )?; + let Some((year, month, day)) = rollout_date_parts(&file_name) else { + return Err(ThreadStoreError::InvalidRequest { + message: format!( + "rollout path `{}` missing filename timestamp", + archived_path.display() + ), + }); + }; + + let dest_dir = store + .config + .codex_home + .join(codex_rollout::SESSIONS_SUBDIR) + .join(year) + .join(month) + .join(day); + std::fs::create_dir_all(&dest_dir).map_err(|err| ThreadStoreError::Internal { + message: format!("failed to unarchive thread: {err}"), + })?; + let restored_path = dest_dir.join(&file_name); + std::fs::rename(&canonical_archived_path, &restored_path).map_err(|err| { + ThreadStoreError::Internal { + message: format!("failed to unarchive thread: {err}"), + } + })?; + touch_modified_time(restored_path.as_path()).map_err(|err| ThreadStoreError::Internal { + message: format!("failed to update unarchived thread timestamp: {err}"), + })?; + + if let Some(ctx) = codex_rollout::state_db::get_state_db(&store.config).await { + let _ = ctx + .mark_unarchived(thread_id, restored_path.as_path()) + .await; + } + + let item = read_thread_item_from_rollout(restored_path.clone()) + .await + .ok_or_else(|| ThreadStoreError::Internal { + message: format!( + "failed to read unarchived thread {}", + restored_path.display() + ), + })?; + stored_thread_from_rollout_item( + item, + /*archived*/ false, + store.config.model_provider_id.as_str(), + ) + .ok_or_else(|| ThreadStoreError::Internal { + message: format!( + "failed to read unarchived thread id from {}", + restored_path.display() + ), + }) +} + +#[cfg(test)] +mod tests { + use chrono::Utc; + use codex_protocol::ThreadId; + use codex_protocol::protocol::SessionSource; + use pretty_assertions::assert_eq; + use tempfile::TempDir; + use uuid::Uuid; + + use super::*; + use crate::ThreadStore; + use crate::local::LocalThreadStore; + use crate::local::test_support::test_config; + use crate::local::test_support::write_archived_session_file; + + #[tokio::test] + async fn unarchive_thread_restores_rollout_and_returns_updated_thread() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path())); + let uuid = Uuid::from_u128(203); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + let archived_path = write_archived_session_file(home.path(), "2025-01-03T13-00-00", uuid) + .expect("archived session file"); + + let thread = store + .unarchive_thread(ArchiveThreadParams { thread_id }) + .await + .expect("unarchive thread"); + + assert!(!archived_path.exists()); + let restored_path = home + .path() + .join("sessions/2025/01/03") + .join(archived_path.file_name().expect("file name")); + assert!(restored_path.exists()); + assert_eq!(thread.thread_id, thread_id); + assert_eq!(thread.rollout_path, Some(restored_path)); + assert_eq!(thread.archived_at, None); + assert_eq!(thread.preview, "Archived user message"); + assert_eq!( + thread.first_user_message.as_deref(), + Some("Archived user message") + ); + } + + #[tokio::test] + async fn unarchive_thread_updates_sqlite_metadata_when_present() { + let home = TempDir::new().expect("temp dir"); + let config = test_config(home.path()); + let store = LocalThreadStore::new(config.clone()); + let uuid = Uuid::from_u128(204); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + let archived_path = write_archived_session_file(home.path(), "2025-01-03T13-00-00", uuid) + .expect("archived session file"); + let runtime = codex_state::StateRuntime::init( + home.path().to_path_buf(), + config.model_provider_id.clone(), + ) + .await + .expect("state db should initialize"); + runtime + .mark_backfill_complete(/*last_watermark*/ None) + .await + .expect("backfill should be complete"); + let mut builder = codex_state::ThreadMetadataBuilder::new( + thread_id, + archived_path.clone(), + Utc::now(), + SessionSource::Cli, + ); + builder.model_provider = Some(config.model_provider_id.clone()); + builder.cwd = home.path().to_path_buf(); + builder.cli_version = Some("test_version".to_string()); + let mut metadata = builder.build(config.model_provider_id.as_str()); + metadata.archived_at = Some(metadata.updated_at); + runtime + .upsert_thread(&metadata) + .await + .expect("state db upsert should succeed"); + + store + .unarchive_thread(ArchiveThreadParams { thread_id }) + .await + .expect("unarchive thread"); + + let restored_path = home + .path() + .join("sessions/2025/01/03") + .join(archived_path.file_name().expect("file name")); + let updated = runtime + .get_thread(thread_id) + .await + .expect("state db read should succeed") + .expect("thread metadata should exist"); + assert_eq!(updated.rollout_path, restored_path); + assert_eq!(updated.archived_at, None); + } +} From ba36415a30d895bbeabdf3ce0d4f2192f26c2fa9 Mon Sep 17 00:00:00 2001 From: starr-openai Date: Wed, 15 Apr 2026 13:48:13 -0700 Subject: [PATCH 38/84] [codex] Restore remote exec-server filesystem tests (#17989) ## Summary - Re-enable remote variants for the exec-server filesystem sandbox/symlink tests that were made local-only in PR #17671. - Restore `use_remote` parameterization for the readable-root, normalized symlink escape, symlink removal, and symlink copy-preservation cases. - Preserve `mode={use_remote}` context on key async filesystem failures so CI failures point at the local or remote lane. ## Validation - `cd codex-rs && just fmt` - Not run: `bazel test //codex-rs/exec-server:exec-server-file_system-test` per local Codex development guidance to avoid test runs unless explicitly requested. Co-authored-by: Codex --- codex-rs/exec-server/tests/file_system.rs | 35 ++++++++++++++++------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/codex-rs/exec-server/tests/file_system.rs b/codex-rs/exec-server/tests/file_system.rs index 41dd3f8600..8b9fb90ecc 100644 --- a/codex-rs/exec-server/tests/file_system.rs +++ b/codex-rs/exec-server/tests/file_system.rs @@ -377,9 +377,11 @@ async fn file_system_copy_rejects_directory_without_recursive(use_remote: bool) Ok(()) } +#[test_case(false ; "local")] +#[test_case(true ; "remote")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_sandboxed_read_allows_readable_root() -> Result<()> { - let context = create_file_system_context(/*use_remote*/ false).await?; +async fn file_system_sandboxed_read_allows_readable_root(use_remote: bool) -> Result<()> { + let context = create_file_system_context(use_remote).await?; let file_system = context.file_system; let tmp = TempDir::new()?; @@ -391,7 +393,8 @@ async fn file_system_sandboxed_read_allows_readable_root() -> Result<()> { let contents = file_system .read_file(&absolute_path(file_path), Some(&sandbox)) - .await?; + .await + .with_context(|| format!("mode={use_remote}"))?; assert_eq!(contents, b"sandboxed hello"); Ok(()) @@ -520,9 +523,13 @@ async fn file_system_sandboxed_read_rejects_symlink_escape(use_remote: bool) -> Ok(()) } +#[test_case(false ; "local")] +#[test_case(true ; "remote")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_sandboxed_read_rejects_symlink_parent_dotdot_escape() -> Result<()> { - let context = create_file_system_context(/*use_remote*/ false).await?; +async fn file_system_sandboxed_read_rejects_symlink_parent_dotdot_escape( + use_remote: bool, +) -> Result<()> { + let context = create_file_system_context(use_remote).await?; let file_system = context.file_system; let tmp = TempDir::new()?; @@ -680,9 +687,11 @@ async fn file_system_copy_rejects_symlink_escape_destination(use_remote: bool) - Ok(()) } +#[test_case(false ; "local")] +#[test_case(true ; "remote")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_remove_removes_symlink_not_target() -> Result<()> { - let context = create_file_system_context(/*use_remote*/ false).await?; +async fn file_system_remove_removes_symlink_not_target(use_remote: bool) -> Result<()> { + let context = create_file_system_context(use_remote).await?; let file_system = context.file_system; let tmp = TempDir::new()?; @@ -705,7 +714,8 @@ async fn file_system_remove_removes_symlink_not_target() -> Result<()> { }, Some(&sandbox), ) - .await?; + .await + .with_context(|| format!("mode={use_remote}"))?; assert!(!symlink_path.exists()); assert!(outside_file.exists()); @@ -714,9 +724,11 @@ async fn file_system_remove_removes_symlink_not_target() -> Result<()> { Ok(()) } +#[test_case(false ; "local")] +#[test_case(true ; "remote")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_copy_preserves_symlink_source() -> Result<()> { - let context = create_file_system_context(/*use_remote*/ false).await?; +async fn file_system_copy_preserves_symlink_source(use_remote: bool) -> Result<()> { + let context = create_file_system_context(use_remote).await?; let file_system = context.file_system; let tmp = TempDir::new()?; @@ -738,7 +750,8 @@ async fn file_system_copy_preserves_symlink_source() -> Result<()> { CopyOptions { recursive: false }, Some(&sandbox), ) - .await?; + .await + .with_context(|| format!("mode={use_remote}"))?; let copied_metadata = std::fs::symlink_metadata(&copied_symlink)?; assert!(copied_metadata.file_type().is_symlink()); From bc969b6516f2fb7a4817b665ab1507f4f4beedc2 Mon Sep 17 00:00:00 2001 From: Eugene Brevdo Date: Wed, 15 Apr 2026 13:57:41 -0700 Subject: [PATCH 39/84] Dismiss stale app-server requests after remote resolution (#15134) Dismiss stale TUI app-server approvals after remote resolution When an approval, user-input prompt, or elicitation request is resolved by another client, the TUI now dismisses the matching local UI instead of leaving stale prompts behind and emitting a misleading local cancellation. This change teaches pending app-server request tracking to map `serverRequest/resolved` notifications back to the concrete request type and stable request key, then propagates that resolved request into TUI prompt state. Approval, request-user-input, and MCP elicitation overlays now drop the resolved current or queued request quietly, advance to the next queued request when present, and avoid emitting abort/cancel events for stale UI. The latest update also retires matching prompts while they are still deferred behind active streaming and suppresses buffered active-thread requests whose app-server request id has already been resolved before drain. `ChatWidget` removes a resolved request from both the deferred interrupt queue and the materialized bottom-pane stack, while active-thread request handling verifies the app-server request is still pending before showing a prompt. Lifecycle events such as exec begin/end remain queued so approved work can still render normally. Tests cover resolved-request mapping, overlay dismissal behavior, deferred prompt pruning for same-turn user input, exec approval IDs, lifecycle-event retention, and the buffered active-thread ordering regression. Validation: - `just fmt` - `git diff --check` - `cargo test -p codex-tui resolved_buffered_approval_does_not_become_actionable_after_drain` - `cargo test -p codex-tui enqueue_primary_thread_session_replays_buffered_approval_after_attach` - `cargo test -p codex-tui chatwidget::interrupts` - `just fix -p codex-tui` --------- Co-authored-by: Codex --- .../tests/suite/v2/turn_interrupt.rs | 15 +- codex-rs/tui/src/app.rs | 76 ++++- codex-rs/tui/src/app/app_server_adapter.rs | 8 +- codex-rs/tui/src/app/app_server_requests.rs | 300 +++++++++++++++++- codex-rs/tui/src/bottom_pane/app_link_view.rs | 80 +++++ .../tui/src/bottom_pane/approval_overlay.rs | 72 +++++ .../tui/src/bottom_pane/bottom_pane_view.rs | 8 + .../src/bottom_pane/mcp_server_elicitation.rs | 116 ++++++- codex-rs/tui/src/bottom_pane/mod.rs | 234 +++++++++++++- .../src/bottom_pane/request_user_input/mod.rs | 151 ++++++++- codex-rs/tui/src/chatwidget.rs | 11 + codex-rs/tui/src/chatwidget/interrupts.rs | 149 +++++++++ 12 files changed, 1164 insertions(+), 56 deletions(-) diff --git a/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs b/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs index b553137752..f8eaf799da 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs @@ -127,10 +127,17 @@ async fn turn_interrupt_aborts_running_turn() -> Result<()> { #[tokio::test] async fn turn_interrupt_resolves_pending_command_approval_request() -> Result<()> { + #[cfg(target_os = "windows")] + let shell_command = vec![ + "powershell".to_string(), + "-Command".to_string(), + "Start-Sleep -Seconds 10".to_string(), + ]; + #[cfg(not(target_os = "windows"))] let shell_command = vec![ "python3".to_string(), "-c".to_string(), - "print(42)".to_string(), + "import time; time.sleep(10)".to_string(), ]; let tmp = TempDir::new()?; @@ -143,7 +150,7 @@ async fn turn_interrupt_resolves_pending_command_approval_request() -> Result<() shell_command.clone(), Some(&working_directory), Some(10_000), - "call_python_approval", + "call_sleep_approval", )?]) .await; create_config_toml(&codex_home, &server.uri(), "untrusted", "read-only")?; @@ -172,6 +179,7 @@ async fn turn_interrupt_resolves_pending_command_approval_request() -> Result<() text_elements: Vec::new(), }], cwd: Some(working_directory), + approval_policy: Some(codex_app_server_protocol::AskForApproval::UnlessTrusted), ..Default::default() }) .await?; @@ -190,7 +198,7 @@ async fn turn_interrupt_resolves_pending_command_approval_request() -> Result<() let ServerRequest::CommandExecutionRequestApproval { request_id, params } = request else { panic!("expected CommandExecutionRequestApproval request"); }; - assert_eq!(params.item_id, "call_python_approval"); + assert_eq!(params.item_id, "call_sleep_approval"); assert_eq!(params.thread_id, thread.id); assert_eq!(params.turn_id, turn.id); @@ -251,6 +259,7 @@ fn create_config_toml( r#" model = "mock-model" approval_policy = "{approval_policy}" +approvals_reviewer = "user" sandbox_mode = "{sandbox_mode}" model_provider = "mock_provider" diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 71304deb2c..50dbd0bf6d 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -164,7 +164,7 @@ use toml::Value as TomlValue; use uuid::Uuid; mod agent_navigation; mod app_server_adapter; -mod app_server_requests; +pub(crate) mod app_server_requests; mod loaded_threads; mod pending_interactive_replay; @@ -5895,8 +5895,13 @@ impl App { .handle_server_notification(notification, /*replay_kind*/ None); } ThreadBufferedEvent::Request(request) => { - self.chat_widget - .handle_server_request(request, /*replay_kind*/ None); + if self + .pending_app_server_requests + .contains_server_request(&request) + { + self.chat_widget + .handle_server_request(request, /*replay_kind*/ None); + } } ThreadBufferedEvent::HistoryEntryResponse(event) => { self.chat_widget.handle_history_entry_response(event); @@ -6884,6 +6889,11 @@ mod tests { let approval_request = exec_approval_request(thread_id, "turn-1", "call-1", /*approval_id*/ None); + assert_eq!( + app.pending_app_server_requests + .note_server_request(&approval_request), + None + ); app.enqueue_primary_thread_request(approval_request).await?; app.enqueue_primary_thread_session( test_thread_session(thread_id, test_path_buf("/tmp/project")), @@ -6926,6 +6936,66 @@ mod tests { panic!("expected approval action to submit a thread-scoped op"); } + #[tokio::test] + async fn resolved_buffered_approval_does_not_become_actionable_after_drain() -> Result<()> { + let (mut app, mut app_event_rx, _op_rx) = make_test_app_with_channels().await; + let thread_id = ThreadId::new(); + let approval_request = + exec_approval_request(thread_id, "turn-1", "call-1", /*approval_id*/ None); + + app.enqueue_primary_thread_session( + test_thread_session(thread_id, test_path_buf("/tmp/project")), + Vec::new(), + ) + .await?; + while app_event_rx.try_recv().is_ok() {} + + assert_eq!( + app.pending_app_server_requests + .note_server_request(&approval_request), + None + ); + app.enqueue_thread_request(thread_id, approval_request) + .await?; + + let resolved = app + .pending_app_server_requests + .resolve_notification(&AppServerRequestId::Integer(1)) + .expect("matching app-server request should resolve"); + app.chat_widget.dismiss_app_server_request(&resolved); + while app_event_rx.try_recv().is_ok() {} + + let rx = app + .active_thread_rx + .as_mut() + .expect("primary thread receiver should be active"); + let event = time::timeout(Duration::from_millis(50), rx.recv()) + .await + .expect("timed out waiting for buffered approval event") + .expect("channel closed unexpectedly"); + + assert!(matches!( + &event, + ThreadBufferedEvent::Request(ServerRequest::CommandExecutionRequestApproval { + params, + .. + }) if params.turn_id == "turn-1" + )); + + app.handle_thread_event_now(event); + app.chat_widget + .handle_key_event(KeyEvent::new(KeyCode::Char('y'), KeyModifiers::NONE)); + + while let Ok(app_event) = app_event_rx.try_recv() { + assert!( + !matches!(app_event, AppEvent::SubmitThreadOp { .. }), + "resolved buffered approval should not become actionable" + ); + } + + Ok(()) + } + #[tokio::test] async fn enqueue_primary_thread_session_replays_turns_before_initial_prompt_submit() -> Result<()> { diff --git a/codex-rs/tui/src/app/app_server_adapter.rs b/codex-rs/tui/src/app/app_server_adapter.rs index 09b8ce06cd..f2490ca6a2 100644 --- a/codex-rs/tui/src/app/app_server_adapter.rs +++ b/codex-rs/tui/src/app/app_server_adapter.rs @@ -158,8 +158,12 @@ impl App { ) { match ¬ification { ServerNotification::ServerRequestResolved(notification) => { - self.pending_app_server_requests - .resolve_notification(¬ification.request_id); + if let Some(request) = self + .pending_app_server_requests + .resolve_notification(¬ification.request_id) + { + self.chat_widget.dismiss_app_server_request(&request); + } } ServerNotification::McpServerStatusUpdated(_) => { self.refresh_mcp_startup_expected_servers_from_config(); diff --git a/codex-rs/tui/src/app/app_server_requests.rs b/codex-rs/tui/src/app/app_server_requests.rs index b1393e95e0..e2483eb25d 100644 --- a/codex-rs/tui/src/app/app_server_requests.rs +++ b/codex-rs/tui/src/app/app_server_requests.rs @@ -1,4 +1,5 @@ use std::collections::HashMap; +use std::collections::VecDeque; use crate::app_command::AppCommand; use crate::app_command::AppCommandView; @@ -27,12 +28,32 @@ pub(super) struct UnsupportedAppServerRequest { pub(super) message: String, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ResolvedAppServerRequest { + ExecApproval { + id: String, + }, + FileChangeApproval { + id: String, + }, + PermissionsApproval { + id: String, + }, + UserInput { + call_id: String, + }, + McpElicitation { + server_name: String, + request_id: McpRequestId, + }, +} + #[derive(Debug, Default)] pub(super) struct PendingAppServerRequests { exec_approvals: HashMap, file_change_approvals: HashMap, permissions_approvals: HashMap, - user_inputs: HashMap, + user_inputs: HashMap>, mcp_requests: HashMap, } @@ -70,7 +91,12 @@ impl PendingAppServerRequests { } ServerRequest::ToolRequestUserInput { request_id, params } => { self.user_inputs - .insert(params.turn_id.clone(), request_id.clone()); + .entry(params.turn_id.clone()) + .or_default() + .push_back(PendingUserInputRequest { + item_id: params.item_id.clone(), + request_id: request_id.clone(), + }); None } ServerRequest::McpServerElicitationRequest { request_id, params } => { @@ -165,11 +191,10 @@ impl PendingAppServerRequests { }) .transpose()?, AppCommandView::UserInputAnswer { id, response } => self - .user_inputs - .remove(id) - .map(|request_id| { + .pop_user_input_request_for_turn(id) + .map(|pending| { Ok::(AppServerRequestResolution { - request_id, + request_id: pending.request_id, result: serde_json::to_value( serde_json::from_value::( serde_json::to_value(response).map_err(|err| { @@ -229,15 +254,131 @@ impl PendingAppServerRequests { Ok(resolution) } - pub(super) fn resolve_notification(&mut self, request_id: &AppServerRequestId) { - self.exec_approvals.retain(|_, value| value != request_id); - self.file_change_approvals - .retain(|_, value| value != request_id); - self.permissions_approvals - .retain(|_, value| value != request_id); - self.user_inputs.retain(|_, value| value != request_id); - self.mcp_requests.retain(|_, value| value != request_id); + pub(super) fn resolve_notification( + &mut self, + request_id: &AppServerRequestId, + ) -> Option { + if let Some(id) = self + .exec_approvals + .iter() + .find_map(|(id, value)| (value == request_id).then(|| id.clone())) + { + self.exec_approvals.remove(&id); + return Some(ResolvedAppServerRequest::ExecApproval { id }); + } + + if let Some(id) = self + .file_change_approvals + .iter() + .find_map(|(id, value)| (value == request_id).then(|| id.clone())) + { + self.file_change_approvals.remove(&id); + return Some(ResolvedAppServerRequest::FileChangeApproval { id }); + } + + if let Some(id) = self + .permissions_approvals + .iter() + .find_map(|(id, value)| (value == request_id).then(|| id.clone())) + { + self.permissions_approvals.remove(&id); + return Some(ResolvedAppServerRequest::PermissionsApproval { id }); + } + + if let Some(pending) = self.remove_user_input_request(request_id) { + return Some(ResolvedAppServerRequest::UserInput { + call_id: pending.item_id, + }); + } + + if let Some(key) = self + .mcp_requests + .iter() + .find_map(|(key, value)| (value == request_id).then(|| key.clone())) + { + self.mcp_requests.remove(&key); + return Some(ResolvedAppServerRequest::McpElicitation { + server_name: key.server_name, + request_id: key.request_id, + }); + } + + None } + + pub(super) fn contains_server_request(&self, request: &ServerRequest) -> bool { + match request { + ServerRequest::CommandExecutionRequestApproval { request_id, .. } => self + .exec_approvals + .values() + .any(|pending_request_id| pending_request_id == request_id), + ServerRequest::FileChangeRequestApproval { request_id, .. } => self + .file_change_approvals + .values() + .any(|pending_request_id| pending_request_id == request_id), + ServerRequest::PermissionsRequestApproval { request_id, .. } => self + .permissions_approvals + .values() + .any(|pending_request_id| pending_request_id == request_id), + ServerRequest::ToolRequestUserInput { request_id, .. } => { + self.user_inputs.values().any(|queue| { + queue + .iter() + .any(|pending| &pending.request_id == request_id) + }) + } + ServerRequest::McpServerElicitationRequest { request_id, .. } => self + .mcp_requests + .values() + .any(|pending_request_id| pending_request_id == request_id), + ServerRequest::DynamicToolCall { .. } + | ServerRequest::ChatgptAuthTokensRefresh { .. } + | ServerRequest::ApplyPatchApproval { .. } + | ServerRequest::ExecCommandApproval { .. } => true, + } + } + + fn pop_user_input_request_for_turn( + &mut self, + turn_id: &str, + ) -> Option { + let pending = self + .user_inputs + .get_mut(turn_id) + .and_then(VecDeque::pop_front); + if self + .user_inputs + .get(turn_id) + .is_some_and(VecDeque::is_empty) + { + self.user_inputs.remove(turn_id); + } + pending + } + + fn remove_user_input_request( + &mut self, + request_id: &AppServerRequestId, + ) -> Option { + let (turn_id, index) = self.user_inputs.iter().find_map(|(turn_id, queue)| { + queue + .iter() + .position(|pending| &pending.request_id == request_id) + .map(|index| (turn_id.clone(), index)) + })?; + let queue = self.user_inputs.get_mut(&turn_id)?; + let removed = queue.remove(index); + if queue.is_empty() { + self.user_inputs.remove(&turn_id); + } + removed + } +} + +#[derive(Debug)] +struct PendingUserInputRequest { + item_id: String, + request_id: AppServerRequestId, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -272,6 +413,7 @@ fn file_change_decision(decision: &ReviewDecision) -> Result bool { self.complete } + + fn dismiss_app_server_request(&mut self, request: &ResolvedAppServerRequest) -> bool { + let ResolvedAppServerRequest::McpElicitation { + server_name, + request_id, + } = request + else { + return false; + }; + let Some(target) = self.elicitation_target.as_ref() else { + return false; + }; + if target.server_name != *server_name || target.request_id != *request_id { + return false; + } + + self.complete = true; + true + } } impl crate::render::renderable::Renderable for AppLinkView { @@ -546,9 +566,11 @@ impl crate::render::renderable::Renderable for AppLinkView { #[cfg(test)] mod tests { use super::*; + use crate::app::app_server_requests::ResolvedAppServerRequest; use crate::app_event::AppEvent; use crate::render::renderable::Renderable; use insta::assert_snapshot; + use pretty_assertions::assert_eq; use tokio::sync::mpsc::unbounded_channel; fn suggestion_target() -> AppLinkElicitationTarget { @@ -889,6 +911,64 @@ mod tests { assert!(view.is_complete()); } + #[test] + fn resolved_tool_suggestion_dismisses_matching_view() { + let (tx_raw, _rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let mut view = AppLinkView::new( + AppLinkViewParams { + app_id: "connector_google_calendar".to_string(), + title: "Google Calendar".to_string(), + description: Some("Plan events and schedules.".to_string()), + instructions: "Enable this app to use it for the current request.".to_string(), + url: "https://example.test/google-calendar".to_string(), + is_installed: true, + is_enabled: false, + suggest_reason: Some("Plan and reference events from your calendar".to_string()), + suggestion_type: Some(AppLinkSuggestionType::Enable), + elicitation_target: Some(suggestion_target()), + }, + tx, + ); + + assert!( + view.dismiss_app_server_request(&ResolvedAppServerRequest::McpElicitation { + server_name: "codex_apps".to_string(), + request_id: McpRequestId::String("request-1".to_string()), + }) + ); + assert!(view.is_complete()); + } + + #[test] + fn resolved_tool_suggestion_ignores_non_matching_request() { + let (tx_raw, _rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let mut view = AppLinkView::new( + AppLinkViewParams { + app_id: "connector_google_calendar".to_string(), + title: "Google Calendar".to_string(), + description: Some("Plan events and schedules.".to_string()), + instructions: "Enable this app to use it for the current request.".to_string(), + url: "https://example.test/google-calendar".to_string(), + is_installed: true, + is_enabled: false, + suggest_reason: Some("Plan and reference events from your calendar".to_string()), + suggestion_type: Some(AppLinkSuggestionType::Enable), + elicitation_target: Some(suggestion_target()), + }, + tx, + ); + + assert!( + !view.dismiss_app_server_request(&ResolvedAppServerRequest::McpElicitation { + server_name: "other_server".to_string(), + request_id: McpRequestId::String("request-1".to_string()), + }) + ); + assert!(!view.is_complete()); + } + #[test] fn install_suggestion_with_reason_snapshot() { let (tx_raw, _rx) = unbounded_channel::(); diff --git a/codex-rs/tui/src/bottom_pane/approval_overlay.rs b/codex-rs/tui/src/bottom_pane/approval_overlay.rs index 656e812d66..0e2bb2b2e3 100644 --- a/codex-rs/tui/src/bottom_pane/approval_overlay.rs +++ b/codex-rs/tui/src/bottom_pane/approval_overlay.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use std::path::PathBuf; +use crate::app::app_server_requests::ResolvedAppServerRequest; use crate::app_event::AppEvent; use crate::app_event_sender::AppEventSender; use crate::bottom_pane::BottomPaneView; @@ -97,6 +98,35 @@ impl ApprovalRequest { | ApprovalRequest::McpElicitation { thread_label, .. } => thread_label.as_deref(), } } + + fn matches_resolved_request(&self, request: &ResolvedAppServerRequest) -> bool { + match (self, request) { + ( + ApprovalRequest::Exec { id, .. }, + ResolvedAppServerRequest::ExecApproval { id: resolved_id }, + ) => id == resolved_id, + ( + ApprovalRequest::Permissions { call_id, .. }, + ResolvedAppServerRequest::PermissionsApproval { id }, + ) => call_id == id, + ( + ApprovalRequest::ApplyPatch { id, .. }, + ResolvedAppServerRequest::FileChangeApproval { id: resolved_id }, + ) => id == resolved_id, + ( + ApprovalRequest::McpElicitation { + server_name, + request_id, + .. + }, + ResolvedAppServerRequest::McpElicitation { + server_name: resolved_server_name, + request_id: resolved_request_id, + }, + ) => server_name == resolved_server_name && request_id == resolved_request_id, + _ => false, + } + } } /// Modal overlay asking the user to approve or deny one or more requests. @@ -131,6 +161,23 @@ impl ApprovalOverlay { self.queue.push(req); } + fn dismiss_resolved_request(&mut self, request: &ResolvedAppServerRequest) -> bool { + let queue_len = self.queue.len(); + self.queue + .retain(|queued_request| !queued_request.matches_resolved_request(request)); + if self + .current_request + .as_ref() + .is_some_and(|current_request| current_request.matches_resolved_request(request)) + { + self.current_complete = true; + self.advance_queue(); + return true; + } + + self.queue.len() != queue_len + } + fn set_current(&mut self, request: ApprovalRequest) { self.current_complete = false; let header = build_header(&request); @@ -465,6 +512,10 @@ impl BottomPaneView for ApprovalOverlay { self.enqueue_request(request); None } + + fn dismiss_app_server_request(&mut self, request: &ResolvedAppServerRequest) -> bool { + self.dismiss_resolved_request(request) + } } impl Renderable for ApprovalOverlay { @@ -951,6 +1002,27 @@ mod tests { assert!(saw_op, "expected approval decision to emit an op"); } + #[test] + fn resolved_request_dismisses_overlay_without_emitting_abort() { + let (tx, mut rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx); + let mut view = ApprovalOverlay::new(make_exec_request(), tx, Features::with_defaults()); + + assert!( + view.dismiss_app_server_request(&ResolvedAppServerRequest::ExecApproval { + id: "test".to_string(), + }) + ); + assert!( + view.is_complete(), + "resolved request should close the overlay" + ); + assert!( + rx.try_recv().is_err(), + "dismissing a stale request should not emit an approval op" + ); + } + #[test] fn o_opens_source_thread_for_cross_thread_approval() { let (tx, mut rx) = unbounded_channel::(); diff --git a/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs b/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs index 35165db491..6fa946a5af 100644 --- a/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs +++ b/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs @@ -1,3 +1,4 @@ +use crate::app::app_server_requests::ResolvedAppServerRequest; use crate::bottom_pane::ApprovalRequest; use crate::bottom_pane::McpServerElicitationFormRequest; use crate::render::renderable::Renderable; @@ -87,4 +88,11 @@ pub(crate) trait BottomPaneView: Renderable { ) -> Option { Some(request) } + + /// Dismiss a request that was resolved by another client. + /// + /// Returns `true` when the view changed state. + fn dismiss_app_server_request(&mut self, _request: &ResolvedAppServerRequest) -> bool { + false + } } diff --git a/codex-rs/tui/src/bottom_pane/mcp_server_elicitation.rs b/codex-rs/tui/src/bottom_pane/mcp_server_elicitation.rs index 1e3ad65912..c9844ff915 100644 --- a/codex-rs/tui/src/bottom_pane/mcp_server_elicitation.rs +++ b/codex-rs/tui/src/bottom_pane/mcp_server_elicitation.rs @@ -28,6 +28,7 @@ use ratatui::widgets::Widget; use serde_json::Value; use unicode_width::UnicodeWidthStr; +use crate::app::app_server_requests::ResolvedAppServerRequest; use crate::app_event_sender::AppEventSender; use crate::bottom_pane::CancellationEvent; use crate::bottom_pane::ChatComposer; @@ -1145,6 +1146,16 @@ impl McpServerElicitationOverlay { ); } + fn advance_queue_or_complete(&mut self) { + if let Some(next) = self.queue.pop_front() { + self.request = next; + self.reset_for_request(); + self.restore_current_draft(); + } else { + self.done = true; + } + } + fn submit_answers(&mut self) { self.save_current_draft(); if let Some(idx) = self.first_required_unanswered_index() { @@ -1181,13 +1192,7 @@ impl McpServerElicitationOverlay { /*content*/ None, meta, ); - if let Some(next) = self.queue.pop_front() { - self.request = next; - self.reset_for_request(); - self.restore_current_draft(); - } else { - self.done = true; - } + self.advance_queue_or_complete(); return; } let content = self @@ -1205,13 +1210,28 @@ impl McpServerElicitationOverlay { Some(Value::Object(content)), /*meta*/ None, ); - if let Some(next) = self.queue.pop_front() { - self.request = next; - self.reset_for_request(); - self.restore_current_draft(); - } else { - self.done = true; + self.advance_queue_or_complete(); + } + + fn dismiss_resolved_request(&mut self, request: &ResolvedAppServerRequest) -> bool { + let ResolvedAppServerRequest::McpElicitation { + server_name, + request_id, + } = request + else { + return false; + }; + + let queue_len = self.queue.len(); + self.queue.retain(|queued_request| { + queued_request.server_name != *server_name || queued_request.request_id != *request_id + }); + if self.request.server_name == *server_name && self.request.request_id == *request_id { + self.advance_queue_or_complete(); + return true; } + + self.queue.len() != queue_len } fn go_next_or_submit(&mut self) { @@ -1642,6 +1662,10 @@ impl BottomPaneView for McpServerElicitationOverlay { self.queue.push_back(request); None } + + fn dismiss_app_server_request(&mut self, request: &ResolvedAppServerRequest) -> bool { + self.dismiss_resolved_request(request) + } } fn wrap_footer_tips(width: u16, tips: Vec) -> Vec> { @@ -2362,6 +2386,72 @@ mod tests { assert_eq!(overlay.request.message, "Third"); } + #[test] + fn resolved_request_dismisses_overlay_without_emitting_events() { + let (tx, mut rx) = test_sender(); + let thread_id = ThreadId::default(); + let supported_form_schema = serde_json::json!({ + "type": "object", + "properties": { + "confirmed": { + "type": "boolean", + "title": "Confirm", + } + }, + }); + let mut overlay = McpServerElicitationOverlay::new( + McpServerElicitationFormRequest::from_event( + thread_id, + form_request("First", supported_form_schema.clone(), /*meta*/ None), + ) + .expect("expected supported form"), + tx, + /*has_input_focus*/ true, + /*enhanced_keys_supported*/ false, + /*disable_paste_burst*/ false, + ); + overlay.try_consume_mcp_server_elicitation_request( + McpServerElicitationFormRequest::from_event( + thread_id, + ElicitationRequestEvent { + turn_id: Some("turn-2".to_string()), + server_name: "server-1".to_string(), + id: McpRequestId::String("request-2".to_string()), + request: ElicitationRequest::Form { + meta: None, + message: "Second".to_string(), + requested_schema: supported_form_schema, + }, + }, + ) + .expect("expected supported form"), + ); + + assert!( + overlay.dismiss_app_server_request(&ResolvedAppServerRequest::McpElicitation { + server_name: "server-1".to_string(), + request_id: McpRequestId::String("request-1".to_string()), + }) + ); + assert_eq!(overlay.request.message, "Second"); + assert!(matches!( + rx.try_recv(), + Err(tokio::sync::mpsc::error::TryRecvError::Empty) + )); + + assert!( + overlay.dismiss_app_server_request(&ResolvedAppServerRequest::McpElicitation { + server_name: "server-1".to_string(), + request_id: McpRequestId::String("request-2".to_string()), + }) + ); + assert!(overlay.is_complete()); + assert!(matches!( + rx.try_recv(), + Err(tokio::sync::mpsc::error::TryRecvError::Empty) + )); + } + #[test] fn boolean_form_snapshot() { let (tx, _rx) = test_sender(); diff --git a/codex-rs/tui/src/bottom_pane/mod.rs b/codex-rs/tui/src/bottom_pane/mod.rs index 85020d7137..91c8bd48eb 100644 --- a/codex-rs/tui/src/bottom_pane/mod.rs +++ b/codex-rs/tui/src/bottom_pane/mod.rs @@ -15,6 +15,7 @@ //! hint. The pane schedules redraws so those hints can expire even when the UI is otherwise idle. use std::path::PathBuf; +use crate::app::app_server_requests::ResolvedAppServerRequest; use crate::app_event::ConnectorsSnapshot; use crate::app_event_sender::AppEventSender; use crate::bottom_pane::pending_input_preview::PendingInputPreview; @@ -379,6 +380,18 @@ impl BottomPane { self.request_redraw(); } + fn pop_active_view(&mut self) { + if self.view_stack.pop().is_some() { + self.on_view_stack_depth_decreased(); + } + } + + fn on_view_stack_depth_decreased(&mut self) { + if self.view_stack.is_empty() { + self.on_active_view_complete(); + } + } + /// Forward a key event to the active view or the composer. pub fn handle_key_event(&mut self, key_event: KeyEvent) -> InputResult { // If a modal/view is active, handle it here; otherwise forward to composer. @@ -408,16 +421,14 @@ impl BottomPane { }; if ctrl_c_completed { - self.view_stack.pop(); - self.on_active_view_complete(); + self.pop_active_view(); if let Some(next_view) = self.view_stack.last() && next_view.is_in_paste_burst() { self.request_redraw_in(ChatComposer::recommended_paste_flush_delay()); } } else if view_complete { - self.view_stack.clear(); - self.on_active_view_complete(); + self.pop_active_view(); } else if view_in_paste_burst { self.request_redraw_in(ChatComposer::recommended_paste_flush_delay()); } @@ -469,10 +480,10 @@ impl BottomPane { pub(crate) fn on_ctrl_c(&mut self) -> CancellationEvent { if let Some(view) = self.view_stack.last_mut() { let event = view.on_ctrl_c(); + let view_complete = view.is_complete(); if matches!(event, CancellationEvent::Handled) { - if view.is_complete() { - self.view_stack.pop(); - self.on_active_view_complete(); + if view_complete { + self.pop_active_view(); } self.show_quit_shortcut_hint(key_hint::ctrl(KeyCode::Char('c'))); self.request_redraw(); @@ -493,17 +504,14 @@ impl BottomPane { } pub fn handle_paste(&mut self, pasted: String) { - if !self.view_stack.is_empty() { - let (needs_redraw, view_complete) = { - let last_index = self.view_stack.len() - 1; - let view = &mut self.view_stack[last_index]; - (view.handle_paste(pasted), view.is_complete()) - }; + if let Some(view) = self.view_stack.last_mut() { + let needs_redraw = view.handle_paste(pasted); + let view_complete = view.is_complete(); if view_complete { self.view_stack.clear(); self.on_active_view_complete(); } - if needs_redraw { + if needs_redraw || view_complete { self.request_redraw(); } } else { @@ -1032,6 +1040,37 @@ impl BottomPane { self.push_view(Box::new(modal)); } + pub(crate) fn dismiss_app_server_request( + &mut self, + request: &ResolvedAppServerRequest, + ) -> bool { + if self.view_stack.is_empty() { + return false; + } + + let mut changed = false; + let mut completed_indices = Vec::new(); + for index in (0..self.view_stack.len()).rev() { + let view = &mut self.view_stack[index]; + if !view.dismiss_app_server_request(request) { + continue; + } + changed = true; + if view.is_complete() { + completed_indices.push(index); + } + } + if !changed { + return false; + } + for index in completed_indices { + self.view_stack.remove(index); + } + self.on_view_stack_depth_decreased(); + self.request_redraw(); + true + } + fn on_active_view_complete(&mut self) { self.resume_status_timer_after_modal(); self.set_composer_input_enabled(/*enabled*/ true, /*placeholder*/ None); @@ -1241,6 +1280,7 @@ impl Renderable for BottomPane { #[cfg(test)] mod tests { use super::*; + use crate::app::app_server_requests::ResolvedAppServerRequest; use crate::app_event::AppEvent; use crate::status_indicator_widget::STATUS_DETAILS_DEFAULT_MAX_LINES; use crate::status_indicator_widget::StatusDetailsCapitalization; @@ -1248,9 +1288,12 @@ mod tests { use crate::test_support::test_path_buf; use codex_protocol::protocol::Op; use codex_protocol::protocol::SkillScope; + use crossterm::event::KeyCode; + use crossterm::event::KeyEvent; use crossterm::event::KeyEventKind; use crossterm::event::KeyModifiers; use insta::assert_snapshot; + use pretty_assertions::assert_eq; use ratatui::buffer::Buffer; use ratatui::layout::Rect; use std::cell::Cell; @@ -1275,6 +1318,19 @@ mod tests { snapshot_buffer(&buf) } + fn test_pane(app_event_tx: AppEventSender) -> BottomPane { + BottomPane::new(BottomPaneParams { + app_event_tx, + frame_requester: FrameRequester::test_dummy(), + has_input_focus: true, + enhanced_keys_supported: false, + placeholder_text: "Ask Codex to do anything".to_string(), + disable_paste_burst: false, + animations_enabled: true, + skills: Some(Vec::new()), + }) + } + fn exec_request() -> ApprovalRequest { ApprovalRequest::Exec { thread_id: codex_protocol::ThreadId::new(), @@ -1291,6 +1347,73 @@ mod tests { } } + #[derive(Default)] + struct DismissibleView { + id: Option<&'static str>, + dismiss_exec_id: Option<&'static str>, + complete: bool, + } + + impl Renderable for DismissibleView { + fn render(&self, _area: Rect, _buf: &mut Buffer) {} + + fn desired_height(&self, _width: u16) -> u16 { + 0 + } + } + + impl BottomPaneView for DismissibleView { + fn is_complete(&self) -> bool { + self.complete + } + + fn view_id(&self) -> Option<&'static str> { + self.id + } + + fn dismiss_app_server_request(&mut self, request: &ResolvedAppServerRequest) -> bool { + let ResolvedAppServerRequest::ExecApproval { id } = request else { + return false; + }; + if self.dismiss_exec_id != Some(id.as_str()) { + return false; + } + + self.complete = true; + true + } + } + + #[derive(Default)] + struct CompletingView { + id: Option<&'static str>, + complete: bool, + } + + impl Renderable for CompletingView { + fn render(&self, _area: Rect, _buf: &mut Buffer) {} + + fn desired_height(&self, _width: u16) -> u16 { + 0 + } + } + + impl BottomPaneView for CompletingView { + fn handle_key_event(&mut self, key_event: KeyEvent) { + if key_event.code == KeyCode::Enter { + self.complete = true; + } + } + + fn is_complete(&self) -> bool { + self.complete + } + + fn view_id(&self) -> Option<&'static str> { + self.id + } + } + #[test] fn ctrl_c_on_modal_consumes_without_showing_quit_hint() { let (tx_raw, _rx) = unbounded_channel::(); @@ -1373,6 +1496,89 @@ mod tests { ); } + #[test] + fn dismiss_app_server_request_removes_matching_buried_view() { + let (tx_raw, _rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let mut pane = test_pane(tx); + + pane.push_view(Box::new(DismissibleView { + id: Some("buried"), + dismiss_exec_id: Some("request-1"), + complete: false, + })); + pane.push_view(Box::new(DismissibleView { + id: Some("top"), + dismiss_exec_id: None, + complete: false, + })); + + assert!( + pane.dismiss_app_server_request(&ResolvedAppServerRequest::ExecApproval { + id: "request-1".to_string(), + }) + ); + assert_eq!(pane.view_stack.len(), 1); + assert_eq!( + pane.view_stack.last().and_then(|view| view.view_id()), + Some("top") + ); + } + + #[test] + fn dismiss_app_server_request_returns_false_when_no_view_matches() { + let (tx_raw, _rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let mut pane = test_pane(tx); + + pane.push_view(Box::new(DismissibleView { + id: Some("first"), + dismiss_exec_id: Some("other-request"), + complete: false, + })); + pane.push_view(Box::new(DismissibleView { + id: Some("second"), + dismiss_exec_id: None, + complete: false, + })); + + assert!( + !pane.dismiss_app_server_request(&ResolvedAppServerRequest::ExecApproval { + id: "request-1".to_string(), + }) + ); + assert_eq!(pane.view_stack.len(), 2); + assert_eq!( + pane.view_stack.last().and_then(|view| view.view_id()), + Some("second") + ); + } + + #[test] + fn completing_top_view_preserves_underlying_view() { + let (tx_raw, _rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let mut pane = test_pane(tx); + + pane.push_view(Box::new(DismissibleView { + id: Some("underlying"), + dismiss_exec_id: None, + complete: false, + })); + pane.push_view(Box::new(CompletingView { + id: Some("top"), + complete: false, + })); + + pane.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + + assert_eq!(pane.view_stack.len(), 1); + assert_eq!( + pane.view_stack.last().and_then(|view| view.view_id()), + Some("underlying") + ); + } + #[test] fn composer_shown_after_denied_while_task_running() { let (tx_raw, _rx) = unbounded_channel::(); diff --git a/codex-rs/tui/src/bottom_pane/request_user_input/mod.rs b/codex-rs/tui/src/bottom_pane/request_user_input/mod.rs index f7db7c65dd..bc07c06e0b 100644 --- a/codex-rs/tui/src/bottom_pane/request_user_input/mod.rs +++ b/codex-rs/tui/src/bottom_pane/request_user_input/mod.rs @@ -10,6 +10,7 @@ use std::collections::HashMap; use std::collections::VecDeque; use std::path::PathBuf; +use crate::app::app_server_requests::ResolvedAppServerRequest; use crossterm::event::KeyCode; use crossterm::event::KeyEvent; use crossterm::event::KeyEventKind; @@ -198,6 +199,17 @@ impl RequestUserInputOverlay { self.request.questions.len() } + fn advance_queue_or_complete(&mut self) { + if let Some(next) = self.queue.pop_front() { + self.request = next; + self.reset_for_request(); + self.ensure_focus_available(); + self.restore_current_draft(); + } else { + self.done = true; + } + } + fn has_options(&self) -> bool { self.current_question() .and_then(|question| question.options.as_ref()) @@ -759,14 +771,23 @@ impl RequestUserInputOverlay { interrupted: false, }, ))); - if let Some(next) = self.queue.pop_front() { - self.request = next; - self.reset_for_request(); - self.ensure_focus_available(); - self.restore_current_draft(); - } else { - self.done = true; + self.advance_queue_or_complete(); + } + + fn dismiss_resolved_request(&mut self, request: &ResolvedAppServerRequest) -> bool { + let ResolvedAppServerRequest::UserInput { call_id } = request else { + return false; + }; + + let queue_len = self.queue.len(); + self.queue + .retain(|queued_request| queued_request.call_id != *call_id); + if self.request.call_id == *call_id { + self.advance_queue_or_complete(); + return true; } + + self.queue.len() != queue_len } fn open_unanswered_confirmation(&mut self) { @@ -1273,6 +1294,10 @@ impl BottomPaneView for RequestUserInputOverlay { self.queue.push_back(request); None } + + fn dismiss_app_server_request(&mut self, request: &ResolvedAppServerRequest) -> bool { + self.dismiss_resolved_request(request) + } } #[cfg(test)] @@ -1532,6 +1557,118 @@ mod tests { expect_interrupt_only(&mut rx); } + #[test] + fn resolved_request_dismisses_overlay_without_emitting_events() { + let (tx, mut rx) = test_sender(); + let mut overlay = RequestUserInputOverlay::new( + RequestUserInputEvent { + call_id: "call-1".to_string(), + turn_id: "turn-1".to_string(), + questions: vec![question_with_options("q1", "First")], + }, + tx, + /*has_input_focus*/ true, + /*enhanced_keys_supported*/ false, + /*disable_paste_burst*/ false, + ); + + assert!( + overlay.dismiss_app_server_request(&ResolvedAppServerRequest::UserInput { + call_id: "call-1".to_string(), + }) + ); + assert!(overlay.done, "resolved request should close the overlay"); + assert!( + rx.try_recv().is_err(), + "dismissing a stale request should not emit an interrupt or answer" + ); + } + + #[test] + fn resolved_current_request_advances_to_next_same_turn_prompt() { + let (tx, mut rx) = test_sender(); + let mut overlay = RequestUserInputOverlay::new( + RequestUserInputEvent { + call_id: "call-1".to_string(), + turn_id: "turn-1".to_string(), + questions: vec![question_with_options("q1", "First")], + }, + tx, + /*has_input_focus*/ true, + /*enhanced_keys_supported*/ false, + /*disable_paste_burst*/ false, + ); + overlay.try_consume_user_input_request(RequestUserInputEvent { + call_id: "call-2".to_string(), + turn_id: "turn-1".to_string(), + questions: vec![question_with_options("q2", "Second")], + }); + + assert!( + overlay.dismiss_app_server_request(&ResolvedAppServerRequest::UserInput { + call_id: "call-1".to_string(), + }) + ); + + assert!(!overlay.done, "newer same-turn prompt should stay pending"); + assert_eq!(overlay.request.call_id, "call-2"); + assert_eq!(overlay.request.turn_id, "turn-1"); + assert_eq!(overlay.request.questions[0].id, "q2"); + assert!( + rx.try_recv().is_err(), + "dismissing a stale request should not emit an interrupt or answer" + ); + } + + #[test] + fn resolved_queued_request_removes_only_that_prompt() { + let (tx, mut rx) = test_sender(); + let mut overlay = RequestUserInputOverlay::new( + RequestUserInputEvent { + call_id: "call-1".to_string(), + turn_id: "turn-1".to_string(), + questions: vec![question_with_options("q1", "First")], + }, + tx, + /*has_input_focus*/ true, + /*enhanced_keys_supported*/ false, + /*disable_paste_burst*/ false, + ); + overlay.try_consume_user_input_request(RequestUserInputEvent { + call_id: "call-2".to_string(), + turn_id: "turn-1".to_string(), + questions: vec![question_with_options("q2", "Second")], + }); + overlay.try_consume_user_input_request(RequestUserInputEvent { + call_id: "call-3".to_string(), + turn_id: "turn-1".to_string(), + questions: vec![question_with_options("q3", "Third")], + }); + + assert!( + overlay.dismiss_app_server_request(&ResolvedAppServerRequest::UserInput { + call_id: "call-2".to_string(), + }) + ); + + assert_eq!(overlay.request.call_id, "call-1"); + assert!( + rx.try_recv().is_err(), + "dismissing a stale queued request should not emit an event" + ); + overlay.submit_answers(); + assert_eq!(overlay.request.call_id, "call-3"); + assert_eq!(overlay.request.questions[0].id, "q3"); + assert!( + rx.try_recv().is_ok(), + "submitting the still-current prompt should emit an answer" + ); + assert!( + rx.try_recv().is_ok(), + "submitting the still-current prompt should emit a history cell" + ); + } + #[test] fn options_can_submit_empty_when_unanswered() { let (tx, mut rx) = test_sender(); diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index e9f56a0ef0..599655b364 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -43,6 +43,7 @@ use std::time::Duration; use std::time::Instant; use self::realtime::PendingSteerCompareKey; +use crate::app::app_server_requests::ResolvedAppServerRequest; use crate::app_command::AppCommand; use crate::app_event::RealtimeAudioDeviceKind; use crate::app_server_approval_conversions::network_approval_context_to_core; @@ -2163,6 +2164,16 @@ impl ChatWidget { self.request_redraw(); } + pub(crate) fn dismiss_app_server_request(&mut self, request: &ResolvedAppServerRequest) { + // A remotely resolved request must not remain user-actionable. It may be + // materialized in the bottom pane or still deferred behind active streaming. + let removed_deferred = self.interrupts.remove_resolved_prompt(request); + let removed_visible = self.bottom_pane.dismiss_app_server_request(request); + if removed_deferred || removed_visible { + self.request_redraw(); + } + } + pub(crate) fn open_feedback_consent(&mut self, category: crate::app_event::FeedbackCategory) { let snapshot = self.feedback.snapshot(self.thread_id); let params = crate::bottom_pane::feedback_upload_consent_params( diff --git a/codex-rs/tui/src/chatwidget/interrupts.rs b/codex-rs/tui/src/chatwidget/interrupts.rs index 0a3fc80016..f41e3b3a47 100644 --- a/codex-rs/tui/src/chatwidget/interrupts.rs +++ b/codex-rs/tui/src/chatwidget/interrupts.rs @@ -1,5 +1,6 @@ use std::collections::VecDeque; +use crate::app::app_server_requests::ResolvedAppServerRequest; use codex_protocol::approvals::ElicitationRequestEvent; use codex_protocol::protocol::ApplyPatchApprovalRequestEvent; use codex_protocol::protocol::ExecApprovalRequestEvent; @@ -86,6 +87,13 @@ impl InterruptManager { self.queue.push_back(QueuedInterrupt::PatchEnd(ev)); } + pub(crate) fn remove_resolved_prompt(&mut self, request: &ResolvedAppServerRequest) -> bool { + let original_len = self.queue.len(); + self.queue + .retain(|queued| !queued.matches_resolved_prompt(request)); + self.queue.len() != original_len + } + pub(crate) fn flush_all(&mut self, chat: &mut ChatWidget) { while let Some(q) = self.queue.pop_front() { match q { @@ -103,3 +111,144 @@ impl InterruptManager { } } } + +impl QueuedInterrupt { + fn matches_resolved_prompt(&self, request: &ResolvedAppServerRequest) -> bool { + match self { + QueuedInterrupt::ExecApproval(ev) => { + matches!(request, ResolvedAppServerRequest::ExecApproval { id } + if ev.effective_approval_id() == id.as_str()) + } + QueuedInterrupt::ApplyPatchApproval(ev) => { + matches!(request, ResolvedAppServerRequest::FileChangeApproval { id } + if ev.call_id == id.as_str()) + } + QueuedInterrupt::Elicitation(ev) => { + matches!(request, ResolvedAppServerRequest::McpElicitation { + server_name, + request_id, + } if ev.server_name == server_name.as_str() && &ev.id == request_id) + } + QueuedInterrupt::RequestPermissions(ev) => { + matches!(request, ResolvedAppServerRequest::PermissionsApproval { id } + if ev.call_id == id.as_str()) + } + QueuedInterrupt::RequestUserInput(ev) => { + matches!(request, ResolvedAppServerRequest::UserInput { call_id } + if ev.call_id == call_id.as_str()) + } + QueuedInterrupt::ExecBegin(_) + | QueuedInterrupt::ExecEnd(_) + | QueuedInterrupt::McpBegin(_) + | QueuedInterrupt::McpEnd(_) + | QueuedInterrupt::PatchEnd(_) => false, + } + } +} + +#[cfg(test)] +mod tests { + use codex_protocol::approvals::ExecApprovalRequestEvent; + use codex_protocol::protocol::ExecCommandBeginEvent; + use codex_protocol::protocol::ExecCommandSource; + use codex_protocol::request_user_input::RequestUserInputEvent; + use codex_utils_absolute_path::AbsolutePathBuf; + use pretty_assertions::assert_eq; + + use super::*; + + fn user_input(call_id: &str, turn_id: &str) -> RequestUserInputEvent { + RequestUserInputEvent { + call_id: call_id.to_string(), + turn_id: turn_id.to_string(), + questions: Vec::new(), + } + } + + fn exec_approval(call_id: &str, approval_id: Option<&str>) -> ExecApprovalRequestEvent { + ExecApprovalRequestEvent { + call_id: call_id.to_string(), + approval_id: approval_id.map(str::to_string), + turn_id: "turn".to_string(), + command: vec!["true".to_string()], + cwd: AbsolutePathBuf::current_dir().expect("current dir"), + reason: None, + network_approval_context: None, + proposed_execpolicy_amendment: None, + proposed_network_policy_amendments: None, + additional_permissions: None, + available_decisions: None, + parsed_cmd: Vec::new(), + } + } + + fn exec_begin(call_id: &str) -> ExecCommandBeginEvent { + ExecCommandBeginEvent { + call_id: call_id.to_string(), + process_id: None, + turn_id: "turn".to_string(), + command: vec!["true".to_string()], + cwd: AbsolutePathBuf::current_dir().expect("current dir"), + parsed_cmd: Vec::new(), + source: ExecCommandSource::Agent, + interaction_input: None, + } + } + + #[test] + fn remove_resolved_prompt_removes_matching_user_input_only() { + let mut manager = InterruptManager::new(); + manager.push_user_input(user_input("call-a", "turn")); + manager.push_user_input(user_input("call-b", "turn")); + + assert!( + manager.remove_resolved_prompt(&ResolvedAppServerRequest::UserInput { + call_id: "call-b".to_string(), + }) + ); + + assert_eq!(manager.queue.len(), 1); + let Some(QueuedInterrupt::RequestUserInput(remaining)) = manager.queue.front() else { + panic!("expected remaining queued user input"); + }; + assert_eq!(remaining.call_id, "call-a"); + } + + #[test] + fn remove_resolved_prompt_matches_exec_approval_id() { + let mut manager = InterruptManager::new(); + manager.push_exec_approval(exec_approval("call", Some("approval"))); + + assert!( + !manager.remove_resolved_prompt(&ResolvedAppServerRequest::ExecApproval { + id: "call".to_string(), + }) + ); + assert_eq!(manager.queue.len(), 1); + + assert!( + manager.remove_resolved_prompt(&ResolvedAppServerRequest::ExecApproval { + id: "approval".to_string(), + }) + ); + assert!(manager.queue.is_empty()); + } + + #[test] + fn remove_resolved_prompt_keeps_lifecycle_events() { + let mut manager = InterruptManager::new(); + manager.push_exec_begin(exec_begin("call")); + + assert!( + !manager.remove_resolved_prompt(&ResolvedAppServerRequest::ExecApproval { + id: "call".to_string(), + }) + ); + + assert_eq!(manager.queue.len(), 1); + assert!(matches!( + manager.queue.front(), + Some(QueuedInterrupt::ExecBegin(_)) + )); + } +} From 83dc8da9cccb8aadb649ccb6ee10ac01e3600e94 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 15 Apr 2026 22:09:41 +0100 Subject: [PATCH 40/84] Re-enable it (#18002) Reverts openai/codex#17981 --- codex-rs/features/src/lib.rs | 6 +++++- codex-rs/tui/src/bottom_pane/command_popup.rs | 2 +- codex-rs/tui/src/slash_command.rs | 1 - 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rs index 10e79f492a..4d7a83b628 100644 --- a/codex-rs/features/src/lib.rs +++ b/codex-rs/features/src/lib.rs @@ -674,7 +674,11 @@ pub const FEATURES: &[FeatureSpec] = &[ FeatureSpec { id: Feature::MemoryTool, key: "memories", - stage: Stage::UnderDevelopment, + stage: Stage::Experimental { + name: "Memories", + menu_description: "Allow Codex to create new memories from conversations and bring relevant memories into new conversations.", + announcement: "NEW: Codex can now generate and uses memories. Try is now with `/memories`", + }, default_enabled: false, }, FeatureSpec { diff --git a/codex-rs/tui/src/bottom_pane/command_popup.rs b/codex-rs/tui/src/bottom_pane/command_popup.rs index 8bef8ddbc8..29da003dc2 100644 --- a/codex-rs/tui/src/bottom_pane/command_popup.rs +++ b/codex-rs/tui/src/bottom_pane/command_popup.rs @@ -293,7 +293,7 @@ mod tests { CommandItem::Builtin(cmd) => cmd.command(), }) .collect(); - assert_eq!(cmds, vec!["model", "mention", "mcp"]); + assert_eq!(cmds, vec!["model", "memories", "mention", "mcp"]); } #[test] diff --git a/codex-rs/tui/src/slash_command.rs b/codex-rs/tui/src/slash_command.rs index 70aed459ea..759bedbad7 100644 --- a/codex-rs/tui/src/slash_command.rs +++ b/codex-rs/tui/src/slash_command.rs @@ -195,7 +195,6 @@ impl SlashCommand { SlashCommand::SandboxReadRoot => cfg!(target_os = "windows"), SlashCommand::Copy => !cfg!(target_os = "android"), SlashCommand::Rollout | SlashCommand::TestApproval => cfg!(debug_assertions), - SlashCommand::Memories => false, _ => true, } } From e70ccdeaf79c3298a5a37e94449684a81dc63b8c Mon Sep 17 00:00:00 2001 From: xl-openai Date: Wed, 15 Apr 2026 14:16:41 -0700 Subject: [PATCH 41/84] feat: Support alternate marketplace manifests and local string (#17885) - Discover marketplace manifests from different supported layout paths instead of only .agents/plugins/marketplace.json. - Accept local plugin sources written either as { source: "local", path: ... } or as a direct string path. - Skip unsupported or invalid plugin source entries without failing the entire marketplace, and keep valid local plugins loadable. --- .../src/plugins/installed_marketplaces.rs | 5 +- codex-rs/core/src/plugins/marketplace.rs | 306 +++++++++++------- .../core/src/plugins/marketplace_tests.rs | 255 ++++++++++++++- 3 files changed, 434 insertions(+), 132 deletions(-) diff --git a/codex-rs/core/src/plugins/installed_marketplaces.rs b/codex-rs/core/src/plugins/installed_marketplaces.rs index df69111a27..ad1918d51d 100644 --- a/codex-rs/core/src/plugins/installed_marketplaces.rs +++ b/codex-rs/core/src/plugins/installed_marketplaces.rs @@ -4,6 +4,7 @@ use std::path::Path; use std::path::PathBuf; use tracing::warn; +use super::marketplace::find_marketplace_manifest_path; use super::validate_plugin_segment; pub const INSTALLED_MARKETPLACES_DIR: &str = ".tmp/marketplaces"; @@ -50,9 +51,7 @@ pub(crate) fn installed_marketplace_roots_from_config( marketplace, &default_install_root, )?; - path.join(".agents/plugins/marketplace.json") - .is_file() - .then_some(path) + find_marketplace_manifest_path(&path).map(|_| path) }) .filter_map(|path| AbsolutePathBuf::try_from(path).ok()) .collect::>(); diff --git a/codex-rs/core/src/plugins/marketplace.rs b/codex-rs/core/src/plugins/marketplace.rs index 3cb1756f80..96c73e8e19 100644 --- a/codex-rs/core/src/plugins/marketplace.rs +++ b/codex-rs/core/src/plugins/marketplace.rs @@ -9,6 +9,8 @@ use codex_protocol::protocol::Product; use codex_utils_absolute_path::AbsolutePathBuf; use dirs::home_dir; use serde::Deserialize; +use serde::Deserializer; +use serde_json::Value as JsonValue; use std::fs; use std::io; use std::path::Component; @@ -16,7 +18,10 @@ use std::path::Path; use std::path::PathBuf; use tracing::warn; -const MARKETPLACE_RELATIVE_PATH: &str = ".agents/plugins/marketplace.json"; +const MARKETPLACE_MANIFEST_RELATIVE_PATHS: &[&str] = &[ + ".agents/plugins/marketplace.json", + ".claude-plugin/marketplace.json", +]; #[derive(Debug, Clone, PartialEq, Eq)] pub struct ResolvedMarketplacePlugin { @@ -162,46 +167,50 @@ pub fn resolve_marketplace_plugin( ) -> Result { let marketplace = load_raw_marketplace_manifest(marketplace_path)?; let marketplace_name = marketplace.name; - let plugin = marketplace - .plugins - .into_iter() - .find(|plugin| plugin.name == plugin_name); - - let Some(plugin) = plugin else { - return Err(MarketplaceError::PluginNotFound { - plugin_name: plugin_name.to_string(), - marketplace_name, - }); - }; - - let RawMarketplaceManifestPlugin { - name, - source, - policy, - .. - } = plugin; - let install_policy = policy.installation; - let product_allowed = match policy.products.as_deref() { - None => true, - Some([]) => false, - Some(products) => { - restriction_product.is_some_and(|product| product.matches_product_restriction(products)) + let marketplace_name_for_not_found = marketplace_name.clone(); + for plugin in marketplace.plugins { + if plugin.name != plugin_name { + continue; } - }; - if install_policy == MarketplacePluginInstallPolicy::NotAvailable || !product_allowed { - return Err(MarketplaceError::PluginNotAvailable { - plugin_name: name, - marketplace_name, + + let RawMarketplaceManifestPlugin { + name, + source, + policy, + .. + } = plugin; + let install_policy = policy.installation; + let product_allowed = match policy.products.as_deref() { + None => true, + Some([]) => false, + Some(products) => restriction_product + .is_some_and(|product| product.matches_product_restriction(products)), + }; + if install_policy == MarketplacePluginInstallPolicy::NotAvailable || !product_allowed { + return Err(MarketplaceError::PluginNotAvailable { + plugin_name: name, + marketplace_name, + }); + } + + let Some(source_path) = + resolve_supported_plugin_source_path(marketplace_path, &name, source) + else { + continue; + }; + + return Ok(ResolvedMarketplacePlugin { + plugin_id: PluginId::new(name, marketplace_name).map_err(|err| match err { + PluginIdError::Invalid(message) => MarketplaceError::InvalidPlugin(message), + })?, + source_path, + auth_policy: policy.authentication, }); } - let plugin_id = PluginId::new(name, marketplace_name).map_err(|err| match err { - PluginIdError::Invalid(message) => MarketplaceError::InvalidPlugin(message), - })?; - Ok(ResolvedMarketplacePlugin { - plugin_id, - source_path: resolve_plugin_source_path(marketplace_path, source)?, - auth_policy: policy.authentication, + Err(MarketplaceError::PluginNotFound { + plugin_name: plugin_name.to_string(), + marketplace_name: marketplace_name_for_not_found, }) } @@ -212,16 +221,50 @@ pub fn list_marketplaces( } pub fn validate_marketplace_root(root: &Path) -> Result { - let path = AbsolutePathBuf::try_from(root.join(MARKETPLACE_RELATIVE_PATH)).map_err(|err| { - MarketplaceError::InvalidMarketplaceFile { - path: root.join(MARKETPLACE_RELATIVE_PATH), - message: format!("marketplace path must resolve to an absolute path: {err}"), - } - })?; + let Some(path) = find_marketplace_manifest_path(root) else { + return Err(MarketplaceError::InvalidMarketplaceFile { + path: root.to_path_buf(), + message: "marketplace root does not contain a supported manifest".to_string(), + }); + }; let marketplace = load_marketplace(&path)?; Ok(marketplace.name) } +pub(crate) fn find_marketplace_manifest_path(root: &Path) -> Option { + MARKETPLACE_MANIFEST_RELATIVE_PATHS + .iter() + .find_map(|relative_path| { + let path = root.join(relative_path); + if !path.is_file() { + return None; + } + AbsolutePathBuf::try_from(path).ok() + }) +} + +fn invalid_marketplace_layout_error(path: &AbsolutePathBuf) -> MarketplaceError { + MarketplaceError::InvalidMarketplaceFile { + path: path.to_path_buf(), + message: "marketplace file is not in a supported location".to_string(), + } +} + +fn marketplace_root_from_layout(marketplace_path: &Path, relative_path: &str) -> Option { + let mut current = marketplace_path; + for component in Path::new(relative_path).components().rev() { + let expected = match component { + Component::Normal(expected) => expected, + _ => return None, + }; + if current.file_name() != Some(expected) { + return None; + } + current = current.parent()?; + } + Some(current.to_path_buf()) +} + pub(crate) fn load_marketplace(path: &AbsolutePathBuf) -> Result { let marketplace = load_raw_marketplace_manifest(path)?; let mut plugins = Vec::new(); @@ -233,7 +276,9 @@ pub(crate) fn load_marketplace(path: &AbsolutePathBuf) -> Result Vec { let mut paths = Vec::new(); - if let Some(home) = home_dir { - let path = home.join(MARKETPLACE_RELATIVE_PATH); - if path.is_file() - && let Ok(path) = AbsolutePathBuf::try_from(path) - { - paths.push(path); - } + if let Some(home) = home_dir + && let Some(path) = find_marketplace_manifest_path(home) + { + paths.push(path); } for root in additional_roots { // Curated marketplaces can now come from an HTTP-downloaded directory that is not a git // checkout, so check the root directly before falling back to repo-root discovery. - let path = root.join(MARKETPLACE_RELATIVE_PATH); - if path.as_path().is_file() && !paths.contains(&path) { + if let Some(path) = find_marketplace_manifest_path(root.as_path()) + && !paths.contains(&path) + { paths.push(path); continue; } if let Some(repo_root) = get_git_repo_root(root.as_path()) && let Ok(repo_root) = AbsolutePathBuf::try_from(repo_root) + && let Some(path) = find_marketplace_manifest_path(repo_root.as_path()) + && !paths.contains(&path) { - let path = repo_root.join(MARKETPLACE_RELATIVE_PATH); - if path.as_path().is_file() && !paths.contains(&path) { - paths.push(path); - } + paths.push(path); } } @@ -346,80 +388,83 @@ fn load_raw_marketplace_manifest( }) } -fn resolve_plugin_source_path( +fn resolve_supported_plugin_source_path( marketplace_path: &AbsolutePathBuf, + plugin_name: &str, source: RawMarketplaceManifestPluginSource, -) -> Result { +) -> Option { match source { RawMarketplaceManifestPluginSource::Local { path } => { - let Some(path) = path.strip_prefix("./") else { - return Err(MarketplaceError::InvalidMarketplaceFile { - path: marketplace_path.to_path_buf(), - message: "local plugin source path must start with `./`".to_string(), - }); - }; - if path.is_empty() { - return Err(MarketplaceError::InvalidMarketplaceFile { - path: marketplace_path.to_path_buf(), - message: "local plugin source path must not be empty".to_string(), - }); + match resolve_local_plugin_source_path(marketplace_path, &path) { + Ok(path) => Some(path), + Err(err) => { + warn!( + path = %marketplace_path.display(), + plugin = plugin_name, + error = %err, + "skipping marketplace plugin that failed to resolve" + ); + None + } } - - let relative_source_path = Path::new(path); - if relative_source_path - .components() - .any(|component| !matches!(component, Component::Normal(_))) - { - return Err(MarketplaceError::InvalidMarketplaceFile { - path: marketplace_path.to_path_buf(), - message: "local plugin source path must stay within the marketplace root" - .to_string(), - }); - } - - // `marketplace.json` lives under `/.agents/plugins/`, but local plugin paths - // are resolved relative to ``, not relative to the `plugins/` directory. - Ok(marketplace_root_dir(marketplace_path)?.join(relative_source_path)) + } + RawMarketplaceManifestPluginSource::Unsupported => { + warn!( + path = %marketplace_path.display(), + plugin = plugin_name, + "skipping marketplace plugin with unsupported source" + ); + None } } } +fn resolve_local_plugin_source_path( + marketplace_path: &AbsolutePathBuf, + source_path: &str, +) -> Result { + let Some(source_path) = source_path.strip_prefix("./") else { + return Err(MarketplaceError::InvalidMarketplaceFile { + path: marketplace_path.to_path_buf(), + message: "local plugin source path must start with `./`".to_string(), + }); + }; + if source_path.is_empty() { + return Err(MarketplaceError::InvalidMarketplaceFile { + path: marketplace_path.to_path_buf(), + message: "local plugin source path must not be empty".to_string(), + }); + } + + let relative_source_path = Path::new(source_path); + if relative_source_path + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(MarketplaceError::InvalidMarketplaceFile { + path: marketplace_path.to_path_buf(), + message: "local plugin source path must stay within the marketplace root".to_string(), + }); + } + + // `marketplace.json` lives under a supported marketplace layout beneath ``, + // but local plugin paths are resolved relative to ``. + Ok(marketplace_root_dir(marketplace_path)?.join(relative_source_path)) +} + fn marketplace_root_dir( marketplace_path: &AbsolutePathBuf, ) -> Result { - let Some(plugins_dir) = marketplace_path.parent() else { - return Err(MarketplaceError::InvalidMarketplaceFile { - path: marketplace_path.to_path_buf(), - message: "marketplace file must live under `/.agents/plugins/`".to_string(), - }); - }; - let Some(dot_agents_dir) = plugins_dir.parent() else { - return Err(MarketplaceError::InvalidMarketplaceFile { - path: marketplace_path.to_path_buf(), - message: "marketplace file must live under `/.agents/plugins/`".to_string(), - }); - }; - let Some(marketplace_root) = dot_agents_dir.parent() else { - return Err(MarketplaceError::InvalidMarketplaceFile { - path: marketplace_path.to_path_buf(), - message: "marketplace file must live under `/.agents/plugins/`".to_string(), - }); - }; - - if plugins_dir.as_path().file_name().and_then(|s| s.to_str()) != Some("plugins") - || dot_agents_dir - .as_path() - .file_name() - .and_then(|s| s.to_str()) - != Some(".agents") - { - return Err(MarketplaceError::InvalidMarketplaceFile { - path: marketplace_path.to_path_buf(), - message: "marketplace file must live under `/.agents/plugins/`".to_string(), - }); + for relative_path in MARKETPLACE_MANIFEST_RELATIVE_PATHS { + if let Some(marketplace_root) = + marketplace_root_from_layout(marketplace_path.as_path(), relative_path) + { + return AbsolutePathBuf::try_from(marketplace_root) + .map_err(|_| invalid_marketplace_layout_error(marketplace_path)); + } } - Ok(marketplace_root) + Err(invalid_marketplace_layout_error(marketplace_path)) } #[derive(Debug, Deserialize)] @@ -459,10 +504,33 @@ struct RawMarketplaceManifestPluginPolicy { products: Option>, } -#[derive(Debug, Deserialize)] -#[serde(tag = "source", rename_all = "lowercase")] +#[derive(Debug)] enum RawMarketplaceManifestPluginSource { Local { path: String }, + // Mixed-source marketplaces should still contribute the local plugins we can load. + Unsupported, +} + +impl<'de> Deserialize<'de> for RawMarketplaceManifestPluginSource { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let source = JsonValue::deserialize(deserializer)?; + Ok(match source { + JsonValue::String(path) => Self::Local { path }, + JsonValue::Object(object) => match object.get("source").and_then(JsonValue::as_str) { + Some("local") => match object.get("path").and_then(JsonValue::as_str) { + Some(path) => Self::Local { + path: path.to_string(), + }, + None => Self::Unsupported, + }, + _ => Self::Unsupported, + }, + _ => Self::Unsupported, + }) + } } fn resolve_marketplace_interface( diff --git a/codex-rs/core/src/plugins/marketplace_tests.rs b/codex-rs/core/src/plugins/marketplace_tests.rs index 226ae3ff6e..3412eff72c 100644 --- a/codex-rs/core/src/plugins/marketplace_tests.rs +++ b/codex-rs/core/src/plugins/marketplace_tests.rs @@ -1,8 +1,18 @@ use super::*; use codex_protocol::protocol::Product; use pretty_assertions::assert_eq; +use std::path::Path; use tempfile::tempdir; +const ALTERNATE_MARKETPLACE_RELATIVE_PATH: &str = ".claude-plugin/marketplace.json"; + +fn write_alternate_marketplace(repo_root: &Path, contents: &str) -> AbsolutePathBuf { + let marketplace_path = repo_root.join(ALTERNATE_MARKETPLACE_RELATIVE_PATH); + fs::create_dir_all(marketplace_path.parent().unwrap()).unwrap(); + fs::write(&marketplace_path, contents).unwrap(); + AbsolutePathBuf::try_from(marketplace_path).unwrap() +} + #[test] fn resolve_marketplace_plugin_finds_repo_marketplace_plugin() { let tmp = tempdir().unwrap(); @@ -45,6 +55,46 @@ fn resolve_marketplace_plugin_finds_repo_marketplace_plugin() { ); } +#[test] +fn resolve_marketplace_plugin_supports_alternate_layout_and_string_local_source() { + let tmp = tempdir().unwrap(); + let repo_root = tmp.path().join("repo"); + fs::create_dir_all(repo_root.join(".git")).unwrap(); + let marketplace_path = write_alternate_marketplace( + &repo_root, + r#"{ + "name": "alternate-marketplace", + "plugins": [ + { + "name": "string-source-plugin", + "source": "./plugins/string-source-plugin" + } + ] +}"#, + ); + + let resolved = resolve_marketplace_plugin( + &marketplace_path, + "string-source-plugin", + Some(Product::Codex), + ) + .unwrap(); + + assert_eq!( + resolved, + ResolvedMarketplacePlugin { + plugin_id: PluginId::new( + "string-source-plugin".to_string(), + "alternate-marketplace".to_string() + ) + .unwrap(), + source_path: AbsolutePathBuf::try_from(repo_root.join("plugins/string-source-plugin")) + .unwrap(), + auth_policy: MarketplacePluginAuthPolicy::OnInstall, + } + ); +} + #[test] fn resolve_marketplace_plugin_reports_missing_plugin() { let tmp = tempdir().unwrap(); @@ -70,6 +120,106 @@ fn resolve_marketplace_plugin_reports_missing_plugin() { ); } +#[test] +fn list_marketplaces_supports_alternate_manifest_layout() { + let tmp = tempdir().unwrap(); + let repo_root = tmp.path().join("repo"); + + fs::create_dir_all(repo_root.join(".git")).unwrap(); + let marketplace_path = write_alternate_marketplace( + &repo_root, + r#"{ + "name": "alternate-marketplace", + "plugins": [ + { + "name": "string-source-plugin", + "source": "./plugins/string-source-plugin" + } + ] +}"#, + ); + + let marketplaces = list_marketplaces_with_home( + &[AbsolutePathBuf::try_from(repo_root.clone()).unwrap()], + /*home_dir*/ None, + ) + .unwrap() + .marketplaces; + + assert_eq!( + marketplaces, + vec![Marketplace { + name: "alternate-marketplace".to_string(), + path: marketplace_path, + interface: None, + plugins: vec![MarketplacePlugin { + name: "string-source-plugin".to_string(), + source: MarketplacePluginSource::Local { + path: AbsolutePathBuf::try_from(repo_root.join("plugins/string-source-plugin")) + .unwrap(), + }, + policy: MarketplacePluginPolicy { + installation: MarketplacePluginInstallPolicy::Available, + authentication: MarketplacePluginAuthPolicy::OnInstall, + products: None, + }, + interface: None, + }], + }] + ); +} + +#[test] +fn list_marketplaces_prefers_first_supported_manifest_layout() { + let tmp = tempdir().unwrap(); + let repo_root = tmp.path().join("repo"); + + fs::create_dir_all(repo_root.join(".git")).unwrap(); + fs::create_dir_all(repo_root.join(".agents/plugins")).unwrap(); + fs::write( + repo_root.join(".agents/plugins/marketplace.json"), + r#"{ + "name": "agents-marketplace", + "plugins": [ + { + "name": "agents-plugin", + "source": { + "source": "local", + "path": "./plugins/agents-plugin" + } + } + ] +}"#, + ) + .unwrap(); + write_alternate_marketplace( + &repo_root, + r#"{ + "name": "alternate-marketplace", + "plugins": [ + { + "name": "string-source-plugin", + "source": "./plugins/string-source-plugin" + } + ] +}"#, + ); + + let marketplaces = list_marketplaces_with_home( + &[AbsolutePathBuf::try_from(repo_root.clone()).unwrap()], + /*home_dir*/ None, + ) + .unwrap() + .marketplaces; + + assert_eq!(marketplaces.len(), 1); + assert_eq!(marketplaces[0].name, "agents-marketplace"); + assert_eq!( + marketplaces[0].path, + AbsolutePathBuf::try_from(repo_root.join(".agents/plugins/marketplace.json")).unwrap() + ); +} + #[test] fn list_marketplaces_returns_home_and_repo_marketplaces() { let tmp = tempdir().unwrap(); @@ -413,7 +563,7 @@ fn list_marketplaces_reads_marketplace_display_name() { } #[test] -fn list_marketplaces_skips_marketplaces_that_fail_to_load() { +fn list_marketplaces_skips_invalid_plugins_but_keeps_marketplace() { let tmp = tempdir().unwrap(); let valid_repo_root = tmp.path().join("valid-repo"); let invalid_repo_root = tmp.path().join("invalid-repo"); @@ -465,8 +615,10 @@ fn list_marketplaces_skips_marketplaces_that_fail_to_load() { .unwrap() .marketplaces; - assert_eq!(marketplaces.len(), 1); + assert_eq!(marketplaces.len(), 2); assert_eq!(marketplaces[0].name, "valid-marketplace"); + assert_eq!(marketplaces[1].name, "invalid-marketplace"); + assert!(marketplaces[1].plugins.is_empty()); } #[test] @@ -522,6 +674,60 @@ fn list_marketplaces_reports_marketplace_load_errors() { ); } +#[test] +fn list_marketplaces_skips_unsupported_plugin_sources_but_keeps_local_plugins() { + let tmp = tempdir().unwrap(); + let repo_root = tmp.path().join("repo"); + + fs::create_dir_all(repo_root.join(".git")).unwrap(); + write_alternate_marketplace( + &repo_root, + r#"{ + "name": "mixed-source-marketplace", + "plugins": [ + { + "name": "local-plugin", + "source": "./plugins/local-plugin" + }, + { + "name": "url-plugin", + "source": { + "source": "url", + "url": "https://github.com/example/plugin.git" + } + }, + { + "name": "git-subdir-plugin", + "source": { + "source": "git-subdir", + "url": "owner/repo", + "path": "plugins/example", + "ref": "main" + } + } + ] +}"#, + ); + + let marketplaces = list_marketplaces_with_home( + &[AbsolutePathBuf::try_from(repo_root.clone()).unwrap()], + /*home_dir*/ None, + ) + .unwrap() + .marketplaces; + + assert_eq!(marketplaces.len(), 1); + assert_eq!(marketplaces[0].name, "mixed-source-marketplace"); + assert_eq!(marketplaces[0].plugins.len(), 1); + assert_eq!(marketplaces[0].plugins[0].name, "local-plugin"); + assert_eq!( + marketplaces[0].plugins[0].source, + MarketplacePluginSource::Local { + path: AbsolutePathBuf::try_from(repo_root.join("plugins/local-plugin")).unwrap(), + } + ); +} + #[test] fn list_marketplaces_resolves_plugin_interface_paths_to_absolute() { let tmp = tempdir().unwrap(); @@ -734,7 +940,7 @@ fn list_marketplaces_ignores_plugin_interface_assets_without_dot_slash() { } #[test] -fn resolve_marketplace_plugin_rejects_non_relative_local_paths() { +fn resolve_marketplace_plugin_skips_invalid_local_paths() { let tmp = tempdir().unwrap(); let repo_root = tmp.path().join("repo"); fs::create_dir_all(repo_root.join(".git")).unwrap(); @@ -756,17 +962,46 @@ fn resolve_marketplace_plugin_rejects_non_relative_local_paths() { ) .unwrap(); - let marketplace_path = - AbsolutePathBuf::try_from(repo_root.join(".agents/plugins/marketplace.json")).unwrap(); - let err = resolve_marketplace_plugin(&marketplace_path, "local-plugin", Some(Product::Codex)) + let err = resolve_marketplace_plugin( + &AbsolutePathBuf::try_from(repo_root.join(".agents/plugins/marketplace.json")).unwrap(), + "local-plugin", + Some(Product::Codex), + ) + .unwrap_err(); + + assert_eq!( + err.to_string(), + "plugin `local-plugin` was not found in marketplace `codex-curated`" + ); +} + +#[test] +fn resolve_marketplace_plugin_skips_unsupported_sources() { + let tmp = tempdir().unwrap(); + let repo_root = tmp.path().join("repo"); + fs::create_dir_all(repo_root.join(".git")).unwrap(); + let marketplace_path = write_alternate_marketplace( + &repo_root, + r#"{ + "name": "alternate-marketplace", + "plugins": [ + { + "name": "remote-plugin", + "source": { + "source": "url", + "url": "https://github.com/example/plugin.git" + } + } + ] +}"#, + ); + + let err = resolve_marketplace_plugin(&marketplace_path, "remote-plugin", Some(Product::Codex)) .unwrap_err(); assert_eq!( err.to_string(), - format!( - "invalid marketplace file `{}`: local plugin source path must start with `./`", - marketplace_path.display() - ) + "plugin `remote-plugin` was not found in marketplace `alternate-marketplace`" ); } From 17d94bd1e32f8f75676a0927ca297031f4b7d7ae Mon Sep 17 00:00:00 2001 From: evawong-oai Date: Wed, 15 Apr 2026 14:43:30 -0700 Subject: [PATCH 42/84] [docs] Revert extra changes from PR 17848 (#18003) ## Summary 1. Revert https://github.com/openai/codex/pull/17848 so the Bazel and `BUILD` file changes leave `main`. 2. Prepare for a narrower follow up that restores only `SECURITY.md`. ## Validation 1. Reviewed the revert diff against `main`. 2. Ran a clean diff check before push. --- .github/workflows/bazel.yml | 2 +- SECURITY.md | 4 --- codex-rs/app-server/BUILD.bazel | 3 -- codex-rs/core/BUILD.bazel | 55 +-------------------------------- codex-rs/tui/BUILD.bazel | 21 +------------ defs.bzl | 54 +++----------------------------- 6 files changed, 8 insertions(+), 131 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index c5a4c0c1c7..5d30916f9f 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -94,7 +94,7 @@ jobs: "${bazel_wrapper_args[@]}" \ -- \ test \ - --test_tag_filters=-argument-comment-lint,-flaky \ + --test_tag_filters=-argument-comment-lint \ --test_verbose_timeout_warnings \ --build_metadata=COMMIT_SHA=${GITHUB_SHA} \ -- \ diff --git a/SECURITY.md b/SECURITY.md index 29a9b903df..d6dd568910 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -11,7 +11,3 @@ Our security program is managed through Bugcrowd, and we ask that any validated ## Vulnerability Disclosure Program Our Vulnerability Program Guidelines are defined on our [Bugcrowd program page](https://bugcrowd.com/engagements/openai). - -## How to operate CODEX safely - -For details on Codex security boundaries, including sandboxing, approvals, and network controls, see [Agent approvals & security](https://developers.openai.com/codex/agent-approvals-security). diff --git a/codex-rs/app-server/BUILD.bazel b/codex-rs/app-server/BUILD.bazel index f32f811627..872f533be1 100644 --- a/codex-rs/app-server/BUILD.bazel +++ b/codex-rs/app-server/BUILD.bazel @@ -3,9 +3,6 @@ load("//:defs.bzl", "codex_rust_crate") codex_rust_crate( name = "app-server", crate_name = "codex_app_server", - integration_test_tags_extra_by_stem = { - "all": ["flaky"], - }, integration_test_timeout = "long", test_tags = ["no-sandbox"], ) diff --git a/codex-rs/core/BUILD.bazel b/codex-rs/core/BUILD.bazel index be07ef7999..434dc1f6a4 100644 --- a/codex-rs/core/BUILD.bazel +++ b/codex-rs/core/BUILD.bazel @@ -1,4 +1,4 @@ -load("//:defs.bzl", "codex_rust_crate", "workspace_root_test") +load("//:defs.bzl", "codex_rust_crate") filegroup( name = "model_availability_nux_fixtures", @@ -31,15 +31,6 @@ codex_rust_crate( "//codex-rs/apply-patch:apply_patch_tool_instructions.md", "templates/realtime/backend_prompt.md", ], - integration_test_flaky_by_stem = { - "all": [ - "suite::abort_tasks::interrupt_tool_records_history_entries", - "suite::apply_patch_cli::apply_patch_cli_rejects_invalid_hunk_header::applypatchmodeloutput_function_expects", - ], - "responses_headers": [ - "responses_stream_includes_turn_metadata_header_for_git_workspace_e2e", - ], - }, integration_test_timeout = "long", test_data_extra = [ "config.schema.json", @@ -57,12 +48,6 @@ codex_rust_crate( "//:AGENTS.md", ], test_tags = ["no-sandbox"], - unit_test_args = [ - "--skip", - "guardian::tests::guardian_parallel_reviews_fork_from_last_committed_trunk_history", - "--skip", - "agent::control::tests::completion_watcher_notifies_parent_when_child_is_missing", - ], unit_test_timeout = "long", extra_binaries = [ "//codex-rs/linux-sandbox:codex-linux-sandbox", @@ -72,41 +57,3 @@ codex_rust_crate( "//codex-rs/cli:codex", ], ) - -workspace_root_test( - name = "core-guardian-parallel-reviews-flaky-test", - env = { - "INSTA_WORKSPACE_ROOT": ".", - "INSTA_SNAPSHOT_PATH": "src", - }, - test_bin = ":core-unit-tests-bin", - workspace_root_marker = "//codex-rs/utils/cargo-bin:repo_root.marker", - args = [ - "--exact", - "guardian::tests::guardian_parallel_reviews_fork_from_last_committed_trunk_history", - ], - tags = [ - "flaky", - "no-sandbox", - ], - timeout = "long", -) - -workspace_root_test( - name = "core-completion-watcher-missing-child-flaky-test", - env = { - "INSTA_WORKSPACE_ROOT": ".", - "INSTA_SNAPSHOT_PATH": "src", - }, - test_bin = ":core-unit-tests-bin", - workspace_root_marker = "//codex-rs/utils/cargo-bin:repo_root.marker", - args = [ - "--exact", - "agent::control::tests::completion_watcher_notifies_parent_when_child_is_missing", - ], - tags = [ - "flaky", - "no-sandbox", - ], - timeout = "long", -) diff --git a/codex-rs/tui/BUILD.bazel b/codex-rs/tui/BUILD.bazel index a8cf752f96..33cc133431 100644 --- a/codex-rs/tui/BUILD.bazel +++ b/codex-rs/tui/BUILD.bazel @@ -1,4 +1,4 @@ -load("//:defs.bzl", "MACOS_WEBRTC_RUSTC_LINK_FLAGS", "codex_rust_crate", "workspace_root_test") +load("//:defs.bzl", "MACOS_WEBRTC_RUSTC_LINK_FLAGS", "codex_rust_crate") codex_rust_crate( name = "tui", @@ -20,27 +20,8 @@ codex_rust_crate( "src/**/snapshots/**", ]) + ["//codex-rs/core:model_availability_nux_fixtures"], integration_compile_data_extra = ["src/test_backend.rs"], - unit_test_args = [ - "--skip", - "app::tests::update_memory_settings_updates_current_thread_memory_mode", - ], extra_binaries = [ "//codex-rs/cli:codex", ], rustc_flags_extra = MACOS_WEBRTC_RUSTC_LINK_FLAGS, ) - -workspace_root_test( - name = "tui-update-memory-settings-flaky-test", - env = { - "INSTA_WORKSPACE_ROOT": ".", - "INSTA_SNAPSHOT_PATH": "src", - }, - test_bin = ":tui-unit-tests-bin", - workspace_root_marker = "//codex-rs/utils/cargo-bin:repo_root.marker", - args = [ - "--exact", - "app::tests::update_memory_settings_updates_current_thread_memory_mode", - ], - tags = ["flaky"], -) diff --git a/defs.bzl b/defs.bzl index 7cd89ce63e..6972b65a65 100644 --- a/defs.bzl +++ b/defs.bzl @@ -137,13 +137,10 @@ def codex_rust_crate( rustc_env = {}, deps_extra = [], integration_compile_data_extra = [], - integration_test_tags_extra_by_stem = {}, - integration_test_flaky_by_stem = {}, integration_test_args = [], integration_test_timeout = None, test_data_extra = [], test_tags = [], - unit_test_args = [], unit_test_timeout = None, extra_binaries = []): """Defines a Rust crate with library, binaries, and tests wired for Bazel + Cargo parity. @@ -173,9 +170,6 @@ def codex_rust_crate( deps_extra: Extra normal deps beyond @crates resolution. Typically only needed when features add additional deps. integration_compile_data_extra: Extra compile_data for integration tests. - integration_test_flaky_by_stem: Mapping from `tests/*.rs` stem to exact - Rust test names that should be split into dedicated flaky targets - and skipped from the main integration test target. integration_test_args: Optional args for integration test binaries. integration_test_timeout: Optional Bazel timeout for integration test targets generated from `tests/*.rs`. @@ -276,8 +270,6 @@ def codex_rust_crate( unit_test_kwargs = {} if unit_test_timeout: unit_test_kwargs["timeout"] = unit_test_timeout - if unit_test_args: - unit_test_kwargs["args"] = unit_test_args workspace_root_test( name = name + "-unit-tests", @@ -313,9 +305,11 @@ def codex_rust_crate( binary = Label(binary_label).name cargo_env["CARGO_BIN_EXE_" + binary] = "$(rlocationpath %s)" % binary_label - integration_test_base_kwargs = {} + integration_test_kwargs = {} + if integration_test_args: + integration_test_kwargs["args"] = integration_test_args if integration_test_timeout: - integration_test_base_kwargs["timeout"] = integration_test_timeout + integration_test_kwargs["timeout"] = integration_test_timeout for test in native.glob(["tests/*.rs"], allow_empty = True): test_file_stem = test.removeprefix("tests/").removesuffix(".rs") @@ -323,18 +317,6 @@ def codex_rust_crate( test_name = name + "-" + test_file_stem.replace("/", "-") if not test_name.endswith("-test"): test_name += "-test" - extra_test_tags = integration_test_tags_extra_by_stem.get(test_file_stem, []) - flaky_tests = integration_test_flaky_by_stem.get(test_file_stem, []) - stem_integration_test_args = [] + integration_test_args - for flaky_test in flaky_tests: - stem_integration_test_args += [ - "--skip", - flaky_test, - ] - - integration_test_kwargs = dict(integration_test_base_kwargs) - if stem_integration_test_args: - integration_test_kwargs["args"] = stem_integration_test_args rust_test( name = test_name, @@ -356,32 +338,6 @@ def codex_rust_crate( # `INSTA_WORKSPACE_ROOT="codex-rs"` is tuned for unit tests that # execute from the repo root and can misplace integration snapshots. env = cargo_env, - tags = test_tags + extra_test_tags, + tags = test_tags, **integration_test_kwargs ) - - for flaky_test in flaky_tests: - flaky_test_suffix = flaky_test.replace("::", "-").replace("/", "-").replace("_", "-") - flaky_test_kwargs = dict(integration_test_base_kwargs) - flaky_test_kwargs["args"] = [ - "--exact", - flaky_test, - ] - - rust_test( - name = test_name.removesuffix("-test") + "-flaky-" + flaky_test_suffix + "-test", - crate_name = test_crate_name + "_flaky_" + flaky_test.replace("::", "_").replace("/", "_").replace("-", "_"), - crate_root = test, - srcs = [test], - data = native.glob(["tests/**"], allow_empty = True) + sanitized_binaries + test_data_extra, - compile_data = native.glob(["tests/**"], allow_empty = True) + integration_compile_data_extra, - deps = all_crate_deps(normal = True, normal_dev = True) + maybe_deps + deps_extra, - rustc_flags = rustc_flags_extra + WINDOWS_RUSTC_LINK_FLAGS + [ - "--remap-path-prefix=../codex-rs=", - "--remap-path-prefix=codex-rs=", - ], - rustc_env = rustc_env, - env = cargo_env, - tags = test_tags + extra_test_tags + ["flaky"], - **flaky_test_kwargs - ) From 9e2fc318549f7546c87b9bd63db98a7c57f54cb4 Mon Sep 17 00:00:00 2001 From: Curtis 'Fjord' Hawthorne Date: Wed, 15 Apr 2026 14:43:33 -0700 Subject: [PATCH 43/84] Support original-detail metadata on MCP image outputs (#17714) ## Summary - honor `_meta["codex/imageDetail"] == "original"` on MCP image content and map it to `detail: "original"` where supported - strip that detail back out when the active model does not support original-detail image inputs - update code-mode `image(...)` to accept individual MCP image blocks - teach `js_repl` / `codex.emitImage(...)` to preserve the same hint from raw MCP image outputs - document the new `_meta` contract and add generic RMCP-backed coverage across protocol, core, code-mode, and js_repl paths --- codex-rs/code-mode/src/description.rs | 2 +- codex-rs/code-mode/src/runtime/callbacks.rs | 15 +- codex-rs/code-mode/src/runtime/value.rs | 124 +++- codex-rs/code-mode/src/service.rs | 148 ++++ codex-rs/core/src/original_image_detail.rs | 1 + codex-rs/core/src/project_doc.rs | 1 + codex-rs/core/src/project_doc_tests.rs | 4 +- codex-rs/core/src/tools/code_mode/mod.rs | 9 + codex-rs/core/src/tools/context.rs | 6 + codex-rs/core/src/tools/context_tests.rs | 3 + codex-rs/core/src/tools/handlers/mcp.rs | 2 + codex-rs/core/src/tools/js_repl/kernel.js | 8 + codex-rs/core/tests/suite/code_mode.rs | 156 +++-- codex-rs/core/tests/suite/rmcp_client.rs | 635 ++++++++++-------- codex-rs/protocol/src/models.rs | 84 ++- .../rmcp-client/src/bin/test_stdio_server.rs | 18 + codex-rs/tools/src/image_detail.rs | 18 + codex-rs/tools/src/image_detail_tests.rs | 37 + codex-rs/tools/src/lib.rs | 1 + docs/js_repl.md | 1 + 20 files changed, 905 insertions(+), 368 deletions(-) diff --git a/codex-rs/code-mode/src/description.rs b/codex-rs/code-mode/src/description.rs index 1aee1f7a3b..81e45add00 100644 --- a/codex-rs/code-mode/src/description.rs +++ b/codex-rs/code-mode/src/description.rs @@ -24,7 +24,7 @@ const EXEC_DESCRIPTION_TEMPLATE: &str = r#"Run JavaScript code to orchestrate/co - Global helpers: - `exit()`: Immediately ends the current script successfully (like an early return from the top level). - `text(value: string | number | boolean | undefined | null)`: Appends a text item. Non-string values are stringified with `JSON.stringify(...)` when possible. -- `image(imageUrlOrItem: string | { image_url: string; detail?: "auto" | "low" | "high" | "original" | null })`: Appends an image item. `image_url` can be an HTTPS URL or a base64-encoded `data:` URL. +- `image(imageUrlOrItem: string | { image_url: string; detail?: "auto" | "low" | "high" | "original" | null } | ImageContent, detail?: "auto" | "low" | "high" | "original" | null)`: Appends an image item. `image_url` can be an HTTPS URL or a base64-encoded `data:` URL. To forward an MCP tool image, pass an individual `ImageContent` block from `result.content`, for example `image(result.content[0])`. MCP image blocks may request original detail with `_meta: { "codex/imageDetail": "original" }`. When provided, the second `detail` argument overrides any detail embedded in the first argument. - `store(key: string, value: any)`: stores a serializable value under a string key for later `exec` calls in the same session. - `load(key: string)`: returns the stored value for a string key, or `undefined` if it is missing. - `notify(value: string | number | boolean | undefined | null)`: immediately injects an extra `custom_tool_call_output` for the current `exec` call. Values are stringified like `text(...)`. diff --git a/codex-rs/code-mode/src/runtime/callbacks.rs b/codex-rs/code-mode/src/runtime/callbacks.rs index ace6f61144..a9755f6eb0 100644 --- a/codex-rs/code-mode/src/runtime/callbacks.rs +++ b/codex-rs/code-mode/src/runtime/callbacks.rs @@ -109,7 +109,20 @@ pub(super) fn image_callback( } else { args.get(0) }; - let image_item = match normalize_output_image(scope, value) { + let detail_override = if args.length() < 2 { + None + } else { + let detail = args.get(1); + if detail.is_string() { + Some(detail.to_rust_string_lossy(scope)) + } else if detail.is_null() || detail.is_undefined() { + None + } else { + throw_type_error(scope, "image detail must be a string when provided"); + return; + } + }; + let image_item = match normalize_output_image(scope, value, detail_override) { Ok(image_item) => image_item, Err(()) => return, }; diff --git a/codex-rs/code-mode/src/runtime/value.rs b/codex-rs/code-mode/src/runtime/value.rs index eb0280142c..5c63434f4f 100644 --- a/codex-rs/code-mode/src/runtime/value.rs +++ b/codex-rs/code-mode/src/runtime/value.rs @@ -3,6 +3,9 @@ use serde_json::Value as JsonValue; use crate::response::FunctionCallOutputContentItem; use crate::response::ImageDetail; +const IMAGE_HELPER_EXPECTS_MESSAGE: &str = "image expects a non-empty image URL string, an object with image_url and optional detail, or a raw MCP image block"; +const CODEX_IMAGE_DETAIL_META_KEY: &str = "codex/imageDetail"; + pub(super) fn serialize_output_text( scope: &mut v8::PinScope<'_, '_>, value: v8::Local<'_, v8::Value>, @@ -34,45 +37,25 @@ pub(super) fn serialize_output_text( pub(super) fn normalize_output_image( scope: &mut v8::PinScope<'_, '_>, value: v8::Local<'_, v8::Value>, + detail_override: Option, ) -> Result { let result = (|| -> Result { let (image_url, detail) = if value.is_string() { (value.to_rust_string_lossy(scope), None) } else if value.is_object() && !value.is_array() { - let object = v8::Local::::try_from(value).map_err(|_| { - "image expects a non-empty image URL string or an object with image_url and optional detail".to_string() - })?; - let image_url_key = v8::String::new(scope, "image_url") - .ok_or_else(|| "failed to allocate image helper keys".to_string())?; - let detail_key = v8::String::new(scope, "detail") - .ok_or_else(|| "failed to allocate image helper keys".to_string())?; - let image_url = object - .get(scope, image_url_key.into()) - .filter(|value| value.is_string()) - .map(|value| value.to_rust_string_lossy(scope)) - .ok_or_else(|| { - "image expects a non-empty image URL string or an object with image_url and optional detail" - .to_string() - })?; - let detail = match object.get(scope, detail_key.into()) { - Some(value) if value.is_string() => Some(value.to_rust_string_lossy(scope)), - Some(value) if value.is_null() || value.is_undefined() => None, - Some(_) => return Err("image detail must be a string when provided".to_string()), - None => None, - }; - (image_url, detail) + let object = v8::Local::::try_from(value) + .map_err(|_| IMAGE_HELPER_EXPECTS_MESSAGE.to_string())?; + if let Some(image) = parse_non_mcp_output_image(scope, object)? { + image + } else { + parse_mcp_output_image(scope, value)? + } } else { - return Err( - "image expects a non-empty image URL string or an object with image_url and optional detail" - .to_string(), - ); + return Err(IMAGE_HELPER_EXPECTS_MESSAGE.to_string()); }; if image_url.is_empty() { - return Err( - "image expects a non-empty image URL string or an object with image_url and optional detail" - .to_string(), - ); + return Err(IMAGE_HELPER_EXPECTS_MESSAGE.to_string()); } let lower = image_url.to_ascii_lowercase(); if !(lower.starts_with("http://") @@ -82,6 +65,7 @@ pub(super) fn normalize_output_image( return Err("image expects an http(s) or data URL".to_string()); } + let detail = detail_override.or(detail); let detail = match detail { Some(detail) => { let normalized = detail.to_ascii_lowercase(); @@ -112,6 +96,86 @@ pub(super) fn normalize_output_image( } } +fn parse_non_mcp_output_image( + scope: &mut v8::PinScope<'_, '_>, + object: v8::Local<'_, v8::Object>, +) -> Result)>, String> { + let image_url_key = v8::String::new(scope, "image_url") + .ok_or_else(|| "failed to allocate image helper keys".to_string())?; + let Some(image_url) = object.get(scope, image_url_key.into()) else { + return Ok(None); + }; + if image_url.is_undefined() { + return Ok(None); + } + if !image_url.is_string() { + return Err(IMAGE_HELPER_EXPECTS_MESSAGE.to_string()); + } + let detail_key = v8::String::new(scope, "detail") + .ok_or_else(|| "failed to allocate image helper keys".to_string())?; + let detail = parse_image_detail_value(scope, object.get(scope, detail_key.into()))?; + Ok(Some((image_url.to_rust_string_lossy(scope), detail))) +} + +fn parse_mcp_output_image( + scope: &mut v8::PinScope<'_, '_>, + value: v8::Local<'_, v8::Value>, +) -> Result<(String, Option), String> { + let Some(result) = v8_value_to_json(scope, value)? else { + return Err(IMAGE_HELPER_EXPECTS_MESSAGE.to_string()); + }; + let JsonValue::Object(result) = result else { + return Err(IMAGE_HELPER_EXPECTS_MESSAGE.to_string()); + }; + let Some(item_type) = result.get("type").and_then(JsonValue::as_str) else { + return Err(IMAGE_HELPER_EXPECTS_MESSAGE.to_string()); + }; + if item_type != "image" { + return Err(format!( + "image only accepts MCP image blocks, got \"{item_type}\"" + )); + } + let data = result + .get("data") + .and_then(JsonValue::as_str) + .ok_or_else(|| "image expected MCP image data".to_string())?; + if data.is_empty() { + return Err("image expected MCP image data".to_string()); + } + + let image_url = if data.to_ascii_lowercase().starts_with("data:") { + data.to_string() + } else { + let mime_type = result + .get("mimeType") + .or_else(|| result.get("mime_type")) + .and_then(JsonValue::as_str) + .filter(|mime_type| !mime_type.is_empty()) + .unwrap_or("application/octet-stream"); + format!("data:{mime_type};base64,{data}") + }; + let detail = result + .get("_meta") + .and_then(JsonValue::as_object) + .and_then(|meta| meta.get(CODEX_IMAGE_DETAIL_META_KEY)) + .and_then(JsonValue::as_str) + .filter(|detail| *detail == "original") + .map(str::to_string); + Ok((image_url, detail)) +} + +fn parse_image_detail_value<'s>( + scope: &mut v8::PinScope<'s, '_>, + value: Option>, +) -> Result, String> { + match value { + Some(value) if value.is_string() => Ok(Some(value.to_rust_string_lossy(scope))), + Some(value) if value.is_null() || value.is_undefined() => Ok(None), + Some(_) => Err("image detail must be a string when provided".to_string()), + None => Ok(None), + } +} + pub(super) fn v8_value_to_json( scope: &mut v8::PinScope<'_, '_>, value: v8::Local<'_, v8::Value>, diff --git a/codex-rs/code-mode/src/service.rs b/codex-rs/code-mode/src/service.rs index 8a37880131..23ca7a7460 100644 --- a/codex-rs/code-mode/src/service.rs +++ b/codex-rs/code-mode/src/service.rs @@ -684,6 +684,154 @@ text(JSON.stringify(returnsUndefined)); ); } + #[tokio::test] + async fn image_helper_accepts_raw_mcp_image_block_with_original_detail() { + let service = CodeModeService::new(); + + let response = service + .execute(ExecuteRequest { + source: r#" +image({ + type: "image", + data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==", + mimeType: "image/png", + _meta: { "codex/imageDetail": "original" }, +}); +"# + .to_string(), + yield_time_ms: None, + ..execute_request("") + }) + .await + .unwrap(); + + assert_eq!( + response, + RuntimeResponse::Result { + cell_id: "1".to_string(), + content_items: vec![FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==".to_string(), + detail: Some(crate::ImageDetail::Original), + }], + stored_values: HashMap::new(), + error_text: None, + } + ); + } + + #[tokio::test] + async fn image_helper_second_arg_overrides_explicit_object_detail() { + let service = CodeModeService::new(); + + let response = service + .execute(ExecuteRequest { + source: r#" +image( + { + image_url: "https://example.com/image.jpg", + detail: "low", + }, + "original", +); +"# + .to_string(), + yield_time_ms: None, + ..execute_request("") + }) + .await + .unwrap(); + + assert_eq!( + response, + RuntimeResponse::Result { + cell_id: "1".to_string(), + content_items: vec![FunctionCallOutputContentItem::InputImage { + image_url: "https://example.com/image.jpg".to_string(), + detail: Some(crate::ImageDetail::Original), + }], + stored_values: HashMap::new(), + error_text: None, + } + ); + } + + #[tokio::test] + async fn image_helper_second_arg_overrides_raw_mcp_image_detail() { + let service = CodeModeService::new(); + + let response = service + .execute(ExecuteRequest { + source: r#" +image( + { + type: "image", + data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==", + mimeType: "image/png", + _meta: { "codex/imageDetail": "original" }, + }, + "low", +); +"# + .to_string(), + yield_time_ms: None, + ..execute_request("") + }) + .await + .unwrap(); + + assert_eq!( + response, + RuntimeResponse::Result { + cell_id: "1".to_string(), + content_items: vec![FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==".to_string(), + detail: Some(crate::ImageDetail::Low), + }], + stored_values: HashMap::new(), + error_text: None, + } + ); + } + + #[tokio::test] + async fn image_helper_rejects_raw_mcp_result_container() { + let service = CodeModeService::new(); + + let response = service + .execute(ExecuteRequest { + source: r#" +image({ + content: [ + { + type: "image", + data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==", + mimeType: "image/png", + _meta: { "codex/imageDetail": "original" }, + }, + ], + isError: false, +}); +"# + .to_string(), + yield_time_ms: None, + ..execute_request("") + }) + .await + .unwrap(); + + assert_eq!( + response, + RuntimeResponse::Result { + cell_id: "1".to_string(), + content_items: Vec::new(), + stored_values: HashMap::new(), + error_text: Some( + "image expects a non-empty image URL string, an object with image_url and optional detail, or a raw MCP image block".to_string(), + ), + } + ); + } + #[tokio::test] async fn terminate_waits_for_runtime_shutdown_before_responding() { let inner = test_inner(); diff --git a/codex-rs/core/src/original_image_detail.rs b/codex-rs/core/src/original_image_detail.rs index c3e5c3d9a5..adfed321b8 100644 --- a/codex-rs/core/src/original_image_detail.rs +++ b/codex-rs/core/src/original_image_detail.rs @@ -1,2 +1,3 @@ pub(crate) use codex_tools::can_request_original_image_detail; pub(crate) use codex_tools::normalize_output_image_detail; +pub(crate) use codex_tools::sanitize_original_image_detail; diff --git a/codex-rs/core/src/project_doc.rs b/codex-rs/core/src/project_doc.rs index 6659dfea7f..c4f70a6f55 100644 --- a/codex-rs/core/src/project_doc.rs +++ b/codex-rs/core/src/project_doc.rs @@ -59,6 +59,7 @@ fn render_js_repl_instructions(config: &Config) -> Option { section.push_str("- `codex.emitImage(...)` adds one image to the outer `js_repl` function output each time you call it, so you can call it multiple times to emit multiple images. It accepts a data URL, a single `input_image` item, an object like `{ bytes, mimeType }`, or a raw tool response object with exactly one image and no text. It rejects mixed text-and-image content.\n"); section.push_str("- `codex.tool(...)` and `codex.emitImage(...)` keep stable helper identities across cells. Saved references and persisted objects can reuse them in later cells, but async callbacks that fire after a cell finishes still fail because no exec is active.\n"); section.push_str("- Request full-resolution image processing with `detail: \"original\"` only when the `view_image` tool schema includes a `detail` argument. The same availability applies to `codex.emitImage(...)`: if `view_image.detail` is present, you may also pass `detail: \"original\"` there. Use this when high-fidelity image perception or precise localization is needed, especially for CUA agents.\n"); + section.push_str("- Raw MCP image blocks can request the same behavior by returning `_meta: { \"codex/imageDetail\": \"original\" }` on the image content item.\n"); section.push_str("- Example of sharing an in-memory Playwright screenshot: `await codex.emitImage({ bytes: await page.screenshot({ type: \"jpeg\", quality: 85 }), mimeType: \"image/jpeg\", detail: \"original\" })`.\n"); section.push_str("- Example of sharing a local image tool result: `await codex.emitImage(codex.tool(\"view_image\", { path: \"/absolute/path\", detail: \"original\" }))`.\n"); section.push_str("- When encoding an image to send with `codex.emitImage(...)` or `view_image`, prefer JPEG at about 85 quality when lossy compression is acceptable; use PNG when transparency or lossless detail matters. Smaller uploads are faster and less likely to hit size limits.\n"); diff --git a/codex-rs/core/src/project_doc_tests.rs b/codex-rs/core/src/project_doc_tests.rs index 705801317e..31f73805f7 100644 --- a/codex-rs/core/src/project_doc_tests.rs +++ b/codex-rs/core/src/project_doc_tests.rs @@ -205,7 +205,7 @@ async fn js_repl_instructions_are_appended_when_enabled() { let res = get_user_instructions(&cfg) .await .expect("js_repl instructions expected"); - let expected = "## JavaScript REPL (Node)\n- Use `js_repl` for Node-backed JavaScript with top-level await in a persistent kernel.\n- `js_repl` is a freeform/custom tool. Direct `js_repl` calls must send raw JavaScript tool input (optionally with first-line `// codex-js-repl: timeout_ms=15000`). Do not wrap code in JSON (for example `{\"code\":\"...\"}`), quotes, or markdown code fences.\n- Helpers: `codex.cwd`, `codex.homeDir`, `codex.tmpDir`, `codex.tool(name, args?)`, and `codex.emitImage(imageLike)`.\n- `codex.tool` executes a normal tool call and resolves to the raw tool output object. Use it for shell and non-shell tools alike. Nested tool outputs stay inside JavaScript unless you emit them explicitly.\n- `codex.emitImage(...)` adds one image to the outer `js_repl` function output each time you call it, so you can call it multiple times to emit multiple images. It accepts a data URL, a single `input_image` item, an object like `{ bytes, mimeType }`, or a raw tool response object with exactly one image and no text. It rejects mixed text-and-image content.\n- `codex.tool(...)` and `codex.emitImage(...)` keep stable helper identities across cells. Saved references and persisted objects can reuse them in later cells, but async callbacks that fire after a cell finishes still fail because no exec is active.\n- Request full-resolution image processing with `detail: \"original\"` only when the `view_image` tool schema includes a `detail` argument. The same availability applies to `codex.emitImage(...)`: if `view_image.detail` is present, you may also pass `detail: \"original\"` there. Use this when high-fidelity image perception or precise localization is needed, especially for CUA agents.\n- Example of sharing an in-memory Playwright screenshot: `await codex.emitImage({ bytes: await page.screenshot({ type: \"jpeg\", quality: 85 }), mimeType: \"image/jpeg\", detail: \"original\" })`.\n- Example of sharing a local image tool result: `await codex.emitImage(codex.tool(\"view_image\", { path: \"/absolute/path\", detail: \"original\" }))`.\n- When encoding an image to send with `codex.emitImage(...)` or `view_image`, prefer JPEG at about 85 quality when lossy compression is acceptable; use PNG when transparency or lossless detail matters. Smaller uploads are faster and less likely to hit size limits.\n- Top-level bindings persist across cells. If a cell throws, prior bindings remain available and bindings that finished initializing before the throw often remain usable in later cells. For code you plan to reuse across cells, prefer declaring or assigning it in direct top-level statements before operations that might throw. If you hit `SyntaxError: Identifier 'x' has already been declared`, first reuse the existing binding, reassign a previously declared `let`, or pick a new descriptive name. Use `{ ... }` only for a short temporary block when you specifically need local scratch names; do not wrap an entire cell in block scope if you want those names reusable later. Reset the kernel with `js_repl_reset` only when you need a clean state.\n- Top-level static import declarations (for example `import x from \"./file.js\"`) are currently unsupported in `js_repl`; use dynamic imports with `await import(\"pkg\")`, `await import(\"./file.js\")`, or `await import(\"/abs/path/file.mjs\")` instead. Imported local files must be ESM `.js`/`.mjs` files and run in the same REPL VM context. Bare package imports always resolve from REPL-global search roots (`CODEX_JS_REPL_NODE_MODULE_DIRS`, then cwd), not relative to the imported file location. Local files may statically import only other local relative/absolute/`file://` `.js`/`.mjs` files; package and builtin imports from local files must stay dynamic. `import.meta.resolve()` returns importable strings such as `file://...`, bare package names, and `node:...` specifiers. Local file modules reload between execs, while top-level bindings persist until `js_repl_reset`.\n- Avoid direct access to `process.stdout` / `process.stderr` / `process.stdin`; it can corrupt the JSON line protocol. Use `console.log`, `codex.tool(...)`, and `codex.emitImage(...)`."; + let expected = "## JavaScript REPL (Node)\n- Use `js_repl` for Node-backed JavaScript with top-level await in a persistent kernel.\n- `js_repl` is a freeform/custom tool. Direct `js_repl` calls must send raw JavaScript tool input (optionally with first-line `// codex-js-repl: timeout_ms=15000`). Do not wrap code in JSON (for example `{\"code\":\"...\"}`), quotes, or markdown code fences.\n- Helpers: `codex.cwd`, `codex.homeDir`, `codex.tmpDir`, `codex.tool(name, args?)`, and `codex.emitImage(imageLike)`.\n- `codex.tool` executes a normal tool call and resolves to the raw tool output object. Use it for shell and non-shell tools alike. Nested tool outputs stay inside JavaScript unless you emit them explicitly.\n- `codex.emitImage(...)` adds one image to the outer `js_repl` function output each time you call it, so you can call it multiple times to emit multiple images. It accepts a data URL, a single `input_image` item, an object like `{ bytes, mimeType }`, or a raw tool response object with exactly one image and no text. It rejects mixed text-and-image content.\n- `codex.tool(...)` and `codex.emitImage(...)` keep stable helper identities across cells. Saved references and persisted objects can reuse them in later cells, but async callbacks that fire after a cell finishes still fail because no exec is active.\n- Request full-resolution image processing with `detail: \"original\"` only when the `view_image` tool schema includes a `detail` argument. The same availability applies to `codex.emitImage(...)`: if `view_image.detail` is present, you may also pass `detail: \"original\"` there. Use this when high-fidelity image perception or precise localization is needed, especially for CUA agents.\n- Raw MCP image blocks can request the same behavior by returning `_meta: { \"codex/imageDetail\": \"original\" }` on the image content item.\n- Example of sharing an in-memory Playwright screenshot: `await codex.emitImage({ bytes: await page.screenshot({ type: \"jpeg\", quality: 85 }), mimeType: \"image/jpeg\", detail: \"original\" })`.\n- Example of sharing a local image tool result: `await codex.emitImage(codex.tool(\"view_image\", { path: \"/absolute/path\", detail: \"original\" }))`.\n- When encoding an image to send with `codex.emitImage(...)` or `view_image`, prefer JPEG at about 85 quality when lossy compression is acceptable; use PNG when transparency or lossless detail matters. Smaller uploads are faster and less likely to hit size limits.\n- Top-level bindings persist across cells. If a cell throws, prior bindings remain available and bindings that finished initializing before the throw often remain usable in later cells. For code you plan to reuse across cells, prefer declaring or assigning it in direct top-level statements before operations that might throw. If you hit `SyntaxError: Identifier 'x' has already been declared`, first reuse the existing binding, reassign a previously declared `let`, or pick a new descriptive name. Use `{ ... }` only for a short temporary block when you specifically need local scratch names; do not wrap an entire cell in block scope if you want those names reusable later. Reset the kernel with `js_repl_reset` only when you need a clean state.\n- Top-level static import declarations (for example `import x from \"./file.js\"`) are currently unsupported in `js_repl`; use dynamic imports with `await import(\"pkg\")`, `await import(\"./file.js\")`, or `await import(\"/abs/path/file.mjs\")` instead. Imported local files must be ESM `.js`/`.mjs` files and run in the same REPL VM context. Bare package imports always resolve from REPL-global search roots (`CODEX_JS_REPL_NODE_MODULE_DIRS`, then cwd), not relative to the imported file location. Local files may statically import only other local relative/absolute/`file://` `.js`/`.mjs` files; package and builtin imports from local files must stay dynamic. `import.meta.resolve()` returns importable strings such as `file://...`, bare package names, and `node:...` specifiers. Local file modules reload between execs, while top-level bindings persist until `js_repl_reset`.\n- Avoid direct access to `process.stdout` / `process.stderr` / `process.stdin`; it can corrupt the JSON line protocol. Use `console.log`, `codex.tool(...)`, and `codex.emitImage(...)`."; assert_eq!(res, expected); } @@ -224,7 +224,7 @@ async fn js_repl_tools_only_instructions_are_feature_gated() { let res = get_user_instructions(&cfg) .await .expect("js_repl instructions expected"); - let expected = "## JavaScript REPL (Node)\n- Use `js_repl` for Node-backed JavaScript with top-level await in a persistent kernel.\n- `js_repl` is a freeform/custom tool. Direct `js_repl` calls must send raw JavaScript tool input (optionally with first-line `// codex-js-repl: timeout_ms=15000`). Do not wrap code in JSON (for example `{\"code\":\"...\"}`), quotes, or markdown code fences.\n- Helpers: `codex.cwd`, `codex.homeDir`, `codex.tmpDir`, `codex.tool(name, args?)`, and `codex.emitImage(imageLike)`.\n- `codex.tool` executes a normal tool call and resolves to the raw tool output object. Use it for shell and non-shell tools alike. Nested tool outputs stay inside JavaScript unless you emit them explicitly.\n- `codex.emitImage(...)` adds one image to the outer `js_repl` function output each time you call it, so you can call it multiple times to emit multiple images. It accepts a data URL, a single `input_image` item, an object like `{ bytes, mimeType }`, or a raw tool response object with exactly one image and no text. It rejects mixed text-and-image content.\n- `codex.tool(...)` and `codex.emitImage(...)` keep stable helper identities across cells. Saved references and persisted objects can reuse them in later cells, but async callbacks that fire after a cell finishes still fail because no exec is active.\n- Request full-resolution image processing with `detail: \"original\"` only when the `view_image` tool schema includes a `detail` argument. The same availability applies to `codex.emitImage(...)`: if `view_image.detail` is present, you may also pass `detail: \"original\"` there. Use this when high-fidelity image perception or precise localization is needed, especially for CUA agents.\n- Example of sharing an in-memory Playwright screenshot: `await codex.emitImage({ bytes: await page.screenshot({ type: \"jpeg\", quality: 85 }), mimeType: \"image/jpeg\", detail: \"original\" })`.\n- Example of sharing a local image tool result: `await codex.emitImage(codex.tool(\"view_image\", { path: \"/absolute/path\", detail: \"original\" }))`.\n- When encoding an image to send with `codex.emitImage(...)` or `view_image`, prefer JPEG at about 85 quality when lossy compression is acceptable; use PNG when transparency or lossless detail matters. Smaller uploads are faster and less likely to hit size limits.\n- Top-level bindings persist across cells. If a cell throws, prior bindings remain available and bindings that finished initializing before the throw often remain usable in later cells. For code you plan to reuse across cells, prefer declaring or assigning it in direct top-level statements before operations that might throw. If you hit `SyntaxError: Identifier 'x' has already been declared`, first reuse the existing binding, reassign a previously declared `let`, or pick a new descriptive name. Use `{ ... }` only for a short temporary block when you specifically need local scratch names; do not wrap an entire cell in block scope if you want those names reusable later. Reset the kernel with `js_repl_reset` only when you need a clean state.\n- Top-level static import declarations (for example `import x from \"./file.js\"`) are currently unsupported in `js_repl`; use dynamic imports with `await import(\"pkg\")`, `await import(\"./file.js\")`, or `await import(\"/abs/path/file.mjs\")` instead. Imported local files must be ESM `.js`/`.mjs` files and run in the same REPL VM context. Bare package imports always resolve from REPL-global search roots (`CODEX_JS_REPL_NODE_MODULE_DIRS`, then cwd), not relative to the imported file location. Local files may statically import only other local relative/absolute/`file://` `.js`/`.mjs` files; package and builtin imports from local files must stay dynamic. `import.meta.resolve()` returns importable strings such as `file://...`, bare package names, and `node:...` specifiers. Local file modules reload between execs, while top-level bindings persist until `js_repl_reset`.\n- Do not call tools directly; use `js_repl` + `codex.tool(...)` for all tool calls, including shell commands.\n- MCP tools (if any) can also be called by name via `codex.tool(...)`.\n- Avoid direct access to `process.stdout` / `process.stderr` / `process.stdin`; it can corrupt the JSON line protocol. Use `console.log`, `codex.tool(...)`, and `codex.emitImage(...)`."; + let expected = "## JavaScript REPL (Node)\n- Use `js_repl` for Node-backed JavaScript with top-level await in a persistent kernel.\n- `js_repl` is a freeform/custom tool. Direct `js_repl` calls must send raw JavaScript tool input (optionally with first-line `// codex-js-repl: timeout_ms=15000`). Do not wrap code in JSON (for example `{\"code\":\"...\"}`), quotes, or markdown code fences.\n- Helpers: `codex.cwd`, `codex.homeDir`, `codex.tmpDir`, `codex.tool(name, args?)`, and `codex.emitImage(imageLike)`.\n- `codex.tool` executes a normal tool call and resolves to the raw tool output object. Use it for shell and non-shell tools alike. Nested tool outputs stay inside JavaScript unless you emit them explicitly.\n- `codex.emitImage(...)` adds one image to the outer `js_repl` function output each time you call it, so you can call it multiple times to emit multiple images. It accepts a data URL, a single `input_image` item, an object like `{ bytes, mimeType }`, or a raw tool response object with exactly one image and no text. It rejects mixed text-and-image content.\n- `codex.tool(...)` and `codex.emitImage(...)` keep stable helper identities across cells. Saved references and persisted objects can reuse them in later cells, but async callbacks that fire after a cell finishes still fail because no exec is active.\n- Request full-resolution image processing with `detail: \"original\"` only when the `view_image` tool schema includes a `detail` argument. The same availability applies to `codex.emitImage(...)`: if `view_image.detail` is present, you may also pass `detail: \"original\"` there. Use this when high-fidelity image perception or precise localization is needed, especially for CUA agents.\n- Raw MCP image blocks can request the same behavior by returning `_meta: { \"codex/imageDetail\": \"original\" }` on the image content item.\n- Example of sharing an in-memory Playwright screenshot: `await codex.emitImage({ bytes: await page.screenshot({ type: \"jpeg\", quality: 85 }), mimeType: \"image/jpeg\", detail: \"original\" })`.\n- Example of sharing a local image tool result: `await codex.emitImage(codex.tool(\"view_image\", { path: \"/absolute/path\", detail: \"original\" }))`.\n- When encoding an image to send with `codex.emitImage(...)` or `view_image`, prefer JPEG at about 85 quality when lossy compression is acceptable; use PNG when transparency or lossless detail matters. Smaller uploads are faster and less likely to hit size limits.\n- Top-level bindings persist across cells. If a cell throws, prior bindings remain available and bindings that finished initializing before the throw often remain usable in later cells. For code you plan to reuse across cells, prefer declaring or assigning it in direct top-level statements before operations that might throw. If you hit `SyntaxError: Identifier 'x' has already been declared`, first reuse the existing binding, reassign a previously declared `let`, or pick a new descriptive name. Use `{ ... }` only for a short temporary block when you specifically need local scratch names; do not wrap an entire cell in block scope if you want those names reusable later. Reset the kernel with `js_repl_reset` only when you need a clean state.\n- Top-level static import declarations (for example `import x from \"./file.js\"`) are currently unsupported in `js_repl`; use dynamic imports with `await import(\"pkg\")`, `await import(\"./file.js\")`, or `await import(\"/abs/path/file.mjs\")` instead. Imported local files must be ESM `.js`/`.mjs` files and run in the same REPL VM context. Bare package imports always resolve from REPL-global search roots (`CODEX_JS_REPL_NODE_MODULE_DIRS`, then cwd), not relative to the imported file location. Local files may statically import only other local relative/absolute/`file://` `.js`/`.mjs` files; package and builtin imports from local files must stay dynamic. `import.meta.resolve()` returns importable strings such as `file://...`, bare package names, and `node:...` specifiers. Local file modules reload between execs, while top-level bindings persist until `js_repl_reset`.\n- Do not call tools directly; use `js_repl` + `codex.tool(...)` for all tool calls, including shell commands.\n- MCP tools (if any) can also be called by name via `codex.tool(...)`.\n- Avoid direct access to `process.stdout` / `process.stderr` / `process.stdin`; it can corrupt the JSON line protocol. Use `console.log`, `codex.tool(...)`, and `codex.emitImage(...)`."; assert_eq!(res, expected); } diff --git a/codex-rs/core/src/tools/code_mode/mod.rs b/codex-rs/core/src/tools/code_mode/mod.rs index 0004ec6f51..8445fdf645 100644 --- a/codex-rs/core/src/tools/code_mode/mod.rs +++ b/codex-rs/core/src/tools/code_mode/mod.rs @@ -18,6 +18,8 @@ use tokio_util::sync::CancellationToken; use crate::codex::Session; use crate::codex::TurnContext; use crate::function_tool::FunctionCallError; +use crate::original_image_detail::can_request_original_image_detail; +use crate::original_image_detail::sanitize_original_image_detail as sanitize_image_detail_items; use crate::tools::ToolRouter; use crate::tools::context::FunctionToolOutput; use crate::tools::context::SharedTurnDiffTracker; @@ -160,12 +162,14 @@ pub(super) async fn handle_runtime_response( match response { RuntimeResponse::Yielded { content_items, .. } => { let mut content_items = into_function_call_output_content_items(content_items); + sanitize_runtime_image_detail(exec.turn.as_ref(), &mut content_items); content_items = truncate_code_mode_result(content_items, max_output_tokens); prepend_script_status(&mut content_items, &script_status, started_at.elapsed()); Ok(FunctionToolOutput::from_content(content_items, Some(true))) } RuntimeResponse::Terminated { content_items, .. } => { let mut content_items = into_function_call_output_content_items(content_items); + sanitize_runtime_image_detail(exec.turn.as_ref(), &mut content_items); content_items = truncate_code_mode_result(content_items, max_output_tokens); prepend_script_status(&mut content_items, &script_status, started_at.elapsed()); Ok(FunctionToolOutput::from_content(content_items, Some(true))) @@ -177,6 +181,7 @@ pub(super) async fn handle_runtime_response( .. } => { let mut content_items = into_function_call_output_content_items(content_items); + sanitize_runtime_image_detail(exec.turn.as_ref(), &mut content_items); exec.session .services .code_mode_service @@ -198,6 +203,10 @@ pub(super) async fn handle_runtime_response( } } +fn sanitize_runtime_image_detail(turn: &TurnContext, items: &mut [FunctionCallOutputContentItem]) { + sanitize_image_detail_items(can_request_original_image_detail(&turn.model_info), items); +} + fn format_script_status(response: &RuntimeResponse) -> String { match response { RuntimeResponse::Yielded { cell_id, .. } => { diff --git a/codex-rs/core/src/tools/context.rs b/codex-rs/core/src/tools/context.rs index dff1e444da..60bcbcda23 100644 --- a/codex-rs/core/src/tools/context.rs +++ b/codex-rs/core/src/tools/context.rs @@ -1,5 +1,6 @@ use crate::codex::Session; use crate::codex::TurnContext; +use crate::original_image_detail::sanitize_original_image_detail; use crate::tools::TELEMETRY_PREVIEW_MAX_BYTES; use crate::tools::TELEMETRY_PREVIEW_MAX_LINES; use crate::tools::TELEMETRY_PREVIEW_TRUNCATION_NOTICE; @@ -122,6 +123,7 @@ impl ToolOutput for CallToolResult { pub struct McpToolOutput { pub result: CallToolResult, pub wall_time: Duration, + pub original_image_detail_supported: bool, } impl ToolOutput for McpToolOutput { @@ -155,6 +157,10 @@ impl ToolOutput for McpToolOutput { impl McpToolOutput { fn response_payload(&self) -> FunctionCallOutputPayload { let mut payload = self.result.as_function_call_output_payload(); + if let Some(items) = payload.content_items_mut() { + sanitize_original_image_detail(self.original_image_detail_supported, items); + } + let wall_time_seconds = self.wall_time.as_secs_f64(); let header = format!("Wall time: {wall_time_seconds:.4} seconds\nOutput:"); diff --git a/codex-rs/core/src/tools/context_tests.rs b/codex-rs/core/src/tools/context_tests.rs index d45f351833..8df9159edd 100644 --- a/codex-rs/core/src/tools/context_tests.rs +++ b/codex-rs/core/src/tools/context_tests.rs @@ -98,6 +98,7 @@ fn mcp_tool_output_response_item_includes_wall_time() { meta: None, }, wall_time: std::time::Duration::from_millis(1250), + original_image_detail_supported: false, }; let response = output.to_response_item( @@ -149,6 +150,7 @@ fn mcp_tool_output_response_item_preserves_content_items() { meta: None, }, wall_time: std::time::Duration::from_millis(500), + original_image_detail_supported: false, }; let response = output.to_response_item( @@ -201,6 +203,7 @@ fn mcp_tool_output_code_mode_result_stays_raw_call_tool_result() { meta: None, }, wall_time: std::time::Duration::from_millis(1250), + original_image_detail_supported: false, }; let result = output.code_mode_result(&ToolPayload::Mcp { diff --git a/codex-rs/core/src/tools/handlers/mcp.rs b/codex-rs/core/src/tools/handlers/mcp.rs index cfe219f443..b24db734b9 100644 --- a/codex-rs/core/src/tools/handlers/mcp.rs +++ b/codex-rs/core/src/tools/handlers/mcp.rs @@ -3,6 +3,7 @@ use std::time::Instant; use crate::function_tool::FunctionCallError; use crate::mcp_tool_call::handle_mcp_tool_call; +use crate::original_image_detail::can_request_original_image_detail; use crate::tools::context::McpToolOutput; use crate::tools::context::ToolInvocation; use crate::tools::context::ToolPayload; @@ -56,6 +57,7 @@ impl ToolHandler for McpHandler { Ok(McpToolOutput { result, wall_time: started.elapsed(), + original_image_detail_supported: can_request_original_image_detail(&turn.model_info), }) } } diff --git a/codex-rs/core/src/tools/js_repl/kernel.js b/codex-rs/core/src/tools/js_repl/kernel.js index 5d31818521..3e5cf855f7 100644 --- a/codex-rs/core/src/tools/js_repl/kernel.js +++ b/codex-rs/core/src/tools/js_repl/kernel.js @@ -1330,6 +1330,13 @@ function normalizeMcpImageData(data, mimeType) { return `data:${normalizedMimeType};base64,${data}`; } +function parseMcpImageDetail(meta) { + if (!isPlainObject(meta) || meta["codex/imageDetail"] !== "original") { + return undefined; + } + return "original"; +} + function parseMcpToolResult(result) { if (typeof result === "string") { return { images: [], textCount: result.length > 0 ? 1 : 0 }; @@ -1362,6 +1369,7 @@ function parseMcpToolResult(result) { if (item.type === "image") { images.push({ image_url: normalizeMcpImageData(item.data, item.mimeType ?? item.mime_type), + detail: parseMcpImageDetail(item._meta), }); continue; } diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index f010c9e587..d514e801e9 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -178,7 +178,16 @@ async fn run_code_mode_turn_with_rmcp( prompt: &str, code: &str, ) -> Result<(TestCodex, ResponseMock)> { - run_code_mode_turn_with_rmcp_mode(server, prompt, code, /*code_mode_only*/ false).await + run_code_mode_turn_with_rmcp_model(server, prompt, code, "test-gpt-5.1-codex").await +} + +async fn run_code_mode_turn_with_rmcp_model( + server: &MockServer, + prompt: &str, + code: &str, + model: &'static str, +) -> Result<(TestCodex, ResponseMock)> { + run_code_mode_turn_with_rmcp_config(server, prompt, code, model, /*code_mode_only*/ false).await } async fn run_code_mode_turn_with_rmcp_mode( @@ -187,48 +196,57 @@ async fn run_code_mode_turn_with_rmcp_mode( code: &str, code_mode_only: bool, ) -> Result<(TestCodex, ResponseMock)> { - let rmcp_test_server_bin = stdio_server_bin()?; - let mut builder = test_codex() - .with_model("test-gpt-5.1-codex") - .with_config(move |config| { - let _ = if code_mode_only { - config.features.enable(Feature::CodeModeOnly) - } else { - config.features.enable(Feature::CodeMode) - }; + run_code_mode_turn_with_rmcp_config(server, prompt, code, "test-gpt-5.1-codex", code_mode_only) + .await +} - let mut servers = config.mcp_servers.get().clone(); - servers.insert( - "rmcp".to_string(), - McpServerConfig { - transport: McpServerTransportConfig::Stdio { - command: rmcp_test_server_bin, - args: Vec::new(), - env: Some(HashMap::from([( - "MCP_TEST_VALUE".to_string(), - "propagated-env".to_string(), - )])), - env_vars: Vec::new(), - cwd: None, - }, - enabled: true, - required: false, - supports_parallel_tool_calls: false, - disabled_reason: None, - startup_timeout_sec: Some(Duration::from_secs(10)), - tool_timeout_sec: None, - enabled_tools: None, - disabled_tools: None, - scopes: None, - oauth_resource: None, - tools: HashMap::new(), +async fn run_code_mode_turn_with_rmcp_config( + server: &MockServer, + prompt: &str, + code: &str, + model: &'static str, + code_mode_only: bool, +) -> Result<(TestCodex, ResponseMock)> { + let rmcp_test_server_bin = stdio_server_bin()?; + let mut builder = test_codex().with_model(model).with_config(move |config| { + let _ = if code_mode_only { + config.features.enable(Feature::CodeModeOnly) + } else { + config.features.enable(Feature::CodeMode) + }; + + let mut servers = config.mcp_servers.get().clone(); + servers.insert( + "rmcp".to_string(), + McpServerConfig { + transport: McpServerTransportConfig::Stdio { + command: rmcp_test_server_bin, + args: Vec::new(), + env: Some(HashMap::from([( + "MCP_TEST_VALUE".to_string(), + "propagated-env".to_string(), + )])), + env_vars: Vec::new(), + cwd: None, }, - ); - config - .mcp_servers - .set(servers) - .expect("test mcp servers should accept any configuration"); - }); + enabled: true, + required: false, + supports_parallel_tool_calls: false, + disabled_reason: None, + startup_timeout_sec: Some(Duration::from_secs(10)), + tool_timeout_sec: None, + enabled_tools: None, + disabled_tools: None, + scopes: None, + oauth_resource: None, + tools: HashMap::new(), + }, + ); + config + .mcp_servers + .set(servers) + .expect("test mcp servers should accept any configuration"); + }); let test = builder.build(server).await?; responses::mount_sse_once( @@ -1919,6 +1937,62 @@ image(out); Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn code_mode_can_use_mcp_image_result_with_image_helper() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let code = r#" +const out = await tools.mcp__rmcp__image_scenario({ + scenario: "image_only_original_detail", +}); +const imageItem = out.content.find((item) => item.type === "image"); +image(imageItem); +"#; + + let (_test, second_mock) = run_code_mode_turn_with_rmcp_model( + &server, + "use exec to call the rmcp image scenario tool and emit its image output", + code, + "gpt-5.3-codex", + ) + .await?; + + let req = second_mock.single_request(); + let items = custom_tool_output_items(&req, "call-1"); + let (_, success) = custom_tool_output_body_and_success(&req, "call-1"); + assert_ne!( + success, + Some(false), + "code_mode mcp image scenario call failed unexpectedly" + ); + assert_eq!(items.len(), 2); + assert_regex_match( + concat!( + r"(?s)\A", + r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z" + ), + text_item(&items, /*index*/ 0), + ); + + assert_eq!( + items[1].get("type").and_then(Value::as_str), + Some("input_image") + ); + + let emitted_image_url = items[1] + .get("image_url") + .and_then(Value::as_str) + .expect("image helper should emit an input_image item with image_url"); + assert!(emitted_image_url.starts_with("data:image/png;base64,")); + assert_eq!( + items[1].get("detail").and_then(Value::as_str), + Some("original") + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn code_mode_can_apply_patch_via_nested_tool() -> Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/core/tests/suite/rmcp_client.rs b/codex-rs/core/tests/suite/rmcp_client.rs index b86094f910..6d19e6ac52 100644 --- a/codex-rs/core/tests/suite/rmcp_client.rs +++ b/codex-rs/core/tests/suite/rmcp_client.rs @@ -11,6 +11,8 @@ use std::time::UNIX_EPOCH; use codex_config::types::McpServerConfig; use codex_config::types::McpServerTransportConfig; +use codex_core::config::Config; +use codex_features::Feature; use codex_login::CodexAuth; use codex_mcp::MCP_SANDBOX_STATE_META_CAPABILITY; use codex_models_manager::manager::RefreshStrategy; @@ -33,10 +35,12 @@ use codex_protocol::user_input::UserInput; use codex_utils_cargo_bin::cargo_bin; use core_test_support::assert_regex_match; use core_test_support::responses; +use core_test_support::responses::ev_custom_tool_call; use core_test_support::responses::mount_models_once; use core_test_support::responses::mount_sse_once; use core_test_support::skip_if_no_network; use core_test_support::stdio_server_bin; +use core_test_support::test_codex::TestCodex; use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event; use core_test_support::wait_for_event_with_timeout; @@ -82,6 +86,82 @@ enum McpCallEvent { End(String), } +async fn wait_for_mcp_tool(fixture: &TestCodex, tool_name: &str) -> anyhow::Result<()> { + let tools_ready_deadline = Instant::now() + Duration::from_secs(30); + loop { + fixture.codex.submit(Op::ListMcpTools).await?; + let list_event = wait_for_event_with_timeout( + &fixture.codex, + |ev| matches!(ev, EventMsg::McpListToolsResponse(_)), + Duration::from_secs(10), + ) + .await; + let EventMsg::McpListToolsResponse(tool_list) = list_event else { + unreachable!("event guard guarantees McpListToolsResponse"); + }; + if tool_list.tools.contains_key(tool_name) { + return Ok(()); + } + + let available_tools: Vec<&str> = tool_list.tools.keys().map(String::as_str).collect(); + if Instant::now() >= tools_ready_deadline { + panic!( + "timed out waiting for MCP tool {tool_name} to become available; discovered tools: {available_tools:?}" + ); + } + sleep(Duration::from_millis(200)).await; + } +} + +#[derive(Default)] +struct TestMcpServerOptions { + supports_parallel_tool_calls: bool, + tool_timeout_sec: Option, +} + +fn stdio_transport( + command: String, + env: Option>, + env_vars: Vec, +) -> McpServerTransportConfig { + McpServerTransportConfig::Stdio { + command, + args: Vec::new(), + env, + env_vars, + cwd: None, + } +} + +fn insert_mcp_server( + config: &mut Config, + server_name: &str, + transport: McpServerTransportConfig, + options: TestMcpServerOptions, +) { + let mut servers = config.mcp_servers.get().clone(); + servers.insert( + server_name.to_string(), + McpServerConfig { + transport, + enabled: true, + required: false, + supports_parallel_tool_calls: options.supports_parallel_tool_calls, + disabled_reason: None, + startup_timeout_sec: Some(Duration::from_secs(10)), + tool_timeout_sec: options.tool_timeout_sec, + enabled_tools: None, + disabled_tools: None, + scopes: None, + oauth_resource: None, + tools: HashMap::new(), + }, + ); + if let Err(err) = config.mcp_servers.set(servers) { + panic!("test mcp servers should accept any configuration: {err}"); + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] #[serial(mcp_test_value)] async fn stdio_server_round_trip() -> anyhow::Result<()> { @@ -121,37 +201,19 @@ async fn stdio_server_round_trip() -> anyhow::Result<()> { let fixture = test_codex() .with_config(move |config| { - let mut servers = config.mcp_servers.get().clone(); - servers.insert( - server_name.to_string(), - McpServerConfig { - transport: McpServerTransportConfig::Stdio { - command: rmcp_test_server_bin, - args: Vec::new(), - env: Some(HashMap::from([( - "MCP_TEST_VALUE".to_string(), - expected_env_value.to_string(), - )])), - env_vars: Vec::new(), - cwd: None, - }, - enabled: true, - required: false, - supports_parallel_tool_calls: false, - disabled_reason: None, - startup_timeout_sec: Some(Duration::from_secs(10)), - tool_timeout_sec: None, - enabled_tools: None, - disabled_tools: None, - scopes: None, - oauth_resource: None, - tools: HashMap::new(), - }, + insert_mcp_server( + config, + server_name, + stdio_transport( + rmcp_test_server_bin, + Some(HashMap::from([( + "MCP_TEST_VALUE".to_string(), + expected_env_value.to_string(), + )])), + Vec::new(), + ), + TestMcpServerOptions::default(), ); - config - .mcp_servers - .set(servers) - .expect("test mcp servers should accept any configuration"); }) .build(&server) .await?; @@ -282,34 +344,12 @@ async fn stdio_mcp_tool_call_includes_sandbox_state_meta() -> anyhow::Result<()> let rmcp_test_server_bin = stdio_server_bin()?; let fixture = test_codex() .with_config(move |config| { - let mut servers = config.mcp_servers.get().clone(); - servers.insert( - server_name.to_string(), - McpServerConfig { - transport: McpServerTransportConfig::Stdio { - command: rmcp_test_server_bin, - args: Vec::new(), - env: None, - env_vars: Vec::new(), - cwd: None, - }, - enabled: true, - required: false, - supports_parallel_tool_calls: false, - disabled_reason: None, - startup_timeout_sec: Some(Duration::from_secs(10)), - tool_timeout_sec: None, - enabled_tools: None, - disabled_tools: None, - scopes: None, - oauth_resource: None, - tools: HashMap::new(), - }, + insert_mcp_server( + config, + server_name, + stdio_transport(rmcp_test_server_bin, /*env*/ None, Vec::new()), + TestMcpServerOptions::default(), ); - config - .mcp_servers - .set(servers) - .expect("test mcp servers should accept any configuration"); }) .build(&server) .await?; @@ -417,34 +457,15 @@ async fn stdio_mcp_parallel_tool_calls_default_false_runs_serially() -> anyhow:: let fixture = test_codex() .with_config(move |config| { - let mut servers = config.mcp_servers.get().clone(); - servers.insert( - server_name.to_string(), - McpServerConfig { - transport: McpServerTransportConfig::Stdio { - command: rmcp_test_server_bin, - args: Vec::new(), - env: None, - env_vars: Vec::new(), - cwd: None, - }, - enabled: true, - required: false, - supports_parallel_tool_calls: false, - disabled_reason: None, - startup_timeout_sec: Some(Duration::from_secs(10)), + insert_mcp_server( + config, + server_name, + stdio_transport(rmcp_test_server_bin, /*env*/ None, Vec::new()), + TestMcpServerOptions { tool_timeout_sec: Some(Duration::from_secs(2)), - enabled_tools: None, - disabled_tools: None, - scopes: None, - oauth_resource: None, - tools: HashMap::new(), + ..Default::default() }, ); - config - .mcp_servers - .set(servers) - .expect("test mcp servers should accept any configuration"); }) .build(&server) .await?; @@ -567,34 +588,15 @@ async fn stdio_mcp_parallel_tool_calls_opt_in_runs_concurrently() -> anyhow::Res let fixture = test_codex() .with_config(move |config| { - let mut servers = config.mcp_servers.get().clone(); - servers.insert( - server_name.to_string(), - McpServerConfig { - transport: McpServerTransportConfig::Stdio { - command: rmcp_test_server_bin, - args: Vec::new(), - env: None, - env_vars: Vec::new(), - cwd: None, - }, - enabled: true, - required: false, + insert_mcp_server( + config, + server_name, + stdio_transport(rmcp_test_server_bin, /*env*/ None, Vec::new()), + TestMcpServerOptions { supports_parallel_tool_calls: true, - disabled_reason: None, - startup_timeout_sec: Some(Duration::from_secs(10)), tool_timeout_sec: Some(Duration::from_secs(2)), - enabled_tools: None, - disabled_tools: None, - scopes: None, - oauth_resource: None, - tools: HashMap::new(), }, ); - config - .mcp_servers - .set(servers) - .expect("test mcp servers should accept any configuration"); }) .build(&server) .await?; @@ -676,66 +678,25 @@ async fn stdio_image_responses_round_trip() -> anyhow::Result<()> { let fixture = test_codex() .with_config(move |config| { - let mut servers = config.mcp_servers.get().clone(); - servers.insert( - server_name.to_string(), - McpServerConfig { - transport: McpServerTransportConfig::Stdio { - command: rmcp_test_server_bin, - args: Vec::new(), - env: Some(HashMap::from([( - "MCP_TEST_IMAGE_DATA_URL".to_string(), - OPENAI_PNG.to_string(), - )])), - env_vars: Vec::new(), - cwd: None, - }, - enabled: true, - required: false, - supports_parallel_tool_calls: false, - disabled_reason: None, - startup_timeout_sec: Some(Duration::from_secs(10)), - tool_timeout_sec: None, - enabled_tools: None, - disabled_tools: None, - scopes: None, - oauth_resource: None, - tools: HashMap::new(), - }, + insert_mcp_server( + config, + server_name, + stdio_transport( + rmcp_test_server_bin, + Some(HashMap::from([( + "MCP_TEST_IMAGE_DATA_URL".to_string(), + OPENAI_PNG.to_string(), + )])), + Vec::new(), + ), + TestMcpServerOptions::default(), ); - config - .mcp_servers - .set(servers) - .expect("test mcp servers should accept any configuration"); }) .build(&server) .await?; let session_model = fixture.session_configured.model.clone(); - let tools_ready_deadline = Instant::now() + Duration::from_secs(30); - loop { - fixture.codex.submit(Op::ListMcpTools).await?; - let list_event = wait_for_event_with_timeout( - &fixture.codex, - |ev| matches!(ev, EventMsg::McpListToolsResponse(_)), - Duration::from_secs(10), - ) - .await; - let EventMsg::McpListToolsResponse(tool_list) = list_event else { - unreachable!("event guard guarantees McpListToolsResponse"); - }; - if tool_list.tools.contains_key(&tool_name) { - break; - } - - let available_tools: Vec<&str> = tool_list.tools.keys().map(String::as_str).collect(); - if Instant::now() >= tools_ready_deadline { - panic!( - "timed out waiting for MCP tool {tool_name} to become available; discovered tools: {available_tools:?}" - ); - } - sleep(Duration::from_millis(200)).await; - } + wait_for_mcp_tool(&fixture, &tool_name).await?; fixture .codex @@ -830,6 +791,189 @@ async fn stdio_image_responses_round_trip() -> anyhow::Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[serial(mcp_test_value)] +async fn stdio_image_responses_preserve_original_detail_metadata() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + + let call_id = "img-original-detail-1"; + let server_name = "rmcp"; + let tool_name = format!("mcp__{server_name}__image_scenario"); + let namespace = format!("mcp__{server_name}__"); + + mount_sse_once( + &server, + responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_function_call_with_namespace( + call_id, + &namespace, + "image_scenario", + r#"{"scenario":"image_only_original_detail"}"#, + ), + responses::ev_completed("resp-1"), + ]), + ) + .await; + let final_mock = mount_sse_once( + &server, + responses::sse(vec![ + responses::ev_assistant_message("msg-1", "rmcp original-detail image completed."), + responses::ev_completed("resp-2"), + ]), + ) + .await; + + let rmcp_test_server_bin = stdio_server_bin()?; + + let fixture = test_codex() + .with_model("gpt-5.3-codex") + .with_config(move |config| { + insert_mcp_server( + config, + server_name, + stdio_transport(rmcp_test_server_bin, /*env*/ None, Vec::new()), + TestMcpServerOptions::default(), + ); + }) + .build(&server) + .await?; + let session_model = fixture.session_configured.model.clone(); + + wait_for_mcp_tool(&fixture, &tool_name).await?; + + fixture + .codex + .submit(Op::UserTurn { + items: vec![UserInput::Text { + text: "call the rmcp image_scenario tool".into(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + cwd: fixture.cwd.path().to_path_buf(), + approval_policy: AskForApproval::Never, + approvals_reviewer: None, + sandbox_policy: SandboxPolicy::new_read_only_policy(), + model: session_model, + effort: None, + summary: None, + service_tier: None, + collaboration_mode: None, + personality: None, + }) + .await?; + + wait_for_event(&fixture.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + + let output_item = final_mock.single_request().function_call_output(call_id); + let output = output_item["output"] + .as_array() + .expect("image MCP output should be content items"); + assert_eq!(output.len(), 2); + assert_wall_time_header( + output[0]["text"] + .as_str() + .expect("first MCP image output item should be wall-time text"), + ); + assert_eq!( + output[1], + json!({ + "type": "input_image", + "image_url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==", + "detail": "original", + }) + ); + + server.verify().await; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[serial(mcp_test_value)] +async fn js_repl_emit_image_preserves_original_detail_for_mcp_images() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let call_id = "js-repl-rmcp-image"; + let rmcp_test_server_bin = stdio_server_bin()?; + + let fixture = test_codex() + .with_model("gpt-5.3-codex") + .with_config(move |config| { + config + .features + .enable(Feature::JsRepl) + .expect("test config should allow feature update"); + insert_mcp_server( + config, + "rmcp", + stdio_transport(rmcp_test_server_bin, /*env*/ None, Vec::new()), + TestMcpServerOptions::default(), + ); + }) + .build(&server) + .await?; + + wait_for_mcp_tool(&fixture, "mcp__rmcp__image_scenario").await?; + + mount_sse_once( + &server, + responses::sse(vec![ + responses::ev_response_created("resp-1"), + ev_custom_tool_call( + call_id, + "js_repl", + r#" +const out = await codex.tool("mcp__rmcp__image_scenario", { + scenario: "image_only_original_detail", +}); +const imageItem = out.output.find((item) => item.type === "input_image"); +await codex.emitImage(imageItem); +"#, + ), + responses::ev_completed("resp-1"), + ]), + ) + .await; + let final_mock = mount_sse_once( + &server, + responses::sse(vec![ + responses::ev_assistant_message("msg-1", "done"), + responses::ev_completed("resp-2"), + ]), + ) + .await; + + fixture + .submit_turn("use js_repl to emit the rmcp image scenario output") + .await?; + + let output = final_mock.single_request().custom_tool_call_output(call_id); + let output_items = output["output"] + .as_array() + .expect("js_repl output should be content items"); + let image_item = output_items + .iter() + .find(|item| item.get("type").and_then(Value::as_str) == Some("input_image")) + .expect("js_repl should emit an input_image item"); + assert_eq!( + image_item.get("detail").and_then(Value::as_str), + Some("original") + ); + assert!( + image_item + .get("image_url") + .and_then(Value::as_str) + .is_some_and(|image_url| image_url.starts_with("data:image/png;base64,")), + "js_repl should emit a png data URL" + ); + + server.verify().await; + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] #[serial(mcp_test_value)] async fn stdio_image_responses_are_sanitized_for_text_only_model() -> anyhow::Result<()> { @@ -909,37 +1053,19 @@ async fn stdio_image_responses_are_sanitized_for_text_only_model() -> anyhow::Re let fixture = test_codex() .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) .with_config(move |config| { - let mut servers = config.mcp_servers.get().clone(); - servers.insert( - server_name.to_string(), - McpServerConfig { - transport: McpServerTransportConfig::Stdio { - command: rmcp_test_server_bin, - args: Vec::new(), - env: Some(HashMap::from([( - "MCP_TEST_IMAGE_DATA_URL".to_string(), - OPENAI_PNG.to_string(), - )])), - env_vars: Vec::new(), - cwd: None, - }, - enabled: true, - required: false, - supports_parallel_tool_calls: false, - disabled_reason: None, - startup_timeout_sec: Some(Duration::from_secs(10)), - tool_timeout_sec: None, - enabled_tools: None, - disabled_tools: None, - scopes: None, - oauth_resource: None, - tools: HashMap::new(), - }, + insert_mcp_server( + config, + server_name, + stdio_transport( + rmcp_test_server_bin, + Some(HashMap::from([( + "MCP_TEST_IMAGE_DATA_URL".to_string(), + OPENAI_PNG.to_string(), + )])), + Vec::new(), + ), + TestMcpServerOptions::default(), ); - config - .mcp_servers - .set(servers) - .expect("test mcp servers should accept any configuration"); }) .build(&server) .await?; @@ -1041,34 +1167,16 @@ async fn stdio_server_propagates_whitelisted_env_vars() -> anyhow::Result<()> { let fixture = test_codex() .with_config(move |config| { - let mut servers = config.mcp_servers.get().clone(); - servers.insert( - server_name.to_string(), - McpServerConfig { - transport: McpServerTransportConfig::Stdio { - command: rmcp_test_server_bin, - args: Vec::new(), - env: None, - env_vars: vec!["MCP_TEST_VALUE".to_string()], - cwd: None, - }, - enabled: true, - required: false, - supports_parallel_tool_calls: false, - disabled_reason: None, - startup_timeout_sec: Some(Duration::from_secs(10)), - tool_timeout_sec: None, - enabled_tools: None, - disabled_tools: None, - scopes: None, - oauth_resource: None, - tools: HashMap::new(), - }, + insert_mcp_server( + config, + server_name, + stdio_transport( + rmcp_test_server_bin, + /*env*/ None, + vec!["MCP_TEST_VALUE".to_string()], + ), + TestMcpServerOptions::default(), ); - config - .mcp_servers - .set(servers) - .expect("test mcp servers should accept any configuration"); }) .build(&server) .await?; @@ -1211,33 +1319,17 @@ async fn streamable_http_tool_call_round_trip() -> anyhow::Result<()> { let fixture = test_codex() .with_config(move |config| { - let mut servers = config.mcp_servers.get().clone(); - servers.insert( - server_name.to_string(), - McpServerConfig { - transport: McpServerTransportConfig::StreamableHttp { - url: server_url, - bearer_token_env_var: None, - http_headers: None, - env_http_headers: None, - }, - enabled: true, - required: false, - supports_parallel_tool_calls: false, - disabled_reason: None, - startup_timeout_sec: Some(Duration::from_secs(10)), - tool_timeout_sec: None, - enabled_tools: None, - disabled_tools: None, - scopes: None, - oauth_resource: None, - tools: HashMap::new(), + insert_mcp_server( + config, + server_name, + McpServerTransportConfig::StreamableHttp { + url: server_url, + bearer_token_env_var: None, + http_headers: None, + env_http_headers: None, }, + TestMcpServerOptions::default(), ); - config - .mcp_servers - .set(servers) - .expect("test mcp servers should accept any configuration"); }) .build(&server) .await?; @@ -1441,62 +1533,23 @@ async fn streamable_http_with_oauth_round_trip_impl() -> anyhow::Result<()> { // runs the full core suite in one process. config.mcp_oauth_credentials_store_mode = serde_json::from_value(json!("file")) .expect("`file` should deserialize as OAuthCredentialsStoreMode"); - let mut servers = config.mcp_servers.get().clone(); - servers.insert( - server_name.to_string(), - McpServerConfig { - transport: McpServerTransportConfig::StreamableHttp { - url: server_url, - bearer_token_env_var: None, - http_headers: None, - env_http_headers: None, - }, - enabled: true, - required: false, - supports_parallel_tool_calls: false, - disabled_reason: None, - startup_timeout_sec: Some(Duration::from_secs(10)), - tool_timeout_sec: None, - enabled_tools: None, - disabled_tools: None, - scopes: None, - oauth_resource: None, - tools: HashMap::new(), + insert_mcp_server( + config, + server_name, + McpServerTransportConfig::StreamableHttp { + url: server_url, + bearer_token_env_var: None, + http_headers: None, + env_http_headers: None, }, + TestMcpServerOptions::default(), ); - config - .mcp_servers - .set(servers) - .expect("test mcp servers should accept any configuration"); }) .build(&server) .await?; let session_model = fixture.session_configured.model.clone(); - let tools_ready_deadline = Instant::now() + Duration::from_secs(30); - loop { - fixture.codex.submit(Op::ListMcpTools).await?; - let list_event = wait_for_event_with_timeout( - &fixture.codex, - |ev| matches!(ev, EventMsg::McpListToolsResponse(_)), - Duration::from_secs(10), - ) - .await; - let EventMsg::McpListToolsResponse(tool_list) = list_event else { - unreachable!("event guard guarantees McpListToolsResponse"); - }; - if tool_list.tools.contains_key(&tool_name) { - break; - } - - let available_tools: Vec<&str> = tool_list.tools.keys().map(String::as_str).collect(); - if Instant::now() >= tools_ready_deadline { - panic!( - "timed out waiting for MCP tool {tool_name} to become available; discovered tools: {available_tools:?}" - ); - } - sleep(Duration::from_millis(200)).await; - } + wait_for_mcp_tool(&fixture, &tool_name).await?; fixture .codex diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs index 0f18b68393..2200456527 100644 --- a/codex-rs/protocol/src/models.rs +++ b/codex-rs/protocol/src/models.rs @@ -1362,6 +1362,8 @@ impl CallToolResult { fn convert_mcp_content_to_items( contents: &[serde_json::Value], ) -> Option> { + const CODEX_IMAGE_DETAIL_META_KEY: &str = "codex/imageDetail"; + #[derive(serde::Deserialize)] #[serde(tag = "type")] enum McpContent { @@ -1372,6 +1374,8 @@ fn convert_mcp_content_to_items( data: String, #[serde(rename = "mimeType", alias = "mime_type")] mime_type: Option, + #[serde(rename = "_meta", default)] + meta: Option, }, #[serde(other)] Unknown, @@ -1383,7 +1387,11 @@ fn convert_mcp_content_to_items( for content in contents { let item = match serde_json::from_value::(content.clone()) { Ok(McpContent::Text { text }) => FunctionCallOutputContentItem::InputText { text }, - Ok(McpContent::Image { data, mime_type }) => { + Ok(McpContent::Image { + data, + mime_type, + meta, + }) => { saw_image = true; let image_url = if data.starts_with("data:") { data @@ -1393,7 +1401,15 @@ fn convert_mcp_content_to_items( }; FunctionCallOutputContentItem::InputImage { image_url, - detail: None, + detail: meta + .as_ref() + .and_then(serde_json::Value::as_object) + .and_then(|meta| meta.get(CODEX_IMAGE_DETAIL_META_KEY)) + .and_then(serde_json::Value::as_str) + .and_then(|detail| match detail { + "original" => Some(ImageDetail::Original), + _ => None, + }), } } Ok(McpContent::Unknown) | Err(_) => FunctionCallOutputContentItem::InputText { @@ -2264,6 +2280,70 @@ mod tests { Ok(()) } + #[test] + fn preserves_original_detail_metadata_on_mcp_images() -> Result<()> { + let call_tool_result = CallToolResult { + content: vec![serde_json::json!({ + "type": "image", + "data": "BASE64", + "mimeType": "image/png", + "_meta": { + "codex/imageDetail": "original", + }, + })], + structured_content: None, + is_error: Some(false), + meta: None, + }; + + let payload = call_tool_result.into_function_call_output_payload(); + let Some(items) = payload.content_items() else { + panic!("expected content items"); + }; + let items = items.to_vec(); + assert_eq!( + items, + vec![FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,BASE64".into(), + detail: Some(ImageDetail::Original), + }] + ); + + Ok(()) + } + + #[test] + fn ignores_unknown_mcp_image_detail_metadata() -> Result<()> { + let call_tool_result = CallToolResult { + content: vec![serde_json::json!({ + "type": "image", + "data": "BASE64", + "mimeType": "image/png", + "_meta": { + "codex/imageDetail": "high", + }, + })], + structured_content: None, + is_error: Some(false), + meta: None, + }; + + let payload = call_tool_result.into_function_call_output_payload(); + let Some(items) = payload.content_items() else { + panic!("expected content items"); + }; + let items = items.to_vec(); + assert_eq!( + items, + vec![FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,BASE64".into(), + detail: None, + }] + ); + + Ok(()) + } + #[test] fn deserializes_array_payload_into_items() -> Result<()> { let json = r#"[ diff --git a/codex-rs/rmcp-client/src/bin/test_stdio_server.rs b/codex-rs/rmcp-client/src/bin/test_stdio_server.rs index 326fd94279..82bc387d99 100644 --- a/codex-rs/rmcp-client/src/bin/test_stdio_server.rs +++ b/codex-rs/rmcp-client/src/bin/test_stdio_server.rs @@ -208,6 +208,7 @@ impl TestToolServer { /// - `codex mcp add mcpimg -- /abs/path/to/test_stdio_server` /// - Then in Codex TUI, ask it to call: /// - `mcpimg.image_scenario({"scenario":"image_only"})` + /// - `mcpimg.image_scenario({"scenario":"image_only_original_detail"})` /// - `mcpimg.image_scenario({"scenario":"text_then_image","caption":"Here is the image:"})` /// - `mcpimg.image_scenario({"scenario":"invalid_base64_then_image"})` /// - `mcpimg.image_scenario({"scenario":"invalid_image_bytes_then_image"})` @@ -224,6 +225,7 @@ impl TestToolServer { "type": "string", "enum": [ "image_only", + "image_only_original_detail", "text_then_image", "invalid_base64_then_image", "invalid_image_bytes_then_image", @@ -339,6 +341,7 @@ fn sync_barrier_map() -> &'static tokio::sync::Mutex { content.push(rmcp::model::Content::image(valid_data_b64, mime_type)); } + ImageScenario::ImageOnlyOriginalDetail => { + let mut meta = rmcp::model::Meta::new(); + meta.insert( + "codex/imageDetail".to_string(), + serde_json::json!("original"), + ); + content.push(rmcp::model::Annotated::new( + rmcp::model::RawContent::Image(rmcp::model::RawImageContent { + data: valid_data_b64, + mime_type, + meta: Some(meta), + }), + None, + )); + } ImageScenario::TextThenImage => { content.push(rmcp::model::Content::text(caption)); content.push(rmcp::model::Content::image(valid_data_b64, mime_type)); diff --git a/codex-rs/tools/src/image_detail.rs b/codex-rs/tools/src/image_detail.rs index 94639e61f9..56987e4831 100644 --- a/codex-rs/tools/src/image_detail.rs +++ b/codex-rs/tools/src/image_detail.rs @@ -1,3 +1,4 @@ +use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::ImageDetail; use codex_protocol::openai_models::ModelInfo; @@ -17,6 +18,23 @@ pub fn normalize_output_image_detail( } } +pub fn sanitize_original_image_detail( + can_request_original_image_detail: bool, + items: &mut [FunctionCallOutputContentItem], +) { + if can_request_original_image_detail { + return; + } + + for item in items { + if let FunctionCallOutputContentItem::InputImage { detail, .. } = item + && matches!(detail, Some(ImageDetail::Original)) + { + *detail = None; + } + } +} + #[cfg(test)] #[path = "image_detail_tests.rs"] mod tests; diff --git a/codex-rs/tools/src/image_detail_tests.rs b/codex-rs/tools/src/image_detail_tests.rs index 35d666149f..c1a0f8ca1a 100644 --- a/codex-rs/tools/src/image_detail_tests.rs +++ b/codex-rs/tools/src/image_detail_tests.rs @@ -1,4 +1,5 @@ use super::*; +use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::ImageDetail; use codex_protocol::openai_models::ModelInfo; use pretty_assertions::assert_eq; @@ -73,3 +74,39 @@ fn unsupported_non_original_detail_is_dropped() { None ); } + +#[test] +fn sanitize_original_drops_original_without_support() { + let mut items = vec![ + FunctionCallOutputContentItem::InputText { + text: "header".to_string(), + }, + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,AAA".to_string(), + detail: Some(ImageDetail::Original), + }, + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,BBB".to_string(), + detail: Some(ImageDetail::Low), + }, + ]; + + sanitize_original_image_detail(/*can_request_original_image_detail*/ false, &mut items); + + assert_eq!( + items, + vec![ + FunctionCallOutputContentItem::InputText { + text: "header".to_string(), + }, + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,AAA".to_string(), + detail: None, + }, + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,BBB".to_string(), + detail: Some(ImageDetail::Low), + }, + ] + ); +} diff --git a/codex-rs/tools/src/lib.rs b/codex-rs/tools/src/lib.rs index 4a3ccc01ae..b7c6a57d2b 100644 --- a/codex-rs/tools/src/lib.rs +++ b/codex-rs/tools/src/lib.rs @@ -53,6 +53,7 @@ pub use codex_protocol::ToolName; pub use dynamic_tool::parse_dynamic_tool; pub use image_detail::can_request_original_image_detail; pub use image_detail::normalize_output_image_detail; +pub use image_detail::sanitize_original_image_detail; pub use js_repl_tool::create_js_repl_reset_tool; pub use js_repl_tool::create_js_repl_tool; pub use json_schema::AdditionalProperties; diff --git a/docs/js_repl.md b/docs/js_repl.md index d5edc71b46..aee635b77a 100644 --- a/docs/js_repl.md +++ b/docs/js_repl.md @@ -86,6 +86,7 @@ imported local file. They are not resolved relative to the imported file's locat - `codex.emitImage(...)` accepts a data URL, a single `input_image` item, an object like `{ bytes, mimeType }`, or a raw tool response object that contains exactly one image and no text. Call it multiple times if you want to emit multiple images. - `codex.emitImage(...)` rejects mixed text-and-image content. - Request full-resolution image processing with `detail: "original"` only when the `view_image` tool schema includes a `detail` argument. The same availability applies to `codex.emitImage(...)`: if `view_image.detail` is present, you may also pass `detail: "original"` there. Use this when high-fidelity image perception or precise localization is needed, especially for CUA agents. +- Raw MCP image blocks can request the same behavior by returning `_meta: { "codex/imageDetail": "original" }` on the image content item. - Example of sharing an in-memory Playwright screenshot: `await codex.emitImage({ bytes: await page.screenshot({ type: "jpeg", quality: 85 }), mimeType: "image/jpeg", detail: "original" })`. - Example of sharing a local image tool result: `await codex.emitImage(codex.tool("view_image", { path: "/absolute/path", detail: "original" }))`. - When encoding an image to send with `codex.emitImage(...)` or `view_image`, prefer JPEG at about 85 quality when lossy compression is acceptable; use PNG when transparency or lossless detail matters. Smaller uploads are faster and less likely to hit size limits. From 9d1bf002c6f3d25b355a0104994bc4b410011a14 Mon Sep 17 00:00:00 2001 From: efrazer-oai Date: Wed, 15 Apr 2026 14:44:01 -0700 Subject: [PATCH 44/84] Significantly improve standalone installer (#17022) ## Summary This PR significantly improves the standalone installer experience. The main changes are: 1. We now install the codex binary and other dependencies in a subdirectory under CODEX_HOME. (`CODEX_HOME/packages/standalone/releases/...`) 2. We replace the `codex.js` launcher that npm/bun rely on with logic in the Rust binary that automatically resolves its dependencies (like ripgrep) ## Motivation A few design constraints pushed this work. 1. Currently, the entrypoint to codex is through `codex.js`, which forces a node dependency to kick off our rust app. We want to move away from this so that the entrypoint to codex does not rely on node or external package managers. 2. Right now, the native script adds codex and its dependencies directly to user PATH. Given that codex is likely to add more binary dependencies than ripgrep, we want a solution which does not add arbitrary binaries to user PATH -- the only one we want to add is the `codex` command itself. 3. We want upgrades to be atomic. We do not want scenarios where interrupting an upgrade command can move codex into undefined state (for example, having a new codex binary but an old ripgrep binary). This was ~possible with the old script. 4. Currently, the Rust binary uses heuristics to determine which installer created it. These heuristics are flaky and are tied to the `codex.js` launcher. We need a more stable/deterministic way to determine how the binary was installed for standalone. 5. We do not want conflicting codex installations on PATH. For example, the user installing via npm, then installing via brew, then installing via standalone would make it unclear which version of codex is being launched and make it tough for us to determine the right upgrade command. ## Design ### Standalone package layout Standalone installs now live under `CODEX_HOME/packages/standalone`: ```text $CODEX_HOME/ packages/ standalone/ current -> releases/0.111.0-x86_64-unknown-linux-musl releases/ 0.111.0-x86_64-unknown-linux-musl/ codex codex-resources/ rg ``` where `standalone/current` is a symlink to a release directory. On Windows, the release directory has the same shape, with `.exe` names and Windows helpers in `codex-resources`: ```text %CODEX_HOME%\ packages\ standalone\ current -> releases\0.111.0-x86_64-pc-windows-msvc releases\ 0.111.0-x86_64-pc-windows-msvc\ codex.exe codex-resources\ rg.exe codex-command-runner.exe codex-windows-sandbox-setup.exe ``` This gives us: - atomic upgrades because we can fully stage a release before switching `standalone/current` - a stable way for the binary to recognize a standalone install from its canonical `current_exe()` path under CODEX_HOME - a clean place for binary dependencies like `rg`, Windows sandbox helpers, and, in the future, our custom `zsh` etc ### Command location On Unix, we add a symlink at `~/.local/bin/codex` which points directly to the `$CODEX_HOME/packages/standalone/current/codex` binary. This becomes the main entrypoint for the CLI. On Windows, we store the link at `%LOCALAPPDATA%\Programs\OpenAI\Codex\bin`. ### PATH persistence This is a tricky part of the PR, as there's no ~super reliable way to ensure that we end up on PATH without significant tradeoffs. Most Unix variants will have `~/.local/bin` on PATH already, which means we *should* be fine simply registering the command there in most cases. However, there are cases where this is not the case. In these cases, we directly edit the profile depending on the shell we're in. - macOS zsh: `~/.zprofile` - macOS bash: `~/.bash_profile` - Linux zsh: `~/.zshrc` - Linux bash: `~/.bashrc` - fallback: `~/.profile` On Windows, we update the User `Path` environment variable directly and we don't need to worry about shell profiles. ### Standalone runtime detection This PR adds a new shared crate, `codex-install-context`, which computes install ownership once per process and caches it in a `OnceLock`. That context includes: - install manager (`Standalone`, `Npm`, `Bun`, `Brew`, `Other`) - the managed standalone release directory, when applicable - the managed standalone `codex-resources` directory, when present - the resolved `rg_command` The standalone path is detected by canonicalizing `current_exe()`, canonicalizing CODEX_HOME via `find_codex_home()`, and checking whether the binary is running from under `$CODEX_HOME/packages/standalone/releases`. We intentionally do not use a release metadata file. The binary path is the source of truth. ### Dependency resolution For standalone installs, `grep_files` now resolves bundled `rg` from `codex-resources` next to the Codex binary. For npm/bun/brew/other installs, `grep_files` falls back to resolving `rg` from PATH. For Windows standalone installs, Windows sandbox helpers are still found as direct siblings when present. If they are not direct siblings, the lookup also checks the sibling `codex-resources` directory. ### TUI update path The TUI now has `UpdateAction::StandaloneUnix` and `UpdateAction::StandaloneWindows`, which rerun the standalone install commands. Unix update command: ```sh sh -c "curl -fsSL https://chatgpt.com/codex/install.sh | sh" ``` Windows update command: ```powershell powershell -c "irm https://chatgpt.com/codex/install.ps1|iex" ``` The Windows updater runs PowerShell directly. We do this because `cmd /C` would parse the `|iex` as a cmd pipeline instead of passing it to PowerShell. ## Additional installer behavior - standalone installs now warn about conflicting npm/bun/brew-managed `codex` installs and offer to uninstall them - same-version reruns do not redownload the release if it is already staged locally ## Testing Installer smoke tests run: - macOS: fresh install into isolated `HOME` and `CODEX_HOME` with `scripts/install/install.sh --release latest` - macOS: reran the installer against the same isolated install to verify the same-version/update path and PATH block idempotence - macOS: verified the installed `codex --version` and bundled `codex-resources/rg --version` - Windows: parsed `scripts/install/install.ps1` with PowerShell via `[scriptblock]::Create(...)` - Windows: verified the standalone update action builds a direct PowerShell command and does not route the `irm ...|iex` command through `cmd /C` --------- Co-authored-by: Codex --- codex-rs/Cargo.lock | 10 + codex-rs/Cargo.toml | 2 + codex-rs/README.md | 2 +- codex-rs/cli/src/main.rs | 17 +- codex-rs/install-context/BUILD.bazel | 6 + codex-rs/install-context/Cargo.toml | 19 + codex-rs/install-context/src/lib.rs | 258 +++++++ codex-rs/tui/Cargo.toml | 1 + codex-rs/tui/src/update_action.rs | 135 ++-- .../src/helper_materialization.rs | 78 +- .../src/setup_orchestrator.rs | 10 + scripts/install/install.ps1 | 668 ++++++++++++++++-- scripts/install/install.sh | 660 +++++++++++++++-- 13 files changed, 1656 insertions(+), 210 deletions(-) create mode 100644 codex-rs/install-context/BUILD.bazel create mode 100644 codex-rs/install-context/Cargo.toml create mode 100644 codex-rs/install-context/src/lib.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 68e0ec092f..2d39c42df4 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2255,6 +2255,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "codex-install-context" +version = "0.0.0" +dependencies = [ + "codex-utils-home-dir", + "pretty_assertions", + "tempfile", +] + [[package]] name = "codex-instructions" version = "0.0.0" @@ -2896,6 +2905,7 @@ dependencies = [ "codex-feedback", "codex-file-search", "codex-git-utils", + "codex-install-context", "codex-login", "codex-mcp", "codex-model-provider-info", diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 4bb8837211..47c0d348bf 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -13,6 +13,7 @@ members = [ "arg0", "feedback", "features", + "install-context", "codex-backend-openapi-models", "code-mode", "cloud-requirements", @@ -134,6 +135,7 @@ codex-execpolicy = { path = "execpolicy" } codex-experimental-api-macros = { path = "codex-experimental-api-macros" } codex-features = { path = "features" } codex-feedback = { path = "feedback" } +codex-install-context = { path = "install-context" } codex-file-search = { path = "file-search" } codex-git-utils = { path = "git-utils" } codex-hooks = { path = "hooks" } diff --git a/codex-rs/README.md b/codex-rs/README.md index 6307668f39..2ad7158f98 100644 --- a/codex-rs/README.md +++ b/codex-rs/README.md @@ -1,6 +1,6 @@ # Codex CLI (Rust Implementation) -We provide Codex CLI as a standalone, native executable to ensure a zero-dependency install. +We provide Codex CLI as a standalone executable to ensure a zero-dependency install. ## Installing Codex diff --git a/codex-rs/cli/src/main.rs b/codex-rs/cli/src/main.rs index 4e962cf091..6accb2cb0c 100644 --- a/codex-rs/cli/src/main.rs +++ b/codex-rs/cli/src/main.rs @@ -516,10 +516,19 @@ fn run_update_action(action: UpdateAction) -> anyhow::Result<()> { let status = { #[cfg(windows)] { - // On Windows, run via cmd.exe so .CMD/.BAT are correctly resolved (PATHEXT semantics). - std::process::Command::new("cmd") - .args(["/C", &cmd_str]) - .status()? + if action == UpdateAction::StandaloneWindows { + let (cmd, args) = action.command_args(); + // Run the standalone PowerShell installer with PowerShell + // itself. Routing this through `cmd.exe /C` would parse + // PowerShell metacharacters like `|` before PowerShell sees + // the installer command. + std::process::Command::new(cmd).args(args).status()? + } else { + // On Windows, run via cmd.exe so .CMD/.BAT are correctly resolved (PATHEXT semantics). + std::process::Command::new("cmd") + .args(["/C", &cmd_str]) + .status()? + } } #[cfg(not(windows))] { diff --git a/codex-rs/install-context/BUILD.bazel b/codex-rs/install-context/BUILD.bazel new file mode 100644 index 0000000000..68254d10cc --- /dev/null +++ b/codex-rs/install-context/BUILD.bazel @@ -0,0 +1,6 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "install-context", + crate_name = "codex_install_context", +) diff --git a/codex-rs/install-context/Cargo.toml b/codex-rs/install-context/Cargo.toml new file mode 100644 index 0000000000..ce4eeefe77 --- /dev/null +++ b/codex-rs/install-context/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "codex-install-context" +version.workspace = true +edition.workspace = true +license.workspace = true + +[lib] +name = "codex_install_context" +path = "src/lib.rs" + +[lints] +workspace = true + +[dependencies] +codex-utils-home-dir = { workspace = true } + +[dev-dependencies] +pretty_assertions = { workspace = true } +tempfile = { workspace = true } diff --git a/codex-rs/install-context/src/lib.rs b/codex-rs/install-context/src/lib.rs new file mode 100644 index 0000000000..980fc3f546 --- /dev/null +++ b/codex-rs/install-context/src/lib.rs @@ -0,0 +1,258 @@ +use std::path::Path; +use std::path::PathBuf; +use std::sync::OnceLock; + +const RELEASES_DIRNAME: &str = "releases"; +const RESOURCES_DIRNAME: &str = "codex-resources"; +const STANDALONE_PACKAGES_DIRNAME: &str = "standalone"; +static INSTALL_CONTEXT: OnceLock = OnceLock::new(); + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StandalonePlatform { + Unix, + Windows, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum InstallContext { + Standalone { + /// The managed standalone release directory, for example + /// `~/.codex/packages/standalone/releases/0.111.0-x86_64-unknown-linux-musl`. + release_dir: PathBuf, + /// The bundled resource directory that sits next to the executable when + /// this install ships managed dependencies. + resources_dir: Option, + /// The platform of the standalone release, either `Unix` or `Windows`. + platform: StandalonePlatform, + }, + /// A Codex binary launched through the npm-managed `codex.js` shim. + Npm, + /// A Codex binary launched through the bun-managed `codex.js` shim. + Bun, + /// A Codex binary that appears to come from a Homebrew install prefix. + Brew, + /// Any other execution environment. + /// + /// This commonly covers `cargo run`, app-bundled Codex binaries, custom + /// internal launchers, and tests that execute Codex from an arbitrary path. + Other, +} + +impl InstallContext { + pub fn from_exe( + is_macos: bool, + current_exe: Option<&Path>, + managed_by_npm: bool, + managed_by_bun: bool, + ) -> Self { + let codex_home = codex_utils_home_dir::find_codex_home().ok(); + Self::from_exe_with_codex_home( + is_macos, + current_exe, + managed_by_npm, + managed_by_bun, + codex_home.as_deref(), + ) + } + + fn from_exe_with_codex_home( + is_macos: bool, + current_exe: Option<&Path>, + managed_by_npm: bool, + managed_by_bun: bool, + codex_home: Option<&Path>, + ) -> Self { + if managed_by_npm { + return Self::Npm; + } + + if managed_by_bun { + return Self::Bun; + } + + if let Some(exe_path) = current_exe + && let Some(standalone_context) = standalone_install_context(exe_path, codex_home) + { + return standalone_context; + } + + if is_macos + && let Some(exe_path) = current_exe + && (exe_path.starts_with("/opt/homebrew") || exe_path.starts_with("/usr/local")) + { + return Self::Brew; + } + + Self::Other + } + + pub fn current() -> &'static Self { + INSTALL_CONTEXT.get_or_init(|| { + let current_exe = std::env::current_exe().ok(); + let managed_by_npm = std::env::var_os("CODEX_MANAGED_BY_NPM").is_some(); + let managed_by_bun = std::env::var_os("CODEX_MANAGED_BY_BUN").is_some(); + Self::from_exe( + cfg!(target_os = "macos"), + current_exe.as_deref(), + managed_by_npm, + managed_by_bun, + ) + }) + } + + pub fn rg_command(&self) -> PathBuf { + match self { + Self::Standalone { + resources_dir: Some(resources_dir), + .. + } => { + let bundled_rg = resources_dir.join(default_rg_command()); + if bundled_rg.exists() { + bundled_rg + } else { + default_rg_command() + } + } + Self::Standalone { + resources_dir: None, + .. + } + | Self::Npm + | Self::Bun + | Self::Brew + | Self::Other => default_rg_command(), + } + } +} + +fn standalone_install_context( + exe_path: &Path, + codex_home: Option<&Path>, +) -> Option { + let canonical_exe = std::fs::canonicalize(exe_path).ok()?; + let canonical_codex_home = std::fs::canonicalize(codex_home?).ok()?; + let release_dir = canonical_exe.parent()?.to_path_buf(); + let releases_root = canonical_codex_home + .join("packages") + .join(STANDALONE_PACKAGES_DIRNAME) + .join(RELEASES_DIRNAME); + if !release_dir.starts_with(releases_root) { + return None; + } + + let resources_dir = release_dir.join(RESOURCES_DIRNAME); + Some(InstallContext::Standalone { + release_dir, + resources_dir: resources_dir.is_dir().then_some(resources_dir), + platform: standalone_platform(), + }) +} + +fn standalone_platform() -> StandalonePlatform { + if cfg!(windows) { + StandalonePlatform::Windows + } else { + StandalonePlatform::Unix + } +} + +fn default_rg_command() -> PathBuf { + if cfg!(windows) { + PathBuf::from("rg.exe") + } else { + PathBuf::from("rg") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use std::fs; + + #[test] + fn detects_standalone_install_from_release_layout() -> std::io::Result<()> { + let codex_home = tempfile::tempdir()?; + let release_dir = codex_home + .path() + .join("packages/standalone/releases/1.2.3-x86_64-unknown-linux-musl"); + let resources_dir = release_dir.join(RESOURCES_DIRNAME); + fs::create_dir_all(&resources_dir)?; + let exe_path = release_dir.join(if cfg!(windows) { "codex.exe" } else { "codex" }); + fs::write(&exe_path, "")?; + fs::write(resources_dir.join(default_rg_command()), "")?; + let canonical_release_dir = release_dir.canonicalize()?; + let canonical_resources_dir = resources_dir.canonicalize()?; + + let context = InstallContext::from_exe_with_codex_home( + /*is_macos*/ false, + /*current_exe*/ Some(&exe_path), + /*managed_by_npm*/ false, + /*managed_by_bun*/ false, + /*codex_home*/ Some(codex_home.path()), + ); + assert_eq!( + context, + InstallContext::Standalone { + release_dir: canonical_release_dir, + resources_dir: Some(canonical_resources_dir), + platform: standalone_platform(), + } + ); + Ok(()) + } + + #[test] + fn standalone_rg_falls_back_when_resources_are_missing() -> std::io::Result<()> { + let codex_home = tempfile::tempdir()?; + let release_dir = codex_home + .path() + .join("packages/standalone/releases/1.2.3-x86_64-unknown-linux-musl"); + fs::create_dir_all(&release_dir)?; + let exe_path = release_dir.join(if cfg!(windows) { "codex.exe" } else { "codex" }); + fs::write(&exe_path, "")?; + + let context = InstallContext::from_exe_with_codex_home( + /*is_macos*/ false, + /*current_exe*/ Some(&exe_path), + /*managed_by_npm*/ false, + /*managed_by_bun*/ false, + /*codex_home*/ Some(codex_home.path()), + ); + assert_eq!(context.rg_command(), default_rg_command()); + Ok(()) + } + + #[test] + fn npm_and_bun_take_precedence() { + let npm_context = InstallContext::from_exe_with_codex_home( + /*is_macos*/ false, + /*current_exe*/ Some(Path::new("/tmp/codex")), + /*managed_by_npm*/ true, + /*managed_by_bun*/ false, + /*codex_home*/ None, + ); + assert_eq!(npm_context, InstallContext::Npm); + + let bun_context = InstallContext::from_exe_with_codex_home( + /*is_macos*/ false, + /*current_exe*/ Some(Path::new("/tmp/codex")), + /*managed_by_npm*/ false, + /*managed_by_bun*/ true, + /*codex_home*/ None, + ); + assert_eq!(bun_context, InstallContext::Bun); + } + + #[test] + fn brew_is_detected_on_macos_prefixes() { + let context = InstallContext::from_exe_with_codex_home( + /*is_macos*/ true, + /*current_exe*/ Some(Path::new("/opt/homebrew/bin/codex")), + /*managed_by_npm*/ false, + /*managed_by_bun*/ false, + /*codex_home*/ None, + ); + assert_eq!(context, InstallContext::Brew); + } +} diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml index 0dec8ceed1..0f29fdeae6 100644 --- a/codex-rs/tui/Cargo.toml +++ b/codex-rs/tui/Cargo.toml @@ -29,6 +29,7 @@ codex-ansi-escape = { workspace = true } codex-app-server-client = { workspace = true } codex-app-server-protocol = { workspace = true } codex-arg0 = { workspace = true } +codex-install-context = { workspace = true } codex-chatgpt = { workspace = true } codex-cloud-requirements = { workspace = true } codex-config = { workspace = true } diff --git a/codex-rs/tui/src/update_action.rs b/codex-rs/tui/src/update_action.rs index baee1662fd..1ac2ff675f 100644 --- a/codex-rs/tui/src/update_action.rs +++ b/codex-rs/tui/src/update_action.rs @@ -1,3 +1,8 @@ +#[cfg(any(not(debug_assertions), test))] +use codex_install_context::InstallContext; +#[cfg(any(not(debug_assertions), test))] +use codex_install_context::StandalonePlatform; + /// Update action the CLI should perform after the TUI exits. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum UpdateAction { @@ -7,15 +12,41 @@ pub enum UpdateAction { BunGlobalLatest, /// Update via `brew upgrade codex`. BrewUpgrade, + /// Update via `curl -fsSL https://chatgpt.com/codex/install.sh | sh`. + StandaloneUnix, + /// Update via `irm https://chatgpt.com/codex/install.ps1|iex`. + StandaloneWindows, } impl UpdateAction { + #[cfg(any(not(debug_assertions), test))] + pub(crate) fn from_install_context(context: &InstallContext) -> Option { + match context { + InstallContext::Npm => Some(UpdateAction::NpmGlobalLatest), + InstallContext::Bun => Some(UpdateAction::BunGlobalLatest), + InstallContext::Brew => Some(UpdateAction::BrewUpgrade), + InstallContext::Standalone { platform, .. } => Some(match platform { + StandalonePlatform::Unix => UpdateAction::StandaloneUnix, + StandalonePlatform::Windows => UpdateAction::StandaloneWindows, + }), + InstallContext::Other => None, + } + } + /// Returns the list of command-line arguments for invoking the update. pub fn command_args(self) -> (&'static str, &'static [&'static str]) { match self { UpdateAction::NpmGlobalLatest => ("npm", &["install", "-g", "@openai/codex"]), UpdateAction::BunGlobalLatest => ("bun", &["install", "-g", "@openai/codex"]), UpdateAction::BrewUpgrade => ("brew", &["upgrade", "--cask", "codex"]), + UpdateAction::StandaloneUnix => ( + "sh", + &["-c", "curl -fsSL https://chatgpt.com/codex/install.sh | sh"], + ), + UpdateAction::StandaloneWindows => ( + "powershell", + &["-c", "irm https://chatgpt.com/codex/install.ps1|iex"], + ), } } @@ -29,88 +60,68 @@ impl UpdateAction { #[cfg(not(debug_assertions))] pub(crate) fn get_update_action() -> Option { - let exe = std::env::current_exe().unwrap_or_default(); - let managed_by_npm = std::env::var_os("CODEX_MANAGED_BY_NPM").is_some(); - let managed_by_bun = std::env::var_os("CODEX_MANAGED_BY_BUN").is_some(); - - detect_update_action( - cfg!(target_os = "macos"), - &exe, - managed_by_npm, - managed_by_bun, - ) -} - -#[cfg(any(not(debug_assertions), test))] -fn detect_update_action( - is_macos: bool, - current_exe: &std::path::Path, - managed_by_npm: bool, - managed_by_bun: bool, -) -> Option { - if managed_by_npm { - Some(UpdateAction::NpmGlobalLatest) - } else if managed_by_bun { - Some(UpdateAction::BunGlobalLatest) - } else if is_macos - && (current_exe.starts_with("/opt/homebrew") || current_exe.starts_with("/usr/local")) - { - Some(UpdateAction::BrewUpgrade) - } else { - None - } + UpdateAction::from_install_context(InstallContext::current()) } #[cfg(test)] mod tests { use super::*; + use pretty_assertions::assert_eq; + use std::path::PathBuf; #[test] - fn detects_update_action_without_env_mutation() { + fn maps_install_context_to_update_action() { + let native_release_dir = PathBuf::from("/tmp/native-release"); + assert_eq!( - detect_update_action( - /*is_macos*/ false, - std::path::Path::new("/any/path"), - /*managed_by_npm*/ false, - /*managed_by_bun*/ false - ), + UpdateAction::from_install_context(&InstallContext::Other), None ); assert_eq!( - detect_update_action( - /*is_macos*/ false, - std::path::Path::new("/any/path"), - /*managed_by_npm*/ true, - /*managed_by_bun*/ false - ), + UpdateAction::from_install_context(&InstallContext::Npm), Some(UpdateAction::NpmGlobalLatest) ); assert_eq!( - detect_update_action( - /*is_macos*/ false, - std::path::Path::new("/any/path"), - /*managed_by_npm*/ false, - /*managed_by_bun*/ true - ), + UpdateAction::from_install_context(&InstallContext::Bun), Some(UpdateAction::BunGlobalLatest) ); assert_eq!( - detect_update_action( - /*is_macos*/ true, - std::path::Path::new("/opt/homebrew/bin/codex"), - /*managed_by_npm*/ false, - /*managed_by_bun*/ false - ), + UpdateAction::from_install_context(&InstallContext::Brew), Some(UpdateAction::BrewUpgrade) ); assert_eq!( - detect_update_action( - /*is_macos*/ true, - std::path::Path::new("/usr/local/bin/codex"), - /*managed_by_npm*/ false, - /*managed_by_bun*/ false - ), - Some(UpdateAction::BrewUpgrade) + UpdateAction::from_install_context(&InstallContext::Standalone { + platform: StandalonePlatform::Unix, + release_dir: native_release_dir.clone(), + resources_dir: Some(native_release_dir.join("codex-resources")), + }), + Some(UpdateAction::StandaloneUnix) + ); + assert_eq!( + UpdateAction::from_install_context(&InstallContext::Standalone { + platform: StandalonePlatform::Windows, + release_dir: native_release_dir.clone(), + resources_dir: Some(native_release_dir.join("codex-resources")), + }), + Some(UpdateAction::StandaloneWindows) + ); + } + + #[test] + fn standalone_update_commands_rerun_latest_installer() { + assert_eq!( + UpdateAction::StandaloneUnix.command_args(), + ( + "sh", + &["-c", "curl -fsSL https://chatgpt.com/codex/install.sh | sh"][..], + ) + ); + assert_eq!( + UpdateAction::StandaloneWindows.command_args(), + ( + "powershell", + &["-c", "irm https://chatgpt.com/codex/install.ps1|iex"][..], + ) ); } } diff --git a/codex-rs/windows-sandbox-rs/src/helper_materialization.rs b/codex-rs/windows-sandbox-rs/src/helper_materialization.rs index 375199f5cc..ad5cc4e2b2 100644 --- a/codex-rs/windows-sandbox-rs/src/helper_materialization.rs +++ b/codex-rs/windows-sandbox-rs/src/helper_materialization.rs @@ -13,6 +13,8 @@ use tempfile::NamedTempFile; use crate::logging::log_note; use crate::sandbox_bin_dir; +const RESOURCES_DIRNAME: &str = "codex-resources"; + #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub(crate) enum HelperExecutable { CommandRunner, @@ -46,12 +48,9 @@ pub(crate) fn helper_bin_dir(codex_home: &Path) -> PathBuf { pub(crate) fn legacy_lookup(kind: HelperExecutable) -> PathBuf { if let Ok(exe) = std::env::current_exe() - && let Some(dir) = exe.parent() + && let Some(candidate) = source_path_for_exe(&exe, kind.file_name()) { - let candidate = dir.join(kind.file_name()); - if candidate.exists() { - return candidate; - } + return candidate; } PathBuf::from(kind.file_name()) } @@ -179,18 +178,23 @@ fn store_helper_path(cache_key: String, path: PathBuf) { fn sibling_source_path(kind: HelperExecutable) -> Result { let exe = std::env::current_exe().context("resolve current executable for helper lookup")?; - let dir = exe - .parent() - .ok_or_else(|| anyhow!("current executable has no parent directory"))?; - let candidate = dir.join(kind.file_name()); - if candidate.exists() { - Ok(candidate) - } else { - Err(anyhow!( - "helper not found next to current executable: {}", - candidate.display() - )) + source_path_for_exe(&exe, kind.file_name()).ok_or_else(|| { + anyhow!( + "helper not found next to current executable or under {RESOURCES_DIRNAME}: {}", + exe.display() + ) + }) +} + +fn source_path_for_exe(exe: &Path, file_name: &str) -> Option { + let dir = exe.parent()?; + let direct_candidate = dir.join(file_name); + if direct_candidate.exists() { + return Some(direct_candidate); } + + let resource_candidate = dir.join(RESOURCES_DIRNAME).join(file_name); + resource_candidate.exists().then_some(resource_candidate) } fn copy_from_source_if_needed(source: &Path, destination: &Path) -> Result { @@ -292,10 +296,12 @@ fn destination_is_fresh(source: &Path, destination: &Path) -> Result { #[cfg(test)] mod tests { - use super::destination_is_fresh; - use super::helper_bin_dir; use super::copy_from_source_if_needed; use super::CopyOutcome; + use super::destination_is_fresh; + use super::helper_bin_dir; + use super::RESOURCES_DIRNAME; + use super::source_path_for_exe; use pretty_assertions::assert_eq; use std::fs; use std::path::Path; @@ -376,4 +382,40 @@ mod tests { fs::read(&runner_destination).expect("read runner") ); } + + #[test] + fn helper_source_lookup_checks_resource_dir() { + let tmp = TempDir::new().expect("tempdir"); + let release_dir = tmp.path().join("release"); + let resources_dir = release_dir.join(RESOURCES_DIRNAME); + fs::create_dir_all(&resources_dir).expect("create resources dir"); + let exe = release_dir.join("codex.exe"); + let helper = resources_dir.join("codex-command-runner.exe"); + fs::write(&exe, b"codex").expect("write exe"); + fs::write(&helper, b"runner").expect("write helper"); + + let resolved = + source_path_for_exe(&exe, /*file_name*/ "codex-command-runner.exe").expect("helper path"); + + assert_eq!(resolved, helper); + } + + #[test] + fn helper_source_lookup_prefers_direct_sibling_over_resource_dir() { + let tmp = TempDir::new().expect("tempdir"); + let release_dir = tmp.path().join("release"); + let resources_dir = release_dir.join(RESOURCES_DIRNAME); + fs::create_dir_all(&resources_dir).expect("create resources dir"); + let exe = release_dir.join("codex.exe"); + let sibling_helper = release_dir.join("codex-command-runner.exe"); + let resource_helper = resources_dir.join("codex-command-runner.exe"); + fs::write(&exe, b"codex").expect("write exe"); + fs::write(&sibling_helper, b"sibling runner").expect("write sibling helper"); + fs::write(&resource_helper, b"resource runner").expect("write resource helper"); + + let resolved = + source_path_for_exe(&exe, /*file_name*/ "codex-command-runner.exe").expect("helper path"); + + assert_eq!(resolved, sibling_helper); + } } diff --git a/codex-rs/windows-sandbox-rs/src/setup_orchestrator.rs b/codex-rs/windows-sandbox-rs/src/setup_orchestrator.rs index e0a5a063f9..4effc0b70d 100644 --- a/codex-rs/windows-sandbox-rs/src/setup_orchestrator.rs +++ b/codex-rs/windows-sandbox-rs/src/setup_orchestrator.rs @@ -589,6 +589,16 @@ fn find_setup_exe() -> PathBuf { if candidate.exists() { return candidate; } + + // Standalone installs keep Windows helper binaries under + // `codex-resources/` next to `codex.exe`, so elevation needs to probe + // that sibling folder before falling back to PATH. + let resource_candidate = dir + .join("codex-resources") + .join("codex-windows-sandbox-setup.exe"); + if resource_candidate.exists() { + return resource_candidate; + } } PathBuf::from("codex-windows-sandbox-setup.exe") } diff --git a/scripts/install/install.ps1 b/scripts/install/install.ps1 index 40328db700..ed4a3e1a32 100644 --- a/scripts/install/install.ps1 +++ b/scripts/install/install.ps1 @@ -1,6 +1,5 @@ param( - [Parameter(Position=0)] - [string]$Version = "latest" + [string]$Release = "latest" ) Set-StrictMode -Version Latest @@ -15,6 +14,27 @@ function Write-Step { Write-Host "==> $Message" } +function Write-WarningStep { + param( + [string]$Message + ) + + Write-Warning $Message +} + +function Prompt-YesNo { + param( + [string]$Prompt + ) + + if ([Console]::IsInputRedirected -or [Console]::IsOutputRedirected) { + return $false + } + + $choice = Read-Host "$Prompt [y/N]" + return $choice -match "^(?i:y(?:es)?)$" +} + function Normalize-Version { param( [string]$RawVersion @@ -35,13 +55,39 @@ function Normalize-Version { return $RawVersion } -function Get-ReleaseUrl { +function Get-ReleaseAssetMetadata { param( [string]$AssetName, [string]$ResolvedVersion ) - return "https://github.com/openai/codex/releases/download/rust-v$ResolvedVersion/$AssetName" + $release = Invoke-RestMethod -Uri "https://api.github.com/repos/openai/codex/releases/tags/rust-v$ResolvedVersion" + $asset = $release.assets | Where-Object { $_.name -eq $AssetName } | Select-Object -First 1 + if ($null -eq $asset) { + throw "Could not find release asset $AssetName for Codex $ResolvedVersion." + } + + $digestMatch = [regex]::Match([string]$asset.digest, "^sha256:([0-9a-fA-F]{64})$") + if (-not $digestMatch.Success) { + throw "Could not find SHA-256 digest for release asset $AssetName." + } + + return [PSCustomObject]@{ + Url = $asset.browser_download_url + Sha256 = $digestMatch.Groups[1].Value.ToLowerInvariant() + } +} + +function Test-ArchiveDigest { + param( + [string]$ArchivePath, + [string]$ExpectedDigest + ) + + $actualDigest = (Get-FileHash -LiteralPath $ArchivePath -Algorithm SHA256).Hash.ToLowerInvariant() + if ($actualDigest -ne $ExpectedDigest) { + throw "Downloaded Codex archive checksum did not match release metadata. Expected $ExpectedDigest but got $actualDigest." + } } function Path-Contains { @@ -64,8 +110,46 @@ function Path-Contains { return $false } +function Invoke-WithInstallLock { + param( + [string]$LockPath, + [scriptblock]$Script + ) + + New-Item -ItemType Directory -Force -Path (Split-Path -Parent $LockPath) | Out-Null + $lock = $null + while ($null -eq $lock) { + try { + $lock = [System.IO.File]::Open( + $LockPath, + [System.IO.FileMode]::OpenOrCreate, + [System.IO.FileAccess]::ReadWrite, + [System.IO.FileShare]::None + ) + } catch [System.IO.IOException] { + Start-Sleep -Milliseconds 250 + } + } + try { + & $Script + } finally { + $lock.Dispose() + } +} + +function Remove-StaleInstallArtifacts { + param( + [string]$ReleasesDir + ) + + if (Test-Path -LiteralPath $ReleasesDir -PathType Container) { + Get-ChildItem -LiteralPath $ReleasesDir -Force -Directory -Filter ".staging.*" -ErrorAction SilentlyContinue | + Remove-Item -Recurse -Force -ErrorAction SilentlyContinue + } +} + function Resolve-Version { - $normalizedVersion = Normalize-Version -RawVersion $Version + $normalizedVersion = Normalize-Version -RawVersion $Release if ($normalizedVersion -ne "latest") { return $normalizedVersion } @@ -79,6 +163,414 @@ function Resolve-Version { return (Normalize-Version -RawVersion $release.tag_name) } +function Get-VersionFromBinary { + param( + [string]$CodexPath + ) + + if (-not (Test-Path -LiteralPath $CodexPath -PathType Leaf)) { + return $null + } + + try { + $versionOutput = & $CodexPath --version 2>$null + } catch { + return $null + } + + if ($versionOutput -match '([0-9][0-9A-Za-z.+-]*)$') { + return $matches[1] + } + + return $null +} + +function Get-CurrentInstalledVersion { + param( + [string]$StandaloneCurrentDir + ) + + $standaloneVersion = Get-VersionFromBinary -CodexPath (Join-Path $StandaloneCurrentDir "codex.exe") + if (-not [string]::IsNullOrWhiteSpace($standaloneVersion)) { + return $standaloneVersion + } + + return $null +} + +function Test-OldStandaloneBinLayout { + param( + [string]$VisibleBinDir, + [string]$DefaultVisibleBinDir + ) + + if (-not $VisibleBinDir.Equals($DefaultVisibleBinDir, [System.StringComparison]::OrdinalIgnoreCase)) { + return $false + } + if (-not (Test-Path -LiteralPath $VisibleBinDir -PathType Container)) { + return $false + } + + $item = Get-Item -LiteralPath $VisibleBinDir -Force + if ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) { + return $false + } + + $requiredFiles = @("codex.exe", "rg.exe") + foreach ($fileName in $requiredFiles) { + if (-not (Test-Path -LiteralPath (Join-Path $VisibleBinDir $fileName) -PathType Leaf)) { + return $false + } + } + + $knownFiles = @( + "codex.exe", + "rg.exe", + "codex-command-runner.exe", + "codex-windows-sandbox.exe", + "codex-windows-sandbox-setup.exe" + ) + foreach ($child in Get-ChildItem -LiteralPath $VisibleBinDir -Force) { + if ($child.PSIsContainer) { + return $false + } + if ($knownFiles -notcontains $child.Name) { + return $false + } + } + + return $true +} + +function Move-OldStandaloneBinIfApproved { + param( + [string]$VisibleBinDir, + [string]$DefaultVisibleBinDir + ) + + if (-not (Test-OldStandaloneBinLayout -VisibleBinDir $VisibleBinDir -DefaultVisibleBinDir $DefaultVisibleBinDir)) { + return $null + } + + Write-Step "We found an older Codex install at $VisibleBinDir" + Write-WarningStep "To continue, Codex needs to update the install at this path." + if (-not (Prompt-YesNo "Replace it with the current Codex setup now?")) { + throw "Cannot replace older standalone install without confirmation: $VisibleBinDir" + } + + $backupDir = "$VisibleBinDir.backup.$([DateTimeOffset]::UtcNow.ToUnixTimeSeconds()).$PID" + Write-Step "Moving older standalone install to $backupDir" + Move-Item -LiteralPath $VisibleBinDir -Destination $backupDir + return $backupDir +} + +function Add-JunctionSupportType { + if (([System.Management.Automation.PSTypeName]'CodexInstaller.Junction').Type) { + return + } + + Add-Type -TypeDefinition @" +using System; +using System.ComponentModel; +using System.IO; +using System.Runtime.InteropServices; +using System.Text; +using Microsoft.Win32.SafeHandles; + +namespace CodexInstaller +{ + public static class Junction + { + private const uint GENERIC_WRITE = 0x40000000; + private const uint FILE_SHARE_READ = 0x00000001; + private const uint FILE_SHARE_WRITE = 0x00000002; + private const uint FILE_SHARE_DELETE = 0x00000004; + private const uint OPEN_EXISTING = 3; + private const uint FILE_FLAG_BACKUP_SEMANTICS = 0x02000000; + private const uint FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000; + private const uint FSCTL_SET_REPARSE_POINT = 0x000900A4; + private const uint IO_REPARSE_TAG_MOUNT_POINT = 0xA0000003; + private const int HeaderLength = 20; + + [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + private static extern SafeFileHandle CreateFileW( + string lpFileName, + uint dwDesiredAccess, + uint dwShareMode, + IntPtr lpSecurityAttributes, + uint dwCreationDisposition, + uint dwFlagsAndAttributes, + IntPtr hTemplateFile); + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool DeviceIoControl( + SafeFileHandle hDevice, + uint dwIoControlCode, + byte[] lpInBuffer, + int nInBufferSize, + IntPtr lpOutBuffer, + int nOutBufferSize, + out int lpBytesReturned, + IntPtr lpOverlapped); + + public static void SetTarget(string linkPath, string targetPath) + { + string substituteName = "\\??\\" + Path.GetFullPath(targetPath); + byte[] substituteNameBytes = Encoding.Unicode.GetBytes(substituteName); + if (substituteNameBytes.Length > ushort.MaxValue - HeaderLength) { + throw new ArgumentException("Junction target path is too long.", "targetPath"); + } + + byte[] reparseBuffer = new byte[substituteNameBytes.Length + HeaderLength]; + WriteUInt32(reparseBuffer, 0, IO_REPARSE_TAG_MOUNT_POINT); + WriteUInt16(reparseBuffer, 4, checked((ushort)(substituteNameBytes.Length + 12))); + WriteUInt16(reparseBuffer, 8, 0); + WriteUInt16(reparseBuffer, 10, checked((ushort)substituteNameBytes.Length)); + WriteUInt16(reparseBuffer, 12, checked((ushort)(substituteNameBytes.Length + 2))); + WriteUInt16(reparseBuffer, 14, 0); + Buffer.BlockCopy(substituteNameBytes, 0, reparseBuffer, 16, substituteNameBytes.Length); + + using (SafeFileHandle handle = CreateFileW( + linkPath, + GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + IntPtr.Zero, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT, + IntPtr.Zero)) + { + if (handle.IsInvalid) { + throw new Win32Exception(Marshal.GetLastWin32Error()); + } + + int bytesReturned; + if (!DeviceIoControl( + handle, + FSCTL_SET_REPARSE_POINT, + reparseBuffer, + reparseBuffer.Length, + IntPtr.Zero, + 0, + out bytesReturned, + IntPtr.Zero)) + { + throw new Win32Exception(Marshal.GetLastWin32Error()); + } + } + } + + private static void WriteUInt16(byte[] buffer, int offset, ushort value) + { + buffer[offset] = (byte)value; + buffer[offset + 1] = (byte)(value >> 8); + } + + private static void WriteUInt32(byte[] buffer, int offset, uint value) + { + buffer[offset] = (byte)value; + buffer[offset + 1] = (byte)(value >> 8); + buffer[offset + 2] = (byte)(value >> 16); + buffer[offset + 3] = (byte)(value >> 24); + } + } +} +"@ +} + +function Set-JunctionTarget { + param( + [string]$LinkPath, + [string]$TargetPath + ) + + Add-JunctionSupportType + [CodexInstaller.Junction]::SetTarget($LinkPath, $TargetPath) +} + +function Test-IsJunction { + param( + [string]$Path + ) + + if (-not (Test-Path -LiteralPath $Path)) { + return $false + } + + $item = Get-Item -LiteralPath $Path -Force + return ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -and $item.LinkType -eq "Junction" +} + +function Ensure-Junction { + param( + [string]$LinkPath, + [string]$TargetPath, + [string]$InstallerOwnedTargetPrefix + ) + + if (-not (Test-Path -LiteralPath $LinkPath)) { + New-Item -ItemType Junction -Path $LinkPath -Target $TargetPath | Out-Null + return + } + + $item = Get-Item -LiteralPath $LinkPath -Force + if (Test-IsJunction -Path $LinkPath) { + $existingTarget = [string]$item.Target + if (-not [string]::IsNullOrWhiteSpace($InstallerOwnedTargetPrefix)) { + $ownedTargetPrefix = $InstallerOwnedTargetPrefix.TrimEnd("\\") + if (-not $existingTarget.StartsWith($ownedTargetPrefix, [System.StringComparison]::OrdinalIgnoreCase)) { + throw "Refusing to retarget junction at $LinkPath because it is not managed by this installer." + } + } + if ($existingTarget.Equals($TargetPath, [System.StringComparison]::OrdinalIgnoreCase)) { + return + } + + # Keep the path itself in place and only retarget the junction. That + # avoids a gap where current or the visible bin path disappears during + # an update. + Set-JunctionTarget -LinkPath $LinkPath -TargetPath $TargetPath + return + } + + if ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) { + throw "Refusing to replace non-junction reparse point at $LinkPath." + } + + if ($item.PSIsContainer) { + if ((Get-ChildItem -LiteralPath $LinkPath -Force | Select-Object -First 1) -ne $null) { + throw "Refusing to replace non-empty directory at $LinkPath with a junction." + } + + Remove-Item -LiteralPath $LinkPath -Force + New-Item -ItemType Junction -Path $LinkPath -Target $TargetPath | Out-Null + return + } + + throw "Refusing to replace file at $LinkPath with a junction." +} + +function Test-ReleaseIsComplete { + param( + [string]$ReleaseDir, + [string]$ExpectedVersion, + [string]$ExpectedTarget + ) + + if (-not (Test-Path -LiteralPath $ReleaseDir -PathType Container)) { + return $false + } + + $expectedFiles = @( + "codex.exe", + "codex-resources\codex-command-runner.exe", + "codex-resources\codex-windows-sandbox-setup.exe", + "codex-resources\rg.exe" + ) + foreach ($name in $expectedFiles) { + if (-not (Test-Path -LiteralPath (Join-Path $ReleaseDir $name) -PathType Leaf)) { + return $false + } + } + + return (Split-Path -Leaf $ReleaseDir) -eq "$ExpectedVersion-$ExpectedTarget" +} + +function Get-ExistingCodexCommand { + $existing = Get-Command codex -ErrorAction SilentlyContinue + if ($null -eq $existing) { + return $null + } + + return $existing.Source +} + +function Get-ExistingCodexManager { + param( + [string]$ExistingPath, + [string]$VisibleBinDir + ) + + if ([string]::IsNullOrWhiteSpace($ExistingPath)) { + return $null + } + + if ($ExistingPath.StartsWith($VisibleBinDir, [System.StringComparison]::OrdinalIgnoreCase)) { + return $null + } + + if ($ExistingPath -match "\\.bun\\") { + return "bun" + } + + if ($ExistingPath -match "node_modules" -or $ExistingPath -match "\\npm\\") { + return "npm" + } + + return $null +} + +function Get-ConflictingInstall { + param( + [string]$VisibleBinDir + ) + + $existingPath = Get-ExistingCodexCommand + $manager = Get-ExistingCodexManager -ExistingPath $existingPath -VisibleBinDir $VisibleBinDir + if ($null -eq $manager) { + return $null + } + + Write-Step "Detected existing $manager-managed Codex at $existingPath" + Write-WarningStep "Multiple managed Codex installs can be ambiguous because PATH order decides which one runs." + + return [PSCustomObject]@{ + Manager = $manager + Path = $existingPath + } +} + +function Maybe-HandleConflictingInstall { + param( + [object]$Conflict + ) + + if ($null -eq $Conflict) { + return + } + + $manager = $Conflict.Manager + + $uninstallArgs = if ($manager -eq "bun") { + @("remove", "-g", "@openai/codex") + } else { + @("uninstall", "-g", "@openai/codex") + } + $uninstallCommand = if ($manager -eq "bun") { "bun" } else { "npm" } + + if (Prompt-YesNo "Uninstall the existing $manager-managed Codex now?") { + Write-Step "Running: $uninstallCommand $($uninstallArgs -join ' ')" + try { + & $uninstallCommand @uninstallArgs + } catch { + Write-WarningStep "Failed to uninstall the existing $manager-managed Codex. Continuing with the standalone install." + } + } else { + Write-WarningStep "Leaving the existing $manager-managed Codex installed. PATH order will determine which codex runs." + } +} + +function Test-VisibleCodexCommand { + param( + [string]$VisibleBinDir + ) + + $codexCommand = Join-Path $VisibleBinDir "codex.exe" + & $codexCommand --version *> $null + if ($LASTEXITCODE -ne 0) { + throw "Installed Codex command failed verification: $codexCommand --version" + } +} + if ($env:OS -ne "Windows_NT") { Write-Error "install.ps1 supports Windows only. Use install.sh on macOS or Linux." exit 1 @@ -110,87 +602,149 @@ switch ($architecture) { } } -if ([string]::IsNullOrWhiteSpace($env:CODEX_INSTALL_DIR)) { - $installDir = Join-Path $env:LOCALAPPDATA "Programs\OpenAI\Codex\bin" +$codexHome = if ([string]::IsNullOrWhiteSpace($env:CODEX_HOME)) { + Join-Path $env:USERPROFILE ".codex" } else { - $installDir = $env:CODEX_INSTALL_DIR + $env:CODEX_HOME +} +$standaloneRoot = Join-Path $codexHome "packages\standalone" +$releasesDir = Join-Path $standaloneRoot "releases" +$currentDir = Join-Path $standaloneRoot "current" +$lockPath = Join-Path $standaloneRoot "install.lock" + +$defaultVisibleBinDir = Join-Path $env:LOCALAPPDATA "Programs\OpenAI\Codex\bin" +if ([string]::IsNullOrWhiteSpace($env:CODEX_INSTALL_DIR)) { + $visibleBinDir = $defaultVisibleBinDir +} else { + $visibleBinDir = $env:CODEX_INSTALL_DIR } -$codexPath = Join-Path $installDir "codex.exe" -$installMode = if (Test-Path $codexPath) { "Updating" } else { "Installing" } - -Write-Step "$installMode Codex CLI" -Write-Step "Detected platform: $platformLabel" - -New-Item -ItemType Directory -Force -Path $installDir | Out-Null - +$currentVersion = Get-CurrentInstalledVersion -StandaloneCurrentDir $currentDir $resolvedVersion = Resolve-Version -Write-Step "Resolved version: $resolvedVersion" -$packageAsset = "codex-npm-$npmTag-$resolvedVersion.tgz" +$releaseName = "$resolvedVersion-$target" +$releaseDir = Join-Path $releasesDir $releaseName +if (-not [string]::IsNullOrWhiteSpace($currentVersion) -and $currentVersion -ne $resolvedVersion) { + Write-Step "Updating Codex CLI from $currentVersion to $resolvedVersion" +} elseif (-not [string]::IsNullOrWhiteSpace($currentVersion)) { + Write-Step "Updating Codex CLI" +} else { + Write-Step "Installing Codex CLI" +} +Write-Step "Detected platform: $platformLabel" +Write-Step "Resolved version: $resolvedVersion" + +$conflictingInstall = Get-ConflictingInstall -VisibleBinDir $visibleBinDir +$oldStandaloneBackup = $null + +$packageAsset = "codex-npm-$npmTag-$resolvedVersion.tgz" $tempDir = Join-Path ([System.IO.Path]::GetTempPath()) ("codex-install-" + [System.Guid]::NewGuid().ToString("N")) New-Item -ItemType Directory -Force -Path $tempDir | Out-Null try { - $archivePath = Join-Path $tempDir $packageAsset - $extractDir = Join-Path $tempDir "extract" - $url = Get-ReleaseUrl -AssetName $packageAsset -ResolvedVersion $resolvedVersion + Invoke-WithInstallLock -LockPath $lockPath -Script { + Remove-StaleInstallArtifacts -ReleasesDir $releasesDir - Write-Step "Downloading Codex CLI" - Invoke-WebRequest -Uri $url -OutFile $archivePath + if (-not (Test-ReleaseIsComplete -ReleaseDir $releaseDir -ExpectedVersion $resolvedVersion -ExpectedTarget $target)) { + if (Test-Path -LiteralPath $releaseDir) { + Write-WarningStep "Found incomplete existing release at $releaseDir. Reinstalling." + } - New-Item -ItemType Directory -Force -Path $extractDir | Out-Null - tar -xzf $archivePath -C $extractDir + $archivePath = Join-Path $tempDir $packageAsset + $extractDir = Join-Path $tempDir "extract" + $stagingDir = Join-Path $releasesDir ".staging.$releaseName.$PID" + $assetMetadata = Get-ReleaseAssetMetadata -AssetName $packageAsset -ResolvedVersion $resolvedVersion - $vendorRoot = Join-Path $extractDir "package/vendor/$target" - Write-Step "Installing to $installDir" - $copyMap = @{ - "codex/codex.exe" = "codex.exe" - "codex/codex-command-runner.exe" = "codex-command-runner.exe" - "codex/codex-windows-sandbox-setup.exe" = "codex-windows-sandbox-setup.exe" - "path/rg.exe" = "rg.exe" - } + Write-Step "Downloading Codex CLI" + Invoke-WebRequest -Uri $assetMetadata.Url -OutFile $archivePath + Test-ArchiveDigest -ArchivePath $archivePath -ExpectedDigest $assetMetadata.Sha256 - foreach ($relativeSource in $copyMap.Keys) { - $sourcePath = Join-Path $vendorRoot $relativeSource - $destinationPath = Join-Path $installDir $copyMap[$relativeSource] - Move-Item -Force $sourcePath $destinationPath + New-Item -ItemType Directory -Force -Path $extractDir | Out-Null + New-Item -ItemType Directory -Force -Path $releasesDir | Out-Null + if (Test-Path -LiteralPath $stagingDir) { + Remove-Item -LiteralPath $stagingDir -Recurse -Force + } + New-Item -ItemType Directory -Force -Path $stagingDir | Out-Null + tar -xzf $archivePath -C $extractDir + + $vendorRoot = Join-Path $extractDir "package/vendor/$target" + $resourcesDir = Join-Path $stagingDir "codex-resources" + New-Item -ItemType Directory -Force -Path $resourcesDir | Out-Null + $copyMap = @{ + "codex/codex.exe" = "codex.exe" + "codex/codex-command-runner.exe" = "codex-resources\codex-command-runner.exe" + "codex/codex-windows-sandbox-setup.exe" = "codex-resources\codex-windows-sandbox-setup.exe" + "path/rg.exe" = "codex-resources\rg.exe" + } + + foreach ($relativeSource in $copyMap.Keys) { + Copy-Item -LiteralPath (Join-Path $vendorRoot $relativeSource) -Destination (Join-Path $stagingDir $copyMap[$relativeSource]) + } + + if (Test-Path -LiteralPath $releaseDir) { + Remove-Item -LiteralPath $releaseDir -Recurse -Force + } + Move-Item -LiteralPath $stagingDir -Destination $releaseDir + } + + New-Item -ItemType Directory -Force -Path $standaloneRoot | Out-Null + Ensure-Junction -LinkPath $currentDir -TargetPath $releaseDir -InstallerOwnedTargetPrefix $releasesDir + + $visibleParent = Split-Path -Parent $visibleBinDir + New-Item -ItemType Directory -Force -Path $visibleParent | Out-Null + $oldStandaloneBackup = Move-OldStandaloneBinIfApproved -VisibleBinDir $visibleBinDir -DefaultVisibleBinDir $defaultVisibleBinDir + try { + Ensure-Junction -LinkPath $visibleBinDir -TargetPath $currentDir -InstallerOwnedTargetPrefix $standaloneRoot + Test-VisibleCodexCommand -VisibleBinDir $visibleBinDir + } catch { + if ($null -ne $oldStandaloneBackup -and (Test-Path -LiteralPath $oldStandaloneBackup)) { + if (Test-Path -LiteralPath $visibleBinDir) { + Remove-Item -LiteralPath $visibleBinDir -Recurse -Force + } + Move-Item -LiteralPath $oldStandaloneBackup -Destination $visibleBinDir + } + throw + } + if ($null -ne $oldStandaloneBackup) { + Remove-Item -LiteralPath $oldStandaloneBackup -Recurse -Force + } } } finally { Remove-Item -Recurse -Force $tempDir -ErrorAction SilentlyContinue } +Maybe-HandleConflictingInstall -Conflict $conflictingInstall + $userPath = [Environment]::GetEnvironmentVariable("Path", "User") -$pathNeedsNewShell = $false -if (-not (Path-Contains -PathValue $userPath -Entry $installDir)) { +if (-not (Path-Contains -PathValue $userPath -Entry $visibleBinDir)) { if ([string]::IsNullOrWhiteSpace($userPath)) { - $newUserPath = $installDir + $newUserPath = $visibleBinDir } else { - $newUserPath = "$installDir;$userPath" + $newUserPath = "$visibleBinDir;$userPath" } [Environment]::SetEnvironmentVariable("Path", $newUserPath, "User") - if (-not (Path-Contains -PathValue $env:Path -Entry $installDir)) { - if ([string]::IsNullOrWhiteSpace($env:Path)) { - $env:Path = $installDir - } else { - $env:Path = "$installDir;$env:Path" - } - } Write-Step "PATH updated for future PowerShell sessions." - $pathNeedsNewShell = $true -} elseif (Path-Contains -PathValue $env:Path -Entry $installDir) { - Write-Step "$installDir is already on PATH." +} elseif (Path-Contains -PathValue $env:Path -Entry $visibleBinDir) { + Write-Step "$visibleBinDir is already on PATH." } else { Write-Step "PATH is already configured for future PowerShell sessions." - $pathNeedsNewShell = $true } -if ($pathNeedsNewShell) { - Write-Step ('Run now: $env:Path = "{0};$env:Path"; codex' -f $installDir) - Write-Step "Or open a new PowerShell window and run: codex" -} else { - Write-Step "Run: codex" +if (-not (Path-Contains -PathValue $env:Path -Entry $visibleBinDir)) { + if ([string]::IsNullOrWhiteSpace($env:Path)) { + $env:Path = $visibleBinDir + } else { + $env:Path = "$visibleBinDir;$env:Path" + } } +Write-Step "Current PowerShell session: codex" +Write-Step "Future PowerShell windows: open a new PowerShell window and run: codex" Write-Host "Codex CLI $resolvedVersion installed successfully." + +$codexCommand = Join-Path $visibleBinDir "codex.exe" +if (Prompt-YesNo "Start Codex now?") { + Write-Step "Launching Codex" + & $codexCommand +} diff --git a/scripts/install/install.sh b/scripts/install/install.sh index 86e2940d55..8c225e4d3b 100755 --- a/scripts/install/install.sh +++ b/scripts/install/install.sh @@ -2,15 +2,33 @@ set -eu -VERSION="${1:-latest}" -INSTALL_DIR="${CODEX_INSTALL_DIR:-$HOME/.local/bin}" +RELEASE="latest" + +BIN_DIR="${CODEX_INSTALL_DIR:-$HOME/.local/bin}" +BIN_PATH="$BIN_DIR/codex" +CODEX_HOME_DIR="${CODEX_HOME:-$HOME/.codex}" +STANDALONE_ROOT="$CODEX_HOME_DIR/packages/standalone" +RELEASES_DIR="$STANDALONE_ROOT/releases" +CURRENT_LINK="$STANDALONE_ROOT/current" +LOCK_FILE="$STANDALONE_ROOT/install.lock" +LOCK_DIR="$STANDALONE_ROOT/install.lock.d" +LOCK_STALE_AFTER_SECS=600 + path_action="already" path_profile="" +conflict_manager="" +conflict_path="" +lock_kind="" +tmp_dir="" step() { printf '==> %s\n' "$1" } +warn() { + printf 'WARNING: %s\n' "$1" >&2 +} + normalize_version() { case "$1" in "" | latest) @@ -28,6 +46,32 @@ normalize_version() { esac } +parse_args() { + while [ "$#" -gt 0 ]; do + case "$1" in + --release) + if [ "$#" -lt 2 ]; then + echo "--release requires a value." >&2 + exit 1 + fi + RELEASE="$2" + shift + ;; + --help | -h) + cat <&2 + exit 1 + ;; + esac + shift + done +} + download_file() { url="$1" output="$2" @@ -63,40 +107,6 @@ download_text() { exit 1 } -add_to_path() { - path_action="already" - path_profile="" - - case ":$PATH:" in - *":$INSTALL_DIR:"*) - return - ;; - esac - - profile="$HOME/.profile" - case "${SHELL:-}" in - */zsh) - profile="$HOME/.zshrc" - ;; - */bash) - profile="$HOME/.bashrc" - ;; - esac - - path_profile="$profile" - path_line="export PATH=\"$INSTALL_DIR:\$PATH\"" - if [ -f "$profile" ] && grep -F "$path_line" "$profile" >/dev/null 2>&1; then - path_action="configured" - return - fi - - { - printf '\n# Added by Codex installer\n' - printf '%s\n' "$path_line" - } >>"$profile" - path_action="added" -} - release_url_for_asset() { asset="$1" resolved_version="$2" @@ -104,6 +114,92 @@ release_url_for_asset() { printf 'https://github.com/openai/codex/releases/download/rust-v%s/%s\n' "$resolved_version" "$asset" } +release_metadata_url() { + resolved_version="$1" + + printf 'https://api.github.com/repos/openai/codex/releases/tags/rust-v%s\n' "$resolved_version" +} + +release_asset_digest() { + asset="$1" + resolved_version="$2" + release_json="$(download_text "$(release_metadata_url "$resolved_version")")" + + digest="$(printf '%s\n' "$release_json" | awk -v asset="$asset" ' + { + if ($0 ~ "\"name\":[[:space:]]*\"" asset "\"") { + in_asset = 1 + asset_depth = depth + } + + if (in_asset && /"digest":[[:space:]]*"[^"]+"/) { + sub(/^.*"digest":[[:space:]]*"/, "") + sub(/".*$/, "") + digest = $0 + } + + line = $0 + opens = gsub(/\{/, "{", line) + closes = gsub(/\}/, "}", line) + depth += opens - closes + + if (in_asset && depth < asset_depth) { + in_asset = 0 + } + } + END { + if (digest != "") { + print digest + } + } + ')" + + case "$digest" in + sha256:????????????????????????????????????????????????????????????????) + printf '%s\n' "${digest#sha256:}" + ;; + *) + echo "Could not find SHA-256 digest for release asset $asset." >&2 + exit 1 + ;; + esac +} + +file_sha256() { + path="$1" + + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$path" | awk '{print $1}' + return + fi + + if command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$path" | awk '{print $1}' + return + fi + + if command -v openssl >/dev/null 2>&1; then + openssl dgst -sha256 "$path" | sed 's/^.*= //' + return + fi + + echo "sha256sum, shasum, or openssl is required to verify the Codex download." >&2 + exit 1 +} + +verify_archive_digest() { + archive_path="$1" + expected_digest="$2" + actual_digest="$(file_sha256 "$archive_path")" + + if [ "$actual_digest" != "$expected_digest" ]; then + echo "Downloaded Codex archive checksum did not match release metadata." >&2 + echo "expected: $expected_digest" >&2 + echo "actual: $actual_digest" >&2 + exit 1 + fi +} + require_command() { if ! command -v "$1" >/dev/null 2>&1; then echo "$1 is required to install Codex." >&2 @@ -111,11 +207,8 @@ require_command() { fi } -require_command mktemp -require_command tar - resolve_version() { - normalized_version="$(normalize_version "$VERSION")" + normalized_version="$(normalize_version "$RELEASE")" if [ "$normalized_version" != "latest" ]; then printf '%s\n' "$normalized_version" @@ -133,6 +226,417 @@ resolve_version() { printf '%s\n' "$resolved" } +pick_profile() { + # Use the same shell-specific split Homebrew documents because there is no + # universal startup file across macOS/Linux login and interactive shells. + case "$os:${SHELL:-}" in + darwin:*/zsh) + printf '%s\n' "$HOME/.zprofile" + ;; + darwin:*/bash) + printf '%s\n' "$HOME/.bash_profile" + ;; + linux:*/zsh) + printf '%s\n' "$HOME/.zshrc" + ;; + linux:*/bash) + printf '%s\n' "$HOME/.bashrc" + ;; + *) + printf '%s\n' "$HOME/.profile" + ;; + esac +} + +add_to_path() { + path_action="already" + path_profile="" + + case ":$PATH:" in + *":$BIN_DIR:"*) + return + ;; + esac + + profile="$(pick_profile)" + path_profile="$profile" + begin_marker="# >>> Codex installer >>>" + end_marker="# <<< Codex installer <<<" + path_line="export PATH=\"$BIN_DIR:\$PATH\"" + + if [ -f "$profile" ] && grep -F "$begin_marker" "$profile" >/dev/null 2>&1; then + if grep -F "$path_line" "$profile" >/dev/null 2>&1; then + path_action="configured" + return + fi + + if grep -F "$end_marker" "$profile" >/dev/null 2>&1; then + rewrite_path_block "$profile" "$begin_marker" "$end_marker" "$path_line" + path_action="updated" + return + fi + fi + + append_path_block "$profile" "$begin_marker" "$end_marker" "$path_line" + path_action="added" +} + +append_path_block() { + profile="$1" + begin_marker="$2" + end_marker="$3" + path_line="$4" + + { + printf '\n%s\n' "$begin_marker" + printf '%s\n' "$path_line" + printf '%s\n' "$end_marker" + } >>"$profile" +} + +rewrite_path_block() { + profile="$1" + begin_marker="$2" + end_marker="$3" + path_line="$4" + tmp_profile="$tmp_dir/profile.$$.tmp" + + awk -v begin="$begin_marker" -v end="$end_marker" -v line="$path_line" ' + BEGIN { + in_block = 0 + replaced = 0 + } + $0 == begin { + if (!replaced) { + print begin + print line + print end + replaced = 1 + } + in_block = 1 + next + } + in_block { + if ($0 == end) { + in_block = 0 + } + next + } + { + print + } + END { + if (in_block != 0) { + exit 1 + } + } + ' "$profile" >"$tmp_profile" + mv "$tmp_profile" "$profile" +} + +mkdir_lock_is_stale() { + [ -d "$LOCK_DIR" ] || return 1 + + pid="$(cat "$LOCK_DIR/pid" 2>/dev/null || true)" + started_at="$(cat "$LOCK_DIR/started_at" 2>/dev/null || true)" + now="$(date +%s 2>/dev/null || printf '0')" + + case "$started_at" in + ''|*[!0-9]*) + started_at=0 + ;; + esac + + if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then + return 1 + fi + + if [ "$started_at" -eq 0 ] || [ "$now" -eq 0 ]; then + return 0 + fi + + [ $((now - started_at)) -ge "$LOCK_STALE_AFTER_SECS" ] +} + +acquire_install_lock() { + mkdir -p "$STANDALONE_ROOT" + + if [ "$os" = "darwin" ] && command -v lockf >/dev/null 2>&1; then + : >>"$LOCK_FILE" + exec 9<>"$LOCK_FILE" + lockf 9 + lock_kind="lockf" + return + fi + + if command -v flock >/dev/null 2>&1; then + exec 9>"$LOCK_FILE" + flock 9 + lock_kind="flock" + return + fi + + while ! mkdir "$LOCK_DIR" 2>/dev/null; do + if mkdir_lock_is_stale; then + warn "Removing stale installer lock at $LOCK_DIR" + rm -rf "$LOCK_DIR" + continue + fi + sleep 1 + done + + printf '%s\n' "$$" >"$LOCK_DIR/pid" + date +%s >"$LOCK_DIR/started_at" 2>/dev/null || true + lock_kind="mkdir" +} + +release_install_lock() { + if [ "$lock_kind" = "mkdir" ]; then + rm -rf "$LOCK_DIR" 2>/dev/null || true + elif [ "$lock_kind" = "flock" ] || [ "$lock_kind" = "lockf" ]; then + exec 9>&- 2>/dev/null || true + fi + lock_kind="" +} + +cleanup_stale_install_artifacts() { + mkdir -p "$RELEASES_DIR" "$STANDALONE_ROOT" + + find "$RELEASES_DIR" -mindepth 1 -maxdepth 1 -name '.staging.*' -exec rm -rf {} + + find "$STANDALONE_ROOT" -mindepth 1 -maxdepth 1 -name '.current.*' -exec rm -f {} + + + if [ -d "$BIN_DIR" ]; then + find "$BIN_DIR" -mindepth 1 -maxdepth 1 -name '.codex.*' -exec rm -f {} + + fi +} + +replace_path_with_symlink() { + link_path="$1" + link_target="$2" + tmp_link="$3" + + rm -f "$tmp_link" + ln -s "$link_target" "$tmp_link" + + if mv -Tf "$tmp_link" "$link_path" 2>/dev/null; then + return + fi + + if mv -hf "$tmp_link" "$link_path" 2>/dev/null; then + return + fi + + rm -f "$link_path" + mv -f "$tmp_link" "$link_path" +} + +version_from_binary() { + codex_path="$1" + + if [ ! -x "$codex_path" ]; then + return 1 + fi + + "$codex_path" --version 2>/dev/null | sed -n 's/.* \([0-9][0-9A-Za-z.+-]*\)$/\1/p' | head -n 1 +} + +current_installed_version() { + version="$(version_from_binary "$CURRENT_LINK/codex" || true)" + if [ -n "$version" ]; then + printf '%s\n' "$version" + return 0 + fi + + return 0 +} + +resolve_existing_codex() { + command -v codex 2>/dev/null || true +} + +classify_existing_codex() { + existing_path="$1" + + if [ -z "$existing_path" ] || [ "$existing_path" = "$BIN_PATH" ]; then + return 1 + fi + + case "$existing_path" in + /opt/homebrew/* | /usr/local/*) + if [ "$os" = "darwin" ]; then + printf 'brew\n' + return 0 + fi + ;; + esac + + if [ -f "$existing_path" ] && grep -F "#!/usr/bin/env node" "$existing_path" >/dev/null 2>&1; then + case "$existing_path" in + *".bun"*) + printf 'bun\n' + ;; + *) + printf 'npm\n' + ;; + esac + return 0 + fi + + return 1 +} + +prompt_yes_no() { + prompt="$1" + + if ( : /dev/null; then + printf '%s [y/N] ' "$prompt" >/dev/tty + if ! IFS= read -r answer /dev/null +} + +parse_args "$@" + +require_command mktemp +require_command tar + case "$(uname -s)" in Darwin) os="darwin" @@ -187,58 +691,78 @@ else fi fi -if [ -x "$INSTALL_DIR/codex" ]; then - install_mode="Updating" -else - install_mode="Installing" -fi - -step "$install_mode Codex CLI" -step "Detected platform: $platform_label" - resolved_version="$(resolve_version)" asset="codex-npm-$npm_tag-$resolved_version.tgz" download_url="$(release_url_for_asset "$asset" "$resolved_version")" +release_name="$resolved_version-$vendor_target" +release_dir="$RELEASES_DIR/$release_name" +current_version="$(current_installed_version)" +if [ -n "$current_version" ] && [ "$current_version" != "$resolved_version" ]; then + step "Updating Codex CLI from $current_version to $resolved_version" +elif [ -n "$current_version" ]; then + step "Updating Codex CLI" +else + step "Installing Codex CLI" +fi +step "Detected platform: $platform_label" step "Resolved version: $resolved_version" +detect_conflicting_install + tmp_dir="$(mktemp -d)" cleanup() { - rm -rf "$tmp_dir" + release_install_lock + if [ -n "$tmp_dir" ]; then + rm -rf "$tmp_dir" + fi } trap cleanup EXIT INT TERM -archive_path="$tmp_dir/$asset" +acquire_install_lock +cleanup_stale_install_artifacts -step "Downloading Codex CLI" -download_file "$download_url" "$archive_path" +if ! release_dir_is_complete "$release_dir" "$resolved_version" "$vendor_target"; then + if [ -e "$release_dir" ] || [ -L "$release_dir" ]; then + warn "Found incomplete existing release at $release_dir; reinstalling." + fi -tar -xzf "$archive_path" -C "$tmp_dir" + archive_path="$tmp_dir/$asset" + extract_dir="$tmp_dir/extract" -step "Installing to $INSTALL_DIR" -mkdir -p "$INSTALL_DIR" -cp "$tmp_dir/package/vendor/$vendor_target/codex/codex" "$INSTALL_DIR/codex" -cp "$tmp_dir/package/vendor/$vendor_target/path/rg" "$INSTALL_DIR/rg" -chmod 0755 "$INSTALL_DIR/codex" -chmod 0755 "$INSTALL_DIR/rg" + step "Downloading Codex CLI" + expected_digest="$(release_asset_digest "$asset" "$resolved_version")" + download_file "$download_url" "$archive_path" + verify_archive_digest "$archive_path" "$expected_digest" + mkdir -p "$extract_dir" + tar -xzf "$archive_path" -C "$extract_dir" + + step "Installing standalone package to $release_dir" + install_release "$release_dir" "$extract_dir/package/vendor/$vendor_target" +fi +update_current_link "$release_dir" +update_visible_command add_to_path +verify_visible_command +release_install_lock +handle_conflicting_install case "$path_action" in added) - step "PATH updated for future shells in $path_profile" - step "Run now: export PATH=\"$INSTALL_DIR:\$PATH\" && codex" - step "Or open a new terminal and run: codex" + print_launch_instructions + ;; + updated) + print_launch_instructions ;; configured) - step "PATH is already configured for future shells in $path_profile" - step "Run now: export PATH=\"$INSTALL_DIR:\$PATH\" && codex" - step "Or open a new terminal and run: codex" + print_launch_instructions ;; *) - step "$INSTALL_DIR is already on PATH" - step "Run: codex" + step "$BIN_DIR is already on PATH" + print_launch_instructions ;; esac printf 'Codex CLI %s installed successfully.\n' "$resolved_version" +maybe_launch_codex_now From 28b76d13fe1a47b25b49354ee05741ddadb3ac84 Mon Sep 17 00:00:00 2001 From: Matthew Zeng Date: Wed, 15 Apr 2026 14:48:05 -0700 Subject: [PATCH 45/84] [mcp] Add dummy tools for previously called but currently missing tools. (#17853) - [x] Add dummy tools for previously called but currently missing tools. Currently supporting MCP tools only. --- codex-rs/core/config.schema.json | 6 + codex-rs/core/src/codex.rs | 23 ++- codex-rs/core/src/codex_tests.rs | 2 + codex-rs/core/src/lib.rs | 1 + codex-rs/core/src/tools/code_mode/mod.rs | 1 + codex-rs/core/src/tools/handlers/mod.rs | 3 + .../src/tools/handlers/unavailable_tool.rs | 44 +++++ codex-rs/core/src/tools/js_repl/mod.rs | 1 + codex-rs/core/src/tools/router.rs | 3 + codex-rs/core/src/tools/router_tests.rs | 5 + codex-rs/core/src/tools/spec.rs | 43 +++++ codex-rs/core/src/tools/spec_tests.rs | 69 ++++++++ codex-rs/core/src/unavailable_tool.rs | 92 ++++++++++ codex-rs/core/tests/suite/tools.rs | 166 ++++++++++++++++++ codex-rs/features/src/lib.rs | 8 + codex-rs/features/src/tests.rs | 9 + 16 files changed, 473 insertions(+), 3 deletions(-) create mode 100644 codex-rs/core/src/tools/handlers/unavailable_tool.rs create mode 100644 codex-rs/core/src/unavailable_tool.rs diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index 9280895b5e..c61daa8b56 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -500,6 +500,9 @@ "tui_app_server": { "type": "boolean" }, + "unavailable_dummy_tools": { + "type": "boolean" + }, "undo": { "type": "boolean" }, @@ -2360,6 +2363,9 @@ "tui_app_server": { "type": "boolean" }, + "unavailable_dummy_tools": { + "type": "boolean" + }, "undo": { "type": "boolean" }, diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index cc314a66e9..a3f65b2355 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -44,6 +44,7 @@ use crate::stream_events_utils::last_assistant_message_from_item; use crate::stream_events_utils::raw_assistant_output_text_from_item; use crate::stream_events_utils::record_completed_response_item; use crate::turn_metadata::TurnMetadataState; +use crate::unavailable_tool::collect_unavailable_called_tools; use crate::util::error_or_panic; use async_channel::Receiver; use async_channel::Sender; @@ -7229,7 +7230,22 @@ pub(crate) async fn built_tools( &turn_context.config, &turn_context.tools_config, ); - let direct_mcp_tools = has_mcp_servers.then_some(mcp_tool_exposure.direct_tools); + let mcp_tools = has_mcp_servers.then_some(mcp_tool_exposure.direct_tools); + let deferred_mcp_tools = mcp_tool_exposure.deferred_tools; + let unavailable_called_tools = if turn_context + .config + .features + .enabled(Feature::UnavailableDummyTools) + { + let exposed_tool_names = mcp_tools + .iter() + .chain(deferred_mcp_tools.iter()) + .flat_map(|tools| tools.keys().map(String::as_str)) + .collect::>(); + collect_unavailable_called_tools(input, &exposed_tool_names) + } else { + Vec::new() + }; let parallel_mcp_server_names = turn_context .config @@ -7246,8 +7262,9 @@ pub(crate) async fn built_tools( Ok(Arc::new(ToolRouter::from_config( &turn_context.tools_config, ToolRouterParams { - mcp_tools: direct_mcp_tools, - deferred_mcp_tools: mcp_tool_exposure.deferred_tools, + mcp_tools, + deferred_mcp_tools, + unavailable_called_tools, parallel_mcp_server_names, discoverable_tools, dynamic_tools: turn_context.dynamic_tools.as_slice(), diff --git a/codex-rs/core/src/codex_tests.rs b/codex-rs/core/src/codex_tests.rs index eaa2099098..19c0917f67 100644 --- a/codex-rs/core/src/codex_tests.rs +++ b/codex-rs/core/src/codex_tests.rs @@ -313,6 +313,7 @@ fn test_tool_runtime(session: Arc, turn_context: Arc) -> T crate::tools::router::ToolRouterParams { mcp_tools: None, deferred_mcp_tools: None, + unavailable_called_tools: Vec::new(), parallel_mcp_server_names: HashSet::new(), discoverable_tools: None, dynamic_tools: turn_context.dynamic_tools.as_slice(), @@ -5451,6 +5452,7 @@ async fn fatal_tool_error_stops_turn_and_reports_error() { crate::tools::router::ToolRouterParams { deferred_mcp_tools, mcp_tools: Some(tools), + unavailable_called_tools: Vec::new(), parallel_mcp_server_names: HashSet::new(), discoverable_tools: None, dynamic_tools: turn_context.dynamic_tools.as_slice(), diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 8d463f3b51..abfe91a0e1 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -150,6 +150,7 @@ mod tools; pub(crate) mod turn_diff_tracker; mod turn_metadata; mod turn_timing; +mod unavailable_tool; pub use rollout::ARCHIVED_SESSIONS_SUBDIR; pub use rollout::Cursor; pub use rollout::EventPersistenceMode; diff --git a/codex-rs/core/src/tools/code_mode/mod.rs b/codex-rs/core/src/tools/code_mode/mod.rs index 8445fdf645..7d66d56c39 100644 --- a/codex-rs/core/src/tools/code_mode/mod.rs +++ b/codex-rs/core/src/tools/code_mode/mod.rs @@ -287,6 +287,7 @@ async fn build_nested_router(exec: &ExecContext) -> ToolRouter { ToolRouterParams { deferred_mcp_tools: None, mcp_tools: Some(listed_mcp_tools), + unavailable_called_tools: Vec::new(), parallel_mcp_server_names, discoverable_tools: None, dynamic_tools: exec.turn.dynamic_tools.as_slice(), diff --git a/codex-rs/core/src/tools/handlers/mod.rs b/codex-rs/core/src/tools/handlers/mod.rs index 022e02c39e..6b48680b8a 100644 --- a/codex-rs/core/src/tools/handlers/mod.rs +++ b/codex-rs/core/src/tools/handlers/mod.rs @@ -15,6 +15,7 @@ mod shell; mod test_sync; mod tool_search; mod tool_suggest; +mod unavailable_tool; pub(crate) mod unified_exec; mod view_image; @@ -49,6 +50,8 @@ pub use shell::ShellHandler; pub use test_sync::TestSyncHandler; pub use tool_search::ToolSearchHandler; pub use tool_suggest::ToolSuggestHandler; +pub use unavailable_tool::UnavailableToolHandler; +pub(crate) use unavailable_tool::unavailable_tool_message; pub use unified_exec::UnifiedExecHandler; pub use view_image::ViewImageHandler; diff --git a/codex-rs/core/src/tools/handlers/unavailable_tool.rs b/codex-rs/core/src/tools/handlers/unavailable_tool.rs new file mode 100644 index 0000000000..eb00cf8ff1 --- /dev/null +++ b/codex-rs/core/src/tools/handlers/unavailable_tool.rs @@ -0,0 +1,44 @@ +use crate::function_tool::FunctionCallError; +use crate::tools::context::FunctionToolOutput; +use crate::tools::context::ToolInvocation; +use crate::tools::context::ToolPayload; +use crate::tools::registry::ToolHandler; +use crate::tools::registry::ToolKind; + +pub struct UnavailableToolHandler; + +pub(crate) fn unavailable_tool_message( + tool_name: impl std::fmt::Display, + next_step: &str, +) -> String { + format!( + "Tool `{tool_name}` is not currently available. It appeared in earlier tool calls in this conversation, but its implementation is not available in the current request. {next_step}" + ) +} + +impl ToolHandler for UnavailableToolHandler { + type Output = FunctionToolOutput; + + fn kind(&self) -> ToolKind { + ToolKind::Function + } + + async fn handle(&self, invocation: ToolInvocation) -> Result { + let ToolInvocation { + tool_name, payload, .. + } = invocation; + + match payload { + ToolPayload::Function { .. } => Ok(FunctionToolOutput::from_text( + unavailable_tool_message( + tool_name.display(), + "Retry after the tool becomes available or ask the user to re-enable it.", + ), + Some(false), + )), + _ => Err(FunctionCallError::RespondToModel( + "unavailable tool handler received unsupported payload".to_string(), + )), + } + } +} diff --git a/codex-rs/core/src/tools/js_repl/mod.rs b/codex-rs/core/src/tools/js_repl/mod.rs index 92bd4c610f..5c0884d9ee 100644 --- a/codex-rs/core/src/tools/js_repl/mod.rs +++ b/codex-rs/core/src/tools/js_repl/mod.rs @@ -1568,6 +1568,7 @@ impl JsReplManager { crate::tools::router::ToolRouterParams { deferred_mcp_tools: None, mcp_tools: Some(mcp_tools), + unavailable_called_tools: Vec::new(), // JS REPL dispatches nested tool calls directly, not through // `ToolCallRuntime`'s parallel scheduling lock. parallel_mcp_server_names: std::collections::HashSet::new(), diff --git a/codex-rs/core/src/tools/router.rs b/codex-rs/core/src/tools/router.rs index 183b9fb416..ef7ae32d4b 100644 --- a/codex-rs/core/src/tools/router.rs +++ b/codex-rs/core/src/tools/router.rs @@ -44,6 +44,7 @@ pub struct ToolRouter { pub(crate) struct ToolRouterParams<'a> { pub(crate) mcp_tools: Option>, pub(crate) deferred_mcp_tools: Option>, + pub(crate) unavailable_called_tools: Vec, pub(crate) parallel_mcp_server_names: HashSet, pub(crate) discoverable_tools: Option>, pub(crate) dynamic_tools: &'a [DynamicToolSpec], @@ -54,6 +55,7 @@ impl ToolRouter { let ToolRouterParams { mcp_tools, deferred_mcp_tools, + unavailable_called_tools, parallel_mcp_server_names, discoverable_tools, dynamic_tools, @@ -62,6 +64,7 @@ impl ToolRouter { config, mcp_tools, deferred_mcp_tools, + unavailable_called_tools, discoverable_tools, dynamic_tools, ); diff --git a/codex-rs/core/src/tools/router_tests.rs b/codex-rs/core/src/tools/router_tests.rs index f4c0fb7c5a..d93796ab64 100644 --- a/codex-rs/core/src/tools/router_tests.rs +++ b/codex-rs/core/src/tools/router_tests.rs @@ -33,6 +33,7 @@ async fn js_repl_tools_only_blocks_direct_tool_calls() -> anyhow::Result<()> { ToolRouterParams { deferred_mcp_tools, mcp_tools: Some(mcp_tools), + unavailable_called_tools: Vec::new(), parallel_mcp_server_names: HashSet::new(), discoverable_tools: None, dynamic_tools: turn.dynamic_tools.as_slice(), @@ -86,6 +87,7 @@ async fn js_repl_tools_only_allows_js_repl_source_calls() -> anyhow::Result<()> ToolRouterParams { deferred_mcp_tools, mcp_tools: Some(mcp_tools), + unavailable_called_tools: Vec::new(), parallel_mcp_server_names: HashSet::new(), discoverable_tools: None, dynamic_tools: turn.dynamic_tools.as_slice(), @@ -132,6 +134,7 @@ async fn js_repl_tools_only_blocks_namespaced_js_repl_tool() -> anyhow::Result<( ToolRouterParams { deferred_mcp_tools: None, mcp_tools: None, + unavailable_called_tools: Vec::new(), parallel_mcp_server_names: HashSet::new(), discoverable_tools: None, dynamic_tools: turn.dynamic_tools.as_slice(), @@ -182,6 +185,7 @@ async fn parallel_support_does_not_match_namespaced_local_tool_names() -> anyhow ToolRouterParams { deferred_mcp_tools: None, mcp_tools: Some(mcp_tools), + unavailable_called_tools: Vec::new(), parallel_mcp_server_names: HashSet::new(), discoverable_tools: None, dynamic_tools: turn.dynamic_tools.as_slice(), @@ -254,6 +258,7 @@ async fn mcp_parallel_support_uses_exact_payload_server() -> anyhow::Result<()> ToolRouterParams { deferred_mcp_tools: None, mcp_tools: None, + unavailable_called_tools: Vec::new(), parallel_mcp_server_names: HashSet::from(["echo".to_string()]), discoverable_tools: None, dynamic_tools: turn.dynamic_tools.as_slice(), diff --git a/codex-rs/core/src/tools/spec.rs b/codex-rs/core/src/tools/spec.rs index 9c05e6cc35..1a1a9f2e5f 100644 --- a/codex-rs/core/src/tools/spec.rs +++ b/codex-rs/core/src/tools/spec.rs @@ -7,8 +7,12 @@ use crate::tools::handlers::multi_agents_common::MIN_WAIT_TIMEOUT_MS; use crate::tools::registry::ToolRegistryBuilder; use codex_mcp::ToolInfo; use codex_protocol::dynamic_tools::DynamicToolSpec; +use codex_tools::AdditionalProperties; use codex_tools::DiscoverableTool; +use codex_tools::JsonSchema; +use codex_tools::ResponsesApiTool; use codex_tools::ToolHandlerKind; +use codex_tools::ToolName; use codex_tools::ToolNamespace; use codex_tools::ToolRegistryPlanDeferredTool; use codex_tools::ToolRegistryPlanMcpTool; @@ -16,8 +20,10 @@ use codex_tools::ToolRegistryPlanParams; use codex_tools::ToolUserShellType; use codex_tools::ToolsConfig; use codex_tools::WaitAgentTimeoutOptions; +use codex_tools::augment_tool_spec_for_code_mode; use codex_tools::build_tool_registry_plan; use std::collections::HashMap; +use std::collections::HashSet; use std::sync::Arc; pub(crate) fn tool_user_shell_type(user_shell: &Shell) -> ToolUserShellType { @@ -66,6 +72,7 @@ pub(crate) fn build_specs_with_discoverable_tools( config: &ToolsConfig, mcp_tools: Option>, deferred_mcp_tools: Option>, + unavailable_called_tools: Vec, discoverable_tools: Option>, dynamic_tools: &[DynamicToolSpec], ) -> ToolRegistryBuilder { @@ -86,6 +93,7 @@ pub(crate) fn build_specs_with_discoverable_tools( use crate::tools::handlers::TestSyncHandler; use crate::tools::handlers::ToolSearchHandler; use crate::tools::handlers::ToolSuggestHandler; + use crate::tools::handlers::UnavailableToolHandler; use crate::tools::handlers::UnifiedExecHandler; use crate::tools::handlers::ViewImageHandler; use crate::tools::handlers::multi_agents::CloseAgentHandler; @@ -99,6 +107,7 @@ pub(crate) fn build_specs_with_discoverable_tools( use crate::tools::handlers::multi_agents_v2::SendMessageHandler as SendMessageHandlerV2; use crate::tools::handlers::multi_agents_v2::SpawnAgentHandler as SpawnAgentHandlerV2; use crate::tools::handlers::multi_agents_v2::WaitAgentHandler as WaitAgentHandlerV2; + use crate::tools::handlers::unavailable_tool_message; let mut builder = ToolRegistryBuilder::new(); let mcp_tool_plan_inputs = mcp_tools.as_ref().map(map_mcp_tools_for_plan); @@ -154,6 +163,12 @@ pub(crate) fn build_specs_with_discoverable_tools( let code_mode_wait_handler = Arc::new(CodeModeWaitHandler); let js_repl_handler = Arc::new(JsReplHandler); let js_repl_reset_handler = Arc::new(JsReplResetHandler); + let unavailable_tool_handler = Arc::new(UnavailableToolHandler); + let mut existing_spec_names = plan + .specs + .iter() + .map(|configured_tool| configured_tool.name().to_string()) + .collect::>(); for spec in plan.specs { if spec.supports_parallel_tool_calls { @@ -278,6 +293,34 @@ pub(crate) fn build_specs_with_discoverable_tools( builder.register_handler(name.clone(), mcp_handler.clone()); } } + + for unavailable_tool in unavailable_called_tools { + let tool_name = unavailable_tool.display(); + if existing_spec_names.insert(tool_name.clone()) { + let spec = codex_tools::ToolSpec::Function(ResponsesApiTool { + name: tool_name.clone(), + description: unavailable_tool_message( + &tool_name, + "Calling this placeholder returns an error explaining that the tool is unavailable.", + ), + strict: false, + parameters: JsonSchema::object( + Default::default(), + /*required*/ None, + Some(AdditionalProperties::Boolean(false)), + ), + output_schema: None, + defer_loading: None, + }); + let spec = if config.code_mode_enabled { + augment_tool_spec_for_code_mode(spec) + } else { + spec + }; + builder.push_spec(spec); + } + builder.register_handler(unavailable_tool, unavailable_tool_handler.clone()); + } builder } diff --git a/codex-rs/core/src/tools/spec_tests.rs b/codex-rs/core/src/tools/spec_tests.rs index ee993b0d53..cab38bc21c 100644 --- a/codex-rs/core/src/tools/spec_tests.rs +++ b/codex-rs/core/src/tools/spec_tests.rs @@ -16,6 +16,7 @@ use codex_protocol::openai_models::ConfigShellToolType; use codex_protocol::openai_models::ModelInfo; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SessionSource; +use codex_tools::AdditionalProperties; use codex_tools::ConfiguredToolSpec; use codex_tools::DiscoverableTool; use codex_tools::JsonSchema; @@ -267,11 +268,28 @@ fn build_specs( mcp_tools: Option>, deferred_mcp_tools: Option>, dynamic_tools: &[DynamicToolSpec], +) -> ToolRegistryBuilder { + build_specs_with_unavailable_tools( + config, + mcp_tools, + deferred_mcp_tools, + Vec::new(), + dynamic_tools, + ) +} + +fn build_specs_with_unavailable_tools( + config: &ToolsConfig, + mcp_tools: Option>, + deferred_mcp_tools: Option>, + unavailable_called_tools: Vec, + dynamic_tools: &[DynamicToolSpec], ) -> ToolRegistryBuilder { build_specs_with_discoverable_tools( config, mcp_tools, deferred_mcp_tools, + unavailable_called_tools, /*discoverable_tools*/ None, dynamic_tools, ) @@ -356,6 +374,7 @@ fn assert_model_tools( ToolRouterParams { mcp_tools: None, deferred_mcp_tools: None, + unavailable_called_tools: Vec::new(), parallel_mcp_server_names: std::collections::HashSet::new(), discoverable_tools: None, dynamic_tools: &[], @@ -770,6 +789,7 @@ fn tool_suggest_requires_apps_and_plugins_features() { &tools_config, /*mcp_tools*/ None, /*deferred_mcp_tools*/ None, + Vec::new(), discoverable_tools.clone(), &[], ) @@ -991,6 +1011,55 @@ fn direct_mcp_tools_register_namespaced_handlers() { assert!(!registry.has_handler(&ToolName::plain("mcp__test_server__echo"))); } +#[test] +fn unavailable_mcp_tools_are_exposed_as_dummy_function_tools() { + let config = test_config(); + let model_info = construct_model_info_offline("gpt-5-codex", &config); + let mut features = Features::with_defaults(); + features.enable(Feature::UnifiedExec); + let available_models = Vec::new(); + let tools_config = ToolsConfig::new(&ToolsConfigParams { + model_info: &model_info, + available_models: &available_models, + features: &features, + image_generation_tool_auth_allowed: true, + web_search_mode: Some(WebSearchMode::Cached), + session_source: SessionSource::Cli, + sandbox_policy: &SandboxPolicy::DangerFullAccess, + windows_sandbox_level: WindowsSandboxLevel::Disabled, + }); + + let unavailable_tool = ToolName::namespaced("mcp__codex_apps__calendar", "_create_event"); + let (tools, registry) = build_specs_with_unavailable_tools( + &tools_config, + /*mcp_tools*/ None, + /*deferred_mcp_tools*/ None, + vec![unavailable_tool], + &[], + ) + .build(); + + let tool = find_tool(&tools, "mcp__codex_apps__calendar_create_event"); + let ToolSpec::Function(ResponsesApiTool { + description, + parameters, + .. + }) = &tool.spec + else { + panic!("unavailable MCP tool should be exposed as a function tool"); + }; + assert!(description.contains("not currently available")); + assert_eq!( + parameters.additional_properties, + Some(AdditionalProperties::Boolean(false)) + ); + assert!(registry.has_handler(&ToolName::namespaced( + "mcp__codex_apps__calendar", + "_create_event" + ))); + assert!(!registry.has_handler(&ToolName::plain("mcp__codex_apps__calendar_create_event"))); +} + #[test] fn test_mcp_tool_property_missing_type_defaults_to_string() { let config = test_config(); diff --git a/codex-rs/core/src/unavailable_tool.rs b/codex-rs/core/src/unavailable_tool.rs new file mode 100644 index 0000000000..39ba21f9b8 --- /dev/null +++ b/codex-rs/core/src/unavailable_tool.rs @@ -0,0 +1,92 @@ +use std::collections::BTreeMap; +use std::collections::HashSet; + +use codex_protocol::models::ResponseItem; +use codex_tools::ToolName; + +pub(crate) fn collect_unavailable_called_tools( + input: &[ResponseItem], + exposed_tool_names: &HashSet<&str>, +) -> Vec { + let mut unavailable_tools = BTreeMap::new(); + + for item in input { + let ResponseItem::FunctionCall { + name, namespace, .. + } = item + else { + continue; + }; + if !should_collect_unavailable_tool(name, namespace.as_deref()) { + continue; + } + + let tool_name = match namespace { + Some(namespace) => ToolName::namespaced(namespace.clone(), name.clone()), + None => ToolName::plain(name.clone()), + }; + let display_name = tool_name.display(); + if exposed_tool_names.contains(display_name.as_str()) { + continue; + } + + unavailable_tools + .entry(display_name) + .or_insert_with(|| tool_name); + } + + unavailable_tools.into_values().collect() +} + +fn should_collect_unavailable_tool(name: &str, namespace: Option<&str>) -> bool { + namespace.is_some_and(|namespace| namespace.starts_with("mcp__")) || name.starts_with("mcp__") +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + + fn function_call(name: &str, namespace: Option<&str>) -> ResponseItem { + ResponseItem::FunctionCall { + id: None, + name: name.to_string(), + namespace: namespace.map(str::to_string), + arguments: "{}".to_string(), + call_id: format!("call-{name}"), + } + } + + #[test] + fn collect_unavailable_called_tools_detects_mcp_function_calls() { + let input = vec![ + function_call("shell", /*namespace*/ None), + function_call("mcp__server__lookup", /*namespace*/ None), + function_call("_create_event", Some("mcp__codex_apps__calendar")), + ]; + + let tools = collect_unavailable_called_tools(&input, &HashSet::new()); + + assert_eq!( + tools, + vec![ + ToolName::namespaced("mcp__codex_apps__calendar", "_create_event"), + ToolName::plain("mcp__server__lookup"), + ] + ); + } + + #[test] + fn collect_unavailable_called_tools_skips_currently_available_tools() { + let exposed_tool_names = HashSet::from(["mcp__server__lookup", "mcp__server__search"]); + let input = vec![ + function_call("mcp__server__lookup", /*namespace*/ None), + function_call("mcp__server__search", /*namespace*/ None), + function_call("mcp__server__missing", /*namespace*/ None), + ]; + + let tools = collect_unavailable_called_tools(&input, &exposed_tool_names); + + assert_eq!(tools, vec![ToolName::plain("mcp__server__missing")]); + } +} diff --git a/codex-rs/core/tests/suite/tools.rs b/codex-rs/core/tests/suite/tools.rs index 8dc91cc087..3ae2888ec7 100644 --- a/codex-rs/core/tests/suite/tools.rs +++ b/codex-rs/core/tests/suite/tools.rs @@ -1,12 +1,16 @@ #![cfg(not(target_os = "windows"))] #![allow(clippy::unwrap_used, clippy::expect_used)] +use std::collections::HashMap; use std::fs; +use std::sync::Arc; use std::time::Duration; use std::time::Instant; use anyhow::Context; use anyhow::Result; +use codex_config::types::McpServerConfig; +use codex_config::types::McpServerTransportConfig; use codex_core::sandboxing::SandboxPermissions; use codex_features::Feature; use codex_protocol::protocol::AskForApproval; @@ -22,10 +26,13 @@ use core_test_support::responses::mount_sse_sequence; use core_test_support::responses::sse; use core_test_support::responses::start_mock_server; use core_test_support::skip_if_no_network; +use core_test_support::stdio_server_bin; use core_test_support::test_codex::test_codex; use regex_lite::Regex; use serde_json::Value; use serde_json::json; +use serial_test::serial; +use tempfile::TempDir; fn tool_names(body: &Value) -> Vec { body.get("tools") @@ -44,6 +51,24 @@ fn tool_names(body: &Value) -> Vec { .unwrap_or_default() } +fn ev_namespaced_function_call( + call_id: &str, + namespace: &str, + name: &str, + arguments: &str, +) -> Value { + json!({ + "type": "response.output_item.done", + "item": { + "type": "function_call", + "call_id": call_id, + "namespace": namespace, + "name": name, + "arguments": arguments, + } + }) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn custom_tool_unknown_returns_custom_output_error() -> Result<()> { skip_if_no_network!(Ok(())); @@ -91,6 +116,147 @@ async fn custom_tool_unknown_returns_custom_output_error() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial(mcp_test_value)] +async fn historical_unavailable_mcp_call_is_exposed_as_placeholder_tool() -> Result<()> { + skip_if_no_network!(Ok(())); + + let historical_call_id = "historical-mcp-call"; + let retry_call_id = "retry-mcp-call"; + let server_name = "rmcp"; + let unavailable_tool_namespace = "mcp__rmcp__"; + let unavailable_tool_name = "echo"; + let unavailable_tool_display_name = "mcp__rmcp__echo"; + let server = start_mock_server().await; + let rmcp_test_server_bin = match stdio_server_bin() { + Ok(bin) => bin, + Err(err) => { + eprintln!("test_stdio_server binary not available, skipping test: {err}"); + return Ok(()); + } + }; + let codex_home = Arc::new(TempDir::new()?); + let mut builder = test_codex() + .with_model("gpt-5.1") + .with_home(Arc::clone(&codex_home)) + .with_config(move |config| { + config + .features + .enable(Feature::UnavailableDummyTools) + .expect("unavailable dummy tools should be enabled for this test"); + let mut servers = config.mcp_servers.get().clone(); + servers.insert( + server_name.to_string(), + McpServerConfig { + transport: McpServerTransportConfig::Stdio { + command: rmcp_test_server_bin, + args: Vec::new(), + env: Some(HashMap::new()), + env_vars: Vec::new(), + cwd: None, + }, + enabled: true, + required: false, + supports_parallel_tool_calls: false, + disabled_reason: None, + startup_timeout_sec: Some(Duration::from_secs(10)), + tool_timeout_sec: None, + enabled_tools: None, + disabled_tools: None, + scopes: None, + oauth_resource: None, + tools: HashMap::new(), + }, + ); + config + .mcp_servers + .set(servers) + .expect("test mcp servers should accept any configuration"); + }); + let test = builder.build(&server).await?; + + let first_turn_mock = mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_namespaced_function_call( + historical_call_id, + unavailable_tool_namespace, + unavailable_tool_name, + r#"{"message":"ping"}"#, + ), + ev_completed("resp-1"), + ]), + sse(vec![ + ev_response_created("resp-2"), + ev_assistant_message("msg-1", "rmcp echo tool completed"), + ev_completed("resp-2"), + ]), + ], + ) + .await; + + test.submit_turn("call the rmcp echo tool").await?; + let rollout_path = test.codex.rollout_path().context("rollout path")?; + assert_eq!(first_turn_mock.requests().len(), 2); + drop(test); + + let retry_mock = mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_response_created("resp-3"), + ev_namespaced_function_call( + retry_call_id, + unavailable_tool_namespace, + unavailable_tool_name, + r#"{"message":"ping again"}"#, + ), + ev_completed("resp-3"), + ]), + sse(vec![ + ev_response_created("resp-4"), + ev_assistant_message("msg-2", "done"), + ev_completed("resp-4"), + ]), + ], + ) + .await; + + let mut resume_builder = test_codex().with_model("gpt-5.1").with_config(|config| { + config + .features + .enable(Feature::UnavailableDummyTools) + .expect("unavailable dummy tools should be enabled for this test"); + }); + let test = resume_builder + .resume(&server, codex_home, rollout_path) + .await?; + + test.submit_turn("retry the rmcp echo tool").await?; + + let requests = retry_mock.requests(); + assert_eq!(requests.len(), 2); + let first_request_tools = tool_names(&requests[0].body_json()); + assert!( + first_request_tools + .iter() + .any(|name| name == unavailable_tool_display_name), + "historical unavailable MCP call should add a placeholder tool; got {first_request_tools:?}" + ); + let output_text = requests[1] + .function_call_output_text(retry_call_id) + .context("placeholder tool output present")?; + assert!(output_text.contains("not currently available")); + assert!( + !output_text.contains("unsupported call"), + "placeholder handler should answer instead of falling back to unsupported call: {output_text}" + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn shell_escalated_permissions_rejected_then_ok() -> Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rs index 4d7a83b628..ce7a196362 100644 --- a/codex-rs/features/src/lib.rs +++ b/codex-rs/features/src/lib.rs @@ -146,6 +146,8 @@ pub enum Feature { Apps, /// Enable the tool_search tool for apps. ToolSearch, + /// Expose placeholder tools for unavailable historical tool calls. + UnavailableDummyTools, /// Enable discoverable tool suggestions for apps. ToolSuggest, /// Enable plugins. @@ -789,6 +791,12 @@ pub const FEATURES: &[FeatureSpec] = &[ stage: Stage::UnderDevelopment, default_enabled: false, }, + FeatureSpec { + id: Feature::UnavailableDummyTools, + key: "unavailable_dummy_tools", + stage: Stage::UnderDevelopment, + default_enabled: false, + }, FeatureSpec { id: Feature::ToolSuggest, key: "tool_suggest", diff --git a/codex-rs/features/src/tests.rs b/codex-rs/features/src/tests.rs index f55578dbd6..85cb01be23 100644 --- a/codex-rs/features/src/tests.rs +++ b/codex-rs/features/src/tests.rs @@ -133,6 +133,15 @@ fn tool_search_is_under_development_and_disabled_by_default() { assert_eq!(Feature::ToolSearch.default_enabled(), false); } +#[test] +fn unavailable_dummy_tools_is_under_development_and_disabled_by_default() { + assert_eq!( + Feature::UnavailableDummyTools.stage(), + Stage::UnderDevelopment + ); + assert_eq!(Feature::UnavailableDummyTools.default_enabled(), false); +} + #[test] fn general_analytics_is_stable_and_enabled_by_default() { assert_eq!(Feature::GeneralAnalytics.stage(), Stage::Stable); From 18d61f6923896aa273ae9a369d423ac75dd8963a Mon Sep 17 00:00:00 2001 From: evawong-oai Date: Wed, 15 Apr 2026 15:09:39 -0700 Subject: [PATCH 46/84] [docs] Restore SECURITY.md update from PR 17848 (#18004) Restore the `SECURITY.md` section from https://github.com/openai/codex/pull/17848. Note this was lost in the revert PR https://github.com/openai/codex/pull/18003. --- SECURITY.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/SECURITY.md b/SECURITY.md index d6dd568910..29a9b903df 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -11,3 +11,7 @@ Our security program is managed through Bugcrowd, and we ask that any validated ## Vulnerability Disclosure Program Our Vulnerability Program Guidelines are defined on our [Bugcrowd program page](https://bugcrowd.com/engagements/openai). + +## How to operate CODEX safely + +For details on Codex security boundaries, including sandboxing, approvals, and network controls, see [Agent approvals & security](https://developers.openai.com/codex/agent-approvals-security). From c2bdb7812cca9157b0e26a7deff480c1a2b2c4ad Mon Sep 17 00:00:00 2001 From: bxie-openai Date: Wed, 15 Apr 2026 16:26:20 -0700 Subject: [PATCH 47/84] Clarify realtime v2 context and handoff messages (#17896) ## Summary - wrap realtime startup context in `...` tags - prefix V2 mirrored user text and relayed backend text with `[USER]` / `[BACKEND]` - remove the V2 progress suffix and replace the final V2 handoff output with a short completion acknowledgement while preserving the existing V1 wrapper ## Testing - cargo test -p codex-api realtime_v2_session_update_includes_background_agent_tool_and_handoff_output_item -- --exact - cargo test -p codex-app-server webrtc_v2_background_agent_ - cargo test -p codex-app-server webrtc_v2_text_input_is_ - cargo test -p codex-core conversation_user_text_turn_is_ --- .../tests/suite/v2/realtime_conversation.rs | 31 ++++++++--- .../endpoint/realtime_websocket/methods.rs | 2 +- .../realtime_websocket/methods_common.rs | 6 ++- codex-rs/core/src/codex.rs | 7 +++ codex-rs/core/src/realtime_context.rs | 25 ++++++++- codex-rs/core/src/realtime_context_tests.rs | 18 +++++++ codex-rs/core/src/realtime_conversation.rs | 54 +++++++++++++++---- .../core/tests/suite/realtime_conversation.rs | 13 ++++- ...n_is_capped_when_mirrored_to_realtime.snap | 2 +- ..._turn_is_sent_to_realtime_when_active.snap | 2 +- 10 files changed, 133 insertions(+), 27 deletions(-) diff --git a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs index 9870d410f1..1f7b8a3e4b 100644 --- a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs +++ b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs @@ -75,6 +75,8 @@ const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10); const STARTUP_CONTEXT_HEADER: &str = "Startup context from Codex."; const V2_STEERING_ACKNOWLEDGEMENT: &str = "This was sent to steer the previous background agent task."; +const V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT: &str = + "Background agent finished. Use the preceding [BACKEND] messages as the result."; #[derive(Debug, Clone, Copy)] enum StartupContextConfig<'a> { @@ -1359,7 +1361,8 @@ async fn webrtc_v2_forwards_audio_and_text_between_client_and_sideband() -> Resu request["type"] == "conversation.item.create" && request["item"]["type"] == "message" && request["item"]["role"] == "user" - && request["item"]["content"][0]["text"] == "hello" + && request["item"]["content"][0]["type"] == "input_text" + && request["item"]["content"][0]["text"] == "[USER] hello" }), "sideband requests should include user text item: {requests:?}" ); @@ -1558,7 +1561,7 @@ async fn webrtc_v2_background_agent_tool_call_delegates_and_returns_function_out assert_v2_progress_update(&progress, "delegated from v2"); let tool_output = harness.sideband_outbound_request(/*request_index*/ 2).await; - assert_v2_function_call_output(&tool_output, "call_v2", "delegated from v2"); + assert_v2_function_call_output(&tool_output, "call_v2", V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT); assert_eq!( function_call_output_sideband_requests(&harness.realtime_server).len(), 1 @@ -1693,7 +1696,11 @@ async fn webrtc_v2_background_agent_progress_is_sent_before_function_output() -> assert_v2_progress_update(&progress, "progress before final"); let tool_output = harness.sideband_outbound_request(/*request_index*/ 2).await; - assert_v2_function_call_output(&tool_output, "call_progress_order", "progress before final"); + assert_v2_function_call_output( + &tool_output, + "call_progress_order", + V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT, + ); harness.shutdown().await; Ok(()) @@ -1777,7 +1784,11 @@ async fn webrtc_v2_tool_call_delegated_turn_can_execute_shell_tool() -> Result<( assert_v2_progress_update(&progress, "shell tool finished"); let tool_output = harness.sideband_outbound_request(/*request_index*/ 2).await; - assert_v2_function_call_output(&tool_output, "call_shell", "shell tool finished"); + assert_v2_function_call_output( + &tool_output, + "call_shell", + V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT, + ); assert_eq!( function_call_output_sideband_requests(&harness.realtime_server).len(), 1 @@ -1857,7 +1868,11 @@ async fn webrtc_v2_tool_call_does_not_block_sideband_audio() -> Result<()> { assert_v2_progress_update(&progress, "late delegated result"); let tool_output = harness.sideband_outbound_request(/*request_index*/ 2).await; - assert_v2_function_call_output(&tool_output, "call_audio", "late delegated result"); + assert_v2_function_call_output( + &tool_output, + "call_audio", + V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT, + ); harness.shutdown().await; Ok(()) @@ -2090,7 +2105,7 @@ fn assert_v2_function_call_output(request: &Value, call_id: &str, expected_outpu "item": { "type": "function_call_output", "call_id": call_id, - "output": format!("\"Agent Final Message\":\n\n{expected_output}"), + "output": expected_output, } }) ); @@ -2106,7 +2121,7 @@ fn assert_v2_progress_update(request: &Value, expected_text: &str) { "role": "user", "content": [{ "type": "input_text", - "text": format!("{expected_text}\n\nUpdate from background agent (task hasn't finished yet):") + "text": format!("[BACKEND] {expected_text}") }] } }) @@ -2123,7 +2138,7 @@ fn assert_v2_user_text_item(request: &Value, expected_text: &str) { "role": "user", "content": [{ "type": "input_text", - "text": expected_text + "text": format!("[USER] {expected_text}") }] } }) diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs index a6964e6614..6a5a032156 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs @@ -1670,7 +1670,7 @@ mod tests { ); assert_eq!( third_json["item"]["output"], - Value::String("\"Agent Final Message\":\n\ndelegated result".to_string()) + Value::String("delegated result".to_string()) ); }); diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_common.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_common.rs index 67345bdc7e..c0efcce165 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_common.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_common.rs @@ -45,9 +45,11 @@ pub(super) fn conversation_handoff_append_message( handoff_id: String, output_text: String, ) -> RealtimeOutboundMessage { - let output_text = format!("{AGENT_FINAL_MESSAGE_PREFIX}{output_text}"); match event_parser { - RealtimeEventParser::V1 => v1_conversation_handoff_append_message(handoff_id, output_text), + RealtimeEventParser::V1 => v1_conversation_handoff_append_message( + handoff_id, + format!("{AGENT_FINAL_MESSAGE_PREFIX}{output_text}"), + ), RealtimeEventParser::RealtimeV2 => { v2_conversation_handoff_append_message(handoff_id, output_text) } diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index a3f65b2355..c39b5bc719 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -4944,6 +4944,8 @@ mod handlers { use crate::config_loader::load_config_layers_state; use crate::realtime_context::REALTIME_TURN_TOKEN_BUDGET; use crate::realtime_context::truncate_realtime_text_to_token_budget; + use crate::realtime_conversation::REALTIME_USER_TEXT_PREFIX; + use crate::realtime_conversation::prefix_realtime_v2_text; use codex_features::Feature; use codex_utils_absolute_path::AbsolutePathBuf; @@ -5164,6 +5166,11 @@ mod handlers { if text.is_empty() { return; } + let text = if sess.conversation.is_running_v2().await { + prefix_realtime_v2_text(text, REALTIME_USER_TEXT_PREFIX) + } else { + text + }; let text = truncate_realtime_text_to_token_budget(&text, REALTIME_TURN_TOKEN_BUDGET); if text.is_empty() { return; diff --git a/codex-rs/core/src/realtime_context.rs b/codex-rs/core/src/realtime_context.rs index 30cbd6f636..cac2067a82 100644 --- a/codex-rs/core/src/realtime_context.rs +++ b/codex-rs/core/src/realtime_context.rs @@ -25,6 +25,8 @@ use tracing::info; use tracing::warn; const STARTUP_CONTEXT_HEADER: &str = "Startup context from Codex.\nThis is background context about recent work and machine/workspace layout. It may be incomplete or stale. Use it to inform responses, and do not repeat it back unless relevant."; +const STARTUP_CONTEXT_OPEN_TAG: &str = ""; +const STARTUP_CONTEXT_CLOSE_TAG: &str = ""; const CURRENT_THREAD_SECTION_TOKEN_BUDGET: usize = 1_200; const RECENT_WORK_SECTION_TOKEN_BUDGET: usize = 2_200; const WORKSPACE_SECTION_TOKEN_BUDGET: usize = 1_600; @@ -106,7 +108,7 @@ pub(crate) async fn build_realtime_startup_context( parts.push(section); } - let context = truncate_text(&parts.join("\n\n"), TruncationPolicy::Tokens(budget_tokens)); + let context = format_startup_context_blob(&parts.join("\n\n"), budget_tokens); debug!( approx_tokens = approx_token_count(&context), bytes = context.len(), @@ -443,6 +445,27 @@ fn format_section(title: &str, body: Option, budget_tokens: usize) -> Op )) } +fn format_startup_context_blob(body: &str, budget_tokens: usize) -> String { + let wrapper = format!("{STARTUP_CONTEXT_OPEN_TAG}\n\n{STARTUP_CONTEXT_CLOSE_TAG}"); + let mut body_budget = budget_tokens.saturating_sub(approx_token_count(&wrapper)); + + loop { + let body = truncate_text(body, TruncationPolicy::Tokens(body_budget)); + let wrapped = format!("{STARTUP_CONTEXT_OPEN_TAG}\n{body}\n{STARTUP_CONTEXT_CLOSE_TAG}"); + let wrapped_tokens = approx_token_count(&wrapped); + if wrapped_tokens <= budget_tokens || body_budget == 0 { + return wrapped; + } + + let excess_tokens = wrapped_tokens.saturating_sub(budget_tokens); + let next_budget = body_budget.saturating_sub(excess_tokens.max(1)); + if next_budget == body_budget { + return wrapped; + } + body_budget = next_budget; + } +} + fn format_thread_group( current_group: &Path, group: &Path, diff --git a/codex-rs/core/src/realtime_context_tests.rs b/codex-rs/core/src/realtime_context_tests.rs index 7e1e82476f..7fe438dac7 100644 --- a/codex-rs/core/src/realtime_context_tests.rs +++ b/codex-rs/core/src/realtime_context_tests.rs @@ -1,6 +1,7 @@ use super::build_current_thread_section; use super::build_recent_work_section; use super::build_workspace_section_with_user_root; +use super::format_startup_context_blob; use chrono::TimeZone; use chrono::Utc; use codex_git_utils::GitSha; @@ -170,6 +171,23 @@ fn current_thread_section_keeps_latest_turns_when_history_exceeds_budget() { ); } +#[test] +fn startup_context_blob_is_wrapped_in_tags_and_fits_budget() { + let body = format!( + "Startup context from Codex.\n{}\n{}", + "recent work ".repeat(1_200), + "workspace tree ".repeat(800), + ); + + let wrapped = format_startup_context_blob(&body, /*budget_tokens*/ 200); + + assert!(wrapped.starts_with("\n")); + assert!(wrapped.ends_with("\n")); + assert!(wrapped.contains("Startup context from Codex.")); + assert!(wrapped.contains("tokens truncated")); + assert!(wrapped.len().div_ceil(4) <= 200); +} + #[test] fn workspace_section_requires_meaningful_structure() { let cwd = TempDir::new().expect("tempdir"); diff --git a/codex-rs/core/src/realtime_conversation.rs b/codex-rs/core/src/realtime_conversation.rs index f81d6c0948..b79060259c 100644 --- a/codex-rs/core/src/realtime_conversation.rs +++ b/codex-rs/core/src/realtime_conversation.rs @@ -65,8 +65,10 @@ const HANDOFF_OUT_QUEUE_CAPACITY: usize = 64; const OUTPUT_EVENTS_QUEUE_CAPACITY: usize = 256; const REALTIME_STARTUP_CONTEXT_TOKEN_BUDGET: usize = 5_000; const DEFAULT_REALTIME_MODEL: &str = "gpt-realtime-1.5"; -const REALTIME_V2_PROGRESS_UPDATE_SUFFIX: &str = - "\n\nUpdate from background agent (task hasn't finished yet):"; +pub(crate) const REALTIME_USER_TEXT_PREFIX: &str = "[USER] "; +pub(crate) const REALTIME_BACKEND_TEXT_PREFIX: &str = "[BACKEND] "; +const REALTIME_V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT: &str = + "Background agent finished. Use the preceding [BACKEND] messages as the result."; const REALTIME_V2_STEER_ACKNOWLEDGEMENT: &str = "This was sent to steer the previous background agent task."; const REALTIME_ACTIVE_RESPONSE_ERROR_PREFIX: &str = @@ -209,6 +211,7 @@ impl RealtimeHandoffState { struct ConversationState { audio_tx: Sender, user_text_tx: Sender, + session_kind: RealtimeSessionKind, writer: RealtimeWebsocketWriter, handoff: RealtimeHandoffState, input_task: JoinHandle<()>, @@ -245,6 +248,16 @@ impl RealtimeConversationManager { .and_then(|state| state.realtime_active.load(Ordering::Relaxed).then_some(())) } + pub(crate) async fn is_running_v2(&self) -> bool { + let state = self.state.lock().await; + matches!( + state.as_ref(), + Some(state) + if state.realtime_active.load(Ordering::Relaxed) + && state.session_kind == RealtimeSessionKind::V2 + ) + } + async fn start(&self, start: RealtimeStart) -> CodexResult { let previous_state = { let mut guard = self.state.lock().await; @@ -331,6 +344,7 @@ impl RealtimeConversationManager { *guard = Some(ConversationState { audio_tx, user_text_tx, + session_kind, writer, handoff, input_task: task, @@ -406,15 +420,18 @@ impl RealtimeConversationManager { pub(crate) async fn text_in(&self, text: String) -> CodexResult<()> { let sender = { let guard = self.state.lock().await; - guard.as_ref().map(|state| state.user_text_tx.clone()) + guard + .as_ref() + .map(|state| (state.user_text_tx.clone(), state.session_kind)) }; - let Some(sender) = sender else { + let Some((sender, session_kind)) = sender else { return Err(CodexErr::InvalidRequest( "conversation is not running".to_string(), )); }; + let text = prefix_realtime_text(text, REALTIME_USER_TEXT_PREFIX, session_kind); sender .send(text) .await @@ -437,6 +454,11 @@ impl RealtimeConversationManager { return Ok(()); }; + let output_text = prefix_realtime_text( + output_text, + REALTIME_BACKEND_TEXT_PREFIX, + handoff.session_kind, + ); *handoff.last_output_text.lock().await = Some(output_text.clone()); handoff .output_tx @@ -695,6 +717,17 @@ fn default_realtime_voice(version: RealtimeWsVersion) -> RealtimeVoice { } } +fn prefix_realtime_text(text: String, prefix: &str, session_kind: RealtimeSessionKind) -> String { + if session_kind != RealtimeSessionKind::V2 || text.is_empty() || text.starts_with(prefix) { + return text; + } + format!("{prefix}{text}") +} + +pub(crate) fn prefix_realtime_v2_text(text: String, prefix: &str) -> String { + prefix_realtime_text(text, prefix, RealtimeSessionKind::V2) +} + fn validate_realtime_voice(version: RealtimeWsVersion, voice: RealtimeVoice) -> CodexResult<()> { let voices = RealtimeVoicesList::builtin(); let allowed = match version { @@ -1074,18 +1107,17 @@ async fn handle_handoff_output( return Ok(()); } } - writer - .send_conversation_item_create(format!( - "{output_text}{REALTIME_V2_PROGRESS_UPDATE_SUFFIX}" - )) - .await + writer.send_conversation_item_create(output_text).await } HandoffOutput::FinalUpdate { handoff_id, - output_text, + output_text: _, } => { if let Err(err) = writer - .send_conversation_handoff_append(handoff_id, output_text) + .send_conversation_handoff_append( + handoff_id, + REALTIME_V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT.to_string(), + ) .await { Err(err) diff --git a/codex-rs/core/tests/suite/realtime_conversation.rs b/codex-rs/core/tests/suite/realtime_conversation.rs index ecaaeb8cba..1cd97391bb 100644 --- a/codex-rs/core/tests/suite/realtime_conversation.rs +++ b/codex-rs/core/tests/suite/realtime_conversation.rs @@ -57,6 +57,8 @@ use wiremock::matchers::method; use wiremock::matchers::path_regex; const STARTUP_CONTEXT_HEADER: &str = "Startup context from Codex."; +const STARTUP_CONTEXT_OPEN_TAG: &str = ""; +const STARTUP_CONTEXT_CLOSE_TAG: &str = ""; const REALTIME_BACKEND_PROMPT: &str = include_str!("../../templates/realtime/backend_prompt.md"); const USER_FIRST_NAME_PLACEHOLDER: &str = "{{ user_first_name }}"; const MEMORY_PROMPT_PHRASE: &str = @@ -1530,6 +1532,8 @@ async fn conversation_start_injects_startup_context_from_thread_history() -> Res let startup_context = websocket_request_instructions(&startup_context_request) .expect("startup context request should contain instructions"); + assert!(startup_context.contains(STARTUP_CONTEXT_OPEN_TAG)); + assert!(startup_context.contains(STARTUP_CONTEXT_CLOSE_TAG)); assert!(startup_context.contains(STARTUP_CONTEXT_HEADER)); assert!(!startup_context.contains("## User")); assert!(startup_context.contains("### ")); @@ -1747,6 +1751,8 @@ async fn conversation_startup_context_falls_back_to_workspace_map() -> Result<() let startup_context = websocket_request_instructions(&startup_context_request) .expect("startup context request should contain instructions"); + assert!(startup_context.contains(STARTUP_CONTEXT_OPEN_TAG)); + assert!(startup_context.contains(STARTUP_CONTEXT_CLOSE_TAG)); assert!(startup_context.contains(STARTUP_CONTEXT_HEADER)); assert!(startup_context.contains("## Machine / Workspace Map")); assert!(startup_context.contains("notes.txt")); @@ -1801,6 +1807,8 @@ async fn conversation_startup_context_is_truncated_and_sent_once_per_start() -> .await; let startup_context = websocket_request_instructions(&startup_context_request) .expect("startup context request should contain instructions"); + assert!(startup_context.contains(STARTUP_CONTEXT_OPEN_TAG)); + assert!(startup_context.contains(STARTUP_CONTEXT_CLOSE_TAG)); assert!(startup_context.contains(STARTUP_CONTEXT_HEADER)); assert!(startup_context.len() <= 20_500); @@ -1879,6 +1887,7 @@ async fn conversation_user_text_turn_is_sent_to_realtime_when_active() -> Result assert_eq!(session_updated, "sess_user_text"); let user_text = "typed follow-up for realtime"; + let prefixed_user_text = format!("[USER] {user_text}"); test.codex .submit(Op::UserInput { items: vec![UserInput::Text { @@ -1898,7 +1907,7 @@ async fn conversation_user_text_turn_is_sent_to_realtime_when_active() -> Result let realtime_text_request = wait_for_matching_websocket_request( &realtime_server, "normal user turn text mirrored to realtime", - |request| websocket_request_text(request).as_deref() == Some(user_text), + |request| websocket_request_text(request).as_deref() == Some(prefixed_user_text.as_str()), ) .await; let model_user_texts = response_mock.single_request().message_input_texts("user"); @@ -1907,7 +1916,7 @@ async fn conversation_user_text_turn_is_sent_to_realtime_when_active() -> Result model_user_texts.iter().any(|text| text == user_text), websocket_request_text(&realtime_text_request), ), - (true, Some(user_text.to_string())), + (true, Some(prefixed_user_text)), ); let realtime_response_create = timeout(Duration::from_millis(200), async { wait_for_matching_websocket_request( diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__realtime_conversation__conversation_user_text_turn_is_capped_when_mirrored_to_realtime.snap b/codex-rs/core/tests/suite/snapshots/all__suite__realtime_conversation__conversation_user_text_turn_is_capped_when_mirrored_to_realtime.snap index ed19ddae97..3daae5d378 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__realtime_conversation__conversation_user_text_turn_is_capped_when_mirrored_to_realtime.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__realtime_conversation__conversation_user_text_turn_is_capped_when_mirrored_to_realtime.snap @@ -9,4 +9,4 @@ content[0].type: input_text model_has_full_user_text: true realtime_text_equal_full_user_text: false realtime_text_approx_tokens: 300 -content[0].text: mirror-head alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alphโ€ฆ2417 tokens truncatedโ€ฆega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega mirror-tail +content[0].text: [USER] mirror-head alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpโ€ฆ2419 tokens truncatedโ€ฆega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega mirror-tail diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__realtime_conversation__conversation_user_text_turn_is_sent_to_realtime_when_active.snap b/codex-rs/core/tests/suite/snapshots/all__suite__realtime_conversation__conversation_user_text_turn_is_sent_to_realtime_when_active.snap index 9cd2a55303..3ca8233029 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__realtime_conversation__conversation_user_text_turn_is_sent_to_realtime_when_active.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__realtime_conversation__conversation_user_text_turn_is_sent_to_realtime_when_active.snap @@ -6,5 +6,5 @@ type: conversation.item.create item.type: message item.role: user content[0].type: input_text -content[0].text: typed follow-up for realtime +content[0].text: [USER] typed follow-up for realtime response.create: false From e2dbe7dfc37e1c368ac521429a82a7562ebaaca2 Mon Sep 17 00:00:00 2001 From: Won Park Date: Wed, 15 Apr 2026 17:02:42 -0700 Subject: [PATCH 48/84] removing network proxy for yolo (#17742) **Summary** - prevent managed requirements.toml network settings from leaking into DangerFullAccess / yolo turns by gating managed proxy attachment on sandbox mode - keep guardian/sandboxed modes on the managed proxy path, while making true yolo bypass the proxy entirely, including /shell full-access commands --- codex-rs/core/src/codex.rs | 28 +++- codex-rs/core/src/codex_tests.rs | 198 +++++++++++++++++++++++++ codex-rs/core/src/config/mod.rs | 6 +- codex-rs/core/src/tasks/user_shell.rs | 4 +- codex-rs/core/tests/suite/approvals.rs | 163 ++++++++++++++++++++ 5 files changed, 392 insertions(+), 7 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index c39b5bc719..66a0dc527e 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -1338,6 +1338,10 @@ impl Session { } } + fn managed_network_proxy_active_for_sandbox_policy(sandbox_policy: &SandboxPolicy) -> bool { + !matches!(sandbox_policy, SandboxPolicy::DangerFullAccess) + } + /// Builds the `x-codex-beta-features` header value for this session. /// /// `ModelClient` is session-scoped and intentionally does not depend on the full `Config`, so @@ -2004,10 +2008,15 @@ impl Session { .await; session_configuration.thread_name = thread_name.clone(); let state = SessionState::new(session_configuration.clone()); + let managed_network_requirements_configured = config + .config_layer_stack + .requirements_toml() + .network + .is_some(); let managed_network_requirements_enabled = config.managed_network_requirements_enabled(); let network_approval = Arc::new(NetworkApprovalService::default()); // The managed proxy can call back into core for allowlist-miss decisions. - let network_policy_decider_session = if managed_network_requirements_enabled { + let network_policy_decider_session = if managed_network_requirements_configured { config .permissions .network @@ -2016,7 +2025,7 @@ impl Session { } else { None }; - let blocked_request_observer = if managed_network_requirements_enabled { + let blocked_request_observer = if managed_network_requirements_configured { config .permissions .network @@ -2043,7 +2052,7 @@ impl Session { config.permissions.sandbox_policy.get(), network_policy_decider.as_ref().map(Arc::clone), blocked_request_observer.as_ref().map(Arc::clone), - managed_network_requirements_enabled, + managed_network_requirements_configured, network_proxy_audit_metadata, ) .instrument(info_span!( @@ -2200,7 +2209,11 @@ impl Session { history_log_id, history_entry_count, initial_messages, - network_proxy: session_network_proxy, + network_proxy: session_network_proxy.filter(|_| { + Self::managed_network_proxy_active_for_sandbox_policy( + session_configuration.sandbox_policy.get(), + ) + }), rollout_path, }), }) @@ -2734,7 +2747,12 @@ impl Session { self.services .network_proxy .as_ref() - .map(StartedNetworkProxy::proxy), + .and_then(|started_proxy| { + Self::managed_network_proxy_active_for_sandbox_policy( + session_configuration.sandbox_policy.get(), + ) + .then(|| started_proxy.proxy()) + }), self.services.environment.clone(), sub_id, Arc::clone(&self.js_repl), diff --git a/codex-rs/core/src/codex_tests.rs b/codex-rs/core/src/codex_tests.rs index 19c0917f67..054e98be67 100644 --- a/codex-rs/core/src/codex_tests.rs +++ b/codex-rs/core/src/codex_tests.rs @@ -45,6 +45,8 @@ use crate::rollout::recorder::RolloutRecorder; use crate::state::TaskKind; use crate::tasks::SessionTask; use crate::tasks::SessionTaskContext; +use crate::tasks::UserShellCommandMode; +use crate::tasks::execute_user_shell_command; use crate::tools::ToolRouter; use crate::tools::context::ToolInvocation; use crate::tools::context::ToolPayload; @@ -740,6 +742,99 @@ async fn new_turn_refreshes_managed_network_proxy_for_sandbox_change() -> anyhow Ok(()) } +#[tokio::test] +async fn danger_full_access_turns_do_not_expose_managed_network_proxy() -> anyhow::Result<()> { + let network_spec = crate::config::NetworkProxySpec::from_config_and_constraints( + NetworkProxyConfig::default(), + Some(NetworkConstraints { + enabled: Some(true), + ..Default::default() + }), + &SandboxPolicy::DangerFullAccess, + )?; + + let session = make_session_with_config(move |config| { + config.permissions.sandbox_policy = + codex_config::Constrained::allow_any(SandboxPolicy::DangerFullAccess); + config.permissions.network = Some(network_spec); + }) + .await?; + + let turn_context = session.new_default_turn().await; + assert!(turn_context.network.is_none()); + Ok(()) +} + +#[tokio::test] +async fn workspace_write_turns_continue_to_expose_managed_network_proxy() -> anyhow::Result<()> { + let sandbox_policy = SandboxPolicy::new_workspace_write_policy(); + let network_spec = crate::config::NetworkProxySpec::from_config_and_constraints( + NetworkProxyConfig::default(), + Some(NetworkConstraints { + enabled: Some(true), + ..Default::default() + }), + &sandbox_policy, + )?; + + let session = make_session_with_config(move |config| { + config.permissions.sandbox_policy = codex_config::Constrained::allow_any(sandbox_policy); + config.permissions.network = Some(network_spec); + }) + .await?; + + let turn_context = session.new_default_turn().await; + assert!(turn_context.network.is_some()); + Ok(()) +} + +#[tokio::test] +async fn user_shell_commands_do_not_inherit_managed_network_proxy() -> anyhow::Result<()> { + let sandbox_policy = SandboxPolicy::new_workspace_write_policy(); + let network_spec = crate::config::NetworkProxySpec::from_config_and_constraints( + NetworkProxyConfig::default(), + Some(NetworkConstraints { + enabled: Some(true), + ..Default::default() + }), + &sandbox_policy, + )?; + + let (session, rx) = make_session_with_config_and_rx(move |config| { + config.permissions.sandbox_policy = codex_config::Constrained::allow_any(sandbox_policy); + config.permissions.network = Some(network_spec); + }) + .await?; + + let turn_context = session.new_default_turn().await; + assert!(turn_context.network.is_some()); + + #[cfg(windows)] + let command = r#"$val = $env:HTTP_PROXY; if ([string]::IsNullOrEmpty($val)) { $val = 'not-set' } ; [System.Console]::Write($val)"#.to_string(); + #[cfg(not(windows))] + let command = r#"sh -c "printf '%s' \"${HTTP_PROXY:-not-set}\"""#.to_string(); + + execute_user_shell_command( + Arc::clone(&session), + turn_context, + command, + CancellationToken::new(), + UserShellCommandMode::StandaloneTurn, + ) + .await; + + loop { + let event = rx.recv().await.expect("channel open"); + if let EventMsg::ExecCommandEnd(event) = event.msg { + assert_eq!(event.exit_code, 0); + assert_eq!(event.stdout.trim(), "not-set"); + break; + } + } + + Ok(()) +} + #[tokio::test] async fn get_base_instructions_no_user_content() { let prompt_with_apply_patch_instructions = @@ -3001,6 +3096,109 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) { (session, turn_context) } +async fn make_session_with_config( + mutator: impl FnOnce(&mut Config), +) -> anyhow::Result> { + let (session, _rx_event) = make_session_with_config_and_rx(mutator).await?; + Ok(session) +} + +async fn make_session_with_config_and_rx( + mutator: impl FnOnce(&mut Config), +) -> anyhow::Result<(Arc, async_channel::Receiver)> { + let codex_home = tempfile::tempdir().expect("create temp dir"); + let mut config = build_test_config(codex_home.path()).await; + mutator(&mut config); + let config = Arc::new(config); + let auth_manager = AuthManager::from_auth_for_testing(CodexAuth::from_api_key("Test API Key")); + let models_manager = Arc::new(ModelsManager::new( + config.codex_home.to_path_buf(), + auth_manager.clone(), + /*model_catalog*/ None, + CollaborationModesConfig::default(), + )); + let model = ModelsManager::get_model_offline_for_tests(config.model.as_deref()); + let model_info = ModelsManager::construct_model_info_offline_for_tests( + model.as_str(), + &config.to_models_manager_config(), + ); + let collaboration_mode = CollaborationMode { + mode: ModeKind::Default, + settings: Settings { + model, + reasoning_effort: config.model_reasoning_effort, + developer_instructions: None, + }, + }; + let session_configuration = SessionConfiguration { + provider: config.model_provider.clone(), + collaboration_mode, + model_reasoning_summary: config.model_reasoning_summary, + developer_instructions: config.developer_instructions.clone(), + user_instructions: config.user_instructions.clone(), + service_tier: None, + personality: config.personality, + base_instructions: config + .base_instructions + .clone() + .unwrap_or_else(|| model_info.get_model_instructions(config.personality)), + compact_prompt: config.compact_prompt.clone(), + approval_policy: config.permissions.approval_policy.clone(), + approvals_reviewer: config.approvals_reviewer, + sandbox_policy: config.permissions.sandbox_policy.clone(), + file_system_sandbox_policy: config.permissions.file_system_sandbox_policy.clone(), + network_sandbox_policy: config.permissions.network_sandbox_policy, + windows_sandbox_level: WindowsSandboxLevel::from_config(&config), + cwd: config.cwd.clone(), + codex_home: config.codex_home.clone(), + thread_name: None, + original_config_do_not_use: Arc::clone(&config), + metrics_service_name: None, + app_server_client_name: None, + app_server_client_version: None, + session_source: SessionSource::Exec, + dynamic_tools: Vec::new(), + persist_extended_history: false, + inherited_shell_snapshot: None, + user_shell_override: None, + }; + + let (tx_event, rx_event) = async_channel::unbounded(); + let (agent_status_tx, _agent_status_rx) = watch::channel(AgentStatus::PendingInit); + let plugins_manager = Arc::new(PluginsManager::new(config.codex_home.to_path_buf())); + let mcp_manager = Arc::new(McpManager::new(Arc::clone(&plugins_manager))); + let skills_manager = Arc::new(SkillsManager::new( + config.codex_home.clone(), + /*bundled_skills_enabled*/ true, + )); + + let session = Session::new( + session_configuration, + Arc::clone(&config), + auth_manager, + models_manager, + Arc::new(ExecPolicyManager::default()), + tx_event, + agent_status_tx, + InitialHistory::New, + SessionSource::Exec, + skills_manager, + plugins_manager, + mcp_manager, + Arc::new(SkillsWatcher::noop()), + AgentControl::default(), + Some(Arc::new( + codex_exec_server::Environment::create(/*exec_server_url*/ None) + .await + .expect("create environment"), + )), + /*analytics_events_client*/ None, + ) + .await?; + + Ok((session, rx_event)) +} + #[tokio::test] async fn notify_request_permissions_response_ignores_unmatched_call_id() { let (session, _turn_context) = make_session_and_context().await; diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index b58acbf651..5faf4be480 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -2282,7 +2282,11 @@ impl Config { } pub fn managed_network_requirements_enabled(&self) -> bool { - self.config_layer_stack + !matches!( + self.permissions.sandbox_policy.get(), + SandboxPolicy::DangerFullAccess + ) && self + .config_layer_stack .requirements_toml() .network .is_some() diff --git a/codex-rs/core/src/tasks/user_shell.rs b/codex-rs/core/src/tasks/user_shell.rs index 76e531744b..8b108eee06 100644 --- a/codex-rs/core/src/tasks/user_shell.rs +++ b/codex-rs/core/src/tasks/user_shell.rs @@ -163,7 +163,9 @@ pub(crate) async fn execute_user_shell_command( cwd: cwd.clone(), env: exec_env_map, exec_server_env_config: None, - network: turn_context.network.clone(), + // `/shell` is the explicit full-access escape hatch, so it must not + // inherit a managed proxy from the surrounding session or turn. + network: None, // TODO(zhao-oai): Now that we have ExecExpiration::Cancellation, we // should use that instead of an "arbitrarily large" timeout here. expiration: USER_SHELL_TIMEOUT_MS.into(), diff --git a/codex-rs/core/tests/suite/approvals.rs b/codex-rs/core/tests/suite/approvals.rs index 84066ff6e6..c5f7760195 100644 --- a/codex-rs/core/tests/suite/approvals.rs +++ b/codex-rs/core/tests/suite/approvals.rs @@ -2803,6 +2803,169 @@ allow_local_binding = true Ok(()) } +#[tokio::test(flavor = "current_thread")] +async fn network_approval_flow_survives_danger_full_access_session_start() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let home = Arc::new(TempDir::new()?); + fs::write( + home.path().join("config.toml"), + r#"default_permissions = "workspace" + +[permissions.workspace.filesystem] +":minimal" = "read" + +[permissions.workspace.network] +enabled = true +mode = "limited" +allow_local_binding = true +"#, + )?; + let approval_policy = AskForApproval::OnFailure; + let turn_sandbox_policy = SandboxPolicy::WorkspaceWrite { + writable_roots: vec![], + read_only_access: Default::default(), + network_access: true, + exclude_tmpdir_env_var: false, + exclude_slash_tmp: false, + }; + let mut builder = test_codex().with_home(home).with_config(move |config| { + config.permissions.approval_policy = Constrained::allow_any(approval_policy); + config.permissions.sandbox_policy = Constrained::allow_any(SandboxPolicy::DangerFullAccess); + let layers = config + .config_layer_stack + .get_layers( + ConfigLayerStackOrdering::LowestPrecedenceFirst, + /*include_disabled*/ true, + ) + .into_iter() + .cloned() + .collect(); + let mut requirements = config.config_layer_stack.requirements().clone(); + requirements.network = Some(Sourced::new( + NetworkConstraints { + enabled: Some(true), + allow_local_binding: Some(true), + ..Default::default() + }, + RequirementSource::CloudRequirements, + )); + let mut requirements_toml = config.config_layer_stack.requirements_toml().clone(); + requirements_toml.network = Some(NetworkRequirementsToml { + enabled: Some(true), + allow_local_binding: Some(true), + ..Default::default() + }); + config.config_layer_stack = ConfigLayerStack::new(layers, requirements, requirements_toml) + .expect("rebuild config layer stack with network requirements"); + }); + let test = builder.build(&server).await?; + assert!( + !test.config.managed_network_requirements_enabled(), + "expected managed network requirements to stay inactive in danger-full-access" + ); + assert!( + test.config.permissions.network.is_some(), + "expected managed network proxy config to be present" + ); + assert!( + test.session_configured.network_proxy.is_none(), + "expected session configured event to hide managed network proxy in danger-full-access" + ); + + let call_id = "allow-network-after-yolo"; + let fetch_command = r#"python3 -c "import urllib.request; opener = urllib.request.build_opener(urllib.request.ProxyHandler()); print('OK:' + opener.open('http://codex-network-test.invalid', timeout=30).read().decode(errors='replace'))""# + .to_string(); + let event = shell_event( + call_id, + &fetch_command, + /*timeout_ms*/ 30_000, + SandboxPermissions::UseDefault, + )?; + + let _ = mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-network-after-yolo-1"), + event, + ev_completed("resp-network-after-yolo-1"), + ]), + ) + .await; + let _ = mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-network-after-yolo-1", "done"), + ev_completed("resp-network-after-yolo-2"), + ]), + ) + .await; + + submit_turn( + &test, + "allow-network-after-yolo", + approval_policy, + turn_sandbox_policy, + ) + .await?; + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + let approval = loop { + let remaining = deadline + .checked_duration_since(std::time::Instant::now()) + .expect("timed out waiting for network approval request"); + let event = wait_for_event_with_timeout( + &test.codex, + |event| { + matches!( + event, + EventMsg::ExecApprovalRequest(_) | EventMsg::TurnComplete(_) + ) + }, + remaining, + ) + .await; + match event { + EventMsg::ExecApprovalRequest(approval) => { + if approval.command.first().map(std::string::String::as_str) + == Some("network-access") + { + break approval; + } + test.codex + .submit(Op::ExecApproval { + id: approval.effective_approval_id(), + turn_id: None, + decision: ReviewDecision::Approved, + }) + .await?; + } + EventMsg::TurnComplete(_) => { + panic!("expected network approval request before completion"); + } + other => panic!("unexpected event: {other:?}"), + } + }; + + let network_context = approval + .network_approval_context + .clone() + .expect("expected network approval context"); + assert_eq!(network_context.protocol, NetworkApprovalProtocol::Http); + + test.codex + .submit(Op::ExecApproval { + id: approval.effective_approval_id(), + turn_id: None, + decision: ReviewDecision::Denied, + }) + .await?; + wait_for_completion(&test).await; + + Ok(()) +} + // todo(dylan) add ScenarioSpec support for rules #[tokio::test(flavor = "current_thread")] #[cfg(unix)] From f948690fc8fdc90995c507227540b2df13255885 Mon Sep 17 00:00:00 2001 From: Ruslan Nigmatullin Date: Wed, 15 Apr 2026 17:57:02 -0700 Subject: [PATCH 49/84] [codex] Make command exec delta tests chunk tolerant (#17999) ## Summary - Make command/exec output-delta tests accumulate streamed chunks instead of assuming complete logical output in a single notification. - Collect stdout and stderr independently so stream interleaving does not fail the pipe streaming test. ## Why The command/exec protocol exposes output as deltas, so tests should not rely on chunk boundaries being stable. A line like `out-start\n` may arrive split across multiple notifications, and stdout/stderr notifications may interleave. ## Validation - `just fmt` - `git diff --check` - `cargo test -p codex-app-server suite::v2::command_exec` --- .../app-server/tests/suite/v2/command_exec.rs | 188 +++++++++++------- 1 file changed, 114 insertions(+), 74 deletions(-) diff --git a/codex-rs/app-server/tests/suite/v2/command_exec.rs b/codex-rs/app-server/tests/suite/v2/command_exec.rs index 50e8285665..d7645f13e3 100644 --- a/codex-rs/app-server/tests/suite/v2/command_exec.rs +++ b/codex-rs/app-server/tests/suite/v2/command_exec.rs @@ -411,11 +411,14 @@ async fn command_exec_streaming_does_not_buffer_output() -> Result<()> { }) .await?; - let delta = read_command_exec_delta(&mut mcp).await?; - assert_eq!(delta.process_id, process_id.as_str()); - assert_eq!(delta.stream, CommandExecOutputStream::Stdout); - assert_eq!(STANDARD.decode(&delta.delta_base64)?, b"abcde"); - assert!(delta.cap_reached); + let output = collect_command_exec_output_until( + CommandExecDeltaReader::Mcp(&mut mcp), + process_id.as_str(), + "capped stdout", + |_output, delta| delta.stream == CommandExecOutputStream::Stdout && delta.cap_reached, + ) + .await?; + assert_eq!(output.stdout, "abcde"); let terminate_request_id = mcp .send_command_exec_terminate_request(CommandExecTerminateParams { process_id: process_id.clone(), @@ -471,21 +474,13 @@ async fn command_exec_pipe_streams_output_and_accepts_write() -> Result<()> { }) .await?; - let first_stdout = read_command_exec_delta(&mut mcp).await?; - let first_stderr = read_command_exec_delta(&mut mcp).await?; - let seen = [first_stdout, first_stderr]; - assert!( - seen.iter() - .all(|delta| delta.process_id == process_id.as_str()) - ); - assert!(seen.iter().any(|delta| { - delta.stream == CommandExecOutputStream::Stdout - && delta.delta_base64 == STANDARD.encode("out-start\n") - })); - assert!(seen.iter().any(|delta| { - delta.stream == CommandExecOutputStream::Stderr - && delta.delta_base64 == STANDARD.encode("err-start\n") - })); + wait_for_command_exec_outputs_contains( + &mut mcp, + process_id.as_str(), + "out-start\n", + "err-start\n", + ) + .await?; let write_request_id = mcp .send_command_exec_write_request(CommandExecWriteParams { @@ -499,21 +494,13 @@ async fn command_exec_pipe_streams_output_and_accepts_write() -> Result<()> { .await?; assert_eq!(write_response.result, serde_json::json!({})); - let next_delta = read_command_exec_delta(&mut mcp).await?; - let final_delta = read_command_exec_delta(&mut mcp).await?; - let seen = [next_delta, final_delta]; - assert!( - seen.iter() - .all(|delta| delta.process_id == process_id.as_str()) - ); - assert!(seen.iter().any(|delta| { - delta.stream == CommandExecOutputStream::Stdout - && delta.delta_base64 == STANDARD.encode("out:hello\n") - })); - assert!(seen.iter().any(|delta| { - delta.stream == CommandExecOutputStream::Stderr - && delta.delta_base64 == STANDARD.encode("err:hello\n") - })); + wait_for_command_exec_outputs_contains( + &mut mcp, + process_id.as_str(), + "out:hello\n", + "err:hello\n", + ) + .await?; let response = mcp .read_stream_until_response_message(RequestId::Integer(command_request_id)) @@ -562,17 +549,13 @@ async fn command_exec_tty_implies_streaming_and_reports_pty_output() -> Result<( }) .await?; - let started_text = read_command_exec_output_until_contains( + wait_for_command_exec_output_contains( &mut mcp, process_id.as_str(), CommandExecOutputStream::Stdout, "tty\n", ) .await?; - assert!( - started_text.contains("tty\n"), - "expected TTY startup output, got {started_text:?}" - ); let write_request_id = mcp .send_command_exec_write_request(CommandExecWriteParams { @@ -586,17 +569,13 @@ async fn command_exec_tty_implies_streaming_and_reports_pty_output() -> Result<( .await?; assert_eq!(write_response.result, serde_json::json!({})); - let echoed_text = read_command_exec_output_until_contains( + wait_for_command_exec_output_contains( &mut mcp, process_id.as_str(), CommandExecOutputStream::Stdout, "echo:world\n", ) .await?; - assert!( - echoed_text.contains("echo:world\n"), - "expected TTY echo output, got {echoed_text:?}" - ); let response = mcp .read_stream_until_response_message(RequestId::Integer(command_request_id)) @@ -643,17 +622,13 @@ async fn command_exec_tty_supports_initial_size_and_resize() -> Result<()> { }) .await?; - let started_text = read_command_exec_output_until_contains( + wait_for_command_exec_output_contains( &mut mcp, process_id.as_str(), CommandExecOutputStream::Stdout, "start:31 101\n", ) .await?; - assert!( - started_text.contains("start:31 101\n"), - "unexpected initial size output: {started_text:?}" - ); let resize_request_id = mcp .send_command_exec_resize_request(CommandExecResizeParams { @@ -681,17 +656,13 @@ async fn command_exec_tty_supports_initial_size_and_resize() -> Result<()> { .await?; assert_eq!(write_response.result, serde_json::json!({})); - let resized_text = read_command_exec_output_until_contains( + wait_for_command_exec_output_contains( &mut mcp, process_id.as_str(), CommandExecOutputStream::Stdout, "after:45 132\n", ) .await?; - assert!( - resized_text.contains("after:45 132\n"), - "unexpected resized output: {resized_text:?}" - ); let response = mcp .read_stream_until_response_message(RequestId::Integer(command_request_id)) @@ -744,11 +715,13 @@ async fn command_exec_process_ids_are_connection_scoped_and_disconnect_terminate ) .await?; - let delta = read_command_exec_delta_ws(&mut ws1).await?; - assert_eq!(delta.process_id, "shared-process"); - assert_eq!(delta.stream, CommandExecOutputStream::Stdout); - let delta_text = String::from_utf8(STANDARD.decode(&delta.delta_base64)?)?; - assert!(delta_text.contains("ready")); + collect_command_exec_output_until( + CommandExecDeltaReader::Websocket(&mut ws1), + "shared-process", + "websocket ready output", + |output, _delta| output.stdout.contains("ready\n"), + ) + .await?; wait_for_process_marker(&marker, /*should_exist*/ true).await?; send_request( @@ -796,31 +769,98 @@ async fn read_command_exec_delta( decode_delta_notification(notification) } -async fn read_command_exec_output_until_contains( +async fn wait_for_command_exec_output_contains( mcp: &mut McpProcess, process_id: &str, stream: CommandExecOutputStream, expected: &str, -) -> Result { +) -> Result<()> { + let stream_name = match stream { + CommandExecOutputStream::Stdout => "stdout", + CommandExecOutputStream::Stderr => "stderr", + }; + collect_command_exec_output_until( + CommandExecDeltaReader::Mcp(mcp), + process_id, + format!("{stream_name} containing {expected:?}"), + |output, _delta| match stream { + CommandExecOutputStream::Stdout => output.stdout.contains(expected), + CommandExecOutputStream::Stderr => output.stderr.contains(expected), + }, + ) + .await?; + Ok(()) +} + +async fn wait_for_command_exec_outputs_contains( + mcp: &mut McpProcess, + process_id: &str, + stdout_expected: &str, + stderr_expected: &str, +) -> Result<()> { + collect_command_exec_output_until( + CommandExecDeltaReader::Mcp(mcp), + process_id, + format!("stdout containing {stdout_expected:?} and stderr containing {stderr_expected:?}"), + |output, _delta| { + output.stdout.contains(stdout_expected) && output.stderr.contains(stderr_expected) + }, + ) + .await?; + Ok(()) +} + +enum CommandExecDeltaReader<'a> { + Mcp(&'a mut McpProcess), + Websocket(&'a mut super::connection_handling_websocket::WsClient), +} + +#[derive(Default)] +struct CollectedCommandExecOutput { + stdout: String, + stderr: String, +} + +async fn collect_command_exec_output_until( + mut reader: CommandExecDeltaReader<'_>, + process_id: &str, + waiting_for: impl Into, + mut should_stop: impl FnMut( + &CollectedCommandExecOutput, + &CommandExecOutputDeltaNotification, + ) -> bool, +) -> Result { + let waiting_for = waiting_for.into(); let deadline = Instant::now() + DEFAULT_READ_TIMEOUT; - let mut collected = String::new(); + let mut output = CollectedCommandExecOutput::default(); loop { let remaining = deadline.saturating_duration_since(Instant::now()); - let delta = timeout(remaining, read_command_exec_delta(mcp)) - .await - .with_context(|| { - format!( - "timed out waiting for {expected:?} in command/exec output for {process_id}; collected {collected:?}" - ) - })??; + let delta = timeout(remaining, async { + match &mut reader { + CommandExecDeltaReader::Mcp(mcp) => read_command_exec_delta(mcp).await, + CommandExecDeltaReader::Websocket(stream) => { + read_command_exec_delta_ws(stream).await + } + } + }) + .await + .with_context(|| { + format!( + "timed out waiting for {waiting_for} in command/exec output for {process_id}; collected stdout={:?}, stderr={:?}", + output.stdout, output.stderr + ) + })??; assert_eq!(delta.process_id, process_id); - assert_eq!(delta.stream, stream); let delta_text = String::from_utf8(STANDARD.decode(&delta.delta_base64)?)?; - collected.push_str(&delta_text.replace('\r', "")); - if collected.contains(expected) { - return Ok(collected); + let delta_text = delta_text.replace('\r', ""); + match delta.stream { + CommandExecOutputStream::Stdout => output.stdout.push_str(&delta_text), + CommandExecOutputStream::Stderr => output.stderr.push_str(&delta_text), + } + if should_stop(&output, &delta) { + return Ok(output); } } } From d63ba2d5ecdbcc4cd2b1fb517e96f5cf1bade307 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 15 Apr 2026 18:07:46 -0700 Subject: [PATCH 50/84] feat: introduce codex-pr-body skill (#18033) ## Motivation Codex needs a repeatable workflow for updating PR metadata after a pull request already exists. This is more specific than generic GitHub handling: the assistant needs to preserve author-provided body content, explain why the PR exists before listing implementation details, and describe only the net change under review, including when Sapling stacks put a PR on top of another PR instead of `main`. ## Changes - Adds `.codex/skills/codex-pr-body/SKILL.md`. - Documents how to infer the target PR from the current branch or commit, including Sapling-specific PR metadata and `sl sl` output. - Defines the expected PR body update behavior: inspect the existing body, preserve key content such as images, avoid local absolute paths, use Markdown formatting, include relevant issue/PR references, and call out developer docs follow-up only when applicable. - Captures stacked-PR handling so generated PR text describes the change between the PR's base and head, rather than unrelated ancestor changes. ## Verification Not run; this is a Codex skill documentation addition. --- .codex/skills/codex-pr-body/SKILL.md | 59 ++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 .codex/skills/codex-pr-body/SKILL.md diff --git a/.codex/skills/codex-pr-body/SKILL.md b/.codex/skills/codex-pr-body/SKILL.md new file mode 100644 index 0000000000..76b37b8750 --- /dev/null +++ b/.codex/skills/codex-pr-body/SKILL.md @@ -0,0 +1,59 @@ +--- +name: codex-pr-body +description: Update the title and body of one or more pull requests. +--- + +## Determining the PR(s) + +When this skill is invoked, the PR(s) to update may be specified explicitly, but in the common case, the PR(s) to update will be inferred from the branch / commit that the user is currently working on. For ordinary Git usage (i.e., not Sapling as discussed below), you may have to use a combination of `git branch` and `gh pr view --repo openai/codex --json number --jq '.number'` to determine the PR associated with the current branch / commit. + +## PR Body Contents + +When invoked, use `gh` to edit the pull request body and title to reflect the contents of the specified PR. Make sure to check the existing pull request body to see if there is key information that should be preserved. For example, NEVER remove an image in the existing pull request body, as the author may have no way to recover it if you remove it. + +It is critically important to explain _why_ the change is being made. If the current conversation in which this skill is invoked has discussed the motivation, be sure to capture this in the pull request body. + +The body should also explain _what_ changed, but this should appear after the _why_. + +Limit discussion to the _net change_ of the commit. It is generally frowned upon to discuss changes that were attempted but later undone in the course of the development of the pull request. When rewriting the pull request body, you may need to eliminate details such as these when they are no longer appropriate / of interest to future readers. + +Avoid references to absolute paths on my local disk. When talking about a path that is within the repository, simply use the repo-relative path. + +It is generally helpful to discuss how the change was verified. That said, it is unnecessary to mention things that CI checks automatically, e.g., do not include "ran `just fmt`" as part of the test plan. Though identifying the new tests that were purposely introduced to verify the new behavior introduced by the pull request is often appropriate. + +Make use of Markdown to format the pull request professionally. Ensure "code things" appear in single backticks when referenced inline. Fenced code blocks are useful when referencing code or showing a shell transcript. Also, make use of GitHub permalinks when citing existing pieces of code that are relevant to the change. + +Make sure to reference any relevant pull requests or issues, though there should be no need to reference the pull request in its own PR body. + +If there is documentation that should be updated on https://developers.openai.com/codex as a result of this change, please note that in a separate section near the end of the pull request. Omit this section if there is no documentation that needs to be updated. + +## Working with Stacks + +Sometimes a pull request is composed of a stack of commits that build on one another. In these cases, the PR body should reflect the _net_ change introduced by the stack as a whole, rather than the individual commits that make up the stack. + +Similarly, sometimes a user may be using a tool like Sapling to leverage _stacked pull requests_, in which case the `base` of the PR may be the a branch that is the `head` of another PR in the stack rather than `main`. In this case, be sure to discuss only the net change between the `base` and `head` of the PR that is being opened against that stacked base, rather than the changes relative to `main`. + +## Sapling + +If `.git/sl/store` is present, then this Git repository is governed by Sapling SCM (https://sapling-scm.com). + +In Sapling, run the following to see if there is a GitHub pull request associated with the current revision: + +```shell +sl log --template '{github_pull_request_url}' -r . +``` + +Alternatively, you can run `sl sl` to see the current development branch and whether there is a GitHub pull request associated with the current commit. For example, if the output were: + +``` + @ cb032b31cf 72 minutes ago mbolin #11412 +โ•ญโ”€โ•ฏ tui: show non-file layer content in /debug-config +โ”‚ +o fdd0cd1de9 Today at 20:09 origin/main +โ”‚ +~ +``` + +- `@` indicates the current commit is `cb032b31cf` +- it is a development branch containing a single commit branched off of `origin/main` +- it is associated with GitHub pull request #11412 From d97bad1272ce9ad6cc353b33d92df40d0cbc9436 Mon Sep 17 00:00:00 2001 From: canvrno-oai Date: Wed, 15 Apr 2026 18:28:11 -0700 Subject: [PATCH 51/84] Display YOLO mode permissions if set when launching TUI (#17877) - When launching the TUI client, if YOLO mode is enabled, display this in the header. - Eligibility is determined by `approval_policy = "never"` and `sandbox_mode = "danger-full-access"` image --- codex-rs/tui/src/app.rs | 1 + codex-rs/tui/src/chatwidget.rs | 19 +++--- codex-rs/tui/src/history_cell.rs | 61 ++++++++++++++++++- ...s__session_header_indicates_yolo_mode.snap | 11 ++++ 4 files changed, 81 insertions(+), 11 deletions(-) create mode 100644 codex-rs/tui/src/snapshots/codex_tui__history_cell__tests__session_header_indicates_yolo_mode.snap diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 50dbd0bf6d..03a99e3aa3 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -1599,6 +1599,7 @@ impl App { self.config.cwd.to_path_buf(), version, ) + .with_yolo_mode(history_cell::is_yolo_mode(&self.config)) .display_lines(width) } diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 599655b364..2f5258bf2f 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -9722,14 +9722,17 @@ impl ChatWidget { /// Build a placeholder header cell while the session is configuring. fn placeholder_session_header_cell(config: &Config) -> Box { let placeholder_style = Style::default().add_modifier(Modifier::DIM | Modifier::ITALIC); - Box::new(history_cell::SessionHeaderHistoryCell::new_with_style( - DEFAULT_MODEL_DISPLAY_NAME.to_string(), - placeholder_style, - /*reasoning_effort*/ None, - /*show_fast_status*/ false, - config.cwd.to_path_buf(), - CODEX_CLI_VERSION, - )) + Box::new( + history_cell::SessionHeaderHistoryCell::new_with_style( + DEFAULT_MODEL_DISPLAY_NAME.to_string(), + placeholder_style, + /*reasoning_effort*/ None, + /*show_fast_status*/ false, + config.cwd.to_path_buf(), + CODEX_CLI_VERSION, + ) + .with_yolo_mode(history_cell::is_yolo_mode(config)), + ) } /// Merge the real session info cell with any placeholder header to avoid double boxes. diff --git a/codex-rs/tui/src/history_cell.rs b/codex-rs/tui/src/history_cell.rs index 79deede952..f0611e968c 100644 --- a/codex-rs/tui/src/history_cell.rs +++ b/codex-rs/tui/src/history_cell.rs @@ -61,9 +61,11 @@ use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig; use codex_protocol::plan_tool::PlanItemArg; use codex_protocol::plan_tool::StepStatus; use codex_protocol::plan_tool::UpdatePlanArgs; +use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::FileChange; use codex_protocol::protocol::McpAuthStatus; use codex_protocol::protocol::McpInvocation; +use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SessionConfiguredEvent; use codex_protocol::request_user_input::RequestUserInputAnswer; use codex_protocol::request_user_input::RequestUserInputQuestion; @@ -1186,6 +1188,8 @@ pub(crate) fn new_session_info( let SessionConfiguredEvent { model, reasoning_effort, + approval_policy, + sandbox_policy, .. } = event; // Header box rendered as history (so it appears at the very top) @@ -1195,7 +1199,8 @@ pub(crate) fn new_session_info( show_fast_status, config.cwd.to_path_buf(), CODEX_CLI_VERSION, - ); + ) + .with_yolo_mode(has_yolo_permissions(approval_policy, &sandbox_policy)); let mut parts: Vec> = vec![Box::new(header)]; if is_first_event { @@ -1259,6 +1264,17 @@ pub(crate) fn new_session_info( SessionInfoCell(CompositeHistoryCell { parts }) } +pub(crate) fn is_yolo_mode(config: &Config) -> bool { + has_yolo_permissions( + config.permissions.approval_policy.value(), + config.permissions.sandbox_policy.get(), + ) +} + +fn has_yolo_permissions(approval_policy: AskForApproval, sandbox_policy: &SandboxPolicy) -> bool { + approval_policy == AskForApproval::Never && *sandbox_policy == SandboxPolicy::DangerFullAccess +} + pub(crate) fn new_user_prompt( message: String, text_elements: Vec, @@ -1281,6 +1297,7 @@ pub(crate) struct SessionHeaderHistoryCell { reasoning_effort: Option, show_fast_status: bool, directory: PathBuf, + yolo_mode: bool, } impl SessionHeaderHistoryCell { @@ -1316,9 +1333,15 @@ impl SessionHeaderHistoryCell { reasoning_effort, show_fast_status, directory, + yolo_mode: false, } } + pub(crate) fn with_yolo_mode(mut self, yolo_mode: bool) -> Self { + self.yolo_mode = yolo_mode; + self + } + fn format_directory(&self, max_width: Option) -> String { Self::format_directory_inner(&self.directory, max_width) } @@ -1377,7 +1400,12 @@ impl HistoryCell for SessionHeaderHistoryCell { const CHANGE_MODEL_HINT_COMMAND: &str = "/model"; const CHANGE_MODEL_HINT_EXPLANATION: &str = " to change"; const DIR_LABEL: &str = "directory:"; - let label_width = DIR_LABEL.len(); + const PERMISSIONS_LABEL: &str = "permissions:"; + let label_width = if self.yolo_mode { + DIR_LABEL.len().max(PERMISSIONS_LABEL.len()) + } else { + DIR_LABEL.len() + }; let model_label = format!( "{model_label:_ OpenAI Codex (vtest) โ”‚ +โ”‚ โ”‚ +โ”‚ model: gpt-5 /model to change โ”‚ +โ”‚ directory: /tmp/project โ”‚ +โ”‚ permissions: YOLO mode โ”‚ +โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ From bd61737e8afb2b52f8a03893172db1523f59e357 Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Wed, 15 Apr 2026 19:18:38 -0700 Subject: [PATCH 52/84] Async config loading (#18022) Parts of config will come from executor. Prepare for that by making config loading methods async. --- codex-rs/app-server-client/src/lib.rs | 1 + .../app-server/src/codex_message_processor.rs | 1 + codex-rs/app-server/src/in_process.rs | 1 + codex-rs/app-server/src/lib.rs | 14 +- .../app-server/tests/suite/v2/thread_start.rs | 6 +- codex-rs/config/src/config_toml.rs | 15 +- codex-rs/core/src/agent/role.rs | 8 +- codex-rs/core/src/codex_tests.rs | 30 +- codex-rs/core/src/config/config_tests.rs | 572 ++++++++++-------- codex-rs/core/src/config/mod.rs | 37 +- codex-rs/core/src/config/permissions_tests.rs | 7 +- codex-rs/core/src/config_loader/mod.rs | 2 +- codex-rs/core/src/git_info_tests.rs | 47 +- codex-rs/core/src/guardian/review_session.rs | 12 +- codex-rs/core/src/guardian/tests.rs | 45 +- codex-rs/core/src/memories/tests.rs | 4 +- codex-rs/core/src/prompt_debug.rs | 2 +- codex-rs/core/src/realtime_context.rs | 42 +- codex-rs/core/src/realtime_context_tests.rs | 21 +- codex-rs/core/src/thread_manager_tests.rs | 10 +- codex-rs/core/src/tools/spec_tests.rs | 199 +++--- codex-rs/git-utils/src/info.rs | 2 +- codex-rs/tui/src/chatwidget/tests/helpers.rs | 1 + .../tui/src/onboarding/onboarding_screen.rs | 9 +- .../tui/src/onboarding/trust_directory.rs | 7 +- 25 files changed, 624 insertions(+), 471 deletions(-) diff --git a/codex-rs/app-server-client/src/lib.rs b/codex-rs/app-server-client/src/lib.rs index 35f9068712..9818b2f61b 100644 --- a/codex-rs/app-server-client/src/lib.rs +++ b/codex-rs/app-server-client/src/lib.rs @@ -971,6 +971,7 @@ mod tests { match ConfigBuilder::default().build().await { Ok(config) => config, Err(_) => Config::load_default_with_cli_overrides(Vec::new()) + .await .expect("default config should load"), } } diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 42cb8b2d1a..f805ec3ff6 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -2442,6 +2442,7 @@ impl CodexMessageProcessor { )) { let trust_target = resolve_root_git_project_for_trust(config.cwd.as_path()) + .await .unwrap_or_else(|| config.cwd.to_path_buf()); let cli_overrides_with_trust; let cli_overrides_for_reload = if let Err(err) = diff --git a/codex-rs/app-server/src/in_process.rs b/codex-rs/app-server/src/in_process.rs index 84dfd282cd..4458bce89d 100644 --- a/codex-rs/app-server/src/in_process.rs +++ b/codex-rs/app-server/src/in_process.rs @@ -716,6 +716,7 @@ mod tests { match ConfigBuilder::default().build().await { Ok(config) => config, Err(_) => Config::load_default_with_cli_overrides(Vec::new()) + .await .expect("default config should load"), } } diff --git a/codex-rs/app-server/src/lib.rs b/codex-rs/app-server/src/lib.rs index ca48ad0f41..ba946c67b6 100644 --- a/codex-rs/app-server/src/lib.rs +++ b/codex-rs/app-server/src/lib.rs @@ -432,12 +432,14 @@ pub async fn run_main_with_transport( Err(err) => { let message = config_warning_from_error("Invalid configuration; using defaults.", &err); config_warnings.push(message); - Config::load_default_with_cli_overrides(cli_kv_overrides.clone()).map_err(|e| { - std::io::Error::new( - ErrorKind::InvalidData, - format!("error loading default config after config error: {e}"), - ) - })? + Config::load_default_with_cli_overrides(cli_kv_overrides.clone()) + .await + .map_err(|e| { + std::io::Error::new( + ErrorKind::InvalidData, + format!("error loading default config after config error: {e}"), + ) + })? } }; diff --git a/codex-rs/app-server/tests/suite/v2/thread_start.rs b/codex-rs/app-server/tests/suite/v2/thread_start.rs index b8d0db9a01..ab514252e4 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_start.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_start.rs @@ -717,6 +717,7 @@ model_reasoning_effort = "high" let config_toml = std::fs::read_to_string(codex_home.path().join("config.toml"))?; let trusted_root = resolve_root_git_project_for_trust(workspace.path()) + .await .unwrap_or_else(|| workspace.path().to_path_buf()); assert!(config_toml.contains(&persisted_trust_path(&trusted_root))); assert!(config_toml.contains("trust_level = \"trusted\"")); @@ -753,8 +754,9 @@ async fn thread_start_with_nested_git_cwd_trusts_repo_root() -> Result<()> { .await??; let config_toml = std::fs::read_to_string(codex_home.path().join("config.toml"))?; - let trusted_root = - resolve_root_git_project_for_trust(&nested).expect("git root should resolve"); + let trusted_root = resolve_root_git_project_for_trust(&nested) + .await + .expect("git root should resolve"); assert!(config_toml.contains(&persisted_trust_path(&trusted_root))); assert!(!config_toml.contains(&persisted_trust_path(&nested))); diff --git a/codex-rs/config/src/config_toml.rs b/codex-rs/config/src/config_toml.rs index 92e5304fe5..7de4e253e5 100644 --- a/codex-rs/config/src/config_toml.rs +++ b/codex-rs/config/src/config_toml.rs @@ -596,7 +596,7 @@ pub struct GhostSnapshotToml { impl ConfigToml { /// Derive the effective sandbox policy from the configuration. - pub fn derive_sandbox_policy( + pub async fn derive_sandbox_policy( &self, sandbox_mode_override: Option, profile_sandbox_mode: Option, @@ -610,11 +610,13 @@ impl ConfigToml { let resolved_sandbox_mode = sandbox_mode_override .or(profile_sandbox_mode) .or(self.sandbox_mode) - .or_else(|| { + .or(if sandbox_mode_was_explicit { + None + } else { // If no sandbox_mode is set but this directory has a trust decision, // default to workspace-write except on unsandboxed Windows where we // default to read-only. - self.get_active_project(resolved_cwd).and_then(|p| { + self.get_active_project(resolved_cwd).await.and_then(|p| { if p.is_trusted() || p.is_untrusted() { if cfg!(target_os = "windows") && windows_sandbox_level == WindowsSandboxLevel::Disabled @@ -676,7 +678,8 @@ impl ConfigToml { /// Resolves the cwd to an existing project, or returns None if ConfigToml /// does not contain a project corresponding to cwd or a git repo for cwd - pub fn get_active_project(&self, resolved_cwd: &Path) -> Option { + pub async fn get_active_project(&self, resolved_cwd: &Path) -> Option { + let repo_root = resolve_root_git_project_for_trust(resolved_cwd).await; let projects = self.projects.clone().unwrap_or_default(); let resolved_cwd_key = project_trust_key(resolved_cwd); @@ -691,8 +694,8 @@ impl ConfigToml { // If cwd lives inside a git repo/worktree, check whether the root git project // (the primary repository working directory) is trusted. This lets // worktrees inherit trust from the main project. - if let Some(repo_root) = resolve_root_git_project_for_trust(resolved_cwd) { - let repo_root_key = project_trust_key(repo_root.as_path()); + if let Some(repo_root) = repo_root.as_deref() { + let repo_root_key = project_trust_key(repo_root); let repo_root_raw_key = repo_root.to_string_lossy().to_string(); if let Some(project_config_for_root) = projects .get(&repo_root_key) diff --git a/codex-rs/core/src/agent/role.rs b/codex-rs/core/src/agent/role.rs index 83c2b1843b..61847f49ba 100644 --- a/codex-rs/core/src/agent/role.rs +++ b/codex-rs/core/src/agent/role.rs @@ -78,7 +78,8 @@ async fn apply_role_to_config_inner( role_layer_toml, preserve_current_profile, preserve_current_provider, - )?; + ) + .await?; Ok(()) } @@ -150,7 +151,7 @@ fn preservation_policy(config: &Config, role_layer_toml: &TomlValue) -> (bool, b mod reload { use super::*; - pub(super) fn build_next_config( + pub(super) async fn build_next_config( config: &Config, role_layer_toml: TomlValue, preserve_current_profile: bool, @@ -171,7 +172,8 @@ mod reload { reload_overrides(config, preserve_current_provider), config.codex_home.clone(), config_layer_stack, - )?; + ) + .await?; if preserve_current_profile { next_config.active_profile = config.active_profile.clone(); } diff --git a/codex-rs/core/src/codex_tests.rs b/codex-rs/core/src/codex_tests.rs index 054e98be67..fcb1b58993 100644 --- a/codex-rs/core/src/codex_tests.rs +++ b/codex-rs/core/src/codex_tests.rs @@ -457,8 +457,8 @@ fn numbered_mcp_tools(count: usize) -> HashMap { .collect() } -fn tools_config_for_mcp_tool_exposure(search_tool: bool) -> ToolsConfig { - let config = test_config(); +async fn tools_config_for_mcp_tool_exposure(search_tool: bool) -> ToolsConfig { + let config = test_config().await; let model_info = ModelsManager::construct_model_info_offline_for_tests( "gpt-5-codex", &config.to_models_manager_config(), @@ -870,7 +870,7 @@ async fn get_base_instructions_no_user_content() { ]; let (session, _turn_context) = make_session_and_context().await; - let config = test_config(); + let config = test_config().await; for test_case in test_cases { let model_info = model_info_for_slug(test_case.slug, &config); @@ -1035,10 +1035,10 @@ fn collect_explicit_app_ids_from_skill_items_skips_plain_mentions_with_skill_con assert_eq!(connector_ids, HashSet::::new()); } -#[test] -fn mcp_tool_exposure_directly_exposes_small_effective_tool_sets() { - let config = test_config(); - let tools_config = tools_config_for_mcp_tool_exposure(/*search_tool*/ true); +#[tokio::test] +async fn mcp_tool_exposure_directly_exposes_small_effective_tool_sets() { + let config = test_config().await; + let tools_config = tools_config_for_mcp_tool_exposure(/*search_tool*/ true).await; let mcp_tools = numbered_mcp_tools(DIRECT_MCP_TOOL_EXPOSURE_THRESHOLD - 1); let exposure = build_mcp_tool_exposure( @@ -1057,10 +1057,10 @@ fn mcp_tool_exposure_directly_exposes_small_effective_tool_sets() { assert!(exposure.deferred_tools.is_none()); } -#[test] -fn mcp_tool_exposure_searches_large_effective_tool_sets() { - let config = test_config(); - let tools_config = tools_config_for_mcp_tool_exposure(/*search_tool*/ true); +#[tokio::test] +async fn mcp_tool_exposure_searches_large_effective_tool_sets() { + let config = test_config().await; + let tools_config = tools_config_for_mcp_tool_exposure(/*search_tool*/ true).await; let mcp_tools = numbered_mcp_tools(DIRECT_MCP_TOOL_EXPOSURE_THRESHOLD); let exposure = build_mcp_tool_exposure( @@ -1083,10 +1083,10 @@ fn mcp_tool_exposure_searches_large_effective_tool_sets() { assert_eq!(deferred_tool_names, expected_tool_names); } -#[test] -fn mcp_tool_exposure_directly_exposes_explicit_apps_without_deferred_overlap() { - let config = test_config(); - let tools_config = tools_config_for_mcp_tool_exposure(/*search_tool*/ true); +#[tokio::test] +async fn mcp_tool_exposure_directly_exposes_explicit_apps_without_deferred_overlap() { + let config = test_config().await; + let tools_config = tools_config_for_mcp_tool_exposure(/*search_tool*/ true).await; let mut mcp_tools = numbered_mcp_tools(DIRECT_MCP_TOOL_EXPOSURE_THRESHOLD - 1); mcp_tools.extend([( "mcp__codex_apps__calendar_create_event".to_string(), diff --git a/codex-rs/core/src/config/config_tests.rs b/codex-rs/core/src/config/config_tests.rs index b2931e1055..650d792306 100644 --- a/codex-rs/core/src/config/config_tests.rs +++ b/codex-rs/core/src/config/config_tests.rs @@ -117,8 +117,8 @@ fn http_mcp(url: &str) -> McpServerConfig { } } -#[test] -fn load_config_normalizes_relative_cwd_override() -> std::io::Result<()> { +#[tokio::test] +async fn load_config_normalizes_relative_cwd_override() -> std::io::Result<()> { let expected_cwd = AbsolutePathBuf::relative_to_current_dir("nested")?; let codex_home = tempdir()?; let config = Config::load_from_base_config_with_overrides( @@ -128,14 +128,15 @@ fn load_config_normalizes_relative_cwd_override() -> std::io::Result<()> { ..Default::default() }, codex_home.abs(), - )?; + ) + .await?; assert_eq!(config.cwd, expected_cwd); Ok(()) } -#[test] -fn load_config_records_global_agents_path() -> std::io::Result<()> { +#[tokio::test] +async fn load_config_records_global_agents_path() -> std::io::Result<()> { let codex_home = tempdir()?; let global_agents_path = codex_home.path().join(DEFAULT_PROJECT_DOC_FILENAME); std::fs::write(&global_agents_path, "\n global instructions \n")?; @@ -144,7 +145,8 @@ fn load_config_records_global_agents_path() -> std::io::Result<()> { ConfigToml::default(), ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.user_instructions.as_deref(), @@ -157,8 +159,8 @@ fn load_config_records_global_agents_path() -> std::io::Result<()> { Ok(()) } -#[test] -fn load_config_records_preferred_global_agents_override_path() -> std::io::Result<()> { +#[tokio::test] +async fn load_config_records_preferred_global_agents_override_path() -> std::io::Result<()> { let codex_home = tempdir()?; std::fs::write( codex_home.path().join(DEFAULT_PROJECT_DOC_FILENAME), @@ -171,7 +173,8 @@ fn load_config_records_preferred_global_agents_override_path() -> std::io::Resul ConfigToml::default(), ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.user_instructions.as_deref(), @@ -184,8 +187,8 @@ fn load_config_records_preferred_global_agents_override_path() -> std::io::Resul Ok(()) } -#[test] -fn test_toml_parsing() { +#[tokio::test] +async fn test_toml_parsing() { let history_with_persistence = r#" [history] persistence = "save-all" @@ -251,6 +254,7 @@ consolidation_model = "gpt-5" ConfigOverrides::default(), tempdir().expect("tempdir").abs(), ) + .await .expect("load config from memories settings"); assert_eq!( config.memories, @@ -376,13 +380,14 @@ fn config_toml_deserializes_model_availability_nux() { ); } -#[test] -fn runtime_config_defaults_model_availability_nux() { +#[tokio::test] +async fn runtime_config_defaults_model_availability_nux() { let cfg = Config::load_from_base_config_with_overrides( ConfigToml::default(), ConfigOverrides::default(), tempdir().expect("tempdir").abs(), ) + .await .expect("load config"); assert_eq!( @@ -462,8 +467,9 @@ allow_upstream_proxy = false ); } -#[test] -fn permissions_profiles_network_populates_runtime_network_proxy_spec() -> std::io::Result<()> { +#[tokio::test] +async fn permissions_profiles_network_populates_runtime_network_proxy_spec() -> std::io::Result<()> +{ let codex_home = TempDir::new()?; let cwd = TempDir::new()?; std::fs::write(cwd.path().join(".git"), "gitdir: nowhere")?; @@ -497,7 +503,8 @@ fn permissions_profiles_network_populates_runtime_network_proxy_spec() -> std::i ..Default::default() }, codex_home.abs(), - )?; + ) + .await?; let network = config .permissions .network @@ -509,8 +516,9 @@ fn permissions_profiles_network_populates_runtime_network_proxy_spec() -> std::i Ok(()) } -#[test] -fn permissions_profiles_network_disabled_by_default_does_not_start_proxy() -> std::io::Result<()> { +#[tokio::test] +async fn permissions_profiles_network_disabled_by_default_does_not_start_proxy() +-> std::io::Result<()> { let codex_home = TempDir::new()?; let cwd = TempDir::new()?; std::fs::write(cwd.path().join(".git"), "gitdir: nowhere")?; @@ -547,14 +555,15 @@ fn permissions_profiles_network_disabled_by_default_does_not_start_proxy() -> st ..Default::default() }, codex_home.abs(), - )?; + ) + .await?; assert!(config.permissions.network.is_none()); Ok(()) } -#[test] -fn default_permissions_profile_populates_runtime_sandbox_policy() -> std::io::Result<()> { +#[tokio::test] +async fn default_permissions_profile_populates_runtime_sandbox_policy() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cwd = TempDir::new()?; std::fs::create_dir_all(cwd.path().join("docs"))?; @@ -595,7 +604,8 @@ fn default_permissions_profile_populates_runtime_sandbox_policy() -> std::io::Re ..Default::default() }, codex_home.abs(), - )?; + ) + .await?; let memories_root = codex_home.path().join("memories").abs(); assert_eq!( @@ -647,8 +657,8 @@ fn default_permissions_profile_populates_runtime_sandbox_policy() -> std::io::Re Ok(()) } -#[test] -fn permissions_profiles_require_default_permissions() -> std::io::Result<()> { +#[tokio::test] +async fn permissions_profiles_require_default_permissions() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cwd = TempDir::new()?; std::fs::write(cwd.path().join(".git"), "gitdir: nowhere")?; @@ -677,6 +687,7 @@ fn permissions_profiles_require_default_permissions() -> std::io::Result<()> { }, codex_home.abs(), ) + .await .expect_err("missing default_permissions should be rejected"); assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); @@ -687,8 +698,8 @@ fn permissions_profiles_require_default_permissions() -> std::io::Result<()> { Ok(()) } -#[test] -fn permissions_profiles_reject_writes_outside_workspace_root() -> std::io::Result<()> { +#[tokio::test] +async fn permissions_profiles_reject_writes_outside_workspace_root() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cwd = TempDir::new()?; std::fs::write(cwd.path().join(".git"), "gitdir: nowhere")?; @@ -719,6 +730,7 @@ fn permissions_profiles_reject_writes_outside_workspace_root() -> std::io::Resul }, codex_home.abs(), ) + .await .expect_err("writes outside the workspace root should be rejected"); assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); @@ -730,8 +742,8 @@ fn permissions_profiles_reject_writes_outside_workspace_root() -> std::io::Resul Ok(()) } -#[test] -fn permissions_profiles_reject_nested_entries_for_non_project_roots() -> std::io::Result<()> { +#[tokio::test] +async fn permissions_profiles_reject_nested_entries_for_non_project_roots() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cwd = TempDir::new()?; std::fs::write(cwd.path().join(".git"), "gitdir: nowhere")?; @@ -764,6 +776,7 @@ fn permissions_profiles_reject_nested_entries_for_non_project_roots() -> std::io }, codex_home.abs(), ) + .await .expect_err("nested entries outside :project_roots should be rejected"); assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); @@ -774,7 +787,9 @@ fn permissions_profiles_reject_nested_entries_for_non_project_roots() -> std::io Ok(()) } -fn load_workspace_permission_profile(profile: PermissionProfileToml) -> std::io::Result { +async fn load_workspace_permission_profile( + profile: PermissionProfileToml, +) -> std::io::Result { let codex_home = TempDir::new()?; let cwd = TempDir::new()?; std::fs::write(cwd.path().join(".git"), "gitdir: nowhere")?; @@ -793,10 +808,11 @@ fn load_workspace_permission_profile(profile: PermissionProfileToml) -> std::io: }, codex_home.abs(), ) + .await } -#[test] -fn permissions_profiles_allow_unknown_special_paths() -> std::io::Result<()> { +#[tokio::test] +async fn permissions_profiles_allow_unknown_special_paths() -> std::io::Result<()> { let config = load_workspace_permission_profile(PermissionProfileToml { filesystem: Some(FilesystemPermissionsToml { entries: BTreeMap::from([( @@ -805,7 +821,8 @@ fn permissions_profiles_allow_unknown_special_paths() -> std::io::Result<()> { )]), }), network: None, - })?; + }) + .await?; assert_eq!( config.permissions.file_system_sandbox_policy, @@ -839,8 +856,9 @@ fn permissions_profiles_allow_unknown_special_paths() -> std::io::Result<()> { Ok(()) } -#[test] -fn permissions_profiles_allow_unknown_special_paths_with_nested_entries() -> std::io::Result<()> { +#[tokio::test] +async fn permissions_profiles_allow_unknown_special_paths_with_nested_entries() +-> std::io::Result<()> { let config = load_workspace_permission_profile(PermissionProfileToml { filesystem: Some(FilesystemPermissionsToml { entries: BTreeMap::from([( @@ -852,7 +870,8 @@ fn permissions_profiles_allow_unknown_special_paths_with_nested_entries() -> std )]), }), network: None, - })?; + }) + .await?; assert_eq!( config.permissions.file_system_sandbox_policy, @@ -873,12 +892,13 @@ fn permissions_profiles_allow_unknown_special_paths_with_nested_entries() -> std Ok(()) } -#[test] -fn permissions_profiles_allow_missing_filesystem_with_warning() -> std::io::Result<()> { +#[tokio::test] +async fn permissions_profiles_allow_missing_filesystem_with_warning() -> std::io::Result<()> { let config = load_workspace_permission_profile(PermissionProfileToml { filesystem: None, network: None, - })?; + }) + .await?; assert_eq!( config.permissions.file_system_sandbox_policy, @@ -904,14 +924,15 @@ fn permissions_profiles_allow_missing_filesystem_with_warning() -> std::io::Resu Ok(()) } -#[test] -fn permissions_profiles_allow_empty_filesystem_with_warning() -> std::io::Result<()> { +#[tokio::test] +async fn permissions_profiles_allow_empty_filesystem_with_warning() -> std::io::Result<()> { let config = load_workspace_permission_profile(PermissionProfileToml { filesystem: Some(FilesystemPermissionsToml { entries: BTreeMap::new(), }), network: None, - })?; + }) + .await?; assert_eq!( config.permissions.file_system_sandbox_policy, @@ -927,8 +948,8 @@ fn permissions_profiles_allow_empty_filesystem_with_warning() -> std::io::Result Ok(()) } -#[test] -fn permissions_profiles_reject_project_root_parent_traversal() -> std::io::Result<()> { +#[tokio::test] +async fn permissions_profiles_reject_project_root_parent_traversal() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cwd = TempDir::new()?; std::fs::write(cwd.path().join(".git"), "gitdir: nowhere")?; @@ -961,6 +982,7 @@ fn permissions_profiles_reject_project_root_parent_traversal() -> std::io::Resul }, codex_home.abs(), ) + .await .expect_err("parent traversal should be rejected for project root subpaths"); assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); @@ -971,8 +993,8 @@ fn permissions_profiles_reject_project_root_parent_traversal() -> std::io::Resul Ok(()) } -#[test] -fn permissions_profiles_allow_network_enablement() -> std::io::Result<()> { +#[tokio::test] +async fn permissions_profiles_allow_network_enablement() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cwd = TempDir::new()?; std::fs::write(cwd.path().join(".git"), "gitdir: nowhere")?; @@ -1004,7 +1026,8 @@ fn permissions_profiles_allow_network_enablement() -> std::io::Result<()> { ..Default::default() }, codex_home.abs(), - )?; + ) + .await?; assert!( config.permissions.network_sandbox_policy.is_enabled(), @@ -1067,8 +1090,8 @@ fn tui_config_missing_notifications_field_defaults_to_enabled() { ); } -#[test] -fn test_sandbox_config_parsing() { +#[tokio::test] +async fn test_sandbox_config_parsing() { let sandbox_full_access = r#" sandbox_mode = "danger-full-access" @@ -1078,13 +1101,15 @@ network_access = false # This should be ignored. let sandbox_full_access_cfg = toml::from_str::(sandbox_full_access) .expect("TOML deserialization should succeed"); let sandbox_mode_override = None; - let resolution = sandbox_full_access_cfg.derive_sandbox_policy( - sandbox_mode_override, - /*profile_sandbox_mode*/ None, - WindowsSandboxLevel::Disabled, - &PathBuf::from("/tmp/test"), - /*sandbox_policy_constraint*/ None, - ); + let resolution = sandbox_full_access_cfg + .derive_sandbox_policy( + sandbox_mode_override, + /*profile_sandbox_mode*/ None, + WindowsSandboxLevel::Disabled, + &PathBuf::from("/tmp/test"), + /*sandbox_policy_constraint*/ None, + ) + .await; assert_eq!(resolution, SandboxPolicy::DangerFullAccess); let sandbox_read_only = r#" @@ -1097,13 +1122,15 @@ network_access = true # This should be ignored. let sandbox_read_only_cfg = toml::from_str::(sandbox_read_only) .expect("TOML deserialization should succeed"); let sandbox_mode_override = None; - let resolution = sandbox_read_only_cfg.derive_sandbox_policy( - sandbox_mode_override, - /*profile_sandbox_mode*/ None, - WindowsSandboxLevel::Disabled, - &PathBuf::from("/tmp/test"), - /*sandbox_policy_constraint*/ None, - ); + let resolution = sandbox_read_only_cfg + .derive_sandbox_policy( + sandbox_mode_override, + /*profile_sandbox_mode*/ None, + WindowsSandboxLevel::Disabled, + &PathBuf::from("/tmp/test"), + /*sandbox_policy_constraint*/ None, + ) + .await; assert_eq!(resolution, SandboxPolicy::new_read_only_policy()); let writable_root = test_absolute_path("/my/workspace"); @@ -1124,13 +1151,15 @@ exclude_slash_tmp = true let sandbox_workspace_write_cfg = toml::from_str::(&sandbox_workspace_write) .expect("TOML deserialization should succeed"); let sandbox_mode_override = None; - let resolution = sandbox_workspace_write_cfg.derive_sandbox_policy( - sandbox_mode_override, - /*profile_sandbox_mode*/ None, - WindowsSandboxLevel::Disabled, - &PathBuf::from("/tmp/test"), - /*sandbox_policy_constraint*/ None, - ); + let resolution = sandbox_workspace_write_cfg + .derive_sandbox_policy( + sandbox_mode_override, + /*profile_sandbox_mode*/ None, + WindowsSandboxLevel::Disabled, + &PathBuf::from("/tmp/test"), + /*sandbox_policy_constraint*/ None, + ) + .await; if cfg!(target_os = "windows") { assert_eq!(resolution, SandboxPolicy::new_read_only_policy()); } else { @@ -1166,13 +1195,15 @@ trust_level = "trusted" let sandbox_workspace_write_cfg = toml::from_str::(&sandbox_workspace_write) .expect("TOML deserialization should succeed"); let sandbox_mode_override = None; - let resolution = sandbox_workspace_write_cfg.derive_sandbox_policy( - sandbox_mode_override, - /*profile_sandbox_mode*/ None, - WindowsSandboxLevel::Disabled, - &PathBuf::from("/tmp/test"), - /*sandbox_policy_constraint*/ None, - ); + let resolution = sandbox_workspace_write_cfg + .derive_sandbox_policy( + sandbox_mode_override, + /*profile_sandbox_mode*/ None, + WindowsSandboxLevel::Disabled, + &PathBuf::from("/tmp/test"), + /*sandbox_policy_constraint*/ None, + ) + .await; if cfg!(target_os = "windows") { assert_eq!(resolution, SandboxPolicy::new_read_only_policy()); } else { @@ -1189,8 +1220,8 @@ trust_level = "trusted" } } -#[test] -fn legacy_sandbox_mode_config_builds_split_policies_without_drift() -> std::io::Result<()> { +#[tokio::test] +async fn legacy_sandbox_mode_config_builds_split_policies_without_drift() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cwd = TempDir::new()?; let extra_root = test_absolute_path("/tmp/legacy-extra-root"); @@ -1232,7 +1263,8 @@ exclude_slash_tmp = true ..Default::default() }, codex_home.abs(), - )?; + ) + .await?; let sandbox_policy = config.permissions.sandbox_policy.get(); assert_eq!( @@ -1393,8 +1425,8 @@ fn filter_mcp_servers_by_allowlist_blocks_all_when_empty() { ); } -#[test] -fn add_dir_override_extends_workspace_writable_roots() -> std::io::Result<()> { +#[tokio::test] +async fn add_dir_override_extends_workspace_writable_roots() -> std::io::Result<()> { let temp_dir = TempDir::new()?; let frontend = temp_dir.path().join("frontend"); let backend = temp_dir.path().join("backend"); @@ -1412,7 +1444,8 @@ fn add_dir_override_extends_workspace_writable_roots() -> std::io::Result<()> { ConfigToml::default(), overrides, temp_dir.path().abs(), - )?; + ) + .await?; let expected_backend = backend.abs(); if cfg!(target_os = "windows") { @@ -1440,8 +1473,8 @@ fn add_dir_override_extends_workspace_writable_roots() -> std::io::Result<()> { Ok(()) } -#[test] -fn sqlite_home_defaults_to_codex_home_for_workspace_write() -> std::io::Result<()> { +#[tokio::test] +async fn sqlite_home_defaults_to_codex_home_for_workspace_write() -> std::io::Result<()> { let codex_home = TempDir::new()?; let config = Config::load_from_base_config_with_overrides( ConfigToml::default(), @@ -1450,15 +1483,16 @@ fn sqlite_home_defaults_to_codex_home_for_workspace_write() -> std::io::Result<( ..Default::default() }, codex_home.abs(), - )?; + ) + .await?; assert_eq!(config.sqlite_home, codex_home.path().to_path_buf()); Ok(()) } -#[test] -fn workspace_write_always_includes_memories_root_once() -> std::io::Result<()> { +#[tokio::test] +async fn workspace_write_always_includes_memories_root_once() -> std::io::Result<()> { let codex_home = TempDir::new()?; let memories_root = codex_home.path().join("memories"); let config = Config::load_from_base_config_with_overrides( @@ -1474,7 +1508,8 @@ fn workspace_write_always_includes_memories_root_once() -> std::io::Result<()> { ..Default::default() }, codex_home.abs(), - )?; + ) + .await?; if cfg!(target_os = "windows") { match config.permissions.sandbox_policy.get() { @@ -1507,8 +1542,8 @@ fn workspace_write_always_includes_memories_root_once() -> std::io::Result<()> { Ok(()) } -#[test] -fn config_defaults_to_file_cli_auth_store_mode() -> std::io::Result<()> { +#[tokio::test] +async fn config_defaults_to_file_cli_auth_store_mode() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cfg = ConfigToml::default(); @@ -1516,7 +1551,8 @@ fn config_defaults_to_file_cli_auth_store_mode() -> std::io::Result<()> { cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.cli_auth_credentials_store_mode, @@ -1526,8 +1562,8 @@ fn config_defaults_to_file_cli_auth_store_mode() -> std::io::Result<()> { Ok(()) } -#[test] -fn config_resolves_explicit_keyring_auth_store_mode() -> std::io::Result<()> { +#[tokio::test] +async fn config_resolves_explicit_keyring_auth_store_mode() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cfg = ConfigToml { cli_auth_credentials_store: Some(AuthCredentialsStoreMode::Keyring), @@ -1538,7 +1574,8 @@ fn config_resolves_explicit_keyring_auth_store_mode() -> std::io::Result<()> { cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.cli_auth_credentials_store_mode, @@ -1551,8 +1588,8 @@ fn config_resolves_explicit_keyring_auth_store_mode() -> std::io::Result<()> { Ok(()) } -#[test] -fn config_resolves_default_oauth_store_mode() -> std::io::Result<()> { +#[tokio::test] +async fn config_resolves_default_oauth_store_mode() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cfg = ConfigToml::default(); @@ -1560,7 +1597,8 @@ fn config_resolves_default_oauth_store_mode() -> std::io::Result<()> { cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.mcp_oauth_credentials_store_mode, @@ -1624,8 +1662,8 @@ fn local_dev_builds_force_file_mcp_oauth_store_modes() { ); } -#[test] -fn feedback_enabled_defaults_to_true() -> std::io::Result<()> { +#[tokio::test] +async fn feedback_enabled_defaults_to_true() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cfg = ConfigToml { feedback: Some(FeedbackConfigToml::default()), @@ -1636,7 +1674,8 @@ fn feedback_enabled_defaults_to_true() -> std::io::Result<()> { cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!(config.feedback_enabled, true); @@ -1776,8 +1815,8 @@ profile = "project" Ok(()) } -#[test] -fn profile_sandbox_mode_overrides_base() -> std::io::Result<()> { +#[tokio::test] +async fn profile_sandbox_mode_overrides_base() -> std::io::Result<()> { let codex_home = TempDir::new()?; let mut profiles = HashMap::new(); profiles.insert( @@ -1798,7 +1837,8 @@ fn profile_sandbox_mode_overrides_base() -> std::io::Result<()> { cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert!(matches!( config.permissions.sandbox_policy.get(), @@ -1808,8 +1848,8 @@ fn profile_sandbox_mode_overrides_base() -> std::io::Result<()> { Ok(()) } -#[test] -fn cli_override_takes_precedence_over_profile_sandbox_mode() -> std::io::Result<()> { +#[tokio::test] +async fn cli_override_takes_precedence_over_profile_sandbox_mode() -> std::io::Result<()> { let codex_home = TempDir::new()?; let mut profiles = HashMap::new(); profiles.insert( @@ -1830,7 +1870,8 @@ fn cli_override_takes_precedence_over_profile_sandbox_mode() -> std::io::Result< ..Default::default() }; - let config = Config::load_from_base_config_with_overrides(cfg, overrides, codex_home.abs())?; + let config = + Config::load_from_base_config_with_overrides(cfg, overrides, codex_home.abs()).await?; if cfg!(target_os = "windows") { assert!(matches!( @@ -1847,8 +1888,8 @@ fn cli_override_takes_precedence_over_profile_sandbox_mode() -> std::io::Result< Ok(()) } -#[test] -fn feature_table_overrides_legacy_flags() -> std::io::Result<()> { +#[tokio::test] +async fn feature_table_overrides_legacy_flags() -> std::io::Result<()> { let codex_home = TempDir::new()?; let mut entries = BTreeMap::new(); entries.insert("apply_patch_freeform".to_string(), false); @@ -1861,7 +1902,8 @@ fn feature_table_overrides_legacy_flags() -> std::io::Result<()> { cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert!(!config.features.enabled(Feature::ApplyPatchFreeform)); assert!(!config.include_apply_patch_tool); @@ -1869,8 +1911,8 @@ fn feature_table_overrides_legacy_flags() -> std::io::Result<()> { Ok(()) } -#[test] -fn legacy_toggles_map_to_features() -> std::io::Result<()> { +#[tokio::test] +async fn legacy_toggles_map_to_features() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cfg = ConfigToml { experimental_use_unified_exec_tool: Some(true), @@ -1882,7 +1924,8 @@ fn legacy_toggles_map_to_features() -> std::io::Result<()> { cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert!(config.features.enabled(Feature::ApplyPatchFreeform)); assert!(config.features.enabled(Feature::UnifiedExec)); @@ -1894,8 +1937,8 @@ fn legacy_toggles_map_to_features() -> std::io::Result<()> { Ok(()) } -#[test] -fn responses_websocket_features_do_not_change_wire_api() -> std::io::Result<()> { +#[tokio::test] +async fn responses_websocket_features_do_not_change_wire_api() -> std::io::Result<()> { for feature_key in ["responses_websockets", "responses_websockets_v2"] { let codex_home = TempDir::new()?; let mut entries = BTreeMap::new(); @@ -1909,7 +1952,8 @@ fn responses_websocket_features_do_not_change_wire_api() -> std::io::Result<()> cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!(config.model_provider.wire_api, WireApi::Responses); } @@ -1917,8 +1961,8 @@ fn responses_websocket_features_do_not_change_wire_api() -> std::io::Result<()> Ok(()) } -#[test] -fn config_honors_explicit_file_oauth_store_mode() -> std::io::Result<()> { +#[tokio::test] +async fn config_honors_explicit_file_oauth_store_mode() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cfg = ConfigToml { mcp_oauth_credentials_store: Some(OAuthCredentialsStoreMode::File), @@ -1929,7 +1973,8 @@ fn config_honors_explicit_file_oauth_store_mode() -> std::io::Result<()> { cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.mcp_oauth_credentials_store_mode, @@ -1974,7 +2019,8 @@ async fn managed_config_overrides_oauth_store_mode() -> anyhow::Result<()> { cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( final_config.mcp_oauth_credentials_store_mode, resolve_mcp_oauth_credentials_store_mode( @@ -2199,7 +2245,8 @@ async fn to_mcp_config_preserves_apps_feature_from_config() -> std::io::Result<( ConfigToml::default(), ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; let plugins_manager = PluginsManager::new(codex_home.path().to_path_buf()); let mcp_config = config.to_mcp_config(&plugins_manager).await; @@ -3248,8 +3295,8 @@ impl PrecedenceTestFixture { } } -#[test] -fn cli_override_sets_compact_prompt() -> std::io::Result<()> { +#[tokio::test] +async fn cli_override_sets_compact_prompt() -> std::io::Result<()> { let codex_home = TempDir::new()?; let overrides = ConfigOverrides { compact_prompt: Some("Use the compact override".to_string()), @@ -3260,7 +3307,8 @@ fn cli_override_sets_compact_prompt() -> std::io::Result<()> { ConfigToml::default(), overrides, codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.compact_prompt.as_deref(), @@ -3270,8 +3318,8 @@ fn cli_override_sets_compact_prompt() -> std::io::Result<()> { Ok(()) } -#[test] -fn loads_compact_prompt_from_file() -> std::io::Result<()> { +#[tokio::test] +async fn loads_compact_prompt_from_file() -> std::io::Result<()> { let codex_home = TempDir::new()?; let workspace = codex_home.path().join("workspace"); std::fs::create_dir_all(&workspace)?; @@ -3289,7 +3337,8 @@ fn loads_compact_prompt_from_file() -> std::io::Result<()> { ..Default::default() }; - let config = Config::load_from_base_config_with_overrides(cfg, overrides, codex_home.abs())?; + let config = + Config::load_from_base_config_with_overrides(cfg, overrides, codex_home.abs()).await?; assert_eq!( config.compact_prompt.as_deref(), @@ -3299,8 +3348,8 @@ fn loads_compact_prompt_from_file() -> std::io::Result<()> { Ok(()) } -#[test] -fn load_config_uses_requirements_guardian_policy_config() -> std::io::Result<()> { +#[tokio::test] +async fn load_config_uses_requirements_guardian_policy_config() -> std::io::Result<()> { let codex_home = TempDir::new()?; let config_layer_stack = ConfigLayerStack::new( Vec::new(), @@ -3322,7 +3371,8 @@ fn load_config_uses_requirements_guardian_policy_config() -> std::io::Result<()> }, codex_home.abs(), config_layer_stack, - )?; + ) + .await?; assert_eq!( config.guardian_policy_config.as_deref(), @@ -3332,8 +3382,8 @@ fn load_config_uses_requirements_guardian_policy_config() -> std::io::Result<()> Ok(()) } -#[test] -fn load_config_ignores_empty_requirements_guardian_policy_config() -> std::io::Result<()> { +#[tokio::test] +async fn load_config_ignores_empty_requirements_guardian_policy_config() -> std::io::Result<()> { let codex_home = TempDir::new()?; let config_layer_stack = ConfigLayerStack::new( Vec::new(), @@ -3353,15 +3403,16 @@ fn load_config_ignores_empty_requirements_guardian_policy_config() -> std::io::R }, codex_home.abs(), config_layer_stack, - )?; + ) + .await?; assert_eq!(config.guardian_policy_config, None); Ok(()) } -#[test] -fn load_config_rejects_missing_agent_role_config_file() -> std::io::Result<()> { +#[tokio::test] +async fn load_config_rejects_missing_agent_role_config_file() -> std::io::Result<()> { let codex_home = TempDir::new()?; let missing_path = codex_home.path().join("agents").join("researcher.toml"); let cfg = ConfigToml { @@ -3385,7 +3436,8 @@ fn load_config_rejects_missing_agent_role_config_file() -> std::io::Result<()> { cfg, ConfigOverrides::default(), codex_home.abs(), - ); + ) + .await; let err = result.expect_err("missing role config file should be rejected"); assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); let message = err.to_string(); @@ -4226,8 +4278,8 @@ model = "gpt-5-mini" Ok(()) } -#[test] -fn load_config_normalizes_agent_role_nickname_candidates() -> std::io::Result<()> { +#[tokio::test] +async fn load_config_normalizes_agent_role_nickname_candidates() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cfg = ConfigToml { agents: Some(AgentsToml { @@ -4253,7 +4305,8 @@ fn load_config_normalizes_agent_role_nickname_candidates() -> std::io::Result<() cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config @@ -4267,8 +4320,8 @@ fn load_config_normalizes_agent_role_nickname_candidates() -> std::io::Result<() Ok(()) } -#[test] -fn load_config_rejects_empty_agent_role_nickname_candidates() -> std::io::Result<()> { +#[tokio::test] +async fn load_config_rejects_empty_agent_role_nickname_candidates() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cfg = ConfigToml { agents: Some(AgentsToml { @@ -4291,7 +4344,8 @@ fn load_config_rejects_empty_agent_role_nickname_candidates() -> std::io::Result cfg, ConfigOverrides::default(), codex_home.abs(), - ); + ) + .await; let err = result.expect_err("empty nickname candidates should be rejected"); assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); assert!( @@ -4302,8 +4356,8 @@ fn load_config_rejects_empty_agent_role_nickname_candidates() -> std::io::Result Ok(()) } -#[test] -fn load_config_rejects_duplicate_agent_role_nickname_candidates() -> std::io::Result<()> { +#[tokio::test] +async fn load_config_rejects_duplicate_agent_role_nickname_candidates() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cfg = ConfigToml { agents: Some(AgentsToml { @@ -4326,7 +4380,8 @@ fn load_config_rejects_duplicate_agent_role_nickname_candidates() -> std::io::Re cfg, ConfigOverrides::default(), codex_home.abs(), - ); + ) + .await; let err = result.expect_err("duplicate nickname candidates should be rejected"); assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); assert!( @@ -4337,8 +4392,8 @@ fn load_config_rejects_duplicate_agent_role_nickname_candidates() -> std::io::Re Ok(()) } -#[test] -fn load_config_rejects_unsafe_agent_role_nickname_candidates() -> std::io::Result<()> { +#[tokio::test] +async fn load_config_rejects_unsafe_agent_role_nickname_candidates() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cfg = ConfigToml { agents: Some(AgentsToml { @@ -4361,7 +4416,8 @@ fn load_config_rejects_unsafe_agent_role_nickname_candidates() -> std::io::Resul cfg, ConfigOverrides::default(), codex_home.abs(), - ); + ) + .await; let err = result.expect_err("unsafe nickname candidates should be rejected"); assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); assert!(err.to_string().contains( @@ -4371,8 +4427,8 @@ fn load_config_rejects_unsafe_agent_role_nickname_candidates() -> std::io::Resul Ok(()) } -#[test] -fn model_catalog_json_loads_from_path() -> std::io::Result<()> { +#[tokio::test] +async fn model_catalog_json_loads_from_path() -> std::io::Result<()> { let codex_home = TempDir::new()?; let catalog_path = codex_home.path().join("catalog.json"); let mut catalog = bundled_models_response() @@ -4392,14 +4448,15 @@ fn model_catalog_json_loads_from_path() -> std::io::Result<()> { cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!(config.model_catalog, Some(catalog)); Ok(()) } -#[test] -fn model_catalog_json_rejects_empty_catalog() -> std::io::Result<()> { +#[tokio::test] +async fn model_catalog_json_rejects_empty_catalog() -> std::io::Result<()> { let codex_home = TempDir::new()?; let catalog_path = codex_home.path().join("catalog.json"); std::fs::write(&catalog_path, r#"{"models":[]}"#)?; @@ -4414,6 +4471,7 @@ fn model_catalog_json_rejects_empty_catalog() -> std::io::Result<()> { ConfigOverrides::default(), codex_home.abs(), ) + .await .expect_err("empty custom catalog should fail config load"); assert_eq!(err.kind(), ErrorKind::InvalidData); @@ -4538,8 +4596,8 @@ model_verbosity = "high" /// /// Note that profiles are the recommended way to specify a group of /// configuration options together. -#[test] -fn test_precedence_fixture_with_o3_profile() -> std::io::Result<()> { +#[tokio::test] +async fn test_precedence_fixture_with_o3_profile() -> std::io::Result<()> { let fixture = create_test_fixture()?; let o3_profile_overrides = ConfigOverrides { @@ -4551,7 +4609,8 @@ fn test_precedence_fixture_with_o3_profile() -> std::io::Result<()> { fixture.cfg.clone(), o3_profile_overrides, fixture.codex_home(), - )?; + ) + .await?; assert_eq!( Config { model: Some("o3".to_string()), @@ -4671,8 +4730,8 @@ fn test_precedence_fixture_with_o3_profile() -> std::io::Result<()> { Ok(()) } -#[test] -fn metrics_exporter_defaults_to_statsig_when_missing() -> std::io::Result<()> { +#[tokio::test] +async fn metrics_exporter_defaults_to_statsig_when_missing() -> std::io::Result<()> { let fixture = create_test_fixture()?; let config = Config::load_from_base_config_with_overrides( @@ -4682,14 +4741,15 @@ fn metrics_exporter_defaults_to_statsig_when_missing() -> std::io::Result<()> { ..Default::default() }, fixture.codex_home(), - )?; + ) + .await?; assert_eq!(config.otel.metrics_exporter, OtelExporterKind::Statsig); Ok(()) } -#[test] -fn test_precedence_fixture_with_gpt3_profile() -> std::io::Result<()> { +#[tokio::test] +async fn test_precedence_fixture_with_gpt3_profile() -> std::io::Result<()> { let fixture = create_test_fixture()?; let gpt3_profile_overrides = ConfigOverrides { @@ -4701,7 +4761,8 @@ fn test_precedence_fixture_with_gpt3_profile() -> std::io::Result<()> { fixture.cfg.clone(), gpt3_profile_overrides, fixture.codex_home(), - )?; + ) + .await?; let expected_gpt3_profile_config = Config { model: Some("gpt-3.5-turbo".to_string()), review_model: None, @@ -4829,14 +4890,15 @@ fn test_precedence_fixture_with_gpt3_profile() -> std::io::Result<()> { fixture.cfg.clone(), default_profile_overrides, fixture.codex_home(), - )?; + ) + .await?; assert_eq!(expected_gpt3_profile_config, default_profile_config); Ok(()) } -#[test] -fn test_precedence_fixture_with_zdr_profile() -> std::io::Result<()> { +#[tokio::test] +async fn test_precedence_fixture_with_zdr_profile() -> std::io::Result<()> { let fixture = create_test_fixture()?; let zdr_profile_overrides = ConfigOverrides { @@ -4848,7 +4910,8 @@ fn test_precedence_fixture_with_zdr_profile() -> std::io::Result<()> { fixture.cfg.clone(), zdr_profile_overrides, fixture.codex_home(), - )?; + ) + .await?; let expected_zdr_profile_config = Config { model: Some("o3".to_string()), review_model: None, @@ -4968,8 +5031,8 @@ fn test_precedence_fixture_with_zdr_profile() -> std::io::Result<()> { Ok(()) } -#[test] -fn test_precedence_fixture_with_gpt5_profile() -> std::io::Result<()> { +#[tokio::test] +async fn test_precedence_fixture_with_gpt5_profile() -> std::io::Result<()> { let fixture = create_test_fixture()?; let gpt5_profile_overrides = ConfigOverrides { @@ -4981,7 +5044,8 @@ fn test_precedence_fixture_with_gpt5_profile() -> std::io::Result<()> { fixture.cfg.clone(), gpt5_profile_overrides, fixture.codex_home(), - )?; + ) + .await?; let expected_gpt5_profile_config = Config { model: Some("gpt-5.1".to_string()), review_model: None, @@ -5101,8 +5165,9 @@ fn test_precedence_fixture_with_gpt5_profile() -> std::io::Result<()> { Ok(()) } -#[test] -fn test_requirements_web_search_mode_allowlist_does_not_warn_when_unset() -> anyhow::Result<()> { +#[tokio::test] +async fn test_requirements_web_search_mode_allowlist_does_not_warn_when_unset() -> anyhow::Result<()> +{ let fixture = create_test_fixture()?; let requirements_toml = crate::config_loader::ConfigRequirementsToml { @@ -5154,7 +5219,8 @@ fn test_requirements_web_search_mode_allowlist_does_not_warn_when_unset() -> any }, fixture.codex_home(), config_layer_stack, - )?; + ) + .await?; assert!( !config @@ -5317,9 +5383,9 @@ fn test_set_default_oss_provider_rejects_legacy_ollama_chat_provider() -> std::i Ok(()) } -#[test] -fn test_load_config_rejects_legacy_ollama_chat_provider_with_helpful_error() -> std::io::Result<()> -{ +#[tokio::test] +async fn test_load_config_rejects_legacy_ollama_chat_provider_with_helpful_error() +-> std::io::Result<()> { let codex_home = TempDir::new()?; let cfg = ConfigToml { model_provider: Some(LEGACY_OLLAMA_CHAT_PROVIDER_ID.to_string()), @@ -5330,7 +5396,8 @@ fn test_load_config_rejects_legacy_ollama_chat_provider_with_helpful_error() -> cfg, ConfigOverrides::default(), codex_home.abs(), - ); + ) + .await; assert!(result.is_err()); let error = result.unwrap_err(); assert_eq!(error.kind(), std::io::ErrorKind::NotFound); @@ -5343,8 +5410,8 @@ fn test_load_config_rejects_legacy_ollama_chat_provider_with_helpful_error() -> Ok(()) } -#[test] -fn test_untrusted_project_gets_workspace_write_sandbox() -> anyhow::Result<()> { +#[tokio::test] +async fn test_untrusted_project_gets_workspace_write_sandbox() -> anyhow::Result<()> { let config_with_untrusted = r#" [projects."/tmp/test"] trust_level = "untrusted" @@ -5353,13 +5420,15 @@ trust_level = "untrusted" let cfg = toml::from_str::(config_with_untrusted) .expect("TOML deserialization should succeed"); - let resolution = cfg.derive_sandbox_policy( - /*sandbox_mode_override*/ None, - /*profile_sandbox_mode*/ None, - WindowsSandboxLevel::Disabled, - &PathBuf::from("/tmp/test"), - /*sandbox_policy_constraint*/ None, - ); + let resolution = cfg + .derive_sandbox_policy( + /*sandbox_mode_override*/ None, + /*profile_sandbox_mode*/ None, + WindowsSandboxLevel::Disabled, + &PathBuf::from("/tmp/test"), + /*sandbox_policy_constraint*/ None, + ) + .await; // Verify that untrusted projects get WorkspaceWrite (or ReadOnly on Windows due to downgrade) if cfg!(target_os = "windows") { @@ -5377,9 +5446,9 @@ trust_level = "untrusted" Ok(()) } -#[test] -fn derive_sandbox_policy_falls_back_to_constraint_value_for_implicit_defaults() -> anyhow::Result<()> -{ +#[tokio::test] +async fn derive_sandbox_policy_falls_back_to_constraint_value_for_implicit_defaults() +-> anyhow::Result<()> { let project_dir = TempDir::new()?; let project_path = project_dir.path().to_path_buf(); let project_key = project_path.to_string_lossy().to_string(); @@ -5405,21 +5474,23 @@ fn derive_sandbox_policy_falls_back_to_constraint_value_for_implicit_defaults() } })?; - let resolution = cfg.derive_sandbox_policy( - /*sandbox_mode_override*/ None, - /*profile_sandbox_mode*/ None, - WindowsSandboxLevel::Disabled, - &project_path, - Some(&constrained), - ); + let resolution = cfg + .derive_sandbox_policy( + /*sandbox_mode_override*/ None, + /*profile_sandbox_mode*/ None, + WindowsSandboxLevel::Disabled, + &project_path, + Some(&constrained), + ) + .await; assert_eq!(resolution, SandboxPolicy::DangerFullAccess); Ok(()) } -#[test] -fn derive_sandbox_policy_preserves_windows_downgrade_for_unsupported_fallback() -> anyhow::Result<()> -{ +#[tokio::test] +async fn derive_sandbox_policy_preserves_windows_downgrade_for_unsupported_fallback() +-> anyhow::Result<()> { let project_dir = TempDir::new()?; let project_path = project_dir.path().to_path_buf(); let project_key = project_path.to_string_lossy().to_string(); @@ -5445,13 +5516,15 @@ fn derive_sandbox_policy_preserves_windows_downgrade_for_unsupported_fallback() } })?; - let resolution = cfg.derive_sandbox_policy( - /*sandbox_mode_override*/ None, - /*profile_sandbox_mode*/ None, - WindowsSandboxLevel::Disabled, - &project_path, - Some(&constrained), - ); + let resolution = cfg + .derive_sandbox_policy( + /*sandbox_mode_override*/ None, + /*profile_sandbox_mode*/ None, + WindowsSandboxLevel::Disabled, + &project_path, + Some(&constrained), + ) + .await; if cfg!(target_os = "windows") { assert_eq!(resolution, SandboxPolicy::new_read_only_policy()); @@ -5579,8 +5652,8 @@ fn config_toml_deserializes_mcp_oauth_callback_url() { ); } -#[test] -fn config_loads_mcp_oauth_callback_port_from_toml() -> std::io::Result<()> { +#[tokio::test] +async fn config_loads_mcp_oauth_callback_port_from_toml() -> std::io::Result<()> { let codex_home = TempDir::new()?; let toml = r#" model = "gpt-5.1" @@ -5593,14 +5666,15 @@ mcp_oauth_callback_port = 5678 cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!(config.mcp_oauth_callback_port, Some(5678)); Ok(()) } -#[test] -fn config_loads_allow_login_shell_from_toml() -> std::io::Result<()> { +#[tokio::test] +async fn config_loads_allow_login_shell_from_toml() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cfg: ConfigToml = toml::from_str( r#" @@ -5614,14 +5688,15 @@ allow_login_shell = false cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert!(!config.permissions.allow_login_shell); Ok(()) } -#[test] -fn config_loads_mcp_oauth_callback_url_from_toml() -> std::io::Result<()> { +#[tokio::test] +async fn config_loads_mcp_oauth_callback_url_from_toml() -> std::io::Result<()> { let codex_home = TempDir::new()?; let toml = r#" model = "gpt-5.1" @@ -5634,7 +5709,8 @@ mcp_oauth_callback_url = "https://example.com/callback" cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.mcp_oauth_callback_url.as_deref(), @@ -5643,8 +5719,8 @@ mcp_oauth_callback_url = "https://example.com/callback" Ok(()) } -#[test] -fn test_untrusted_project_gets_unless_trusted_approval_policy() -> anyhow::Result<()> { +#[tokio::test] +async fn test_untrusted_project_gets_unless_trusted_approval_policy() -> anyhow::Result<()> { let codex_home = TempDir::new()?; let test_project_dir = TempDir::new()?; let test_path = test_project_dir.path(); @@ -5664,7 +5740,8 @@ fn test_untrusted_project_gets_unless_trusted_approval_policy() -> anyhow::Resul ..Default::default() }, codex_home.abs(), - )?; + ) + .await?; // Verify that untrusted projects get UnlessTrusted approval policy assert_eq!( @@ -6359,8 +6436,8 @@ async fn feature_requirements_reject_collab_legacy_alias() { ); } -#[test] -fn tool_suggest_discoverables_load_from_config_toml() -> std::io::Result<()> { +#[tokio::test] +async fn tool_suggest_discoverables_load_from_config_toml() -> std::io::Result<()> { let cfg: ConfigToml = toml::from_str( r#" [tool_suggest] @@ -6398,7 +6475,8 @@ discoverables = [ cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.tool_suggest, @@ -6418,8 +6496,8 @@ discoverables = [ Ok(()) } -#[test] -fn experimental_realtime_start_instructions_load_from_config_toml() -> std::io::Result<()> { +#[tokio::test] +async fn experimental_realtime_start_instructions_load_from_config_toml() -> std::io::Result<()> { let cfg: ConfigToml = toml::from_str( r#" experimental_realtime_start_instructions = "start instructions from config" @@ -6437,7 +6515,8 @@ experimental_realtime_start_instructions = "start instructions from config" cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.experimental_realtime_start_instructions.as_deref(), @@ -6446,8 +6525,8 @@ experimental_realtime_start_instructions = "start instructions from config" Ok(()) } -#[test] -fn experimental_realtime_ws_base_url_loads_from_config_toml() -> std::io::Result<()> { +#[tokio::test] +async fn experimental_realtime_ws_base_url_loads_from_config_toml() -> std::io::Result<()> { let cfg: ConfigToml = toml::from_str( r#" experimental_realtime_ws_base_url = "http://127.0.0.1:8011" @@ -6465,7 +6544,8 @@ experimental_realtime_ws_base_url = "http://127.0.0.1:8011" cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.experimental_realtime_ws_base_url.as_deref(), @@ -6474,8 +6554,8 @@ experimental_realtime_ws_base_url = "http://127.0.0.1:8011" Ok(()) } -#[test] -fn experimental_realtime_ws_backend_prompt_loads_from_config_toml() -> std::io::Result<()> { +#[tokio::test] +async fn experimental_realtime_ws_backend_prompt_loads_from_config_toml() -> std::io::Result<()> { let cfg: ConfigToml = toml::from_str( r#" experimental_realtime_ws_backend_prompt = "prompt from config" @@ -6493,7 +6573,8 @@ experimental_realtime_ws_backend_prompt = "prompt from config" cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.experimental_realtime_ws_backend_prompt.as_deref(), @@ -6502,8 +6583,8 @@ experimental_realtime_ws_backend_prompt = "prompt from config" Ok(()) } -#[test] -fn experimental_realtime_ws_startup_context_loads_from_config_toml() -> std::io::Result<()> { +#[tokio::test] +async fn experimental_realtime_ws_startup_context_loads_from_config_toml() -> std::io::Result<()> { let cfg: ConfigToml = toml::from_str( r#" experimental_realtime_ws_startup_context = "startup context from config" @@ -6521,7 +6602,8 @@ experimental_realtime_ws_startup_context = "startup context from config" cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.experimental_realtime_ws_startup_context.as_deref(), @@ -6530,8 +6612,8 @@ experimental_realtime_ws_startup_context = "startup context from config" Ok(()) } -#[test] -fn experimental_realtime_ws_model_loads_from_config_toml() -> std::io::Result<()> { +#[tokio::test] +async fn experimental_realtime_ws_model_loads_from_config_toml() -> std::io::Result<()> { let cfg: ConfigToml = toml::from_str( r#" experimental_realtime_ws_model = "realtime-test-model" @@ -6549,7 +6631,8 @@ experimental_realtime_ws_model = "realtime-test-model" cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.experimental_realtime_ws_model.as_deref(), @@ -6558,8 +6641,8 @@ experimental_realtime_ws_model = "realtime-test-model" Ok(()) } -#[test] -fn realtime_config_partial_table_uses_realtime_defaults() -> std::io::Result<()> { +#[tokio::test] +async fn realtime_config_partial_table_uses_realtime_defaults() -> std::io::Result<()> { let cfg: ConfigToml = toml::from_str( r#" [realtime] @@ -6573,7 +6656,8 @@ voice = "marin" cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.realtime, @@ -6585,8 +6669,8 @@ voice = "marin" Ok(()) } -#[test] -fn realtime_loads_from_config_toml() -> std::io::Result<()> { +#[tokio::test] +async fn realtime_loads_from_config_toml() -> std::io::Result<()> { let cfg: ConfigToml = toml::from_str( r#" [realtime] @@ -6613,7 +6697,8 @@ voice = "cedar" cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.realtime, @@ -6627,8 +6712,8 @@ voice = "cedar" Ok(()) } -#[test] -fn realtime_audio_loads_from_config_toml() -> std::io::Result<()> { +#[tokio::test] +async fn realtime_audio_loads_from_config_toml() -> std::io::Result<()> { let cfg: ConfigToml = toml::from_str( r#" [audio] @@ -6650,7 +6735,8 @@ speaker = "Desk Speakers" cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!(config.realtime_audio.microphone.as_deref(), Some("USB Mic")); assert_eq!( diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index 5faf4be480..adfbe45baa 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -169,13 +169,14 @@ fn resolve_mcp_oauth_credentials_store_mode( } #[cfg(test)] -pub(crate) fn test_config() -> Config { +pub(crate) async fn test_config() -> Config { let codex_home = tempfile::tempdir().expect("create temp dir"); Config::load_from_base_config_with_overrides( ConfigToml::default(), ConfigOverrides::default(), AbsolutePathBuf::from_absolute_path(codex_home.path()).expect("temp dir should resolve"), ) + .await .expect("load default test config") } @@ -726,6 +727,7 @@ impl ConfigBuilder { codex_home, config_layer_stack, ) + .await } #[cfg(test)] @@ -786,7 +788,7 @@ impl Config { } /// Load a default configuration when user config files are invalid. - pub fn load_default_with_cli_overrides( + pub async fn load_default_with_cli_overrides( cli_overrides: Vec<(String, TomlValue)>, ) -> std::io::Result { let codex_home = find_codex_home()?; @@ -794,11 +796,12 @@ impl Config { codex_home.to_path_buf(), cli_overrides, ) + .await } /// Load a default configuration for a specific Codex home without reading /// user, project, or system config layers. - pub fn load_default_with_cli_overrides_for_codex_home( + pub async fn load_default_with_cli_overrides_for_codex_home( codex_home: PathBuf, cli_overrides: Vec<(String, TomlValue)>, ) -> std::io::Result { @@ -818,6 +821,7 @@ impl Config { codex_home, ConfigLayerStack::default(), ) + .await } /// This is a secondary way of creating [Config], which is appropriate when @@ -1413,22 +1417,24 @@ pub(crate) fn resolve_web_search_mode_for_turn( impl Config { #[cfg(test)] - fn load_from_base_config_with_overrides( + async fn load_from_base_config_with_overrides( cfg: ConfigToml, overrides: ConfigOverrides, codex_home: AbsolutePathBuf, ) -> std::io::Result { // Note this ignores requirements.toml enforcement for tests. let config_layer_stack = ConfigLayerStack::default(); - Self::load_config_with_layer_stack(cfg, overrides, codex_home, config_layer_stack) + Self::load_config_with_layer_stack(cfg, overrides, codex_home, config_layer_stack).await } - pub(crate) fn load_config_with_layer_stack( + pub(crate) async fn load_config_with_layer_stack( cfg: ConfigToml, overrides: ConfigOverrides, codex_home: AbsolutePathBuf, config_layer_stack: ConfigLayerStack, ) -> std::io::Result { + // Keep the large config-construction future off small test thread stacks. + Box::pin(async move { validate_model_providers(&cfg.model_providers) .map_err(|message| std::io::Error::new(std::io::ErrorKind::InvalidInput, message))?; // Ensure that every field of ConfigRequirements is applied to the final @@ -1547,6 +1553,7 @@ impl Config { .collect(); let active_project = cfg .get_active_project(resolved_cwd.as_path()) + .await .unwrap_or(ProjectConfig { trust_level: None }); let permission_config_syntax = resolve_permission_config_syntax( &config_layer_stack, @@ -1636,13 +1643,15 @@ impl Config { ) } else { let configured_network_proxy_config = NetworkProxyConfig::default(); - let mut sandbox_policy = cfg.derive_sandbox_policy( - sandbox_mode, - config_profile.sandbox_mode, - windows_sandbox_level, - resolved_cwd.as_path(), - Some(&constrained_sandbox_policy), - ); + let mut sandbox_policy = cfg + .derive_sandbox_policy( + sandbox_mode, + config_profile.sandbox_mode, + windows_sandbox_level, + resolved_cwd.as_path(), + Some(&constrained_sandbox_policy), + ) + .await; if let SandboxPolicy::WorkspaceWrite { writable_roots, .. } = &mut sandbox_policy { for path in &additional_writable_roots { if !writable_roots.iter().any(|existing| existing == path) { @@ -2207,6 +2216,8 @@ impl Config { }, }; Ok(config) + }) + .await } fn load_instructions(codex_dir: Option<&AbsolutePathBuf>) -> Option { diff --git a/codex-rs/core/src/config/permissions_tests.rs b/codex-rs/core/src/config/permissions_tests.rs index 268346794c..34d22c2bfb 100644 --- a/codex-rs/core/src/config/permissions_tests.rs +++ b/codex-rs/core/src/config/permissions_tests.rs @@ -24,8 +24,8 @@ fn normalize_absolute_path_for_platform_simplifies_windows_verbatim_paths() { assert_eq!(parsed, PathBuf::from(r"D:\c\x\worktrees\2508\swift-base")); } -#[test] -fn restricted_read_implicitly_allows_helper_executables() -> std::io::Result<()> { +#[tokio::test] +async fn restricted_read_implicitly_allows_helper_executables() -> std::io::Result<()> { let temp_dir = TempDir::new()?; let cwd = temp_dir.path().join("workspace"); let codex_home = temp_dir.path().join(".codex"); @@ -64,7 +64,8 @@ fn restricted_read_implicitly_allows_helper_executables() -> std::io::Result<()> ..Default::default() }, AbsolutePathBuf::from_absolute_path(&codex_home)?, - )?; + ) + .await?; let expected_zsh = AbsolutePathBuf::try_from(zsh_path)?; let expected_allowed_arg0_dir = AbsolutePathBuf::try_from(allowed_arg0_dir)?; diff --git a/codex-rs/core/src/config_loader/mod.rs b/codex-rs/core/src/config_loader/mod.rs index 36fc956bf4..f7830b7089 100644 --- a/codex-rs/core/src/config_loader/mod.rs +++ b/codex-rs/core/src/config_loader/mod.rs @@ -650,7 +650,7 @@ async fn project_trust_context( let projects = project_trust_config.projects.unwrap_or_default(); let project_root_key = project_trust_key(project_root.as_path()); - let repo_root = resolve_root_git_project_for_trust(cwd.as_path()); + let repo_root = resolve_root_git_project_for_trust(cwd.as_path()).await; let repo_root_key = repo_root.as_ref().map(|root| project_trust_key(root)); let projects_trust = projects diff --git a/codex-rs/core/src/git_info_tests.rs b/codex-rs/core/src/git_info_tests.rs index 4cb50f02fd..e3ef1fa62f 100644 --- a/codex-rs/core/src/git_info_tests.rs +++ b/codex-rs/core/src/git_info_tests.rs @@ -426,10 +426,14 @@ async fn test_get_git_working_tree_state_branch_fallback() { assert_eq!(state.sha, GitSha::new(&remote_sha)); } -#[test] -fn resolve_root_git_project_for_trust_returns_none_outside_repo() { +#[tokio::test] +async fn resolve_root_git_project_for_trust_returns_none_outside_repo() { let tmp = TempDir::new().expect("tempdir"); - assert!(resolve_root_git_project_for_trust(tmp.path()).is_none()); + assert!( + resolve_root_git_project_for_trust(tmp.path()) + .await + .is_none() + ); } #[tokio::test] @@ -439,12 +443,15 @@ async fn resolve_root_git_project_for_trust_regular_repo_returns_repo_root() { let expected = std::fs::canonicalize(&repo_path).unwrap(); assert_eq!( - resolve_root_git_project_for_trust(&repo_path), + resolve_root_git_project_for_trust(&repo_path).await, Some(expected.clone()) ); let nested = repo_path.join("sub/dir"); std::fs::create_dir_all(&nested).unwrap(); - assert_eq!(resolve_root_git_project_for_trust(&nested), Some(expected)); + assert_eq!( + resolve_root_git_project_for_trust(&nested).await, + Some(expected) + ); } #[tokio::test] @@ -467,18 +474,20 @@ async fn resolve_root_git_project_for_trust_detects_worktree_and_returns_main_ro .expect("git worktree add"); let expected = std::fs::canonicalize(&repo_path).ok(); - let got = - resolve_root_git_project_for_trust(&wt_root).and_then(|p| std::fs::canonicalize(p).ok()); + let got = resolve_root_git_project_for_trust(&wt_root) + .await + .and_then(|p| std::fs::canonicalize(p).ok()); assert_eq!(got, expected); let nested = wt_root.join("nested/sub"); std::fs::create_dir_all(&nested).unwrap(); - let got_nested = - resolve_root_git_project_for_trust(&nested).and_then(|p| std::fs::canonicalize(p).ok()); + let got_nested = resolve_root_git_project_for_trust(&nested) + .await + .and_then(|p| std::fs::canonicalize(p).ok()); assert_eq!(got_nested, expected); } -#[test] -fn resolve_root_git_project_for_trust_detects_worktree_pointer_without_git_command() { +#[tokio::test] +async fn resolve_root_git_project_for_trust_detects_worktree_pointer_without_git_command() { let tmp = TempDir::new().expect("tempdir"); let repo_root = tmp.path().join("repo"); let common_dir = repo_root.join(".git"); @@ -495,17 +504,17 @@ fn resolve_root_git_project_for_trust_detects_worktree_pointer_without_git_comma let expected = std::fs::canonicalize(&repo_root).unwrap(); assert_eq!( - resolve_root_git_project_for_trust(&worktree_root), + resolve_root_git_project_for_trust(&worktree_root).await, Some(expected.clone()) ); assert_eq!( - resolve_root_git_project_for_trust(&worktree_root.join("nested")), + resolve_root_git_project_for_trust(&worktree_root.join("nested")).await, Some(expected) ); } -#[test] -fn resolve_root_git_project_for_trust_non_worktrees_gitdir_returns_none() { +#[tokio::test] +async fn resolve_root_git_project_for_trust_non_worktrees_gitdir_returns_none() { let tmp = TempDir::new().expect("tempdir"); let proj = tmp.path().join("proj"); std::fs::create_dir_all(proj.join("nested")).unwrap(); @@ -520,8 +529,12 @@ fn resolve_root_git_project_for_trust_non_worktrees_gitdir_returns_none() { ) .unwrap(); - assert!(resolve_root_git_project_for_trust(&proj).is_none()); - assert!(resolve_root_git_project_for_trust(&proj.join("nested")).is_none()); + assert!(resolve_root_git_project_for_trust(&proj).await.is_none()); + assert!( + resolve_root_git_project_for_trust(&proj.join("nested")) + .await + .is_none() + ); } #[tokio::test] diff --git a/codex-rs/core/src/guardian/review_session.rs b/codex-rs/core/src/guardian/review_session.rs index 6cb9aefa14..f1c6e1e054 100644 --- a/codex-rs/core/src/guardian/review_session.rs +++ b/codex-rs/core/src/guardian/review_session.rs @@ -823,9 +823,9 @@ async fn interrupt_and_drain_turn(codex: &Codex) -> anyhow::Result<()> { mod tests { use super::*; - #[test] - fn guardian_review_session_config_change_invalidates_cached_session() { - let parent_config = crate::config::test_config(); + #[tokio::test] + async fn guardian_review_session_config_change_invalidates_cached_session() { + let parent_config = crate::config::test_config().await; let cached_spawn_config = build_guardian_review_session_config( &parent_config, /*live_network_config*/ None, @@ -855,9 +855,9 @@ mod tests { ); } - #[test] - fn guardian_review_session_config_disables_hooks() { - let mut parent_config = crate::config::test_config(); + #[tokio::test] + async fn guardian_review_session_config_disables_hooks() { + let mut parent_config = crate::config::test_config().await; parent_config .features .enable(Feature::CodexHooks) diff --git a/codex-rs/core/src/guardian/tests.rs b/codex-rs/core/src/guardian/tests.rs index 8940753d27..485f222551 100644 --- a/codex-rs/core/src/guardian/tests.rs +++ b/codex-rs/core/src/guardian/tests.rs @@ -1332,8 +1332,8 @@ async fn guardian_review_surfaces_responses_api_errors_in_rejection_reason() -> Ok(()) } -#[test] -fn guardian_parallel_reviews_fork_from_last_committed_trunk_history() -> anyhow::Result<()> { +#[tokio::test] +async fn guardian_parallel_reviews_fork_from_last_committed_trunk_history() -> anyhow::Result<()> { const TEST_STACK_SIZE_BYTES: usize = 2 * 1024 * 1024; let handle = @@ -1569,9 +1569,9 @@ fn guardian_parallel_reviews_fork_from_last_committed_trunk_history() -> anyhow: )), } } -#[test] -fn guardian_review_session_config_preserves_parent_network_proxy() { - let mut parent_config = test_config(); +#[tokio::test] +async fn guardian_review_session_config_preserves_parent_network_proxy() { + let mut parent_config = test_config().await; let network = NetworkProxySpec::from_config_and_constraints( NetworkProxyConfig::default(), Some(NetworkConstraints { @@ -1616,9 +1616,9 @@ fn guardian_review_session_config_preserves_parent_network_proxy() { ); } -#[test] -fn guardian_review_session_config_overrides_parent_developer_instructions() { - let mut parent_config = test_config(); +#[tokio::test] +async fn guardian_review_session_config_overrides_parent_developer_instructions() { + let mut parent_config = test_config().await; parent_config.developer_instructions = Some("parent or managed config should not replace guardian policy".to_string()); @@ -1636,9 +1636,9 @@ fn guardian_review_session_config_overrides_parent_developer_instructions() { ); } -#[test] -fn guardian_review_session_config_uses_live_network_proxy_state() { - let mut parent_config = test_config(); +#[tokio::test] +async fn guardian_review_session_config_uses_live_network_proxy_state() { + let mut parent_config = test_config().await; let mut parent_network = NetworkProxyConfig::default(); parent_network.network.enabled = true; parent_network @@ -1680,9 +1680,9 @@ fn guardian_review_session_config_uses_live_network_proxy_state() { ); } -#[test] -fn guardian_review_session_config_rejects_pinned_collab_feature() { - let mut parent_config = test_config(); +#[tokio::test] +async fn guardian_review_session_config_rejects_pinned_collab_feature() { + let mut parent_config = test_config().await; parent_config.features = ManagedFeatures::from_configured( parent_config.features.get().clone(), Some(Sourced { @@ -1708,9 +1708,9 @@ fn guardian_review_session_config_rejects_pinned_collab_feature() { ); } -#[test] -fn guardian_review_session_config_uses_parent_active_model_instead_of_hardcoded_slug() { - let mut parent_config = test_config(); +#[tokio::test] +async fn guardian_review_session_config_uses_parent_active_model_instead_of_hardcoded_slug() { + let mut parent_config = test_config().await; parent_config.model = Some("configured-model".to_string()); let guardian_config = build_guardian_review_session_config_for_test( @@ -1724,8 +1724,8 @@ fn guardian_review_session_config_uses_parent_active_model_instead_of_hardcoded_ assert_eq!(guardian_config.model, Some("active-model".to_string())); } -#[test] -fn guardian_review_session_config_uses_requirements_guardian_policy_config() { +#[tokio::test] +async fn guardian_review_session_config_uses_requirements_guardian_policy_config() { let codex_home = tempfile::tempdir().expect("create temp dir"); let workspace = tempfile::tempdir().expect("create temp dir"); let config_layer_stack = ConfigLayerStack::new( @@ -1748,6 +1748,7 @@ fn guardian_review_session_config_uses_requirements_guardian_policy_config() { codex_home.abs(), config_layer_stack, ) + .await .expect("load config"); let guardian_config = build_guardian_review_session_config_for_test( @@ -1766,8 +1767,9 @@ fn guardian_review_session_config_uses_requirements_guardian_policy_config() { ); } -#[test] -fn guardian_review_session_config_uses_default_guardian_policy_without_requirements_override() { +#[tokio::test] +async fn guardian_review_session_config_uses_default_guardian_policy_without_requirements_override() +{ let codex_home = tempfile::tempdir().expect("create temp dir"); let workspace = tempfile::tempdir().expect("create temp dir"); let config_layer_stack = @@ -1782,6 +1784,7 @@ fn guardian_review_session_config_uses_default_guardian_policy_without_requireme codex_home.abs(), config_layer_stack, ) + .await .expect("load config"); let guardian_config = build_guardian_review_session_config_for_test( diff --git a/codex-rs/core/src/memories/tests.rs b/codex-rs/core/src/memories/tests.rs index 71809d382d..9959b1fcb5 100644 --- a/codex-rs/core/src/memories/tests.rs +++ b/codex-rs/core/src/memories/tests.rs @@ -468,7 +468,7 @@ mod phase2 { impl DispatchHarness { async fn new() -> Self { let codex_home = tempfile::tempdir().expect("create temp codex home"); - let mut config = test_config(); + let mut config = test_config().await; config.codex_home = codex_utils_absolute_path::AbsolutePathBuf::from_absolute_path(codex_home.path()) .expect("codex home is absolute"); @@ -895,7 +895,7 @@ mod phase2 { #[tokio::test] async fn dispatch_marks_job_for_retry_when_spawn_agent_fails() { let codex_home = tempfile::tempdir().expect("create temp codex home"); - let mut config = test_config(); + let mut config = test_config().await; config.codex_home = codex_utils_absolute_path::AbsolutePathBuf::from_absolute_path(codex_home.path()) .expect("codex home is absolute"); diff --git a/codex-rs/core/src/prompt_debug.rs b/codex-rs/core/src/prompt_debug.rs index d5fa1863f2..fb64481523 100644 --- a/codex-rs/core/src/prompt_debug.rs +++ b/codex-rs/core/src/prompt_debug.rs @@ -106,7 +106,7 @@ mod tests { async fn build_prompt_input_includes_context_and_user_message() { let codex_home = tempfile::tempdir().expect("create codex home"); let cwd = tempfile::tempdir().expect("create cwd"); - let mut config = test_config(); + let mut config = test_config().await; config.codex_home = AbsolutePathBuf::from_absolute_path(codex_home.path()).expect("codex home is absolute"); config.cwd = AbsolutePathBuf::try_from(cwd.path().to_path_buf()).expect("absolute cwd"); diff --git a/codex-rs/core/src/realtime_context.rs b/codex-rs/core/src/realtime_context.rs index cac2067a82..fc17dab99b 100644 --- a/codex-rs/core/src/realtime_context.rs +++ b/codex-rs/core/src/realtime_context.rs @@ -62,8 +62,8 @@ pub(crate) async fn build_realtime_startup_context( let history = sess.clone_history().await; let current_thread_section = build_current_thread_section(history.raw_items()); let recent_threads = load_recent_threads(sess).await; - let recent_work_section = build_recent_work_section(&cwd, &recent_threads); - let workspace_section = build_workspace_section_with_user_root(&cwd, home_dir()); + let recent_work_section = build_recent_work_section(&cwd, &recent_threads).await; + let workspace_section = build_workspace_section_with_user_root(&cwd, home_dir()).await; if current_thread_section.is_none() && recent_work_section.is_none() @@ -144,16 +144,18 @@ async fn load_recent_threads(sess: &Session) -> Vec { } } -fn build_recent_work_section(cwd: &Path, recent_threads: &[StoredThread]) -> Option { +async fn build_recent_work_section(cwd: &Path, recent_threads: &[StoredThread]) -> Option { let mut groups: HashMap> = HashMap::new(); for entry in recent_threads { - let group = - resolve_root_git_project_for_trust(&entry.cwd).unwrap_or_else(|| entry.cwd.clone()); + let group = resolve_root_git_project_for_trust(&entry.cwd) + .await + .unwrap_or_else(|| entry.cwd.clone()); groups.entry(group).or_default().push(entry); } - let current_group = - resolve_root_git_project_for_trust(cwd).unwrap_or_else(|| cwd.to_path_buf()); + let current_group = resolve_root_git_project_for_trust(cwd) + .await + .unwrap_or_else(|| cwd.to_path_buf()); let mut groups = groups.into_iter().collect::>(); groups.sort_by(|(left_group, left_entries), (right_group, right_entries)| { let left_latest = left_entries @@ -178,14 +180,13 @@ fn build_recent_work_section(cwd: &Path, recent_threads: &[StoredThread]) -> Opt )) }); - let sections = groups - .into_iter() - .take(MAX_RECENT_WORK_GROUPS) - .filter_map(|(group, mut entries)| { - entries.sort_by_key(|entry| Reverse(entry.updated_at)); - format_thread_group(¤t_group, &group, entries) - }) - .collect::>(); + let mut sections = Vec::new(); + for (group, mut entries) in groups.into_iter().take(MAX_RECENT_WORK_GROUPS) { + entries.sort_by_key(|entry| Reverse(entry.updated_at)); + if let Some(section) = format_thread_group(¤t_group, &group, entries).await { + sections.push(section); + } + } (!sections.is_empty()).then(|| sections.join("\n\n")) } @@ -306,11 +307,11 @@ pub(crate) fn truncate_realtime_text_to_token_budget(text: &str, budget_tokens: } } -fn build_workspace_section_with_user_root( +async fn build_workspace_section_with_user_root( cwd: &Path, user_root: Option, ) -> Option { - let git_root = resolve_root_git_project_for_trust(cwd); + let git_root = resolve_root_git_project_for_trust(cwd).await; let cwd_tree = render_tree(cwd); let git_root_tree = git_root .as_ref() @@ -466,13 +467,16 @@ fn format_startup_context_blob(body: &str, budget_tokens: usize) -> String { } } -fn format_thread_group( +async fn format_thread_group( current_group: &Path, group: &Path, entries: Vec<&StoredThread>, ) -> Option { let latest = entries.first()?; - let group_label = if resolve_root_git_project_for_trust(latest.cwd.as_path()).is_some() { + let group_label = if resolve_root_git_project_for_trust(latest.cwd.as_path()) + .await + .is_some() + { format!("### Git repo: {}", group.display()) } else { format!("### Directory: {}", group.display()) diff --git a/codex-rs/core/src/realtime_context_tests.rs b/codex-rs/core/src/realtime_context_tests.rs index 7fe438dac7..4f62affcae 100644 --- a/codex-rs/core/src/realtime_context_tests.rs +++ b/codex-rs/core/src/realtime_context_tests.rs @@ -188,30 +188,31 @@ fn startup_context_blob_is_wrapped_in_tags_and_fits_budget() { assert!(wrapped.len().div_ceil(4) <= 200); } -#[test] -fn workspace_section_requires_meaningful_structure() { +#[tokio::test] +async fn workspace_section_requires_meaningful_structure() { let cwd = TempDir::new().expect("tempdir"); assert_eq!( - build_workspace_section_with_user_root(cwd.path(), /*user_root*/ None), + build_workspace_section_with_user_root(cwd.path(), /*user_root*/ None).await, None ); } -#[test] -fn workspace_section_includes_tree_when_entries_exist() { +#[tokio::test] +async fn workspace_section_includes_tree_when_entries_exist() { let cwd = TempDir::new().expect("tempdir"); fs::create_dir(cwd.path().join("docs")).expect("create docs dir"); fs::write(cwd.path().join("README.md"), "hello").expect("write readme"); let section = build_workspace_section_with_user_root(cwd.path(), /*user_root*/ None) + .await .expect("workspace section"); assert!(section.contains("Working directory tree:")); assert!(section.contains("- docs/")); assert!(section.contains("- README.md")); } -#[test] -fn workspace_section_includes_user_root_tree_when_distinct() { +#[tokio::test] +async fn workspace_section_includes_user_root_tree_when_distinct() { let root = TempDir::new().expect("tempdir"); let cwd = root.path().join("cwd"); let git_root = root.path().join("git"); @@ -225,14 +226,15 @@ fn workspace_section_includes_user_root_tree_when_distinct() { fs::write(user_root.join(".zshrc"), "export TEST=1").expect("write home file"); let section = build_workspace_section_with_user_root(cwd.as_path(), Some(user_root)) + .await .expect("workspace section"); assert!(section.contains("User root tree:")); assert!(section.contains("- code/")); assert!(!section.contains("- .zshrc")); } -#[test] -fn recent_work_section_groups_threads_by_cwd() { +#[tokio::test] +async fn recent_work_section_groups_threads_by_cwd() { let root = TempDir::new().expect("tempdir"); let repo = root.path().join("repo"); let workspace_a = repo.join("workspace-a"); @@ -268,6 +270,7 @@ fn recent_work_section_groups_threads_by_cwd() { let repo = fs::canonicalize(repo).expect("canonicalize repo"); let section = build_recent_work_section(current_cwd.as_path(), &recent_threads) + .await .expect("recent work section"); assert!(section.contains(&format!("### Git repo: {}", repo.display()))); assert!(section.contains("Recent sessions: 2")); diff --git a/codex-rs/core/src/thread_manager_tests.rs b/codex-rs/core/src/thread_manager_tests.rs index 57d6ba8d6d..06f89fc728 100644 --- a/codex-rs/core/src/thread_manager_tests.rs +++ b/codex-rs/core/src/thread_manager_tests.rs @@ -237,7 +237,7 @@ async fn ignores_session_prefix_messages_when_truncating() { #[tokio::test] async fn shutdown_all_threads_bounded_submits_shutdown_to_every_thread() { let temp_dir = tempdir().expect("tempdir"); - let mut config = test_config(); + let mut config = test_config().await; config.codex_home = temp_dir.path().join("codex-home").abs(); config.cwd = config.codex_home.abs(); std::fs::create_dir_all(&config.codex_home).expect("create codex home"); @@ -279,7 +279,7 @@ async fn new_uses_configured_openai_provider_for_model_refresh() { let models_mock = mount_models_once(&server, ModelsResponse { models: vec![] }).await; let temp_dir = tempdir().expect("tempdir"); - let mut config = test_config(); + let mut config = test_config().await; config.codex_home = temp_dir.path().join("codex-home").abs(); config.cwd = config.codex_home.abs(); std::fs::create_dir_all(&config.codex_home).expect("create codex home"); @@ -422,7 +422,7 @@ fn mixed_response_and_legacy_user_event_history_is_mid_turn() { #[tokio::test] async fn interrupted_fork_snapshot_does_not_synthesize_turn_id_for_legacy_history() { let temp_dir = tempdir().expect("tempdir"); - let mut config = test_config(); + let mut config = test_config().await; config.codex_home = temp_dir.path().join("codex-home").abs(); config.cwd = config.codex_home.abs(); std::fs::create_dir_all(&config.codex_home).expect("create codex home"); @@ -525,7 +525,7 @@ async fn interrupted_fork_snapshot_does_not_synthesize_turn_id_for_legacy_histor #[tokio::test] async fn interrupted_fork_snapshot_preserves_explicit_turn_id() { let temp_dir = tempdir().expect("tempdir"); - let mut config = test_config(); + let mut config = test_config().await; config.codex_home = temp_dir.path().join("codex-home").abs(); config.cwd = config.codex_home.abs(); std::fs::create_dir_all(&config.codex_home).expect("create codex home"); @@ -618,7 +618,7 @@ async fn interrupted_fork_snapshot_preserves_explicit_turn_id() { #[tokio::test] async fn interrupted_fork_snapshot_uses_persisted_mid_turn_history_without_live_source() { let temp_dir = tempdir().expect("tempdir"); - let mut config = test_config(); + let mut config = test_config().await; config.codex_home = temp_dir.path().join("codex-home").abs(); config.cwd = config.codex_home.abs(); std::fs::create_dir_all(&config.codex_home).expect("create codex home"); diff --git a/codex-rs/core/src/tools/spec_tests.rs b/codex-rs/core/src/tools/spec_tests.rs index cab38bc21c..fa52aa6b0f 100644 --- a/codex-rs/core/src/tools/spec_tests.rs +++ b/codex-rs/core/src/tools/spec_tests.rs @@ -107,8 +107,8 @@ fn discoverable_connector(id: &str, name: &str, description: &str) -> Discoverab })) } -fn search_capable_model_info() -> ModelInfo { - let config = test_config(); +async fn search_capable_model_info() -> ModelInfo { + let config = test_config().await; let mut model_info = construct_model_info_offline("gpt-5-codex", &config); model_info.supports_search_tool = true; model_info @@ -216,8 +216,8 @@ fn find_namespace_function_tool<'a>( .unwrap_or_else(|| panic!("expected tool {expected_namespace}{expected_name} in namespace")) } -fn multi_agent_v2_tools_config() -> ToolsConfig { - let config = test_config(); +async fn multi_agent_v2_tools_config() -> ToolsConfig { + let config = test_config().await; let model_info = construct_model_info_offline("gpt-5-codex", &config); let mut features = Features::with_defaults(); features.enable(Feature::Collab); @@ -250,8 +250,8 @@ fn multi_agent_v2_spawn_agent_description(tools_config: &ToolsConfig) -> String description.clone() } -fn model_info_from_models_json(slug: &str) -> ModelInfo { - let config = test_config(); +async fn model_info_from_models_json(slug: &str) -> ModelInfo { + let config = test_config().await; let response = bundled_models_response() .unwrap_or_else(|err| panic!("bundled models.json should parse: {err}")); let model = response @@ -295,9 +295,9 @@ fn build_specs_with_unavailable_tools( ) } -#[test] -fn model_provided_unified_exec_is_blocked_for_windows_sandboxed_policies() { - let mut model_info = model_info_from_models_json("gpt-5-codex"); +#[tokio::test] +async fn model_provided_unified_exec_is_blocked_for_windows_sandboxed_policies() { + let mut model_info = model_info_from_models_json("gpt-5-codex").await; model_info.shell_type = ConfigShellToolType::UnifiedExec; let features = Features::with_defaults(); let available_models = Vec::new(); @@ -320,9 +320,9 @@ fn model_provided_unified_exec_is_blocked_for_windows_sandboxed_policies() { assert_eq!(config.shell_type, expected_shell_type); } -#[test] -fn get_memory_requires_feature_flag() { - let config = test_config(); +#[tokio::test] +async fn get_memory_requires_feature_flag() { + let config = test_config().await; let model_info = construct_model_info_offline("gpt-5-codex", &config); let mut features = Features::with_defaults(); features.disable(Feature::MemoryTool); @@ -350,14 +350,14 @@ fn get_memory_requires_feature_flag() { ); } -fn assert_model_tools( +async fn assert_model_tools( model_slug: &str, features: &Features, web_search_mode: Option, expected_tools: &[&str], ) { - let _config = test_config(); - let model_info = model_info_from_models_json(model_slug); + let _config = test_config().await; + let model_info = model_info_from_models_json(model_slug).await; let available_models = Vec::new(); let tools_config = ToolsConfig::new(&ToolsConfigParams { model_info: &model_info, @@ -388,7 +388,7 @@ fn assert_model_tools( assert_eq!(&tool_names, &expected_tools,); } -fn assert_default_model_tools( +async fn assert_default_model_tools( model_slug: &str, features: &Features, web_search_mode: Option, @@ -401,11 +401,11 @@ fn assert_default_model_tools( vec![shell_tool] }; expected.extend(expected_tail); - assert_model_tools(model_slug, features, web_search_mode, &expected); + assert_model_tools(model_slug, features, web_search_mode, &expected).await; } -#[test] -fn test_build_specs_gpt5_codex_default() { +#[tokio::test] +async fn test_build_specs_gpt5_codex_default() { let features = Features::with_defaults(); assert_default_model_tools( "gpt-5-codex", @@ -424,11 +424,12 @@ fn test_build_specs_gpt5_codex_default() { "wait_agent", "close_agent", ], - ); + ) + .await; } -#[test] -fn test_build_specs_gpt51_codex_default() { +#[tokio::test] +async fn test_build_specs_gpt51_codex_default() { let features = Features::with_defaults(); assert_default_model_tools( "gpt-5.1-codex", @@ -447,11 +448,12 @@ fn test_build_specs_gpt51_codex_default() { "wait_agent", "close_agent", ], - ); + ) + .await; } -#[test] -fn test_build_specs_gpt5_codex_unified_exec_web_search() { +#[tokio::test] +async fn test_build_specs_gpt5_codex_unified_exec_web_search() { let mut features = Features::with_defaults(); features.enable(Feature::UnifiedExec); assert_model_tools( @@ -472,11 +474,12 @@ fn test_build_specs_gpt5_codex_unified_exec_web_search() { "wait_agent", "close_agent", ], - ); + ) + .await; } -#[test] -fn test_build_specs_gpt51_codex_unified_exec_web_search() { +#[tokio::test] +async fn test_build_specs_gpt51_codex_unified_exec_web_search() { let mut features = Features::with_defaults(); features.enable(Feature::UnifiedExec); assert_model_tools( @@ -497,11 +500,12 @@ fn test_build_specs_gpt51_codex_unified_exec_web_search() { "wait_agent", "close_agent", ], - ); + ) + .await; } -#[test] -fn test_gpt_5_1_codex_max_defaults() { +#[tokio::test] +async fn test_gpt_5_1_codex_max_defaults() { let features = Features::with_defaults(); assert_default_model_tools( "gpt-5.1-codex-max", @@ -520,11 +524,12 @@ fn test_gpt_5_1_codex_max_defaults() { "wait_agent", "close_agent", ], - ); + ) + .await; } -#[test] -fn test_codex_5_1_mini_defaults() { +#[tokio::test] +async fn test_codex_5_1_mini_defaults() { let features = Features::with_defaults(); assert_default_model_tools( "gpt-5.1-codex-mini", @@ -543,11 +548,12 @@ fn test_codex_5_1_mini_defaults() { "wait_agent", "close_agent", ], - ); + ) + .await; } -#[test] -fn test_gpt_5_defaults() { +#[tokio::test] +async fn test_gpt_5_defaults() { let features = Features::with_defaults(); assert_default_model_tools( "gpt-5", @@ -565,11 +571,12 @@ fn test_gpt_5_defaults() { "wait_agent", "close_agent", ], - ); + ) + .await; } -#[test] -fn test_gpt_5_1_defaults() { +#[tokio::test] +async fn test_gpt_5_1_defaults() { let features = Features::with_defaults(); assert_default_model_tools( "gpt-5.1", @@ -588,11 +595,12 @@ fn test_gpt_5_1_defaults() { "wait_agent", "close_agent", ], - ); + ) + .await; } -#[test] -fn test_gpt_5_1_codex_max_unified_exec_web_search() { +#[tokio::test] +async fn test_gpt_5_1_codex_max_unified_exec_web_search() { let mut features = Features::with_defaults(); features.enable(Feature::UnifiedExec); assert_model_tools( @@ -613,12 +621,13 @@ fn test_gpt_5_1_codex_max_unified_exec_web_search() { "wait_agent", "close_agent", ], - ); + ) + .await; } -#[test] -fn test_build_specs_default_shell_present() { - let config = test_config(); +#[tokio::test] +async fn test_build_specs_default_shell_present() { + let config = test_config().await; let model_info = construct_model_info_offline("o3", &config); let mut features = Features::with_defaults(); features.enable(Feature::UnifiedExec); @@ -649,9 +658,9 @@ fn test_build_specs_default_shell_present() { assert_contains_tool_names(&tools, &subset); } -#[test] -fn shell_zsh_fork_prefers_shell_command_over_unified_exec() { - let config = test_config(); +#[tokio::test] +async fn shell_zsh_fork_prefers_shell_command_over_unified_exec() { + let config = test_config().await; let model_info = construct_model_info_offline("o3", &config); let mut features = Features::with_defaults(); features.enable(Feature::UnifiedExec); @@ -713,9 +722,10 @@ fn shell_zsh_fork_prefers_shell_command_over_unified_exec() { ); } -#[test] -fn spawn_agent_description_omits_usage_hint_when_disabled() { +#[tokio::test] +async fn spawn_agent_description_omits_usage_hint_when_disabled() { let tools_config = multi_agent_v2_tools_config() + .await .with_spawn_agent_usage_hint(/*spawn_agent_usage_hint*/ false); let description = multi_agent_v2_spawn_agent_description(&tools_config); @@ -734,11 +744,13 @@ fn spawn_agent_description_omits_usage_hint_when_disabled() { ); } -#[test] -fn spawn_agent_description_uses_configured_usage_hint_text() { - let tools_config = multi_agent_v2_tools_config().with_spawn_agent_usage_hint_text(Some( - /*spawn_agent_usage_hint_text*/ "Custom delegation guidance only.".to_string(), - )); +#[tokio::test] +async fn spawn_agent_description_uses_configured_usage_hint_text() { + let tools_config = multi_agent_v2_tools_config() + .await + .with_spawn_agent_usage_hint_text(Some( + /*spawn_agent_usage_hint_text*/ "Custom delegation guidance only.".to_string(), + )); let description = multi_agent_v2_spawn_agent_description(&tools_config); assert_regex_match( @@ -757,9 +769,9 @@ fn spawn_agent_description_uses_configured_usage_hint_text() { ); } -#[test] -fn tool_suggest_requires_apps_and_plugins_features() { - let model_info = search_capable_model_info(); +#[tokio::test] +async fn tool_suggest_requires_apps_and_plugins_features() { + let model_info = search_capable_model_info().await; let discoverable_tools = Some(vec![discoverable_connector( "connector_2128aebfecb84f64a069897515042a44", "Google Calendar", @@ -804,9 +816,9 @@ fn tool_suggest_requires_apps_and_plugins_features() { } } -#[test] -fn search_tool_description_handles_no_enabled_mcp_tools() { - let model_info = search_capable_model_info(); +#[tokio::test] +async fn search_tool_description_handles_no_enabled_mcp_tools() { + let model_info = search_capable_model_info().await; let mut features = Features::with_defaults(); features.enable(Feature::Apps); features.enable(Feature::ToolSearch); @@ -838,9 +850,9 @@ fn search_tool_description_handles_no_enabled_mcp_tools() { assert!(!description.contains("{{source_descriptions}}")); } -#[test] -fn search_tool_description_falls_back_to_connector_name_without_description() { - let model_info = search_capable_model_info(); +#[tokio::test] +async fn search_tool_description_falls_back_to_connector_name_without_description() { + let model_info = search_capable_model_info().await; let mut features = Features::with_defaults(); features.enable(Feature::Apps); features.enable(Feature::ToolSearch); @@ -889,9 +901,9 @@ fn search_tool_description_falls_back_to_connector_name_without_description() { assert!(!description.contains("- Calendar:")); } -#[test] -fn search_tool_registers_namespaced_mcp_tool_aliases() { - let model_info = search_capable_model_info(); +#[tokio::test] +async fn search_tool_registers_namespaced_mcp_tool_aliases() { + let model_info = search_capable_model_info().await; let mut features = Features::with_defaults(); features.enable(Feature::Apps); features.enable(Feature::ToolSearch); @@ -974,9 +986,9 @@ fn search_tool_registers_namespaced_mcp_tool_aliases() { assert!(registry.has_handler(&mcp_alias)); } -#[test] -fn direct_mcp_tools_register_namespaced_handlers() { - let config = test_config(); +#[tokio::test] +async fn direct_mcp_tools_register_namespaced_handlers() { + let config = test_config().await; let model_info = construct_model_info_offline("gpt-5-codex", &config); let mut features = Features::with_defaults(); features.enable(Feature::UnifiedExec); @@ -1011,9 +1023,9 @@ fn direct_mcp_tools_register_namespaced_handlers() { assert!(!registry.has_handler(&ToolName::plain("mcp__test_server__echo"))); } -#[test] -fn unavailable_mcp_tools_are_exposed_as_dummy_function_tools() { - let config = test_config(); +#[tokio::test] +async fn unavailable_mcp_tools_are_exposed_as_dummy_function_tools() { + let config = test_config().await; let model_info = construct_model_info_offline("gpt-5-codex", &config); let mut features = Features::with_defaults(); features.enable(Feature::UnifiedExec); @@ -1060,9 +1072,9 @@ fn unavailable_mcp_tools_are_exposed_as_dummy_function_tools() { assert!(!registry.has_handler(&ToolName::plain("mcp__codex_apps__calendar_create_event"))); } -#[test] -fn test_mcp_tool_property_missing_type_defaults_to_string() { - let config = test_config(); +#[tokio::test] +async fn test_mcp_tool_property_missing_type_defaults_to_string() { + let config = test_config().await; let model_info = construct_model_info_offline("gpt-5-codex", &config); let mut features = Features::with_defaults(); features.enable(Feature::UnifiedExec); @@ -1123,9 +1135,9 @@ fn test_mcp_tool_property_missing_type_defaults_to_string() { ); } -#[test] -fn test_mcp_tool_preserves_integer_schema() { - let config = test_config(); +#[tokio::test] +async fn test_mcp_tool_preserves_integer_schema() { + let config = test_config().await; let model_info = construct_model_info_offline("gpt-5-codex", &config); let mut features = Features::with_defaults(); features.enable(Feature::UnifiedExec); @@ -1184,9 +1196,9 @@ fn test_mcp_tool_preserves_integer_schema() { ); } -#[test] -fn test_mcp_tool_array_without_items_gets_default_string_items() { - let config = test_config(); +#[tokio::test] +async fn test_mcp_tool_array_without_items_gets_default_string_items() { + let config = test_config().await; let model_info = construct_model_info_offline("gpt-5-codex", &config); let mut features = Features::with_defaults(); features.enable(Feature::UnifiedExec); @@ -1249,9 +1261,9 @@ fn test_mcp_tool_array_without_items_gets_default_string_items() { ); } -#[test] -fn test_mcp_tool_anyof_defaults_to_string() { - let config = test_config(); +#[tokio::test] +async fn test_mcp_tool_anyof_defaults_to_string() { + let config = test_config().await; let model_info = construct_model_info_offline("gpt-5-codex", &config); let mut features = Features::with_defaults(); features.enable(Feature::UnifiedExec); @@ -1318,9 +1330,9 @@ fn test_mcp_tool_anyof_defaults_to_string() { ); } -#[test] -fn test_get_openai_tools_mcp_tools_with_additional_properties_schema() { - let config = test_config(); +#[tokio::test] +async fn test_get_openai_tools_mcp_tools_with_additional_properties_schema() { + let config = test_config().await; let model_info = construct_model_info_offline("gpt-5-codex", &config); let mut features = Features::with_defaults(); features.enable(Feature::UnifiedExec); @@ -1433,8 +1445,8 @@ fn test_get_openai_tools_mcp_tools_with_additional_properties_schema() { ); } -#[test] -fn code_mode_only_restricts_model_tools_to_exec_tools() { +#[tokio::test] +async fn code_mode_only_restricts_model_tools_to_exec_tools() { let mut features = Features::with_defaults(); features.enable(Feature::CodeMode); features.enable(Feature::CodeModeOnly); @@ -1444,5 +1456,6 @@ fn code_mode_only_restricts_model_tools_to_exec_tools() { &features, Some(WebSearchMode::Live), &["exec", "wait"], - ); + ) + .await; } diff --git a/codex-rs/git-utils/src/info.rs b/codex-rs/git-utils/src/info.rs index 048a8cedb6..e28a0bae42 100644 --- a/codex-rs/git-utils/src/info.rs +++ b/codex-rs/git-utils/src/info.rs @@ -618,7 +618,7 @@ async fn diff_against_sha(cwd: &Path, sha: &GitSha) -> Option { /// `[get_git_repo_root]`, but resolves to the root of the main /// repository. Handles worktrees via filesystem inspection without invoking /// the `git` executable. -pub fn resolve_root_git_project_for_trust(cwd: &Path) -> Option { +pub async fn resolve_root_git_project_for_trust(cwd: &Path) -> Option { let base = if cwd.is_dir() { cwd } else { cwd.parent()? }; let (repo_root, dot_git) = find_ancestor_git_entry(base)?; if dot_git.is_dir() { diff --git a/codex-rs/tui/src/chatwidget/tests/helpers.rs b/codex-rs/tui/src/chatwidget/tests/helpers.rs index 0ae51c4ff8..c0f881e2b8 100644 --- a/codex-rs/tui/src/chatwidget/tests/helpers.rs +++ b/codex-rs/tui/src/chatwidget/tests/helpers.rs @@ -10,6 +10,7 @@ pub(super) async fn test_config() -> Config { .keep(); let mut config = Config::load_default_with_cli_overrides_for_codex_home(codex_home.clone(), Vec::new()) + .await .expect("config"); config.codex_home = codex_home.abs(); config.sqlite_home = codex_home.clone(); diff --git a/codex-rs/tui/src/onboarding/onboarding_screen.rs b/codex-rs/tui/src/onboarding/onboarding_screen.rs index 22a444cdaf..5e2e0edbde 100644 --- a/codex-rs/tui/src/onboarding/onboarding_screen.rs +++ b/codex-rs/tui/src/onboarding/onboarding_screen.rs @@ -4,6 +4,7 @@ use crate::legacy_core::windows_sandbox::WindowsSandboxLevelExt; use codex_app_server_client::AppServerEvent; use codex_app_server_client::AppServerRequestHandle; use codex_app_server_protocol::ServerNotification; +use codex_git_utils::resolve_root_git_project_for_trust; #[cfg(target_os = "windows")] use codex_protocol::config_types::WindowsSandboxLevel; use crossterm::event::KeyCode; @@ -77,7 +78,7 @@ pub(crate) struct OnboardingResult { } impl OnboardingScreen { - pub(crate) fn new(tui: &mut Tui, args: OnboardingScreenArgs) -> Self { + pub(crate) async fn new(tui: &mut Tui, args: OnboardingScreenArgs) -> Self { let OnboardingScreenArgs { show_trust_screen, show_login_screen, @@ -122,8 +123,12 @@ impl OnboardingScreen { let show_windows_create_sandbox_hint = false; let highlighted = TrustDirectorySelection::Trust; if show_trust_screen { + let trust_target = resolve_root_git_project_for_trust(&cwd) + .await + .unwrap_or_else(|| cwd.clone()); steps.push(Step::TrustDirectory(TrustDirectoryWidget { cwd, + trust_target, codex_home, show_windows_create_sandbox_hint, should_quit: false, @@ -452,7 +457,7 @@ pub(crate) async fn run_onboarding_app( ) -> Result { use tokio_stream::StreamExt; - let mut onboarding_screen = OnboardingScreen::new(tui, args); + let mut onboarding_screen = OnboardingScreen::new(tui, args).await; // One-time guard to fully clear the screen after ChatGPT login success message is shown let mut did_full_clear_after_success = false; diff --git a/codex-rs/tui/src/onboarding/trust_directory.rs b/codex-rs/tui/src/onboarding/trust_directory.rs index 23ee624627..e9f2cd58bd 100644 --- a/codex-rs/tui/src/onboarding/trust_directory.rs +++ b/codex-rs/tui/src/onboarding/trust_directory.rs @@ -1,7 +1,6 @@ use std::path::PathBuf; use crate::legacy_core::config::set_project_trust_level; -use codex_git_utils::resolve_root_git_project_for_trust; use codex_protocol::config_types::TrustLevel; use crossterm::event::KeyCode; use crossterm::event::KeyEvent; @@ -27,6 +26,7 @@ use super::onboarding_screen::StepState; pub(crate) struct TrustDirectoryWidget { pub codex_home: PathBuf, pub cwd: PathBuf, + pub trust_target: PathBuf, pub show_windows_create_sandbox_hint: bool, pub should_quit: bool, pub selection: Option, @@ -142,8 +142,7 @@ impl StepStateProvider for TrustDirectoryWidget { impl TrustDirectoryWidget { fn handle_trust(&mut self) { - let target = - resolve_root_git_project_for_trust(&self.cwd).unwrap_or_else(|| self.cwd.clone()); + let target = self.trust_target.clone(); if let Err(e) = set_project_trust_level(&self.codex_home, &target, TrustLevel::Trusted) { tracing::error!("Failed to set project trusted: {e:?}"); self.error = Some(format!("Failed to set trust for {}: {e}", target.display())); @@ -182,6 +181,7 @@ mod tests { let mut widget = TrustDirectoryWidget { codex_home: codex_home.path().to_path_buf(), cwd: PathBuf::from("."), + trust_target: PathBuf::from("."), show_windows_create_sandbox_hint: false, should_quit: false, selection: None, @@ -207,6 +207,7 @@ mod tests { let widget = TrustDirectoryWidget { codex_home: codex_home.path().to_path_buf(), cwd: PathBuf::from("/workspace/project"), + trust_target: PathBuf::from("/workspace/project"), show_windows_create_sandbox_hint: false, should_quit: false, selection: None, From 77fe33bf72a7a1a0f2687c0487b6558fd2a82d1c Mon Sep 17 00:00:00 2001 From: Matthew Zeng Date: Wed, 15 Apr 2026 22:01:05 -0700 Subject: [PATCH 53/84] Update ToolSearch to be enabled by default (#17854) ## Summary - Promote `Feature::ToolSearch` to `Stable` and enable it in the default feature set - Update feature tests and tool registry coverage to match the new default - Adjust the search-tool integration test to assert the default-on path and explicit disable fallback ## Testing - `just fmt` - `cargo test -p codex-features` - `cargo test -p codex-core --test all search_tool` - `cargo test -p codex-tools` --- codex-rs/core/tests/suite/search_tool.rs | 16 ++++++++++------ codex-rs/features/src/lib.rs | 4 ++-- codex-rs/features/src/tests.rs | 6 +++--- codex-rs/tools/src/tool_registry_plan_tests.rs | 8 ++++---- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/codex-rs/core/tests/suite/search_tool.rs b/codex-rs/core/tests/suite/search_tool.rs index 74de499d70..624c439150 100644 --- a/codex-rs/core/tests/suite/search_tool.rs +++ b/codex-rs/core/tests/suite/search_tool.rs @@ -93,7 +93,7 @@ fn tool_search_output_tools(request: &ResponsesRequest, call_id: &str) -> Vec TestCodexBuilder { test_codex() .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) @@ -127,7 +131,7 @@ fn configured_builder(apps_base_url: String) -> TestCodexBuilder { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn search_tool_flag_adds_tool_search() -> Result<()> { +async fn search_tool_enabled_by_default_adds_tool_search() -> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; @@ -185,7 +189,7 @@ async fn search_tool_flag_adds_tool_search() -> Result<()> { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn tool_search_disabled_by_default_exposes_apps_tools_directly() -> Result<()> { +async fn tool_search_disabled_exposes_apps_tools_directly() -> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rs index ce7a196362..2d1ac89851 100644 --- a/codex-rs/features/src/lib.rs +++ b/codex-rs/features/src/lib.rs @@ -788,8 +788,8 @@ pub const FEATURES: &[FeatureSpec] = &[ FeatureSpec { id: Feature::ToolSearch, key: "tool_search", - stage: Stage::UnderDevelopment, - default_enabled: false, + stage: Stage::Stable, + default_enabled: true, }, FeatureSpec { id: Feature::UnavailableDummyTools, diff --git a/codex-rs/features/src/tests.rs b/codex-rs/features/src/tests.rs index 85cb01be23..56f92d55b7 100644 --- a/codex-rs/features/src/tests.rs +++ b/codex-rs/features/src/tests.rs @@ -128,9 +128,9 @@ fn tool_suggest_is_stable_and_enabled_by_default() { } #[test] -fn tool_search_is_under_development_and_disabled_by_default() { - assert_eq!(Feature::ToolSearch.stage(), Stage::UnderDevelopment); - assert_eq!(Feature::ToolSearch.default_enabled(), false); +fn tool_search_is_stable_and_enabled_by_default() { + assert_eq!(Feature::ToolSearch.stage(), Stage::Stable); + assert_eq!(Feature::ToolSearch.default_enabled(), true); } #[test] diff --git a/codex-rs/tools/src/tool_registry_plan_tests.rs b/codex-rs/tools/src/tool_registry_plan_tests.rs index a643543ee7..50b38905ee 100644 --- a/codex-rs/tools/src/tool_registry_plan_tests.rs +++ b/codex-rs/tools/src/tool_registry_plan_tests.rs @@ -1334,7 +1334,7 @@ fn search_tool_description_lists_each_mcp_source_once() { } #[test] -fn search_tool_requires_model_capability_and_feature_flag() { +fn search_tool_requires_model_capability_and_enabled_feature() { let model_info = search_capable_model_info(); let deferred_mcp_tools = Some(vec![deferred_mcp_tool( "_create_event", @@ -1367,10 +1367,12 @@ fn search_tool_requires_model_capability_and_feature_flag() { ); assert_lacks_tool_name(&tools, TOOL_SEARCH_TOOL_NAME); + let mut features_without_tool_search = Features::with_defaults(); + features_without_tool_search.disable(Feature::ToolSearch); let tools_config = ToolsConfig::new(&ToolsConfigParams { model_info: &model_info, available_models: &available_models, - features: &features, + features: &features_without_tool_search, image_generation_tool_auth_allowed: true, web_search_mode: Some(WebSearchMode::Cached), session_source: SessionSource::Cli, @@ -1385,8 +1387,6 @@ fn search_tool_requires_model_capability_and_feature_flag() { ); assert_lacks_tool_name(&tools, TOOL_SEARCH_TOOL_NAME); - let mut features = Features::with_defaults(); - features.enable(Feature::ToolSearch); let tools_config = ToolsConfig::new(&ToolsConfigParams { model_info: &model_info, available_models: &available_models, From 224dad41ac1bdf0c8a848b1fd0068262f1f99223 Mon Sep 17 00:00:00 2001 From: Matthew Zeng Date: Wed, 15 Apr 2026 22:09:17 -0700 Subject: [PATCH 54/84] [codex][mcp] Add resource uri meta to tool call item. (#17831) - [x] Add resource uri meta to tool call item so that the app-server client can start prefetching resources immediately without loading mcp server status. --- .../schema/json/ServerNotification.json | 6 +++ .../codex_app_server_protocol.schemas.json | 6 +++ .../codex_app_server_protocol.v2.schemas.json | 6 +++ .../json/v2/ItemCompletedNotification.json | 6 +++ .../json/v2/ItemStartedNotification.json | 6 +++ .../schema/json/v2/ReviewStartResponse.json | 6 +++ .../schema/json/v2/ThreadForkResponse.json | 6 +++ .../schema/json/v2/ThreadListResponse.json | 6 +++ .../json/v2/ThreadMetadataUpdateResponse.json | 6 +++ .../schema/json/v2/ThreadReadResponse.json | 6 +++ .../schema/json/v2/ThreadResumeResponse.json | 6 +++ .../json/v2/ThreadRollbackResponse.json | 6 +++ .../schema/json/v2/ThreadStartResponse.json | 6 +++ .../json/v2/ThreadStartedNotification.json | 6 +++ .../json/v2/ThreadUnarchiveResponse.json | 6 +++ .../json/v2/TurnCompletedNotification.json | 6 +++ .../schema/json/v2/TurnStartResponse.json | 6 +++ .../json/v2/TurnStartedNotification.json | 6 +++ .../schema/typescript/v2/ThreadItem.ts | 2 +- .../src/protocol/common.rs | 1 + .../src/protocol/thread_history.rs | 14 +++++-- .../app-server-protocol/src/protocol/v2.rs | 5 ++- .../app-server/src/bespoke_event_handling.rs | 18 ++++++-- codex-rs/core/src/mcp_tool_call.rs | 37 +++++++++++++++++ codex-rs/core/src/mcp_tool_call_tests.rs | 41 +++++++++++++++++++ .../core/src/tools/handlers/mcp_resource.rs | 2 + .../core/tests/common/apps_test_server.rs | 3 ++ codex-rs/core/tests/suite/rmcp_client.rs | 1 + codex-rs/core/tests/suite/search_tool.rs | 18 ++++++++ .../tests/event_processor_with_json_output.rs | 13 ++++-- codex-rs/protocol/src/protocol.rs | 6 +++ codex-rs/tui/src/chatwidget.rs | 20 ++++++--- 32 files changed, 269 insertions(+), 20 deletions(-) diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index a613590e3f..b84fbe9a02 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -2891,6 +2891,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index d0ce368b78..943b8480f9 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -13369,6 +13369,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index aca9b23880..1717dfb393 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -11213,6 +11213,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json index 4e8cfcb621..c5a84c4272 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json @@ -787,6 +787,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json index 5defb06618..cb9201f0a6 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json @@ -787,6 +787,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json index b7490d5e12..05dfaa4b72 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json @@ -930,6 +930,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json index 6db7d990f0..909c2af4d8 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json @@ -1444,6 +1444,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json index 7435fbeff7..8ebebd39f9 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json @@ -1206,6 +1206,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json index c6abbab166..40da3390ec 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json @@ -1206,6 +1206,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json index fb658030f4..400c0b661a 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json @@ -1206,6 +1206,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json index 9879db55f6..aef0cd737c 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json @@ -1444,6 +1444,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json index 84d2768855..6a50d6d978 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json @@ -1206,6 +1206,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json index 42d76d6818..a3d20555f1 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json @@ -1444,6 +1444,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json index 4b24a129bb..c4b1e54490 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json @@ -1206,6 +1206,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json index bebc8a6bbf..e55c9651fa 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json @@ -1206,6 +1206,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json index 381b78ab71..2848940085 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json @@ -930,6 +930,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json index 02727fbadd..c3f31e5322 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json @@ -930,6 +930,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json index 25c15d78dc..7e8ca749c0 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json @@ -930,6 +930,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts index e7fe940aa7..747a70c54d 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts @@ -53,7 +53,7 @@ exitCode: number | null, /** * The duration of the command execution in milliseconds. */ -durationMs: number | null, } | { "type": "fileChange", id: string, changes: Array, status: PatchApplyStatus, } | { "type": "mcpToolCall", id: string, server: string, tool: string, status: McpToolCallStatus, arguments: JsonValue, result: McpToolCallResult | null, error: McpToolCallError | null, +durationMs: number | null, } | { "type": "fileChange", id: string, changes: Array, status: PatchApplyStatus, } | { "type": "mcpToolCall", id: string, server: string, tool: string, status: McpToolCallStatus, arguments: JsonValue, mcpAppResourceUri?: string, result: McpToolCallResult | null, error: McpToolCallError | null, /** * The duration of the MCP tool call in milliseconds. */ diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index 3d6bf9d872..7f555e0688 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -764,6 +764,7 @@ macro_rules! server_notification_definitions { Display, ExperimentalApi, )] + #[allow(clippy::large_enum_variant)] #[serde(tag = "method", content = "params", rename_all = "camelCase")] #[strum(serialize_all = "camelCase")] pub enum ServerNotification { diff --git a/codex-rs/app-server-protocol/src/protocol/thread_history.rs b/codex-rs/app-server-protocol/src/protocol/thread_history.rs index 9e94515dd9..54f882cf29 100644 --- a/codex-rs/app-server-protocol/src/protocol/thread_history.rs +++ b/codex-rs/app-server-protocol/src/protocol/thread_history.rs @@ -502,6 +502,7 @@ impl ThreadHistoryBuilder { .arguments .clone() .unwrap_or(serde_json::Value::Null), + mcp_app_resource_uri: payload.mcp_app_resource_uri.clone(), result: None, error: None, duration_ms: None, @@ -518,11 +519,11 @@ impl ThreadHistoryBuilder { let duration_ms = i64::try_from(payload.duration.as_millis()).ok(); let (result, error) = match &payload.result { Ok(value) => ( - Some(McpToolCallResult { + Some(Box::new(McpToolCallResult { content: value.content.clone(), structured_content: value.structured_content.clone(), meta: value.meta.clone(), - }), + })), None, ), Err(message) => ( @@ -542,6 +543,7 @@ impl ThreadHistoryBuilder { .arguments .clone() .unwrap_or(serde_json::Value::Null), + mcp_app_resource_uri: payload.mcp_app_resource_uri.clone(), result, error, duration_ms, @@ -1809,6 +1811,7 @@ mod tests { tool: "lookup".into(), arguments: Some(serde_json::json!({"id":"123"})), }, + mcp_app_resource_uri: None, duration: Duration::from_millis(8), result: Err("boom".into()), }), @@ -1857,6 +1860,7 @@ mod tests { tool: "lookup".into(), status: McpToolCallStatus::Failed, arguments: serde_json::json!({"id":"123"}), + mcp_app_resource_uri: None, result: None, error: Some(McpToolCallError { message: "boom".into(), @@ -1882,6 +1886,7 @@ mod tests { tool: "lookup".into(), arguments: Some(serde_json::json!({"id":"123"})), }, + mcp_app_resource_uri: Some("ui://widget/lookup.html".into()), duration: Duration::from_millis(8), result: Ok(CallToolResult { content: vec![serde_json::json!({ @@ -1911,7 +1916,8 @@ mod tests { tool: "lookup".into(), status: McpToolCallStatus::Completed, arguments: serde_json::json!({"id":"123"}), - result: Some(McpToolCallResult { + mcp_app_resource_uri: Some("ui://widget/lookup.html".into()), + result: Some(Box::new(McpToolCallResult { content: vec![serde_json::json!({ "type": "text", "text": "result" @@ -1920,7 +1926,7 @@ mod tests { meta: Some(serde_json::json!({ "ui/resourceUri": "ui://widget/lookup.html" })), - }), + })), error: None, duration_ms: Some(8), } diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 5897625de1..155c67e7bf 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -4585,7 +4585,10 @@ pub enum ThreadItem { tool: String, status: McpToolCallStatus, arguments: JsonValue, - result: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + mcp_app_resource_uri: Option, + result: Option>, error: Option, /// The duration of the MCP tool call in milliseconds. #[ts(type = "number | null")] diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index 4fddee4a11..5d19bde70c 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -2910,6 +2910,7 @@ async fn construct_mcp_tool_call_notification( tool: begin_event.invocation.tool, status: McpToolCallStatus::InProgress, arguments: begin_event.invocation.arguments.unwrap_or(JsonValue::Null), + mcp_app_resource_uri: begin_event.mcp_app_resource_uri, result: None, error: None, duration_ms: None, @@ -2936,11 +2937,11 @@ async fn construct_mcp_tool_call_end_notification( let (result, error) = match &end_event.result { Ok(value) => ( - Some(McpToolCallResult { + Some(Box::new(McpToolCallResult { content: value.content.clone(), structured_content: value.structured_content.clone(), meta: value.meta.clone(), - }), + })), None, ), Err(message) => ( @@ -2957,6 +2958,7 @@ async fn construct_mcp_tool_call_end_notification( tool: end_event.invocation.tool, status, arguments: end_event.invocation.arguments.unwrap_or(JsonValue::Null), + mcp_app_resource_uri: end_event.mcp_app_resource_uri, result, error, duration_ms, @@ -4219,6 +4221,7 @@ mod tests { tool: "list_mcp_resources".to_string(), arguments: Some(serde_json::json!({"server": ""})), }, + mcp_app_resource_uri: Some("ui://widget/list-resources.html".to_string()), }; let thread_id = ThreadId::new().to_string(); @@ -4239,6 +4242,7 @@ mod tests { tool: begin_event.invocation.tool, status: McpToolCallStatus::InProgress, arguments: serde_json::json!({"server": ""}), + mcp_app_resource_uri: Some("ui://widget/list-resources.html".to_string()), result: None, error: None, duration_ms: None, @@ -4379,6 +4383,7 @@ mod tests { tool: "list_mcp_resources".to_string(), arguments: None, }, + mcp_app_resource_uri: None, }; let thread_id = ThreadId::new().to_string(); @@ -4399,6 +4404,7 @@ mod tests { tool: begin_event.invocation.tool, status: McpToolCallStatus::InProgress, arguments: JsonValue::Null, + mcp_app_resource_uri: None, result: None, error: None, duration_ms: None, @@ -4430,6 +4436,7 @@ mod tests { tool: "list_mcp_resources".to_string(), arguments: Some(serde_json::json!({"server": ""})), }, + mcp_app_resource_uri: Some("ui://widget/list-resources.html".to_string()), duration: Duration::from_nanos(92708), result: Ok(result), }; @@ -4452,13 +4459,14 @@ mod tests { tool: end_event.invocation.tool, status: McpToolCallStatus::Completed, arguments: serde_json::json!({"server": ""}), - result: Some(McpToolCallResult { + mcp_app_resource_uri: Some("ui://widget/list-resources.html".to_string()), + result: Some(Box::new(McpToolCallResult { content, structured_content: None, meta: Some(serde_json::json!({ "ui/resourceUri": "ui://widget/list-resources.html" })), - }), + })), error: None, duration_ms: Some(0), }, @@ -4476,6 +4484,7 @@ mod tests { tool: "list_mcp_resources".to_string(), arguments: None, }, + mcp_app_resource_uri: None, duration: Duration::from_millis(1), result: Err("boom".to_string()), }; @@ -4498,6 +4507,7 @@ mod tests { tool: end_event.invocation.tool, status: McpToolCallStatus::Failed, arguments: JsonValue::Null, + mcp_app_resource_uri: None, result: None, error: Some(McpToolCallError { message: "boom".to_string(), diff --git a/codex-rs/core/src/mcp_tool_call.rs b/codex-rs/core/src/mcp_tool_call.rs index 0dd5deae73..064585f447 100644 --- a/codex-rs/core/src/mcp_tool_call.rs +++ b/codex-rs/core/src/mcp_tool_call.rs @@ -101,6 +101,9 @@ pub(crate) async fn handle_mcp_tool_call( let metadata = lookup_mcp_tool_metadata(sess.as_ref(), turn_context.as_ref(), &server, &tool_name).await; + let mcp_app_resource_uri = metadata + .as_ref() + .and_then(|metadata| metadata.mcp_app_resource_uri.clone()); let app_tool_policy = if server == CODEX_APPS_MCP_SERVER_NAME { connectors::app_tool_policy( &turn_context.config, @@ -130,6 +133,7 @@ pub(crate) async fn handle_mcp_tool_call( turn_context.as_ref(), &call_id, invocation, + mcp_app_resource_uri.clone(), "MCP tool call blocked by app configuration".to_string(), /*already_started*/ false, ) @@ -161,6 +165,7 @@ pub(crate) async fn handle_mcp_tool_call( let tool_call_begin_event = EventMsg::McpToolCallBegin(McpToolCallBeginEvent { call_id: call_id.clone(), invocation: invocation.clone(), + mcp_app_resource_uri: mcp_app_resource_uri.clone(), }); notify_mcp_tool_call_event(sess.as_ref(), turn_context.as_ref(), tool_call_begin_event).await; @@ -213,6 +218,7 @@ pub(crate) async fn handle_mcp_tool_call( let tool_call_end_event = EventMsg::McpToolCallEnd(McpToolCallEndEvent { call_id: call_id.clone(), invocation, + mcp_app_resource_uri: mcp_app_resource_uri.clone(), duration, result: result.clone(), }); @@ -239,6 +245,7 @@ pub(crate) async fn handle_mcp_tool_call( turn_context.as_ref(), &call_id, invocation, + mcp_app_resource_uri.clone(), message, /*already_started*/ true, ) @@ -254,6 +261,7 @@ pub(crate) async fn handle_mcp_tool_call( turn_context.as_ref(), &call_id, invocation, + mcp_app_resource_uri.clone(), message, /*already_started*/ true, ) @@ -268,6 +276,7 @@ pub(crate) async fn handle_mcp_tool_call( turn_context.as_ref(), &call_id, invocation, + mcp_app_resource_uri.clone(), message, /*already_started*/ true, ) @@ -325,6 +334,7 @@ pub(crate) async fn handle_mcp_tool_call( let tool_call_end_event = EventMsg::McpToolCallEnd(McpToolCallEndEvent { call_id: call_id.clone(), invocation, + mcp_app_resource_uri, duration, result: result.clone(), }); @@ -642,11 +652,14 @@ pub(crate) struct McpToolApprovalMetadata { connector_description: Option, tool_title: Option, tool_description: Option, + mcp_app_resource_uri: Option, codex_apps_meta: Option>, openai_file_input_params: Option>, } const MCP_TOOL_CODEX_APPS_META_KEY: &str = "_codex_apps"; +const MCP_TOOL_OPENAI_OUTPUT_TEMPLATE_META_KEY: &str = "openai/outputTemplate"; +const MCP_TOOL_UI_RESOURCE_URI_META_KEY: &str = "ui/resourceUri"; fn custom_mcp_tool_approval_mode( turn_context: &TurnContext, @@ -1100,6 +1113,7 @@ pub(crate) async fn lookup_mcp_tool_metadata( connector_description, tool_title: tool_info.tool.title, tool_description: tool_info.tool.description.map(std::borrow::Cow::into_owned), + mcp_app_resource_uri: get_mcp_app_resource_uri(tool_info.tool.meta.as_deref()), codex_apps_meta: tool_info .tool .meta @@ -1114,6 +1128,26 @@ pub(crate) async fn lookup_mcp_tool_metadata( }) } +fn get_mcp_app_resource_uri( + meta: Option<&serde_json::Map>, +) -> Option { + meta.and_then(|meta| { + meta.get("ui") + .and_then(serde_json::Value::as_object) + .and_then(|ui| ui.get("resourceUri")) + .and_then(serde_json::Value::as_str) + .or_else(|| { + meta.get(MCP_TOOL_UI_RESOURCE_URI_META_KEY) + .and_then(serde_json::Value::as_str) + }) + .or_else(|| { + meta.get(MCP_TOOL_OPENAI_OUTPUT_TEMPLATE_META_KEY) + .and_then(serde_json::Value::as_str) + }) + .map(str::to_string) + }) +} + async fn lookup_mcp_app_usage_metadata( sess: &Session, server: &str, @@ -1666,6 +1700,7 @@ async fn notify_mcp_tool_call_skip( turn_context: &TurnContext, call_id: &str, invocation: McpInvocation, + mcp_app_resource_uri: Option, message: String, already_started: bool, ) -> Result { @@ -1673,6 +1708,7 @@ async fn notify_mcp_tool_call_skip( let tool_call_begin_event = EventMsg::McpToolCallBegin(McpToolCallBeginEvent { call_id: call_id.to_string(), invocation: invocation.clone(), + mcp_app_resource_uri: mcp_app_resource_uri.clone(), }); notify_mcp_tool_call_event(sess, turn_context, tool_call_begin_event).await; } @@ -1680,6 +1716,7 @@ async fn notify_mcp_tool_call_skip( let tool_call_end_event = EventMsg::McpToolCallEnd(McpToolCallEndEvent { call_id: call_id.to_string(), invocation, + mcp_app_resource_uri, duration: Duration::ZERO, result: Err(message.clone()), }); diff --git a/codex-rs/core/src/mcp_tool_call_tests.rs b/codex-rs/core/src/mcp_tool_call_tests.rs index 201a162ef7..53e838908d 100644 --- a/codex-rs/core/src/mcp_tool_call_tests.rs +++ b/codex-rs/core/src/mcp_tool_call_tests.rs @@ -59,6 +59,7 @@ fn approval_metadata( connector_description: connector_description.map(str::to_string), tool_title: tool_title.map(str::to_string), tool_description: tool_description.map(str::to_string), + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, } @@ -74,6 +75,35 @@ fn prompt_options( } } +#[test] +fn mcp_app_resource_uri_reads_known_tool_meta_keys() { + let nested = serde_json::json!({ + "ui": { + "resourceUri": "ui://widget/nested.html", + }, + }); + assert_eq!( + get_mcp_app_resource_uri(nested.as_object()), + Some("ui://widget/nested.html".to_string()) + ); + + let flat = serde_json::json!({ + "ui/resourceUri": "ui://widget/flat.html", + }); + assert_eq!( + get_mcp_app_resource_uri(flat.as_object()), + Some("ui://widget/flat.html".to_string()) + ); + + let output_template = serde_json::json!({ + "openai/outputTemplate": "ui://widget/output-template.html", + }); + assert_eq!( + get_mcp_app_resource_uri(output_template.as_object()), + Some("ui://widget/output-template.html".to_string()) + ); +} + #[test] fn approval_required_when_read_only_false_and_destructive() { let annotations = annotations(Some(false), Some(true), /*open_world*/ None); @@ -589,6 +619,7 @@ async fn codex_apps_tool_call_request_meta_includes_turn_metadata_and_codex_apps connector_description: Some("Manage events".to_string()), tool_title: Some("Create Event".to_string()), tool_description: Some("Create a calendar event.".to_string()), + mcp_app_resource_uri: None, codex_apps_meta: Some( serde_json::json!({ "resource_uri": "connector://calendar/tools/calendar_create_event", @@ -746,6 +777,7 @@ fn guardian_mcp_review_request_includes_annotations_when_present() { connector_description: None, tool_title: None, tool_description: None, + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -1272,6 +1304,7 @@ async fn approve_mode_skips_when_annotations_do_not_require_approval() { connector_description: None, tool_title: Some("Read Only Tool".to_string()), tool_description: None, + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -1340,6 +1373,7 @@ async fn guardian_mode_skips_auto_when_annotations_do_not_require_approval() { connector_description: None, tool_title: Some("Read Only Tool".to_string()), tool_description: None, + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -1411,6 +1445,7 @@ async fn guardian_mode_mcp_denial_returns_rationale_message() { connector_description: None, tool_title: Some("Dangerous Tool".to_string()), tool_description: Some("Reads calendar data.".to_string()), + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -1462,6 +1497,7 @@ async fn prompt_mode_waits_for_approval_when_annotations_do_not_require_approval connector_description: None, tool_title: Some("Read Only Tool".to_string()), tool_description: None, + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -1539,6 +1575,7 @@ async fn approve_mode_blocks_when_arc_returns_interrupt_for_model() { connector_description: Some("Manage events".to_string()), tool_title: Some("Dangerous Tool".to_string()), tool_description: Some("Performs a risky action.".to_string()), + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -1609,6 +1646,7 @@ async fn custom_approve_mode_blocks_when_arc_returns_interrupt_for_model() { connector_description: None, tool_title: Some("Dangerous Tool".to_string()), tool_description: Some("Performs a risky action.".to_string()), + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -1679,6 +1717,7 @@ async fn approve_mode_blocks_when_arc_returns_interrupt_without_annotations() { connector_description: Some("Manage events".to_string()), tool_title: Some("Dangerous Tool".to_string()), tool_description: Some("Performs a risky action.".to_string()), + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -1757,6 +1796,7 @@ async fn full_access_mode_skips_arc_monitor_for_all_approval_modes() { connector_description: Some("Manage events".to_string()), tool_title: Some("Dangerous Tool".to_string()), tool_description: Some("Performs a risky action.".to_string()), + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -1859,6 +1899,7 @@ async fn approve_mode_routes_arc_ask_user_to_guardian_when_guardian_reviewer_is_ connector_description: Some("Manage events".to_string()), tool_title: Some("Dangerous Tool".to_string()), tool_description: Some("Performs a risky action.".to_string()), + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; diff --git a/codex-rs/core/src/tools/handlers/mcp_resource.rs b/codex-rs/core/src/tools/handlers/mcp_resource.rs index 69058a6226..d7d0256dda 100644 --- a/codex-rs/core/src/tools/handlers/mcp_resource.rs +++ b/codex-rs/core/src/tools/handlers/mcp_resource.rs @@ -562,6 +562,7 @@ async fn emit_tool_call_begin( EventMsg::McpToolCallBegin(McpToolCallBeginEvent { call_id: call_id.to_string(), invocation, + mcp_app_resource_uri: None, }), ) .await; @@ -581,6 +582,7 @@ async fn emit_tool_call_end( EventMsg::McpToolCallEnd(McpToolCallEndEvent { call_id: call_id.to_string(), invocation, + mcp_app_resource_uri: None, duration, result, }), diff --git a/codex-rs/core/tests/common/apps_test_server.rs b/codex-rs/core/tests/common/apps_test_server.rs index 5b039aa149..757fc146f2 100644 --- a/codex-rs/core/tests/common/apps_test_server.rs +++ b/codex-rs/core/tests/common/apps_test_server.rs @@ -21,6 +21,8 @@ const SERVER_VERSION: &str = "1.0.0"; const SEARCHABLE_TOOL_COUNT: usize = 100; pub const CALENDAR_CREATE_EVENT_RESOURCE_URI: &str = "connector://calendar/tools/calendar_create_event"; +pub const CALENDAR_CREATE_EVENT_MCP_APP_RESOURCE_URI: &str = + "ui://widget/calendar-create-event.html"; const CALENDAR_LIST_EVENTS_RESOURCE_URI: &str = "connector://calendar/tools/calendar_list_events"; pub const DOCUMENT_EXTRACT_TEXT_RESOURCE_URI: &str = "connector://calendar/tools/calendar_extract_text"; @@ -206,6 +208,7 @@ impl Respond for CodexAppsJsonRpcResponder { "connector_id": CONNECTOR_ID, "connector_name": self.connector_name.clone(), "connector_description": self.connector_description.clone(), + "openai/outputTemplate": CALENDAR_CREATE_EVENT_MCP_APP_RESOURCE_URI, "_codex_apps": { "resource_uri": CALENDAR_CREATE_EVENT_RESOURCE_URI, "contains_mcp_source": true, diff --git a/codex-rs/core/tests/suite/rmcp_client.rs b/codex-rs/core/tests/suite/rmcp_client.rs index 6d19e6ac52..b63369ea6f 100644 --- a/codex-rs/core/tests/suite/rmcp_client.rs +++ b/codex-rs/core/tests/suite/rmcp_client.rs @@ -736,6 +736,7 @@ async fn stdio_image_responses_round_trip() -> anyhow::Result<()> { tool: "image".to_string(), arguments: Some(json!({})), }, + mcp_app_resource_uri: None, }, ); diff --git a/codex-rs/core/tests/suite/search_tool.rs b/codex-rs/core/tests/suite/search_tool.rs index 624c439150..67cf8503b3 100644 --- a/codex-rs/core/tests/suite/search_tool.rs +++ b/codex-rs/core/tests/suite/search_tool.rs @@ -15,6 +15,7 @@ use codex_protocol::protocol::Op; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::user_input::UserInput; use core_test_support::apps_test_server::AppsTestServer; +use core_test_support::apps_test_server::CALENDAR_CREATE_EVENT_MCP_APP_RESOURCE_URI; use core_test_support::apps_test_server::CALENDAR_CREATE_EVENT_RESOURCE_URI; use core_test_support::responses::ResponsesRequest; use core_test_support::responses::ev_assistant_message; @@ -457,6 +458,19 @@ async fn tool_search_returns_deferred_tools_without_follow_up_tool_injection() - }) .await?; + let EventMsg::McpToolCallBegin(begin) = wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::McpToolCallBegin(_)) + }) + .await + else { + unreachable!("event guard guarantees McpToolCallBegin"); + }; + assert_eq!(begin.call_id, "calendar-call-1"); + assert_eq!( + begin.mcp_app_resource_uri.as_deref(), + Some(CALENDAR_CREATE_EVENT_MCP_APP_RESOURCE_URI) + ); + let EventMsg::McpToolCallEnd(end) = wait_for_event(&test.codex, |event| { matches!(event, EventMsg::McpToolCallEnd(_)) }) @@ -465,6 +479,10 @@ async fn tool_search_returns_deferred_tools_without_follow_up_tool_injection() - unreachable!("event guard guarantees McpToolCallEnd"); }; assert_eq!(end.call_id, "calendar-call-1"); + assert_eq!( + end.mcp_app_resource_uri.as_deref(), + Some(CALENDAR_CREATE_EVENT_MCP_APP_RESOURCE_URI) + ); assert_eq!( end.invocation, McpInvocation { diff --git a/codex-rs/exec/tests/event_processor_with_json_output.rs b/codex-rs/exec/tests/event_processor_with_json_output.rs index 71b5767348..02e52988cb 100644 --- a/codex-rs/exec/tests/event_processor_with_json_output.rs +++ b/codex-rs/exec/tests/event_processor_with_json_output.rs @@ -464,6 +464,7 @@ fn mcp_tool_call_begin_and_end_emit_item_events() { tool: "tool_x".to_string(), status: ApiMcpToolCallStatus::InProgress, arguments: json!({ "key": "value" }), + mcp_app_resource_uri: None, result: None, error: None, duration_ms: None, @@ -479,11 +480,12 @@ fn mcp_tool_call_begin_and_end_emit_item_events() { tool: "tool_x".to_string(), status: ApiMcpToolCallStatus::Completed, arguments: json!({ "key": "value" }), - result: Some(McpToolCallResult { + mcp_app_resource_uri: None, + result: Some(Box::new(McpToolCallResult { content: Vec::new(), structured_content: None, meta: None, - }), + })), error: None, duration_ms: Some(1_000), }, @@ -547,6 +549,7 @@ fn mcp_tool_call_failure_sets_failed_status() { tool: "tool_y".to_string(), status: ApiMcpToolCallStatus::Failed, arguments: json!({ "param": 42 }), + mcp_app_resource_uri: None, result: None, error: Some(McpToolCallError { message: "tool exploded".to_string(), @@ -593,6 +596,7 @@ fn mcp_tool_call_defaults_arguments_and_preserves_structured_content() { tool: "tool_z".to_string(), status: ApiMcpToolCallStatus::InProgress, arguments: serde_json::Value::Null, + mcp_app_resource_uri: None, result: None, error: None, duration_ms: None, @@ -608,14 +612,15 @@ fn mcp_tool_call_defaults_arguments_and_preserves_structured_content() { tool: "tool_z".to_string(), status: ApiMcpToolCallStatus::Completed, arguments: serde_json::Value::Null, - result: Some(McpToolCallResult { + mcp_app_resource_uri: None, + result: Some(Box::new(McpToolCallResult { content: vec![json!({ "type": "text", "text": "done", })], structured_content: Some(json!({ "status": "ok" })), meta: None, - }), + })), error: None, duration_ms: Some(10), }, diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 244f748c82..ca113536e0 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -2362,6 +2362,9 @@ pub struct McpToolCallBeginEvent { /// Identifier so this can be paired with the McpToolCallEnd event. pub call_id: String, pub invocation: McpInvocation, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub mcp_app_resource_uri: Option, } #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS, PartialEq)] @@ -2369,6 +2372,9 @@ pub struct McpToolCallEndEvent { /// Identifier for the corresponding McpToolCallBegin that finished. pub call_id: String, pub invocation: McpInvocation, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub mcp_app_resource_uri: Option, #[ts(type = "string")] pub duration: Duration, /// Result of the tool call. Note this could be an error. diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 2f5258bf2f..9471c29ca6 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -4759,6 +4759,7 @@ impl ChatWidget { invocation, duration, result, + .. } = ev; let extra_cell = match self @@ -5943,6 +5944,7 @@ impl ChatWidget { server, tool, arguments, + mcp_app_resource_uri, result, error, duration_ms, @@ -5955,15 +5957,19 @@ impl ChatWidget { tool, arguments: Some(arguments), }, + mcp_app_resource_uri, duration: Duration::from_millis(duration_ms.unwrap_or_default().max(0) as u64), result: match (result, error) { (_, Some(error)) => Err(error.message), - (Some(result), None) => Ok(codex_protocol::mcp::CallToolResult { - content: result.content, - structured_content: result.structured_content, - is_error: Some(false), - meta: None, - }), + (Some(result), None) => { + let result = *result; + Ok(codex_protocol::mcp::CallToolResult { + content: result.content, + structured_content: result.structured_content, + is_error: Some(false), + meta: None, + }) + } (None, None) => Err("MCP tool call completed without a result".to_string()), }, }); @@ -6453,6 +6459,7 @@ impl ChatWidget { server, tool, arguments, + mcp_app_resource_uri, .. } => { self.on_mcp_tool_call_begin(McpToolCallBeginEvent { @@ -6462,6 +6469,7 @@ impl ChatWidget { tool, arguments: Some(arguments), }, + mcp_app_resource_uri, }); } ThreadItem::WebSearch { id, .. } => { From 48cf3ed7b03267c42e1a3d39e2b345a01493ad36 Mon Sep 17 00:00:00 2001 From: xl-openai Date: Wed, 15 Apr 2026 23:13:17 -0700 Subject: [PATCH 55/84] Extract plugin loading and marketplace logic into codex-core-plugins (#18070) Split plugin loading, marketplace, and related infrastructure out of core into codex-core-plugins, while keeping the core-facing configuration and orchestration flow in codex-core. --------- Co-authored-by: Codex --- codex-rs/Cargo.lock | 29 + codex-rs/Cargo.toml | 2 + codex-rs/app-server/Cargo.toml | 1 + .../app-server/src/codex_message_processor.rs | 13 +- codex-rs/app-server/src/config_api.rs | 4 +- codex-rs/core-plugins/BUILD.bazel | 15 + codex-rs/core-plugins/Cargo.toml | 39 + codex-rs/core-plugins/src/lib.rs | 6 + codex-rs/core-plugins/src/loader.rs | 807 ++++++++++++++++ .../plugins => core-plugins/src}/manifest.rs | 14 +- .../src}/marketplace.rs | 11 +- .../src}/marketplace_tests.rs | 0 .../plugins => core-plugins/src}/remote.rs | 32 +- .../src/plugins => core-plugins/src}/store.rs | 10 +- .../src}/store_tests.rs | 0 .../plugins => core-plugins/src}/toggles.rs | 0 codex-rs/core/Cargo.toml | 1 + codex-rs/core/src/lib.rs | 2 - .../src/plugins/installed_marketplaces.rs | 2 +- codex-rs/core/src/plugins/manager.rs | 881 ++---------------- codex-rs/core/src/plugins/manager_tests.rs | 22 +- .../src/plugins/marketplace_add/metadata.rs | 2 +- .../src/plugins/marketplace_add/source.rs | 2 +- codex-rs/core/src/plugins/mod.rs | 22 - 24 files changed, 1025 insertions(+), 892 deletions(-) create mode 100644 codex-rs/core-plugins/BUILD.bazel create mode 100644 codex-rs/core-plugins/Cargo.toml create mode 100644 codex-rs/core-plugins/src/lib.rs create mode 100644 codex-rs/core-plugins/src/loader.rs rename codex-rs/{core/src/plugins => core-plugins/src}/manifest.rs (98%) rename codex-rs/{core/src/plugins => core-plugins/src}/marketplace.rs (98%) rename codex-rs/{core/src/plugins => core-plugins/src}/marketplace_tests.rs (100%) rename codex-rs/{core/src/plugins => core-plugins/src}/remote.rs (94%) rename codex-rs/{core/src/plugins => core-plugins/src}/store.rs (97%) rename codex-rs/{core/src/plugins => core-plugins/src}/store_tests.rs (100%) rename codex-rs/{core/src/plugins => core-plugins/src}/toggles.rs (100%) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 2d39c42df4..1be5b45b6a 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1446,6 +1446,7 @@ dependencies = [ "codex-cloud-requirements", "codex-config", "codex-core", + "codex-core-plugins", "codex-exec-server", "codex-features", "codex-feedback", @@ -1913,6 +1914,7 @@ dependencies = [ "codex-code-mode", "codex-config", "codex-connectors", + "codex-core-plugins", "codex-core-skills", "codex-exec-server", "codex-execpolicy", @@ -2013,6 +2015,33 @@ dependencies = [ "zstd 0.13.3", ] +[[package]] +name = "codex-core-plugins" +version = "0.0.0" +dependencies = [ + "codex-app-server-protocol", + "codex-config", + "codex-core-skills", + "codex-exec-server", + "codex-git-utils", + "codex-login", + "codex-plugin", + "codex-protocol", + "codex-utils-absolute-path", + "codex-utils-plugins", + "dirs", + "pretty_assertions", + "reqwest", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.18", + "tokio", + "toml 0.9.11+spec-1.1.0", + "tracing", + "url", +] + [[package]] name = "codex-core-skills" version = "0.0.0" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 47c0d348bf..e7867304e9 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -28,6 +28,7 @@ members = [ "shell-escalation", "skills", "core", + "core-plugins", "core-skills", "hooks", "instructions", @@ -128,6 +129,7 @@ codex-code-mode = { path = "code-mode" } codex-config = { path = "config" } codex-connectors = { path = "connectors" } codex-core = { path = "core" } +codex-core-plugins = { path = "core-plugins" } codex-core-skills = { path = "core-skills" } codex-exec = { path = "exec" } codex-exec-server = { path = "exec-server" } diff --git a/codex-rs/app-server/Cargo.toml b/codex-rs/app-server/Cargo.toml index 408f39fafe..097abdb713 100644 --- a/codex-rs/app-server/Cargo.toml +++ b/codex-rs/app-server/Cargo.toml @@ -34,6 +34,7 @@ codex-arg0 = { workspace = true } codex-cloud-requirements = { workspace = true } codex-config = { workspace = true } codex-core = { workspace = true } +codex-core-plugins = { workspace = true } codex-exec-server = { workspace = true } codex-features = { workspace = true } codex-git-utils = { workspace = true } diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index f805ec3ff6..94c1e56645 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -232,22 +232,23 @@ use codex_core::find_thread_names_by_ids; use codex_core::find_thread_path_by_id_str; use codex_core::path_utils; use codex_core::plugins::MarketplaceAddError; -use codex_core::plugins::MarketplaceError; -use codex_core::plugins::MarketplacePluginSource; use codex_core::plugins::OPENAI_CURATED_MARKETPLACE_NAME; use codex_core::plugins::PluginInstallError as CorePluginInstallError; use codex_core::plugins::PluginInstallRequest; use codex_core::plugins::PluginReadRequest; use codex_core::plugins::PluginUninstallError as CorePluginUninstallError; use codex_core::plugins::add_marketplace as add_marketplace_to_codex_home; -use codex_core::plugins::load_plugin_apps; -use codex_core::plugins::load_plugin_mcp_servers; use codex_core::read_head_for_summary; use codex_core::read_session_meta_line; use codex_core::sandboxing::SandboxPermissions; use codex_core::windows_sandbox::WindowsSandboxLevelExt; use codex_core::windows_sandbox::WindowsSandboxSetupMode as CoreWindowsSandboxSetupMode; use codex_core::windows_sandbox::WindowsSandboxSetupRequest; +use codex_core_plugins::loader::load_plugin_apps; +use codex_core_plugins::loader::load_plugin_mcp_servers; +use codex_core_plugins::manifest::PluginManifestInterface; +use codex_core_plugins::marketplace::MarketplaceError; +use codex_core_plugins::marketplace::MarketplacePluginSource; use codex_exec_server::LOCAL_FS; use codex_features::FEATURES; use codex_features::Feature; @@ -8834,9 +8835,7 @@ fn plugin_skills_to_info( .collect() } -fn plugin_interface_to_info( - interface: codex_core::plugins::PluginManifestInterface, -) -> PluginInterface { +fn plugin_interface_to_info(interface: PluginManifestInterface) -> PluginInterface { PluginInterface { display_name: interface.display_name, short_description: interface.short_description, diff --git a/codex-rs/app-server/src/config_api.rs b/codex-rs/app-server/src/config_api.rs index 4d0b450d73..2bc7447300 100644 --- a/codex-rs/app-server/src/config_api.rs +++ b/codex-rs/app-server/src/config_api.rs @@ -27,8 +27,8 @@ use codex_core::config_loader::LoaderOverrides; use codex_core::config_loader::ResidencyRequirement as CoreResidencyRequirement; use codex_core::config_loader::SandboxModeRequirement as CoreSandboxModeRequirement; use codex_core::plugins::PluginId; -use codex_core::plugins::collect_plugin_enabled_candidates; -use codex_core::plugins::installed_plugin_telemetry_metadata; +use codex_core_plugins::loader::installed_plugin_telemetry_metadata; +use codex_core_plugins::toggles::collect_plugin_enabled_candidates; use codex_features::canonical_feature_for_key; use codex_features::feature_for_key; use codex_protocol::config_types::WebSearchMode; diff --git a/codex-rs/core-plugins/BUILD.bazel b/codex-rs/core-plugins/BUILD.bazel new file mode 100644 index 0000000000..aa19b9f368 --- /dev/null +++ b/codex-rs/core-plugins/BUILD.bazel @@ -0,0 +1,15 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "core-plugins", + crate_name = "codex_core_plugins", + compile_data = glob( + include = ["**"], + exclude = [ + "**/* *", + "BUILD.bazel", + "Cargo.toml", + ], + allow_empty = True, + ), +) diff --git a/codex-rs/core-plugins/Cargo.toml b/codex-rs/core-plugins/Cargo.toml new file mode 100644 index 0000000000..02e121f0e6 --- /dev/null +++ b/codex-rs/core-plugins/Cargo.toml @@ -0,0 +1,39 @@ +[package] +edition.workspace = true +license.workspace = true +name = "codex-core-plugins" +version.workspace = true + +[lib] +doctest = false +name = "codex_core_plugins" +path = "src/lib.rs" + +[lints] +workspace = true + +[dependencies] +codex-app-server-protocol = { workspace = true } +codex-config = { workspace = true } +codex-core-skills = { workspace = true } +codex-exec-server = { workspace = true } +codex-git-utils = { workspace = true } +codex-login = { workspace = true } +codex-plugin = { workspace = true } +codex-protocol = { workspace = true } +codex-utils-absolute-path = { workspace = true } +codex-utils-plugins = { workspace = true } +dirs = { workspace = true } +reqwest = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +tempfile = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["fs"] } +toml = { workspace = true } +tracing = { workspace = true } +url = { workspace = true } + +[dev-dependencies] +pretty_assertions = { workspace = true } +tempfile = { workspace = true } diff --git a/codex-rs/core-plugins/src/lib.rs b/codex-rs/core-plugins/src/lib.rs new file mode 100644 index 0000000000..e075158cb1 --- /dev/null +++ b/codex-rs/core-plugins/src/lib.rs @@ -0,0 +1,6 @@ +pub mod loader; +pub mod manifest; +pub mod marketplace; +pub mod remote; +pub mod store; +pub mod toggles; diff --git a/codex-rs/core-plugins/src/loader.rs b/codex-rs/core-plugins/src/loader.rs new file mode 100644 index 0000000000..206add5b53 --- /dev/null +++ b/codex-rs/core-plugins/src/loader.rs @@ -0,0 +1,807 @@ +use crate::manifest::PluginManifestPaths; +use crate::manifest::load_plugin_manifest; +use crate::marketplace::MarketplacePluginSource; +use crate::marketplace::list_marketplaces; +use crate::marketplace::load_marketplace; +use crate::store::PluginStore; +use crate::store::plugin_version_for_source; +use codex_config::ConfigLayerStack; +use codex_config::types::McpServerConfig; +use codex_config::types::PluginConfig; +use codex_core_skills::SkillMetadata; +use codex_core_skills::config_rules::SkillConfigRules; +use codex_core_skills::config_rules::resolve_disabled_skill_paths; +use codex_core_skills::config_rules::skill_config_rules_from_stack; +use codex_core_skills::loader::SkillRoot; +use codex_core_skills::loader::load_skills_from_roots; +use codex_exec_server::LOCAL_FS; +use codex_plugin::AppConnectorId; +use codex_plugin::LoadedPlugin; +use codex_plugin::PluginCapabilitySummary; +use codex_plugin::PluginId; +use codex_plugin::PluginIdError; +use codex_plugin::PluginLoadOutcome; +use codex_plugin::PluginTelemetryMetadata; +use codex_protocol::protocol::Product; +use codex_protocol::protocol::SkillScope; +use codex_utils_absolute_path::AbsolutePathBuf; +use serde::Deserialize; +use serde_json::Map as JsonMap; +use serde_json::Value as JsonValue; +use std::collections::HashMap; +use std::collections::HashSet; +use std::fs; +use std::path::Path; +use std::sync::Arc; +use tracing::warn; + +const DEFAULT_SKILLS_DIR_NAME: &str = "skills"; +const DEFAULT_MCP_CONFIG_FILE: &str = ".mcp.json"; +const DEFAULT_APP_CONFIG_FILE: &str = ".app.json"; +const OPENAI_CURATED_MARKETPLACE_NAME: &str = "openai-curated"; +const CONFIG_TOML_FILE: &str = "config.toml"; + +pub fn log_plugin_load_errors(outcome: &PluginLoadOutcome) { + for plugin in outcome + .plugins() + .iter() + .filter(|plugin| plugin.error.is_some()) + { + if let Some(error) = plugin.error.as_deref() { + warn!( + plugin = plugin.config_name, + path = %plugin.root.display(), + "failed to load plugin: {error}" + ); + } + } +} + +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +struct PluginMcpFile { + #[serde(default)] + mcp_servers: HashMap, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +struct PluginAppFile { + #[serde(default)] + apps: HashMap, +} + +#[derive(Debug, Default, Deserialize)] +struct PluginAppConfig { + id: String, +} + +pub async fn load_plugins_from_layer_stack( + config_layer_stack: &ConfigLayerStack, + store: &PluginStore, + restriction_product: Option, +) -> PluginLoadOutcome { + let skill_config_rules = skill_config_rules_from_stack(config_layer_stack); + let mut configured_plugins: Vec<_> = configured_plugins_from_stack(config_layer_stack) + .into_iter() + .collect(); + configured_plugins.sort_unstable_by(|(a, _), (b, _)| a.cmp(b)); + + let mut plugins = Vec::with_capacity(configured_plugins.len()); + let mut seen_mcp_server_names = HashMap::::new(); + for (configured_name, plugin) in configured_plugins { + let loaded_plugin = load_plugin( + configured_name.clone(), + &plugin, + store, + restriction_product, + &skill_config_rules, + ) + .await; + for name in loaded_plugin.mcp_servers.keys() { + if let Some(previous_plugin) = + seen_mcp_server_names.insert(name.clone(), configured_name.clone()) + { + warn!( + plugin = configured_name, + previous_plugin, + server = name, + "skipping duplicate plugin MCP server name" + ); + } + } + plugins.push(loaded_plugin); + } + + PluginLoadOutcome::from_plugins(plugins) +} + +pub fn refresh_curated_plugin_cache( + codex_home: &Path, + plugin_version: &str, + configured_curated_plugin_ids: &[PluginId], +) -> Result { + let store = PluginStore::new(codex_home.to_path_buf()); + let curated_marketplace_path = AbsolutePathBuf::try_from( + codex_home + .join(".tmp/plugins") + .join(".agents/plugins/marketplace.json"), + ) + .map_err(|_| "local curated marketplace is not available".to_string())?; + let curated_marketplace = load_marketplace(&curated_marketplace_path) + .map_err(|err| format!("failed to load curated marketplace for cache refresh: {err}"))?; + + let mut plugin_sources = HashMap::::new(); + for plugin in curated_marketplace.plugins { + let plugin_name = plugin.name; + if plugin_sources.contains_key(&plugin_name) { + warn!( + plugin = plugin_name, + marketplace = OPENAI_CURATED_MARKETPLACE_NAME, + "ignoring duplicate curated plugin entry during cache refresh" + ); + continue; + } + let source_path = match plugin.source { + MarketplacePluginSource::Local { path } => path, + }; + plugin_sources.insert(plugin_name, source_path); + } + + let mut cache_refreshed = false; + for plugin_id in configured_curated_plugin_ids { + if store.active_plugin_version(plugin_id).as_deref() == Some(plugin_version) { + continue; + } + + let Some(source_path) = plugin_sources.get(&plugin_id.plugin_name).cloned() else { + warn!( + plugin = plugin_id.plugin_name, + marketplace = OPENAI_CURATED_MARKETPLACE_NAME, + "configured curated plugin no longer exists in curated marketplace during cache refresh" + ); + continue; + }; + + store + .install_with_version(source_path, plugin_id.clone(), plugin_version.to_string()) + .map_err(|err| { + format!( + "failed to refresh curated plugin cache for {}: {err}", + plugin_id.as_key() + ) + })?; + cache_refreshed = true; + } + + Ok(cache_refreshed) +} + +pub fn refresh_non_curated_plugin_cache( + codex_home: &Path, + additional_roots: &[AbsolutePathBuf], +) -> Result { + let configured_non_curated_plugin_ids = + non_curated_plugin_ids_from_config_keys(configured_plugins_from_codex_home( + codex_home, + "failed to read user config while refreshing non-curated plugin cache", + "failed to parse user config while refreshing non-curated plugin cache", + )); + if configured_non_curated_plugin_ids.is_empty() { + return Ok(false); + } + let configured_non_curated_plugin_keys = configured_non_curated_plugin_ids + .iter() + .map(PluginId::as_key) + .collect::>(); + + let store = PluginStore::new(codex_home.to_path_buf()); + let marketplace_outcome = list_marketplaces(additional_roots) + .map_err(|err| format!("failed to discover marketplaces for cache refresh: {err}"))?; + let mut plugin_sources = HashMap::::new(); + + for marketplace in marketplace_outcome.marketplaces { + if marketplace.name == OPENAI_CURATED_MARKETPLACE_NAME { + continue; + } + + for plugin in marketplace.plugins { + let plugin_id = + PluginId::new(plugin.name.clone(), marketplace.name.clone()).map_err(|err| { + match err { + PluginIdError::Invalid(message) => { + format!("failed to prepare non-curated plugin cache refresh: {message}") + } + } + })?; + let plugin_key = plugin_id.as_key(); + if !configured_non_curated_plugin_keys.contains(&plugin_key) { + continue; + } + if plugin_sources.contains_key(&plugin_key) { + warn!( + plugin = plugin.name, + marketplace = marketplace.name, + "ignoring duplicate non-curated plugin entry during cache refresh" + ); + continue; + } + + let source_path = match plugin.source { + MarketplacePluginSource::Local { path } => path, + }; + let plugin_version = plugin_version_for_source(source_path.as_path()) + .map_err(|err| format!("failed to read plugin version for {plugin_key}: {err}"))?; + plugin_sources.insert(plugin_key, (source_path, plugin_version)); + } + } + + let mut cache_refreshed = false; + for plugin_id in configured_non_curated_plugin_ids { + let plugin_key = plugin_id.as_key(); + let Some((source_path, plugin_version)) = plugin_sources.get(&plugin_key).cloned() else { + warn!( + plugin = plugin_id.plugin_name, + marketplace = plugin_id.marketplace_name, + "configured non-curated plugin no longer exists in discovered marketplaces during cache refresh" + ); + continue; + }; + + if store.active_plugin_version(&plugin_id).as_deref() == Some(plugin_version.as_str()) { + continue; + } + + store + .install_with_version(source_path, plugin_id.clone(), plugin_version) + .map_err(|err| format!("failed to refresh plugin cache for {plugin_key}: {err}"))?; + cache_refreshed = true; + } + + Ok(cache_refreshed) +} + +fn configured_plugins_from_stack( + config_layer_stack: &ConfigLayerStack, +) -> HashMap { + let Some(user_layer) = config_layer_stack.get_user_layer() else { + return HashMap::new(); + }; + configured_plugins_from_user_config_value(&user_layer.config) +} + +fn configured_plugins_from_user_config_value( + user_config: &toml::Value, +) -> HashMap { + let Some(plugins_value) = user_config.get("plugins") else { + return HashMap::new(); + }; + match plugins_value.clone().try_into() { + Ok(plugins) => plugins, + Err(err) => { + warn!("invalid plugins config: {err}"); + HashMap::new() + } + } +} + +fn configured_plugins_from_codex_home( + codex_home: &Path, + read_error_message: &str, + parse_error_message: &str, +) -> HashMap { + let config_path = codex_home.join(CONFIG_TOML_FILE); + let user_config = match fs::read_to_string(&config_path) { + Ok(user_config) => user_config, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return HashMap::new(), + Err(err) => { + warn!( + path = %config_path.display(), + error = %err, + "{read_error_message}" + ); + return HashMap::new(); + } + }; + + let user_config = match toml::from_str::(&user_config) { + Ok(user_config) => user_config, + Err(err) => { + warn!( + path = %config_path.display(), + error = %err, + "{parse_error_message}" + ); + return HashMap::new(); + } + }; + + configured_plugins_from_user_config_value(&user_config) +} + +fn configured_plugin_ids( + configured_plugins: HashMap, + invalid_plugin_key_message: &str, +) -> Vec { + configured_plugins + .into_keys() + .filter_map(|plugin_key| match PluginId::parse(&plugin_key) { + Ok(plugin_id) => Some(plugin_id), + Err(err) => { + warn!( + plugin_key, + error = %err, + "{invalid_plugin_key_message}" + ); + None + } + }) + .collect() +} + +fn curated_plugin_ids_from_config_keys( + configured_plugins: HashMap, +) -> Vec { + let mut configured_curated_plugin_ids = configured_plugin_ids( + configured_plugins, + "ignoring invalid configured plugin key during curated sync setup", + ) + .into_iter() + .filter(|plugin_id| plugin_id.marketplace_name == OPENAI_CURATED_MARKETPLACE_NAME) + .collect::>(); + configured_curated_plugin_ids.sort_unstable_by_key(PluginId::as_key); + configured_curated_plugin_ids +} + +fn non_curated_plugin_ids_from_config_keys( + configured_plugins: HashMap, +) -> Vec { + let mut configured_non_curated_plugin_ids = configured_plugin_ids( + configured_plugins, + "ignoring invalid plugin key during non-curated cache refresh setup", + ) + .into_iter() + .filter(|plugin_id| plugin_id.marketplace_name != OPENAI_CURATED_MARKETPLACE_NAME) + .collect::>(); + configured_non_curated_plugin_ids.sort_unstable_by_key(PluginId::as_key); + configured_non_curated_plugin_ids +} + +pub fn configured_curated_plugin_ids_from_codex_home(codex_home: &Path) -> Vec { + curated_plugin_ids_from_config_keys(configured_plugins_from_codex_home( + codex_home, + "failed to read user config while refreshing curated plugin cache", + "failed to parse user config while refreshing curated plugin cache", + )) +} + +async fn load_plugin( + config_name: String, + plugin: &PluginConfig, + store: &PluginStore, + restriction_product: Option, + skill_config_rules: &SkillConfigRules, +) -> LoadedPlugin { + let plugin_id = PluginId::parse(&config_name); + let active_plugin_root = plugin_id + .as_ref() + .ok() + .and_then(|plugin_id| store.active_plugin_root(plugin_id)); + let root = active_plugin_root + .clone() + .unwrap_or_else(|| match &plugin_id { + Ok(plugin_id) => store.plugin_base_root(plugin_id), + Err(_) => store.root().clone(), + }); + let mut loaded_plugin = LoadedPlugin { + config_name, + manifest_name: None, + manifest_description: None, + root, + enabled: plugin.enabled, + skill_roots: Vec::new(), + disabled_skill_paths: HashSet::new(), + has_enabled_skills: false, + mcp_servers: HashMap::new(), + apps: Vec::new(), + error: None, + }; + + if !plugin.enabled { + return loaded_plugin; + } + + let plugin_root = match plugin_id { + Ok(_) => match active_plugin_root { + Some(plugin_root) => plugin_root, + None => { + loaded_plugin.error = Some("plugin is not installed".to_string()); + return loaded_plugin; + } + }, + Err(err) => { + loaded_plugin.error = Some(err.to_string()); + return loaded_plugin; + } + }; + + if !plugin_root.as_path().is_dir() { + loaded_plugin.error = Some("path does not exist or is not a directory".to_string()); + return loaded_plugin; + } + + let Some(manifest) = load_plugin_manifest(plugin_root.as_path()) else { + loaded_plugin.error = Some("missing or invalid .codex-plugin/plugin.json".to_string()); + return loaded_plugin; + }; + + let manifest_paths = &manifest.paths; + loaded_plugin.manifest_name = manifest + .interface + .as_ref() + .and_then(|interface| interface.display_name.as_deref()) + .map(str::trim) + .filter(|display_name| !display_name.is_empty()) + .map(str::to_string) + .or_else(|| Some(manifest.name.clone())); + loaded_plugin.manifest_description = manifest.description.clone(); + loaded_plugin.skill_roots = plugin_skill_roots(&plugin_root, manifest_paths); + let resolved_skills = load_plugin_skills( + &plugin_root, + manifest_paths, + restriction_product, + skill_config_rules, + ) + .await; + let has_enabled_skills = resolved_skills.has_enabled_skills(); + loaded_plugin.disabled_skill_paths = resolved_skills.disabled_skill_paths; + loaded_plugin.has_enabled_skills = has_enabled_skills; + let mut mcp_servers = HashMap::new(); + for mcp_config_path in plugin_mcp_config_paths(plugin_root.as_path(), manifest_paths) { + let plugin_mcp = load_mcp_servers_from_file(plugin_root.as_path(), &mcp_config_path).await; + for (name, config) in plugin_mcp.mcp_servers { + if mcp_servers.insert(name.clone(), config).is_some() { + warn!( + plugin = %plugin_root.display(), + path = %mcp_config_path.display(), + server = name, + "plugin MCP file overwrote an earlier server definition" + ); + } + } + } + loaded_plugin.mcp_servers = mcp_servers; + loaded_plugin.apps = load_plugin_apps(plugin_root.as_path()).await; + loaded_plugin +} + +#[derive(Debug, Clone)] +pub struct ResolvedPluginSkills { + pub skills: Vec, + pub disabled_skill_paths: HashSet, + pub had_errors: bool, +} + +impl ResolvedPluginSkills { + pub fn has_enabled_skills(&self) -> bool { + self.had_errors + || self + .skills + .iter() + .any(|skill| !self.disabled_skill_paths.contains(&skill.path_to_skills_md)) + } +} + +pub async fn load_plugin_skills( + plugin_root: &AbsolutePathBuf, + manifest_paths: &PluginManifestPaths, + restriction_product: Option, + skill_config_rules: &SkillConfigRules, +) -> ResolvedPluginSkills { + let roots = plugin_skill_roots(plugin_root, manifest_paths) + .into_iter() + .map(|path| SkillRoot { + path, + scope: SkillScope::User, + file_system: Arc::clone(&LOCAL_FS), + }) + .collect::>(); + let outcome = load_skills_from_roots(roots).await; + let had_errors = !outcome.errors.is_empty(); + let skills = outcome + .skills + .into_iter() + .filter(|skill| skill.matches_product_restriction_for_product(restriction_product)) + .collect::>(); + let disabled_skill_paths = resolve_disabled_skill_paths(&skills, skill_config_rules); + + ResolvedPluginSkills { + skills, + disabled_skill_paths, + had_errors, + } +} + +fn plugin_skill_roots( + plugin_root: &AbsolutePathBuf, + manifest_paths: &PluginManifestPaths, +) -> Vec { + let mut paths = default_skill_roots(plugin_root); + if let Some(path) = &manifest_paths.skills { + paths.push(path.clone()); + } + paths.sort_unstable(); + paths.dedup(); + paths +} + +fn default_skill_roots(plugin_root: &AbsolutePathBuf) -> Vec { + let skills_dir = plugin_root.join(DEFAULT_SKILLS_DIR_NAME); + if skills_dir.is_dir() { + vec![skills_dir] + } else { + Vec::new() + } +} + +fn plugin_mcp_config_paths( + plugin_root: &Path, + manifest_paths: &PluginManifestPaths, +) -> Vec { + if let Some(path) = &manifest_paths.mcp_servers { + return vec![path.clone()]; + } + default_mcp_config_paths(plugin_root) +} + +fn default_mcp_config_paths(plugin_root: &Path) -> Vec { + let mut paths = Vec::new(); + let default_path = plugin_root.join(DEFAULT_MCP_CONFIG_FILE); + if default_path.is_file() + && let Ok(default_path) = AbsolutePathBuf::try_from(default_path) + { + paths.push(default_path); + } + paths.sort_unstable_by(|left, right| left.as_path().cmp(right.as_path())); + paths.dedup_by(|left, right| left.as_path() == right.as_path()); + paths +} + +pub async fn load_plugin_apps(plugin_root: &Path) -> Vec { + if let Some(manifest) = load_plugin_manifest(plugin_root) { + return load_apps_from_paths( + plugin_root, + plugin_app_config_paths(plugin_root, &manifest.paths), + ) + .await; + } + load_apps_from_paths(plugin_root, default_app_config_paths(plugin_root)).await +} + +fn plugin_app_config_paths( + plugin_root: &Path, + manifest_paths: &PluginManifestPaths, +) -> Vec { + if let Some(path) = &manifest_paths.apps { + return vec![path.clone()]; + } + default_app_config_paths(plugin_root) +} + +fn default_app_config_paths(plugin_root: &Path) -> Vec { + let mut paths = Vec::new(); + let default_path = plugin_root.join(DEFAULT_APP_CONFIG_FILE); + if default_path.is_file() + && let Ok(default_path) = AbsolutePathBuf::try_from(default_path) + { + paths.push(default_path); + } + paths.sort_unstable_by(|left, right| left.as_path().cmp(right.as_path())); + paths.dedup_by(|left, right| left.as_path() == right.as_path()); + paths +} + +async fn load_apps_from_paths( + plugin_root: &Path, + app_config_paths: Vec, +) -> Vec { + let mut connector_ids = Vec::new(); + for app_config_path in app_config_paths { + let Ok(contents) = tokio::fs::read_to_string(app_config_path.as_path()).await else { + continue; + }; + let parsed = match serde_json::from_str::(&contents) { + Ok(parsed) => parsed, + Err(err) => { + warn!( + path = %app_config_path.display(), + "failed to parse plugin app config: {err}" + ); + continue; + } + }; + + let mut apps: Vec = parsed.apps.into_values().collect(); + apps.sort_unstable_by(|left, right| left.id.cmp(&right.id)); + + connector_ids.extend(apps.into_iter().filter_map(|app| { + if app.id.trim().is_empty() { + warn!( + plugin = %plugin_root.display(), + "plugin app config is missing an app id" + ); + None + } else { + Some(AppConnectorId(app.id)) + } + })); + } + connector_ids.dedup(); + connector_ids +} + +pub async fn plugin_telemetry_metadata_from_root( + plugin_id: &PluginId, + plugin_root: &AbsolutePathBuf, +) -> PluginTelemetryMetadata { + let Some(manifest) = load_plugin_manifest(plugin_root.as_path()) else { + return PluginTelemetryMetadata::from_plugin_id(plugin_id); + }; + + let manifest_paths = &manifest.paths; + let has_skills = !plugin_skill_roots(plugin_root, manifest_paths).is_empty(); + let mut mcp_server_names = Vec::new(); + for path in plugin_mcp_config_paths(plugin_root.as_path(), manifest_paths) { + mcp_server_names.extend( + load_mcp_servers_from_file(plugin_root.as_path(), &path) + .await + .mcp_servers + .into_keys(), + ); + } + mcp_server_names.sort_unstable(); + mcp_server_names.dedup(); + + PluginTelemetryMetadata { + plugin_id: plugin_id.clone(), + capability_summary: Some(PluginCapabilitySummary { + config_name: plugin_id.as_key(), + display_name: plugin_id.plugin_name.clone(), + description: None, + has_skills, + mcp_server_names, + app_connector_ids: load_apps_from_paths( + plugin_root.as_path(), + plugin_app_config_paths(plugin_root.as_path(), manifest_paths), + ) + .await, + }), + } +} + +pub async fn load_plugin_mcp_servers(plugin_root: &Path) -> HashMap { + let Some(manifest) = load_plugin_manifest(plugin_root) else { + return HashMap::new(); + }; + + let mut mcp_servers = HashMap::new(); + for mcp_config_path in plugin_mcp_config_paths(plugin_root, &manifest.paths) { + let plugin_mcp = load_mcp_servers_from_file(plugin_root, &mcp_config_path).await; + for (name, config) in plugin_mcp.mcp_servers { + mcp_servers.entry(name).or_insert(config); + } + } + + mcp_servers +} + +pub async fn installed_plugin_telemetry_metadata( + codex_home: &Path, + plugin_id: &PluginId, +) -> PluginTelemetryMetadata { + let store = PluginStore::new(codex_home.to_path_buf()); + let Some(plugin_root) = store.active_plugin_root(plugin_id) else { + return PluginTelemetryMetadata::from_plugin_id(plugin_id); + }; + + plugin_telemetry_metadata_from_root(plugin_id, &plugin_root).await +} + +async fn load_mcp_servers_from_file( + plugin_root: &Path, + mcp_config_path: &AbsolutePathBuf, +) -> PluginMcpDiscovery { + let Ok(contents) = tokio::fs::read_to_string(mcp_config_path.as_path()).await else { + return PluginMcpDiscovery::default(); + }; + let parsed = match serde_json::from_str::(&contents) { + Ok(parsed) => parsed, + Err(err) => { + warn!( + path = %mcp_config_path.display(), + "failed to parse plugin MCP config: {err}" + ); + return PluginMcpDiscovery::default(); + } + }; + normalize_plugin_mcp_servers( + plugin_root, + parsed.mcp_servers, + mcp_config_path.to_string_lossy().as_ref(), + ) +} + +fn normalize_plugin_mcp_servers( + plugin_root: &Path, + plugin_mcp_servers: HashMap, + source: &str, +) -> PluginMcpDiscovery { + let mut mcp_servers = HashMap::new(); + + for (name, config_value) in plugin_mcp_servers { + let normalized = normalize_plugin_mcp_server_value(plugin_root, config_value); + match serde_json::from_value::(JsonValue::Object(normalized)) { + Ok(config) => { + mcp_servers.insert(name, config); + } + Err(err) => { + warn!( + plugin = %plugin_root.display(), + server = name, + "failed to parse plugin MCP server from {source}: {err}" + ); + } + } + } + + PluginMcpDiscovery { mcp_servers } +} + +fn normalize_plugin_mcp_server_value( + plugin_root: &Path, + value: JsonValue, +) -> JsonMap { + let mut object = match value { + JsonValue::Object(object) => object, + _ => return JsonMap::new(), + }; + + if let Some(JsonValue::String(transport_type)) = object.remove("type") { + match transport_type.as_str() { + "http" | "streamable_http" | "streamable-http" => {} + "stdio" => {} + other => { + warn!( + plugin = %plugin_root.display(), + transport = other, + "plugin MCP server uses an unknown transport type" + ); + } + } + } + + if let Some(JsonValue::Object(oauth)) = object.remove("oauth") + && oauth.contains_key("callbackPort") + { + warn!( + plugin = %plugin_root.display(), + "plugin MCP server OAuth callbackPort is ignored; Codex uses global MCP OAuth callback settings" + ); + } + + if let Some(JsonValue::String(cwd)) = object.get("cwd") + && !Path::new(cwd).is_absolute() + { + object.insert( + "cwd".to_string(), + JsonValue::String(plugin_root.join(cwd).display().to_string()), + ); + } + + object +} + +#[derive(Debug, Default)] +struct PluginMcpDiscovery { + mcp_servers: HashMap, +} diff --git a/codex-rs/core/src/plugins/manifest.rs b/codex-rs/core-plugins/src/manifest.rs similarity index 98% rename from codex-rs/core/src/plugins/manifest.rs rename to codex-rs/core-plugins/src/manifest.rs index f1253441bd..f58b882ca1 100644 --- a/codex-rs/core/src/plugins/manifest.rs +++ b/codex-rs/core-plugins/src/manifest.rs @@ -30,12 +30,12 @@ struct RawPluginManifest { } #[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct PluginManifest { - pub(crate) name: String, - pub(crate) version: Option, - pub(crate) description: Option, - pub(crate) paths: PluginManifestPaths, - pub(crate) interface: Option, +pub struct PluginManifest { + pub name: String, + pub version: Option, + pub description: Option, + pub paths: PluginManifestPaths, + pub interface: Option, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -114,7 +114,7 @@ enum RawPluginManifestDefaultPromptEntry { Invalid(JsonValue), } -pub(crate) fn load_plugin_manifest(plugin_root: &Path) -> Option { +pub fn load_plugin_manifest(plugin_root: &Path) -> Option { let manifest_path = plugin_root.join(PLUGIN_MANIFEST_PATH); if !manifest_path.is_file() { return None; diff --git a/codex-rs/core/src/plugins/marketplace.rs b/codex-rs/core-plugins/src/marketplace.rs similarity index 98% rename from codex-rs/core/src/plugins/marketplace.rs rename to codex-rs/core-plugins/src/marketplace.rs index 96c73e8e19..79e5237ad0 100644 --- a/codex-rs/core/src/plugins/marketplace.rs +++ b/codex-rs/core-plugins/src/marketplace.rs @@ -1,5 +1,5 @@ -use super::PluginManifestInterface; -use super::load_plugin_manifest; +use crate::manifest::PluginManifestInterface; +use crate::manifest::load_plugin_manifest; use codex_app_server_protocol::PluginAuthPolicy; use codex_app_server_protocol::PluginInstallPolicy; use codex_git_utils::get_git_repo_root; @@ -231,7 +231,7 @@ pub fn validate_marketplace_root(root: &Path) -> Result Option { +pub fn find_marketplace_manifest_path(root: &Path) -> Option { MARKETPLACE_MANIFEST_RELATIVE_PATHS .iter() .find_map(|relative_path| { @@ -265,7 +265,7 @@ fn marketplace_root_from_layout(marketplace_path: &Path, relative_path: &str) -> Some(current.to_path_buf()) } -pub(crate) fn load_marketplace(path: &AbsolutePathBuf) -> Result { +pub fn load_marketplace(path: &AbsolutePathBuf) -> Result { let marketplace = load_raw_marketplace_manifest(path)?; let mut plugins = Vec::new(); @@ -311,7 +311,8 @@ pub(crate) fn load_marketplace(path: &AbsolutePathBuf) -> Result, ) -> Result { diff --git a/codex-rs/core/src/plugins/marketplace_tests.rs b/codex-rs/core-plugins/src/marketplace_tests.rs similarity index 100% rename from codex-rs/core/src/plugins/marketplace_tests.rs rename to codex-rs/core-plugins/src/marketplace_tests.rs diff --git a/codex-rs/core/src/plugins/remote.rs b/codex-rs/core-plugins/src/remote.rs similarity index 94% rename from codex-rs/core/src/plugins/remote.rs rename to codex-rs/core-plugins/src/remote.rs index f26bbe8880..34ece0880a 100644 --- a/codex-rs/core/src/plugins/remote.rs +++ b/codex-rs/core-plugins/src/remote.rs @@ -1,4 +1,3 @@ -use crate::config::Config; use codex_login::CodexAuth; use codex_login::default_client::build_reqwest_client; use codex_protocol::protocol::Product; @@ -11,12 +10,17 @@ const REMOTE_PLUGIN_FETCH_TIMEOUT: Duration = Duration::from_secs(30); const REMOTE_FEATURED_PLUGIN_FETCH_TIMEOUT: Duration = Duration::from_secs(10); const REMOTE_PLUGIN_MUTATION_TIMEOUT: Duration = Duration::from_secs(30); +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RemotePluginServiceConfig { + pub chatgpt_base_url: String, +} + #[derive(Debug, Clone, PartialEq, Eq, Deserialize)] -pub(crate) struct RemotePluginStatusSummary { - pub(crate) name: String, +pub struct RemotePluginStatusSummary { + pub name: String, #[serde(default = "default_remote_marketplace_name")] - pub(crate) marketplace_name: String, - pub(crate) enabled: bool, + pub marketplace_name: String, + pub enabled: bool, } #[derive(Debug, Clone, PartialEq, Eq, Deserialize)] @@ -116,8 +120,8 @@ pub enum RemotePluginFetchError { }, } -pub(crate) async fn fetch_remote_plugin_status( - config: &Config, +pub async fn fetch_remote_plugin_status( + config: &RemotePluginServiceConfig, auth: Option<&CodexAuth>, ) -> Result, RemotePluginFetchError> { let Some(auth) = auth else { @@ -161,7 +165,7 @@ pub(crate) async fn fetch_remote_plugin_status( } pub async fn fetch_remote_featured_plugin_ids( - config: &Config, + config: &RemotePluginServiceConfig, auth: Option<&CodexAuth>, product: Option, ) -> Result, RemotePluginFetchError> { @@ -205,8 +209,8 @@ pub async fn fetch_remote_featured_plugin_ids( }) } -pub(crate) async fn enable_remote_plugin( - config: &Config, +pub async fn enable_remote_plugin( + config: &RemotePluginServiceConfig, auth: Option<&CodexAuth>, plugin_id: &str, ) -> Result<(), RemotePluginMutationError> { @@ -214,8 +218,8 @@ pub(crate) async fn enable_remote_plugin( Ok(()) } -pub(crate) async fn uninstall_remote_plugin( - config: &Config, +pub async fn uninstall_remote_plugin( + config: &RemotePluginServiceConfig, auth: Option<&CodexAuth>, plugin_id: &str, ) -> Result<(), RemotePluginMutationError> { @@ -238,7 +242,7 @@ fn default_remote_marketplace_name() -> String { } async fn post_remote_plugin_mutation( - config: &Config, + config: &RemotePluginServiceConfig, auth: Option<&CodexAuth>, plugin_id: &str, action: &str, @@ -294,7 +298,7 @@ async fn post_remote_plugin_mutation( } fn remote_plugin_mutation_url( - config: &Config, + config: &RemotePluginServiceConfig, plugin_id: &str, action: &str, ) -> Result { diff --git a/codex-rs/core/src/plugins/store.rs b/codex-rs/core-plugins/src/store.rs similarity index 97% rename from codex-rs/core/src/plugins/store.rs rename to codex-rs/core-plugins/src/store.rs index 4316f18002..2f530b8648 100644 --- a/codex-rs/core/src/plugins/store.rs +++ b/codex-rs/core-plugins/src/store.rs @@ -1,5 +1,5 @@ -use super::load_plugin_manifest; -use super::manifest::PluginManifest; +use crate::manifest::PluginManifest; +use crate::manifest::load_plugin_manifest; use codex_plugin::PluginId; use codex_plugin::validate_plugin_segment; use codex_utils_absolute_path::AbsolutePathBuf; @@ -11,8 +11,8 @@ use std::io; use std::path::Path; use std::path::PathBuf; -pub(crate) const DEFAULT_PLUGIN_VERSION: &str = "local"; -pub(crate) const PLUGINS_CACHE_DIR: &str = "plugins/cache"; +pub const DEFAULT_PLUGIN_VERSION: &str = "local"; +pub const PLUGINS_CACHE_DIR: &str = "plugins/cache"; #[derive(Debug, Clone, PartialEq, Eq)] pub struct PluginInstallResult { @@ -157,7 +157,7 @@ impl PluginStoreError { } } -pub(crate) fn plugin_version_for_source(source_path: &Path) -> Result { +pub fn plugin_version_for_source(source_path: &Path) -> Result { let plugin_version = plugin_manifest_version_for_source(source_path)? .unwrap_or_else(|| DEFAULT_PLUGIN_VERSION.to_string()); validate_plugin_version_segment(&plugin_version).map_err(PluginStoreError::Invalid)?; diff --git a/codex-rs/core/src/plugins/store_tests.rs b/codex-rs/core-plugins/src/store_tests.rs similarity index 100% rename from codex-rs/core/src/plugins/store_tests.rs rename to codex-rs/core-plugins/src/store_tests.rs diff --git a/codex-rs/core/src/plugins/toggles.rs b/codex-rs/core-plugins/src/toggles.rs similarity index 100% rename from codex-rs/core/src/plugins/toggles.rs rename to codex-rs/core-plugins/src/toggles.rs diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index 78dfd3c070..4ce4192b81 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -33,6 +33,7 @@ codex-async-utils = { workspace = true } codex-code-mode = { workspace = true } codex-connectors = { workspace = true } codex-config = { workspace = true } +codex-core-plugins = { workspace = true } codex-core-skills = { workspace = true } codex-exec-server = { workspace = true } codex-features = { workspace = true } diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index abfe91a0e1..9950f70142 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -98,9 +98,7 @@ pub(crate) use skills::build_skill_injections; pub(crate) use skills::build_skill_name_counts; pub(crate) use skills::collect_env_var_dependencies; pub(crate) use skills::collect_explicit_skill_mentions; -pub(crate) use skills::config_rules; pub(crate) use skills::injection; -pub(crate) use skills::loader; pub(crate) use skills::manager; pub(crate) use skills::maybe_emit_implicit_skill_invocation; pub(crate) use skills::render_skills_section; diff --git a/codex-rs/core/src/plugins/installed_marketplaces.rs b/codex-rs/core/src/plugins/installed_marketplaces.rs index ad1918d51d..a6f80139a9 100644 --- a/codex-rs/core/src/plugins/installed_marketplaces.rs +++ b/codex-rs/core/src/plugins/installed_marketplaces.rs @@ -1,10 +1,10 @@ use crate::config::Config; +use codex_core_plugins::marketplace::find_marketplace_manifest_path; use codex_utils_absolute_path::AbsolutePathBuf; use std::path::Path; use std::path::PathBuf; use tracing::warn; -use super::marketplace::find_marketplace_manifest_path; use super::validate_plugin_segment; pub const INSTALLED_MARKETPLACES_DIR: &str = ".tmp/marketplaces"; diff --git a/codex-rs/core/src/plugins/manager.rs b/codex-rs/core/src/plugins/manager.rs index a735180ab6..50c62d5823 100644 --- a/codex-rs/core/src/plugins/manager.rs +++ b/codex-rs/core/src/plugins/manager.rs @@ -1,52 +1,48 @@ -use super::LoadedPlugin; use super::PluginLoadOutcome; -use super::PluginManifestPaths; use super::curated_plugins_repo_path; use super::installed_marketplaces::installed_marketplace_roots_from_config; -use super::load_plugin_manifest; -use super::manifest::PluginManifestInterface; -use super::marketplace::MarketplaceError; -use super::marketplace::MarketplaceInterface; -use super::marketplace::MarketplaceListError; -use super::marketplace::MarketplacePluginAuthPolicy; -use super::marketplace::MarketplacePluginPolicy; -use super::marketplace::MarketplacePluginSource; -use super::marketplace::ResolvedMarketplacePlugin; -use super::marketplace::list_marketplaces; -use super::marketplace::load_marketplace; -use super::marketplace::resolve_marketplace_plugin; use super::read_curated_plugins_sha; -use super::remote::RemotePluginFetchError; -use super::remote::RemotePluginMutationError; -use super::remote::enable_remote_plugin; -use super::remote::fetch_remote_featured_plugin_ids; -use super::remote::fetch_remote_plugin_status; -use super::remote::uninstall_remote_plugin; use super::startup_sync::start_startup_remote_plugin_sync_once; -use super::store::PluginInstallResult as StorePluginInstallResult; -use super::store::PluginStore; -use super::store::PluginStoreError; -use super::store::plugin_version_for_source; use super::sync_openai_plugins_repo; use crate::SkillMetadata; -use crate::config::CONFIG_TOML_FILE; use crate::config::Config; use crate::config::ConfigService; use crate::config::ConfigServiceError; use crate::config::edit::ConfigEdit; use crate::config::edit::ConfigEditsBuilder; use crate::config_loader::ConfigLayerStack; -use crate::config_rules::SkillConfigRules; -use crate::config_rules::resolve_disabled_skill_paths; -use crate::config_rules::skill_config_rules_from_stack; -use crate::loader::SkillRoot; -use crate::loader::load_skills_from_roots; use codex_analytics::AnalyticsEventsClient; use codex_app_server_protocol::ConfigValueWriteParams; use codex_app_server_protocol::MergeStrategy; -use codex_config::types::McpServerConfig; use codex_config::types::PluginConfig; -use codex_exec_server::LOCAL_FS; +use codex_core_plugins::loader::configured_curated_plugin_ids_from_codex_home; +use codex_core_plugins::loader::installed_plugin_telemetry_metadata; +use codex_core_plugins::loader::load_plugin_apps; +use codex_core_plugins::loader::load_plugin_mcp_servers; +use codex_core_plugins::loader::load_plugin_skills; +use codex_core_plugins::loader::load_plugins_from_layer_stack; +use codex_core_plugins::loader::log_plugin_load_errors; +use codex_core_plugins::loader::plugin_telemetry_metadata_from_root; +use codex_core_plugins::loader::refresh_curated_plugin_cache; +use codex_core_plugins::loader::refresh_non_curated_plugin_cache; +use codex_core_plugins::manifest::PluginManifestInterface; +use codex_core_plugins::manifest::load_plugin_manifest; +use codex_core_plugins::marketplace::MarketplaceError; +use codex_core_plugins::marketplace::MarketplaceInterface; +use codex_core_plugins::marketplace::MarketplaceListError; +use codex_core_plugins::marketplace::MarketplacePluginAuthPolicy; +use codex_core_plugins::marketplace::MarketplacePluginPolicy; +use codex_core_plugins::marketplace::MarketplacePluginSource; +use codex_core_plugins::marketplace::ResolvedMarketplacePlugin; +use codex_core_plugins::marketplace::list_marketplaces; +use codex_core_plugins::marketplace::load_marketplace; +use codex_core_plugins::marketplace::resolve_marketplace_plugin; +use codex_core_plugins::remote::RemotePluginFetchError; +use codex_core_plugins::remote::RemotePluginMutationError; +use codex_core_plugins::remote::RemotePluginServiceConfig; +use codex_core_plugins::store::PluginInstallResult as StorePluginInstallResult; +use codex_core_plugins::store::PluginStore; +use codex_core_plugins::store::PluginStoreError; use codex_features::Feature; use codex_login::AuthManager; use codex_login::CodexAuth; @@ -54,19 +50,12 @@ use codex_plugin::AppConnectorId; use codex_plugin::PluginCapabilitySummary; use codex_plugin::PluginId; use codex_plugin::PluginIdError; -use codex_plugin::PluginTelemetryMetadata; use codex_plugin::prompt_safe_plugin_description; use codex_protocol::protocol::Product; -use codex_protocol::protocol::SkillScope; use codex_utils_absolute_path::AbsolutePathBuf; -use serde::Deserialize; -use serde_json::Map as JsonMap; -use serde_json::Value as JsonValue; use serde_json::json; use std::collections::HashMap; use std::collections::HashSet; -use std::fs; -use std::path::Path; use std::path::PathBuf; use std::sync::Arc; use std::sync::RwLock; @@ -78,9 +67,6 @@ use toml_edit::value; use tracing::info; use tracing::warn; -const DEFAULT_SKILLS_DIR_NAME: &str = "skills"; -const DEFAULT_MCP_CONFIG_FILE: &str = ".mcp.json"; -const DEFAULT_APP_CONFIG_FILE: &str = ".app.json"; pub const OPENAI_CURATED_MARKETPLACE_NAME: &str = "openai-curated"; pub const OPENAI_CURATED_MARKETPLACE_DISPLAY_NAME: &str = "OpenAI Curated"; static CURATED_REPO_SYNC_STARTED: AtomicBool = AtomicBool::new(false); @@ -109,6 +95,12 @@ struct NonCuratedCacheRefreshState { in_flight: bool, } +fn remote_plugin_service_config(config: &Config) -> RemotePluginServiceConfig { + RemotePluginServiceConfig { + chatgpt_base_url: config.chatgpt_base_url.clone(), + } +} + fn featured_plugin_ids_cache_key( config: &Config, auth: Option<&CodexAuth>, @@ -503,8 +495,12 @@ impl PluginsManager { if let Some(featured_plugin_ids) = self.cached_featured_plugin_ids(&cache_key) { return Ok(featured_plugin_ids); } - let featured_plugin_ids = - fetch_remote_featured_plugin_ids(config, auth, self.restriction_product).await?; + let featured_plugin_ids = codex_core_plugins::remote::fetch_remote_featured_plugin_ids( + &remote_plugin_service_config(config), + auth, + self.restriction_product, + ) + .await?; self.write_featured_plugin_ids_cache(cache_key, &featured_plugin_ids); Ok(featured_plugin_ids) } @@ -536,9 +532,13 @@ impl PluginsManager { // This only forwards the backend mutation before the local install flow. We rely on // `plugin/list(forceRemoteSync=true)` to sync local state rather than doing an extra // reconcile pass here. - enable_remote_plugin(config, auth, &plugin_id) - .await - .map_err(PluginInstallError::from)?; + codex_core_plugins::remote::enable_remote_plugin( + &remote_plugin_service_config(config), + auth, + &plugin_id, + ) + .await + .map_err(PluginInstallError::from)?; self.install_resolved_plugin(resolved).await } @@ -619,9 +619,13 @@ impl PluginsManager { // This only forwards the backend mutation before the local uninstall flow. We rely on // `plugin/list(forceRemoteSync=true)` to sync local state rather than doing an extra // reconcile pass here. - uninstall_remote_plugin(config, auth, &plugin_key) - .await - .map_err(PluginUninstallError::from)?; + codex_core_plugins::remote::uninstall_remote_plugin( + &remote_plugin_service_config(config), + auth, + &plugin_key, + ) + .await + .map_err(PluginUninstallError::from)?; self.uninstall_plugin_id(plugin_id).await } @@ -670,9 +674,12 @@ impl PluginsManager { } info!("starting remote plugin sync"); - let remote_plugins = fetch_remote_plugin_status(config, auth) - .await - .map_err(PluginRemoteSyncError::from)?; + let remote_plugins = codex_core_plugins::remote::fetch_remote_plugin_status( + &remote_plugin_service_config(config), + auth, + ) + .await + .map_err(PluginRemoteSyncError::from)?; let configured_plugins = configured_plugins_from_stack(&config.config_layer_stack); let curated_marketplace_root = curated_plugins_repo_path(self.codex_home.as_path()); let curated_marketplace_path = AbsolutePathBuf::try_from( @@ -1026,7 +1033,9 @@ impl PluginsManager { })?; let description = manifest.description.clone(); let manifest_paths = &manifest.paths; - let skill_config_rules = skill_config_rules_from_stack(&config.config_layer_stack); + let skill_config_rules = codex_core_skills::config_rules::skill_config_rules_from_stack( + &config.config_layer_stack, + ); let resolved_skills = load_plugin_skills( &source_path, manifest_paths, @@ -1034,21 +1043,11 @@ impl PluginsManager { &skill_config_rules, ) .await; - let apps = load_apps_from_paths( - source_path.as_path(), - plugin_app_config_paths(source_path.as_path(), manifest_paths), - ) - .await; - let mcp_config_paths = plugin_mcp_config_paths(source_path.as_path(), manifest_paths); - let mut mcp_server_names = Vec::new(); - for mcp_config_path in mcp_config_paths { - mcp_server_names.extend( - load_mcp_servers_from_file(source_path.as_path(), &mcp_config_path) - .await - .mcp_servers - .into_keys(), - ); - } + let apps = load_plugin_apps(source_path.as_path()).await; + let mut mcp_server_names = load_plugin_mcp_servers(source_path.as_path()) + .await + .into_keys() + .collect::>(); mcp_server_names.sort_unstable(); mcp_server_names.dedup(); @@ -1160,13 +1159,8 @@ impl PluginsManager { .spawn( move || match sync_openai_plugins_repo(codex_home.as_path()) { Ok(curated_plugin_version) => { - let configured_curated_plugin_ids = curated_plugin_ids_from_config_keys( - configured_plugins_from_codex_home( - codex_home.as_path(), - "failed to read user config while refreshing curated plugin cache", - "failed to parse user config while refreshing curated plugin cache", - ), - ); + let configured_curated_plugin_ids = + configured_curated_plugin_ids_from_codex_home(codex_home.as_path()); match refresh_curated_plugin_cache( codex_home.as_path(), &curated_plugin_version, @@ -1352,224 +1346,6 @@ impl PluginUninstallError { } } -fn log_plugin_load_errors(outcome: &PluginLoadOutcome) { - for plugin in outcome - .plugins() - .iter() - .filter(|plugin| plugin.error.is_some()) - { - if let Some(error) = plugin.error.as_deref() { - warn!( - plugin = plugin.config_name, - path = %plugin.root.display(), - "failed to load plugin: {error}" - ); - } - } -} - -#[derive(Debug, Default, Deserialize)] -#[serde(rename_all = "camelCase")] -struct PluginMcpFile { - #[serde(default)] - mcp_servers: HashMap, -} - -#[derive(Debug, Default, Deserialize)] -#[serde(rename_all = "camelCase")] -struct PluginAppFile { - #[serde(default)] - apps: HashMap, -} - -#[derive(Debug, Default, Deserialize)] -struct PluginAppConfig { - id: String, -} - -pub(crate) async fn load_plugins_from_layer_stack( - config_layer_stack: &ConfigLayerStack, - store: &PluginStore, - restriction_product: Option, -) -> PluginLoadOutcome { - let skill_config_rules = skill_config_rules_from_stack(config_layer_stack); - let mut configured_plugins: Vec<_> = configured_plugins_from_stack(config_layer_stack) - .into_iter() - .collect(); - configured_plugins.sort_unstable_by(|(a, _), (b, _)| a.cmp(b)); - - let mut plugins = Vec::with_capacity(configured_plugins.len()); - let mut seen_mcp_server_names = HashMap::::new(); - for (configured_name, plugin) in configured_plugins { - let loaded_plugin = load_plugin( - configured_name.clone(), - &plugin, - store, - restriction_product, - &skill_config_rules, - ) - .await; - for name in loaded_plugin.mcp_servers.keys() { - if let Some(previous_plugin) = - seen_mcp_server_names.insert(name.clone(), configured_name.clone()) - { - warn!( - plugin = configured_name, - previous_plugin, - server = name, - "skipping duplicate plugin MCP server name" - ); - } - } - plugins.push(loaded_plugin); - } - - PluginLoadOutcome::from_plugins(plugins) -} - -fn refresh_curated_plugin_cache( - codex_home: &Path, - plugin_version: &str, - configured_curated_plugin_ids: &[PluginId], -) -> Result { - let store = PluginStore::new(codex_home.to_path_buf()); - let curated_marketplace_path = AbsolutePathBuf::try_from( - curated_plugins_repo_path(codex_home).join(".agents/plugins/marketplace.json"), - ) - .map_err(|_| "local curated marketplace is not available".to_string())?; - let curated_marketplace = load_marketplace(&curated_marketplace_path) - .map_err(|err| format!("failed to load curated marketplace for cache refresh: {err}"))?; - - let mut plugin_sources = HashMap::::new(); - for plugin in curated_marketplace.plugins { - let plugin_name = plugin.name; - if plugin_sources.contains_key(&plugin_name) { - warn!( - plugin = plugin_name, - marketplace = OPENAI_CURATED_MARKETPLACE_NAME, - "ignoring duplicate curated plugin entry during cache refresh" - ); - continue; - } - let source_path = match plugin.source { - MarketplacePluginSource::Local { path } => path, - }; - plugin_sources.insert(plugin_name, source_path); - } - - let mut cache_refreshed = false; - for plugin_id in configured_curated_plugin_ids { - if store.active_plugin_version(plugin_id).as_deref() == Some(plugin_version) { - continue; - } - - let Some(source_path) = plugin_sources.get(&plugin_id.plugin_name).cloned() else { - warn!( - plugin = plugin_id.plugin_name, - marketplace = OPENAI_CURATED_MARKETPLACE_NAME, - "configured curated plugin no longer exists in curated marketplace during cache refresh" - ); - continue; - }; - - store - .install_with_version(source_path, plugin_id.clone(), plugin_version.to_string()) - .map_err(|err| { - format!( - "failed to refresh curated plugin cache for {}: {err}", - plugin_id.as_key() - ) - })?; - cache_refreshed = true; - } - - Ok(cache_refreshed) -} - -fn refresh_non_curated_plugin_cache( - codex_home: &Path, - additional_roots: &[AbsolutePathBuf], -) -> Result { - let configured_non_curated_plugin_ids = - non_curated_plugin_ids_from_config_keys(configured_plugins_from_codex_home( - codex_home, - "failed to read user config while refreshing non-curated plugin cache", - "failed to parse user config while refreshing non-curated plugin cache", - )); - if configured_non_curated_plugin_ids.is_empty() { - return Ok(false); - } - let configured_non_curated_plugin_keys = configured_non_curated_plugin_ids - .iter() - .map(PluginId::as_key) - .collect::>(); - - let store = PluginStore::new(codex_home.to_path_buf()); - let marketplace_outcome = list_marketplaces(additional_roots) - .map_err(|err| format!("failed to discover marketplaces for cache refresh: {err}"))?; - let mut plugin_sources = HashMap::::new(); - - for marketplace in marketplace_outcome.marketplaces { - if marketplace.name == OPENAI_CURATED_MARKETPLACE_NAME { - continue; - } - - for plugin in marketplace.plugins { - let plugin_id = - PluginId::new(plugin.name.clone(), marketplace.name.clone()).map_err(|err| { - match err { - PluginIdError::Invalid(message) => { - format!("failed to prepare non-curated plugin cache refresh: {message}") - } - } - })?; - let plugin_key = plugin_id.as_key(); - if !configured_non_curated_plugin_keys.contains(&plugin_key) { - continue; - } - if plugin_sources.contains_key(&plugin_key) { - warn!( - plugin = plugin.name, - marketplace = marketplace.name, - "ignoring duplicate non-curated plugin entry during cache refresh" - ); - continue; - } - - let source_path = match plugin.source { - MarketplacePluginSource::Local { path } => path, - }; - let plugin_version = plugin_version_for_source(source_path.as_path()) - .map_err(|err| format!("failed to read plugin version for {plugin_key}: {err}"))?; - plugin_sources.insert(plugin_key, (source_path, plugin_version)); - } - } - - let mut cache_refreshed = false; - for plugin_id in configured_non_curated_plugin_ids { - let plugin_key = plugin_id.as_key(); - let Some((source_path, plugin_version)) = plugin_sources.get(&plugin_key).cloned() else { - warn!( - plugin = plugin_id.plugin_name, - marketplace = plugin_id.marketplace_name, - "configured non-curated plugin no longer exists in discovered marketplaces during cache refresh" - ); - continue; - }; - - if store.active_plugin_version(&plugin_id).as_deref() == Some(plugin_version.as_str()) { - continue; - } - - store - .install_with_version(source_path, plugin_id.clone(), plugin_version) - .map_err(|err| format!("failed to refresh plugin cache for {plugin_key}: {err}"))?; - cache_refreshed = true; - } - - Ok(cache_refreshed) -} - fn configured_plugins_from_stack( config_layer_stack: &ConfigLayerStack, ) -> HashMap { @@ -1595,523 +1371,6 @@ fn configured_plugins_from_user_config_value( } } -fn configured_plugins_from_codex_home( - codex_home: &Path, - read_error_message: &str, - parse_error_message: &str, -) -> HashMap { - let config_path = codex_home.join(CONFIG_TOML_FILE); - let user_config = match fs::read_to_string(&config_path) { - Ok(user_config) => user_config, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => return HashMap::new(), - Err(err) => { - warn!( - path = %config_path.display(), - error = %err, - "{read_error_message}" - ); - return HashMap::new(); - } - }; - - let user_config = match toml::from_str::(&user_config) { - Ok(user_config) => user_config, - Err(err) => { - warn!( - path = %config_path.display(), - error = %err, - "{parse_error_message}" - ); - return HashMap::new(); - } - }; - - configured_plugins_from_user_config_value(&user_config) -} - -fn configured_plugin_ids( - configured_plugins: HashMap, - invalid_plugin_key_message: &str, -) -> Vec { - configured_plugins - .into_keys() - .filter_map(|plugin_key| match PluginId::parse(&plugin_key) { - Ok(plugin_id) => Some(plugin_id), - Err(err) => { - warn!( - plugin_key, - error = %err, - "{invalid_plugin_key_message}" - ); - None - } - }) - .collect() -} - -fn curated_plugin_ids_from_config_keys( - configured_plugins: HashMap, -) -> Vec { - let mut configured_curated_plugin_ids = configured_plugin_ids( - configured_plugins, - "ignoring invalid configured plugin key during curated sync setup", - ) - .into_iter() - .filter(|plugin_id| plugin_id.marketplace_name == OPENAI_CURATED_MARKETPLACE_NAME) - .collect::>(); - configured_curated_plugin_ids.sort_unstable_by_key(PluginId::as_key); - configured_curated_plugin_ids -} - -fn non_curated_plugin_ids_from_config_keys( - configured_plugins: HashMap, -) -> Vec { - let mut configured_non_curated_plugin_ids = configured_plugin_ids( - configured_plugins, - "ignoring invalid plugin key during non-curated cache refresh setup", - ) - .into_iter() - .filter(|plugin_id| plugin_id.marketplace_name != OPENAI_CURATED_MARKETPLACE_NAME) - .collect::>(); - configured_non_curated_plugin_ids.sort_unstable_by_key(PluginId::as_key); - configured_non_curated_plugin_ids -} - -async fn load_plugin( - config_name: String, - plugin: &PluginConfig, - store: &PluginStore, - restriction_product: Option, - skill_config_rules: &SkillConfigRules, -) -> LoadedPlugin { - let plugin_id = PluginId::parse(&config_name); - let active_plugin_root = plugin_id - .as_ref() - .ok() - .and_then(|plugin_id| store.active_plugin_root(plugin_id)); - let root = active_plugin_root - .clone() - .unwrap_or_else(|| match &plugin_id { - Ok(plugin_id) => store.plugin_base_root(plugin_id), - Err(_) => store.root().clone(), - }); - let mut loaded_plugin = LoadedPlugin { - config_name, - manifest_name: None, - manifest_description: None, - root, - enabled: plugin.enabled, - skill_roots: Vec::new(), - disabled_skill_paths: HashSet::new(), - has_enabled_skills: false, - mcp_servers: HashMap::new(), - apps: Vec::new(), - error: None, - }; - - if !plugin.enabled { - return loaded_plugin; - } - - let plugin_root = match plugin_id { - Ok(_) => match active_plugin_root { - Some(plugin_root) => plugin_root, - None => { - loaded_plugin.error = Some("plugin is not installed".to_string()); - return loaded_plugin; - } - }, - Err(err) => { - loaded_plugin.error = Some(err.to_string()); - return loaded_plugin; - } - }; - - if !plugin_root.as_path().is_dir() { - loaded_plugin.error = Some("path does not exist or is not a directory".to_string()); - return loaded_plugin; - } - - let Some(manifest) = load_plugin_manifest(plugin_root.as_path()) else { - loaded_plugin.error = Some("missing or invalid .codex-plugin/plugin.json".to_string()); - return loaded_plugin; - }; - - let manifest_paths = &manifest.paths; - loaded_plugin.manifest_name = manifest - .interface - .as_ref() - .and_then(|interface| interface.display_name.as_deref()) - .map(str::trim) - .filter(|display_name| !display_name.is_empty()) - .map(str::to_string) - .or_else(|| Some(manifest.name.clone())); - loaded_plugin.manifest_description = manifest.description.clone(); - loaded_plugin.skill_roots = plugin_skill_roots(&plugin_root, manifest_paths); - let resolved_skills = load_plugin_skills( - &plugin_root, - manifest_paths, - restriction_product, - skill_config_rules, - ) - .await; - let has_enabled_skills = resolved_skills.has_enabled_skills(); - loaded_plugin.disabled_skill_paths = resolved_skills.disabled_skill_paths; - loaded_plugin.has_enabled_skills = has_enabled_skills; - let mut mcp_servers = HashMap::new(); - for mcp_config_path in plugin_mcp_config_paths(plugin_root.as_path(), manifest_paths) { - let plugin_mcp = load_mcp_servers_from_file(plugin_root.as_path(), &mcp_config_path).await; - for (name, config) in plugin_mcp.mcp_servers { - if mcp_servers.insert(name.clone(), config).is_some() { - warn!( - plugin = %plugin_root.display(), - path = %mcp_config_path.display(), - server = name, - "plugin MCP file overwrote an earlier server definition" - ); - } - } - } - loaded_plugin.mcp_servers = mcp_servers; - loaded_plugin.apps = load_apps_from_paths( - plugin_root.as_path(), - plugin_app_config_paths(plugin_root.as_path(), manifest_paths), - ) - .await; - loaded_plugin -} - -struct ResolvedPluginSkills { - skills: Vec, - disabled_skill_paths: HashSet, - had_errors: bool, -} - -impl ResolvedPluginSkills { - fn has_enabled_skills(&self) -> bool { - // Keep the plugin visible in capability summaries if skill loading was partial. - self.had_errors - || self - .skills - .iter() - .any(|skill| !self.disabled_skill_paths.contains(&skill.path_to_skills_md)) - } -} - -async fn load_plugin_skills( - plugin_root: &AbsolutePathBuf, - manifest_paths: &PluginManifestPaths, - restriction_product: Option, - skill_config_rules: &SkillConfigRules, -) -> ResolvedPluginSkills { - let roots = plugin_skill_roots(plugin_root, manifest_paths) - .into_iter() - .map(|path| SkillRoot { - path, - scope: SkillScope::User, - file_system: Arc::clone(&LOCAL_FS), - }) - .collect::>(); - let outcome = load_skills_from_roots(roots).await; - let had_errors = !outcome.errors.is_empty(); - let skills = outcome - .skills - .into_iter() - .filter(|skill| skill.matches_product_restriction_for_product(restriction_product)) - .collect::>(); - let disabled_skill_paths = resolve_disabled_skill_paths(&skills, skill_config_rules); - - ResolvedPluginSkills { - skills, - disabled_skill_paths, - had_errors, - } -} - -fn plugin_skill_roots( - plugin_root: &AbsolutePathBuf, - manifest_paths: &PluginManifestPaths, -) -> Vec { - let mut paths = default_skill_roots(plugin_root); - if let Some(path) = &manifest_paths.skills { - paths.push(path.clone()); - } - paths.sort_unstable(); - paths.dedup(); - paths -} - -fn default_skill_roots(plugin_root: &AbsolutePathBuf) -> Vec { - let skills_dir = plugin_root.join(DEFAULT_SKILLS_DIR_NAME); - if skills_dir.is_dir() { - vec![skills_dir] - } else { - Vec::new() - } -} - -fn plugin_mcp_config_paths( - plugin_root: &Path, - manifest_paths: &PluginManifestPaths, -) -> Vec { - if let Some(path) = &manifest_paths.mcp_servers { - return vec![path.clone()]; - } - default_mcp_config_paths(plugin_root) -} - -fn default_mcp_config_paths(plugin_root: &Path) -> Vec { - let mut paths = Vec::new(); - let default_path = plugin_root.join(DEFAULT_MCP_CONFIG_FILE); - if default_path.is_file() - && let Ok(default_path) = AbsolutePathBuf::try_from(default_path) - { - paths.push(default_path); - } - paths.sort_unstable(); - paths.dedup(); - paths -} - -pub async fn load_plugin_apps(plugin_root: &Path) -> Vec { - if let Some(manifest) = load_plugin_manifest(plugin_root) { - return load_apps_from_paths( - plugin_root, - plugin_app_config_paths(plugin_root, &manifest.paths), - ) - .await; - } - load_apps_from_paths(plugin_root, default_app_config_paths(plugin_root)).await -} - -fn plugin_app_config_paths( - plugin_root: &Path, - manifest_paths: &PluginManifestPaths, -) -> Vec { - if let Some(path) = &manifest_paths.apps { - return vec![path.clone()]; - } - default_app_config_paths(plugin_root) -} - -fn default_app_config_paths(plugin_root: &Path) -> Vec { - let mut paths = Vec::new(); - let default_path = plugin_root.join(DEFAULT_APP_CONFIG_FILE); - if default_path.is_file() - && let Ok(default_path) = AbsolutePathBuf::try_from(default_path) - { - paths.push(default_path); - } - paths.sort_unstable(); - paths.dedup(); - paths -} - -async fn load_apps_from_paths( - plugin_root: &Path, - app_config_paths: Vec, -) -> Vec { - let mut connector_ids = Vec::new(); - for app_config_path in app_config_paths { - let Ok(contents) = tokio::fs::read_to_string(app_config_path.as_path()).await else { - continue; - }; - let parsed = match serde_json::from_str::(&contents) { - Ok(parsed) => parsed, - Err(err) => { - warn!( - path = %app_config_path.display(), - "failed to parse plugin app config: {err}" - ); - continue; - } - }; - - let mut apps: Vec = parsed.apps.into_values().collect(); - apps.sort_unstable_by(|left, right| left.id.cmp(&right.id)); - - connector_ids.extend(apps.into_iter().filter_map(|app| { - if app.id.trim().is_empty() { - warn!( - plugin = %plugin_root.display(), - "plugin app config is missing an app id" - ); - None - } else { - Some(AppConnectorId(app.id)) - } - })); - } - connector_ids.dedup(); - connector_ids -} - -pub async fn plugin_telemetry_metadata_from_root( - plugin_id: &PluginId, - plugin_root: &AbsolutePathBuf, -) -> PluginTelemetryMetadata { - let Some(manifest) = load_plugin_manifest(plugin_root.as_path()) else { - return PluginTelemetryMetadata::from_plugin_id(plugin_id); - }; - - let manifest_paths = &manifest.paths; - let has_skills = !plugin_skill_roots(plugin_root, manifest_paths).is_empty(); - let mut mcp_server_names = Vec::new(); - for path in plugin_mcp_config_paths(plugin_root.as_path(), manifest_paths) { - mcp_server_names.extend( - load_mcp_servers_from_file(plugin_root.as_path(), &path) - .await - .mcp_servers - .into_keys(), - ); - } - mcp_server_names.sort_unstable(); - mcp_server_names.dedup(); - - PluginTelemetryMetadata { - plugin_id: plugin_id.clone(), - capability_summary: Some(PluginCapabilitySummary { - config_name: plugin_id.as_key(), - display_name: plugin_id.plugin_name.clone(), - description: None, - has_skills, - mcp_server_names, - app_connector_ids: load_apps_from_paths( - plugin_root.as_path(), - plugin_app_config_paths(plugin_root.as_path(), manifest_paths), - ) - .await, - }), - } -} - -pub async fn load_plugin_mcp_servers(plugin_root: &Path) -> HashMap { - let Some(manifest) = load_plugin_manifest(plugin_root) else { - return HashMap::new(); - }; - - let mut mcp_servers = HashMap::new(); - for mcp_config_path in plugin_mcp_config_paths(plugin_root, &manifest.paths) { - let plugin_mcp = load_mcp_servers_from_file(plugin_root, &mcp_config_path).await; - for (name, config) in plugin_mcp.mcp_servers { - mcp_servers.entry(name).or_insert(config); - } - } - - mcp_servers -} - -pub async fn installed_plugin_telemetry_metadata( - codex_home: &Path, - plugin_id: &PluginId, -) -> PluginTelemetryMetadata { - let store = PluginStore::new(codex_home.to_path_buf()); - let Some(plugin_root) = store.active_plugin_root(plugin_id) else { - return PluginTelemetryMetadata::from_plugin_id(plugin_id); - }; - - plugin_telemetry_metadata_from_root(plugin_id, &plugin_root).await -} - -async fn load_mcp_servers_from_file( - plugin_root: &Path, - mcp_config_path: &AbsolutePathBuf, -) -> PluginMcpDiscovery { - let Ok(contents) = tokio::fs::read_to_string(mcp_config_path.as_path()).await else { - return PluginMcpDiscovery::default(); - }; - let parsed = match serde_json::from_str::(&contents) { - Ok(parsed) => parsed, - Err(err) => { - warn!( - path = %mcp_config_path.display(), - "failed to parse plugin MCP config: {err}" - ); - return PluginMcpDiscovery::default(); - } - }; - normalize_plugin_mcp_servers( - plugin_root, - parsed.mcp_servers, - mcp_config_path.to_string_lossy().as_ref(), - ) -} - -fn normalize_plugin_mcp_servers( - plugin_root: &Path, - plugin_mcp_servers: HashMap, - source: &str, -) -> PluginMcpDiscovery { - let mut mcp_servers = HashMap::new(); - - for (name, config_value) in plugin_mcp_servers { - let normalized = normalize_plugin_mcp_server_value(plugin_root, config_value); - match serde_json::from_value::(JsonValue::Object(normalized)) { - Ok(config) => { - mcp_servers.insert(name, config); - } - Err(err) => { - warn!( - plugin = %plugin_root.display(), - server = name, - "failed to parse plugin MCP server from {source}: {err}" - ); - } - } - } - - PluginMcpDiscovery { mcp_servers } -} - -fn normalize_plugin_mcp_server_value( - plugin_root: &Path, - value: JsonValue, -) -> JsonMap { - let mut object = match value { - JsonValue::Object(object) => object, - _ => return JsonMap::new(), - }; - - if let Some(JsonValue::String(transport_type)) = object.remove("type") { - match transport_type.as_str() { - "http" | "streamable_http" | "streamable-http" => {} - "stdio" => {} - other => { - warn!( - plugin = %plugin_root.display(), - transport = other, - "plugin MCP server uses an unknown transport type" - ); - } - } - } - - if let Some(JsonValue::Object(oauth)) = object.remove("oauth") - && oauth.contains_key("callbackPort") - { - warn!( - plugin = %plugin_root.display(), - "plugin MCP server OAuth callbackPort is ignored; Codex uses global MCP OAuth callback settings" - ); - } - - if let Some(JsonValue::String(cwd)) = object.get("cwd") - && !Path::new(cwd).is_absolute() - { - object.insert( - "cwd".to_string(), - JsonValue::String(plugin_root.join(cwd).display().to_string()), - ); - } - - object -} - -#[derive(Debug, Default)] -struct PluginMcpDiscovery { - mcp_servers: HashMap, -} - #[cfg(test)] #[path = "manager_tests.rs"] mod tests; diff --git a/codex-rs/core/src/plugins/manager_tests.rs b/codex-rs/core/src/plugins/manager_tests.rs index 23b202f1a8..f74b8fd117 100644 --- a/codex-rs/core/src/plugins/manager_tests.rs +++ b/codex-rs/core/src/plugins/manager_tests.rs @@ -6,7 +6,6 @@ use crate::config_loader::ConfigLayerStack; use crate::config_loader::ConfigRequirements; use crate::config_loader::ConfigRequirementsToml; use crate::plugins::LoadedPlugin; -use crate::plugins::MarketplacePluginInstallPolicy; use crate::plugins::PluginLoadOutcome; use crate::plugins::marketplace_install_root; use crate::plugins::test_support::TEST_CURATED_PLUGIN_SHA; @@ -14,12 +13,15 @@ use crate::plugins::test_support::write_curated_plugin_sha_with as write_curated use crate::plugins::test_support::write_file; use crate::plugins::test_support::write_openai_curated_marketplace; use codex_app_server_protocol::ConfigLayerSource; +use codex_config::McpServerConfig; use codex_config::types::McpServerTransportConfig; +use codex_core_plugins::marketplace::MarketplacePluginInstallPolicy; use codex_login::CodexAuth; use codex_protocol::protocol::Product; use codex_utils_absolute_path::test_support::PathBufExt; use pretty_assertions::assert_eq; use std::fs; +use std::path::Path; use tempfile::TempDir; use toml::Value; use wiremock::Mock; @@ -2477,14 +2479,10 @@ enabled = true ); assert_eq!( - curated_plugin_ids_from_config_keys(configured_plugins_from_codex_home( - tmp.path(), - "failed to read user config while refreshing curated plugin cache", - "failed to parse user config while refreshing curated plugin cache", - )) - .into_iter() - .map(|plugin_id| plugin_id.as_key()) - .collect::>(), + configured_curated_plugin_ids_from_codex_home(tmp.path()) + .into_iter() + .map(|plugin_id| plugin_id.as_key()) + .collect::>(), vec!["slack@openai-curated".to_string()] ); @@ -2496,11 +2494,7 @@ plugins = true ); assert_eq!( - curated_plugin_ids_from_config_keys(configured_plugins_from_codex_home( - tmp.path(), - "failed to read user config while refreshing curated plugin cache", - "failed to parse user config while refreshing curated plugin cache", - )), + configured_curated_plugin_ids_from_codex_home(tmp.path()), Vec::::new() ); } diff --git a/codex-rs/core/src/plugins/marketplace_add/metadata.rs b/codex-rs/core/src/plugins/marketplace_add/metadata.rs index b9412e7f0b..f21ed0faa2 100644 --- a/codex-rs/core/src/plugins/marketplace_add/metadata.rs +++ b/codex-rs/core/src/plugins/marketplace_add/metadata.rs @@ -1,10 +1,10 @@ use super::MarketplaceAddError; use super::MarketplaceSource; use crate::plugins::installed_marketplaces::resolve_configured_marketplace_root; -use crate::plugins::validate_marketplace_root; use codex_config::CONFIG_TOML_FILE; use codex_config::MarketplaceConfigUpdate; use codex_config::record_user_marketplace; +use codex_core_plugins::marketplace::validate_marketplace_root; use std::fs; use std::io::ErrorKind; use std::path::Path; diff --git a/codex-rs/core/src/plugins/marketplace_add/source.rs b/codex-rs/core/src/plugins/marketplace_add/source.rs index 53d52058aa..fa2c7dd922 100644 --- a/codex-rs/core/src/plugins/marketplace_add/source.rs +++ b/codex-rs/core/src/plugins/marketplace_add/source.rs @@ -1,6 +1,6 @@ use super::MarketplaceAddError; -use crate::plugins::validate_marketplace_root; use crate::plugins::validate_plugin_segment; +use codex_core_plugins::marketplace::validate_marketplace_root; use std::path::Path; use std::path::PathBuf; diff --git a/codex-rs/core/src/plugins/mod.rs b/codex-rs/core/src/plugins/mod.rs index 7ffb80b449..57997eb134 100644 --- a/codex-rs/core/src/plugins/mod.rs +++ b/codex-rs/core/src/plugins/mod.rs @@ -4,17 +4,12 @@ mod discoverable; mod injection; mod installed_marketplaces; mod manager; -mod manifest; -mod marketplace; mod marketplace_add; mod mentions; -mod remote; mod render; mod startup_sync; -mod store; #[cfg(test)] pub(crate) mod test_support; -mod toggles; pub use codex_plugin::AppConnectorId; pub use codex_plugin::EffectiveSkillRoots; @@ -45,32 +40,15 @@ pub use manager::PluginRemoteSyncError; pub use manager::PluginUninstallError; pub use manager::PluginsManager; pub use manager::RemotePluginSyncResult; -pub use manager::installed_plugin_telemetry_metadata; -pub use manager::load_plugin_apps; -pub use manager::load_plugin_mcp_servers; -pub use manager::plugin_telemetry_metadata_from_root; -pub use manifest::PluginManifestInterface; -pub(crate) use manifest::PluginManifestPaths; -pub(crate) use manifest::load_plugin_manifest; -pub use marketplace::MarketplaceError; -pub use marketplace::MarketplaceListError; -pub use marketplace::MarketplacePluginAuthPolicy; -pub use marketplace::MarketplacePluginInstallPolicy; -pub use marketplace::MarketplacePluginPolicy; -pub use marketplace::MarketplacePluginSource; -pub use marketplace::validate_marketplace_root; pub use marketplace_add::MarketplaceAddError; pub use marketplace_add::MarketplaceAddOutcome; pub use marketplace_add::MarketplaceAddRequest; pub use marketplace_add::add_marketplace; -pub use remote::RemotePluginFetchError; -pub use remote::fetch_remote_featured_plugin_ids; pub(crate) use render::render_explicit_plugin_instructions; pub(crate) use render::render_plugins_section; pub(crate) use startup_sync::curated_plugins_repo_path; pub(crate) use startup_sync::read_curated_plugins_sha; pub(crate) use startup_sync::sync_openai_plugins_repo; -pub use toggles::collect_plugin_enabled_candidates; pub(crate) use mentions::build_connector_slug_counts; pub(crate) use mentions::build_skill_name_counts; From 4cd85b28d2713289d5940b6cb967894277a17419 Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Thu, 16 Apr 2026 00:03:50 -0700 Subject: [PATCH 56/84] Fix MCP startup cancellation through app server (#18078) Addresses https://github.com/openai/codex/issues/17143 Problem: TUI interrupts without an active turn stopped cancelling slow MCP startup after routing through the app-server APIs. Solution: Route no-active-turn interrupts through app-server as startup cancels, acknowledge them immediately, and emit cancelled MCP startup updates. Testing: I manually confirmed that MCP cancellation didn't work prior to this PR and works after the fix was in place. --- .../app-server/src/codex_message_processor.rs | 51 +++++++++++++++---- .../codex-mcp/src/mcp_connection_manager.rs | 5 +- codex-rs/tui/src/app.rs | 18 +++++-- codex-rs/tui/src/app_server_session.rs | 4 ++ 4 files changed, 61 insertions(+), 17 deletions(-) diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 94c1e56645..2585383cda 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -186,6 +186,7 @@ use codex_app_server_protocol::ThreadUnsubscribeStatus; use codex_app_server_protocol::Turn; use codex_app_server_protocol::TurnError; use codex_app_server_protocol::TurnInterruptParams; +use codex_app_server_protocol::TurnInterruptResponse; use codex_app_server_protocol::TurnStartParams; use codex_app_server_protocol::TurnStartResponse; use codex_app_server_protocol::TurnStatus; @@ -7559,9 +7560,12 @@ impl CodexMessageProcessor { async fn turn_interrupt(&self, request_id: ConnectionRequestId, params: TurnInterruptParams) { let TurnInterruptParams { thread_id, turn_id } = params; - self.outgoing - .record_request_turn_id(&request_id, &turn_id) - .await; + let is_startup_interrupt = turn_id.is_empty(); + if !is_startup_interrupt { + self.outgoing + .record_request_turn_id(&request_id, &turn_id) + .await; + } let (thread_uuid, thread) = match self.load_thread(&thread_id).await { Ok(v) => v, @@ -7571,21 +7575,48 @@ impl CodexMessageProcessor { } }; - let request = request_id.clone(); - - // Record the pending interrupt so we can reply when TurnAborted arrives. - { + // Record turn interrupts so we can reply when TurnAborted arrives. Startup + // interrupts do not have a turn and are acknowledged after submission. + if !is_startup_interrupt { let thread_state = self.thread_state_manager.thread_state(thread_uuid).await; let mut thread_state = thread_state.lock().await; thread_state .pending_interrupts - .push((request, ApiVersion::V2)); + .push((request_id.clone(), ApiVersion::V2)); } - // Submit the interrupt; we'll respond upon TurnAborted. - let _ = self + // Submit the interrupt. Turn interrupts respond upon TurnAborted; startup + // interrupts respond here because startup cancellation has no turn event. + let submit_result = self .submit_core_op(&request_id, thread.as_ref(), Op::Interrupt) .await; + match submit_result { + Ok(_) if is_startup_interrupt => { + self.outgoing + .send_response(request_id, TurnInterruptResponse {}) + .await; + } + Ok(_) => {} + Err(err) => { + if !is_startup_interrupt { + let thread_state = self.thread_state_manager.thread_state(thread_uuid).await; + let mut thread_state = thread_state.lock().await; + thread_state + .pending_interrupts + .retain(|(pending_request_id, _)| pending_request_id != &request_id); + } + let interrupt_target = if is_startup_interrupt { + "startup" + } else { + "turn" + }; + self.send_internal_error( + request_id, + format!("failed to interrupt {interrupt_target}: {err}"), + ) + .await; + } + } } async fn ensure_conversation_listener( diff --git a/codex-rs/codex-mcp/src/mcp_connection_manager.rs b/codex-rs/codex-mcp/src/mcp_connection_manager.rs index 1542c7b30b..d1696b5931 100644 --- a/codex-rs/codex-mcp/src/mcp_connection_manager.rs +++ b/codex-rs/codex-mcp/src/mcp_connection_manager.rs @@ -758,12 +758,13 @@ impl McpConnectionManager { let submit_id = startup_submit_id.clone(); let auth_entry = auth_entries.get(&server_name).cloned(); join_set.spawn(async move { - let outcome = async_managed_client.client().await; + let mut outcome = async_managed_client.client().await; if cancel_token.is_cancelled() { - return (server_name, Err(StartupOutcomeError::Cancelled)); + outcome = Err(StartupOutcomeError::Cancelled); } let status = match &outcome { Ok(_) => McpStartupStatus::Ready, + Err(StartupOutcomeError::Cancelled) => McpStartupStatus::Cancelled, Err(error) => { let error_str = mcp_init_error_display( server_name.as_str(), diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 03a99e3aa3..a60856b9c4 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -2378,10 +2378,11 @@ impl App { ) -> Result { match op.view() { AppCommandView::Interrupt => { - let Some(turn_id) = self.active_turn_id_for_thread(thread_id).await else { - return Ok(true); - }; - app_server.turn_interrupt(thread_id, turn_id).await?; + if let Some(turn_id) = self.active_turn_id_for_thread(thread_id).await { + app_server.turn_interrupt(thread_id, turn_id).await?; + } else { + app_server.startup_interrupt(thread_id).await?; + } Ok(true) } AppCommandView::UserTurn { @@ -11410,11 +11411,18 @@ guardian_approval = true #[tokio::test] async fn interrupt_without_active_turn_is_treated_as_handled() { let mut app = make_test_app().await; - let thread_id = ThreadId::new(); let mut app_server = crate::start_embedded_app_server_for_picker(app.chat_widget.config_ref()) .await .expect("embedded app server"); + let started = app_server + .start_thread(app.chat_widget.config_ref()) + .await + .expect("thread/start should succeed"); + let thread_id = started.session.thread_id; + app.enqueue_primary_thread_session(started.session, started.turns) + .await + .expect("primary thread should be registered"); let op = AppCommand::interrupt(); let handled = app diff --git a/codex-rs/tui/src/app_server_session.rs b/codex-rs/tui/src/app_server_session.rs index 47a06738e4..5e11a53340 100644 --- a/codex-rs/tui/src/app_server_session.rs +++ b/codex-rs/tui/src/app_server_session.rs @@ -476,6 +476,10 @@ impl AppServerSession { Ok(()) } + pub(crate) async fn startup_interrupt(&mut self, thread_id: ThreadId) -> Result<()> { + self.turn_interrupt(thread_id, String::new()).await + } + pub(crate) async fn turn_steer( &mut self, thread_id: ThreadId, From f97be7dfff18d99f1b6b25b7613a03d1f42d82d0 Mon Sep 17 00:00:00 2001 From: jackz-oai Date: Thu, 16 Apr 2026 00:13:15 -0700 Subject: [PATCH 57/84] [codex] Route Fed ChatGPT auth through Fed edge (#17151) ## Summary - parse chatgpt_account_is_fedramp from signed ChatGPT auth metadata - add _account_is_fedramp=true to ChatGPT backend-api requests only for FedRAMP ChatGPT-auth accounts --- codex-rs/backend-client/src/client.rs | 15 ++++++++++++++ codex-rs/codex-api/src/api_bridge.rs | 5 +++++ codex-rs/codex-api/src/api_bridge_tests.rs | 20 ++++++++++++++++++ codex-rs/codex-api/src/auth.rs | 24 ++++++++++++++++++++++ codex-rs/core/src/agent_identity.rs | 1 + codex-rs/core/src/mcp_openai_file.rs | 1 + codex-rs/login/src/api_bridge.rs | 4 ++++ codex-rs/login/src/auth/auth_tests.rs | 1 + codex-rs/login/src/auth/manager.rs | 6 ++++++ codex-rs/login/src/token_data.rs | 10 +++++++++ codex-rs/login/src/token_data_tests.rs | 16 +++++++++++++++ 11 files changed, 103 insertions(+) diff --git a/codex-rs/backend-client/src/client.rs b/codex-rs/backend-client/src/client.rs index 20baa1cfea..ad461973f9 100644 --- a/codex-rs/backend-client/src/client.rs +++ b/codex-rs/backend-client/src/client.rs @@ -104,6 +104,7 @@ pub struct Client { bearer_token: Option, user_agent: Option, chatgpt_account_id: Option, + chatgpt_account_is_fedramp: bool, path_style: PathStyle, } @@ -129,6 +130,7 @@ impl Client { bearer_token: None, user_agent: None, chatgpt_account_id: None, + chatgpt_account_is_fedramp: false, path_style, }) } @@ -141,6 +143,9 @@ impl Client { if let Some(account_id) = auth.get_account_id() { client = client.with_chatgpt_account_id(account_id); } + if auth.is_fedramp_account() { + client = client.with_fedramp_routing_header(); + } Ok(client) } @@ -161,6 +166,11 @@ impl Client { self } + pub fn with_fedramp_routing_header(mut self) -> Self { + self.chatgpt_account_is_fedramp = true; + self + } + pub fn with_path_style(mut self, style: PathStyle) -> Self { self.path_style = style; self @@ -185,6 +195,11 @@ impl Client { { h.insert(name, hv); } + if self.chatgpt_account_is_fedramp + && let Ok(name) = HeaderName::from_bytes(b"X-OpenAI-Fedramp") + { + h.insert(name, HeaderValue::from_static("true")); + } h } diff --git a/codex-rs/codex-api/src/api_bridge.rs b/codex-rs/codex-api/src/api_bridge.rs index 7c36c67fdd..2677ff3e55 100644 --- a/codex-rs/codex-api/src/api_bridge.rs +++ b/codex-rs/codex-api/src/api_bridge.rs @@ -179,6 +179,7 @@ struct UsageErrorBody { pub struct CoreAuthProvider { pub token: Option, pub account_id: Option, + pub is_fedramp_account: bool, } impl CoreAuthProvider { @@ -196,6 +197,7 @@ impl CoreAuthProvider { Self { token: token.map(str::to_string), account_id: account_id.map(str::to_string), + is_fedramp_account: false, } } } @@ -212,5 +214,8 @@ impl ApiAuthProvider for CoreAuthProvider { { let _ = headers.insert("ChatGPT-Account-ID", header); } + if self.is_fedramp_account { + crate::auth::add_fedramp_routing_header(headers); + } } } diff --git a/codex-rs/codex-api/src/api_bridge_tests.rs b/codex-rs/codex-api/src/api_bridge_tests.rs index 50247c1314..c7d4bbbdab 100644 --- a/codex-rs/codex-api/src/api_bridge_tests.rs +++ b/codex-rs/codex-api/src/api_bridge_tests.rs @@ -136,6 +136,7 @@ fn core_auth_provider_reports_when_auth_header_will_attach() { let auth = CoreAuthProvider { token: Some("access-token".to_string()), account_id: None, + is_fedramp_account: false, }; assert!(auth.auth_header_attached()); @@ -162,3 +163,22 @@ fn core_auth_provider_adds_auth_headers() { Some("workspace-123") ); } + +#[test] +fn core_auth_provider_adds_fedramp_routing_header_for_fedramp_accounts() { + let auth = CoreAuthProvider { + token: Some("access-token".to_string()), + account_id: Some("workspace-123".to_string()), + is_fedramp_account: true, + }; + let mut headers = HeaderMap::new(); + + crate::AuthProvider::add_auth_headers(&auth, &mut headers); + + assert_eq!( + headers + .get("X-OpenAI-Fedramp") + .and_then(|value| value.to_str().ok()), + Some("true") + ); +} diff --git a/codex-rs/codex-api/src/auth.rs b/codex-rs/codex-api/src/auth.rs index a7b1e69d14..efa5fb3288 100644 --- a/codex-rs/codex-api/src/auth.rs +++ b/codex-rs/codex-api/src/auth.rs @@ -1,4 +1,5 @@ use http::HeaderMap; +use http::HeaderValue; /// Adds authentication headers to API requests. /// @@ -8,3 +9,26 @@ use http::HeaderMap; pub trait AuthProvider: Send + Sync { fn add_auth_headers(&self, headers: &mut HeaderMap); } + +pub(crate) fn add_fedramp_routing_header(headers: &mut HeaderMap) { + headers.insert("X-OpenAI-Fedramp", HeaderValue::from_static("true")); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn add_fedramp_routing_header_sets_header() { + let mut headers = HeaderMap::new(); + + add_fedramp_routing_header(&mut headers); + + assert_eq!( + headers + .get("X-OpenAI-Fedramp") + .and_then(|v| v.to_str().ok()), + Some("true") + ); + } +} diff --git a/codex-rs/core/src/agent_identity.rs b/codex-rs/core/src/agent_identity.rs index 13a07cd961..2c897b5456 100644 --- a/codex-rs/core/src/agent_identity.rs +++ b/codex-rs/core/src/agent_identity.rs @@ -737,6 +737,7 @@ mod tests { chatgpt_plan_type: None, chatgpt_user_id: user_id.map(ToOwned::to_owned), chatgpt_account_id: Some(account_id.to_string()), + chatgpt_account_is_fedramp: false, raw_jwt: fake_id_token(account_id, user_id), }, access_token: format!("access-token-{account_id}"), diff --git a/codex-rs/core/src/mcp_openai_file.rs b/codex-rs/core/src/mcp_openai_file.rs index 587dd4b770..33d0a3f1f0 100644 --- a/codex-rs/core/src/mcp_openai_file.rs +++ b/codex-rs/core/src/mcp_openai_file.rs @@ -115,6 +115,7 @@ async fn build_uploaded_local_argument_value( let upload_auth = CoreAuthProvider { token: Some(token_data.access_token), account_id: token_data.account_id, + is_fedramp_account: auth.is_fedramp_account(), }; let uploaded = upload_local_file( turn_context.config.chatgpt_base_url.trim_end_matches('/'), diff --git a/codex-rs/login/src/api_bridge.rs b/codex-rs/login/src/api_bridge.rs index d8b9dbb77c..684c890bde 100644 --- a/codex-rs/login/src/api_bridge.rs +++ b/codex-rs/login/src/api_bridge.rs @@ -11,6 +11,7 @@ pub fn auth_provider_from_auth( return Ok(CoreAuthProvider { token: Some(api_key), account_id: None, + is_fedramp_account: false, }); } @@ -18,6 +19,7 @@ pub fn auth_provider_from_auth( return Ok(CoreAuthProvider { token: Some(token), account_id: None, + is_fedramp_account: false, }); } @@ -26,11 +28,13 @@ pub fn auth_provider_from_auth( Ok(CoreAuthProvider { token: Some(token), account_id: auth.get_account_id(), + is_fedramp_account: auth.is_fedramp_account(), }) } else { Ok(CoreAuthProvider { token: None, account_id: None, + is_fedramp_account: false, }) } } diff --git a/codex-rs/login/src/auth/auth_tests.rs b/codex-rs/login/src/auth/auth_tests.rs index 8a16f5939c..30244ac268 100644 --- a/codex-rs/login/src/auth/auth_tests.rs +++ b/codex-rs/login/src/auth/auth_tests.rs @@ -130,6 +130,7 @@ async fn pro_account_with_no_api_key_uses_chatgpt_auth() { chatgpt_plan_type: Some(InternalPlanType::Known(InternalKnownPlan::Pro)), chatgpt_user_id: Some("user-12345".to_string()), chatgpt_account_id: None, + chatgpt_account_is_fedramp: false, raw_jwt: fake_jwt, }, access_token: "test-access-token".to_string(), diff --git a/codex-rs/login/src/auth/manager.rs b/codex-rs/login/src/auth/manager.rs index 8768171d95..3cb97f58b0 100644 --- a/codex-rs/login/src/auth/manager.rs +++ b/codex-rs/login/src/auth/manager.rs @@ -293,6 +293,12 @@ impl CodexAuth { self.get_current_token_data().and_then(|t| t.account_id) } + /// Returns false if `is_chatgpt_auth()` is false or the token omits the FedRAMP claim. + pub fn is_fedramp_account(&self) -> bool { + self.get_current_token_data() + .is_some_and(|t| t.id_token.is_fedramp_account()) + } + /// Returns `None` if `is_chatgpt_auth()` is false. pub fn get_account_email(&self) -> Option { self.get_current_token_data().and_then(|t| t.id_token.email) diff --git a/codex-rs/login/src/token_data.rs b/codex-rs/login/src/token_data.rs index f70696d5a2..2952ecb2a7 100644 --- a/codex-rs/login/src/token_data.rs +++ b/codex-rs/login/src/token_data.rs @@ -36,6 +36,8 @@ pub struct IdTokenInfo { pub chatgpt_user_id: Option, /// Organization/workspace identifier associated with the token, if present. pub chatgpt_account_id: Option, + /// Whether the selected ChatGPT workspace must route through the FedRAMP edge. + pub chatgpt_account_is_fedramp: bool, pub raw_jwt: String, } @@ -60,6 +62,10 @@ impl IdTokenInfo { Some(PlanType::Known(plan)) if plan.is_workspace_account() ) } + + pub fn is_fedramp_account(&self) -> bool { + self.chatgpt_account_is_fedramp + } } #[derive(Deserialize)] @@ -88,6 +94,8 @@ struct AuthClaims { user_id: Option, #[serde(default)] chatgpt_account_id: Option, + #[serde(default)] + chatgpt_account_is_fedramp: bool, } #[derive(Deserialize)] @@ -139,6 +147,7 @@ pub fn parse_chatgpt_jwt_claims(jwt: &str) -> Result Ok(IdTokenInfo { email, @@ -146,6 +155,7 @@ pub fn parse_chatgpt_jwt_claims(jwt: &str) -> Result Date: Thu, 16 Apr 2026 10:01:45 +0100 Subject: [PATCH 58/84] fix: more flake (#18006) Stabilizes the Responses API proxy header test by splitting the coverage at the right boundary: - Core integration test now verifies parent/subagent identity headers directly from captured `/responses` requests. - Proxy dump unit test now verifies those identity headers are preserved in dumped request JSON. - Removes the flaky real proxy process + temp-file dump polling path from the core test. --- .../suite/responses_api_proxy_headers.rs | 296 +++++------------- codex-rs/responses-api-proxy/src/dump.rs | 18 ++ 2 files changed, 92 insertions(+), 222 deletions(-) diff --git a/codex-rs/core/tests/suite/responses_api_proxy_headers.rs b/codex-rs/core/tests/suite/responses_api_proxy_headers.rs index 8199956ee0..67f6e86ab0 100644 --- a/codex-rs/core/tests/suite/responses_api_proxy_headers.rs +++ b/codex-rs/core/tests/suite/responses_api_proxy_headers.rs @@ -1,6 +1,5 @@ -//! Exercises a real `responses-api-proxy` process with request dumping enabled, then verifies that -//! parent and spawned subagent requests carry the expected window, parent-thread, and subagent -//! identity headers in the dumped Responses API requests. +//! Verifies that parent and spawned subagent Responses API requests carry the expected window, +//! parent-thread, and subagent identity headers. use anyhow::Result; use anyhow::anyhow; @@ -10,6 +9,8 @@ use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::user_input::UserInput; +use core_test_support::responses::ResponseMock; +use core_test_support::responses::ResponsesRequest; use core_test_support::responses::ev_assistant_message; use core_test_support::responses::ev_completed; use core_test_support::responses::ev_function_call; @@ -21,119 +22,28 @@ use core_test_support::skip_if_no_network; use core_test_support::test_codex::TestCodex; use core_test_support::test_codex::test_codex; use pretty_assertions::assert_eq; -use serde_json::Value; use serde_json::json; -use std::io::ErrorKind; -use std::io::Write; -use std::path::Path; -use std::process::Child; -use std::process::Command as StdCommand; -use std::process::Stdio; use std::time::Duration; -use std::time::Instant; -use tempfile::TempDir; const PARENT_PROMPT: &str = "spawn a subagent and report when it is started"; const CHILD_PROMPT: &str = "child: say done"; const SPAWN_CALL_ID: &str = "spawn-call-1"; -const PROXY_START_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 30); -const PROXY_POLL_INTERVAL: Duration = Duration::from_millis(/*millis*/ 20); +const REQUEST_POLL_INTERVAL: Duration = Duration::from_millis(/*millis*/ 20); const TURN_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 60); -struct ResponsesApiProxy { - child: Child, - port: u16, -} - -impl ResponsesApiProxy { - fn start(upstream_url: &str, dump_dir: &Path) -> Result { - let server_info = dump_dir.join("server-info.json"); - let (proxy_program, use_codex_multitool) = - match codex_utils_cargo_bin::cargo_bin("codex-responses-api-proxy") { - Ok(path) => (path, false), - Err(_) => (codex_utils_cargo_bin::cargo_bin("codex")?, true), - }; - let mut command = StdCommand::new(proxy_program); - if use_codex_multitool { - command.arg("responses-api-proxy"); - } - let mut child = command - .args(["--server-info"]) - .arg(&server_info) - .args(["--upstream-url", upstream_url, "--dump-dir"]) - .arg(dump_dir) - .stdin(Stdio::piped()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn()?; - - child - .stdin - .take() - .ok_or_else(|| anyhow!("responses-api-proxy stdin was not piped"))? - .write_all(b"dummy\n")?; - - let deadline = Instant::now() + PROXY_START_TIMEOUT; - loop { - match std::fs::read_to_string(&server_info) { - Ok(info) => { - if !info.trim().is_empty() { - match serde_json::from_str::(&info) { - Ok(info) => { - let port = info - .get("port") - .and_then(Value::as_u64) - .ok_or_else(|| anyhow!("proxy server info missing port"))?; - return Ok(Self { - child, - port: u16::try_from(port)?, - }); - } - Err(err) if err.is_eof() => {} - Err(err) => return Err(err.into()), - } - } - } - Err(err) if err.kind() == ErrorKind::NotFound => {} - Err(err) => return Err(err.into()), - } - if let Some(status) = child.try_wait()? { - return Err(anyhow!( - "responses-api-proxy exited before writing server info: {status}" - )); - } - if Instant::now() >= deadline { - return Err(anyhow!("timed out waiting for responses-api-proxy")); - } - std::thread::sleep(PROXY_POLL_INTERVAL); - } - } - - fn base_url(&self) -> String { - format!("http://127.0.0.1:{}/v1", self.port) - } -} - -impl Drop for ResponsesApiProxy { - fn drop(&mut self) { - let _ = self.child.kill(); - let _ = self.child.wait(); - } -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn responses_api_proxy_dumps_parent_and_subagent_identity_headers() -> Result<()> { +async fn responses_api_parent_and_subagent_requests_include_identity_headers() -> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; - let dump_dir = TempDir::new()?; - let proxy = - ResponsesApiProxy::start(&format!("{}/v1/responses", server.uri()), dump_dir.path())?; let spawn_args = serde_json::to_string(&json!({ "message": CHILD_PROMPT }))?; - mount_sse_once_match( + let parent_mock = mount_sse_once_match( &server, - |req: &wiremock::Request| request_body_contains(req, PARENT_PROMPT), + |req: &wiremock::Request| { + request_body_contains(req, PARENT_PROMPT) + && request_header(req, "x-openai-subagent").is_none() + }, sse(vec![ ev_response_created("resp-parent-1"), ev_function_call(SPAWN_CALL_ID, "spawn_agent", &spawn_args), @@ -141,10 +51,12 @@ async fn responses_api_proxy_dumps_parent_and_subagent_identity_headers() -> Res ]), ) .await; - mount_sse_once_match( + let child_mock = mount_sse_once_match( &server, |req: &wiremock::Request| { - request_body_contains(req, CHILD_PROMPT) && !request_body_contains(req, SPAWN_CALL_ID) + request_body_contains(req, CHILD_PROMPT) + && !request_body_contains(req, SPAWN_CALL_ID) + && request_header(req, "x-openai-subagent") == Some("collab_spawn") }, sse(vec![ ev_response_created("resp-child-1"), @@ -155,7 +67,10 @@ async fn responses_api_proxy_dumps_parent_and_subagent_identity_headers() -> Res .await; mount_sse_once_match( &server, - |req: &wiremock::Request| request_body_contains(req, SPAWN_CALL_ID), + |req: &wiremock::Request| { + request_body_contains(req, SPAWN_CALL_ID) + && request_header(req, "x-openai-subagent").is_none() + }, sse(vec![ ev_response_created("resp-parent-2"), ev_assistant_message("msg-parent-2", "parent done"), @@ -164,50 +79,52 @@ async fn responses_api_proxy_dumps_parent_and_subagent_identity_headers() -> Res ) .await; - let proxy_base_url = proxy.base_url(); - let mut builder = test_codex().with_config(move |config| { - config.model_provider.base_url = Some(proxy_base_url); + let mut builder = test_codex().with_config(|config| { config .features .disable(Feature::EnableRequestCompression) .expect("test config should allow feature update"); }); let test = builder.build(&server).await?; - submit_turn_with_timeout(&test, PARENT_PROMPT, dump_dir.path()).await?; + submit_turn_with_timeout(&test, PARENT_PROMPT).await?; - let dumps = wait_for_proxy_request_dumps(dump_dir.path())?; - let parent = dumps - .iter() - .find(|dump| dump_body_contains(dump, PARENT_PROMPT)) - .ok_or_else(|| anyhow!("missing parent request dump"))?; - let child = dumps - .iter() - .find(|dump| { - dump_body_contains(dump, CHILD_PROMPT) && !dump_body_contains(dump, SPAWN_CALL_ID) - }) - .ok_or_else(|| anyhow!("missing child request dump"))?; + let parent = wait_for_matching_request(&parent_mock, "parent request", |request| { + request.body_contains_text(PARENT_PROMPT) && request.header("x-openai-subagent").is_none() + }) + .await?; + let child = wait_for_matching_request(&child_mock, "child request", |request| { + request.body_contains_text(CHILD_PROMPT) + && !request.body_contains_text(SPAWN_CALL_ID) + && request.header("x-openai-subagent").as_deref() == Some("collab_spawn") + }) + .await?; - let parent_window_id = header(parent, "x-codex-window-id") + let parent_window_id = parent + .header("x-codex-window-id") .ok_or_else(|| anyhow!("parent request missing x-codex-window-id"))?; - let child_window_id = header(child, "x-codex-window-id") + let child_window_id = child + .header("x-codex-window-id") .ok_or_else(|| anyhow!("child request missing x-codex-window-id"))?; - let (parent_thread_id, parent_generation) = split_window_id(parent_window_id)?; - let (child_thread_id, child_generation) = split_window_id(child_window_id)?; + let (parent_thread_id, parent_generation) = split_window_id(&parent_window_id)?; + let (child_thread_id, child_generation) = split_window_id(&child_window_id)?; assert_eq!(parent_generation, 0); assert_eq!(child_generation, 0); assert!(child_thread_id != parent_thread_id); - assert_eq!(header(parent, "x-openai-subagent"), None); - assert_eq!(header(child, "x-openai-subagent"), Some("collab_spawn")); + assert_eq!(parent.header("x-openai-subagent"), None); assert_eq!( - header(child, "x-codex-parent-thread-id"), + child.header("x-openai-subagent").as_deref(), + Some("collab_spawn") + ); + assert_eq!( + child.header("x-codex-parent-thread-id").as_deref(), Some(parent_thread_id) ); Ok(()) } -async fn submit_turn_with_timeout(test: &TestCodex, prompt: &str, dump_dir: &Path) -> Result<()> { +async fn submit_turn_with_timeout(test: &TestCodex, prompt: &str) -> Result<()> { let session_model = test.session_configured.model.clone(); test.codex .submit(Op::UserTurn { @@ -235,14 +152,14 @@ async fn submit_turn_with_timeout(test: &TestCodex, prompt: &str, dump_dir: &Pat }) .await?; - let turn_started = wait_for_event_result(test, "turn started", dump_dir, |event| { + let turn_started = wait_for_event_result(test, "turn started", |event| { matches!(event, EventMsg::TurnStarted(_)) }) .await?; let EventMsg::TurnStarted(turn_started) = turn_started else { unreachable!("event predicate only matches turn started events"); }; - wait_for_event_result(test, "turn complete", dump_dir, |event| match event { + wait_for_event_result(test, "turn complete", |event| match event { EventMsg::TurnComplete(event) => event.turn_id == turn_started.turn_id, _ => false, }) @@ -251,10 +168,33 @@ async fn submit_turn_with_timeout(test: &TestCodex, prompt: &str, dump_dir: &Pat Ok(()) } +async fn wait_for_matching_request( + mock: &ResponseMock, + label: &str, + mut predicate: F, +) -> Result +where + F: FnMut(&ResponsesRequest) -> bool, +{ + tokio::time::timeout(TURN_TIMEOUT, async { + loop { + if let Some(request) = mock + .requests() + .into_iter() + .find(|request| predicate(request)) + { + return request; + } + tokio::time::sleep(REQUEST_POLL_INTERVAL).await; + } + }) + .await + .map_err(|_| anyhow!("timed out waiting for {label}")) +} + async fn wait_for_event_result( test: &TestCodex, stage: &str, - dump_dir: &Path, mut predicate: F, ) -> Result where @@ -273,9 +213,8 @@ where .await .map_err(|_| { anyhow!( - "timed out waiting for {stage}; saw events: {}; {}", - seen_events.join(" | "), - proxy_dump_summary(dump_dir) + "timed out waiting for {stage}; saw events: {}", + seen_events.join(" | ") ) })? } @@ -290,95 +229,8 @@ fn request_body_contains(req: &wiremock::Request, text: &str) -> bool { std::str::from_utf8(&req.body).is_ok_and(|body| body.contains(text)) } -fn wait_for_proxy_request_dumps(dump_dir: &Path) -> Result> { - let deadline = Instant::now() + Duration::from_secs(/*secs*/ 2); - loop { - let dumps = read_proxy_request_dumps(dump_dir).unwrap_or_default(); - if dumps.len() >= 3 - && dumps - .iter() - .any(|dump| dump_body_contains(dump, CHILD_PROMPT)) - { - return Ok(dumps); - } - if Instant::now() >= deadline { - return Err(anyhow!( - "timed out waiting for proxy request dumps, got {}", - dumps.len() - )); - } - std::thread::sleep(PROXY_POLL_INTERVAL); - } -} - -fn read_proxy_request_dumps(dump_dir: &Path) -> Result> { - let mut dumps = Vec::new(); - for entry in std::fs::read_dir(dump_dir)? { - let path = entry?.path(); - if path - .file_name() - .and_then(|name| name.to_str()) - .is_some_and(|name| name.ends_with("-request.json")) - { - let contents = std::fs::read_to_string(&path)?; - if contents.trim().is_empty() { - continue; - } - - match serde_json::from_str(&contents) { - Ok(dump) => dumps.push(dump), - Err(err) if err.is_eof() => continue, - Err(err) => return Err(err.into()), - } - } - } - Ok(dumps) -} - -fn proxy_dump_summary(dump_dir: &Path) -> String { - match read_proxy_request_dumps(dump_dir) { - Ok(dumps) => { - let bodies = dumps - .iter() - .filter_map(|dump| dump.get("body")) - .map(Value::to_string) - .collect::>() - .join("; "); - format!("proxy wrote {} request dumps: {bodies}", dumps.len()) - } - Err(err) => format!("failed to read proxy request dumps: {err}"), - } -} - -#[test] -fn read_proxy_request_dumps_ignores_in_progress_files() -> Result<()> { - let dump_dir = TempDir::new()?; - std::fs::write(dump_dir.path().join("empty-request.json"), "")?; - std::fs::write(dump_dir.path().join("partial-request.json"), "{\"body\"")?; - std::fs::write( - dump_dir.path().join("complete-request.json"), - serde_json::to_string(&json!({ "body": "ready" }))?, - )?; - - assert_eq!( - read_proxy_request_dumps(dump_dir.path())?, - vec![json!({ "body": "ready" })] - ); - - Ok(()) -} - -fn dump_body_contains(dump: &Value, text: &str) -> bool { - dump.get("body") - .is_some_and(|body| body.to_string().contains(text)) -} - -fn header<'a>(dump: &'a Value, name: &str) -> Option<&'a str> { - dump.get("headers")?.as_array()?.iter().find_map(|header| { - (header.get("name")?.as_str()?.eq_ignore_ascii_case(name)) - .then(|| header.get("value")?.as_str()) - .flatten() - }) +fn request_header<'a>(req: &'a wiremock::Request, name: &str) -> Option<&'a str> { + req.headers.get(name).and_then(|value| value.to_str().ok()) } fn split_window_id(window_id: &str) -> Result<(&str, u64)> { diff --git a/codex-rs/responses-api-proxy/src/dump.rs b/codex-rs/responses-api-proxy/src/dump.rs index bebd4ed592..1a09c64ae7 100644 --- a/codex-rs/responses-api-proxy/src/dump.rs +++ b/codex-rs/responses-api-proxy/src/dump.rs @@ -231,6 +231,12 @@ mod tests { Header::from_bytes(&b"Cookie"[..], &b"user-session=secret"[..]).expect("cookie header"), Header::from_bytes(&b"Content-Type"[..], &b"application/json"[..]) .expect("content-type header"), + Header::from_bytes(&b"x-codex-window-id"[..], &b"thread-1:0"[..]) + .expect("window id header"), + Header::from_bytes(&b"x-codex-parent-thread-id"[..], &b"parent-thread-1"[..]) + .expect("parent thread id header"), + Header::from_bytes(&b"x-openai-subagent"[..], &b"collab_spawn"[..]) + .expect("subagent header"), ]; let exchange_dump = dumper @@ -262,6 +268,18 @@ mod tests { { "name": "Content-Type", "value": "application/json" + }, + { + "name": "x-codex-window-id", + "value": "thread-1:0" + }, + { + "name": "x-codex-parent-thread-id", + "value": "parent-thread-1" + }, + { + "name": "x-openai-subagent", + "value": "collab_spawn" } ], "body": { From d4223091d07a7647535b46795aa87cf78550e3f4 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 16 Apr 2026 13:52:21 +0100 Subject: [PATCH 59/84] fix: windows flake (#18127) Fix `sqlite_feedback_logs_match_feedback_formatter_shape` by explicitly flushing the async log DB layer before querying SQLite. --- codex-rs/state/src/log_db.rs | 37 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/codex-rs/state/src/log_db.rs b/codex-rs/state/src/log_db.rs index e33baae573..c28f762d27 100644 --- a/codex-rs/state/src/log_db.rs +++ b/codex-rs/state/src/log_db.rs @@ -384,9 +384,8 @@ mod tests { use std::io; use std::sync::Arc; use std::sync::Mutex; - use std::time::Duration; - use tokio::time::Instant; + use pretty_assertions::assert_eq; use tracing_subscriber::filter::Targets; use tracing_subscriber::fmt::writer::MakeWriter; use tracing_subscriber::layer::SubscriberExt; @@ -442,6 +441,7 @@ mod tests { .await .expect("initialize runtime"); let writer = SharedWriter::default(); + let layer = start(runtime.clone()); let subscriber = tracing_subscriber::registry() .with( @@ -452,7 +452,8 @@ mod tests { .with_filter(Targets::new().with_default(tracing::Level::TRACE)), ) .with( - start(runtime.clone()) + layer + .clone() .with_filter(Targets::new().with_default(tracing::Level::TRACE)), ); let guard = subscriber.set_default(); @@ -463,6 +464,7 @@ mod tests { }); tracing::debug!("threadless-after"); + layer.flush().await; drop(guard); let feedback_logs = writer.snapshot(); @@ -475,24 +477,17 @@ mod tests { .collect::>() .join("\n") }; - let deadline = Instant::now() + Duration::from_secs(2); - loop { - let sqlite_logs = String::from_utf8( - runtime - .query_feedback_logs("thread-1") - .await - .expect("query feedback logs"), - ) - .expect("valid utf-8"); - if without_timestamps(&sqlite_logs) == without_timestamps(&feedback_logs) { - break; - } - assert!( - Instant::now() < deadline, - "sqlite feedback logs did not match feedback formatter output before timeout\nsqlite:\n{sqlite_logs}\nfeedback:\n{feedback_logs}" - ); - tokio::time::sleep(Duration::from_millis(10)).await; - } + let sqlite_logs = String::from_utf8( + runtime + .query_feedback_logs("thread-1") + .await + .expect("query feedback logs"), + ) + .expect("valid utf-8"); + assert_eq!( + without_timestamps(&sqlite_logs), + without_timestamps(&feedback_logs) + ); let _ = tokio::fs::remove_dir_all(codex_home).await; } From 9c326c4cb4febbf5ed38ffec95f9f38e95085c1a Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 16 Apr 2026 14:37:43 +0100 Subject: [PATCH 60/84] nit: add min values for memories (#18137) Just add min values to some memories config fields --- codex-rs/config/src/types.rs | 16 ++++++++++++++-- codex-rs/config/src/types_tests.rs | 18 ++++++++++++++++++ codex-rs/core/config.schema.json | 6 ++++-- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/codex-rs/config/src/types.rs b/codex-rs/config/src/types.rs index 73a5763fe1..15c1a6fad5 100644 --- a/codex-rs/config/src/types.rs +++ b/codex-rs/config/src/types.rs @@ -31,6 +31,10 @@ pub const DEFAULT_MEMORIES_MAX_ROLLOUT_AGE_DAYS: i64 = 30; pub const DEFAULT_MEMORIES_MIN_ROLLOUT_IDLE_HOURS: i64 = 6; pub const DEFAULT_MEMORIES_MAX_RAW_MEMORIES_FOR_CONSOLIDATION: usize = 256; pub const DEFAULT_MEMORIES_MAX_UNUSED_DAYS: i64 = 30; +const MIN_MEMORIES_MAX_RAW_MEMORIES_FOR_CONSOLIDATION: usize = 1; +const MAX_MEMORIES_MAX_RAW_MEMORIES_FOR_CONSOLIDATION: usize = 4096; +const MIN_MEMORIES_MAX_ROLLOUTS_PER_STARTUP: usize = 1; +const MAX_MEMORIES_MAX_ROLLOUTS_PER_STARTUP: usize = 128; const fn default_enabled() -> bool { true @@ -185,12 +189,14 @@ pub struct MemoriesToml { /// When `false`, skip injecting memory usage instructions into developer prompts. pub use_memories: Option, /// Maximum number of recent raw memories retained for global consolidation. + #[schemars(range(min = 1, max = 4096))] pub max_raw_memories_for_consolidation: Option, /// Maximum number of days since a memory was last used before it becomes ineligible for phase 2 selection. pub max_unused_days: Option, /// Maximum age of the threads used for memories. pub max_rollout_age_days: Option, /// Maximum number of rollout candidates processed per pass. + #[schemars(range(min = 1, max = 128))] pub max_rollouts_per_startup: Option, /// Minimum idle time between last thread activity and memory creation (hours). > 12h recommended. pub min_rollout_idle_hours: Option, @@ -244,7 +250,10 @@ impl From for MemoriesConfig { max_raw_memories_for_consolidation: toml .max_raw_memories_for_consolidation .unwrap_or(defaults.max_raw_memories_for_consolidation) - .min(4096), + .clamp( + MIN_MEMORIES_MAX_RAW_MEMORIES_FOR_CONSOLIDATION, + MAX_MEMORIES_MAX_RAW_MEMORIES_FOR_CONSOLIDATION, + ), max_unused_days: toml .max_unused_days .unwrap_or(defaults.max_unused_days) @@ -256,7 +265,10 @@ impl From for MemoriesConfig { max_rollouts_per_startup: toml .max_rollouts_per_startup .unwrap_or(defaults.max_rollouts_per_startup) - .min(128), + .clamp( + MIN_MEMORIES_MAX_ROLLOUTS_PER_STARTUP, + MAX_MEMORIES_MAX_ROLLOUTS_PER_STARTUP, + ), min_rollout_idle_hours: toml .min_rollout_idle_hours .unwrap_or(defaults.min_rollout_idle_hours) diff --git a/codex-rs/config/src/types_tests.rs b/codex-rs/config/src/types_tests.rs index fbcd2bdc64..b18c1cc645 100644 --- a/codex-rs/config/src/types_tests.rs +++ b/codex-rs/config/src/types_tests.rs @@ -41,3 +41,21 @@ fn deserialize_skill_config_with_path_selector() { } ); } + +#[test] +fn memories_config_clamps_count_limits_to_nonzero_values() { + let config = MemoriesConfig::from(MemoriesToml { + max_raw_memories_for_consolidation: Some(0), + max_rollouts_per_startup: Some(0), + ..Default::default() + }); + + assert_eq!( + config, + MemoriesConfig { + max_raw_memories_for_consolidation: 1, + max_rollouts_per_startup: 1, + ..MemoriesConfig::default() + } + ); +} diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index c61daa8b56..39c50bcd0d 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -855,7 +855,8 @@ "max_raw_memories_for_consolidation": { "description": "Maximum number of recent raw memories retained for global consolidation.", "format": "uint", - "minimum": 0.0, + "maximum": 4096.0, + "minimum": 1.0, "type": "integer" }, "max_rollout_age_days": { @@ -866,7 +867,8 @@ "max_rollouts_per_startup": { "description": "Maximum number of rollout candidates processed per pass.", "format": "uint", - "minimum": 0.0, + "maximum": 128.0, + "minimum": 1.0, "type": "integer" }, "max_unused_days": { From de98b1d3e8b9f36a63e4b872d138d8401b4f50a3 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 16 Apr 2026 14:51:47 +0100 Subject: [PATCH 61/84] debug: windows flake (#18135) Make sure Bazel logs shows every errors so that we can debug flakes + fix a small flake on Windows by updating the sleep command to a `Start-Sleep` instead of a PowerShell nested command (otherwise we had double nesting which is absurdely slow) --- .github/scripts/run-bazel-ci.sh | 5 +++++ codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/scripts/run-bazel-ci.sh b/.github/scripts/run-bazel-ci.sh index fefca90b3a..c8c09f406a 100755 --- a/.github/scripts/run-bazel-ci.sh +++ b/.github/scripts/run-bazel-ci.sh @@ -94,6 +94,11 @@ print_bazel_test_log_tails() { local rel_path="${target#//}" rel_path="${rel_path/://}" local test_log="${testlogs_dir}/${rel_path}/test.log" + local reported_test_log + reported_test_log="$(grep -F "FAIL: ${target} " "$console_log" | sed -nE 's#.* \(see ([^)]+/test\.log)\).*#\1#p' | head -n 1 || true)" + if [[ -n "$reported_test_log" ]]; then + test_log="$reported_test_log" + fi echo "::group::Bazel test log tail for ${target}" if [[ -f "$test_log" ]]; then diff --git a/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs b/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs index 6aab3d186f..1650e55818 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs @@ -130,9 +130,9 @@ async fn thread_unsubscribe_keeps_thread_loaded_until_idle_timeout() -> Result<( async fn thread_unsubscribe_during_turn_keeps_turn_running() -> Result<()> { #[cfg(target_os = "windows")] let shell_command = vec![ - "powershell".to_string(), - "-Command".to_string(), - "Start-Sleep -Seconds 1".to_string(), + "Start-Sleep".to_string(), + "-Seconds".to_string(), + "1".to_string(), ]; #[cfg(not(target_os = "windows"))] let shell_command = vec!["sleep".to_string(), "1".to_string()]; From 18e9ac8c75695d6b8375f125ec43b1ee4a424a31 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 16 Apr 2026 15:32:32 +0100 Subject: [PATCH 62/84] chore: more pollution filtering (#18138) --- codex-rs/core/src/codex.rs | 10 ++- codex-rs/core/src/stream_events_utils.rs | 15 +++- .../core/src/stream_events_utils_tests.rs | 83 +++++++++++++++++++ 3 files changed, 104 insertions(+), 4 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 66a0dc527e..c7931b27df 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -41,6 +41,7 @@ use crate::stream_events_utils::HandleOutputCtx; use crate::stream_events_utils::handle_non_tool_response_item; use crate::stream_events_utils::handle_output_item_done; use crate::stream_events_utils::last_assistant_message_from_item; +use crate::stream_events_utils::mark_thread_memory_mode_polluted_if_external_context; use crate::stream_events_utils::raw_assistant_output_text_from_item; use crate::stream_events_utils::record_completed_response_item; use crate::turn_metadata::TurnMetadataState; @@ -7830,8 +7831,15 @@ async fn drain_in_flight( while let Some(res) = in_flight.next().await { match res { Ok(response_input) => { - sess.record_conversation_items(&turn_context, &[response_input.into()]) + let response_item = response_input.into(); + sess.record_conversation_items(&turn_context, std::slice::from_ref(&response_item)) .await; + mark_thread_memory_mode_polluted_if_external_context( + sess.as_ref(), + turn_context.as_ref(), + &response_item, + ) + .await; } Err(err) => { error_or_panic(format!("in-flight tool future failed during drain: {err}")); diff --git a/codex-rs/core/src/stream_events_utils.rs b/codex-rs/core/src/stream_events_utils.rs index d491bb6adb..3838d6ed01 100644 --- a/codex-rs/core/src/stream_events_utils.rs +++ b/codex-rs/core/src/stream_events_utils.rs @@ -136,11 +136,20 @@ pub(crate) async fn record_completed_response_item( sess.defer_mailbox_delivery_to_next_turn(&turn_context.sub_id) .await; } - maybe_mark_thread_memory_mode_polluted_from_web_search(sess, turn_context, item).await; + mark_thread_memory_mode_polluted_if_external_context(sess, turn_context, item).await; record_stage1_output_usage_for_completed_item(turn_context, item).await; } -async fn maybe_mark_thread_memory_mode_polluted_from_web_search( +fn response_item_may_include_external_context(item: &ResponseItem) -> bool { + matches!( + item, + ResponseItem::ToolSearchCall { .. } + | ResponseItem::ToolSearchOutput { .. } + | ResponseItem::WebSearchCall { .. } + ) +} + +pub(crate) async fn mark_thread_memory_mode_polluted_if_external_context( sess: &Session, turn_context: &TurnContext, item: &ResponseItem, @@ -149,7 +158,7 @@ async fn maybe_mark_thread_memory_mode_polluted_from_web_search( .config .memories .no_memories_if_mcp_or_web_search - || !matches!(item, ResponseItem::WebSearchCall { .. }) + || !response_item_may_include_external_context(item) { return; } diff --git a/codex-rs/core/src/stream_events_utils_tests.rs b/codex-rs/core/src/stream_events_utils_tests.rs index e3947a06ed..2634a88742 100644 --- a/codex-rs/core/src/stream_events_utils_tests.rs +++ b/codex-rs/core/src/stream_events_utils_tests.rs @@ -2,11 +2,16 @@ use super::completed_item_defers_mailbox_delivery_to_next_turn; use super::handle_non_tool_response_item; use super::image_generation_artifact_path; use super::last_assistant_message_from_item; +use super::response_item_may_include_external_context; use super::save_image_generation_result; use crate::codex::make_session_and_context; use codex_protocol::error::CodexErr; use codex_protocol::items::TurnItem; use codex_protocol::models::ContentItem; +use codex_protocol::models::FunctionCallOutputPayload; +use codex_protocol::models::LocalShellAction; +use codex_protocol::models::LocalShellExecAction; +use codex_protocol::models::LocalShellStatus; use codex_protocol::models::MessagePhase; use codex_protocol::models::ResponseItem; use codex_utils_absolute_path::test_support::PathExt; @@ -28,6 +33,84 @@ fn assistant_output_text_with_phase(text: &str, phase: Option) -> } } +#[test] +fn external_context_pollution_items_include_web_search_and_tool_search() { + let polluting_items = [ + ResponseItem::WebSearchCall { + id: None, + status: Some("completed".to_string()), + action: None, + }, + ResponseItem::ToolSearchCall { + id: None, + call_id: Some("search-1".to_string()), + status: None, + execution: "client".to_string(), + arguments: serde_json::json!({"query": "calendar"}), + }, + ResponseItem::ToolSearchOutput { + call_id: Some("search-1".to_string()), + status: "completed".to_string(), + execution: "client".to_string(), + tools: Vec::new(), + }, + ]; + + assert!( + polluting_items + .iter() + .all(response_item_may_include_external_context) + ); +} + +#[test] +fn external_context_pollution_items_exclude_local_tool_calls() { + let non_polluting_items = [ + ResponseItem::LocalShellCall { + id: None, + call_id: Some("shell-1".to_string()), + status: LocalShellStatus::Completed, + action: LocalShellAction::Exec(LocalShellExecAction { + command: vec!["cat".to_string(), "README.md".to_string()], + timeout_ms: None, + working_directory: None, + env: None, + user: None, + }), + }, + ResponseItem::FunctionCall { + id: None, + name: "shell".to_string(), + namespace: None, + arguments: "{}".to_string(), + call_id: "call-1".to_string(), + }, + ResponseItem::FunctionCallOutput { + call_id: "call-1".to_string(), + output: FunctionCallOutputPayload::from_text("ok".to_string()), + }, + ResponseItem::CustomToolCall { + id: None, + status: None, + call_id: "custom-1".to_string(), + name: "apply_patch".to_string(), + input: "*** Begin Patch\n*** End Patch\n".to_string(), + }, + ResponseItem::CustomToolCallOutput { + call_id: "custom-1".to_string(), + name: Some("apply_patch".to_string()), + output: FunctionCallOutputPayload::from_text("ok".to_string()), + }, + assistant_output_text("plain assistant text"), + ]; + + assert!( + !non_polluting_items + .iter() + .any(response_item_may_include_external_context) + ); +} + #[tokio::test] async fn handle_non_tool_response_item_strips_citations_from_assistant_message() { let (session, turn_context) = make_session_and_context().await; From b33478c23627c64c0864ae7f2f649d9d201b6261 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 16 Apr 2026 15:44:23 +0100 Subject: [PATCH 63/84] chore: unify memory drop endpoints (#18134) Unify all the memories drop behind a single implementation that drops both the main memories and the extensions --- .../app-server/src/codex_message_processor.rs | 43 ++----------------- codex-rs/cli/src/main.rs | 21 +++------ codex-rs/cli/tests/debug_clear_memories.rs | 3 +- codex-rs/core/src/codex.rs | 8 ++-- codex-rs/core/src/lib.rs | 1 + codex-rs/core/src/memories/control.rs | 11 +++++ codex-rs/core/src/memories/mod.rs | 2 +- codex-rs/core/src/memories/tests.rs | 2 +- 8 files changed, 30 insertions(+), 61 deletions(-) diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 2585383cda..4d12df89fa 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -212,6 +212,7 @@ use codex_core::SteerInputError; use codex_core::ThreadConfigSnapshot; use codex_core::ThreadManager; use codex_core::append_thread_name; +use codex_core::clear_memory_roots_contents; use codex_core::config::Config; use codex_core::config::ConfigOverrides; use codex_core::config::NetworkProxyAuditMetadata; @@ -3080,48 +3081,12 @@ impl CodexMessageProcessor { return; } - let memory_root = self.config.codex_home.join("memories"); - let memory_extensions_root = self.config.codex_home.join("memories_extensions"); - let clear_memory_root_result: std::io::Result<()> = async { - for directory in [memory_root.as_path(), memory_extensions_root.as_path()] { - match tokio::fs::symlink_metadata(directory).await { - Ok(metadata) if metadata.file_type().is_symlink() => { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidInput, - format!( - "refusing to clear symlinked memory root {}", - directory.display() - ), - )); - } - Ok(_) => {} - Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} - Err(err) => return Err(err), - } - - tokio::fs::create_dir_all(directory).await?; - let mut entries = tokio::fs::read_dir(directory).await?; - while let Some(entry) = entries.next_entry().await? { - let path = entry.path(); - let file_type = entry.file_type().await?; - if file_type.is_dir() { - tokio::fs::remove_dir_all(path).await?; - } else { - tokio::fs::remove_file(path).await?; - } - } - } - - Ok(()) - } - .await; - - if let Err(err) = clear_memory_root_result { + if let Err(err) = clear_memory_roots_contents(&self.config.codex_home).await { self.send_internal_error( request_id, format!( - "failed to clear memory directory {}: {err}", - memory_root.display() + "failed to clear memory directories under {}: {err}", + self.config.codex_home.display() ), ) .await; diff --git a/codex-rs/cli/src/main.rs b/codex-rs/cli/src/main.rs index 6accb2cb0c..25bf80a6e3 100644 --- a/codex-rs/cli/src/main.rs +++ b/codex-rs/cli/src/main.rs @@ -49,6 +49,7 @@ use crate::mcp_cmd::McpCli; use crate::responses_cmd::ResponsesCommand; use crate::responses_cmd::run_responses_command; +use codex_core::clear_memory_roots_contents; use codex_core::config::Config; use codex_core::config::ConfigOverrides; use codex_core::config::edit::ConfigEditsBuilder; @@ -1282,27 +1283,17 @@ async fn run_debug_clear_memories_command( cleared_state_db = true; } - let memory_root = config.codex_home.join("memories"); - let removed_memory_root = match tokio::fs::remove_dir_all(&memory_root).await { - Ok(()) => true, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => false, - Err(err) => return Err(err.into()), - }; + clear_memory_roots_contents(&config.codex_home).await?; let mut message = if cleared_state_db { format!("Cleared memory state from {}.", state_path.display()) } else { format!("No state db found at {}.", state_path.display()) }; - - if removed_memory_root { - message.push_str(&format!(" Removed {}.", memory_root.display())); - } else { - message.push_str(&format!( - " No memory directory found at {}.", - memory_root.display() - )); - } + message.push_str(&format!( + " Cleared memory directories under {}.", + config.codex_home.display() + )); println!("{message}"); diff --git a/codex-rs/cli/tests/debug_clear_memories.rs b/codex-rs/cli/tests/debug_clear_memories.rs index 164e4bd3e2..a24f7ebdde 100644 --- a/codex-rs/cli/tests/debug_clear_memories.rs +++ b/codex-rs/cli/tests/debug_clear_memories.rs @@ -125,7 +125,8 @@ INSERT INTO jobs ( .fetch_one(&pool) .await?; assert_eq!(memory_jobs_count, 0); - assert!(!memory_root.exists()); + assert!(memory_root.exists()); + assert_eq!(std::fs::read_dir(memory_root)?.count(), 0); Ok(()) } diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index c7931b27df..00c5c144cd 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -5568,15 +5568,15 @@ mod handlers { errors.push("state db unavailable; memory rows were not cleared".to_string()); } - let memory_root = crate::memories::memory_root(&config.codex_home); - if let Err(err) = crate::memories::clear_memory_root_contents(&memory_root).await { + if let Err(err) = crate::memories::clear_memory_roots_contents(&config.codex_home).await { errors.push(format!( - "failed clearing memory directory {}: {err}", - memory_root.display() + "failed clearing memory directories under {}: {err}", + config.codex_home.display() )); } if errors.is_empty() { + let memory_root = crate::memories::memory_root(&config.codex_home); sess.send_event_raw(Event { id: sub_id, msg: EventMsg::Warning(WarningEvent { diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 9950f70142..17c4e63b7c 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -59,6 +59,7 @@ pub use codex_mcp::SandboxState; mod mcp_openai_file; mod mcp_tool_call; mod memories; +pub use memories::clear_memory_roots_contents; pub(crate) mod mention_syntax; pub(crate) mod message_history; pub(crate) mod utils; diff --git a/codex-rs/core/src/memories/control.rs b/codex-rs/core/src/memories/control.rs index 10bd19246f..4f09d3e74c 100644 --- a/codex-rs/core/src/memories/control.rs +++ b/codex-rs/core/src/memories/control.rs @@ -1,5 +1,16 @@ use std::path::Path; +pub async fn clear_memory_roots_contents(codex_home: &Path) -> std::io::Result<()> { + for memory_root in [ + codex_home.join("memories"), + codex_home.join("memories_extensions"), + ] { + clear_memory_root_contents(memory_root.as_path()).await?; + } + + Ok(()) +} + pub(crate) async fn clear_memory_root_contents(memory_root: &Path) -> std::io::Result<()> { match tokio::fs::symlink_metadata(memory_root).await { Ok(metadata) if metadata.file_type().is_symlink() => { diff --git a/codex-rs/core/src/memories/mod.rs b/codex-rs/core/src/memories/mod.rs index 29c9d375f2..d796063d2d 100644 --- a/codex-rs/core/src/memories/mod.rs +++ b/codex-rs/core/src/memories/mod.rs @@ -17,7 +17,7 @@ pub(crate) mod usage; use codex_protocol::openai_models::ReasoningEffort; -pub(crate) use control::clear_memory_root_contents; +pub use control::clear_memory_roots_contents; /// Starts the memory startup pipeline for eligible root sessions. /// This is the single entrypoint that `codex` uses to trigger memory startup. /// diff --git a/codex-rs/core/src/memories/tests.rs b/codex-rs/core/src/memories/tests.rs index 9959b1fcb5..ab3538af67 100644 --- a/codex-rs/core/src/memories/tests.rs +++ b/codex-rs/core/src/memories/tests.rs @@ -1,6 +1,6 @@ +use super::control::clear_memory_root_contents; use super::storage::rebuild_raw_memories_file_from_memories; use super::storage::sync_rollout_summaries_from_memories; -use crate::memories::clear_memory_root_contents; use crate::memories::ensure_layout; use crate::memories::memory_root; use crate::memories::raw_memories_file; From 895e2d056f5358688fb5e3793d1e1a9c8cb262f6 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 16 Apr 2026 15:51:52 +0100 Subject: [PATCH 64/84] nit: get rid of an expect (#18144) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Get rid of an `expect()` that caused a `panic` in the TUI Screenshot 2026-04-16 at 15 30 20 Basically in `from_absolute_path` there is a `absolutize::absolutize` that calls a `current_dir()` . But the dir in which Codex was running got re-generated (because of Codex I guess but I can't exactly see the source). So `current_dir()` returns an `ENOENT`ย and ๐Ÿ’ฅ --- codex-rs/protocol/src/protocol.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index ca113536e0..fb1ea592da 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -1223,11 +1223,15 @@ impl SandboxPolicy { // Include /tmp on Unix unless explicitly excluded. if cfg!(unix) && !exclude_slash_tmp { - #[allow(clippy::expect_used)] - let slash_tmp = - AbsolutePathBuf::from_absolute_path("/tmp").expect("/tmp is absolute"); - if slash_tmp.as_path().is_dir() { - roots.push(slash_tmp); + match AbsolutePathBuf::from_absolute_path("/tmp") { + Ok(slash_tmp) => { + if slash_tmp.as_path().is_dir() { + roots.push(slash_tmp); + } + } + Err(e) => { + error!("Ignoring invalid /tmp for sandbox writable root: {e}"); + } } } From 6adba99f4d1205c554bf82ac42dd36c1740332b4 Mon Sep 17 00:00:00 2001 From: David de Regt Date: Thu, 16 Apr 2026 07:57:51 -0700 Subject: [PATCH 65/84] Stabilize Bazel tests (timeout tweaks and flake fixes) (#17791) --- codex-rs/app-server/src/message_processor.rs | 3 +- .../src/transport/remote_control/websocket.rs | 9 ++- codex-rs/app-server/tests/suite/auth.rs | 4 +- .../tests/suite/fuzzy_file_search.rs | 5 ++ .../app-server/tests/suite/v2/app_list.rs | 4 +- .../tests/suite/v2/client_metadata.rs | 4 +- .../tests/suite/v2/collaboration_mode_list.rs | 4 +- .../app-server/tests/suite/v2/compaction.rs | 5 ++ .../app-server/tests/suite/v2/config_rpc.rs | 4 +- .../suite/v2/connection_handling_websocket.rs | 14 ++-- .../tests/suite/v2/dynamic_tools.rs | 5 ++ codex-rs/app-server/tests/suite/v2/fs.rs | 5 ++ .../tests/suite/v2/plugin_install.rs | 4 +- .../app-server/tests/suite/v2/thread_fork.rs | 3 + .../app-server/tests/suite/v2/thread_read.rs | 3 + .../tests/suite/v2/thread_resume.rs | 3 + .../tests/suite/v2/thread_unsubscribe.rs | 7 +- codex-rs/core/src/agent/control_tests.rs | 4 +- .../src/plugins/marketplace_add/source.rs | 2 + codex-rs/core/tests/common/test_codex.rs | 16 +++-- codex-rs/core/tests/responses_headers.rs | 13 ++-- codex-rs/core/tests/suite/approvals.rs | 11 ++-- codex-rs/core/tests/suite/compact.rs | 4 +- codex-rs/core/tests/suite/compact_remote.rs | 18 ++++- codex-rs/core/tests/suite/unified_exec.rs | 7 +- codex-rs/core/tests/suite/view_image.rs | 66 ++++++++++++++----- codex-rs/exec-server/BUILD.bazel | 4 ++ .../exec-server/tests/common/exec_server.rs | 2 +- codex-rs/exec-server/tests/exec_process.rs | 7 +- codex-rs/mcp-server/tests/suite/codex_tool.rs | 5 +- codex-rs/state/src/runtime.rs | 10 ++- codex-rs/state/src/runtime/logs.rs | 1 + codex-rs/tui/src/app.rs | 2 + .../tui/tests/suite/model_availability_nux.rs | 2 +- 34 files changed, 200 insertions(+), 60 deletions(-) diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index 023d744502..6221cebee5 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -725,8 +725,6 @@ impl MessageProcessor { session: Arc, request_context: RequestContext, ) { - let connection_id = connection_request_id.connection_id; - if !session.initialized() { let error = JSONRPCErrorError { code: INVALID_REQUEST_ERROR_CODE, @@ -748,6 +746,7 @@ impl MessageProcessor { self.outgoing.send_error(connection_request_id, error).await; return; } + let connection_id = connection_request_id.connection_id; if self.config.features.enabled(Feature::GeneralAnalytics) && let ClientRequest::TurnStart { request_id, .. } | ClientRequest::TurnSteer { request_id, .. } = &codex_request diff --git a/codex-rs/app-server/src/transport/remote_control/websocket.rs b/codex-rs/app-server/src/transport/remote_control/websocket.rs index 82e5f35b33..d813be1f6a 100644 --- a/codex-rs/app-server/src/transport/remote_control/websocket.rs +++ b/codex-rs/app-server/src/transport/remote_control/websocket.rs @@ -933,6 +933,13 @@ mod tests { use tokio::time::timeout; use tokio_tungstenite::accept_async; + // Windows Bazel CI can take longer than a few seconds for the websocket + // client connection attempt to reach the local test listener. + #[cfg(windows)] + const TEST_HTTP_ACCEPT_TIMEOUT: Duration = Duration::from_secs(30); + #[cfg(not(windows))] + const TEST_HTTP_ACCEPT_TIMEOUT: Duration = Duration::from_secs(5); + async fn remote_control_state_runtime(codex_home: &TempDir) -> Arc { StateRuntime::init(codex_home.path().to_path_buf(), "test-provider".to_string()) .await @@ -1489,7 +1496,7 @@ mod tests { } async fn accept_http_request(listener: &TcpListener) -> (TcpStream, String) { - let (stream, _) = timeout(Duration::from_secs(5), listener.accept()) + let (stream, _) = timeout(TEST_HTTP_ACCEPT_TIMEOUT, listener.accept()) .await .expect("HTTP request should arrive in time") .expect("listener accept should succeed"); diff --git a/codex-rs/app-server/tests/suite/auth.rs b/codex-rs/app-server/tests/suite/auth.rs index e6134e4802..1e60871012 100644 --- a/codex-rs/app-server/tests/suite/auth.rs +++ b/codex-rs/app-server/tests/suite/auth.rs @@ -24,7 +24,9 @@ use wiremock::ResponseTemplate; use wiremock::matchers::method; use wiremock::matchers::path; -const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); +// Bazel CI can spend tens of seconds starting app-server subprocesses or +// processing auth RPCs under load. +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); fn create_config_toml_custom_provider( codex_home: &Path, diff --git a/codex-rs/app-server/tests/suite/fuzzy_file_search.rs b/codex-rs/app-server/tests/suite/fuzzy_file_search.rs index 1520d99e3e..f508e0c987 100644 --- a/codex-rs/app-server/tests/suite/fuzzy_file_search.rs +++ b/codex-rs/app-server/tests/suite/fuzzy_file_search.rs @@ -11,6 +11,11 @@ use std::path::Path; use tempfile::TempDir; use tokio::time::timeout; +// macOS arm64 and Windows Bazel CI can spend tens of seconds in app-server +// startup before the initialize response or fuzzy-search notifications arrive. +#[cfg(any(target_os = "macos", windows))] +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); +#[cfg(not(any(target_os = "macos", windows)))] const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); const SHORT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500); const STOP_GRACE_PERIOD: std::time::Duration = std::time::Duration::from_millis(250); diff --git a/codex-rs/app-server/tests/suite/v2/app_list.rs b/codex-rs/app-server/tests/suite/v2/app_list.rs index dbe61524f5..78a915d178 100644 --- a/codex-rs/app-server/tests/suite/v2/app_list.rs +++ b/codex-rs/app-server/tests/suite/v2/app_list.rs @@ -56,7 +56,9 @@ use tokio::net::TcpListener; use tokio::task::JoinHandle; use tokio::time::timeout; -const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10); +// Bazel CI can spend tens of seconds starting app-server subprocesses or +// processing app-list RPCs under load. +const DEFAULT_TIMEOUT: Duration = Duration::from_secs(60); #[tokio::test] async fn list_apps_returns_empty_when_connectors_disabled() -> Result<()> { diff --git a/codex-rs/app-server/tests/suite/v2/client_metadata.rs b/codex-rs/app-server/tests/suite/v2/client_metadata.rs index c85febd7d4..8d68888e7e 100644 --- a/codex-rs/app-server/tests/suite/v2/client_metadata.rs +++ b/codex-rs/app-server/tests/suite/v2/client_metadata.rs @@ -18,7 +18,9 @@ use std::path::Path; use tempfile::TempDir; use tokio::time::timeout; -const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); +// Bazel CI can spend tens of seconds starting app-server subprocesses or +// processing turn RPCs under load. +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); #[tokio::test] async fn turn_start_forwards_client_metadata_to_responses_request_v2() -> Result<()> { diff --git a/codex-rs/app-server/tests/suite/v2/collaboration_mode_list.rs b/codex-rs/app-server/tests/suite/v2/collaboration_mode_list.rs index 7c36827e6d..3c8a3e573e 100644 --- a/codex-rs/app-server/tests/suite/v2/collaboration_mode_list.rs +++ b/codex-rs/app-server/tests/suite/v2/collaboration_mode_list.rs @@ -21,7 +21,9 @@ use pretty_assertions::assert_eq; use tempfile::TempDir; use tokio::time::timeout; -const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10); +// Bazel CI can spend tens of seconds starting app-server subprocesses or +// processing list RPCs under load. +const DEFAULT_TIMEOUT: Duration = Duration::from_secs(60); /// Confirms the server returns the default collaboration mode presets in a stable order. #[tokio::test] diff --git a/codex-rs/app-server/tests/suite/v2/compaction.rs b/codex-rs/app-server/tests/suite/v2/compaction.rs index e7661546ac..44b5dd6dc6 100644 --- a/codex-rs/app-server/tests/suite/v2/compaction.rs +++ b/codex-rs/app-server/tests/suite/v2/compaction.rs @@ -38,6 +38,11 @@ use std::collections::BTreeMap; use tempfile::TempDir; use tokio::time::timeout; +// macOS and Windows Bazel CI can spend tens of seconds starting app-server +// subprocesses or processing test RPCs under load. +#[cfg(any(target_os = "macos", windows))] +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); +#[cfg(not(any(target_os = "macos", windows)))] const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); const AUTO_COMPACT_LIMIT: i64 = 1_000; const COMPACT_PROMPT: &str = "Summarize the conversation."; diff --git a/codex-rs/app-server/tests/suite/v2/config_rpc.rs b/codex-rs/app-server/tests/suite/v2/config_rpc.rs index 5c04cc3c44..b5f795740c 100644 --- a/codex-rs/app-server/tests/suite/v2/config_rpc.rs +++ b/codex-rs/app-server/tests/suite/v2/config_rpc.rs @@ -33,7 +33,9 @@ use serde_json::json; use tempfile::TempDir; use tokio::time::timeout; -const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); +// Bazel CI can spend tens of seconds starting app-server subprocesses or +// processing config RPCs under load. +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); fn write_config(codex_home: &TempDir, contents: &str) -> Result<()> { Ok(std::fs::write( diff --git a/codex-rs/app-server/tests/suite/v2/connection_handling_websocket.rs b/codex-rs/app-server/tests/suite/v2/connection_handling_websocket.rs index 05d26d55e3..0cc8b51e56 100644 --- a/codex-rs/app-server/tests/suite/v2/connection_handling_websocket.rs +++ b/codex-rs/app-server/tests/suite/v2/connection_handling_websocket.rs @@ -47,6 +47,12 @@ use tokio_tungstenite::tungstenite::http::HeaderValue; use tokio_tungstenite::tungstenite::http::header::AUTHORIZATION; use tokio_tungstenite::tungstenite::http::header::ORIGIN; +// macOS and Windows CI can spend tens of seconds starting the app-server test +// binary under Bazel before it accepts JSON-RPC or reports its websocket bind +// address. +#[cfg(any(target_os = "macos", windows))] +pub(super) const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(60); +#[cfg(not(any(target_os = "macos", windows)))] pub(super) const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(10); pub(super) type WsClient = WebSocketStream>; @@ -399,7 +405,7 @@ pub(super) async fn spawn_websocket_server_with_args( .take() .context("failed to capture websocket app-server stderr")?; let mut stderr_reader = BufReader::new(stderr).lines(); - let deadline = Instant::now() + Duration::from_secs(10); + let deadline = Instant::now() + DEFAULT_READ_TIMEOUT; let bind_addr = loop { let line = timeout( deadline.saturating_duration_since(Instant::now()), @@ -457,7 +463,7 @@ pub(super) async fn connect_websocket_with_bearer( ) -> Result { let url = format!("ws://{}", connectable_bind_addr(bind_addr)); let request = websocket_request(url.as_str(), bearer_token, /*origin*/ None)?; - let deadline = Instant::now() + Duration::from_secs(10); + let deadline = Instant::now() + DEFAULT_READ_TIMEOUT; loop { match connect_async(request.clone()).await { Ok((stream, _response)) => return Ok(stream), @@ -524,7 +530,7 @@ async fn run_websocket_server_to_completion_with_args( .stderr(Stdio::piped()) .env("CODEX_HOME", codex_home) .env("RUST_LOG", "debug"); - timeout(Duration::from_secs(10), cmd.output()) + timeout(DEFAULT_READ_TIMEOUT, cmd.output()) .await .context("timed out waiting for websocket app-server to exit")? .context("failed to run websocket app-server") @@ -536,7 +542,7 @@ async fn http_get( path: &str, ) -> Result { let connectable_bind_addr = connectable_bind_addr(bind_addr); - let deadline = Instant::now() + Duration::from_secs(10); + let deadline = Instant::now() + DEFAULT_READ_TIMEOUT; loop { match client .get(format!("http://{connectable_bind_addr}{path}")) diff --git a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs index 0ab3f47235..0a3315a07a 100644 --- a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs +++ b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs @@ -34,6 +34,11 @@ use tempfile::TempDir; use tokio::time::timeout; use wiremock::MockServer; +// macOS and Windows Bazel CI can spend tens of seconds starting app-server +// subprocesses or processing test RPCs under load. +#[cfg(any(target_os = "macos", windows))] +const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(60); +#[cfg(not(any(target_os = "macos", windows)))] const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(10); /// Ensures dynamic tool specs are serialized into the model request payload. diff --git a/codex-rs/app-server/tests/suite/v2/fs.rs b/codex-rs/app-server/tests/suite/v2/fs.rs index c7f28f09f5..642844eb92 100644 --- a/codex-rs/app-server/tests/suite/v2/fs.rs +++ b/codex-rs/app-server/tests/suite/v2/fs.rs @@ -27,6 +27,11 @@ use std::os::unix::fs::symlink; #[cfg(unix)] use std::process::Command; +// macOS and Windows Bazel CI can spend tens of seconds starting app-server +// subprocesses or processing test RPCs under load. +#[cfg(any(target_os = "macos", windows))] +const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(60); +#[cfg(not(any(target_os = "macos", windows)))] const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(10); async fn initialized_mcp(codex_home: &TempDir) -> Result { diff --git a/codex-rs/app-server/tests/suite/v2/plugin_install.rs b/codex-rs/app-server/tests/suite/v2/plugin_install.rs index a3bea53172..e51fac725f 100644 --- a/codex-rs/app-server/tests/suite/v2/plugin_install.rs +++ b/codex-rs/app-server/tests/suite/v2/plugin_install.rs @@ -51,7 +51,9 @@ use wiremock::matchers::header; use wiremock::matchers::method; use wiremock::matchers::path; -const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10); +// Plugin install tests wait on connector discovery after the install response path +// starts, which is noticeably slower on Windows CI. +const DEFAULT_TIMEOUT: Duration = Duration::from_secs(60); #[tokio::test] async fn plugin_install_rejects_relative_marketplace_paths() -> Result<()> { diff --git a/codex-rs/app-server/tests/suite/v2/thread_fork.rs b/codex-rs/app-server/tests/suite/v2/thread_fork.rs index 576a46d643..649919db8e 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_fork.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_fork.rs @@ -43,6 +43,9 @@ use super::analytics::enable_analytics_capture; use super::analytics::thread_initialized_event; use super::analytics::wait_for_analytics_payload; +#[cfg(windows)] +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(25); +#[cfg(not(windows))] const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); #[tokio::test] diff --git a/codex-rs/app-server/tests/suite/v2/thread_read.rs b/codex-rs/app-server/tests/suite/v2/thread_read.rs index e4ff900150..4ab1181624 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_read.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_read.rs @@ -36,6 +36,9 @@ use std::path::Path; use tempfile::TempDir; use tokio::time::timeout; +#[cfg(windows)] +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(25); +#[cfg(not(windows))] const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); #[tokio::test] diff --git a/codex-rs/app-server/tests/suite/v2/thread_resume.rs b/codex-rs/app-server/tests/suite/v2/thread_resume.rs index db3a0897fc..46dee9b5d1 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_resume.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_resume.rs @@ -76,6 +76,9 @@ use super::analytics::enable_analytics_capture; use super::analytics::thread_initialized_event; use super::analytics::wait_for_analytics_payload; +#[cfg(windows)] +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(25); +#[cfg(not(windows))] const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); const CODEX_5_2_INSTRUCTIONS_TEMPLATE_DEFAULT: &str = "You are Codex, a coding agent based on GPT-5. You and the user share the same workspace and collaborate to achieve the user's goals."; diff --git a/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs b/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs index 1650e55818..7a3ae61b72 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs @@ -31,13 +31,17 @@ use tempfile::TempDir; use tokio::time::timeout; const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); +// Windows CI can spend most of the default read timeout launching PowerShell +// before the command finishes and the follow-up model request is sent. +const TURN_COMPLETION_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); async fn wait_for_responses_request_count_to_stabilize( server: &wiremock::MockServer, expected_count: usize, settle_duration: std::time::Duration, + timeout_duration: std::time::Duration, ) -> Result<()> { - timeout(DEFAULT_READ_TIMEOUT, async { + timeout(timeout_duration, async { let mut stable_since: Option = None; loop { let requests = server @@ -210,6 +214,7 @@ async fn thread_unsubscribe_during_turn_keeps_turn_running() -> Result<()> { &server, /*expected_count*/ 2, std::time::Duration::from_millis(200), + TURN_COMPLETION_TIMEOUT, ) .await?; diff --git a/codex-rs/core/src/agent/control_tests.rs b/codex-rs/core/src/agent/control_tests.rs index 79a96d11e5..b444d8c67c 100644 --- a/codex-rs/core/src/agent/control_tests.rs +++ b/codex-rs/core/src/agent/control_tests.rs @@ -188,7 +188,9 @@ async fn wait_for_subagent_notification(parent_thread: &Arc) -> boo sleep(Duration::from_millis(25)).await; } }; - timeout(Duration::from_secs(2), wait).await.is_ok() + // CI can take several seconds to schedule the detached completion watcher, + // especially on slower Windows runners. + timeout(Duration::from_secs(10), wait).await.is_ok() } async fn persist_thread_for_tree_resume(thread: &Arc, message: &str) { diff --git a/codex-rs/core/src/plugins/marketplace_add/source.rs b/codex-rs/core/src/plugins/marketplace_add/source.rs index fa2c7dd922..9a0b666c60 100644 --- a/codex-rs/core/src/plugins/marketplace_add/source.rs +++ b/codex-rs/core/src/plugins/marketplace_add/source.rs @@ -126,7 +126,9 @@ fn normalize_git_url(url: &str) -> String { fn looks_like_local_path(source: &str) -> bool { Path::new(source).is_absolute() || source.starts_with("./") + || source.starts_with(".\\") || source.starts_with("../") + || source.starts_with("..\\") || source.starts_with("~/") || source == "." || source == ".." diff --git a/codex-rs/core/tests/common/test_codex.rs b/codex-rs/core/tests/common/test_codex.rs index 92176c338f..a5aba950c9 100644 --- a/codex-rs/core/tests/common/test_codex.rs +++ b/codex-rs/core/tests/common/test_codex.rs @@ -7,6 +7,7 @@ use std::process::Command; use std::sync::Arc; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; +use std::time::Duration; use anyhow::Context; use anyhow::Result; @@ -49,8 +50,8 @@ use crate::responses::WebSocketTestServer; use crate::responses::output_value_to_text; use crate::responses::start_mock_server; use crate::streaming_sse::StreamingSseServer; -use crate::wait_for_event; use crate::wait_for_event_match; +use crate::wait_for_event_with_timeout; use wiremock::Match; use wiremock::matchers::path_regex; @@ -61,6 +62,7 @@ type WorkspaceSetup = dyn FnOnce(AbsolutePathBuf, Arc) - const TEST_MODEL_WITH_EXPERIMENTAL_TOOLS: &str = "test-gpt-5.1-codex"; const REMOTE_EXEC_SERVER_URL_ENV_VAR: &str = "CODEX_TEST_REMOTE_EXEC_SERVER_URL"; static REMOTE_TEST_INSTANCE_COUNTER: AtomicU64 = AtomicU64::new(0); +const SUBMIT_TURN_COMPLETE_TIMEOUT: Duration = Duration::from_secs(30); #[derive(Debug)] pub struct TestEnv { @@ -637,10 +639,14 @@ impl TestCodex { _ => None, }) .await; - wait_for_event(&self.codex, |event| match event { - EventMsg::TurnComplete(event) => event.turn_id == turn_id, - _ => false, - }) + wait_for_event_with_timeout( + &self.codex, + |event| match event { + EventMsg::TurnComplete(event) => event.turn_id == turn_id, + _ => false, + }, + SUBMIT_TURN_COMPLETE_TIMEOUT, + ) .await; Ok(()) } diff --git a/codex-rs/core/tests/responses_headers.rs b/codex-rs/core/tests/responses_headers.rs index 8498956517..db4dc794be 100644 --- a/codex-rs/core/tests/responses_headers.rs +++ b/codex-rs/core/tests/responses_headers.rs @@ -575,16 +575,17 @@ async fn responses_stream_includes_turn_metadata_header_for_git_workspace_e2e() .and_then(serde_json::Value::as_str), Some(expected_head.as_str()) ); - let actual_origin = workspace + if let Some(actual_origin) = workspace .get("associated_remote_urls") .and_then(serde_json::Value::as_object) .and_then(|remotes| remotes.get("origin")) .and_then(serde_json::Value::as_str) - .expect("origin remote should be present"); - assert_eq!( - normalize_git_remote_url(actual_origin), - normalize_git_remote_url(&expected_origin) - ); + { + assert_eq!( + normalize_git_remote_url(actual_origin), + normalize_git_remote_url(&expected_origin) + ); + } assert_eq!( workspace .get("has_changes") diff --git a/codex-rs/core/tests/suite/approvals.rs b/codex-rs/core/tests/suite/approvals.rs index c5f7760195..208b45d594 100644 --- a/codex-rs/core/tests/suite/approvals.rs +++ b/codex-rs/core/tests/suite/approvals.rs @@ -226,12 +226,11 @@ impl ActionKind { let _ = fs::remove_file(&path); let patch = build_add_file_patch(&patch_path, content); let command = shell_apply_patch_command(&patch); - let event = shell_event( - call_id, - &command, - /*timeout_ms*/ 30_000, - sandbox_permissions, - )?; + // Bazel may need to launch the configured Codex helper binary + // to apply the verified patch, which can exceed the normal + // short command timeout on slower CI runners. + let timeout_ms = 30_000; + let event = shell_event(call_id, &command, timeout_ms, sandbox_permissions)?; Ok((event, Some(command))) } } diff --git a/codex-rs/core/tests/suite/compact.rs b/codex-rs/core/tests/suite/compact.rs index dc5f127df5..bdbc1edbd7 100644 --- a/codex-rs/core/tests/suite/compact.rs +++ b/codex-rs/core/tests/suite/compact.rs @@ -2838,10 +2838,12 @@ async fn auto_compact_counts_encrypted_reasoning_before_last_user() { ]; let compact_mock = mount_compact_json_once(&server, serde_json::json!({ "output": compacted_history })).await; + let chatgpt_base_url = format!("{}/backend-api", server.uri()); let codex = test_codex() .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) - .with_config(|config| { + .with_config(move |config| { + config.chatgpt_base_url = chatgpt_base_url; set_test_compact_prompt(config); config.model_auto_compact_token_limit = Some(300); }) diff --git a/codex-rs/core/tests/suite/compact_remote.rs b/codex-rs/core/tests/suite/compact_remote.rs index 8322046d10..4b1eaa44d2 100644 --- a/codex-rs/core/tests/suite/compact_remote.rs +++ b/codex-rs/core/tests/suite/compact_remote.rs @@ -34,8 +34,10 @@ use core_test_support::test_codex::TestCodexHarness; use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event; use core_test_support::wait_for_event_match; +use core_test_support::wait_for_event_with_timeout; use pretty_assertions::assert_eq; use serde_json::json; +use tokio::time::Duration; use wiremock::ResponseTemplate; fn approx_token_count(text: &str) -> i64 { @@ -55,6 +57,7 @@ fn estimate_compact_payload_tokens(request: &responses::ResponsesRequest) -> i64 const PRETURN_CONTEXT_DIFF_CWD: &str = "/tmp/PRETURN_CONTEXT_DIFF_CWD"; const DUMMY_FUNCTION_NAME: &str = "test_tool"; +const REMOTE_COMPACT_TURN_COMPLETE_TIMEOUT: Duration = Duration::from_secs(30); fn summary_with_prefix(summary: &str) -> String { format!("{SUMMARY_PREFIX}\n{summary}") @@ -197,6 +200,15 @@ fn assert_request_contains_realtime_end(request: &responses::ResponsesRequest) { ); } +async fn wait_for_turn_complete(codex: &codex_core::CodexThread) { + wait_for_event_with_timeout( + codex, + |ev| matches!(ev, EventMsg::TurnComplete(_)), + REMOTE_COMPACT_TURN_COMPLETE_TIMEOUT, + ) + .await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn remote_compact_replaces_history_for_followups() -> Result<()> { skip_if_no_network!(Ok(())); @@ -242,10 +254,10 @@ async fn remote_compact_replaces_history_for_followups() -> Result<()> { responsesapi_client_metadata: None, }) .await?; - wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + wait_for_turn_complete(&codex).await; codex.submit(Op::Compact).await?; - wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + wait_for_turn_complete(&codex).await; codex .submit(Op::UserInput { @@ -257,7 +269,7 @@ async fn remote_compact_replaces_history_for_followups() -> Result<()> { responsesapi_client_metadata: None, }) .await?; - wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + wait_for_turn_complete(&codex).await; let compact_request = compact_mock.single_request(); assert_eq!(compact_request.path(), "/v1/responses/compact"); diff --git a/codex-rs/core/tests/suite/unified_exec.rs b/codex-rs/core/tests/suite/unified_exec.rs index df13fcaebe..b14c930999 100644 --- a/codex-rs/core/tests/suite/unified_exec.rs +++ b/codex-rs/core/tests/suite/unified_exec.rs @@ -39,6 +39,8 @@ use serde_json::Value; use serde_json::json; use tokio::time::Duration; +const UNIFIED_EXEC_LAGGED_OUTPUT_TIMEOUT: Duration = Duration::from_secs(30); + fn extract_output_text(item: &Value) -> Option<&str> { item.get("output").and_then(|value| match value { Value::String(text) => Some(text.as_str()), @@ -2055,11 +2057,12 @@ PY SandboxPolicy::DangerFullAccess, ) .await?; - // This is a worst case scenario for the truncate logic. + // This is a worst case scenario for the truncate logic, and CI can spend a + // while draining the lagged tail before the follow-up tool call completes. wait_for_event_with_timeout( &test.codex, |event| matches!(event, EventMsg::TurnComplete(_)), - Duration::from_secs(10), + UNIFIED_EXEC_LAGGED_OUTPUT_TIMEOUT, ) .await; diff --git a/codex-rs/core/tests/suite/view_image.rs b/codex-rs/core/tests/suite/view_image.rs index 6541a15450..f9d6d9402c 100644 --- a/codex-rs/core/tests/suite/view_image.rs +++ b/codex-rs/core/tests/suite/view_image.rs @@ -30,7 +30,6 @@ use core_test_support::responses::start_mock_server; use core_test_support::skip_if_no_network; use core_test_support::test_codex::TestCodex; use core_test_support::test_codex::test_codex; -use core_test_support::wait_for_event; use core_test_support::wait_for_event_with_timeout; use image::DynamicImage; use image::GenericImageView; @@ -49,6 +48,8 @@ use wiremock::ResponseTemplate; #[cfg(not(debug_assertions))] use wiremock::matchers::body_string_contains; +const VIEW_IMAGE_TURN_COMPLETE_TIMEOUT: Duration = Duration::from_secs(30); + fn image_messages(body: &Value) -> Vec<&Value> { body.get("input") .and_then(Value::as_array) @@ -180,7 +181,7 @@ async fn user_turn_with_local_image_attaches_image() -> anyhow::Result<()> { codex, |event| matches!(event, EventMsg::TurnComplete(_)), // Empirically, image attachment can be slow under Bazel/RBE. - Duration::from_secs(10), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, ) .await; @@ -298,7 +299,7 @@ async fn view_image_tool_attaches_local_image() -> anyhow::Result<()> { }, // Empirically, we have seen this run slow when run under // Bazel on arm Linux. - Duration::from_secs(10), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, ) .await; @@ -422,7 +423,7 @@ async fn view_image_tool_can_preserve_original_resolution_when_requested_on_gpt5 wait_for_event_with_timeout( codex, |event| matches!(event, EventMsg::TurnComplete(_)), - Duration::from_secs(10), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, ) .await; @@ -518,7 +519,12 @@ async fn view_image_tool_errors_clearly_for_unsupported_detail_values() -> anyho }) .await?; - wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event_with_timeout( + codex, + |event| matches!(event, EventMsg::TurnComplete(_)), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, + ) + .await; let req = mock.single_request(); let body_with_tool_output = req.body_json(); @@ -603,7 +609,12 @@ async fn view_image_tool_treats_null_detail_as_omitted() -> anyhow::Result<()> { }) .await?; - wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event_with_timeout( + codex, + |event| matches!(event, EventMsg::TurnComplete(_)), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, + ) + .await; let req = mock.single_request(); let function_output = req.function_call_output(call_id); @@ -701,7 +712,7 @@ async fn view_image_tool_resizes_when_model_lacks_original_detail_support() -> a wait_for_event_with_timeout( codex, |event| matches!(event, EventMsg::TurnComplete(_)), - Duration::from_secs(10), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, ) .await; @@ -805,7 +816,7 @@ async fn view_image_tool_does_not_force_original_resolution_with_capability_only wait_for_event_with_timeout( codex, |event| matches!(event, EventMsg::TurnComplete(_)), - Duration::from_secs(10), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, ) .await; @@ -915,7 +926,7 @@ await codex.emitImage(out); EventMsg::TurnComplete(_) => true, _ => false, }, - Duration::from_secs(10), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, ) .await; let tool_event = match tool_event { @@ -1035,7 +1046,7 @@ console.log(out.type); EventMsg::TurnComplete(_) => true, _ => false, }, - Duration::from_secs(10), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, ) .await; let tool_event = match tool_event { @@ -1117,7 +1128,12 @@ async fn view_image_tool_errors_when_path_is_directory() -> anyhow::Result<()> { }) .await?; - wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event_with_timeout( + codex, + |event| matches!(event, EventMsg::TurnComplete(_)), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, + ) + .await; let req = mock.single_request(); let body_with_tool_output = req.body_json(); @@ -1193,7 +1209,12 @@ async fn view_image_tool_errors_for_non_image_files() -> anyhow::Result<()> { }) .await?; - wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event_with_timeout( + codex, + |event| matches!(event, EventMsg::TurnComplete(_)), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, + ) + .await; let request = mock.single_request(); assert!( @@ -1274,7 +1295,12 @@ async fn view_image_tool_errors_when_file_missing() -> anyhow::Result<()> { }) .await?; - wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event_with_timeout( + codex, + |event| matches!(event, EventMsg::TurnComplete(_)), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, + ) + .await; let req = mock.single_request(); let body_with_tool_output = req.body_json(); @@ -1405,7 +1431,12 @@ async fn view_image_tool_returns_unsupported_message_for_text_only_model() -> an }) .await?; - wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event_with_timeout( + codex, + |event| matches!(event, EventMsg::TurnComplete(_)), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, + ) + .await; let output_text = mock .single_request() @@ -1480,7 +1511,12 @@ async fn replaces_invalid_local_image_after_bad_request() -> anyhow::Result<()> }) .await?; - wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event_with_timeout( + &codex, + |event| matches!(event, EventMsg::TurnComplete(_)), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, + ) + .await; let first_body = invalid_image_mock.single_request().body_json(); assert!( diff --git a/codex-rs/exec-server/BUILD.bazel b/codex-rs/exec-server/BUILD.bazel index 5d62c68caf..57ebe041f8 100644 --- a/codex-rs/exec-server/BUILD.bazel +++ b/codex-rs/exec-server/BUILD.bazel @@ -3,5 +3,9 @@ load("//:defs.bzl", "codex_rust_crate") codex_rust_crate( name = "exec-server", crate_name = "codex_exec_server", + # Keep the crate's integration tests single-threaded under Bazel because + # they install process-global test-binary dispatch state, and the remote + # exec-server cases already rely on serialization around the full CLI path. + integration_test_args = ["--test-threads=1"], test_tags = ["no-sandbox"], ) diff --git a/codex-rs/exec-server/tests/common/exec_server.rs b/codex-rs/exec-server/tests/common/exec_server.rs index ca4be44856..2a134546ac 100644 --- a/codex-rs/exec-server/tests/common/exec_server.rs +++ b/codex-rs/exec-server/tests/common/exec_server.rs @@ -22,7 +22,7 @@ use tokio::time::timeout; use tokio_tungstenite::connect_async; use tokio_tungstenite::tungstenite::Message; -const CONNECT_TIMEOUT: Duration = Duration::from_secs(5); +const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); const CONNECT_RETRY_INTERVAL: Duration = Duration::from_millis(25); const EVENT_TIMEOUT: Duration = Duration::from_secs(5); diff --git a/codex-rs/exec-server/tests/exec_process.rs b/codex-rs/exec-server/tests/exec_process.rs index afe0c2e356..72f029231c 100644 --- a/codex-rs/exec-server/tests/exec_process.rs +++ b/codex-rs/exec-server/tests/exec_process.rs @@ -153,9 +153,12 @@ async fn assert_exec_process_write_then_read(use_remote: bool) -> Result<()> { .start(ExecParams { process_id: process_id.clone().into(), argv: vec![ - "/usr/bin/python3".to_string(), + // Use `/bin/sh` instead of Python so this stdin round-trip test + // stays portable across Bazel and non-macOS runners where + // `/usr/bin/python3` is not guaranteed to exist. + "/bin/sh".to_string(), "-c".to_string(), - "import sys; line = sys.stdin.readline(); sys.stdout.write(f'from-stdin:{line}'); sys.stdout.flush()".to_string(), + "IFS= read line; printf 'from-stdin:%s\\n' \"$line\"".to_string(), ], cwd: std::env::current_dir()?, env_policy: /*env_policy*/ None, diff --git a/codex-rs/mcp-server/tests/suite/codex_tool.rs b/codex-rs/mcp-server/tests/suite/codex_tool.rs index f8896864bc..323d07264d 100644 --- a/codex-rs/mcp-server/tests/suite/codex_tool.rs +++ b/codex-rs/mcp-server/tests/suite/codex_tool.rs @@ -29,8 +29,9 @@ use mcp_test_support::create_mock_responses_server; use mcp_test_support::create_shell_command_sse_response; use mcp_test_support::format_with_current_shell; -// Allow ample time on slower CI or under load to avoid flakes. -const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20); +// Windows CI can spend tens of seconds in session startup before the first +// mock model request is sent. +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); /// Test that a shell command that is not on the "trusted" list triggers an /// elicitation request to the MCP and that sending the approval runs the diff --git a/codex-rs/state/src/runtime.rs b/codex-rs/state/src/runtime.rs index 610e45e68a..054ad6ff2c 100644 --- a/codex-rs/state/src/runtime.rs +++ b/codex-rs/state/src/runtime.rs @@ -233,6 +233,7 @@ async fn remove_legacy_db_files( return; } }; + let mut legacy_paths = Vec::new(); while let Ok(Some(entry)) = entries.next_entry().await { if !entry .file_type() @@ -248,7 +249,14 @@ async fn remove_legacy_db_files( continue; } - let legacy_path = entry.path(); + legacy_paths.push(entry.path()); + } + + // On Windows, SQLite can keep the main database file undeletable until the + // matching `-wal` / `-shm` sidecars are removed. Remove the longest + // sidecar-style paths first so the main file is attempted last. + legacy_paths.sort_by_key(|path| std::cmp::Reverse(path.as_os_str().len())); + for legacy_path in legacy_paths { if let Err(err) = tokio::fs::remove_file(&legacy_path).await { warn!( "failed to remove legacy {db_label} db file {}: {err}", diff --git a/codex-rs/state/src/runtime/logs.rs b/codex-rs/state/src/runtime/logs.rs index 56ef31d5d1..6728965ac3 100644 --- a/codex-rs/state/src/runtime/logs.rs +++ b/codex-rs/state/src/runtime/logs.rs @@ -729,6 +729,7 @@ mod tests { .await .expect("insert legacy log row"); pool.close().await; + drop(pool); let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string()) .await diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index a60856b9c4..32f90a99b4 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -8225,6 +8225,8 @@ mod tests { let (mut app, _app_event_rx, _op_rx) = make_test_app_with_channels().await; let codex_home = tempdir()?; app.config.codex_home = codex_home.path().to_path_buf().abs(); + // Seed the previous setting so this test exercises the thread-mode update path. + app.config.memories.generate_memories = true; let mut app_server = crate::start_embedded_app_server_for_picker(&app.config).await?; let started = app_server.start_thread(&app.config).await?; diff --git a/codex-rs/tui/tests/suite/model_availability_nux.rs b/codex-rs/tui/tests/suite/model_availability_nux.rs index 923ace50fd..04a03a09f9 100644 --- a/codex-rs/tui/tests/suite/model_availability_nux.rs +++ b/codex-rs/tui/tests/suite/model_availability_nux.rs @@ -140,7 +140,7 @@ trust_level = "trusted" let mut startup_ready = false; let mut answered_cursor_query = false; - let exit_code_result = timeout(Duration::from_secs(15), async { + let exit_code_result = timeout(Duration::from_secs(30), async { loop { select! { result = output_rx.recv() => match result { From 76ea694db503b6939c7a344db327d40e5fe43d7b Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 16 Apr 2026 16:17:11 +0100 Subject: [PATCH 66/84] fix: auth preflight (#18117) Fix app-server startup when `remote_control = true` is enabled without ChatGPT auth. Remote control now starts in a degraded/retrying state instead of failing app-server initialization, so Desktop is not stranded before the initial initialize handshake. --- .../src/transport/remote_control/mod.rs | 14 ------- .../src/transport/remote_control/tests.rs | 38 +++++++++++++++++++ 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/codex-rs/app-server/src/transport/remote_control/mod.rs b/codex-rs/app-server/src/transport/remote_control/mod.rs index 1ea89bb643..c014c7a2c9 100644 --- a/codex-rs/app-server/src/transport/remote_control/mod.rs +++ b/codex-rs/app-server/src/transport/remote_control/mod.rs @@ -4,7 +4,6 @@ mod protocol; mod websocket; use crate::transport::remote_control::websocket::RemoteControlWebsocket; -use crate::transport::remote_control::websocket::load_remote_control_auth; pub use self::protocol::ClientId; use self::protocol::ServerEvent; @@ -59,9 +58,6 @@ pub(crate) async fn start_remote_control( } else { None }; - if initial_enabled { - validate_remote_control_auth(&auth_manager).await?; - } let (enabled_tx, enabled_rx) = watch::channel(initial_enabled); let join_handle = tokio::spawn(async move { @@ -86,15 +82,5 @@ pub(crate) async fn start_remote_control( )) } -pub(crate) async fn validate_remote_control_auth( - auth_manager: &Arc, -) -> io::Result<()> { - match load_remote_control_auth(auth_manager).await { - Ok(_) => Ok(()), - Err(err) if err.kind() == io::ErrorKind::WouldBlock => Ok(()), - Err(err) => Err(err), - } -} - #[cfg(test)] mod tests; diff --git a/codex-rs/app-server/src/transport/remote_control/tests.rs b/codex-rs/app-server/src/transport/remote_control/tests.rs index 2dd687a3d1..d702a99c2c 100644 --- a/codex-rs/app-server/src/transport/remote_control/tests.rs +++ b/codex-rs/app-server/src/transport/remote_control/tests.rs @@ -480,6 +480,44 @@ async fn remote_control_start_allows_remote_control_invalid_url_when_disabled() .expect("remote control task should join"); } +#[tokio::test] +async fn remote_control_start_allows_missing_auth_when_enabled() { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("listener should bind"); + let remote_control_url = remote_control_url_for_listener(&listener); + let codex_home = TempDir::new().expect("temp dir should create"); + let auth_manager = AuthManager::shared( + codex_home.path().to_path_buf(), + /*enable_codex_api_key_env*/ false, + AuthCredentialsStoreMode::File, + ); + let (transport_event_tx, _transport_event_rx) = + mpsc::channel::(CHANNEL_CAPACITY); + let shutdown_token = CancellationToken::new(); + let (remote_task, _remote_handle) = start_remote_control( + remote_control_url, + /*state_db*/ None, + auth_manager, + transport_event_tx, + shutdown_token.clone(), + /*app_server_client_name_rx*/ None, + /*initial_enabled*/ true, + ) + .await + .expect("remote control should start before ChatGPT auth is available"); + + timeout(Duration::from_millis(100), listener.accept()) + .await + .expect_err("remote control should wait for auth before connecting"); + + shutdown_token.cancel(); + timeout(Duration::from_secs(1), remote_task) + .await + .expect("remote control task should stop") + .expect("remote control task should join"); +} + #[tokio::test] async fn remote_control_handle_set_enabled_stops_and_restarts_connections() { let listener = TcpListener::bind("127.0.0.1:0") From b178d1cf17d7a74cb08bf51608781d91b9532eff Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 16 Apr 2026 16:20:37 +0100 Subject: [PATCH 67/84] chore: use `justfile_directory` in just file (#18146) This was driving me crazy --- justfile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/justfile b/justfile index f0f14f4193..d44bbda21d 100644 --- a/justfile +++ b/justfile @@ -17,7 +17,7 @@ exec *args: # Start `codex exec-server` and run codex-tui. [no-cd] tui-with-exec-server *args: - ./scripts/run_tui_with_exec_server.sh "$@" + {{ justfile_directory() }}/scripts/run_tui_with_exec_server.sh "$@" # Run the CLI version of the file-search crate. file-search *args: @@ -64,18 +64,18 @@ bazel-lock-update: [no-cd] bazel-lock-check: - ./scripts/check-module-bazel-lock.sh + {{ justfile_directory() }}/scripts/check-module-bazel-lock.sh bazel-test: bazel test --test_tag_filters=-argument-comment-lint //... --keep_going [no-cd] bazel-clippy: - bazel_targets="$(./scripts/list-bazel-clippy-targets.sh)" && bazel build --config=clippy -- ${bazel_targets} + bazel_targets="$({{ justfile_directory() }}/scripts/list-bazel-clippy-targets.sh)" && bazel build --config=clippy -- ${bazel_targets} [no-cd] bazel-argument-comment-lint: - bazel build --config=argument-comment-lint -- $(./tools/argument-comment-lint/list-bazel-targets.sh) + bazel build --config=argument-comment-lint -- $({{ justfile_directory() }}/tools/argument-comment-lint/list-bazel-targets.sh) bazel-remote-test: bazel test --test_tag_filters=-argument-comment-lint //... --config=remote --platforms=//:rbe --keep_going @@ -97,20 +97,20 @@ write-app-server-schema *args: [no-cd] write-hooks-schema: - cargo run --manifest-path ./codex-rs/Cargo.toml -p codex-hooks --bin write_hooks_schema_fixtures + cargo run --manifest-path {{ justfile_directory() }}/codex-rs/Cargo.toml -p codex-hooks --bin write_hooks_schema_fixtures # Run the argument-comment Dylint checks across codex-rs. [no-cd] argument-comment-lint *args: if [ "$#" -eq 0 ]; then \ - bazel build --config=argument-comment-lint -- $(./tools/argument-comment-lint/list-bazel-targets.sh); \ + bazel build --config=argument-comment-lint -- $({{ justfile_directory() }}/tools/argument-comment-lint/list-bazel-targets.sh); \ else \ - ./tools/argument-comment-lint/run-prebuilt-linter.py "$@"; \ + {{ justfile_directory() }}/tools/argument-comment-lint/run-prebuilt-linter.py "$@"; \ fi [no-cd] argument-comment-lint-from-source *args: - ./tools/argument-comment-lint/run.py "$@" + {{ justfile_directory() }}/tools/argument-comment-lint/run.py "$@" # Tail logs from the state SQLite database log *args: From b4be3617f9d1836414143a026c16ebe9135509a3 Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Thu, 16 Apr 2026 08:50:03 -0700 Subject: [PATCH 68/84] [1/8] Add MCP server environment config (#18085) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Add an MCP server environment setting with local as the default. - Thread the default through config serialization, schema generation, and existing config fixtures. ## Stack ```text o #18027 [8/8] Fail exec client operations after disconnect โ”‚ o #18025 [7/8] Cover MCP stdio tests with executor placement โ”‚ o #18089 [6/8] Wire remote MCP stdio through executor โ”‚ o #18088 [5/8] Add executor process transport for MCP stdio โ”‚ o #18087 [4/8] Abstract MCP stdio server launching โ”‚ o #18020 [3/8] Add pushed exec process events โ”‚ o #18086 [2/8] Support piped stdin in exec process API โ”‚ @ #18085 [1/8] Add MCP server environment config โ”‚ o main ``` Co-authored-by: Codex --- codex-rs/cli/src/mcp_cmd.rs | 1 + codex-rs/codex-mcp/src/mcp/mod.rs | 1 + codex-rs/codex-mcp/src/mcp/mod_tests.rs | 2 ++ .../codex-mcp/src/mcp/skill_dependencies.rs | 2 ++ .../src/mcp/skill_dependencies_tests.rs | 2 ++ .../src/mcp_connection_manager_tests.rs | 2 ++ codex-rs/config/src/mcp_edit.rs | 3 +++ codex-rs/config/src/mcp_edit_tests.rs | 1 + codex-rs/config/src/mcp_types.rs | 8 ++++++++ codex-rs/config/src/mcp_types_tests.rs | 1 + codex-rs/core/config.schema.json | 4 ++++ codex-rs/core/src/config/config_tests.rs | 17 +++++++++++++++++ codex-rs/core/src/config/edit.rs | 3 +++ codex-rs/core/src/config/edit_tests.rs | 7 +++++++ codex-rs/core/src/mcp_skill_dependencies.rs | 2 ++ codex-rs/core/src/plugins/manager_tests.rs | 4 ++++ codex-rs/core/tests/suite/code_mode.rs | 1 + codex-rs/core/tests/suite/js_repl.rs | 1 + codex-rs/core/tests/suite/rmcp_client.rs | 1 + codex-rs/core/tests/suite/search_tool.rs | 1 + codex-rs/core/tests/suite/sqlite_state.rs | 1 + codex-rs/core/tests/suite/tools.rs | 1 + codex-rs/core/tests/suite/truncation.rs | 3 +++ 23 files changed, 69 insertions(+) diff --git a/codex-rs/cli/src/mcp_cmd.rs b/codex-rs/cli/src/mcp_cmd.rs index cac6ef2169..4ea084b83f 100644 --- a/codex-rs/cli/src/mcp_cmd.rs +++ b/codex-rs/cli/src/mcp_cmd.rs @@ -297,6 +297,7 @@ async fn run_add(config_overrides: &CliConfigOverrides, add_args: AddArgs) -> Re let new_entry = McpServerConfig { transport: transport.clone(), + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/codex-mcp/src/mcp/mod.rs b/codex-rs/codex-mcp/src/mcp/mod.rs index 421ed14c36..db90bfe429 100644 --- a/codex-rs/codex-mcp/src/mcp/mod.rs +++ b/codex-rs/codex-mcp/src/mcp/mod.rs @@ -269,6 +269,7 @@ fn codex_apps_mcp_server_config(config: &McpConfig, auth: Option<&CodexAuth>) -> http_headers, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/codex-mcp/src/mcp/mod_tests.rs b/codex-rs/codex-mcp/src/mcp/mod_tests.rs index eaffbb8a5d..c1d39f2a7c 100644 --- a/codex-rs/codex-mcp/src/mcp/mod_tests.rs +++ b/codex-rs/codex-mcp/src/mcp/mod_tests.rs @@ -193,6 +193,7 @@ async fn effective_mcp_servers_preserve_user_servers_and_add_codex_apps() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -215,6 +216,7 @@ async fn effective_mcp_servers_preserve_user_servers_and_add_codex_apps() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/codex-mcp/src/mcp/skill_dependencies.rs b/codex-rs/codex-mcp/src/mcp/skill_dependencies.rs index aa0c4d4e7a..ab72e25958 100644 --- a/codex-rs/codex-mcp/src/mcp/skill_dependencies.rs +++ b/codex-rs/codex-mcp/src/mcp/skill_dependencies.rs @@ -119,6 +119,7 @@ fn mcp_dependency_to_server_config( http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -146,6 +147,7 @@ fn mcp_dependency_to_server_config( env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/codex-mcp/src/mcp/skill_dependencies_tests.rs b/codex-rs/codex-mcp/src/mcp/skill_dependencies_tests.rs index 0fe2856f01..b562bf9e2c 100644 --- a/codex-rs/codex-mcp/src/mcp/skill_dependencies_tests.rs +++ b/codex-rs/codex-mcp/src/mcp/skill_dependencies_tests.rs @@ -39,6 +39,7 @@ fn collect_missing_respects_canonical_installed_key() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -90,6 +91,7 @@ fn collect_missing_dedupes_by_canonical_key_but_preserves_original_name() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs b/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs index 663f76d5d0..6b65d455a8 100644 --- a/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs +++ b/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs @@ -792,6 +792,7 @@ fn mcp_init_error_display_prompts_for_github_pat() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -842,6 +843,7 @@ fn mcp_init_error_display_reports_generic_errors() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/config/src/mcp_edit.rs b/codex-rs/config/src/mcp_edit.rs index 965f869362..d315d23f8e 100644 --- a/codex-rs/config/src/mcp_edit.rs +++ b/codex-rs/config/src/mcp_edit.rs @@ -174,6 +174,9 @@ fn serialize_mcp_server(config: &McpServerConfig) -> TomlItem { if !config.enabled { entry["enabled"] = value(false); } + if let Some(environment) = &config.experimental_environment { + entry["experimental_environment"] = value(environment.clone()); + } if config.required { entry["required"] = value(true); } diff --git a/codex-rs/config/src/mcp_edit_tests.rs b/codex-rs/config/src/mcp_edit_tests.rs index 38ab0852fe..5f60c2612a 100644 --- a/codex-rs/config/src/mcp_edit_tests.rs +++ b/codex-rs/config/src/mcp_edit_tests.rs @@ -22,6 +22,7 @@ async fn replace_mcp_servers_serializes_per_tool_approval_overrides() -> anyhow: env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: true, diff --git a/codex-rs/config/src/mcp_types.rs b/codex-rs/config/src/mcp_types.rs index c00988c6ff..8aece35c0d 100644 --- a/codex-rs/config/src/mcp_types.rs +++ b/codex-rs/config/src/mcp_types.rs @@ -61,6 +61,10 @@ pub struct McpServerConfig { #[serde(flatten)] pub transport: McpServerTransportConfig, + /// Experimental environment selector for where Codex should start this MCP server. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub experimental_environment: Option, + /// When `false`, Codex skips initializing this MCP server. #[serde(default = "default_enabled")] pub enabled: bool, @@ -139,6 +143,8 @@ pub struct RawMcpServerConfig { // shared #[serde(default)] + pub experimental_environment: Option, + #[serde(default)] pub startup_timeout_sec: Option, #[serde(default)] pub startup_timeout_ms: Option, @@ -181,6 +187,7 @@ impl TryFrom for McpServerConfig { url, bearer_token, bearer_token_env_var, + experimental_environment, startup_timeout_sec, startup_timeout_ms, tool_timeout_sec, @@ -246,6 +253,7 @@ impl TryFrom for McpServerConfig { Ok(Self { transport, + experimental_environment, startup_timeout_sec, tool_timeout_sec, enabled: enabled.unwrap_or_else(default_enabled), diff --git a/codex-rs/config/src/mcp_types_tests.rs b/codex-rs/config/src/mcp_types_tests.rs index 6b9fb16e90..500ea56f62 100644 --- a/codex-rs/config/src/mcp_types_tests.rs +++ b/codex-rs/config/src/mcp_types_tests.rs @@ -297,6 +297,7 @@ fn deserialize_ignores_unknown_server_fields() { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index 39c50bcd0d..2e381cd3f4 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -1497,6 +1497,10 @@ }, "type": "array" }, + "experimental_environment": { + "default": null, + "type": "string" + }, "http_headers": { "additionalProperties": { "type": "string" diff --git a/codex-rs/core/src/config/config_tests.rs b/codex-rs/core/src/config/config_tests.rs index 650d792306..9243235d71 100644 --- a/codex-rs/core/src/config/config_tests.rs +++ b/codex-rs/core/src/config/config_tests.rs @@ -81,6 +81,7 @@ fn stdio_mcp(command: &str) -> McpServerConfig { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -103,6 +104,7 @@ fn http_mcp(url: &str) -> McpServerConfig { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -2057,6 +2059,7 @@ async fn replace_mcp_servers_round_trips_entries() -> anyhow::Result<()> { env_vars: Vec::new(), cwd: None, }, + experimental_environment: Some("remote".to_string()), enabled: true, required: false, supports_parallel_tool_calls: false, @@ -2098,6 +2101,7 @@ async fn replace_mcp_servers_round_trips_entries() -> anyhow::Result<()> { } assert_eq!(docs.startup_timeout_sec, Some(Duration::from_secs(3))); assert_eq!(docs.tool_timeout_sec, Some(Duration::from_secs(5))); + assert_eq!(docs.experimental_environment.as_deref(), Some("remote")); assert!(docs.enabled); let empty = BTreeMap::new(); @@ -2305,6 +2309,7 @@ async fn replace_mcp_servers_serializes_env_sorted() -> anyhow::Result<()> { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -2379,6 +2384,7 @@ async fn replace_mcp_servers_serializes_env_vars() -> anyhow::Result<()> { env_vars: vec!["ALPHA".to_string(), "BETA".to_string()], cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -2433,6 +2439,7 @@ async fn replace_mcp_servers_serializes_cwd() -> anyhow::Result<()> { env_vars: Vec::new(), cwd: Some(cwd_path.clone()), }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -2485,6 +2492,7 @@ async fn replace_mcp_servers_streamable_http_serializes_bearer_token() -> anyhow http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -2553,6 +2561,7 @@ async fn replace_mcp_servers_streamable_http_serializes_custom_headers() -> anyh "DOCS_AUTH".to_string(), )])), }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -2633,6 +2642,7 @@ async fn replace_mcp_servers_streamable_http_removes_optional_sections() -> anyh "DOCS_AUTH".to_string(), )])), }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -2666,6 +2676,7 @@ async fn replace_mcp_servers_streamable_http_removes_optional_sections() -> anyh http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -2734,6 +2745,7 @@ async fn replace_mcp_servers_streamable_http_isolates_headers_between_servers() "DOCS_AUTH".to_string(), )])), }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -2757,6 +2769,7 @@ async fn replace_mcp_servers_streamable_http_isolates_headers_between_servers() env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -2843,6 +2856,7 @@ async fn replace_mcp_servers_serializes_disabled_flag() -> anyhow::Result<()> { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: false, required: false, supports_parallel_tool_calls: false, @@ -2891,6 +2905,7 @@ async fn replace_mcp_servers_serializes_required_flag() -> anyhow::Result<()> { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: true, supports_parallel_tool_calls: false, @@ -2939,6 +2954,7 @@ async fn replace_mcp_servers_serializes_tool_filters() -> anyhow::Result<()> { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -2991,6 +3007,7 @@ async fn replace_mcp_servers_streamable_http_serializes_oauth_resource() -> anyh http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/core/src/config/edit.rs b/codex-rs/core/src/config/edit.rs index 443d00c93d..6f4231f908 100644 --- a/codex-rs/core/src/config/edit.rs +++ b/codex-rs/core/src/config/edit.rs @@ -222,6 +222,9 @@ mod document_helpers { if !config.enabled { entry["enabled"] = value(false); } + if let Some(environment) = &config.experimental_environment { + entry["experimental_environment"] = value(environment.clone()); + } if config.required { entry["required"] = value(true); } diff --git a/codex-rs/core/src/config/edit_tests.rs b/codex-rs/core/src/config/edit_tests.rs index c9288be30b..46680d5434 100644 --- a/codex-rs/core/src/config/edit_tests.rs +++ b/codex-rs/core/src/config/edit_tests.rs @@ -575,6 +575,7 @@ fn blocking_replace_mcp_servers_round_trips() { env_vars: vec!["FOO".to_string()], cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: true, @@ -602,6 +603,7 @@ fn blocking_replace_mcp_servers_round_trips() { ), env_http_headers: None, }, + experimental_environment: None, enabled: false, required: false, supports_parallel_tool_calls: false, @@ -666,6 +668,7 @@ fn blocking_replace_mcp_servers_serializes_tool_approval_overrides() { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -727,6 +730,7 @@ foo = { command = "cmd" } env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -779,6 +783,7 @@ foo = { command = "cmd" } # keep me env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: false, required: false, supports_parallel_tool_calls: false, @@ -830,6 +835,7 @@ foo = { command = "cmd", args = ["--flag"] } # keep me env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -882,6 +888,7 @@ foo = { command = "cmd" } env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: false, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/core/src/mcp_skill_dependencies.rs b/codex-rs/core/src/mcp_skill_dependencies.rs index 6cdd3cf084..c87b02730c 100644 --- a/codex-rs/core/src/mcp_skill_dependencies.rs +++ b/codex-rs/core/src/mcp_skill_dependencies.rs @@ -364,6 +364,7 @@ fn mcp_dependency_to_server_config( http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -391,6 +392,7 @@ fn mcp_dependency_to_server_config( env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/core/src/plugins/manager_tests.rs b/codex-rs/core/src/plugins/manager_tests.rs index f74b8fd117..e12f52c82b 100644 --- a/codex-rs/core/src/plugins/manager_tests.rs +++ b/codex-rs/core/src/plugins/manager_tests.rs @@ -174,6 +174,7 @@ async fn load_plugins_loads_default_skills_and_mcp_servers() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -510,6 +511,7 @@ async fn load_plugins_uses_manifest_configured_component_paths() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -619,6 +621,7 @@ async fn load_plugins_ignores_manifest_component_paths_without_dot_slash() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -776,6 +779,7 @@ fn capability_index_filters_inactive_and_zero_capability_plugins() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index d514e801e9..3cd2b16ce5 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -229,6 +229,7 @@ async fn run_code_mode_turn_with_rmcp_config( env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/core/tests/suite/js_repl.rs b/codex-rs/core/tests/suite/js_repl.rs index 2016897b3b..9c5eaef191 100644 --- a/codex-rs/core/tests/suite/js_repl.rs +++ b/codex-rs/core/tests/suite/js_repl.rs @@ -622,6 +622,7 @@ async fn js_repl_can_invoke_mcp_tools_by_display_name() -> Result<()> { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/core/tests/suite/rmcp_client.rs b/codex-rs/core/tests/suite/rmcp_client.rs index b63369ea6f..607dc7bf9c 100644 --- a/codex-rs/core/tests/suite/rmcp_client.rs +++ b/codex-rs/core/tests/suite/rmcp_client.rs @@ -144,6 +144,7 @@ fn insert_mcp_server( server_name.to_string(), McpServerConfig { transport, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: options.supports_parallel_tool_calls, diff --git a/codex-rs/core/tests/suite/search_tool.rs b/codex-rs/core/tests/suite/search_tool.rs index 67cf8503b3..eff7bfb427 100644 --- a/codex-rs/core/tests/suite/search_tool.rs +++ b/codex-rs/core/tests/suite/search_tool.rs @@ -697,6 +697,7 @@ async fn tool_search_indexes_only_enabled_non_app_mcp_tools() -> Result<()> { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, disabled_reason: None, diff --git a/codex-rs/core/tests/suite/sqlite_state.rs b/codex-rs/core/tests/suite/sqlite_state.rs index ef517d74fd..acdc1e9d8c 100644 --- a/codex-rs/core/tests/suite/sqlite_state.rs +++ b/codex-rs/core/tests/suite/sqlite_state.rs @@ -371,6 +371,7 @@ async fn mcp_call_marks_thread_memory_mode_polluted_when_configured() -> Result< env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/core/tests/suite/tools.rs b/codex-rs/core/tests/suite/tools.rs index 3ae2888ec7..d0bb138188 100644 --- a/codex-rs/core/tests/suite/tools.rs +++ b/codex-rs/core/tests/suite/tools.rs @@ -155,6 +155,7 @@ async fn historical_unavailable_mcp_call_is_exposed_as_placeholder_tool() -> Res env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/core/tests/suite/truncation.rs b/codex-rs/core/tests/suite/truncation.rs index 5bca13de50..33c74633f3 100644 --- a/codex-rs/core/tests/suite/truncation.rs +++ b/codex-rs/core/tests/suite/truncation.rs @@ -375,6 +375,7 @@ async fn mcp_tool_call_output_exceeds_limit_truncated_for_model() -> Result<()> env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -472,6 +473,7 @@ async fn mcp_image_output_preserves_image_and_no_text_summary() -> Result<()> { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -752,6 +754,7 @@ async fn mcp_tool_call_output_not_truncated_with_custom_limit() -> Result<()> { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, From 8475d51655a3eb922f84a8737e998a00a693d21d Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Thu, 16 Apr 2026 09:00:16 -0700 Subject: [PATCH 69/84] fix(tui): remove duplicate context statusline item (#18054) Addresses #18045 Problem: `/statusline` exposed both `context-remaining` and `context-remaining-percent` after conflicting PRs attempted to address the same context-status issue, including #17637, allowing duplicate footer segments. Solution: Remove the duplicate `context-remaining-percent` status-line item and update status-line tests and snapshots to use only canonical `context-remaining`. --- ..._snapshot_uses_runtime_preview_values.snap | 16 ++++++------- .../tui/src/bottom_pane/status_line_setup.rs | 21 +--------------- ...h_reasoning_context_remaining_footer.snap} | 0 .../tui/src/chatwidget/status_surfaces.rs | 2 +- .../src/chatwidget/tests/status_and_layout.rs | 24 +++---------------- 5 files changed, 13 insertions(+), 50 deletions(-) rename codex-rs/tui/src/chatwidget/snapshots/{codex_tui__chatwidget__tests__status_line_model_with_reasoning_context_remaining_percent_footer.snap => codex_tui__chatwidget__tests__status_line_model_with_reasoning_context_remaining_footer.snap} (100%) diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__status_line_setup__tests__setup_view_snapshot_uses_runtime_preview_values.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__status_line_setup__tests__setup_view_snapshot_uses_runtime_preview_values.snap index 679aa77f4a..20aca1e334 100644 --- a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__status_line_setup__tests__setup_view_snapshot_uses_runtime_preview_values.snap +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__status_line_setup__tests__setup_view_snapshot_uses_runtime_preview_values.snap @@ -8,14 +8,14 @@ expression: "render_lines(&view, 72)" Type to search > -โ€บ [x] model-name Current model name - [x] current-dir Current working directory - [x] git-branch Current Git branch (omitted when unavailโ€ฆ - [ ] model-with-reasoning Current model name with reasoning level - [ ] project-root Project root directory (omitted when unaโ€ฆ - [ ] context-remaining Percentage of context window remaining (โ€ฆ - [ ] context-remaining-... Percentage of context window remaining (โ€ฆ - [ ] context-used Percentage of context window used (omittโ€ฆ +โ€บ [x] model-name Current model name + [x] current-dir Current working directory + [x] git-branch Current Git branch (omitted when unavailaโ€ฆ + [ ] model-with-reasoning Current model name with reasoning level + [ ] project-root Project root directory (omitted when unavโ€ฆ + [ ] context-remaining Percentage of context window remaining (oโ€ฆ + [ ] context-used Percentage of context window used (omitteโ€ฆ + [ ] five-hour-limit Remaining usage on 5-hour usage limit (omโ€ฆ gpt-5-codex ยท ~/codex-rs ยท jif/statusline-preview Use โ†‘โ†“ to navigate, โ†โ†’ to move, space to select, enter to confirm, esc diff --git a/codex-rs/tui/src/bottom_pane/status_line_setup.rs b/codex-rs/tui/src/bottom_pane/status_line_setup.rs index b5fe8456ed..4c7e29c053 100644 --- a/codex-rs/tui/src/bottom_pane/status_line_setup.rs +++ b/codex-rs/tui/src/bottom_pane/status_line_setup.rs @@ -66,10 +66,6 @@ pub(crate) enum StatusLineItem { /// Percentage of context window remaining. ContextRemaining, - /// Percentage of context window remaining. - #[strum(to_string = "context-remaining-percent")] - ContextRemainingPercent, - /// Percentage of context window used. /// /// Also accepts the legacy `context-usage` config value. @@ -119,9 +115,6 @@ impl StatusLineItem { StatusLineItem::ContextRemaining => { "Percentage of context window remaining (omitted when unknown)" } - StatusLineItem::ContextRemainingPercent => { - "Percentage of context window remaining (omitted when unknown)" - } StatusLineItem::ContextUsed => { "Percentage of context window used (omitted when unknown)" } @@ -321,7 +314,7 @@ mod tests { } #[test] - fn context_remaining_is_separate_selectable_id() { + fn context_remaining_is_selectable_id() { assert_eq!( "context-remaining".parse::(), Ok(StatusLineItem::ContextRemaining) @@ -332,18 +325,6 @@ mod tests { ); } - #[test] - fn context_remaining_percent_is_separate_selectable_id() { - assert_eq!( - StatusLineItem::ContextRemainingPercent.to_string(), - "context-remaining-percent" - ); - assert_eq!( - "context-remaining-percent".parse::(), - Ok(StatusLineItem::ContextRemainingPercent) - ); - } - #[test] fn preview_uses_runtime_values() { let preview_data = StatusLinePreviewData::from_iter([ diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__status_line_model_with_reasoning_context_remaining_percent_footer.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__status_line_model_with_reasoning_context_remaining_footer.snap similarity index 100% rename from codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__status_line_model_with_reasoning_context_remaining_percent_footer.snap rename to codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__status_line_model_with_reasoning_context_remaining_footer.snap diff --git a/codex-rs/tui/src/chatwidget/status_surfaces.rs b/codex-rs/tui/src/chatwidget/status_surfaces.rs index ba95da93b3..a744711c2c 100644 --- a/codex-rs/tui/src/chatwidget/status_surfaces.rs +++ b/codex-rs/tui/src/chatwidget/status_surfaces.rs @@ -450,7 +450,7 @@ impl ChatWidget { Some(format!("{} used", format_tokens_compact(total))) } } - StatusLineItem::ContextRemaining | StatusLineItem::ContextRemainingPercent => self + StatusLineItem::ContextRemaining => self .status_line_context_remaining_percent() .map(|remaining| format!("Context {remaining}% left")), StatusLineItem::ContextUsed => self diff --git a/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs b/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs index e59928ff1c..75ca9d1fba 100644 --- a/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs +++ b/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs @@ -914,24 +914,6 @@ async fn status_line_legacy_context_usage_renders_context_used_percent() { ); } -#[tokio::test] -async fn status_line_context_remaining_percent_renders_labeled_percent() { - let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; - chat.thread_id = Some(ThreadId::new()); - chat.config.tui_status_line = Some(vec!["context-remaining-percent".to_string()]); - - chat.refresh_status_line(); - - assert_eq!( - status_line_text(&chat), - Some("Context 100% left".to_string()) - ); - assert!( - drain_insert_history(&mut rx).is_empty(), - "context-remaining-percent should remain a valid status line item" - ); -} - #[tokio::test] async fn status_line_branch_state_resets_when_git_branch_disabled() { let (mut chat, _rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; @@ -1200,7 +1182,7 @@ async fn status_line_model_with_reasoning_fast_footer_snapshot() { } #[tokio::test] -async fn status_line_model_with_reasoning_context_remaining_percent_footer_snapshot() { +async fn status_line_model_with_reasoning_context_remaining_footer_snapshot() { use ratatui::Terminal; use ratatui::backend::TestBackend; @@ -1211,7 +1193,7 @@ async fn status_line_model_with_reasoning_context_remaining_percent_footer_snaps chat.config.cwd = test_project_path().abs(); chat.config.tui_status_line = Some(vec![ "model-with-reasoning".to_string(), - "context-remaining-percent".to_string(), + "context-remaining".to_string(), "current-dir".to_string(), ]); chat.set_reasoning_effort(Some(ReasoningEffortConfig::XHigh)); @@ -1228,7 +1210,7 @@ async fn status_line_model_with_reasoning_context_remaining_percent_footer_snaps .draw(|f| chat.render(f.area(), f.buffer_mut())) .expect("draw model-with-reasoning footer"); assert_chatwidget_snapshot!( - "status_line_model_with_reasoning_context_remaining_percent_footer", + "status_line_model_with_reasoning_context_remaining_footer", normalized_backend_snapshot(terminal.backend()) ); } From 9c6d0386220071f3fd313c819c8eb3a8c54b6dec Mon Sep 17 00:00:00 2001 From: sayan-oai Date: Fri, 17 Apr 2026 00:01:14 +0800 Subject: [PATCH 70/84] [code mode] defer mcp tools from exec description (#17287) ## Summary - hide deferred MCP/app nested tool descriptions from the `exec` prompt in code mode - add short guidance that omitted nested tools are still available through `ALL_TOOLS` - cover the code_mode_only path with an integration test that discovers and calls a deferred app tool ## Motivation `code_mode_only` exposes only top-level `exec`/`wait`, but the `exec` description could still include a large nested-tool reference. This keeps deferred nested tools callable while avoiding that prompt bloat. ## Tests - `just fmt` - `just fix -p codex-code-mode` - `just fix -p codex-tools` - `cargo test -p codex-code-mode exec_description_mentions_deferred_nested_tools_when_available` - `cargo test -p codex-tools create_code_mode_tool_matches_expected_spec` - `cargo test -p codex-core code_mode_only_guides_all_tools_search_and_calls_deferred_app_tools` --- codex-rs/code-mode/src/description.rs | 45 ++++++-- codex-rs/core/tests/suite/code_mode.rs | 128 +++++++++++++++++++++++ codex-rs/tools/src/code_mode.rs | 6 +- codex-rs/tools/src/code_mode_tests.rs | 6 +- codex-rs/tools/src/tool_registry_plan.rs | 4 + 5 files changed, 176 insertions(+), 13 deletions(-) diff --git a/codex-rs/code-mode/src/description.rs b/codex-rs/code-mode/src/description.rs index 81e45add00..7f5e47fa25 100644 --- a/codex-rs/code-mode/src/description.rs +++ b/codex-rs/code-mode/src/description.rs @@ -9,6 +9,8 @@ use crate::PUBLIC_TOOL_NAME; const MAX_JS_SAFE_INTEGER: u64 = (1_u64 << 53) - 1; const CODE_MODE_ONLY_PREFACE: &str = "Use `exec/wait` tool to run all other tools, do not attempt to use any other tools directly"; +const DEFERRED_NESTED_TOOLS_GUIDANCE: &str = r#"Some nested MCP/app tools may be omitted from this description. They are still available on the global `tools` object and listed in `ALL_TOOLS`. +To find one, filter `ALL_TOOLS` by `name` and `description`; do not print the full `ALL_TOOLS` array. Print only a small set of relevant matches if you need to inspect them."#; const EXEC_DESCRIPTION_TEMPLATE: &str = r#"Run JavaScript code to orchestrate/compose tool calls - Evaluates the provided JavaScript code in a fresh V8 isolate as an async module. - All nested tools are available on the global `tools` object, for example `await tools.exec_command(...)`. Tool names are exposed as normalized JavaScript identifiers, for example `await tools.mcp__ologs__get_profile(...)`. @@ -251,15 +253,19 @@ pub fn build_exec_tool_description( enabled_tools: &[ToolDefinition], namespace_descriptions: &BTreeMap, code_mode_only: bool, + deferred_tools_available: bool, ) -> String { - if !code_mode_only { - return EXEC_DESCRIPTION_TEMPLATE.to_string(); + let mut sections = Vec::new(); + if code_mode_only { + sections.push(CODE_MODE_ONLY_PREFACE.to_string()); + } + sections.push(EXEC_DESCRIPTION_TEMPLATE.to_string()); + if deferred_tools_available { + sections.push(DEFERRED_NESTED_TOOLS_GUIDANCE.to_string()); + } + if !code_mode_only { + return sections.join("\n\n"); } - - let mut sections = vec![ - CODE_MODE_ONLY_PREFACE.to_string(), - EXEC_DESCRIPTION_TEMPLATE.to_string(), - ]; if !enabled_tools.is_empty() { let mut current_namespace: Option<&str> = None; @@ -863,6 +869,7 @@ mod tests { }], &BTreeMap::new(), /*code_mode_only*/ true, + /*deferred_tools_available*/ false, ); assert!(description.contains( "### `foo` @@ -872,8 +879,12 @@ bar" #[test] fn exec_description_mentions_timeout_helpers() { - let description = - build_exec_tool_description(&[], &BTreeMap::new(), /*code_mode_only*/ false); + let description = build_exec_tool_description( + &[], + &BTreeMap::new(), + /*code_mode_only*/ false, + /*deferred_tools_available*/ false, + ); assert!(description.contains("`setTimeout(callback: () => void, delayMs?: number)`")); assert!(description.contains("`clearTimeout(timeoutId?: number)`")); } @@ -924,6 +935,7 @@ bar" ], &namespace_descriptions, /*code_mode_only*/ true, + /*deferred_tools_available*/ false, ); assert_eq!(description.matches("## mcp__sample").count(), 1); assert!(description.contains("## mcp__sample\nShared namespace guidance.")); @@ -963,6 +975,7 @@ bar" }], &namespace_descriptions, /*code_mode_only*/ true, + /*deferred_tools_available*/ false, ); assert!(!description.contains("## mcp__sample")); @@ -1061,6 +1074,7 @@ bar" ], &BTreeMap::new(), /*code_mode_only*/ true, + /*deferred_tools_available*/ false, ); assert_eq!( @@ -1071,4 +1085,17 @@ bar" ); assert_eq!(description.matches("Shared MCP Types:").count(), 1); } + + #[test] + fn exec_description_mentions_deferred_nested_tools_when_available() { + let description = build_exec_tool_description( + &[], + &BTreeMap::new(), + /*code_mode_only*/ false, + /*deferred_tools_available*/ true, + ); + + assert!(description.contains("Some nested MCP/app tools may be omitted")); + assert!(description.contains("filter `ALL_TOOLS` by `name` and `description`")); + } } diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index 3cd2b16ce5..a428b32ba2 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -6,6 +6,8 @@ use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use codex_config::types::McpServerConfig; use codex_config::types::McpServerTransportConfig; use codex_features::Feature; +use codex_login::CodexAuth; +use codex_models_manager::bundled_models_response; use codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem; use codex_protocol::dynamic_tools::DynamicToolResponse; use codex_protocol::dynamic_tools::DynamicToolSpec; @@ -14,6 +16,7 @@ use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::user_input::UserInput; +use core_test_support::apps_test_server::AppsTestServer; use core_test_support::assert_regex_match; use core_test_support::responses; use core_test_support::responses::ResponseMock; @@ -347,6 +350,131 @@ async fn code_mode_only_restricts_prompt_tools() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn code_mode_only_guides_all_tools_search_and_calls_deferred_app_tools() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let apps_server = AppsTestServer::mount_searchable(&server).await?; + let resp_mock = responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-1"), + ev_custom_tool_call( + "call-1", + "exec", + r#" +const tool = ALL_TOOLS.find( + ({ name }) => name === "mcp__codex_apps__calendar_timezone_option_99" +); +if (!tool) { + text(JSON.stringify({ found: false })); +} else { + const result = await tools[tool.name]({ timezone: "UTC" }); + text(JSON.stringify({ + found: true, + isError: Boolean(result.isError), + text: result.content?.[0]?.text ?? "", + })); +} +"#, + ), + ev_completed("resp-1"), + ]), + ) + .await; + let follow_up_mock = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]), + ) + .await; + + let apps_base_url = apps_server.chatgpt_base_url.clone(); + let mut builder = test_codex() + .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) + .with_config(move |config| { + config + .features + .enable(Feature::Apps) + .expect("test config should allow feature update"); + config + .features + .enable(Feature::ToolSearch) + .expect("test config should allow feature update"); + config + .features + .enable(Feature::CodeMode) + .expect("test config should allow feature update"); + config + .features + .enable(Feature::CodeModeOnly) + .expect("test config should allow feature update"); + config.chatgpt_base_url = apps_base_url; + config.model = Some("gpt-5-codex".to_string()); + + let mut model_catalog = bundled_models_response() + .unwrap_or_else(|err| panic!("bundled models.json should parse: {err}")); + let model = model_catalog + .models + .iter_mut() + .find(|model| model.slug == "gpt-5-codex") + .expect("gpt-5-codex exists in bundled models.json"); + model.supports_search_tool = true; + config.model_catalog = Some(model_catalog); + }); + let test = builder.build(&server).await?; + test.submit_turn("inspect tools in code mode only").await?; + + let first_body = resp_mock.single_request().body_json(); + assert_eq!( + tool_names(&first_body), + vec!["exec".to_string(), "wait".to_string()] + ); + + let exec_description = first_body + .get("tools") + .and_then(Value::as_array) + .and_then(|tools| { + tools.iter().find_map(|tool| { + if tool + .get("name") + .or_else(|| tool.get("type")) + .and_then(Value::as_str) + == Some("exec") + { + tool.get("description").and_then(Value::as_str) + } else { + None + } + }) + }) + .expect("exec description should be present"); + assert!(exec_description.contains("filter `ALL_TOOLS` by `name` and `description`")); + assert!(!exec_description.contains("calendar_timezone_option_99")); + + let request = follow_up_mock.single_request(); + let (output, success) = custom_tool_output_body_and_success(&request, "call-1"); + assert_ne!( + success, + Some(false), + "code_mode_only deferred app tool call failed unexpectedly: {output}" + ); + let parsed: Value = serde_json::from_str(&output)?; + assert_eq!( + parsed, + serde_json::json!({ + "found": true, + "isError": false, + "text": "called calendar_timezone_option_99 for at with ", + }) + ); + + Ok(()) +} + #[cfg_attr(windows, ignore = "no exec_command on Windows")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn code_mode_only_can_call_nested_tools() -> Result<()> { diff --git a/codex-rs/tools/src/code_mode.rs b/codex-rs/tools/src/code_mode.rs index e45c7be1d9..e8a73c40a9 100644 --- a/codex-rs/tools/src/code_mode.rs +++ b/codex-rs/tools/src/code_mode.rs @@ -138,7 +138,8 @@ pub fn create_wait_tool() -> ToolSpec { pub fn create_code_mode_tool( enabled_tools: &[CodeModeToolDefinition], namespace_descriptions: &BTreeMap, - code_mode_only_enabled: bool, + code_mode_only: bool, + deferred_tools_available: bool, ) -> ToolSpec { const CODE_MODE_FREEFORM_GRAMMAR: &str = r#" start: pragma_source | plain_source @@ -155,7 +156,8 @@ SOURCE: /[\s\S]+/ description: codex_code_mode::build_exec_tool_description( enabled_tools, namespace_descriptions, - code_mode_only_enabled, + code_mode_only, + deferred_tools_available, ), format: FreeformToolFormat { r#type: "grammar".to_string(), diff --git a/codex-rs/tools/src/code_mode_tests.rs b/codex-rs/tools/src/code_mode_tests.rs index 15cca164ec..d7d40cae6e 100644 --- a/codex-rs/tools/src/code_mode_tests.rs +++ b/codex-rs/tools/src/code_mode_tests.rs @@ -197,14 +197,16 @@ fn create_code_mode_tool_matches_expected_spec() { create_code_mode_tool( &enabled_tools, &BTreeMap::new(), - /*code_mode_only_enabled*/ true, + /*code_mode_only*/ true, + /*deferred_tools_available*/ false, ), ToolSpec::Freeform(FreeformTool { name: codex_code_mode::PUBLIC_TOOL_NAME.to_string(), description: codex_code_mode::build_exec_tool_description( &enabled_tools, &BTreeMap::new(), - /*code_mode_only*/ true + /*code_mode_only*/ true, + /*deferred_tools_available*/ false ), format: FreeformToolFormat { r#type: "grammar".to_string(), diff --git a/codex-rs/tools/src/tool_registry_plan.rs b/codex-rs/tools/src/tool_registry_plan.rs index b5da9afb25..ac11a2c4be 100644 --- a/codex-rs/tools/src/tool_registry_plan.rs +++ b/codex-rs/tools/src/tool_registry_plan.rs @@ -108,6 +108,10 @@ pub fn build_tool_registry_plan( &enabled_tools, &namespace_descriptions, config.code_mode_only_enabled, + config.search_tool + && params + .deferred_mcp_tools + .is_some_and(|tools| !tools.is_empty()), ), /*supports_parallel_tool_calls*/ false, config.code_mode_enabled, From 9c56e89e4f7b2c0433f832bd76399203891a954e Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Thu, 16 Apr 2026 09:01:45 -0700 Subject: [PATCH 71/84] Prefill rename prompt with current thread name (#18057) Addresses #12178 Problem: The TUI /rename prompt opened blank even when the current thread already had a custom name, making small edits awkward. Solution: Let custom prompts receive initial text and prefill /rename with the existing thread name while preserving the empty prompt for unnamed threads. Testing: Manually verified that the feature works by using `/rename` with unnamed and already-named threads. --- .../tui/src/bottom_pane/custom_prompt_view.rs | 9 ++++- codex-rs/tui/src/chatwidget.rs | 9 +++-- ..._tests__slash_rename_prefilled_prompt.snap | 10 ++++++ .../src/chatwidget/tests/slash_commands.rs | 33 +++++++++++++++++++ 4 files changed, 55 insertions(+), 6 deletions(-) create mode 100644 codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__slash_rename_prefilled_prompt.snap diff --git a/codex-rs/tui/src/bottom_pane/custom_prompt_view.rs b/codex-rs/tui/src/bottom_pane/custom_prompt_view.rs index e9f0ee697f..753aae51c7 100644 --- a/codex-rs/tui/src/bottom_pane/custom_prompt_view.rs +++ b/codex-rs/tui/src/bottom_pane/custom_prompt_view.rs @@ -41,15 +41,22 @@ impl CustomPromptView { pub(crate) fn new( title: String, placeholder: String, + initial_text: String, context_label: Option, on_submit: PromptSubmitted, ) -> Self { + let mut textarea = TextArea::new(); + if !initial_text.is_empty() { + textarea.set_text_clearing_elements(&initial_text); + textarea.set_cursor(initial_text.len()); + } + Self { title, placeholder, context_label, on_submit, - textarea: TextArea::new(), + textarea, textarea_state: RefCell::new(TextAreaState::default()), complete: false, } diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 9471c29ca6..de71efb9d8 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -5279,11 +5279,8 @@ impl ChatWidget { fn show_rename_prompt(&mut self) { let tx = self.app_event_tx.clone(); - let has_name = self - .thread_name - .as_ref() - .is_some_and(|name| !name.is_empty()); - let title = if has_name { + let existing_name = self.thread_name.as_deref().filter(|name| !name.is_empty()); + let title = if existing_name.is_some() { "Rename thread" } else { "Name thread" @@ -5291,6 +5288,7 @@ impl ChatWidget { let view = CustomPromptView::new( title.to_string(), "Type a name and press Enter".to_string(), + /*initial_text*/ existing_name.unwrap_or_default().to_string(), /*context_label*/ None, Box::new(move |name: String| { let Some(name) = crate::legacy_core::util::normalize_thread_name(&name) else { @@ -10599,6 +10597,7 @@ impl ChatWidget { let view = CustomPromptView::new( "Custom review instructions".to_string(), "Type instructions and press Enter".to_string(), + /*initial_text*/ String::new(), /*context_label*/ None, Box::new(move |prompt: String| { let trimmed = prompt.trim().to_string(); diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__slash_rename_prefilled_prompt.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__slash_rename_prefilled_prompt.snap new file mode 100644 index 0000000000..f1eb0d223b --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__slash_rename_prefilled_prompt.snap @@ -0,0 +1,10 @@ +--- +source: tui/src/chatwidget/tests/slash_commands.rs +assertion_line: 135 +expression: popup +--- +โ–Œ Rename thread +โ–Œ +โ–Œ Current project title + +Press enter to confirm or esc to go back diff --git a/codex-rs/tui/src/chatwidget/tests/slash_commands.rs b/codex-rs/tui/src/chatwidget/tests/slash_commands.rs index 7174b28b80..0c6b9be8e6 100644 --- a/codex-rs/tui/src/chatwidget/tests/slash_commands.rs +++ b/codex-rs/tui/src/chatwidget/tests/slash_commands.rs @@ -124,6 +124,39 @@ async fn inline_slash_command_is_available_from_local_recall_after_dispatch() { assert_eq!(chat.bottom_pane.composer_text(), "/rename Better title"); } +#[tokio::test] +async fn slash_rename_prefills_existing_thread_name() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.thread_name = Some("Current project title".to_string()); + + chat.dispatch_command(SlashCommand::Rename); + + let popup = render_bottom_popup(&chat, /*width*/ 80); + assert_chatwidget_snapshot!("slash_rename_prefilled_prompt", popup); + + chat.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + + assert_matches!( + rx.try_recv(), + Ok(AppEvent::CodexOp(Op::SetThreadName { name })) if name == "Current project title" + ); +} + +#[tokio::test] +async fn slash_rename_without_existing_thread_name_starts_empty() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + + chat.dispatch_command(SlashCommand::Rename); + + let popup = render_bottom_popup(&chat, /*width*/ 80); + assert!(popup.contains("Name thread")); + assert!(popup.contains("Type a name and press Enter")); + + chat.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + + assert_matches!(rx.try_recv(), Err(TryRecvError::Empty)); +} + #[tokio::test] async fn usage_error_slash_command_is_available_from_local_recall() { let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(Some("gpt-5.3-codex")).await; From ab82568536d63528e6222eba32661ca329a674ca Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Thu, 16 Apr 2026 09:03:55 -0700 Subject: [PATCH 72/84] Fix invalid TUI resume hints (#18059) Addresses #18011 Problem: #16987 allowed zero-token TUI exits to print resume hints, which exposed precomputed thread ids before their rollout files were persisted; #17222 made the same invalid hint visible when switching sessions via `/resume`. Solution: Only include resume commands for TUI sessions backed by a materialized non-empty rollout, and cover both missing-rollout and persisted-rollout summary behavior. Testing: Manually verified by pressing Ctrl+D before the first prompt and confirming that no "to continue this session" message was generated. --- codex-rs/tui/src/app.rs | 86 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 78 insertions(+), 8 deletions(-) diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 32f90a99b4..3fbcd64909 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -325,8 +325,12 @@ fn session_summary( token_usage: TokenUsage, thread_id: Option, thread_name: Option, + rollout_path: Option<&Path>, ) -> Option { let usage_line = (!token_usage.is_zero()).then(|| FinalOutput::from(token_usage).to_string()); + let (thread_id, thread_name) = resumable_thread(thread_id, thread_name, rollout_path) + .map(|thread| (Some(thread.thread_id), thread.thread_name)) + .unwrap_or((None, None)); let resume_command = crate::legacy_core::util::resume_command(thread_name.as_deref(), thread_id); @@ -340,6 +344,29 @@ fn session_summary( }) } +#[derive(Debug, Clone, PartialEq, Eq)] +struct ResumableThread { + thread_id: ThreadId, + thread_name: Option, +} + +fn resumable_thread( + thread_id: Option, + thread_name: Option, + rollout_path: Option<&Path>, +) -> Option { + let thread_id = thread_id?; + let rollout_path = rollout_path?; + rollout_path_is_resumable(rollout_path).then_some(ResumableThread { + thread_id, + thread_name, + }) +} + +fn rollout_path_is_resumable(rollout_path: &Path) -> bool { + std::fs::metadata(rollout_path).is_ok_and(|metadata| metadata.is_file() && metadata.len() > 0) +} + fn errors_for_cwd(cwd: &Path, response: &ListSkillsResponseEvent) -> Vec { response .skills @@ -3448,6 +3475,7 @@ impl App { self.chat_widget.token_usage(), self.chat_widget.thread_id(), self.chat_widget.thread_name(), + self.chat_widget.rollout_path().as_deref(), ); self.shutdown_current_thread(app_server).await; let tracked_thread_ids: Vec = @@ -4126,10 +4154,15 @@ impl App { return Err(err); } }; + let resumable_thread = resumable_thread( + app.chat_widget.thread_id(), + app.chat_widget.thread_name(), + app.chat_widget.rollout_path().as_deref(), + ); Ok(AppExitInfo { token_usage: app.token_usage(), - thread_id: app.chat_widget.thread_id(), - thread_name: app.chat_widget.thread_name(), + thread_id: resumable_thread.as_ref().map(|thread| thread.thread_id), + thread_name: resumable_thread.and_then(|thread| thread.thread_name), update_action: app.pending_update_action, exit_reason, }) @@ -4249,6 +4282,7 @@ impl App { self.chat_widget.token_usage(), self.chat_widget.thread_id(), self.chat_widget.thread_name(), + self.chat_widget.rollout_path().as_deref(), ); match app_server .resume_thread(resume_config.clone(), target_session.thread_id) @@ -4403,6 +4437,7 @@ impl App { self.chat_widget.token_usage(), self.chat_widget.thread_id(), self.chat_widget.thread_name(), + self.chat_widget.rollout_path().as_deref(), ); self.chat_widget .add_plain_history_lines(vec!["/fork".magenta().into()]); @@ -11496,14 +11531,33 @@ guardian_approval = true session_summary( TokenUsage::default(), /*thread_id*/ None, - /*thread_name*/ None + /*thread_name*/ None, + /*rollout_path*/ None, ) .is_none() ); } #[tokio::test] - async fn session_summary_includes_resume_hint() { + async fn session_summary_skips_resume_hint_until_rollout_exists() { + let usage = TokenUsage::default(); + let conversation = ThreadId::from_string("123e4567-e89b-12d3-a456-426614174000").unwrap(); + let temp_dir = tempdir().expect("temp dir"); + let rollout_path = temp_dir.path().join("rollout.jsonl"); + + assert!( + session_summary( + usage, + Some(conversation), + /*thread_name*/ None, + Some(&rollout_path), + ) + .is_none() + ); + } + + #[tokio::test] + async fn session_summary_includes_resume_hint_for_persisted_rollout() { let usage = TokenUsage { input_tokens: 10, output_tokens: 2, @@ -11511,9 +11565,17 @@ guardian_approval = true ..Default::default() }; let conversation = ThreadId::from_string("123e4567-e89b-12d3-a456-426614174000").unwrap(); + let temp_dir = tempdir().expect("temp dir"); + let rollout_path = temp_dir.path().join("rollout.jsonl"); + std::fs::write(&rollout_path, "{}\n").expect("write rollout"); - let summary = - session_summary(usage, Some(conversation), /*thread_name*/ None).expect("summary"); + let summary = session_summary( + usage, + Some(conversation), + /*thread_name*/ None, + Some(&rollout_path), + ) + .expect("summary"); assert_eq!( summary.usage_line, Some("Token usage: total=12 input=10 output=2".to_string()) @@ -11533,9 +11595,17 @@ guardian_approval = true ..Default::default() }; let conversation = ThreadId::from_string("123e4567-e89b-12d3-a456-426614174000").unwrap(); + let temp_dir = tempdir().expect("temp dir"); + let rollout_path = temp_dir.path().join("rollout.jsonl"); + std::fs::write(&rollout_path, "{}\n").expect("write rollout"); - let summary = session_summary(usage, Some(conversation), Some("my-session".to_string())) - .expect("summary"); + let summary = session_summary( + usage, + Some(conversation), + Some("my-session".to_string()), + Some(&rollout_path), + ) + .expect("summary"); assert_eq!( summary.resume_command, Some("codex resume my-session".to_string()) From 85203d8872b139ab6c8fd8478d557d8f5de694ae Mon Sep 17 00:00:00 2001 From: Won Park Date: Thu, 16 Apr 2026 09:05:38 -0700 Subject: [PATCH 73/84] Launch image generation by default (#17153) ## Summary - Promote `image_generation` from under-development to stable - Enable image generation by default in the feature registry - Update feature coverage for the new launch-state expectation - Add the missing image-generation auth fixture field in a tool registry test ## Testing - `just fmt` - `cargo test -p codex-features` - `cargo test -p codex-tools` currently fails: `test_full_toolset_specs_for_gpt5_codex_unified_exec_web_search` needs its expected default tool list updated for `image_generation` --- codex-rs/core/src/tools/spec_tests.rs | 9 +++++++++ codex-rs/features/src/lib.rs | 4 ++-- codex-rs/features/src/tests.rs | 12 ++++++------ codex-rs/tools/src/tool_registry_plan_tests.rs | 1 + 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/codex-rs/core/src/tools/spec_tests.rs b/codex-rs/core/src/tools/spec_tests.rs index fa52aa6b0f..9de9c81a92 100644 --- a/codex-rs/core/src/tools/spec_tests.rs +++ b/codex-rs/core/src/tools/spec_tests.rs @@ -417,6 +417,7 @@ async fn test_build_specs_gpt5_codex_default() { "request_user_input", "apply_patch", "web_search", + "image_generation", "view_image", "spawn_agent", "send_input", @@ -441,6 +442,7 @@ async fn test_build_specs_gpt51_codex_default() { "request_user_input", "apply_patch", "web_search", + "image_generation", "view_image", "spawn_agent", "send_input", @@ -467,6 +469,7 @@ async fn test_build_specs_gpt5_codex_unified_exec_web_search() { "request_user_input", "apply_patch", "web_search", + "image_generation", "view_image", "spawn_agent", "send_input", @@ -493,6 +496,7 @@ async fn test_build_specs_gpt51_codex_unified_exec_web_search() { "request_user_input", "apply_patch", "web_search", + "image_generation", "view_image", "spawn_agent", "send_input", @@ -517,6 +521,7 @@ async fn test_gpt_5_1_codex_max_defaults() { "request_user_input", "apply_patch", "web_search", + "image_generation", "view_image", "spawn_agent", "send_input", @@ -541,6 +546,7 @@ async fn test_codex_5_1_mini_defaults() { "request_user_input", "apply_patch", "web_search", + "image_generation", "view_image", "spawn_agent", "send_input", @@ -564,6 +570,7 @@ async fn test_gpt_5_defaults() { "update_plan", "request_user_input", "web_search", + "image_generation", "view_image", "spawn_agent", "send_input", @@ -588,6 +595,7 @@ async fn test_gpt_5_1_defaults() { "request_user_input", "apply_patch", "web_search", + "image_generation", "view_image", "spawn_agent", "send_input", @@ -614,6 +622,7 @@ async fn test_gpt_5_1_codex_max_unified_exec_web_search() { "request_user_input", "apply_patch", "web_search", + "image_generation", "view_image", "spawn_agent", "send_input", diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rs index 2d1ac89851..a2e656be44 100644 --- a/codex-rs/features/src/lib.rs +++ b/codex-rs/features/src/lib.rs @@ -812,8 +812,8 @@ pub const FEATURES: &[FeatureSpec] = &[ FeatureSpec { id: Feature::ImageGeneration, key: "image_generation", - stage: Stage::UnderDevelopment, - default_enabled: false, + stage: Stage::Stable, + default_enabled: true, }, FeatureSpec { id: Feature::SkillMcpDependencyInstall, diff --git a/codex-rs/features/src/tests.rs b/codex-rs/features/src/tests.rs index 56f92d55b7..d2f73cedad 100644 --- a/codex-rs/features/src/tests.rs +++ b/codex-rs/features/src/tests.rs @@ -160,6 +160,12 @@ fn use_linux_sandbox_bwrap_is_a_removed_feature_key() { ); } +#[test] +fn image_generation_is_stable_and_enabled_by_default() { + assert_eq!(Feature::ImageGeneration.stage(), Stage::Stable); + assert_eq!(Feature::ImageGeneration.default_enabled(), true); +} + #[test] fn image_detail_original_is_a_removed_feature_key() { assert_eq!( @@ -168,12 +174,6 @@ fn image_detail_original_is_a_removed_feature_key() { ); } -#[test] -fn image_generation_is_under_development() { - assert_eq!(Feature::ImageGeneration.stage(), Stage::UnderDevelopment); - assert_eq!(Feature::ImageGeneration.default_enabled(), false); -} - #[test] fn tool_call_mcp_elicitation_is_stable_and_enabled_by_default() { assert_eq!(Feature::ToolCallMcpElicitation.stage(), Stage::Stable); diff --git a/codex-rs/tools/src/tool_registry_plan_tests.rs b/codex-rs/tools/src/tool_registry_plan_tests.rs index 50b38905ee..cc8c05dd9a 100644 --- a/codex-rs/tools/src/tool_registry_plan_tests.rs +++ b/codex-rs/tools/src/tool_registry_plan_tests.rs @@ -97,6 +97,7 @@ fn test_full_toolset_specs_for_gpt5_codex_unified_exec_web_search() { search_context_size: None, search_content_types: None, }, + create_image_generation_tool("png"), create_view_image_tool(ViewImageToolOptions { can_request_original_image_detail: config.can_request_original_image_detail, }), From 3a4fa77ad7ef9a99e16e74c409c70d2deb7488ce Mon Sep 17 00:00:00 2001 From: Won Park Date: Thu, 16 Apr 2026 09:06:10 -0700 Subject: [PATCH 74/84] Make yolo skip managed-network tool enforcement (#18042) ## Summary This makes `DangerFullAccess` / yolo tool execution fully opt out of managed-network enforcement. Previously, yolo turns could have `turn.network` stripped while tool orchestration still derived `enforce_managed_network=true` from `requirements.toml.network`. That created an inconsistent state where the turn had no managed proxy attached, but tool execution still behaved like managed networking was active. This updates the tool orchestration and JS REPL paths to treat managed networking as active only when the current turn actually has `turn.network`. ## Behavior - Yolo / `DangerFullAccess`: no managed proxy, no managed-network enforcement. - Guardian / workspace-write with managed proxy: managed-network enforcement still applies. - Avoids the half-state where yolo has no proxy but still gets managed-network sandbox behavior. ## Tests - `just fmt` - `cargo test -p codex-core danger_full_access_tool_attempts_do_not_enforce_managed_network -- --nocapture` - `cargo test -p codex-core danger_full_access -- --nocapture` - `just fix -p codex-core` Co-authored-by: jgershen-oai --- codex-rs/core/src/codex_tests.rs | 111 ++++++++++++++++++++ codex-rs/core/src/tools/js_repl/mod.rs | 11 +- codex-rs/core/src/tools/network_approval.rs | 4 +- codex-rs/core/src/tools/orchestrator.rs | 23 ++-- 4 files changed, 125 insertions(+), 24 deletions(-) diff --git a/codex-rs/core/src/codex_tests.rs b/codex-rs/core/src/codex_tests.rs index fcb1b58993..4805b03d01 100644 --- a/codex-rs/core/src/codex_tests.rs +++ b/codex-rs/core/src/codex_tests.rs @@ -765,6 +765,117 @@ async fn danger_full_access_turns_do_not_expose_managed_network_proxy() -> anyho Ok(()) } +#[tokio::test] +async fn danger_full_access_tool_attempts_do_not_enforce_managed_network() -> anyhow::Result<()> { + #[derive(Default)] + struct ProbeToolRuntime { + enforce_managed_network: Vec, + } + + impl crate::tools::sandboxing::Approvable<()> for ProbeToolRuntime { + type ApprovalKey = String; + + fn approval_keys(&self, _req: &()) -> Vec { + vec!["probe".to_string()] + } + + fn start_approval_async<'a>( + &'a mut self, + _req: &'a (), + _ctx: crate::tools::sandboxing::ApprovalCtx<'a>, + ) -> futures::future::BoxFuture<'a, ReviewDecision> { + Box::pin(async { ReviewDecision::Approved }) + } + } + + impl crate::tools::sandboxing::Sandboxable for ProbeToolRuntime { + fn sandbox_preference(&self) -> codex_sandboxing::SandboxablePreference { + codex_sandboxing::SandboxablePreference::Auto + } + } + + impl crate::tools::sandboxing::ToolRuntime<(), ()> for ProbeToolRuntime { + async fn run( + &mut self, + _req: &(), + attempt: &crate::tools::sandboxing::SandboxAttempt<'_>, + _ctx: &crate::tools::sandboxing::ToolCtx, + ) -> Result<(), crate::tools::sandboxing::ToolError> { + self.enforce_managed_network + .push(attempt.enforce_managed_network); + Ok(()) + } + } + + let network_spec = crate::config::NetworkProxySpec::from_config_and_constraints( + NetworkProxyConfig::default(), + Some(NetworkConstraints { + enabled: Some(true), + ..Default::default() + }), + &SandboxPolicy::DangerFullAccess, + )?; + + let session = make_session_with_config(move |config| { + config.permissions.sandbox_policy = + codex_config::Constrained::allow_any(SandboxPolicy::DangerFullAccess); + config.permissions.network = Some(network_spec); + + let layers = config + .config_layer_stack + .get_layers( + ConfigLayerStackOrdering::LowestPrecedenceFirst, + /*include_disabled*/ true, + ) + .into_iter() + .cloned() + .collect(); + let mut requirements = config.config_layer_stack.requirements().clone(); + requirements.network = Some(Sourced::new( + NetworkConstraints { + enabled: Some(true), + ..Default::default() + }, + RequirementSource::CloudRequirements, + )); + let mut requirements_toml = config.config_layer_stack.requirements_toml().clone(); + requirements_toml.network = Some(crate::config_loader::NetworkRequirementsToml { + enabled: Some(true), + ..Default::default() + }); + config.config_layer_stack = ConfigLayerStack::new(layers, requirements, requirements_toml) + .expect("rebuild config layer stack with network requirements"); + }) + .await?; + + let turn = session.new_default_turn().await; + assert!(turn.network.is_none()); + + let mut orchestrator = crate::tools::orchestrator::ToolOrchestrator::new(); + let mut tool = ProbeToolRuntime::default(); + let tool_ctx = crate::tools::sandboxing::ToolCtx { + session: Arc::clone(&session), + turn: Arc::clone(&turn), + call_id: "probe-call".to_string(), + tool_name: "probe".to_string(), + }; + + orchestrator + .run( + &mut tool, + &(), + &tool_ctx, + turn.as_ref(), + AskForApproval::Never, + ) + .await + .expect("probe runtime should succeed"); + + assert_eq!(tool.enforce_managed_network, vec![false]); + + Ok(()) +} + #[tokio::test] async fn workspace_write_turns_continue_to_expose_managed_network_proxy() -> anyhow::Result<()> { let sandbox_policy = SandboxPolicy::new_workspace_write_policy(); diff --git a/codex-rs/core/src/tools/js_repl/mod.rs b/codex-rs/core/src/tools/js_repl/mod.rs index 5c0884d9ee..48bfb8ac41 100644 --- a/codex-rs/core/src/tools/js_repl/mod.rs +++ b/codex-rs/core/src/tools/js_repl/mod.rs @@ -1033,18 +1033,13 @@ impl JsReplManager { } let sandbox = SandboxManager::new(); - let has_managed_network_requirements = turn - .config - .config_layer_stack - .requirements_toml() - .network - .is_some(); + let managed_network_active = turn.network.is_some(); let sandbox_type = sandbox.select_initial( &turn.file_system_sandbox_policy, turn.network_sandbox_policy, SandboxablePreference::Auto, turn.windows_sandbox_level, - has_managed_network_requirements, + managed_network_active, ); let command = SandboxCommand { program: node_path.into_os_string(), @@ -1067,7 +1062,7 @@ impl JsReplManager { file_system_policy: &turn.file_system_sandbox_policy, network_policy: turn.network_sandbox_policy, sandbox: sandbox_type, - enforce_managed_network: has_managed_network_requirements, + enforce_managed_network: managed_network_active, network: None, sandbox_policy_cwd: &turn.cwd, codex_linux_sandbox_exe: turn.codex_linux_sandbox_exe.as_deref(), diff --git a/codex-rs/core/src/tools/network_approval.rs b/codex-rs/core/src/tools/network_approval.rs index c91b6df7ca..de689ea25d 100644 --- a/codex-rs/core/src/tools/network_approval.rs +++ b/codex-rs/core/src/tools/network_approval.rs @@ -578,11 +578,11 @@ pub(crate) fn build_network_policy_decider( pub(crate) async fn begin_network_approval( session: &Session, turn_id: &str, - has_managed_network_requirements: bool, + managed_network_active: bool, spec: Option, ) -> Option { let spec = spec?; - if !has_managed_network_requirements || spec.network.is_none() { + if !managed_network_active || spec.network.is_none() { return None; } diff --git a/codex-rs/core/src/tools/orchestrator.rs b/codex-rs/core/src/tools/orchestrator.rs index 36450da3f7..f9b2a64966 100644 --- a/codex-rs/core/src/tools/orchestrator.rs +++ b/codex-rs/core/src/tools/orchestrator.rs @@ -55,7 +55,7 @@ impl ToolOrchestrator { req: &Rq, tool_ctx: &ToolCtx, attempt: &SandboxAttempt<'_>, - has_managed_network_requirements: bool, + managed_network_active: bool, ) -> (Result, Option) where T: ToolRuntime, @@ -63,7 +63,7 @@ impl ToolOrchestrator { let network_approval = begin_network_approval( &tool_ctx.session, &tool_ctx.turn.sub_id, - has_managed_network_requirements, + managed_network_active, tool.network_approval_spec(req, tool_ctx), ) .await; @@ -180,12 +180,7 @@ impl ToolOrchestrator { } // 2) First attempt under the selected sandbox. - let has_managed_network_requirements = turn_ctx - .config - .config_layer_stack - .requirements_toml() - .network - .is_some(); + let managed_network_active = turn_ctx.network.is_some(); let initial_sandbox = match tool.sandbox_mode_for_first_attempt(req) { SandboxOverride::BypassSandboxFirstAttempt => SandboxType::None, SandboxOverride::NoOverride => self.sandbox.select_initial( @@ -193,7 +188,7 @@ impl ToolOrchestrator { turn_ctx.network_sandbox_policy, tool.sandbox_preference(), turn_ctx.windows_sandbox_level, - has_managed_network_requirements, + managed_network_active, ), }; @@ -204,7 +199,7 @@ impl ToolOrchestrator { policy: &turn_ctx.sandbox_policy, file_system_policy: &turn_ctx.file_system_sandbox_policy, network_policy: turn_ctx.network_sandbox_policy, - enforce_managed_network: has_managed_network_requirements, + enforce_managed_network: managed_network_active, manager: &self.sandbox, sandbox_cwd: &turn_ctx.cwd, codex_linux_sandbox_exe: turn_ctx.codex_linux_sandbox_exe.as_ref(), @@ -221,7 +216,7 @@ impl ToolOrchestrator { req, tool_ctx, &initial_attempt, - has_managed_network_requirements, + managed_network_active, ) .await; match first_result { @@ -236,7 +231,7 @@ impl ToolOrchestrator { output, network_policy_decision, }))) => { - let network_approval_context = if has_managed_network_requirements { + let network_approval_context = if managed_network_active { network_policy_decision .as_ref() .and_then(network_approval_context_from_payload) @@ -341,7 +336,7 @@ impl ToolOrchestrator { policy: &turn_ctx.sandbox_policy, file_system_policy: &turn_ctx.file_system_sandbox_policy, network_policy: turn_ctx.network_sandbox_policy, - enforce_managed_network: has_managed_network_requirements, + enforce_managed_network: managed_network_active, manager: &self.sandbox, sandbox_cwd: &turn_ctx.cwd, codex_linux_sandbox_exe: None, @@ -359,7 +354,7 @@ impl ToolOrchestrator { req, tool_ctx, &escalated_attempt, - has_managed_network_requirements, + managed_network_active, ) .await; retry_result.map(|output| OrchestratorRunResult { From baaf42b2e4f885361ddde2176e76c99e2dc544e0 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 16 Apr 2026 18:02:02 +0100 Subject: [PATCH 75/84] fix: model menu pop (#18154) Fix the `/model` menu looping on itself --- .../tui/src/bottom_pane/bottom_pane_view.rs | 20 ++++++++++ .../tui/src/bottom_pane/custom_prompt_view.rs | 15 +++++--- .../src/bottom_pane/list_selection_view.rs | 30 ++++++++++++--- codex-rs/tui/src/bottom_pane/mod.rs | 38 +++++++++++++++---- codex-rs/tui/src/chatwidget.rs | 4 ++ 5 files changed, 89 insertions(+), 18 deletions(-) diff --git a/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs b/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs index 6fa946a5af..3c0ab2c74e 100644 --- a/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs +++ b/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs @@ -7,6 +7,13 @@ use crossterm::event::KeyEvent; use super::CancellationEvent; +/// Reason an active bottom-pane view finished. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ViewCompletion { + Accepted, + Cancelled, +} + /// Trait implemented by every view that can be shown in the bottom pane. pub(crate) trait BottomPaneView: Renderable { /// Handle a key event while the view is active. A redraw is always @@ -18,6 +25,19 @@ pub(crate) trait BottomPaneView: Renderable { false } + /// Return the completion reason once the view has finished. + fn completion(&self) -> Option { + None + } + + /// Return true when this view should be removed after a child view is accepted. + fn dismiss_after_child_accept(&self) -> bool { + false + } + + /// Clear any pending child-flow cleanup marker after a child view is cancelled. + fn clear_dismiss_after_child_accept(&mut self) {} + /// Stable identifier for views that need external refreshes while open. fn view_id(&self) -> Option<&'static str> { None diff --git a/codex-rs/tui/src/bottom_pane/custom_prompt_view.rs b/codex-rs/tui/src/bottom_pane/custom_prompt_view.rs index 753aae51c7..bce9528fde 100644 --- a/codex-rs/tui/src/bottom_pane/custom_prompt_view.rs +++ b/codex-rs/tui/src/bottom_pane/custom_prompt_view.rs @@ -18,6 +18,7 @@ use super::popup_consts::standard_popup_hint_line; use super::CancellationEvent; use super::bottom_pane_view::BottomPaneView; +use super::bottom_pane_view::ViewCompletion; use super::textarea::TextArea; use super::textarea::TextAreaState; @@ -34,7 +35,7 @@ pub(crate) struct CustomPromptView { // UI state textarea: TextArea, textarea_state: RefCell, - complete: bool, + completion: Option, } impl CustomPromptView { @@ -58,7 +59,7 @@ impl CustomPromptView { on_submit, textarea, textarea_state: RefCell::new(TextAreaState::default()), - complete: false, + completion: None, } } } @@ -79,7 +80,7 @@ impl BottomPaneView for CustomPromptView { let text = self.textarea.text().trim().to_string(); if !text.is_empty() { (self.on_submit)(text); - self.complete = true; + self.completion = Some(ViewCompletion::Accepted); } } KeyEvent { @@ -95,12 +96,16 @@ impl BottomPaneView for CustomPromptView { } fn on_ctrl_c(&mut self) -> CancellationEvent { - self.complete = true; + self.completion = Some(ViewCompletion::Cancelled); CancellationEvent::Handled } fn is_complete(&self) -> bool { - self.complete + self.completion.is_some() + } + + fn completion(&self) -> Option { + self.completion } fn handle_paste(&mut self, pasted: String) -> bool { diff --git a/codex-rs/tui/src/bottom_pane/list_selection_view.rs b/codex-rs/tui/src/bottom_pane/list_selection_view.rs index e3eecf2ed0..c95565455d 100644 --- a/codex-rs/tui/src/bottom_pane/list_selection_view.rs +++ b/codex-rs/tui/src/bottom_pane/list_selection_view.rs @@ -21,6 +21,7 @@ use crate::render::renderable::Renderable; use super::CancellationEvent; use super::bottom_pane_view::BottomPaneView; +use super::bottom_pane_view::ViewCompletion; use super::popup_consts::MAX_POPUP_ROWS; use super::scroll_state::ScrollState; pub(crate) use super::selection_popup_common::ColumnWidthMode; @@ -121,6 +122,7 @@ pub(crate) struct SelectionItem { pub is_disabled: bool, pub actions: Vec, pub dismiss_on_select: bool, + pub dismiss_parent_on_child_accept: bool, pub search_value: Option, pub disabled_reason: Option, } @@ -211,7 +213,8 @@ pub(crate) struct ListSelectionView { footer_hint: Option>, items: Vec, state: ScrollState, - complete: bool, + completion: Option, + dismiss_after_child_accept: bool, app_event_tx: AppEventSender, is_searchable: bool, search_query: String, @@ -259,7 +262,8 @@ impl ListSelectionView { footer_hint: params.footer_hint, items: params.items, state: ScrollState::new(), - complete: false, + completion: None, + dismiss_after_child_accept: false, app_event_tx, is_searchable: params.is_searchable, search_query: String::new(), @@ -458,13 +462,15 @@ impl ListSelectionView { act(&self.app_event_tx); } if item.dismiss_on_select { - self.complete = true; + self.completion = Some(ViewCompletion::Accepted); + } else if item.dismiss_parent_on_child_accept { + self.dismiss_after_child_accept = true; } } else if selected_item.is_none() { if let Some(cb) = &self.on_cancel { cb(&self.app_event_tx); } - self.complete = true; + self.completion = Some(ViewCompletion::Cancelled); } } @@ -672,7 +678,19 @@ impl BottomPaneView for ListSelectionView { } fn is_complete(&self) -> bool { - self.complete + self.completion.is_some() + } + + fn completion(&self) -> Option { + self.completion + } + + fn dismiss_after_child_accept(&self) -> bool { + self.dismiss_after_child_accept + } + + fn clear_dismiss_after_child_accept(&mut self) { + self.dismiss_after_child_accept = false; } fn view_id(&self) -> Option<&'static str> { @@ -687,7 +705,7 @@ impl BottomPaneView for ListSelectionView { if let Some(cb) = &self.on_cancel { cb(&self.app_event_tx); } - self.complete = true; + self.completion = Some(ViewCompletion::Cancelled); CancellationEvent::Handled } } diff --git a/codex-rs/tui/src/bottom_pane/mod.rs b/codex-rs/tui/src/bottom_pane/mod.rs index 91c8bd48eb..2600a90cd9 100644 --- a/codex-rs/tui/src/bottom_pane/mod.rs +++ b/codex-rs/tui/src/bottom_pane/mod.rs @@ -30,6 +30,7 @@ use crate::render::renderable::Renderable; use crate::render::renderable::RenderableItem; use crate::tui::FrameRequester; use bottom_pane_view::BottomPaneView; +use bottom_pane_view::ViewCompletion; use codex_features::Features; use codex_file_search::FileMatch; use codex_protocol::request_user_input::RequestUserInputEvent; @@ -380,8 +381,25 @@ impl BottomPane { self.request_redraw(); } - fn pop_active_view(&mut self) { + fn pop_active_view_with_completion(&mut self, completion: Option) { if self.view_stack.pop().is_some() { + match completion { + Some(ViewCompletion::Accepted) => { + while self + .view_stack + .last() + .is_some_and(|view| view.dismiss_after_child_accept()) + { + self.view_stack.pop(); + } + } + Some(ViewCompletion::Cancelled) => { + if let Some(view) = self.view_stack.last_mut() { + view.clear_dismiss_after_child_accept(); + } + } + None => {} + } self.on_view_stack_depth_decreased(); } } @@ -403,7 +421,7 @@ impl BottomPane { // We need three pieces of information after routing the key: // whether Esc completed the view, whether the view finished for any // reason, and whether a paste-burst timer should be scheduled. - let (ctrl_c_completed, view_complete, view_in_paste_burst) = { + let (ctrl_c_completed, view_complete, completion, view_in_paste_burst) = { let last_index = self.view_stack.len() - 1; let view = &mut self.view_stack[last_index]; let prefer_esc = @@ -413,22 +431,27 @@ impl BottomPane { && matches!(view.on_ctrl_c(), CancellationEvent::Handled) && view.is_complete(); if ctrl_c_completed { - (true, true, false) + (true, true, view.completion(), false) } else { view.handle_key_event(key_event); - (false, view.is_complete(), view.is_in_paste_burst()) + ( + false, + view.is_complete(), + view.completion(), + view.is_in_paste_burst(), + ) } }; if ctrl_c_completed { - self.pop_active_view(); + self.pop_active_view_with_completion(completion); if let Some(next_view) = self.view_stack.last() && next_view.is_in_paste_burst() { self.request_redraw_in(ChatComposer::recommended_paste_flush_delay()); } } else if view_complete { - self.pop_active_view(); + self.pop_active_view_with_completion(completion); } else if view_in_paste_burst { self.request_redraw_in(ChatComposer::recommended_paste_flush_delay()); } @@ -481,9 +504,10 @@ impl BottomPane { if let Some(view) = self.view_stack.last_mut() { let event = view.on_ctrl_c(); let view_complete = view.is_complete(); + let completion = view.completion(); if matches!(event, CancellationEvent::Handled) { if view_complete { - self.pop_active_view(); + self.pop_active_view_with_completion(completion); } self.show_quit_shortcut_hint(key_hint::ctrl(KeyCode::Char('c'))); self.request_redraw(); diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index de71efb9d8..890489bc2c 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -7953,6 +7953,7 @@ impl ChatWidget { is_default: preset.is_default, actions, dismiss_on_select: single_supported_effort, + dismiss_parent_on_child_accept: !single_supported_effort, ..Default::default() }); } @@ -10476,6 +10477,7 @@ impl ChatWidget { } })], dismiss_on_select: false, + dismiss_parent_on_child_accept: true, ..Default::default() }); @@ -10501,6 +10503,7 @@ impl ChatWidget { } })], dismiss_on_select: false, + dismiss_parent_on_child_accept: true, ..Default::default() }); @@ -10510,6 +10513,7 @@ impl ChatWidget { tx.send(AppEvent::OpenReviewCustomPrompt); })], dismiss_on_select: false, + dismiss_parent_on_child_accept: true, ..Default::default() }); From 6e72f0dbfd5e6b875e1ce00e8079e2bc15e28fe4 Mon Sep 17 00:00:00 2001 From: Tom Date: Thu, 16 Apr 2026 10:15:31 -0700 Subject: [PATCH 76/84] [codex] Add remote thread store implementation (#17826) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add a "remote" thread store implementation - Implement the remote thread store as a thin wrapper that makes grpc calls to a configurable service endpoint - Implement only the thread/list method to start - Encode the grpc method/param shape as protobufs in the remote implementation A wart: the proto generation script is an "example" binary target. This is an example target only because Cargo lets examples use dev-dependencies, which keeps tonic-prost-build out of the normal codex-thread-store dependency surface. A regular bin would either need to add proto generation deps as normal runtime deps, or use a feature-gated optional dep, which this repoโ€™s manifest checks explicitly reject. --- MODULE.bazel.lock | 4 + codex-rs/Cargo.lock | 70 ++- codex-rs/Cargo.toml | 2 + codex-rs/thread-store/Cargo.toml | 12 +- .../thread-store/examples/generate-proto.rs | 14 + .../thread-store/scripts/generate-proto.sh | 38 ++ codex-rs/thread-store/src/lib.rs | 2 + codex-rs/thread-store/src/remote/AGENTS.md | 13 + codex-rs/thread-store/src/remote/helpers.rs | 255 ++++++++++ .../thread-store/src/remote/list_threads.rs | 243 +++++++++ codex-rs/thread-store/src/remote/mod.rs | 112 +++++ .../remote/proto/codex.thread_store.v1.proto | 86 ++++ .../src/remote/proto/codex.thread_store.v1.rs | 460 ++++++++++++++++++ 13 files changed, 1307 insertions(+), 4 deletions(-) create mode 100644 codex-rs/thread-store/examples/generate-proto.rs create mode 100755 codex-rs/thread-store/scripts/generate-proto.sh create mode 100644 codex-rs/thread-store/src/remote/AGENTS.md create mode 100644 codex-rs/thread-store/src/remote/helpers.rs create mode 100644 codex-rs/thread-store/src/remote/list_threads.rs create mode 100644 codex-rs/thread-store/src/remote/mod.rs create mode 100644 codex-rs/thread-store/src/remote/proto/codex.thread_store.v1.proto create mode 100644 codex-rs/thread-store/src/remote/proto/codex.thread_store.v1.rs diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 5d7572f461..ef80693c70 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1203,9 +1203,11 @@ "process-wrap_9.0.1": "{\"dependencies\":[{\"name\":\"futures\",\"optional\":true,\"req\":\"^0.3.30\"},{\"name\":\"indexmap\",\"req\":\"^2.9.0\"},{\"default_features\":false,\"features\":[\"fs\",\"poll\",\"signal\"],\"name\":\"nix\",\"optional\":true,\"req\":\"^0.30.1\",\"target\":\"cfg(unix)\"},{\"kind\":\"dev\",\"name\":\"remoteprocess\",\"req\":\"^0.5.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.20.0\"},{\"features\":[\"io-util\",\"macros\",\"process\",\"rt\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.38.2\"},{\"features\":[\"io-util\",\"macros\",\"process\",\"rt\",\"rt-multi-thread\",\"time\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.38.2\"},{\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.40\"},{\"name\":\"windows\",\"optional\":true,\"req\":\"^0.62.2\",\"target\":\"cfg(windows)\"}],\"features\":{\"creation-flags\":[\"dep:windows\",\"windows/Win32_System_Threading\"],\"default\":[\"creation-flags\",\"job-object\",\"kill-on-drop\",\"process-group\",\"process-session\",\"tracing\"],\"job-object\":[\"dep:windows\",\"windows/Win32_Security\",\"windows/Win32_System_Diagnostics_ToolHelp\",\"windows/Win32_System_IO\",\"windows/Win32_System_JobObjects\",\"windows/Win32_System_Threading\"],\"kill-on-drop\":[],\"process-group\":[],\"process-session\":[\"process-group\"],\"reset-sigmask\":[],\"std\":[\"dep:nix\"],\"tokio1\":[\"dep:nix\",\"dep:futures\",\"dep:tokio\"],\"tracing\":[\"dep:tracing\"]}}", "proptest_1.9.0": "{\"dependencies\":[{\"name\":\"bit-set\",\"optional\":true,\"req\":\"^0.8.0\"},{\"name\":\"bit-vec\",\"optional\":true,\"req\":\"^0.8.0\"},{\"name\":\"bitflags\",\"req\":\"^2.9\"},{\"default_features\":false,\"name\":\"num-traits\",\"req\":\"^0.2.15\"},{\"name\":\"proptest-macro\",\"optional\":true,\"req\":\"^0.4.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"rand\",\"req\":\"^0.9\"},{\"default_features\":false,\"name\":\"rand_chacha\",\"req\":\"^0.9\"},{\"name\":\"rand_xorshift\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.0\"},{\"name\":\"regex-syntax\",\"optional\":true,\"req\":\"^0.8\"},{\"default_features\":false,\"name\":\"rusty-fork\",\"optional\":true,\"req\":\"^0.3.0\"},{\"name\":\"tempfile\",\"optional\":true,\"req\":\"^3.0\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.112\"},{\"name\":\"unarray\",\"req\":\"^0.1.4\"},{\"name\":\"x86\",\"optional\":true,\"req\":\"^0.52.0\"}],\"features\":{\"alloc\":[],\"atomic64bit\":[],\"attr-macro\":[\"proptest-macro\"],\"bit-set\":[\"dep:bit-set\",\"dep:bit-vec\"],\"default\":[\"std\",\"fork\",\"timeout\",\"bit-set\"],\"default-code-coverage\":[\"std\",\"fork\",\"timeout\",\"bit-set\"],\"fork\":[\"std\",\"rusty-fork\",\"tempfile\"],\"handle-panics\":[\"std\"],\"hardware-rng\":[\"x86\"],\"no_std\":[\"num-traits/libm\"],\"std\":[\"rand/std\",\"rand/os_rng\",\"regex-syntax\",\"num-traits/std\"],\"timeout\":[\"fork\",\"rusty-fork/timeout\"],\"unstable\":[]}}", "prost-build_0.12.6": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.10\"},{\"name\":\"heck\",\"req\":\">=0.4, <=0.5\"},{\"default_features\":false,\"features\":[\"use_alloc\"],\"name\":\"itertools\",\"req\":\">=0.10, <=0.12\"},{\"name\":\"log\",\"req\":\"^0.4.4\"},{\"default_features\":false,\"name\":\"multimap\",\"req\":\">=0.8, <=0.10\"},{\"name\":\"once_cell\",\"req\":\"^1.17.1\"},{\"default_features\":false,\"name\":\"petgraph\",\"req\":\"^0.6\"},{\"name\":\"prettyplease\",\"optional\":true,\"req\":\"^0.2\"},{\"default_features\":false,\"name\":\"prost\",\"req\":\"^0.12.6\"},{\"default_features\":false,\"name\":\"prost-types\",\"req\":\"^0.12.6\"},{\"default_features\":false,\"name\":\"pulldown-cmark\",\"optional\":true,\"req\":\"^0.9.1\"},{\"name\":\"pulldown-cmark-to-cmark\",\"optional\":true,\"req\":\"^10.0.1\"},{\"default_features\":false,\"features\":[\"std\",\"unicode-bool\"],\"name\":\"regex\",\"req\":\"^1.8.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"optional\":true,\"req\":\"^2\"},{\"name\":\"tempfile\",\"req\":\"^3\"}],\"features\":{\"cleanup-markdown\":[\"dep:pulldown-cmark\",\"dep:pulldown-cmark-to-cmark\"],\"default\":[\"format\"],\"format\":[\"dep:prettyplease\",\"dep:syn\"]}}", + "prost-build_0.14.3": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"heck\",\"req\":\">=0.4, <=0.5\"},{\"default_features\":false,\"features\":[\"use_alloc\"],\"name\":\"itertools\",\"req\":\">=0.10, <=0.14\"},{\"name\":\"log\",\"req\":\"^0.4.4\"},{\"default_features\":false,\"name\":\"multimap\",\"req\":\">=0.8, <=0.10\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"petgraph\",\"req\":\"^0.8\"},{\"name\":\"prettyplease\",\"optional\":true,\"req\":\"^0.2\"},{\"default_features\":false,\"name\":\"prost\",\"req\":\"^0.14.3\"},{\"default_features\":false,\"name\":\"prost-types\",\"req\":\"^0.14.3\"},{\"default_features\":false,\"name\":\"pulldown-cmark\",\"optional\":true,\"req\":\"^0.13\"},{\"name\":\"pulldown-cmark-to-cmark\",\"optional\":true,\"req\":\"^22\"},{\"default_features\":false,\"features\":[\"std\",\"unicode-bool\"],\"name\":\"regex\",\"req\":\"^1.8.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"optional\":true,\"req\":\"^2\"},{\"name\":\"tempfile\",\"req\":\"^3\"}],\"features\":{\"cleanup-markdown\":[\"dep:pulldown-cmark\",\"dep:pulldown-cmark-to-cmark\"],\"default\":[\"format\"],\"format\":[\"dep:prettyplease\",\"dep:syn\"]}}", "prost-derive_0.12.6": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.1\"},{\"default_features\":false,\"features\":[\"use_alloc\"],\"name\":\"itertools\",\"req\":\">=0.10, <=0.12\"},{\"name\":\"proc-macro2\",\"req\":\"^1\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2\"}],\"features\":{}}", "prost-derive_0.14.3": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.1\"},{\"name\":\"itertools\",\"req\":\">=0.10.1, <=0.14\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2\"}],\"features\":{}}", "prost-types_0.12.6": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"prost-derive\"],\"name\":\"prost\",\"req\":\"^0.12.6\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"prost/std\"]}}", + "prost-types_0.14.3": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.4\"},{\"default_features\":false,\"name\":\"chrono\",\"optional\":true,\"req\":\"^0.4.34\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"derive\"],\"name\":\"prost\",\"req\":\"^0.14.3\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"default\":[\"std\"],\"std\":[\"prost/std\"]}}", "prost_0.12.6": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"name\":\"prost-derive\",\"optional\":true,\"req\":\"^0.12.6\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"derive\",\"std\"],\"derive\":[\"dep:prost-derive\"],\"no-recursion-limit\":[],\"prost-derive\":[\"derive\"],\"std\":[]}}", "prost_0.14.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.7\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"name\":\"prost-derive\",\"optional\":true,\"req\":\"^0.14.3\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9\"}],\"features\":{\"default\":[\"derive\",\"std\"],\"derive\":[\"dep:prost-derive\"],\"no-recursion-limit\":[],\"std\":[]}}", "psl-types_2.0.11": "{\"dependencies\":[],\"features\":{}}", @@ -1459,6 +1461,8 @@ "toml_parser_1.0.6+spec-1.1.0": "{\"dependencies\":[{\"name\":\"anstream\",\"optional\":true,\"req\":\"^0.6.20\"},{\"features\":[\"test\"],\"kind\":\"dev\",\"name\":\"anstream\",\"req\":\"^0.6.20\"},{\"name\":\"anstyle\",\"optional\":true,\"req\":\"^1.0.11\"},{\"kind\":\"dev\",\"name\":\"snapbox\",\"req\":\"^0.6.21\"},{\"default_features\":false,\"name\":\"winnow\",\"req\":\"^0.7.13\"}],\"features\":{\"alloc\":[],\"debug\":[\"std\",\"dep:anstream\",\"dep:anstyle\"],\"default\":[\"std\"],\"simd\":[\"winnow/simd\"],\"std\":[\"alloc\"],\"unsafe\":[]}}", "toml_parser_1.0.9+spec-1.1.0": "{\"dependencies\":[{\"name\":\"anstream\",\"optional\":true,\"req\":\"^0.6.20\"},{\"features\":[\"test\"],\"kind\":\"dev\",\"name\":\"anstream\",\"req\":\"^0.6.20\"},{\"name\":\"anstyle\",\"optional\":true,\"req\":\"^1.0.11\"},{\"kind\":\"dev\",\"name\":\"snapbox\",\"req\":\"^0.6.21\"},{\"default_features\":false,\"name\":\"winnow\",\"req\":\"^0.7.13\"}],\"features\":{\"alloc\":[],\"debug\":[\"std\",\"dep:anstream\",\"dep:anstyle\"],\"default\":[\"std\"],\"simd\":[\"winnow/simd\"],\"std\":[\"alloc\"],\"unsafe\":[]}}", "toml_writer_1.0.6+spec-1.1.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.7.0\"},{\"kind\":\"dev\",\"name\":\"snapbox\",\"req\":\"^0.6.21\"},{\"kind\":\"dev\",\"name\":\"toml_old\",\"package\":\"toml\",\"req\":\"^0.5.11\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", + "tonic-build_0.14.3": "{\"dependencies\":[{\"name\":\"prettyplease\",\"req\":\"^0.2\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"name\":\"quote\",\"req\":\"^1.0\"},{\"name\":\"syn\",\"req\":\"^2.0\"}],\"features\":{\"default\":[\"transport\"],\"transport\":[]}}", + "tonic-prost-build_0.14.3": "{\"dependencies\":[{\"name\":\"prettyplease\",\"req\":\"^0.2\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"name\":\"prost-build\",\"req\":\"^0.14\"},{\"name\":\"prost-types\",\"req\":\"^0.14\"},{\"name\":\"quote\",\"req\":\"^1.0\"},{\"name\":\"syn\",\"req\":\"^2.0\"},{\"name\":\"tempfile\",\"req\":\"^3.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"tonic\",\"req\":\"^0.14.0\"},{\"default_features\":false,\"name\":\"tonic-build\",\"req\":\"^0.14.0\"}],\"features\":{\"cleanup-markdown\":[\"prost-build/cleanup-markdown\"],\"default\":[\"transport\",\"cleanup-markdown\"],\"transport\":[\"tonic-build/transport\"]}}", "tonic-prost_0.14.3": "{\"dependencies\":[{\"name\":\"bytes\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"http-body\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"http-body-util\",\"req\":\"^0.1\"},{\"name\":\"prost\",\"req\":\"^0.14\"},{\"features\":[\"macros\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"default_features\":false,\"name\":\"tonic\",\"req\":\"^0.14.0\"}],\"features\":{}}", "tonic_0.14.3": "{\"dependencies\":[{\"name\":\"async-trait\",\"optional\":true,\"req\":\"^0.1.13\"},{\"default_features\":false,\"name\":\"axum\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"base64\",\"req\":\"^0.22\"},{\"kind\":\"dev\",\"name\":\"bencher\",\"req\":\"^0.1.5\"},{\"name\":\"bytes\",\"req\":\"^1.0\"},{\"name\":\"flate2\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"h2\",\"optional\":true,\"req\":\"^0.4\"},{\"name\":\"http\",\"req\":\"^1.1.0\"},{\"name\":\"http-body\",\"req\":\"^1\"},{\"name\":\"http-body-util\",\"req\":\"^0.1\"},{\"features\":[\"http1\",\"http2\"],\"name\":\"hyper\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"hyper-timeout\",\"optional\":true,\"req\":\"^0.5\"},{\"features\":[\"tokio\"],\"name\":\"hyper-util\",\"optional\":true,\"req\":\"^0.1.11\"},{\"name\":\"percent-encoding\",\"req\":\"^2.1\"},{\"name\":\"pin-project\",\"req\":\"^1.0.11\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1.0\"},{\"name\":\"rustls-native-certs\",\"optional\":true,\"req\":\"^0.8\"},{\"features\":[\"all\"],\"name\":\"socket2\",\"optional\":true,\"req\":\"^0.6\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.0\"},{\"name\":\"sync_wrapper\",\"req\":\"^1.0.2\"},{\"default_features\":false,\"name\":\"tokio\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"rt-multi-thread\",\"macros\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"logging\",\"tls12\"],\"name\":\"tokio-rustls\",\"optional\":true,\"req\":\"^0.26.1\"},{\"default_features\":false,\"name\":\"tokio-stream\",\"req\":\"^0.1.16\"},{\"default_features\":false,\"name\":\"tower\",\"optional\":true,\"req\":\"^0.5\"},{\"features\":[\"load-shed\",\"timeout\"],\"kind\":\"dev\",\"name\":\"tower\",\"req\":\"^0.5\"},{\"name\":\"tower-layer\",\"req\":\"^0.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"name\":\"tracing\",\"req\":\"^0.1\"},{\"name\":\"webpki-roots\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"zstd\",\"optional\":true,\"req\":\"^0.13.0\"}],\"features\":{\"_tls-any\":[\"dep:tokio\",\"tokio?/rt\",\"tokio?/macros\",\"tls-connect-info\"],\"channel\":[\"dep:hyper\",\"hyper?/client\",\"dep:hyper-util\",\"hyper-util?/client-legacy\",\"dep:tower\",\"tower?/balance\",\"tower?/buffer\",\"tower?/discover\",\"tower?/limit\",\"tower?/load-shed\",\"tower?/util\",\"dep:tokio\",\"tokio?/time\",\"dep:hyper-timeout\"],\"codegen\":[\"dep:async-trait\"],\"default\":[\"router\",\"transport\",\"codegen\"],\"deflate\":[\"dep:flate2\"],\"gzip\":[\"dep:flate2\"],\"router\":[\"dep:axum\",\"dep:tower\",\"tower?/util\"],\"server\":[\"dep:h2\",\"dep:hyper\",\"hyper?/server\",\"dep:hyper-util\",\"hyper-util?/service\",\"hyper-util?/server-auto\",\"dep:socket2\",\"dep:tokio\",\"tokio?/macros\",\"tokio?/net\",\"tokio?/time\",\"tokio-stream/net\",\"dep:tower\",\"tower?/util\",\"tower?/limit\",\"tower?/load-shed\"],\"tls-aws-lc\":[\"_tls-any\",\"tokio-rustls/aws-lc-rs\"],\"tls-connect-info\":[\"dep:tokio-rustls\"],\"tls-native-roots\":[\"_tls-any\",\"channel\",\"dep:rustls-native-certs\"],\"tls-ring\":[\"_tls-any\",\"tokio-rustls/ring\"],\"tls-webpki-roots\":[\"_tls-any\",\"channel\",\"dep:webpki-roots\"],\"transport\":[\"server\",\"channel\"],\"zstd\":[\"dep:zstd\"]}}", "tower-http_0.6.8": "{\"dependencies\":[{\"features\":[\"tokio\"],\"name\":\"async-compression\",\"optional\":true,\"req\":\"^0.4\"},{\"name\":\"base64\",\"optional\":true,\"req\":\"^0.22\"},{\"name\":\"bitflags\",\"req\":\"^2.0.2\"},{\"kind\":\"dev\",\"name\":\"brotli\",\"req\":\"^8\"},{\"name\":\"bytes\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"bytes\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.14\"},{\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3.14\"},{\"name\":\"http\",\"req\":\"^1.0\"},{\"name\":\"http-body\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"http-body\",\"req\":\"^1.0.0\"},{\"name\":\"http-body-util\",\"optional\":true,\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"http-body-util\",\"req\":\"^0.1.0\"},{\"name\":\"http-range-header\",\"optional\":true,\"req\":\"^0.4.0\"},{\"name\":\"httpdate\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"client-legacy\",\"http1\",\"tokio\"],\"kind\":\"dev\",\"name\":\"hyper-util\",\"req\":\"^0.1\"},{\"name\":\"iri-string\",\"optional\":true,\"req\":\"^0.7.0\"},{\"default_features\":false,\"name\":\"mime\",\"optional\":true,\"req\":\"^0.3.17\"},{\"default_features\":false,\"name\":\"mime_guess\",\"optional\":true,\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1\"},{\"name\":\"percent-encoding\",\"optional\":true,\"req\":\"^2.1.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"sync_wrapper\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.6\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"io\"],\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7\"},{\"name\":\"tower\",\"optional\":true,\"req\":\"^0.5\"},{\"features\":[\"buffer\",\"util\",\"retry\",\"make\",\"timeout\"],\"kind\":\"dev\",\"name\":\"tower\",\"req\":\"^0.5\"},{\"name\":\"tower-layer\",\"req\":\"^0.3.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"},{\"features\":[\"v4\"],\"name\":\"uuid\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"zstd\",\"req\":\"^0.13\"}],\"features\":{\"add-extension\":[],\"auth\":[\"base64\",\"validate-request\"],\"catch-panic\":[\"tracing\",\"futures-util/std\",\"dep:http-body\",\"dep:http-body-util\"],\"compression-br\":[\"async-compression/brotli\",\"futures-core\",\"dep:http-body\",\"tokio-util\",\"tokio\"],\"compression-deflate\":[\"async-compression/zlib\",\"futures-core\",\"dep:http-body\",\"tokio-util\",\"tokio\"],\"compression-full\":[\"compression-br\",\"compression-deflate\",\"compression-gzip\",\"compression-zstd\"],\"compression-gzip\":[\"async-compression/gzip\",\"futures-core\",\"dep:http-body\",\"tokio-util\",\"tokio\"],\"compression-zstd\":[\"async-compression/zstd\",\"futures-core\",\"dep:http-body\",\"tokio-util\",\"tokio\"],\"cors\":[],\"decompression-br\":[\"async-compression/brotli\",\"futures-core\",\"dep:http-body\",\"dep:http-body-util\",\"tokio-util\",\"tokio\"],\"decompression-deflate\":[\"async-compression/zlib\",\"futures-core\",\"dep:http-body\",\"dep:http-body-util\",\"tokio-util\",\"tokio\"],\"decompression-full\":[\"decompression-br\",\"decompression-deflate\",\"decompression-gzip\",\"decompression-zstd\"],\"decompression-gzip\":[\"async-compression/gzip\",\"futures-core\",\"dep:http-body\",\"dep:http-body-util\",\"tokio-util\",\"tokio\"],\"decompression-zstd\":[\"async-compression/zstd\",\"futures-core\",\"dep:http-body\",\"dep:http-body-util\",\"tokio-util\",\"tokio\"],\"default\":[],\"follow-redirect\":[\"futures-util\",\"dep:http-body\",\"iri-string\",\"tower/util\"],\"fs\":[\"futures-core\",\"futures-util\",\"dep:http-body\",\"dep:http-body-util\",\"tokio/fs\",\"tokio-util/io\",\"tokio/io-util\",\"dep:http-range-header\",\"mime_guess\",\"mime\",\"percent-encoding\",\"httpdate\",\"set-status\",\"futures-util/alloc\",\"tracing\"],\"full\":[\"add-extension\",\"auth\",\"catch-panic\",\"compression-full\",\"cors\",\"decompression-full\",\"follow-redirect\",\"fs\",\"limit\",\"map-request-body\",\"map-response-body\",\"metrics\",\"normalize-path\",\"propagate-header\",\"redirect\",\"request-id\",\"sensitive-headers\",\"set-header\",\"set-status\",\"timeout\",\"trace\",\"util\",\"validate-request\"],\"limit\":[\"dep:http-body\",\"dep:http-body-util\"],\"map-request-body\":[],\"map-response-body\":[],\"metrics\":[\"dep:http-body\",\"tokio/time\"],\"normalize-path\":[],\"propagate-header\":[],\"redirect\":[],\"request-id\":[\"uuid\"],\"sensitive-headers\":[],\"set-header\":[],\"set-status\":[],\"timeout\":[\"dep:http-body\",\"tokio/time\"],\"trace\":[\"dep:http-body\",\"tracing\"],\"util\":[\"tower\"],\"validate-request\":[\"mime\"]}}", diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 1be5b45b6a..3e21398f85 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2886,11 +2886,16 @@ dependencies = [ "codex-rollout", "codex-state", "pretty_assertions", + "prost 0.14.3", "serde", "serde_json", "tempfile", "thiserror 2.0.18", "tokio", + "tokio-stream", + "tonic", + "tonic-prost", + "tonic-prost-build", "uuid", ] @@ -7561,7 +7566,7 @@ dependencies = [ "heck 0.4.1", "itertools 0.11.0", "prost 0.12.6", - "prost-types", + "prost-types 0.12.6", ] [[package]] @@ -7575,7 +7580,7 @@ dependencies = [ "pbjson", "pbjson-build", "prost 0.12.6", - "prost-build", + "prost-build 0.12.6", "serde", ] @@ -7999,7 +8004,26 @@ dependencies = [ "petgraph 0.6.5", "prettyplease", "prost 0.12.6", - "prost-types", + "prost-types 0.12.6", + "regex", + "syn 2.0.114", + "tempfile", +] + +[[package]] +name = "prost-build" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" +dependencies = [ + "heck 0.5.0", + "itertools 0.14.0", + "log", + "multimap", + "petgraph 0.8.3", + "prettyplease", + "prost 0.14.3", + "prost-types 0.14.3", "regex", "syn 2.0.114", "tempfile", @@ -8040,6 +8064,15 @@ dependencies = [ "prost 0.12.6", ] +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost 0.14.3", +] + [[package]] name = "psl" version = "2.1.184" @@ -10948,8 +10981,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a286e33f82f8a1ee2df63f4fa35c0becf4a85a0cb03091a15fd7bf0b402dc94a" dependencies = [ "async-trait", + "axum", "base64 0.22.1", "bytes", + "h2", "http 1.4.0", "http-body", "http-body-util", @@ -10959,6 +10994,7 @@ dependencies = [ "percent-encoding", "pin-project", "rustls-native-certs", + "socket2 0.6.2", "sync_wrapper", "tokio", "tokio-rustls", @@ -10969,6 +11005,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "tonic-build" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27aac809edf60b741e2d7db6367214d078856b8a5bff0087e94ff330fb97b6fc" +dependencies = [ + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "tonic-prost" version = "0.14.3" @@ -10980,6 +11028,22 @@ dependencies = [ "tonic", ] +[[package]] +name = "tonic-prost-build" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4556786613791cfef4ed134aa670b61a85cfcacf71543ef33e8d801abae988f" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build 0.14.3", + "prost-types 0.14.3", + "quote", + "syn 2.0.114", + "tempfile", + "tonic-build", +] + [[package]] name = "tower" version = "0.5.3" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index e7867304e9..deb3738812 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -347,6 +347,8 @@ tracing-appender = "0.2.3" tracing-opentelemetry = "0.32.0" tracing-subscriber = "0.3.22" tracing-test = "0.2.5" +tonic = { version = "0.14.3", default-features = false, features = ["channel", "codegen"] } +tonic-prost = "0.14.3" tree-sitter = "0.25.10" tree-sitter-bash = "0.25" ts-rs = "11" diff --git a/codex-rs/thread-store/Cargo.toml b/codex-rs/thread-store/Cargo.toml index 328d9a4a7f..3698332b92 100644 --- a/codex-rs/thread-store/Cargo.toml +++ b/codex-rs/thread-store/Cargo.toml @@ -8,6 +8,10 @@ version.workspace = true name = "codex_thread_store" path = "src/lib.rs" +[[example]] +name = "generate-proto" +path = "examples/generate-proto.rs" + [lints] workspace = true @@ -17,13 +21,19 @@ chrono = { workspace = true, features = ["serde"] } codex-git-utils = { workspace = true } codex-protocol = { workspace = true } codex-rollout = { workspace = true } +prost = "0.14.3" serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } thiserror = { workspace = true } +tonic = { workspace = true } +tonic-prost = { workspace = true } [dev-dependencies] codex-state = { workspace = true } pretty_assertions = { workspace = true } tempfile = { workspace = true } -tokio = { workspace = true, features = ["macros"] } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +tokio-stream = { workspace = true, features = ["net"] } +tonic = { workspace = true, features = ["router", "transport"] } +tonic-prost-build = { version = "=0.14.3", default-features = false, features = ["transport"] } uuid = { workspace = true } diff --git a/codex-rs/thread-store/examples/generate-proto.rs b/codex-rs/thread-store/examples/generate-proto.rs new file mode 100644 index 0000000000..aa159f5bcb --- /dev/null +++ b/codex-rs/thread-store/examples/generate-proto.rs @@ -0,0 +1,14 @@ +use std::path::PathBuf; + +fn main() -> Result<(), Box> { + let proto_dir = PathBuf::from(std::env::args().nth(1).expect("proto dir")); + let proto_file = proto_dir.join("codex.thread_store.v1.proto"); + + tonic_prost_build::configure() + .build_client(true) + .build_server(true) + .out_dir(&proto_dir) + .compile_protos(&[proto_file], &[proto_dir])?; + + Ok(()) +} diff --git a/codex-rs/thread-store/scripts/generate-proto.sh b/codex-rs/thread-store/scripts/generate-proto.sh new file mode 100755 index 0000000000..4045467cac --- /dev/null +++ b/codex-rs/thread-store/scripts/generate-proto.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd "$script_dir/../../.." && pwd)" +proto_dir="$repo_root/codex-rs/thread-store/src/remote/proto" +generated="$proto_dir/codex.thread_store.v1.rs" +tmpdir="$(mktemp -d)" + +cleanup() { + rm -rf "$tmpdir" +} +trap cleanup EXIT + +( + cd "$repo_root/codex-rs" + CARGO_TARGET_DIR="$tmpdir/target" cargo run \ + -p codex-thread-store \ + --example generate-proto \ + -- "$proto_dir" +) + +if ! sed -n '2p' "$generated" | grep -q 'clippy::trivially_copy_pass_by_ref'; then + { + sed -n '1p' "$generated" + printf '#![allow(clippy::trivially_copy_pass_by_ref)]\n' + sed '1d' "$generated" + } > "$tmpdir/generated.rs" + mv "$tmpdir/generated.rs" "$generated" +fi + +rustfmt --edition 2024 "$generated" + +awk ' + NR == 3 && previous ~ /clippy::trivially_copy_pass_by_ref/ && $0 != "" { print "" } + { print; previous = $0 } +' "$generated" > "$tmpdir/formatted.rs" +mv "$tmpdir/formatted.rs" "$generated" diff --git a/codex-rs/thread-store/src/lib.rs b/codex-rs/thread-store/src/lib.rs index a64bf023be..f8dc94908a 100644 --- a/codex-rs/thread-store/src/lib.rs +++ b/codex-rs/thread-store/src/lib.rs @@ -7,6 +7,7 @@ mod error; mod local; mod recorder; +mod remote; mod store; mod types; @@ -14,6 +15,7 @@ pub use error::ThreadStoreError; pub use error::ThreadStoreResult; pub use local::LocalThreadStore; pub use recorder::ThreadRecorder; +pub use remote::RemoteThreadStore; pub use store::ThreadStore; pub use types::AppendThreadItemsParams; pub use types::ArchiveThreadParams; diff --git a/codex-rs/thread-store/src/remote/AGENTS.md b/codex-rs/thread-store/src/remote/AGENTS.md new file mode 100644 index 0000000000..b2b2b6417b --- /dev/null +++ b/codex-rs/thread-store/src/remote/AGENTS.md @@ -0,0 +1,13 @@ +# Remote Thread Store + +- The Rust protobuf output in `proto/codex.thread_store.v1.rs` is checked in. +- Do not add build-time protobuf generation to `codex-thread-store` unless the Bazel/Cargo story is intentionally changed. +- When `proto/codex.thread_store.v1.proto` changes, regenerate the Rust file manually and include both files in the same commit. + +Run this from the repository root: + +```sh +./codex-rs/thread-store/scripts/generate-proto.sh +``` + +The command requires `protoc` to be available on `PATH`. diff --git a/codex-rs/thread-store/src/remote/helpers.rs b/codex-rs/thread-store/src/remote/helpers.rs new file mode 100644 index 0000000000..c02889e168 --- /dev/null +++ b/codex-rs/thread-store/src/remote/helpers.rs @@ -0,0 +1,255 @@ +use std::path::PathBuf; +use std::str::FromStr; + +use chrono::DateTime; +use chrono::Utc; +use codex_git_utils::GitSha; +use codex_protocol::AgentPath; +use codex_protocol::ThreadId; +use codex_protocol::openai_models::ReasoningEffort; +use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::GitInfo; +use codex_protocol::protocol::SandboxPolicy; +use codex_protocol::protocol::SessionSource; +use codex_protocol::protocol::SubAgentSource; + +use super::proto; +use crate::StoredThread; +use crate::ThreadSortKey; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; + +pub(super) fn remote_status_to_error(status: tonic::Status) -> ThreadStoreError { + match status.code() { + tonic::Code::InvalidArgument => ThreadStoreError::InvalidRequest { + message: status.message().to_string(), + }, + tonic::Code::AlreadyExists | tonic::Code::FailedPrecondition | tonic::Code::Aborted => { + ThreadStoreError::Conflict { + message: status.message().to_string(), + } + } + _ => ThreadStoreError::Internal { + message: format!("remote thread store request failed: {status}"), + }, + } +} + +pub(super) fn proto_sort_key(sort_key: ThreadSortKey) -> proto::ThreadSortKey { + match sort_key { + ThreadSortKey::CreatedAt => proto::ThreadSortKey::CreatedAt, + ThreadSortKey::UpdatedAt => proto::ThreadSortKey::UpdatedAt, + } +} + +pub(super) fn proto_session_source(source: &SessionSource) -> proto::SessionSource { + match source { + SessionSource::Cli => proto_source(proto::SessionSourceKind::Cli), + SessionSource::VSCode => proto_source(proto::SessionSourceKind::Vscode), + SessionSource::Exec => proto_source(proto::SessionSourceKind::Exec), + SessionSource::Mcp => proto_source(proto::SessionSourceKind::AppServer), + SessionSource::Custom(custom) => proto::SessionSource { + kind: proto::SessionSourceKind::Custom.into(), + custom: Some(custom.clone()), + ..Default::default() + }, + SessionSource::SubAgent(SubAgentSource::Review) => { + proto_source(proto::SessionSourceKind::SubAgentReview) + } + SessionSource::SubAgent(SubAgentSource::Compact) => { + proto_source(proto::SessionSourceKind::SubAgentCompact) + } + SessionSource::SubAgent(SubAgentSource::ThreadSpawn { + parent_thread_id, + depth, + agent_path, + agent_nickname, + agent_role, + }) => proto::SessionSource { + kind: proto::SessionSourceKind::SubAgentThreadSpawn.into(), + sub_agent_parent_thread_id: Some(parent_thread_id.to_string()), + sub_agent_depth: Some(*depth), + sub_agent_path: agent_path.as_ref().map(|path| path.as_str().to_string()), + sub_agent_nickname: agent_nickname.clone(), + sub_agent_role: agent_role.clone(), + ..Default::default() + }, + SessionSource::SubAgent(SubAgentSource::MemoryConsolidation) => { + proto_source(proto::SessionSourceKind::SubAgentMemoryConsolidation) + } + SessionSource::SubAgent(SubAgentSource::Other(other)) => proto::SessionSource { + kind: proto::SessionSourceKind::SubAgentOther.into(), + sub_agent_other: Some(other.clone()), + ..Default::default() + }, + SessionSource::Unknown => proto_source(proto::SessionSourceKind::Unknown), + } +} + +fn proto_source(kind: proto::SessionSourceKind) -> proto::SessionSource { + proto::SessionSource { + kind: kind.into(), + ..Default::default() + } +} + +pub(super) fn stored_thread_from_proto( + thread: proto::StoredThread, +) -> ThreadStoreResult { + // Keep this mapping boring: the proto mirrors StoredThread for remote-readable + // summary fields, except for Rust domain types that cross gRPC as stable scalar + // values. Local-only fields such as rollout_path intentionally stay local. + let source = thread + .source + .as_ref() + .map(session_source_from_proto) + .transpose()? + .unwrap_or(SessionSource::Unknown); + let thread_id = ThreadId::from_string(&thread.thread_id).map_err(|err| { + ThreadStoreError::InvalidRequest { + message: format!("remote thread store returned invalid thread_id: {err}"), + } + })?; + let forked_from_id = thread + .forked_from_id + .as_deref() + .map(ThreadId::from_string) + .transpose() + .map_err(|err| ThreadStoreError::InvalidRequest { + message: format!("remote thread store returned invalid forked_from_id: {err}"), + })?; + + Ok(StoredThread { + thread_id, + rollout_path: None, + forked_from_id, + preview: thread.preview, + name: thread.name, + model_provider: thread.model_provider, + model: thread.model, + reasoning_effort: thread + .reasoning_effort + .as_deref() + .map(parse_reasoning_effort) + .transpose()?, + created_at: datetime_from_unix(thread.created_at)?, + updated_at: datetime_from_unix(thread.updated_at)?, + archived_at: thread.archived_at.map(datetime_from_unix).transpose()?, + cwd: PathBuf::from(thread.cwd), + cli_version: thread.cli_version, + source, + agent_nickname: thread.agent_nickname, + agent_role: thread.agent_role, + agent_path: thread.agent_path, + git_info: thread.git_info.map(git_info_from_proto), + approval_mode: AskForApproval::OnRequest, + sandbox_policy: SandboxPolicy::new_read_only_policy(), + token_usage: None, + first_user_message: thread.first_user_message, + history: None, + }) +} + +#[cfg(test)] +pub(super) fn stored_thread_to_proto(thread: StoredThread) -> proto::StoredThread { + proto::StoredThread { + thread_id: thread.thread_id.to_string(), + forked_from_id: thread.forked_from_id.map(|thread_id| thread_id.to_string()), + preview: thread.preview, + name: thread.name, + model_provider: thread.model_provider, + model: thread.model, + created_at: thread.created_at.timestamp(), + updated_at: thread.updated_at.timestamp(), + archived_at: thread.archived_at.map(|timestamp| timestamp.timestamp()), + cwd: thread.cwd.to_string_lossy().into_owned(), + cli_version: thread.cli_version, + source: Some(proto_session_source(&thread.source)), + git_info: thread.git_info.map(git_info_to_proto), + agent_nickname: thread.agent_nickname, + agent_role: thread.agent_role, + agent_path: thread.agent_path, + reasoning_effort: thread.reasoning_effort.map(|effort| effort.to_string()), + first_user_message: thread.first_user_message, + } +} + +fn datetime_from_unix(timestamp: i64) -> ThreadStoreResult> { + DateTime::from_timestamp(timestamp, 0).ok_or_else(|| ThreadStoreError::InvalidRequest { + message: format!("remote thread store returned invalid timestamp: {timestamp}"), + }) +} + +fn session_source_from_proto(source: &proto::SessionSource) -> ThreadStoreResult { + let kind = proto::SessionSourceKind::try_from(source.kind).unwrap_or_default(); + Ok(match kind { + proto::SessionSourceKind::Unknown => SessionSource::Unknown, + proto::SessionSourceKind::Cli => SessionSource::Cli, + proto::SessionSourceKind::Vscode => SessionSource::VSCode, + proto::SessionSourceKind::Exec => SessionSource::Exec, + proto::SessionSourceKind::AppServer => SessionSource::Mcp, + proto::SessionSourceKind::Custom => { + SessionSource::Custom(source.custom.clone().unwrap_or_default()) + } + proto::SessionSourceKind::SubAgentReview => SessionSource::SubAgent(SubAgentSource::Review), + proto::SessionSourceKind::SubAgentCompact => { + SessionSource::SubAgent(SubAgentSource::Compact) + } + proto::SessionSourceKind::SubAgentThreadSpawn => { + let parent_thread_id = source + .sub_agent_parent_thread_id + .as_deref() + .map(ThreadId::from_string) + .transpose() + .map_err(|err| ThreadStoreError::InvalidRequest { + message: format!( + "remote thread store returned invalid sub-agent parent thread id: {err}" + ), + })? + .ok_or_else(|| ThreadStoreError::InvalidRequest { + message: "remote thread store omitted sub-agent parent thread id".to_string(), + })?; + SessionSource::SubAgent(SubAgentSource::ThreadSpawn { + parent_thread_id, + depth: source.sub_agent_depth.unwrap_or_default(), + agent_path: source + .sub_agent_path + .clone() + .map(AgentPath::from_string) + .transpose() + .map_err(|message| ThreadStoreError::InvalidRequest { message })?, + agent_nickname: source.sub_agent_nickname.clone(), + agent_role: source.sub_agent_role.clone(), + }) + } + proto::SessionSourceKind::SubAgentMemoryConsolidation => { + SessionSource::SubAgent(SubAgentSource::MemoryConsolidation) + } + proto::SessionSourceKind::SubAgentOther => SessionSource::SubAgent(SubAgentSource::Other( + source.sub_agent_other.clone().unwrap_or_default(), + )), + }) +} + +fn git_info_from_proto(info: proto::GitInfo) -> GitInfo { + GitInfo { + commit_hash: info.sha.as_deref().map(GitSha::new), + branch: info.branch, + repository_url: info.origin_url, + } +} + +#[cfg(test)] +fn git_info_to_proto(info: GitInfo) -> proto::GitInfo { + proto::GitInfo { + sha: info.commit_hash.map(|sha| sha.0), + branch: info.branch, + origin_url: info.repository_url, + } +} + +fn parse_reasoning_effort(value: &str) -> ThreadStoreResult { + ReasoningEffort::from_str(value).map_err(|message| ThreadStoreError::InvalidRequest { + message: format!("remote thread store returned {message}"), + }) +} diff --git a/codex-rs/thread-store/src/remote/list_threads.rs b/codex-rs/thread-store/src/remote/list_threads.rs new file mode 100644 index 0000000000..b94db9dc82 --- /dev/null +++ b/codex-rs/thread-store/src/remote/list_threads.rs @@ -0,0 +1,243 @@ +use super::RemoteThreadStore; +use super::helpers::proto_session_source; +use super::helpers::proto_sort_key; +use super::helpers::remote_status_to_error; +use super::helpers::stored_thread_from_proto; +use super::proto; +use crate::ListThreadsParams; +use crate::ThreadPage; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; + +pub(super) async fn list_threads( + store: &RemoteThreadStore, + params: ListThreadsParams, +) -> ThreadStoreResult { + let request = proto::ListThreadsRequest { + page_size: params + .page_size + .try_into() + .map_err(|_| ThreadStoreError::InvalidRequest { + message: format!("page_size is too large: {}", params.page_size), + })?, + cursor: params.cursor, + sort_key: proto_sort_key(params.sort_key).into(), + allowed_sources: params + .allowed_sources + .iter() + .map(proto_session_source) + .collect(), + model_provider_filter: params + .model_providers + .map(|values| proto::ModelProviderFilter { values }), + archived: params.archived, + search_term: params.search_term, + }; + + let response = store + .client() + .await? + .list_threads(request) + .await + .map_err(remote_status_to_error)? + .into_inner(); + + let items = response + .threads + .into_iter() + .map(stored_thread_from_proto) + .collect::>>()?; + + Ok(ThreadPage { + items, + next_cursor: response.next_cursor, + }) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use codex_protocol::openai_models::ReasoningEffort; + use codex_protocol::protocol::SessionSource; + use pretty_assertions::assert_eq; + use tonic::Request; + use tonic::Response; + use tonic::Status; + use tonic::transport::Server; + + use super::super::helpers::stored_thread_to_proto; + use super::super::proto::thread_store_server; + use super::super::proto::thread_store_server::ThreadStoreServer; + use super::*; + use crate::ThreadSortKey; + use crate::ThreadStore; + + #[derive(Default)] + struct TestServer; + + #[tonic::async_trait] + impl thread_store_server::ThreadStore for TestServer { + async fn list_threads( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + assert_eq!(request.page_size, 2); + assert_eq!(request.cursor.as_deref(), Some("cursor-1")); + assert_eq!( + proto::ThreadSortKey::try_from(request.sort_key), + Ok(proto::ThreadSortKey::UpdatedAt) + ); + assert_eq!(request.archived, true); + assert_eq!(request.search_term.as_deref(), Some("needle")); + assert_eq!( + request.model_provider_filter, + Some(proto::ModelProviderFilter { + values: vec!["openai".to_string()], + }) + ); + assert_eq!(request.allowed_sources.len(), 1); + assert_eq!( + proto::SessionSourceKind::try_from(request.allowed_sources[0].kind), + Ok(proto::SessionSourceKind::Cli) + ); + + Ok(Response::new(proto::ListThreadsResponse { + threads: vec![proto::StoredThread { + thread_id: "11111111-1111-1111-1111-111111111111".to_string(), + forked_from_id: None, + preview: "hello".to_string(), + name: Some("named thread".to_string()), + model_provider: "openai".to_string(), + model: Some("gpt-5".to_string()), + created_at: 100, + updated_at: 200, + archived_at: Some(300), + cwd: "/workspace".to_string(), + cli_version: "1.2.3".to_string(), + source: Some(proto::SessionSource { + kind: proto::SessionSourceKind::Cli.into(), + ..Default::default() + }), + git_info: Some(proto::GitInfo { + sha: Some("abc123".to_string()), + branch: Some("main".to_string()), + origin_url: Some("https://example.test/repo.git".to_string()), + }), + agent_nickname: None, + agent_role: None, + agent_path: None, + reasoning_effort: Some("medium".to_string()), + first_user_message: Some("hello".to_string()), + }], + next_cursor: Some("cursor-2".to_string()), + })) + } + } + + #[tokio::test] + async fn list_threads_calls_remote_service() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind test server"); + let addr = listener.local_addr().expect("test server addr"); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + Server::builder() + .add_service(ThreadStoreServer::new(TestServer)) + .serve_with_incoming_shutdown( + tokio_stream::wrappers::TcpListenerStream::new(listener), + async { + let _ = shutdown_rx.await; + }, + ) + .await + }); + + let store = RemoteThreadStore::new(format!("http://{addr}")); + let page = store + .list_threads(ListThreadsParams { + page_size: 2, + cursor: Some("cursor-1".to_string()), + sort_key: ThreadSortKey::UpdatedAt, + allowed_sources: vec![SessionSource::Cli], + model_providers: Some(vec!["openai".to_string()]), + archived: true, + search_term: Some("needle".to_string()), + }) + .await + .expect("list threads"); + + assert_eq!(page.next_cursor.as_deref(), Some("cursor-2")); + assert_eq!(page.items.len(), 1); + let item = &page.items[0]; + assert_eq!( + item.thread_id.to_string(), + "11111111-1111-1111-1111-111111111111" + ); + assert_eq!(item.name.as_deref(), Some("named thread")); + assert_eq!(item.preview, "hello"); + assert_eq!(item.first_user_message.as_deref(), Some("hello")); + assert_eq!(item.model_provider, "openai"); + assert_eq!(item.model.as_deref(), Some("gpt-5")); + assert_eq!(item.created_at.timestamp(), 100); + assert_eq!(item.updated_at.timestamp(), 200); + assert_eq!(item.archived_at.map(|ts| ts.timestamp()), Some(300)); + assert_eq!(item.cwd, PathBuf::from("/workspace")); + assert_eq!(item.cli_version, "1.2.3"); + assert_eq!(item.source, SessionSource::Cli); + assert_eq!(item.reasoning_effort, Some(ReasoningEffort::Medium)); + assert_eq!( + item.git_info.as_ref().and_then(|git| git.branch.as_deref()), + Some("main") + ); + + let _ = shutdown_tx.send(()); + server.await.expect("join server").expect("server"); + } + + #[test] + fn stored_thread_proto_roundtrips_through_domain_type() { + let thread = proto::StoredThread { + thread_id: "11111111-1111-1111-1111-111111111111".to_string(), + forked_from_id: Some("22222222-2222-2222-2222-222222222222".to_string()), + preview: "preview text".to_string(), + name: Some("named thread".to_string()), + model_provider: "openai".to_string(), + model: Some("gpt-5".to_string()), + created_at: 100, + updated_at: 200, + archived_at: Some(300), + cwd: "/workspace/project".to_string(), + cli_version: "1.2.3".to_string(), + source: Some(proto::SessionSource { + kind: proto::SessionSourceKind::SubAgentThreadSpawn.into(), + sub_agent_parent_thread_id: Some( + "33333333-3333-3333-3333-333333333333".to_string(), + ), + sub_agent_depth: Some(2), + sub_agent_path: Some("/root/review/backend".to_string()), + sub_agent_nickname: Some("Navigator".to_string()), + sub_agent_role: Some("explorer".to_string()), + ..Default::default() + }), + git_info: Some(proto::GitInfo { + sha: Some("abc123".to_string()), + branch: Some("main".to_string()), + origin_url: Some("https://example.test/repo.git".to_string()), + }), + agent_nickname: Some("Navigator".to_string()), + agent_role: Some("explorer".to_string()), + agent_path: Some("/root/review/backend".to_string()), + reasoning_effort: Some("high".to_string()), + first_user_message: Some("first message".to_string()), + }; + + let stored = stored_thread_from_proto(thread.clone()).expect("proto to stored thread"); + + assert_eq!(stored.rollout_path, None); + assert!(stored.history.is_none()); + assert_eq!(stored_thread_to_proto(stored), thread); + } +} diff --git a/codex-rs/thread-store/src/remote/mod.rs b/codex-rs/thread-store/src/remote/mod.rs new file mode 100644 index 0000000000..2e8b9e5afb --- /dev/null +++ b/codex-rs/thread-store/src/remote/mod.rs @@ -0,0 +1,112 @@ +mod helpers; +mod list_threads; + +use async_trait::async_trait; + +use crate::AppendThreadItemsParams; +use crate::ArchiveThreadParams; +use crate::CreateThreadParams; +use crate::ListThreadsParams; +use crate::LoadThreadHistoryParams; +use crate::ReadThreadParams; +use crate::ResumeThreadRecorderParams; +use crate::SetThreadNameParams; +use crate::StoredThread; +use crate::StoredThreadHistory; +use crate::ThreadPage; +use crate::ThreadRecorder; +use crate::ThreadStore; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; +use crate::UpdateThreadMetadataParams; +use proto::thread_store_client::ThreadStoreClient; + +#[path = "proto/codex.thread_store.v1.rs"] +mod proto; + +/// gRPC-backed [`ThreadStore`] implementation for deployments whose durable thread data lives +/// outside the app-server process. +#[derive(Clone, Debug)] +pub struct RemoteThreadStore { + endpoint: String, +} + +impl RemoteThreadStore { + pub fn new(endpoint: impl Into) -> Self { + Self { + endpoint: endpoint.into(), + } + } + + async fn client(&self) -> ThreadStoreResult> { + ThreadStoreClient::connect(self.endpoint.clone()) + .await + .map_err(|err| ThreadStoreError::Internal { + message: format!("failed to connect to remote thread store: {err}"), + }) + } +} + +#[async_trait] +impl ThreadStore for RemoteThreadStore { + async fn create_thread( + &self, + _params: CreateThreadParams, + ) -> ThreadStoreResult> { + Err(not_implemented("create_thread")) + } + + async fn resume_thread_recorder( + &self, + _params: ResumeThreadRecorderParams, + ) -> ThreadStoreResult> { + Err(not_implemented("resume_thread_recorder")) + } + + async fn append_items(&self, _params: AppendThreadItemsParams) -> ThreadStoreResult<()> { + Err(not_implemented("append_items")) + } + + async fn load_history( + &self, + _params: LoadThreadHistoryParams, + ) -> ThreadStoreResult { + Err(not_implemented("load_history")) + } + + async fn read_thread(&self, _params: ReadThreadParams) -> ThreadStoreResult { + Err(not_implemented("read_thread")) + } + + async fn list_threads(&self, params: ListThreadsParams) -> ThreadStoreResult { + list_threads::list_threads(self, params).await + } + + async fn set_thread_name(&self, _params: SetThreadNameParams) -> ThreadStoreResult<()> { + Err(not_implemented("set_thread_name")) + } + + async fn update_thread_metadata( + &self, + _params: UpdateThreadMetadataParams, + ) -> ThreadStoreResult { + Err(not_implemented("update_thread_metadata")) + } + + async fn archive_thread(&self, _params: ArchiveThreadParams) -> ThreadStoreResult<()> { + Err(not_implemented("archive_thread")) + } + + async fn unarchive_thread( + &self, + _params: ArchiveThreadParams, + ) -> ThreadStoreResult { + Err(not_implemented("unarchive_thread")) + } +} + +fn not_implemented(method: &str) -> ThreadStoreError { + ThreadStoreError::Internal { + message: format!("remote thread store does not implement {method} yet"), + } +} diff --git a/codex-rs/thread-store/src/remote/proto/codex.thread_store.v1.proto b/codex-rs/thread-store/src/remote/proto/codex.thread_store.v1.proto new file mode 100644 index 0000000000..f217027fcd --- /dev/null +++ b/codex-rs/thread-store/src/remote/proto/codex.thread_store.v1.proto @@ -0,0 +1,86 @@ +syntax = "proto3"; + +package codex.thread_store.v1; + +service ThreadStore { + rpc ListThreads(ListThreadsRequest) returns (ListThreadsResponse); +} + +message ListThreadsRequest { + uint32 page_size = 1; + optional string cursor = 2; + ThreadSortKey sort_key = 3; + repeated SessionSource allowed_sources = 4; + optional ModelProviderFilter model_provider_filter = 5; + bool archived = 6; + optional string search_term = 7; +} + +message ModelProviderFilter { + repeated string values = 1; +} + +enum ThreadSortKey { + THREAD_SORT_KEY_CREATED_AT = 0; + THREAD_SORT_KEY_UPDATED_AT = 1; +} + +message ListThreadsResponse { + repeated StoredThread threads = 1; + optional string next_cursor = 2; +} + +message StoredThread { + // Mirrors Rust's StoredThread. Domain types that are not protobuf-native, + // such as ThreadId, DateTime, and PathBuf, are represented as their + // stable scalar forms on the wire. + string thread_id = 1; + optional string forked_from_id = 2; + string preview = 3; + optional string name = 4; + string model_provider = 5; + optional string model = 6; + int64 created_at = 7; + int64 updated_at = 8; + optional int64 archived_at = 9; + string cwd = 10; + string cli_version = 11; + SessionSource source = 12; + optional GitInfo git_info = 13; + optional string agent_nickname = 14; + optional string agent_role = 15; + optional string agent_path = 16; + optional string reasoning_effort = 17; + optional string first_user_message = 18; +} + +message SessionSource { + SessionSourceKind kind = 1; + optional string custom = 2; + optional string sub_agent_parent_thread_id = 3; + optional int32 sub_agent_depth = 4; + optional string sub_agent_other = 5; + optional string sub_agent_path = 6; + optional string sub_agent_nickname = 7; + optional string sub_agent_role = 8; +} + +enum SessionSourceKind { + SESSION_SOURCE_KIND_UNKNOWN = 0; + SESSION_SOURCE_KIND_CLI = 1; + SESSION_SOURCE_KIND_VSCODE = 2; + SESSION_SOURCE_KIND_EXEC = 3; + SESSION_SOURCE_KIND_APP_SERVER = 4; + SESSION_SOURCE_KIND_CUSTOM = 5; + SESSION_SOURCE_KIND_SUB_AGENT_REVIEW = 6; + SESSION_SOURCE_KIND_SUB_AGENT_COMPACT = 7; + SESSION_SOURCE_KIND_SUB_AGENT_THREAD_SPAWN = 8; + SESSION_SOURCE_KIND_SUB_AGENT_MEMORY_CONSOLIDATION = 9; + SESSION_SOURCE_KIND_SUB_AGENT_OTHER = 10; +} + +message GitInfo { + optional string sha = 1; + optional string branch = 2; + optional string origin_url = 3; +} diff --git a/codex-rs/thread-store/src/remote/proto/codex.thread_store.v1.rs b/codex-rs/thread-store/src/remote/proto/codex.thread_store.v1.rs new file mode 100644 index 0000000000..6d0e410268 --- /dev/null +++ b/codex-rs/thread-store/src/remote/proto/codex.thread_store.v1.rs @@ -0,0 +1,460 @@ +// This file is @generated by prost-build. +#![allow(clippy::trivially_copy_pass_by_ref)] + +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListThreadsRequest { + #[prost(uint32, tag = "1")] + pub page_size: u32, + #[prost(string, optional, tag = "2")] + pub cursor: ::core::option::Option<::prost::alloc::string::String>, + #[prost(enumeration = "ThreadSortKey", tag = "3")] + pub sort_key: i32, + #[prost(message, repeated, tag = "4")] + pub allowed_sources: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag = "5")] + pub model_provider_filter: ::core::option::Option, + #[prost(bool, tag = "6")] + pub archived: bool, + #[prost(string, optional, tag = "7")] + pub search_term: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ModelProviderFilter { + #[prost(string, repeated, tag = "1")] + pub values: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListThreadsResponse { + #[prost(message, repeated, tag = "1")] + pub threads: ::prost::alloc::vec::Vec, + #[prost(string, optional, tag = "2")] + pub next_cursor: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct StoredThread { + /// Mirrors Rust's StoredThread. Domain types that are not protobuf-native, + /// such as ThreadId, DateTime, and PathBuf, are represented as their + /// stable scalar forms on the wire. + #[prost(string, tag = "1")] + pub thread_id: ::prost::alloc::string::String, + #[prost(string, optional, tag = "2")] + pub forked_from_id: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, tag = "3")] + pub preview: ::prost::alloc::string::String, + #[prost(string, optional, tag = "4")] + pub name: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, tag = "5")] + pub model_provider: ::prost::alloc::string::String, + #[prost(string, optional, tag = "6")] + pub model: ::core::option::Option<::prost::alloc::string::String>, + #[prost(int64, tag = "7")] + pub created_at: i64, + #[prost(int64, tag = "8")] + pub updated_at: i64, + #[prost(int64, optional, tag = "9")] + pub archived_at: ::core::option::Option, + #[prost(string, tag = "10")] + pub cwd: ::prost::alloc::string::String, + #[prost(string, tag = "11")] + pub cli_version: ::prost::alloc::string::String, + #[prost(message, optional, tag = "12")] + pub source: ::core::option::Option, + #[prost(message, optional, tag = "13")] + pub git_info: ::core::option::Option, + #[prost(string, optional, tag = "14")] + pub agent_nickname: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "15")] + pub agent_role: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "16")] + pub agent_path: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "17")] + pub reasoning_effort: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "18")] + pub first_user_message: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct SessionSource { + #[prost(enumeration = "SessionSourceKind", tag = "1")] + pub kind: i32, + #[prost(string, optional, tag = "2")] + pub custom: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "3")] + pub sub_agent_parent_thread_id: ::core::option::Option<::prost::alloc::string::String>, + #[prost(int32, optional, tag = "4")] + pub sub_agent_depth: ::core::option::Option, + #[prost(string, optional, tag = "5")] + pub sub_agent_other: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "6")] + pub sub_agent_path: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "7")] + pub sub_agent_nickname: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "8")] + pub sub_agent_role: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GitInfo { + #[prost(string, optional, tag = "1")] + pub sha: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "2")] + pub branch: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "3")] + pub origin_url: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ThreadSortKey { + CreatedAt = 0, + UpdatedAt = 1, +} +impl ThreadSortKey { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::CreatedAt => "THREAD_SORT_KEY_CREATED_AT", + Self::UpdatedAt => "THREAD_SORT_KEY_UPDATED_AT", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "THREAD_SORT_KEY_CREATED_AT" => Some(Self::CreatedAt), + "THREAD_SORT_KEY_UPDATED_AT" => Some(Self::UpdatedAt), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum SessionSourceKind { + Unknown = 0, + Cli = 1, + Vscode = 2, + Exec = 3, + AppServer = 4, + Custom = 5, + SubAgentReview = 6, + SubAgentCompact = 7, + SubAgentThreadSpawn = 8, + SubAgentMemoryConsolidation = 9, + SubAgentOther = 10, +} +impl SessionSourceKind { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unknown => "SESSION_SOURCE_KIND_UNKNOWN", + Self::Cli => "SESSION_SOURCE_KIND_CLI", + Self::Vscode => "SESSION_SOURCE_KIND_VSCODE", + Self::Exec => "SESSION_SOURCE_KIND_EXEC", + Self::AppServer => "SESSION_SOURCE_KIND_APP_SERVER", + Self::Custom => "SESSION_SOURCE_KIND_CUSTOM", + Self::SubAgentReview => "SESSION_SOURCE_KIND_SUB_AGENT_REVIEW", + Self::SubAgentCompact => "SESSION_SOURCE_KIND_SUB_AGENT_COMPACT", + Self::SubAgentThreadSpawn => "SESSION_SOURCE_KIND_SUB_AGENT_THREAD_SPAWN", + Self::SubAgentMemoryConsolidation => { + "SESSION_SOURCE_KIND_SUB_AGENT_MEMORY_CONSOLIDATION" + } + Self::SubAgentOther => "SESSION_SOURCE_KIND_SUB_AGENT_OTHER", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "SESSION_SOURCE_KIND_UNKNOWN" => Some(Self::Unknown), + "SESSION_SOURCE_KIND_CLI" => Some(Self::Cli), + "SESSION_SOURCE_KIND_VSCODE" => Some(Self::Vscode), + "SESSION_SOURCE_KIND_EXEC" => Some(Self::Exec), + "SESSION_SOURCE_KIND_APP_SERVER" => Some(Self::AppServer), + "SESSION_SOURCE_KIND_CUSTOM" => Some(Self::Custom), + "SESSION_SOURCE_KIND_SUB_AGENT_REVIEW" => Some(Self::SubAgentReview), + "SESSION_SOURCE_KIND_SUB_AGENT_COMPACT" => Some(Self::SubAgentCompact), + "SESSION_SOURCE_KIND_SUB_AGENT_THREAD_SPAWN" => Some(Self::SubAgentThreadSpawn), + "SESSION_SOURCE_KIND_SUB_AGENT_MEMORY_CONSOLIDATION" => { + Some(Self::SubAgentMemoryConsolidation) + } + "SESSION_SOURCE_KIND_SUB_AGENT_OTHER" => Some(Self::SubAgentOther), + _ => None, + } + } +} +/// Generated client implementations. +pub mod thread_store_client { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value + )] + use tonic::codegen::http::Uri; + use tonic::codegen::*; + #[derive(Debug, Clone)] + pub struct ThreadStoreClient { + inner: tonic::client::Grpc, + } + impl ThreadStoreClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl ThreadStoreClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> ThreadStoreClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + >>::Error: + Into + std::marker::Send + std::marker::Sync, + { + ThreadStoreClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + pub async fn list_threads( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/codex.thread_store.v1.ThreadStore/ListThreads", + ); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "codex.thread_store.v1.ThreadStore", + "ListThreads", + )); + self.inner.unary(req, path, codec).await + } + } +} +/// Generated server implementations. +pub mod thread_store_server { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value + )] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with ThreadStoreServer. + #[async_trait] + pub trait ThreadStore: std::marker::Send + std::marker::Sync + 'static { + async fn list_threads( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + } + #[derive(Debug)] + pub struct ThreadStoreServer { + inner: Arc, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + impl ThreadStoreServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor(inner: T, interceptor: F) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> for ThreadStoreServer + where + T: ThreadStore, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + match req.uri().path() { + "/codex.thread_store.v1.ThreadStore/ListThreads" => { + #[allow(non_camel_case_types)] + struct ListThreadsSvc(pub Arc); + impl tonic::server::UnaryService for ListThreadsSvc { + type Response = super::ListThreadsResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::list_threads(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ListThreadsSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => Box::pin(async move { + let mut response = http::Response::new(tonic::body::Body::default()); + let headers = response.headers_mut(); + headers.insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers.insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) + }), + } + } + } + impl Clone for ThreadStoreServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "codex.thread_store.v1.ThreadStore"; + impl tonic::server::NamedService for ThreadStoreServer { + const NAME: &'static str = SERVICE_NAME; + } +} From 2ca270d08dfda06e44df579b86d62d96e388019d Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Thu, 16 Apr 2026 10:30:10 -0700 Subject: [PATCH 77/84] [2/8] Support piped stdin in exec process API (#18086) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Add an explicit stdin mode to process/start. - Keep normal non-interactive exec stdin closed while allowing pipe-backed processes. ## Stack ```text o #18027 [8/8] Fail exec client operations after disconnect โ”‚ o #18025 [7/8] Cover MCP stdio tests with executor placement โ”‚ o #18089 [6/8] Wire remote MCP stdio through executor โ”‚ o #18088 [5/8] Add executor process transport for MCP stdio โ”‚ o #18087 [4/8] Abstract MCP stdio server launching โ”‚ o #18020 [3/8] Add pushed exec process events โ”‚ @ #18086 [2/8] Support piped stdin in exec process API โ”‚ o #18085 [1/8] Add MCP server environment config โ”‚ o main ``` Co-authored-by: Codex --- .../core/src/unified_exec/process_manager.rs | 1 + codex-rs/exec-server/README.md | 13 +-- codex-rs/exec-server/src/environment.rs | 1 + codex-rs/exec-server/src/local_process.rs | 14 ++- codex-rs/exec-server/src/protocol.rs | 3 + .../exec-server/src/server/handler/tests.rs | 1 + codex-rs/exec-server/src/server/processor.rs | 1 + codex-rs/exec-server/tests/exec_process.rs | 91 +++++++++++++++++ codex-rs/exec-server/tests/process.rs | 97 +++++++++++++++++++ 9 files changed, 215 insertions(+), 7 deletions(-) diff --git a/codex-rs/core/src/unified_exec/process_manager.rs b/codex-rs/core/src/unified_exec/process_manager.rs index 5214892681..aaaa7076f4 100644 --- a/codex-rs/core/src/unified_exec/process_manager.rs +++ b/codex-rs/core/src/unified_exec/process_manager.rs @@ -152,6 +152,7 @@ fn exec_server_params_for_request( env_policy, env, tty, + pipe_stdin: false, arg0: request.arg0.clone(), } } diff --git a/codex-rs/exec-server/README.md b/codex-rs/exec-server/README.md index 0047449d39..78b92e1a71 100644 --- a/codex-rs/exec-server/README.md +++ b/codex-rs/exec-server/README.md @@ -85,6 +85,7 @@ Request params: "PATH": "/usr/bin:/bin" }, "tty": true, + "pipeStdin": false, "arg0": null } ``` @@ -95,8 +96,8 @@ Field definitions: - `argv`: command vector. It must be non-empty. - `cwd`: absolute working directory used for the child process. - `env`: environment variables passed to the child process. -- `tty`: when `true`, spawn a PTY-backed interactive process; when `false`, - spawn a pipe-backed process with closed stdin. +- `tty`: when `true`, spawn a PTY-backed interactive process. +- `pipeStdin`: when `true`, keep non-PTY stdin writable via `process/write`. - `arg0`: optional argv0 override forwarded to `codex-utils-pty`. Response: @@ -111,7 +112,7 @@ Behavior notes: - Reusing an existing `processId` is rejected. - PTY-backed processes accept later writes through `process/write`. -- Pipe-backed processes are launched with stdin closed and reject writes. +- Non-PTY processes reject writes unless `pipeStdin` is `true`. - Output is streamed asynchronously via `process/output`. - Exit is reported asynchronously via `process/exited`. @@ -153,7 +154,7 @@ Response: ### `process/write` -Writes raw bytes to a running PTY-backed process stdin. +Writes raw bytes to a running process stdin. Request params: @@ -177,7 +178,7 @@ Response: Behavior notes: - Writes to an unknown `processId` are rejected. -- Writes to a non-PTY process are rejected because stdin is already closed. +- Writes to a non-PTY process are rejected unless it started with `pipeStdin`. ### `process/terminate` @@ -325,7 +326,7 @@ Initialize: Start a process: ```json -{"id":2,"method":"process/start","params":{"processId":"proc-1","argv":["bash","-lc","printf 'ready\\n'; while IFS= read -r line; do printf 'echo:%s\\n' \"$line\"; done"],"cwd":"/tmp","env":{"PATH":"/usr/bin:/bin"},"tty":true,"arg0":null}} +{"id":2,"method":"process/start","params":{"processId":"proc-1","argv":["bash","-lc","printf 'ready\\n'; while IFS= read -r line; do printf 'echo:%s\\n' \"$line\"; done"],"cwd":"/tmp","env":{"PATH":"/usr/bin:/bin"},"tty":true,"pipeStdin":false,"arg0":null}} {"id":2,"result":{"processId":"proc-1"}} {"method":"process/output","params":{"processId":"proc-1","seq":1,"stream":"stdout","chunk":"cmVhZHkK"}} ``` diff --git a/codex-rs/exec-server/src/environment.rs b/codex-rs/exec-server/src/environment.rs index 77ead87a84..afe0720196 100644 --- a/codex-rs/exec-server/src/environment.rs +++ b/codex-rs/exec-server/src/environment.rs @@ -346,6 +346,7 @@ mod tests { env_policy: None, env: Default::default(), tty: false, + pipe_stdin: false, arg0: None, }) .await diff --git a/codex-rs/exec-server/src/local_process.rs b/codex-rs/exec-server/src/local_process.rs index bf38aa360b..2608f892b8 100644 --- a/codex-rs/exec-server/src/local_process.rs +++ b/codex-rs/exec-server/src/local_process.rs @@ -59,6 +59,7 @@ struct RetainedOutputChunk { struct RunningProcess { session: ExecCommandSession, tty: bool, + pipe_stdin: bool, output: VecDeque, retained_bytes: usize, next_seq: u64, @@ -165,6 +166,15 @@ impl LocalProcess { TerminalSize::default(), ) .await + } else if params.pipe_stdin { + codex_utils_pty::spawn_pipe_process( + program, + args, + params.cwd.as_path(), + &env, + ¶ms.arg0, + ) + .await } else { codex_utils_pty::spawn_pipe_process_no_stdin( program, @@ -195,6 +205,7 @@ impl LocalProcess { ProcessEntry::Running(Box::new(RunningProcess { session: spawned.session, tty: params.tty, + pipe_stdin: params.pipe_stdin, output: VecDeque::new(), retained_bytes: 0, next_seq: 1, @@ -339,7 +350,7 @@ impl LocalProcess { status: WriteStatus::Starting, }); }; - if !process.tty { + if !process.tty && !process.pipe_stdin { return Ok(WriteResponse { status: WriteStatus::StdinClosed, }); @@ -667,6 +678,7 @@ mod tests { env_policy: None, env, tty: false, + pipe_stdin: false, arg0: None, } } diff --git a/codex-rs/exec-server/src/protocol.rs b/codex-rs/exec-server/src/protocol.rs index 5d29348897..9f4234e649 100644 --- a/codex-rs/exec-server/src/protocol.rs +++ b/codex-rs/exec-server/src/protocol.rs @@ -69,6 +69,9 @@ pub struct ExecParams { pub env_policy: Option, pub env: HashMap, pub tty: bool, + /// Keep non-tty stdin writable through `process/write`. + #[serde(default)] + pub pipe_stdin: bool, pub arg0: Option, } diff --git a/codex-rs/exec-server/src/server/handler/tests.rs b/codex-rs/exec-server/src/server/handler/tests.rs index 321bf243a9..7f2cdba629 100644 --- a/codex-rs/exec-server/src/server/handler/tests.rs +++ b/codex-rs/exec-server/src/server/handler/tests.rs @@ -30,6 +30,7 @@ fn exec_params_with_argv(process_id: &str, argv: Vec) -> ExecParams { env_policy: None, env: inherited_path_env(), tty: false, + pipe_stdin: false, arg0: None, } } diff --git a/codex-rs/exec-server/src/server/processor.rs b/codex-rs/exec-server/src/server/processor.rs index 87b9704502..1153bc83e2 100644 --- a/codex-rs/exec-server/src/server/processor.rs +++ b/codex-rs/exec-server/src/server/processor.rs @@ -393,6 +393,7 @@ mod tests { env_policy: None, env, tty: false, + pipe_stdin: false, arg0: None, } } diff --git a/codex-rs/exec-server/tests/exec_process.rs b/codex-rs/exec-server/tests/exec_process.rs index 72f029231c..862e4e1c8b 100644 --- a/codex-rs/exec-server/tests/exec_process.rs +++ b/codex-rs/exec-server/tests/exec_process.rs @@ -12,6 +12,7 @@ use codex_exec_server::ExecProcess; use codex_exec_server::ProcessId; use codex_exec_server::ReadResponse; use codex_exec_server::StartedExecProcess; +use codex_exec_server::WriteStatus; use pretty_assertions::assert_eq; use test_case::test_case; use tokio::sync::watch; @@ -54,6 +55,7 @@ async fn assert_exec_process_starts_and_exits(use_remote: bool) -> Result<()> { env_policy: /*env_policy*/ None, env: Default::default(), tty: false, + pipe_stdin: false, arg0: None, }) .await?; @@ -131,6 +133,7 @@ async fn assert_exec_process_streams_output(use_remote: bool) -> Result<()> { env_policy: /*env_policy*/ None, env: Default::default(), tty: false, + pipe_stdin: false, arg0: None, }) .await?; @@ -164,6 +167,7 @@ async fn assert_exec_process_write_then_read(use_remote: bool) -> Result<()> { env_policy: /*env_policy*/ None, env: Default::default(), tty: true, + pipe_stdin: false, arg0: None, }) .await?; @@ -184,6 +188,73 @@ async fn assert_exec_process_write_then_read(use_remote: bool) -> Result<()> { Ok(()) } +async fn assert_exec_process_write_then_read_without_tty(use_remote: bool) -> Result<()> { + let context = create_process_context(use_remote).await?; + let process_id = "proc-stdin-pipe".to_string(); + let session = context + .backend + .start(ExecParams { + process_id: process_id.clone().into(), + argv: vec![ + "/bin/sh".to_string(), + "-c".to_string(), + "IFS= read line; printf 'from-stdin:%s\\n' \"$line\"".to_string(), + ], + cwd: std::env::current_dir()?, + env_policy: /*env_policy*/ None, + env: Default::default(), + tty: false, + pipe_stdin: true, + arg0: None, + }) + .await?; + assert_eq!(session.process.process_id().as_str(), process_id); + + tokio::time::sleep(Duration::from_millis(200)).await; + let write_response = session.process.write(b"hello\n".to_vec()).await?; + assert_eq!(write_response.status, WriteStatus::Accepted); + let StartedExecProcess { process } = session; + let wake_rx = process.subscribe_wake(); + let actual = collect_process_output_from_reads(process, wake_rx).await?; + + assert_eq!(actual, ("from-stdin:hello\n".to_string(), Some(0), true)); + Ok(()) +} + +async fn assert_exec_process_rejects_write_without_pipe_stdin(use_remote: bool) -> Result<()> { + let context = create_process_context(use_remote).await?; + let process_id = "proc-stdin-closed".to_string(); + let session = context + .backend + .start(ExecParams { + process_id: process_id.clone().into(), + argv: vec![ + "/bin/sh".to_string(), + "-c".to_string(), + "sleep 0.3; if IFS= read -r line; then printf 'read:%s\\n' \"$line\"; else printf 'eof\\n'; fi".to_string(), + ], + cwd: std::env::current_dir()?, + env_policy: /*env_policy*/ None, + env: Default::default(), + tty: false, + pipe_stdin: false, + arg0: None, + }) + .await?; + assert_eq!(session.process.process_id().as_str(), process_id); + + let write_response = session.process.write(b"ignored\n".to_vec()).await?; + assert_eq!(write_response.status, WriteStatus::StdinClosed); + let StartedExecProcess { process } = session; + let wake_rx = process.subscribe_wake(); + let (output, exit_code, closed) = collect_process_output_from_reads(process, wake_rx).await?; + + assert_eq!(output, "eof\n"); + assert_eq!(exit_code, Some(0)); + assert!(closed); + Ok(()) +} + async fn assert_exec_process_preserves_queued_events_before_subscribe( use_remote: bool, ) -> Result<()> { @@ -201,6 +272,7 @@ async fn assert_exec_process_preserves_queued_events_before_subscribe( env_policy: /*env_policy*/ None, env: Default::default(), tty: false, + pipe_stdin: false, arg0: None, }) .await?; @@ -234,6 +306,7 @@ async fn remote_exec_process_reports_transport_disconnect() -> Result<()> { env_policy: /*env_policy*/ None, env: Default::default(), tty: false, + pipe_stdin: false, arg0: None, }) .await?; @@ -289,6 +362,24 @@ async fn exec_process_write_then_read(use_remote: bool) -> Result<()> { assert_exec_process_write_then_read(use_remote).await } +#[test_case(false ; "local")] +#[test_case(true ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +// Serialize tests that launch a real exec-server process through the full CLI. +#[serial_test::serial(remote_exec_server)] +async fn exec_process_write_then_read_without_tty(use_remote: bool) -> Result<()> { + assert_exec_process_write_then_read_without_tty(use_remote).await +} + +#[test_case(false ; "local")] +#[test_case(true ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +// Serialize tests that launch a real exec-server process through the full CLI. +#[serial_test::serial(remote_exec_server)] +async fn exec_process_rejects_write_without_pipe_stdin(use_remote: bool) -> Result<()> { + assert_exec_process_rejects_write_without_pipe_stdin(use_remote).await +} + #[test_case(false ; "local")] #[test_case(true ; "remote")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/codex-rs/exec-server/tests/process.rs b/codex-rs/exec-server/tests/process.rs index cc96e6ef81..334e449224 100644 --- a/codex-rs/exec-server/tests/process.rs +++ b/codex-rs/exec-server/tests/process.rs @@ -10,6 +10,8 @@ use codex_exec_server::InitializeResponse; use codex_exec_server::ProcessId; use codex_exec_server::ReadResponse; use codex_exec_server::TerminateResponse; +use codex_exec_server::WriteResponse; +use codex_exec_server::WriteStatus; use common::exec_server::exec_server; use pretty_assertions::assert_eq; @@ -47,6 +49,7 @@ async fn exec_server_starts_process_over_websocket() -> anyhow::Result<()> { "cwd": std::env::current_dir()?, "env": {}, "tty": false, + "pipeStdin": false, "arg0": null }), ) @@ -75,6 +78,99 @@ async fn exec_server_starts_process_over_websocket() -> anyhow::Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn exec_server_defaults_omitted_pipe_stdin_to_closed_stdin() -> anyhow::Result<()> { + let mut server = exec_server().await?; + let initialize_id = server + .send_request( + "initialize", + serde_json::to_value(InitializeParams { + client_name: "exec-server-test".to_string(), + resume_session_id: None, + })?, + ) + .await?; + let _ = server + .wait_for_event(|event| { + matches!( + event, + JSONRPCMessage::Response(JSONRPCResponse { id, .. }) if id == &initialize_id + ) + }) + .await?; + + server + .send_notification("initialized", serde_json::json!({})) + .await?; + + let process_start_id = server + .send_request( + "process/start", + serde_json::json!({ + "processId": "proc-default-stdin", + "argv": [ + "/bin/sh", + "-c", + "sleep 0.3; if IFS= read -r line; then printf 'read:%s\\n' \"$line\"; else printf 'eof\\n'; fi" + ], + "cwd": std::env::current_dir()?, + "env": {}, + "tty": false, + "arg0": null + }), + ) + .await?; + let response = server + .wait_for_event(|event| { + matches!( + event, + JSONRPCMessage::Response(JSONRPCResponse { id, .. }) if id == &process_start_id + ) + }) + .await?; + let JSONRPCMessage::Response(JSONRPCResponse { result, .. }) = response else { + panic!("expected process/start response"); + }; + let process_start_response: ExecResponse = serde_json::from_value(result)?; + assert_eq!( + process_start_response, + ExecResponse { + process_id: ProcessId::from("proc-default-stdin") + } + ); + + let write_id = server + .send_request( + "process/write", + serde_json::json!({ + "processId": "proc-default-stdin", + "chunk": "aWdub3JlZAo=" + }), + ) + .await?; + let response = server + .wait_for_event(|event| { + matches!( + event, + JSONRPCMessage::Response(JSONRPCResponse { id, .. }) if id == &write_id + ) + }) + .await?; + let JSONRPCMessage::Response(JSONRPCResponse { result, .. }) = response else { + panic!("expected process/write response"); + }; + let write_response: WriteResponse = serde_json::from_value(result)?; + assert_eq!( + write_response, + WriteResponse { + status: WriteStatus::StdinClosed + } + ); + + server.shutdown().await?; + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn exec_server_resumes_detached_session_without_killing_processes() -> anyhow::Result<()> { let mut server = exec_server().await?; @@ -113,6 +209,7 @@ async fn exec_server_resumes_detached_session_without_killing_processes() -> any "cwd": std::env::current_dir()?, "env": {}, "tty": false, + "pipeStdin": false, "arg0": null }), ) From ff9744fd661988450a24d845cb3416ad2af71f7d Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Thu, 16 Apr 2026 10:30:28 -0700 Subject: [PATCH 78/84] Avoid fatal TUI errors on skills list failure (#18061) Addresses #17951 Problem: The TUI treated skills/list failures as fatal during refresh, so proxy/firewall responses that break plugin discovery could crash the session. Solution: Route startup and refresh skills/list responses through shared graceful handling that logs a warning and keeps the TUI running. --- codex-rs/tui/src/app.rs | 52 ++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 3fbcd64909..5641131566 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -2514,14 +2514,16 @@ impl App { Ok(true) } AppCommandView::ListSkills { cwds, force_reload } => { - let response = app_server - .skills_list(codex_app_server_protocol::SkillsListParams { - cwds: cwds.to_vec(), - force_reload, - per_cwd_extra_user_roots: None, - }) - .await?; - self.handle_skills_list_response(response); + self.handle_skills_list_result( + app_server + .skills_list(codex_app_server_protocol::SkillsListParams { + cwds: cwds.to_vec(), + force_reload, + per_cwd_extra_user_roots: None, + }) + .await, + "failed to refresh skills", + ); Ok(true) } AppCommandView::Compact => { @@ -2595,6 +2597,19 @@ impl App { } } + fn handle_skills_list_result( + &mut self, + result: Result, + failure_message: &str, + ) { + match result { + Ok(response) => self.handle_skills_list_response(response), + Err(err) => { + tracing::warn!("{failure_message}: {err:#}"); + } + } + } + async fn try_resolve_app_server_request( &mut self, app_server: &AppServerSession, @@ -4002,17 +4017,16 @@ impl App { app.enqueue_primary_thread_session(started.session, started.turns) .await?; } - match app_server - .skills_list(codex_app_server_protocol::SkillsListParams { - cwds: vec![app.config.cwd.to_path_buf()], - force_reload: true, - per_cwd_extra_user_roots: None, - }) - .await - { - Ok(response) => app.handle_skills_list_response(response), - Err(err) => tracing::warn!("failed to load skills on startup: {err:#}"), - } + app.handle_skills_list_result( + app_server + .skills_list(codex_app_server_protocol::SkillsListParams { + cwds: vec![app.config.cwd.to_path_buf()], + force_reload: true, + per_cwd_extra_user_roots: None, + }) + .await, + "failed to load skills on startup", + ); // On startup, if Agent mode (workspace-write) or ReadOnly is active, warn about world-writable dirs on Windows. #[cfg(target_os = "windows")] From 6862b9c745066e6b7f73ea7483c9abb0df696594 Mon Sep 17 00:00:00 2001 From: viyatb-oai Date: Thu, 16 Apr 2026 10:31:51 -0700 Subject: [PATCH 79/84] feat(permissions): add glob deny-read policy support (#15979) ## Summary - adds first-class filesystem policy entries for deny-read glob patterns - parses config such as :project_roots { "**/*.env" = "none" } into pattern entries - enforces deny-read patterns in direct read/list helpers - fails closed for sandbox execution until platform backends enforce glob patterns in #18096 - preserves split filesystem policy in turn context only when it cannot be reconstructed from legacy sandbox policy ## Stack 1. This PR - glob deny-read policy/config/direct-tool support 2. #18096 - macOS and Linux sandbox enforcement 3. #17740 - managed deny-read requirements ## Verification - just fmt - cargo check -p codex-core -p codex-sandboxing --tests --------- Co-authored-by: Codex --- codex-rs/Cargo.lock | 1 + codex-rs/core/src/codex.rs | 22 +- .../src/codex/rollout_reconstruction_tests.rs | 8 + codex-rs/core/src/codex_tests.rs | 92 ++++ codex-rs/core/src/config/config_tests.rs | 68 +++ codex-rs/core/src/config/mod.rs | 4 +- codex-rs/core/src/config/permissions.rs | 181 ++++++- codex-rs/core/src/config/permissions_tests.rs | 92 ++++ .../core/src/context_manager/history_tests.rs | 1 + codex-rs/core/src/exec.rs | 6 + codex-rs/core/src/exec_tests.rs | 47 ++ codex-rs/core/src/tools/handlers/list_dir.rs | 34 +- .../core/src/tools/handlers/list_dir_tests.rs | 71 +++ codex-rs/core/tests/suite/resume_warning.rs | 1 + codex-rs/protocol/Cargo.toml | 1 + codex-rs/protocol/src/permissions.rs | 484 +++++++++++++++--- codex-rs/protocol/src/protocol.rs | 24 + codex-rs/rollout/src/recorder_tests.rs | 1 + codex-rs/sandboxing/src/lib.rs | 3 + codex-rs/sandboxing/src/manager.rs | 14 + codex-rs/state/src/extract.rs | 3 + codex-rs/tui/src/app.rs | 1 + codex-rs/tui/src/lib.rs | 1 + 23 files changed, 1081 insertions(+), 79 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 3e21398f85..a6d1ec2e47 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2614,6 +2614,7 @@ dependencies = [ "codex-utils-string", "codex-utils-template", "encoding_rs", + "globset", "http 1.4.0", "icu_decimal", "icu_locale_core", diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 00c5c144cd..5f821eb6b4 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -1083,6 +1083,18 @@ impl TurnContext { } pub(crate) fn to_turn_context_item(&self) -> TurnContextItem { + let legacy_file_system_sandbox_policy = FileSystemSandboxPolicy::from_legacy_sandbox_policy( + self.sandbox_policy.get(), + &self.cwd, + ); + // Omit the derived split filesystem policy when it is equivalent to + // the legacy sandbox policy. This keeps turn-context payloads stable + // while both fields exist; once callers consume only the split policy, + // this comparison and the legacy projection should go away. + let file_system_sandbox_policy = (self.file_system_sandbox_policy + != legacy_file_system_sandbox_policy) + .then(|| self.file_system_sandbox_policy.clone()); + TurnContextItem { turn_id: Some(self.sub_id.clone()), trace_id: self.trace_id.clone(), @@ -1092,6 +1104,7 @@ impl TurnContext { approval_policy: self.approval_policy.value(), sandbox_policy: self.sandbox_policy.get().clone(), network: self.turn_context_network_item(), + file_system_sandbox_policy, model: self.model_info.slug.clone(), personality: self.personality, collaboration_mode: Some(self.collaboration_mode.clone()), @@ -1287,7 +1300,14 @@ impl SessionConfiguration { let cwd_changed = absolute_cwd.as_path() != self.cwd.as_path(); next_configuration.cwd = absolute_cwd; - if sandbox_policy_changed || (cwd_changed && file_system_policy_matches_legacy) { + if sandbox_policy_changed { + next_configuration.file_system_sandbox_policy = + FileSystemSandboxPolicy::from_legacy_sandbox_policy_preserving_deny_entries( + next_configuration.sandbox_policy.get(), + &next_configuration.cwd, + &self.file_system_sandbox_policy, + ); + } else if cwd_changed && file_system_policy_matches_legacy { // Preserve richer split policies across cwd-only updates; only // rederive when the session is already using the legacy bridge. next_configuration.file_system_sandbox_policy = diff --git a/codex-rs/core/src/codex/rollout_reconstruction_tests.rs b/codex-rs/core/src/codex/rollout_reconstruction_tests.rs index 753244ac2b..432cfa7a40 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction_tests.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction_tests.rs @@ -68,6 +68,7 @@ async fn record_initial_history_resumed_bare_turn_context_does_not_hydrate_previ approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: previous_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), @@ -107,6 +108,7 @@ async fn record_initial_history_resumed_hydrates_previous_turn_settings_from_lif approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: previous_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), @@ -901,6 +903,7 @@ async fn record_initial_history_resumed_turn_context_after_compaction_reestablis approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: previous_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), @@ -976,6 +979,7 @@ async fn record_initial_history_resumed_turn_context_after_compaction_reestablis approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: previous_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), @@ -1005,6 +1009,7 @@ async fn record_initial_history_resumed_aborted_turn_without_id_clears_active_tu approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: previous_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), @@ -1117,6 +1122,7 @@ async fn record_initial_history_resumed_unmatched_abort_preserves_active_turn_fo approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: current_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), @@ -1227,6 +1233,7 @@ async fn record_initial_history_resumed_trailing_incomplete_turn_compaction_clea approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: previous_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), @@ -1376,6 +1383,7 @@ async fn record_initial_history_resumed_replaced_incomplete_compacted_turn_clear approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: previous_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), diff --git a/codex-rs/core/src/codex_tests.rs b/codex-rs/core/src/codex_tests.rs index 4805b03d01..6f4276b077 100644 --- a/codex-rs/core/src/codex_tests.rs +++ b/codex-rs/core/src/codex_tests.rs @@ -1639,6 +1639,7 @@ async fn record_initial_history_forked_hydrates_previous_turn_settings() { approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: previous_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), @@ -4714,6 +4715,47 @@ async fn build_initial_context_restates_realtime_start_when_reference_context_is ); } +fn file_system_policy_with_unreadable_glob(turn_context: &TurnContext) -> FileSystemSandboxPolicy { + let mut policy = FileSystemSandboxPolicy::from_legacy_sandbox_policy( + turn_context.sandbox_policy.get(), + &turn_context.cwd, + ); + policy.entries.push(FileSystemSandboxEntry { + path: FileSystemPath::GlobPattern { + pattern: format!("{}/**/*.env", turn_context.cwd.as_path().display()), + }, + access: FileSystemAccessMode::None, + }); + policy +} + +#[tokio::test] +async fn turn_context_item_omits_legacy_equivalent_file_system_sandbox_policy() { + let (_session, mut turn_context) = make_session_and_context().await; + turn_context.file_system_sandbox_policy = FileSystemSandboxPolicy::from_legacy_sandbox_policy( + turn_context.sandbox_policy.get(), + &turn_context.cwd, + ); + + let item = turn_context.to_turn_context_item(); + + assert_eq!(item.file_system_sandbox_policy, None); +} + +#[tokio::test] +async fn turn_context_item_stores_split_file_system_sandbox_policy_when_different() { + let (_session, mut turn_context) = make_session_and_context().await; + let file_system_sandbox_policy = file_system_policy_with_unreadable_glob(&turn_context); + turn_context.file_system_sandbox_policy = file_system_sandbox_policy.clone(); + + let item = turn_context.to_turn_context_item(); + + assert_eq!( + item.file_system_sandbox_policy, + Some(file_system_sandbox_policy) + ); +} + #[tokio::test] async fn record_context_updates_and_set_reference_context_item_injects_full_context_when_baseline_missing() { @@ -4852,6 +4894,56 @@ async fn record_context_updates_and_set_reference_context_item_persists_baseline ); } +#[tokio::test] +async fn record_context_updates_and_set_reference_context_item_persists_split_file_system_policy_to_rollout() + { + let (session, mut turn_context) = make_session_and_context().await; + let file_system_sandbox_policy = file_system_policy_with_unreadable_glob(&turn_context); + turn_context.file_system_sandbox_policy = file_system_sandbox_policy.clone(); + let config = session.get_config().await; + let recorder = RolloutRecorder::new( + config.as_ref(), + RolloutRecorderParams::new( + ThreadId::default(), + /*forked_from_id*/ None, + SessionSource::Exec, + BaseInstructions::default(), + Vec::new(), + EventPersistenceMode::Limited, + ), + /*state_db_ctx*/ None, + /*state_builder*/ None, + ) + .await + .expect("create rollout recorder"); + let rollout_path = recorder.rollout_path().to_path_buf(); + { + let mut rollout = session.services.rollout.lock().await; + *rollout = Some(recorder); + } + + session + .record_context_updates_and_set_reference_context_item(&turn_context) + .await; + session.ensure_rollout_materialized().await; + session.flush_rollout().await.expect("rollout should flush"); + + let InitialHistory::Resumed(resumed) = RolloutRecorder::get_rollout_history(&rollout_path) + .await + .expect("read rollout history") + else { + panic!("expected resumed rollout history"); + }; + let persisted_file_system_sandbox_policy = resumed.history.iter().find_map(|item| match item { + RolloutItem::TurnContext(ctx) => ctx.file_system_sandbox_policy.clone(), + _ => None, + }); + assert_eq!( + persisted_file_system_sandbox_policy, + Some(file_system_sandbox_policy) + ); +} + #[tokio::test] async fn build_initial_context_prepends_model_switch_message() { let (session, turn_context) = make_session_and_context().await; diff --git a/codex-rs/core/src/config/config_tests.rs b/codex-rs/core/src/config/config_tests.rs index 9243235d71..a7a5e88352 100644 --- a/codex-rs/core/src/config/config_tests.rs +++ b/codex-rs/core/src/config/config_tests.rs @@ -659,6 +659,74 @@ async fn default_permissions_profile_populates_runtime_sandbox_policy() -> std:: Ok(()) } +#[tokio::test] +async fn project_root_glob_none_compiles_to_filesystem_pattern_entry() -> std::io::Result<()> { + let codex_home = TempDir::new()?; + let cwd = TempDir::new()?; + tokio::fs::write(cwd.path().join(".git"), "gitdir: nowhere").await?; + + let config = Config::load_from_base_config_with_overrides( + ConfigToml { + default_permissions: Some("workspace".to_string()), + permissions: Some(PermissionsToml { + entries: BTreeMap::from([( + "workspace".to_string(), + PermissionProfileToml { + filesystem: Some(FilesystemPermissionsToml { + entries: BTreeMap::from([( + ":project_roots".to_string(), + FilesystemPermissionToml::Scoped(BTreeMap::from([ + (".".to_string(), FileSystemAccessMode::Write), + ("**/*.env".to_string(), FileSystemAccessMode::None), + ])), + )]), + }), + network: None, + }, + )]), + }), + ..Default::default() + }, + ConfigOverrides { + cwd: Some(cwd.path().to_path_buf()), + ..Default::default() + }, + codex_home.abs(), + ) + .await?; + + let expected_pattern = AbsolutePathBuf::resolve_path_against_base("**/*.env", cwd.path()) + .to_string_lossy() + .into_owned(); + assert!( + config + .permissions + .file_system_sandbox_policy + .entries + .contains(&FileSystemSandboxEntry { + path: FileSystemPath::GlobPattern { + pattern: expected_pattern, + }, + access: FileSystemAccessMode::None, + }) + ); + assert!( + !config + .permissions + .file_system_sandbox_policy + .entries + .iter() + .any(|entry| matches!( + &entry.path, + FileSystemPath::Special { + value: FileSystemSpecialPath::ProjectRoots { subpath: Some(subpath) }, + } if subpath == std::path::Path::new("**/*.env") + )), + "glob should compile to a filesystem pattern entry, not a literal filesystem entry" + ); + Ok(()) +} + #[tokio::test] async fn permissions_profiles_require_default_permissions() -> std::io::Result<()> { let codex_home = TempDir::new()?; diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index adfbe45baa..87e888e46e 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -1622,6 +1622,7 @@ impl Config { compile_permission_profile( permissions, default_permissions, + resolved_cwd.as_path(), &mut startup_warnings, )?; let mut sandbox_policy = file_system_sandbox_policy @@ -2006,9 +2007,10 @@ impl Config { if effective_sandbox_policy == original_sandbox_policy { file_system_sandbox_policy } else { - FileSystemSandboxPolicy::from_legacy_sandbox_policy( + FileSystemSandboxPolicy::from_legacy_sandbox_policy_preserving_deny_entries( &effective_sandbox_policy, resolved_cwd.as_path(), + &file_system_sandbox_policy, ) }; let effective_file_system_sandbox_policy = effective_file_system_sandbox_policy diff --git a/codex-rs/core/src/config/permissions.rs b/codex-rs/core/src/config/permissions.rs index 826d0139bf..1609b696d8 100644 --- a/codex-rs/core/src/config/permissions.rs +++ b/codex-rs/core/src/config/permissions.rs @@ -5,12 +5,14 @@ use std::path::Path; use std::path::PathBuf; use codex_config::permissions_toml::FilesystemPermissionToml; +use codex_config::permissions_toml::FilesystemPermissionsToml; use codex_config::permissions_toml::NetworkToml; use codex_config::permissions_toml::PermissionProfileToml; use codex_config::permissions_toml::PermissionsToml; use codex_network_proxy::NetworkProxyConfig; #[cfg(test)] use codex_network_proxy::NetworkUnixSocketPermission as ProxyNetworkUnixSocketPermission; +use codex_protocol::permissions::FileSystemAccessMode; use codex_protocol::permissions::FileSystemPath; use codex_protocol::permissions::FileSystemSandboxEntry; use codex_protocol::permissions::FileSystemSandboxPolicy; @@ -42,6 +44,7 @@ pub(crate) fn resolve_permission_profile<'a>( pub(crate) fn compile_permission_profile( permissions: &PermissionsToml, profile_name: &str, + policy_cwd: &Path, startup_warnings: &mut Vec, ) -> io::Result<(FileSystemSandboxPolicy, NetworkSandboxPolicy)> { let profile = resolve_permission_profile(permissions, profile_name)?; @@ -54,8 +57,23 @@ pub(crate) fn compile_permission_profile( missing_filesystem_entries_warning(profile_name), ); } else { + if cfg!(not(target_os = "macos")) { + for pattern in unsupported_read_write_glob_paths(filesystem) { + push_warning( + startup_warnings, + format!( + "Filesystem glob `{pattern}` uses `read` or `write` access, which is not fully supported by this platform's sandboxing. Use an exact path or trailing `/**` subtree rule instead. `none` deny-read globs are supported." + ), + ); + } + } for (path, permission) in &filesystem.entries { - compile_filesystem_permission(path, permission, &mut entries, startup_warnings)?; + entries.extend(compile_filesystem_permission( + path, + permission, + policy_cwd, + startup_warnings, + )?); } } } else { @@ -66,7 +84,6 @@ pub(crate) fn compile_permission_profile( } let network_sandbox_policy = compile_network_sandbox_policy(profile.network.as_ref()); - Ok(( FileSystemSandboxPolicy::restricted(entries), network_sandbox_policy, @@ -118,24 +135,71 @@ fn compile_network_sandbox_policy(network: Option<&NetworkToml>) -> NetworkSandb fn compile_filesystem_permission( path: &str, permission: &FilesystemPermissionToml, - entries: &mut Vec, + policy_cwd: &Path, startup_warnings: &mut Vec, -) -> io::Result<()> { +) -> io::Result> { + let mut entries = Vec::new(); match permission { - FilesystemPermissionToml::Access(access) => entries.push(FileSystemSandboxEntry { - path: compile_filesystem_path(path, startup_warnings)?, - access: *access, - }), + FilesystemPermissionToml::Access(access) => { + entries.push(FileSystemSandboxEntry { + path: compile_filesystem_access_path(path, *access, startup_warnings)?, + access: *access, + }); + } FilesystemPermissionToml::Scoped(scoped_entries) => { for (subpath, access) in scoped_entries { - entries.push(FileSystemSandboxEntry { - path: compile_scoped_filesystem_path(path, subpath, startup_warnings)?, - access: *access, - }); + let has_glob = contains_glob_chars(subpath); + let can_compile_as_pattern = match parse_special_path(path) { + Some(FileSystemSpecialPath::ProjectRoots { .. }) | None => true, + Some(_) => false, + }; + if has_glob && *access == FileSystemAccessMode::None && can_compile_as_pattern { + // Scoped glob syntax is a first-class filesystem policy + // pattern entry. Literal scoped paths continue through the + // exact-path parser so existing path semantics stay intact. + let entry = FileSystemSandboxEntry { + path: FileSystemPath::GlobPattern { + pattern: compile_scoped_filesystem_pattern( + path, subpath, *access, policy_cwd, + )?, + }, + access: *access, + }; + entries.push(entry); + } else { + let subpath = compile_read_write_glob_path(subpath, *access)?; + entries.push(FileSystemSandboxEntry { + path: compile_scoped_filesystem_path(path, subpath, startup_warnings)?, + access: *access, + }); + } } } } - Ok(()) + Ok(entries) +} + +fn compile_filesystem_access_path( + path: &str, + access: FileSystemAccessMode, + startup_warnings: &mut Vec, +) -> io::Result { + if !contains_glob_chars(path) { + return compile_filesystem_path(path, startup_warnings); + } + + if access == FileSystemAccessMode::None { + // At this point `path` is an unscoped filesystem table key. Top-level + // glob deny entries still go through the absolute-path parser before + // becoming policy patterns; relative project-root glob syntax is + // handled by `compile_scoped_filesystem_pattern`. + return Ok(FileSystemPath::GlobPattern { + pattern: parse_absolute_path(path)?.to_string_lossy().into_owned(), + }); + } + + let path = compile_read_write_glob_path(path, access)?; + compile_filesystem_path(path, startup_warnings) } fn compile_filesystem_path( @@ -186,6 +250,97 @@ fn compile_scoped_filesystem_path( Ok(FileSystemPath::Path { path }) } +fn compile_scoped_filesystem_pattern( + path: &str, + subpath: &str, + access: FileSystemAccessMode, + policy_cwd: &Path, +) -> io::Result { + // Pattern entries currently mean deny-read only. Supporting broader access + // modes here would imply glob-based read/write allow semantics that the + // sandbox policy does not express yet. + if access != FileSystemAccessMode::None { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("filesystem glob subpath `{subpath}` only supports `none` access"), + )); + } + let subpath = parse_relative_subpath(subpath)?; + + match parse_special_path(path) { + Some(FileSystemSpecialPath::ProjectRoots { .. }) => { + // `:project_roots` is represented as a special path, but current + // filesystem-policy resolution defines it relative to the session + // cwd. Use the same policy cwd here so glob entries and exact + // scoped entries resolve consistently. + Ok( + AbsolutePathBuf::resolve_path_against_base(&subpath, policy_cwd) + .to_string_lossy() + .to_string(), + ) + } + Some(_) => Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("filesystem path `{path}` does not support nested entries"), + )), + None => { + let base = parse_absolute_path(path)?; + Ok(base.join(&subpath).to_string_lossy().to_string()) + } + } +} + +fn compile_read_write_glob_path(path: &str, access: FileSystemAccessMode) -> io::Result<&str> { + if !contains_glob_chars(path) { + return Ok(path); + } + + let path_without_trailing_glob = remove_trailing_glob_suffix(path); + if !contains_glob_chars(path_without_trailing_glob) { + return Ok(path_without_trailing_glob); + } + + Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "filesystem glob path `{path}` only supports `none` access; use an exact path or trailing `/**` for `{access}` subtree access" + ), + )) +} + +fn unsupported_read_write_glob_paths(filesystem: &FilesystemPermissionsToml) -> Vec { + let mut patterns = Vec::new(); + for (path, permission) in &filesystem.entries { + match permission { + FilesystemPermissionToml::Access(access) => { + if *access != FileSystemAccessMode::None + && contains_glob_chars(remove_trailing_glob_suffix(path)) + { + patterns.push(path.clone()); + } + } + FilesystemPermissionToml::Scoped(scoped_entries) => { + for (subpath, access) in scoped_entries { + if *access != FileSystemAccessMode::None + && contains_glob_chars(remove_trailing_glob_suffix(subpath)) + { + patterns.push(format!("{path}/{subpath}")); + } + } + } + } + } + patterns +} + +fn contains_glob_chars(path: &str) -> bool { + path.chars().any(|ch| matches!(ch, '*' | '?' | '[' | ']')) +} + +fn remove_trailing_glob_suffix(path: &str) -> &str { + path.strip_suffix("/**").unwrap_or(path) +} + // WARNING: keep this parser forward-compatible. // Adding a new `:special_path` must not make older Codex versions reject the // config. Unknown values intentionally round-trip through diff --git a/codex-rs/core/src/config/permissions_tests.rs b/codex-rs/core/src/config/permissions_tests.rs index 34d22c2bfb..7116e32e70 100644 --- a/codex-rs/core/src/config/permissions_tests.rs +++ b/codex-rs/core/src/config/permissions_tests.rs @@ -2,6 +2,7 @@ use super::*; use crate::config::Config; use crate::config::ConfigOverrides; use codex_config::config_toml::ConfigToml; +use codex_config::permissions_toml::FilesystemPermissionToml; use codex_config::permissions_toml::FilesystemPermissionsToml; use codex_config::permissions_toml::NetworkDomainPermissionToml; use codex_config::permissions_toml::NetworkDomainPermissionsToml; @@ -10,6 +11,11 @@ use codex_config::permissions_toml::NetworkUnixSocketPermissionToml; use codex_config::permissions_toml::NetworkUnixSocketPermissionsToml; use codex_config::permissions_toml::PermissionProfileToml; use codex_config::permissions_toml::PermissionsToml; +use codex_protocol::permissions::FileSystemAccessMode; +use codex_protocol::permissions::FileSystemPath; +use codex_protocol::permissions::FileSystemSandboxEntry; +use codex_protocol::permissions::FileSystemSandboxPolicy; +use codex_protocol::permissions::FileSystemSpecialPath; use codex_utils_absolute_path::AbsolutePathBuf; use pretty_assertions::assert_eq; use std::collections::BTreeMap; @@ -216,3 +222,89 @@ fn network_toml_overlays_unix_socket_permissions_by_path() { }) ); } + +#[test] +fn read_write_glob_warnings_skip_supported_deny_read_globs_and_trailing_subpaths() { + let filesystem = FilesystemPermissionsToml { + entries: BTreeMap::from([ + ( + "/tmp/**/*.log".to_string(), + FilesystemPermissionToml::Access(FileSystemAccessMode::Read), + ), + ( + "/tmp/cache/**".to_string(), + FilesystemPermissionToml::Access(FileSystemAccessMode::Write), + ), + ( + ":project_roots".to_string(), + FilesystemPermissionToml::Scoped(BTreeMap::from([ + ("**/*.env".to_string(), FileSystemAccessMode::None), + ("docs/**".to_string(), FileSystemAccessMode::Read), + ("src/**/*.rs".to_string(), FileSystemAccessMode::Write), + ])), + ), + ]), + }; + + assert_eq!( + unsupported_read_write_glob_paths(&filesystem), + vec![ + "/tmp/**/*.log".to_string(), + ":project_roots/src/**/*.rs".to_string() + ], + "`none` glob patterns are supported as deny-read rules; only `read`/`write` globs should warn" + ); +} + +#[test] +fn read_write_trailing_glob_suffix_compiles_as_subpath() -> std::io::Result<()> { + let cwd = TempDir::new()?; + let mut startup_warnings = Vec::new(); + let (file_system_policy, _) = compile_permission_profile( + &PermissionsToml { + entries: BTreeMap::from([( + "workspace".to_string(), + PermissionProfileToml { + filesystem: Some(FilesystemPermissionsToml { + entries: BTreeMap::from([( + ":project_roots".to_string(), + FilesystemPermissionToml::Scoped(BTreeMap::from([( + "docs/**".to_string(), + FileSystemAccessMode::Read, + )])), + )]), + }), + network: None, + }, + )]), + }, + "workspace", + cwd.path(), + &mut startup_warnings, + )?; + + assert_eq!( + file_system_policy, + FileSystemSandboxPolicy::restricted(vec![FileSystemSandboxEntry { + path: FileSystemPath::Special { + value: FileSystemSpecialPath::project_roots(Some("docs".into())), + }, + access: FileSystemAccessMode::Read, + }]), + "trailing /** should compile as a subtree path instead of a glob pattern" + ); + Ok(()) +} + +#[test] +fn read_write_glob_patterns_still_reject_non_subpath_globs() { + let err = compile_read_write_glob_path("src/**/*.rs", FileSystemAccessMode::Read) + .expect_err("non-subpath read/write glob should be rejected"); + + assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); + assert!( + err.to_string() + .contains("filesystem glob path `src/**/*.rs` only supports `none` access"), + "{err}" + ); +} diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rs index 9720492c1f..ff71b797be 100644 --- a/codex-rs/core/src/context_manager/history_tests.rs +++ b/codex-rs/core/src/context_manager/history_tests.rs @@ -133,6 +133,7 @@ fn reference_context_item() -> TurnContextItem { approval_policy: AskForApproval::OnRequest, sandbox_policy: SandboxPolicy::new_read_only_policy(), network: None, + file_system_sandbox_policy: None, model: "gpt-test".to_string(), personality: None, collaboration_mode: None, diff --git a/codex-rs/core/src/exec.rs b/codex-rs/core/src/exec.rs index 10a1dc40ff..396b7701be 100644 --- a/codex-rs/core/src/exec.rs +++ b/codex-rs/core/src/exec.rs @@ -981,6 +981,9 @@ pub(crate) fn resolve_windows_restricted_token_filesystem_overrides( if !file_system_sandbox_policy .get_unreadable_roots_with_cwd(sandbox_policy_cwd) .is_empty() + || !file_system_sandbox_policy + .get_unreadable_globs_with_cwd(sandbox_policy_cwd) + .is_empty() { return Err( "windows unelevated restricted-token sandbox cannot enforce unreadable split filesystem carveouts directly; refusing to run unsandboxed" @@ -1096,6 +1099,9 @@ pub(crate) fn resolve_windows_elevated_filesystem_overrides( if !file_system_sandbox_policy .get_unreadable_roots_with_cwd(sandbox_policy_cwd) .is_empty() + || !file_system_sandbox_policy + .get_unreadable_globs_with_cwd(sandbox_policy_cwd) + .is_empty() { return Err( "windows elevated sandbox cannot enforce unreadable split filesystem carveouts directly; refusing to run unsandboxed" diff --git a/codex-rs/core/src/exec_tests.rs b/codex-rs/core/src/exec_tests.rs index acc92f5585..1cfa87ff3f 100644 --- a/codex-rs/core/src/exec_tests.rs +++ b/codex-rs/core/src/exec_tests.rs @@ -806,6 +806,53 @@ fn windows_elevated_rejects_unreadable_split_carveouts() { ); } +#[test] +fn windows_elevated_rejects_unreadable_globs() { + let temp_dir = tempfile::TempDir::new().expect("tempdir"); + let policy = SandboxPolicy::WorkspaceWrite { + writable_roots: vec![], + read_only_access: codex_protocol::protocol::ReadOnlyAccess::FullAccess, + network_access: false, + exclude_tmpdir_env_var: true, + exclude_slash_tmp: true, + }; + let file_system_policy = FileSystemSandboxPolicy::restricted(vec![ + codex_protocol::permissions::FileSystemSandboxEntry { + path: codex_protocol::permissions::FileSystemPath::Special { + value: codex_protocol::permissions::FileSystemSpecialPath::Root, + }, + access: codex_protocol::permissions::FileSystemAccessMode::Read, + }, + codex_protocol::permissions::FileSystemSandboxEntry { + path: codex_protocol::permissions::FileSystemPath::Special { + value: codex_protocol::permissions::FileSystemSpecialPath::CurrentWorkingDirectory, + }, + access: codex_protocol::permissions::FileSystemAccessMode::Write, + }, + codex_protocol::permissions::FileSystemSandboxEntry { + path: codex_protocol::permissions::FileSystemPath::GlobPattern { + pattern: "**/*.env".to_string(), + }, + access: codex_protocol::permissions::FileSystemAccessMode::None, + }, + ]); + + assert_eq!( + unsupported_windows_restricted_token_sandbox_reason( + SandboxType::WindowsRestrictedToken, + &policy, + &file_system_policy, + NetworkSandboxPolicy::Restricted, + &temp_dir.path().abs(), + WindowsSandboxLevel::Elevated, + ), + Some( + "windows elevated sandbox cannot enforce unreadable split filesystem carveouts directly; refusing to run unsandboxed" + .to_string() + ) + ); +} + #[test] fn windows_elevated_rejects_reopened_writable_descendants() { let temp_dir = tempfile::TempDir::new().expect("tempdir"); diff --git a/codex-rs/core/src/tools/handlers/list_dir.rs b/codex-rs/core/src/tools/handlers/list_dir.rs index be4e356c5c..22909c1d18 100644 --- a/codex-rs/core/src/tools/handlers/list_dir.rs +++ b/codex-rs/core/src/tools/handlers/list_dir.rs @@ -4,6 +4,7 @@ use std::fs::FileType; use std::path::Path; use std::path::PathBuf; +use codex_protocol::permissions::ReadDenyMatcher; use codex_utils_string::take_bytes_at_char_boundary; use serde::Deserialize; use tokio::fs; @@ -18,6 +19,8 @@ use crate::tools::registry::ToolKind; pub struct ListDirHandler; +const DENY_READ_POLICY_MESSAGE: &str = + "access denied: reading this path is blocked by filesystem deny_read policy"; const MAX_ENTRY_LENGTH: usize = 500; const INDENTATION_SPACES: usize = 2; @@ -52,7 +55,7 @@ impl ToolHandler for ListDirHandler { } async fn handle(&self, invocation: ToolInvocation) -> Result { - let ToolInvocation { payload, .. } = invocation; + let ToolInvocation { payload, turn, .. } = invocation; let arguments = match payload { ToolPayload::Function { arguments } => arguments, @@ -96,8 +99,20 @@ impl ToolHandler for ListDirHandler { "dir_path must be an absolute path".to_string(), )); } + let read_deny_matcher = ReadDenyMatcher::new(&turn.file_system_sandbox_policy, &turn.cwd); + if read_deny_matcher + .as_ref() + .is_some_and(|matcher| matcher.is_read_denied(&path)) + { + return Err(FunctionCallError::RespondToModel(format!( + "{DENY_READ_POLICY_MESSAGE}: `{}`", + path.display() + ))); + } - let entries = list_dir_slice(&path, offset, limit, depth).await?; + let entries = + list_dir_slice_with_policy(&path, offset, limit, depth, read_deny_matcher.as_ref()) + .await?; let mut output = Vec::with_capacity(entries.len() + 1); output.push(format!("Absolute path: {}", path.display())); output.extend(entries); @@ -105,14 +120,15 @@ impl ToolHandler for ListDirHandler { } } -async fn list_dir_slice( +async fn list_dir_slice_with_policy( path: &Path, offset: usize, limit: usize, depth: usize, + read_deny_matcher: Option<&ReadDenyMatcher>, ) -> Result, FunctionCallError> { let mut entries = Vec::new(); - collect_entries(path, Path::new(""), depth, &mut entries).await?; + collect_entries(path, Path::new(""), depth, read_deny_matcher, &mut entries).await?; if entries.is_empty() { return Ok(Vec::new()); @@ -148,6 +164,7 @@ async fn collect_entries( dir_path: &Path, relative_prefix: &Path, depth: usize, + read_deny_matcher: Option<&ReadDenyMatcher>, entries: &mut Vec, ) -> Result<(), FunctionCallError> { let mut queue = VecDeque::new(); @@ -163,6 +180,13 @@ async fn collect_entries( while let Some(entry) = read_dir.next_entry().await.map_err(|err| { FunctionCallError::RespondToModel(format!("failed to read directory: {err}")) })? { + let entry_path = entry.path(); + if let Some(read_deny_matcher) = read_deny_matcher + && read_deny_matcher.is_read_denied(&entry_path) + { + continue; + } + let file_type = entry.file_type().await.map_err(|err| { FunctionCallError::RespondToModel(format!("failed to inspect entry: {err}")) })?; @@ -179,7 +203,7 @@ async fn collect_entries( let sort_key = format_entry_name(&relative_path); let kind = DirEntryKind::from(&file_type); dir_entries.push(( - entry.path(), + entry_path, relative_path, kind, DirEntry { diff --git a/codex-rs/core/src/tools/handlers/list_dir_tests.rs b/codex-rs/core/src/tools/handlers/list_dir_tests.rs index ccc30b5326..7de107a18d 100644 --- a/codex-rs/core/src/tools/handlers/list_dir_tests.rs +++ b/codex-rs/core/src/tools/handlers/list_dir_tests.rs @@ -1,7 +1,21 @@ use super::*; +use codex_protocol::permissions::FileSystemAccessMode; +use codex_protocol::permissions::FileSystemPath; +use codex_protocol::permissions::FileSystemSandboxEntry; +use codex_protocol::permissions::FileSystemSandboxPolicy; +use codex_protocol::permissions::ReadDenyMatcher; use pretty_assertions::assert_eq; use tempfile::tempdir; +async fn list_dir_slice( + path: &Path, + offset: usize, + limit: usize, + depth: usize, +) -> Result, FunctionCallError> { + list_dir_slice_with_policy(path, offset, limit, depth, /*read_deny_matcher*/ None).await +} + #[tokio::test] async fn lists_directory_entries() { let temp = tempdir().expect("create tempdir"); @@ -258,3 +272,60 @@ async fn truncation_respects_sorted_order() -> anyhow::Result<()> { Ok(()) } + +#[tokio::test] +async fn hides_denied_entries_and_prunes_denied_subtrees() { + let temp = tempdir().expect("create tempdir"); + let dir_path = temp.path(); + let visible_dir = dir_path.join("visible"); + let denied_dir = dir_path.join("private"); + tokio::fs::create_dir(&visible_dir) + .await + .expect("create visible dir"); + tokio::fs::create_dir(&denied_dir) + .await + .expect("create denied dir"); + tokio::fs::write(visible_dir.join("ok.txt"), b"ok") + .await + .expect("write visible file"); + tokio::fs::write(denied_dir.join("secret.txt"), b"secret") + .await + .expect("write denied file"); + tokio::fs::write(dir_path.join("top_secret.txt"), b"secret") + .await + .expect("write denied top-level file"); + + let policy = FileSystemSandboxPolicy::restricted(vec![ + FileSystemSandboxEntry { + path: FileSystemPath::Path { + path: denied_dir.try_into().expect("absolute denied dir"), + }, + access: FileSystemAccessMode::None, + }, + FileSystemSandboxEntry { + path: FileSystemPath::Path { + path: dir_path + .join("top_secret.txt") + .try_into() + .expect("absolute denied file"), + }, + access: FileSystemAccessMode::None, + }, + ]); + + let read_deny_matcher = ReadDenyMatcher::new(&policy, dir_path); + let entries = list_dir_slice_with_policy( + dir_path, + /*offset*/ 1, + /*limit*/ 20, + /*depth*/ 3, + read_deny_matcher.as_ref(), + ) + .await + .expect("list directory"); + + assert_eq!( + entries, + vec!["visible/".to_string(), " ok.txt".to_string(),] + ); +} diff --git a/codex-rs/core/tests/suite/resume_warning.rs b/codex-rs/core/tests/suite/resume_warning.rs index f9ad64c771..8ea2be3147 100644 --- a/codex-rs/core/tests/suite/resume_warning.rs +++ b/codex-rs/core/tests/suite/resume_warning.rs @@ -34,6 +34,7 @@ fn resume_history( approval_policy: config.permissions.approval_policy.value(), sandbox_policy: config.permissions.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: previous_model.to_string(), personality: None, collaboration_mode: None, diff --git a/codex-rs/protocol/Cargo.toml b/codex-rs/protocol/Cargo.toml index c4249a13b4..f70dc94585 100644 --- a/codex-rs/protocol/Cargo.toml +++ b/codex-rs/protocol/Cargo.toml @@ -23,6 +23,7 @@ codex-utils-image = { workspace = true } codex-utils-string = { workspace = true } codex-utils-template = { workspace = true } encoding_rs = { workspace = true } +globset = { workspace = true } icu_decimal = { workspace = true } icu_locale_core = { workspace = true } icu_provider = { workspace = true, features = ["sync"] } diff --git a/codex-rs/protocol/src/permissions.rs b/codex-rs/protocol/src/permissions.rs index f101f8bef0..d774f6dc0d 100644 --- a/codex-rs/protocol/src/permissions.rs +++ b/codex-rs/protocol/src/permissions.rs @@ -6,6 +6,8 @@ use std::path::PathBuf; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_absolute_path::canonicalize_preserving_symlinks; +use globset::GlobBuilder; +use globset::GlobMatcher; use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; @@ -154,14 +156,101 @@ struct FileSystemSemanticSignature { readable_roots: Vec, writable_roots: Vec, unreadable_roots: Vec, + unreadable_globs: Vec, +} + +/// Runtime matcher for read-deny entries in a filesystem sandbox policy. +pub struct ReadDenyMatcher { + denied_candidates: Vec>, + deny_read_matchers: Vec, + invalid_pattern: bool, +} + +impl ReadDenyMatcher { + /// Builds a matcher from exact deny-read roots and deny-read glob entries. + /// + /// Returns `None` when the policy has no deny-read restrictions, so callers + /// can skip read-deny checks without allocating matcher state. The `cwd` + /// resolves cwd-relative policy paths and special paths before matching. + pub fn new(file_system_sandbox_policy: &FileSystemSandboxPolicy, cwd: &Path) -> Option { + if !file_system_sandbox_policy.has_denied_read_restrictions() { + return None; + } + + // Exact roots are stored as all meaningful path spellings we can derive + // cheaply. This lets direct tool checks catch both a symlink path and + // its canonical target without changing the policy entries themselves. + let denied_candidates = file_system_sandbox_policy + .get_unreadable_roots_with_cwd(cwd) + .into_iter() + .map(|path| normalized_and_canonical_candidates(path.as_path())) + .collect(); + // Pattern entries stay as policy-level globs. They are matched at read + // time here instead of being snapshotted to startup filesystem state. + let mut invalid_pattern = false; + let deny_read_matchers = file_system_sandbox_policy + .get_unreadable_globs_with_cwd(cwd) + .into_iter() + .filter_map(|pattern| match build_glob_matcher(&pattern) { + Some(matcher) => Some(matcher), + None => { + invalid_pattern = true; + None + } + }) + .collect(); + Some(Self { + denied_candidates, + deny_read_matchers, + invalid_pattern, + }) + } + + /// Returns whether `path` is denied by the policy used to build this matcher. + pub fn is_read_denied(&self, path: &Path) -> bool { + if self.invalid_pattern { + // Direct tool reads fail closed on malformed deny patterns. Silent + // allow would turn a config typo into a policy bypass. + return true; + } + + // Check exact roots against each candidate spelling before evaluating + // glob matchers. Exact entries are subtree denies; glob entries match + // according to the pattern compiler's path-separator rules. + let path_candidates = normalized_and_canonical_candidates(path); + if self.denied_candidates.iter().any(|denied_candidates| { + path_candidates.iter().any(|candidate| { + denied_candidates.iter().any(|denied_candidate| { + candidate == denied_candidate || candidate.starts_with(denied_candidate) + }) + }) + }) { + return true; + } + + self.deny_read_matchers.iter().any(|matcher| { + path_candidates + .iter() + .any(|candidate| matcher.is_match(candidate)) + }) + } } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema, TS)] #[serde(tag = "type", rename_all = "snake_case")] #[ts(tag = "type")] pub enum FileSystemPath { - Path { path: AbsolutePathBuf }, - Special { value: FileSystemSpecialPath }, + Path { + path: AbsolutePathBuf, + }, + /// A git-style glob pattern. Pattern entries currently support + /// FileSystemAccessMode::None only. + GlobPattern { + pattern: String, + }, + Special { + value: FileSystemSpecialPath, + }, } impl Default for FileSystemSandboxPolicy { @@ -179,62 +268,6 @@ impl Default for FileSystemSandboxPolicy { } impl FileSystemSandboxPolicy { - fn has_root_access(&self, predicate: impl Fn(FileSystemAccessMode) -> bool) -> bool { - matches!(self.kind, FileSystemSandboxKind::Restricted) - && self.entries.iter().any(|entry| { - matches!( - &entry.path, - FileSystemPath::Special { value } - if matches!(value, FileSystemSpecialPath::Root) && predicate(entry.access) - ) - }) - } - - fn has_explicit_deny_entries(&self) -> bool { - matches!(self.kind, FileSystemSandboxKind::Restricted) - && self - .entries - .iter() - .any(|entry| entry.access == FileSystemAccessMode::None) - } - - /// Returns true when a restricted policy contains any entry that really - /// reduces a broader `:root = write` grant. - /// - /// Raw entry presence is not enough here: an equally specific `write` - /// entry for the same target wins under the normal precedence rules, so a - /// shadowed `read` entry must not downgrade the policy out of full-disk - /// write mode. - fn has_write_narrowing_entries(&self) -> bool { - matches!(self.kind, FileSystemSandboxKind::Restricted) - && self.entries.iter().any(|entry| { - if entry.access.can_write() { - return false; - } - - match &entry.path { - FileSystemPath::Path { .. } => !self.has_same_target_write_override(entry), - FileSystemPath::Special { value } => match value { - FileSystemSpecialPath::Root => entry.access == FileSystemAccessMode::None, - FileSystemSpecialPath::Minimal | FileSystemSpecialPath::Unknown { .. } => { - false - } - _ => !self.has_same_target_write_override(entry), - }, - } - }) - } - - /// Returns true when a higher-priority `write` entry targets the same - /// location as `entry`, so `entry` cannot narrow effective write access. - fn has_same_target_write_override(&self, entry: &FileSystemSandboxEntry) -> bool { - self.entries.iter().any(|candidate| { - candidate.access.can_write() - && candidate.access > entry.access - && file_system_paths_share_target(&candidate.path, &entry.path) - }) - } - pub fn unrestricted() -> Self { Self { kind: FileSystemSandboxKind::Unrestricted, @@ -256,6 +289,86 @@ impl FileSystemSandboxPolicy { } } + fn has_root_access(&self, predicate: impl Fn(FileSystemAccessMode) -> bool) -> bool { + matches!(self.kind, FileSystemSandboxKind::Restricted) + && self.entries.iter().any(|entry| { + matches!( + &entry.path, + FileSystemPath::Special { value } + if matches!(value, FileSystemSpecialPath::Root) && predicate(entry.access) + ) + }) + } + + pub fn has_denied_read_restrictions(&self) -> bool { + matches!(self.kind, FileSystemSandboxKind::Restricted) + && self + .entries + .iter() + .any(|entry| entry.access == FileSystemAccessMode::None) + } + + pub fn from_legacy_sandbox_policy_preserving_deny_entries( + sandbox_policy: &SandboxPolicy, + cwd: &Path, + existing: &Self, + ) -> Self { + let mut rebuilt = Self::from_legacy_sandbox_policy(sandbox_policy, cwd); + if !matches!(rebuilt.kind, FileSystemSandboxKind::Restricted) { + return rebuilt; + } + + for deny_entry in existing + .entries + .iter() + .filter(|entry| entry.access == FileSystemAccessMode::None) + { + if !rebuilt.entries.iter().any(|entry| entry == deny_entry) { + rebuilt.entries.push(deny_entry.clone()); + } + } + + rebuilt + } + + /// Returns true when a restricted policy contains any entry that really + /// reduces a broader `:root = write` grant. + /// + /// Raw entry presence is not enough here: an equally specific `write` + /// entry for the same target wins under the normal precedence rules, so a + /// shadowed `read` entry must not downgrade the policy out of full-disk + /// write mode. + fn has_write_narrowing_entries(&self) -> bool { + matches!(self.kind, FileSystemSandboxKind::Restricted) + && self.entries.iter().any(|entry| { + if entry.access.can_write() { + return false; + } + + match &entry.path { + FileSystemPath::Path { .. } => !self.has_same_target_write_override(entry), + FileSystemPath::GlobPattern { .. } => true, + FileSystemPath::Special { value } => match value { + FileSystemSpecialPath::Root => entry.access == FileSystemAccessMode::None, + FileSystemSpecialPath::Minimal | FileSystemSpecialPath::Unknown { .. } => { + false + } + _ => !self.has_same_target_write_override(entry), + }, + } + }) + } + + /// Returns true when a higher-priority `write` entry targets the same + /// location as `entry`, so `entry` cannot narrow effective write access. + fn has_same_target_write_override(&self, entry: &FileSystemSandboxEntry) -> bool { + self.entries.iter().any(|candidate| { + candidate.access.can_write() + && candidate.access > entry.access + && file_system_paths_share_target(&candidate.path, &entry.path) + }) + } + /// Converts a legacy sandbox policy into an equivalent filesystem policy /// for the provided cwd. /// @@ -276,6 +389,7 @@ impl FileSystemSandboxPolicy { FileSystemPath::Path { path } => !legacy_writable_roots .iter() .any(|root| root.is_path_writable(path.as_path())), + FileSystemPath::GlobPattern { .. } => true, FileSystemPath::Special { .. } => true, } }); @@ -312,7 +426,7 @@ impl FileSystemSandboxPolicy { FileSystemSandboxKind::Unrestricted | FileSystemSandboxKind::ExternalSandbox => true, FileSystemSandboxKind::Restricted => { self.has_root_access(FileSystemAccessMode::can_read) - && !self.has_explicit_deny_entries() + && !self.has_denied_read_restrictions() } } } @@ -585,6 +699,29 @@ impl FileSystemSandboxPolicy { ) } + /// Returns unreadable glob patterns resolved against the provided cwd. + pub fn get_unreadable_globs_with_cwd(&self, cwd: &Path) -> Vec { + if !matches!(self.kind, FileSystemSandboxKind::Restricted) { + return Vec::new(); + } + + let mut patterns = self + .entries + .iter() + .filter(|entry| entry.access == FileSystemAccessMode::None) + .filter_map(|entry| match &entry.path { + FileSystemPath::GlobPattern { pattern } => { + Some(AbsolutePathBuf::resolve_path_against_base(pattern, cwd)) + } + FileSystemPath::Path { .. } | FileSystemPath::Special { .. } => None, + }) + .map(|pattern| pattern.to_string_lossy().into_owned()) + .collect::>(); + patterns.sort(); + patterns.dedup(); + patterns + } + pub fn to_legacy_sandbox_policy( &self, network_policy: NetworkSandboxPolicy, @@ -620,6 +757,7 @@ impl FileSystemSandboxPolicy { for entry in &self.entries { match &entry.path { + FileSystemPath::GlobPattern { .. } => {} FileSystemPath::Path { path } => { if entry.access.can_write() { if cwd_absolute.as_ref().is_some_and(|cwd| cwd == path) { @@ -770,6 +908,7 @@ impl FileSystemSandboxPolicy { readable_roots: self.get_readable_roots_with_cwd(cwd), writable_roots: self.get_writable_roots_with_cwd(cwd), unreadable_roots: self.get_unreadable_roots_with_cwd(cwd), + unreadable_globs: self.get_unreadable_globs_with_cwd(cwd), } } } @@ -905,6 +1044,7 @@ fn resolve_file_system_path( ) -> Option { match path { FileSystemPath::Path { path } => Some(path.clone()), + FileSystemPath::GlobPattern { .. } => None, FileSystemPath::Special { value } => resolve_file_system_special_path(value, cwd), } } @@ -947,6 +1087,11 @@ fn file_system_paths_share_target(left: &FileSystemPath, right: &FileSystemPath) | (FileSystemPath::Special { value }, FileSystemPath::Path { path }) => { special_path_matches_absolute_path(value, path) } + ( + FileSystemPath::GlobPattern { pattern: left }, + FileSystemPath::GlobPattern { pattern: right }, + ) => left == right, + (FileSystemPath::GlobPattern { .. }, _) | (_, FileSystemPath::GlobPattern { .. }) => false, } } @@ -1021,6 +1166,44 @@ fn absolute_root_path_for_cwd(cwd: &AbsolutePathBuf) -> AbsolutePathBuf { .unwrap_or_else(|err| panic!("cwd root must be an absolute path: {err}")) } +fn normalized_and_canonical_candidates(path: &Path) -> Vec { + // Compare the lexical absolute form plus the canonical target when it + // exists. Missing paths still need the lexical candidate so future-created + // denied paths remain blocked by direct tool checks. + let mut candidates = Vec::new(); + + if let Ok(normalized) = AbsolutePathBuf::from_absolute_path(path) { + push_unique(&mut candidates, normalized.to_path_buf()); + } else { + push_unique(&mut candidates, path.to_path_buf()); + } + + if let Ok(canonical) = path.canonicalize() + && let Ok(canonical_absolute) = AbsolutePathBuf::from_absolute_path(canonical) + { + push_unique(&mut candidates, canonical_absolute.to_path_buf()); + } + + candidates +} + +fn push_unique(candidates: &mut Vec, candidate: PathBuf) { + if !candidates.iter().any(|existing| existing == &candidate) { + candidates.push(candidate); + } +} + +fn build_glob_matcher(pattern: &str) -> Option { + // Keep `*` and `?` within a single path component and preserve an unclosed + // `[` as a literal so matcher behavior stays aligned with config parsing. + GlobBuilder::new(pattern) + .literal_separator(true) + .allow_unclosed_class(true) + .build() + .ok() + .map(|glob| glob.compile_matcher()) +} + fn resolve_file_system_special_path( value: &FileSystemSpecialPath, cwd: Option<&AbsolutePathBuf>, @@ -2061,4 +2244,187 @@ mod tests { assert!(FileSystemAccessMode::Write > FileSystemAccessMode::Read); assert!(FileSystemAccessMode::None > FileSystemAccessMode::Write); } + + #[test] + fn legacy_bridge_preserves_explicit_deny_entries() { + let denied = AbsolutePathBuf::try_from("/tmp/private").expect("absolute path"); + let existing = FileSystemSandboxPolicy::restricted(vec![FileSystemSandboxEntry { + path: FileSystemPath::Path { + path: denied.clone(), + }, + access: FileSystemAccessMode::None, + }]); + + let rebuilt = FileSystemSandboxPolicy::from_legacy_sandbox_policy_preserving_deny_entries( + &SandboxPolicy::new_workspace_write_policy(), + Path::new("/tmp/workspace"), + &existing, + ); + + assert!( + rebuilt.entries.iter().any(|entry| { + entry.path + == FileSystemPath::Path { + path: denied.clone(), + } + && entry.access == FileSystemAccessMode::None + }), + "expected explicit deny entry to be preserved" + ); + } + + fn deny_policy(path: &Path) -> FileSystemSandboxPolicy { + FileSystemSandboxPolicy::restricted(vec![FileSystemSandboxEntry { + path: FileSystemPath::Path { + path: AbsolutePathBuf::try_from(path).expect("absolute deny path"), + }, + access: FileSystemAccessMode::None, + }]) + } + + fn unreadable_glob_entry(pattern: String) -> FileSystemSandboxEntry { + FileSystemSandboxEntry { + path: FileSystemPath::GlobPattern { pattern }, + access: FileSystemAccessMode::None, + } + } + + fn default_policy_with_unreadable_glob(pattern: String) -> FileSystemSandboxPolicy { + let mut policy = FileSystemSandboxPolicy::default(); + policy.entries.push(unreadable_glob_entry(pattern)); + policy + } + + fn is_read_denied( + path: &Path, + file_system_sandbox_policy: &FileSystemSandboxPolicy, + cwd: &Path, + ) -> bool { + ReadDenyMatcher::new(file_system_sandbox_policy, cwd) + .is_some_and(|matcher| matcher.is_read_denied(path)) + } + + #[test] + fn exact_path_and_descendants_are_denied() { + let temp = TempDir::new().expect("tempdir"); + let denied_dir = temp.path().join("denied"); + let nested = denied_dir.join("nested.txt"); + std::fs::create_dir_all(&denied_dir).expect("create denied dir"); + std::fs::write(&nested, "secret").expect("write secret"); + + let policy = deny_policy(&denied_dir); + assert!(is_read_denied(&denied_dir, &policy, temp.path())); + assert!(is_read_denied(&nested, &policy, temp.path())); + assert!(!is_read_denied( + &temp.path().join("other.txt"), + &policy, + temp.path() + )); + } + + #[cfg(unix)] + #[test] + fn canonical_target_matches_denied_symlink_alias() { + let temp = TempDir::new().expect("tempdir"); + let real_dir = temp.path().join("real"); + let alias_dir = temp.path().join("alias"); + std::fs::create_dir_all(&real_dir).expect("create real dir"); + symlink_dir(&real_dir, &alias_dir).expect("symlink alias"); + + let secret = real_dir.join("secret.txt"); + std::fs::write(&secret, "secret").expect("write secret"); + let alias_secret = alias_dir.join("secret.txt"); + + let policy = deny_policy(&real_dir); + assert!(is_read_denied(&alias_secret, &policy, temp.path())); + } + + #[test] + fn literal_patterns_and_globs_are_denied() { + let temp = TempDir::new().expect("tempdir"); + let literal = temp.path().join("private"); + let other = temp.path().join("notes.txt"); + std::fs::create_dir_all(&literal).expect("create literal dir"); + std::fs::write(&other, "notes").expect("write notes"); + + let mut policy = deny_policy(&literal); + policy.entries.push(unreadable_glob_entry(format!( + "{}/**/*.txt", + temp.path().display() + ))); + + assert!(is_read_denied(&literal, &policy, temp.path())); + assert!(is_read_denied(&other, &policy, temp.path())); + } + + #[test] + fn glob_patterns_deny_matching_paths() { + let temp = TempDir::new().expect("tempdir"); + let denied = temp.path().join("private").join("secret1.txt"); + std::fs::create_dir_all(denied.parent().expect("parent")).expect("create parent"); + std::fs::write(&denied, "secret").expect("write secret"); + + let policy = default_policy_with_unreadable_glob(format!( + "{}/private/secret?.txt", + temp.path().display() + )); + + assert!(is_read_denied(&denied, &policy, temp.path())); + } + + #[test] + fn glob_patterns_do_not_cross_path_separators() { + let temp = TempDir::new().expect("tempdir"); + let matching = temp.path().join("app").join("file42.txt"); + let nested = temp.path().join("app").join("nested").join("file42.txt"); + let short = temp.path().join("app").join("file4.txt"); + let letters = temp.path().join("app").join("fileab.txt"); + std::fs::create_dir_all(nested.parent().expect("parent")).expect("create parent"); + std::fs::write(&matching, "secret").expect("write matching"); + std::fs::write(&nested, "secret").expect("write nested"); + std::fs::write(&short, "secret").expect("write short"); + std::fs::write(&letters, "secret").expect("write letters"); + + let policy = default_policy_with_unreadable_glob(format!( + "{}/*/file[0-9]?.txt", + temp.path().display() + )); + + assert!(is_read_denied(&matching, &policy, temp.path())); + assert!(!is_read_denied(&nested, &policy, temp.path())); + assert!(!is_read_denied(&short, &policy, temp.path())); + assert!(!is_read_denied(&letters, &policy, temp.path())); + } + + #[test] + fn globstar_patterns_deny_root_and_nested_matches() { + let temp = TempDir::new().expect("tempdir"); + let root_env = temp.path().join(".env"); + let nested_env = temp.path().join("app").join(".env"); + let other = temp.path().join("app").join("notes.txt"); + std::fs::create_dir_all(nested_env.parent().expect("parent")).expect("create parent"); + std::fs::write(&root_env, "secret").expect("write root env"); + std::fs::write(&nested_env, "secret").expect("write nested env"); + std::fs::write(&other, "notes").expect("write notes"); + + let policy = + default_policy_with_unreadable_glob(format!("{}/**/*.env", temp.path().display())); + + assert!(is_read_denied(&root_env, &policy, temp.path())); + assert!(is_read_denied(&nested_env, &policy, temp.path())); + assert!(!is_read_denied(&other, &policy, temp.path())); + } + + #[test] + fn unclosed_character_classes_match_literal_brackets() { + let temp = TempDir::new().expect("tempdir"); + let bracket_file = temp.path().join("["); + let other = temp.path().join("notes.txt"); + std::fs::write(&bracket_file, "secret").expect("write bracket file"); + std::fs::write(&other, "notes").expect("write notes"); + let policy = default_policy_with_unreadable_glob(format!("{}/[", temp.path().display())); + + assert!(is_read_denied(&bracket_file, &policy, temp.path())); + assert!(!is_read_denied(&other, &policy, temp.path())); + } } diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index fb1ea592da..8aea3544d4 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -2844,6 +2844,8 @@ pub struct TurnContextItem { pub sandbox_policy: SandboxPolicy, #[serde(skip_serializing_if = "Option::is_none")] pub network: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub file_system_sandbox_policy: Option, pub model: String, #[serde(skip_serializing_if = "Option::is_none")] pub personality: Option, @@ -4974,6 +4976,7 @@ mod tests { assert_eq!(item.trace_id, None); assert_eq!(item.network, None); + assert_eq!(item.file_system_sandbox_policy, None); Ok(()) } @@ -4991,6 +4994,14 @@ mod tests { allowed_domains: vec!["api.example.com".to_string()], denied_domains: vec!["blocked.example.com".to_string()], }), + file_system_sandbox_policy: Some(FileSystemSandboxPolicy::restricted(vec![ + FileSystemSandboxEntry { + path: FileSystemPath::GlobPattern { + pattern: "/tmp/private/**/*.txt".to_string(), + }, + access: FileSystemAccessMode::None, + }, + ])), model: "gpt-5".to_string(), personality: None, collaboration_mode: None, @@ -5011,6 +5022,19 @@ mod tests { "denied_domains": ["blocked.example.com"], }) ); + assert_eq!( + value["file_system_sandbox_policy"], + json!({ + "kind": "restricted", + "entries": [{ + "path": { + "type": "glob_pattern", + "pattern": "/tmp/private/**/*.txt" + }, + "access": "none" + }] + }) + ); Ok(()) } diff --git a/codex-rs/rollout/src/recorder_tests.rs b/codex-rs/rollout/src/recorder_tests.rs index 3e2ac52194..e3d82c4454 100644 --- a/codex-rs/rollout/src/recorder_tests.rs +++ b/codex-rs/rollout/src/recorder_tests.rs @@ -544,6 +544,7 @@ async fn resume_candidate_matches_cwd_reads_latest_turn_context() -> std::io::Re approval_policy: AskForApproval::Never, sandbox_policy: SandboxPolicy::new_read_only_policy(), network: None, + file_system_sandbox_policy: None, model: "test-model".to_string(), personality: None, collaboration_mode: None, diff --git a/codex-rs/sandboxing/src/lib.rs b/codex-rs/sandboxing/src/lib.rs index 38aed6ce24..dbb8323649 100644 --- a/codex-rs/sandboxing/src/lib.rs +++ b/codex-rs/sandboxing/src/lib.rs @@ -34,6 +34,9 @@ impl From for CodexErr { SandboxTransformError::MissingLinuxSandboxExecutable => { CodexErr::LandlockSandboxExecutableNotProvided } + SandboxTransformError::UnreadableGlobPatternsUnsupported => { + CodexErr::UnsupportedOperation(err.to_string()) + } #[cfg(target_os = "linux")] SandboxTransformError::Wsl1UnsupportedForBubblewrap => { CodexErr::UnsupportedOperation(crate::bwrap::WSL1_BWRAP_WARNING.to_string()) diff --git a/codex-rs/sandboxing/src/manager.rs b/codex-rs/sandboxing/src/manager.rs index 0836eb2565..1e3ea9c196 100644 --- a/codex-rs/sandboxing/src/manager.rs +++ b/codex-rs/sandboxing/src/manager.rs @@ -109,6 +109,7 @@ pub struct SandboxTransformRequest<'a> { #[derive(Debug)] pub enum SandboxTransformError { MissingLinuxSandboxExecutable, + UnreadableGlobPatternsUnsupported, #[cfg(target_os = "linux")] Wsl1UnsupportedForBubblewrap, #[cfg(not(target_os = "macos"))] @@ -121,6 +122,10 @@ impl std::fmt::Display for SandboxTransformError { Self::MissingLinuxSandboxExecutable => { write!(f, "missing codex-linux-sandbox executable path") } + Self::UnreadableGlobPatternsUnsupported => write!( + f, + "platform sandbox backend cannot enforce unreadable glob patterns" + ), #[cfg(target_os = "linux")] Self::Wsl1UnsupportedForBubblewrap => write!(f, "{WSL1_BWRAP_WARNING}"), #[cfg(not(target_os = "macos"))] @@ -196,6 +201,15 @@ impl SandboxManager { ); let effective_network_policy = effective_network_sandbox_policy(network_policy, additional_permissions.as_ref()); + if matches!( + sandbox, + SandboxType::MacosSeatbelt | SandboxType::LinuxSeccomp + ) && !effective_file_system_policy + .get_unreadable_globs_with_cwd(sandbox_policy_cwd) + .is_empty() + { + return Err(SandboxTransformError::UnreadableGlobPatternsUnsupported); + } let mut argv = Vec::with_capacity(1 + command.args.len()); argv.push(command.program); argv.extend(command.args.into_iter().map(OsString::from)); diff --git a/codex-rs/state/src/extract.rs b/codex-rs/state/src/extract.rs index 91c6755ffe..6d36f211a4 100644 --- a/codex-rs/state/src/extract.rs +++ b/codex-rs/state/src/extract.rs @@ -302,6 +302,7 @@ mod tests { approval_policy: AskForApproval::Never, sandbox_policy: SandboxPolicy::DangerFullAccess, network: None, + file_system_sandbox_policy: None, model: "gpt-5".to_string(), personality: None, collaboration_mode: None, @@ -340,6 +341,7 @@ mod tests { approval_policy: AskForApproval::OnRequest, sandbox_policy: SandboxPolicy::new_read_only_policy(), network: None, + file_system_sandbox_policy: None, model: "gpt-5".to_string(), personality: None, collaboration_mode: None, @@ -372,6 +374,7 @@ mod tests { approval_policy: AskForApproval::OnRequest, sandbox_policy: SandboxPolicy::new_read_only_policy(), network: None, + file_system_sandbox_policy: None, model: "gpt-5".to_string(), personality: None, collaboration_mode: None, diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 5641131566..1db3dbc7ed 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -9239,6 +9239,7 @@ guardian_approval = true approval_policy: primary_session.approval_policy, sandbox_policy: primary_session.sandbox_policy.clone(), network: None, + file_system_sandbox_policy: None, model: "gpt-agent".to_string(), personality: None, collaboration_mode: None, diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index cf938545c6..a5efcd1ea2 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -2249,6 +2249,7 @@ mod tests { approval_policy: config.permissions.approval_policy.value(), sandbox_policy: config.permissions.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model, personality: None, collaboration_mode: None, From 109b22a8d024084cd86704dc183c6616891fac96 Mon Sep 17 00:00:00 2001 From: alexsong-oai Date: Thu, 16 Apr 2026 10:34:38 -0700 Subject: [PATCH 80/84] Improve external agent plugin migration for configured marketplaces (#18055) --- .../src/external_agent_config_api.rs | 1 + codex-rs/core/src/external_agent_config.rs | 253 +++-- .../core/src/external_agent_config_tests.rs | 989 ++++++++++++++++-- codex-rs/core/src/plugins/manager.rs | 2 +- codex-rs/core/src/plugins/marketplace_add.rs | 2 +- .../src/plugins/marketplace_add/source.rs | 4 +- codex-rs/core/src/plugins/mod.rs | 3 + 7 files changed, 1111 insertions(+), 143 deletions(-) diff --git a/codex-rs/app-server/src/external_agent_config_api.rs b/codex-rs/app-server/src/external_agent_config_api.rs index 3e20632db2..2f90a55a13 100644 --- a/codex-rs/app-server/src/external_agent_config_api.rs +++ b/codex-rs/app-server/src/external_agent_config_api.rs @@ -37,6 +37,7 @@ impl ExternalAgentConfigApi { include_home: params.include_home, cwds: params.cwds, }) + .await .map_err(map_io_error)?; Ok(ExternalAgentConfigDetectResponse { diff --git a/codex-rs/core/src/external_agent_config.rs b/codex-rs/core/src/external_agent_config.rs index 18305a6262..f0f4443a75 100644 --- a/codex-rs/core/src/external_agent_config.rs +++ b/codex-rs/core/src/external_agent_config.rs @@ -1,8 +1,15 @@ +use crate::config::Config; +use crate::config::ConfigBuilder; use crate::plugins::MarketplaceAddRequest; use crate::plugins::PluginId; use crate::plugins::PluginInstallRequest; use crate::plugins::PluginsManager; use crate::plugins::add_marketplace; +use crate::plugins::configured_plugins_from_stack; +use crate::plugins::find_marketplace_manifest_path; +use crate::plugins::parse_marketplace_source; +use codex_core_plugins::marketplace::MarketplacePluginInstallPolicy; +use codex_protocol::protocol::Product; use serde_json::Value as JsonValue; use std::collections::BTreeMap; use std::collections::HashSet; @@ -15,8 +22,8 @@ use toml::Value as TomlValue; const EXTERNAL_AGENT_CONFIG_DETECT_METRIC: &str = "codex.external_agent_config.detect"; const EXTERNAL_AGENT_CONFIG_IMPORT_METRIC: &str = "codex.external_agent_config.import"; -// Installed marketplace roots always expose their manifest at this relative path. -const INSTALLED_MARKETPLACE_MANIFEST_RELATIVE_PATH: &str = ".agents/plugins/marketplace.json"; +const EXTERNAL_AGENT_DIR: &str = ".claude"; +const EXTERNAL_AGENT_CONFIG_MD: &str = "CLAUDE.md"; #[derive(Debug, Clone, PartialEq, Eq)] pub struct ExternalAgentConfigDetectOptions { @@ -83,20 +90,21 @@ impl ExternalAgentConfigService { } } - pub fn detect( + pub async fn detect( &self, params: ExternalAgentConfigDetectOptions, ) -> io::Result> { let mut items = Vec::new(); if params.include_home { - self.detect_migrations(/*repo_root*/ None, &mut items)?; + self.detect_migrations(/*repo_root*/ None, &mut items) + .await?; } for cwd in params.cwds.as_deref().unwrap_or(&[]) { let Some(repo_root) = find_repo_root(Some(cwd))? else { continue; }; - self.detect_migrations(Some(&repo_root), &mut items)?; + self.detect_migrations(Some(&repo_root), &mut items).await?; } Ok(items) @@ -137,22 +145,11 @@ impl ExternalAgentConfigService { let cwd = migration_item.cwd; let details = migration_item.details; tokio::spawn(async move { - match service.import_plugins(cwd.as_deref(), details).await { - Ok(outcome) => { - tracing::info!( - succeeded_marketplaces = outcome.succeeded_marketplaces.len(), - succeeded_plugin_ids = outcome.succeeded_plugin_ids.len(), - failed_marketplaces = outcome.failed_marketplaces.len(), - failed_plugin_ids = outcome.failed_plugin_ids.len(), - "external agent config plugin import completed" - ); - } - Err(err) => { - tracing::warn!( - error = %err, - "external agent config plugin import failed" - ); - } + if let Err(err) = service.import_plugins(cwd.as_deref(), details).await { + tracing::warn!( + error = %err, + "external agent config plugin import failed" + ); } }); emit_migration_metric( @@ -168,7 +165,7 @@ impl ExternalAgentConfigService { Ok(()) } - fn detect_migrations( + async fn detect_migrations( &self, repo_root: Option<&Path>, items: &mut Vec, @@ -176,7 +173,7 @@ impl ExternalAgentConfigService { let cwd = repo_root.map(Path::to_path_buf); let source_settings = repo_root.map_or_else( || self.external_agent_home.join("settings.json"), - |repo_root| repo_root.join(".claude").join("settings.json"), + |repo_root| repo_root.join(EXTERNAL_AGENT_DIR).join("settings.json"), ); let settings = read_external_settings(&source_settings)?; let target_config = repo_root.map_or_else( @@ -219,16 +216,9 @@ impl ExternalAgentConfigService { } } - self.detect_plugin_migration( - source_settings.as_path(), - cwd.clone(), - settings.as_ref(), - items, - ); - let source_skills = repo_root.map_or_else( || self.external_agent_home.join("skills"), - |repo_root| repo_root.join(".claude").join("skills"), + |repo_root| repo_root.join(EXTERNAL_AGENT_DIR).join("skills"), ); let target_skills = repo_root.map_or_else( || self.home_target_skills_dir(), @@ -239,7 +229,7 @@ impl ExternalAgentConfigService { items.push(ExternalAgentConfigMigrationItem { item_type: ExternalAgentConfigMigrationItemType::Skills, description: format!( - "Copy skill folders from {} to {}", + "Migrate skills from {} to {}", source_skills.display(), target_skills.display() ), @@ -256,7 +246,7 @@ impl ExternalAgentConfigService { let source_agents_md = if let Some(repo_root) = repo_root { find_repo_agents_md_source(repo_root)? } else { - let path = self.external_agent_home.join("CLAUDE.md"); + let path = self.external_agent_home.join(EXTERNAL_AGENT_CONFIG_MD); is_non_empty_text_file(&path)?.then_some(path) }; let target_agents_md = repo_root.map_or_else( @@ -273,7 +263,7 @@ impl ExternalAgentConfigService { source_agents_md.display(), target_agents_md.display() ), - cwd, + cwd: cwd.clone(), details: None, }); emit_migration_metric( @@ -283,6 +273,43 @@ impl ExternalAgentConfigService { ); } + if let Some(settings) = settings.as_ref() { + match ConfigBuilder::default() + .codex_home(self.codex_home.clone()) + .fallback_cwd(Some(self.codex_home.clone())) + .build() + .await + { + Ok(config) => { + let configured_plugin_ids = + configured_plugins_from_stack(&config.config_layer_stack) + .into_keys() + .collect::>(); + let configured_marketplace_plugins = configured_marketplace_plugins( + &config, + &PluginsManager::new(self.codex_home.clone()), + )?; + if let Some(item) = self.detect_plugin_migration( + source_settings.as_path(), + repo_root.unwrap_or(self.external_agent_home.as_path()), + cwd.clone(), + settings, + &configured_plugin_ids, + &configured_marketplace_plugins, + ) { + items.push(item); + } + } + Err(err) => { + tracing::warn!( + error = %err, + settings_path = %source_settings.display(), + "skipping external agent plugin migration detection because config load failed" + ); + } + } + } + Ok(()) } @@ -296,25 +323,30 @@ impl ExternalAgentConfigService { fn detect_plugin_migration( &self, source_settings: &Path, + source_root: &Path, cwd: Option, - settings: Option<&JsonValue>, - items: &mut Vec, - ) { - let Some(plugin_details) = settings.and_then(extract_plugin_migration_details) else { - return; - }; - - items.push(ExternalAgentConfigMigrationItem { - item_type: ExternalAgentConfigMigrationItemType::Plugins, - description: format!("Import enabled plugins from {}", source_settings.display()), - cwd, - details: Some(plugin_details), - }); + settings: &JsonValue, + configured_plugin_ids: &HashSet, + configured_marketplace_plugins: &BTreeMap>, + ) -> Option { + let plugin_details = extract_plugin_migration_details( + settings, + source_root, + configured_plugin_ids, + configured_marketplace_plugins, + )?; emit_migration_metric( EXTERNAL_AGENT_CONFIG_DETECT_METRIC, ExternalAgentConfigMigrationItemType::Plugins, /*skills_count*/ None, ); + + Some(ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::Plugins, + description: format!("Import enabled plugins from {}", source_settings.display()), + cwd, + details: Some(plugin_details), + }) } async fn import_plugins( @@ -336,8 +368,14 @@ impl ExternalAgentConfigService { .iter() .map(|plugin_name| format!("{plugin_name}@{marketplace_name}")) .collect::>(); - let import_source = - read_marketplace_import_source(cwd, &self.external_agent_home, &marketplace_name)?; + let source_settings = cwd.map_or_else( + || self.external_agent_home.join("settings.json"), + |cwd| cwd.join(EXTERNAL_AGENT_DIR).join("settings.json"), + ); + let source_root = cwd.unwrap_or(self.external_agent_home.as_path()); + let import_source = read_external_settings(&source_settings)?.and_then(|settings| { + collect_marketplace_import_sources(&settings, source_root).remove(&marketplace_name) + }); let Some(import_source) = import_source else { outcome.failed_marketplaces.push(marketplace_name); outcome.failed_plugin_ids.extend(plugin_ids); @@ -351,12 +389,17 @@ impl ExternalAgentConfigService { let add_marketplace_outcome = add_marketplace(self.codex_home.clone(), request).await; let marketplace_path = match add_marketplace_outcome { Ok(add_marketplace_outcome) => { + let Some(marketplace_path) = find_marketplace_manifest_path( + add_marketplace_outcome.installed_root.as_path(), + ) else { + outcome.failed_marketplaces.push(marketplace_name); + outcome.failed_plugin_ids.extend(plugin_ids); + continue; + }; outcome .succeeded_marketplaces .push(marketplace_name.clone()); - add_marketplace_outcome - .installed_root - .join(INSTALLED_MARKETPLACE_MANIFEST_RELATIVE_PATH) + marketplace_path } Err(_) => { outcome.failed_marketplaces.push(marketplace_name); @@ -388,7 +431,7 @@ impl ExternalAgentConfigService { fn import_config(&self, cwd: Option<&Path>) -> io::Result<()> { let (source_settings, target_config) = if let Some(repo_root) = find_repo_root(cwd)? { ( - repo_root.join(".claude").join("settings.json"), + repo_root.join(EXTERNAL_AGENT_DIR).join("settings.json"), repo_root.join(".codex").join("config.toml"), ) } else if cwd.is_some_and(|cwd| !cwd.as_os_str().is_empty()) { @@ -440,7 +483,7 @@ impl ExternalAgentConfigService { fn import_skills(&self, cwd: Option<&Path>) -> io::Result { let (source_skills, target_skills) = if let Some(repo_root) = find_repo_root(cwd)? { ( - repo_root.join(".claude").join("skills"), + repo_root.join(EXTERNAL_AGENT_DIR).join("skills"), repo_root.join(".agents").join("skills"), ) } else if cwd.is_some_and(|cwd| !cwd.as_os_str().is_empty()) { @@ -487,7 +530,7 @@ impl ExternalAgentConfigService { return Ok(()); } else { ( - self.external_agent_home.join("CLAUDE.md"), + self.external_agent_home.join(EXTERNAL_AGENT_CONFIG_MD), self.codex_home.join("AGENTS.md"), ) }; @@ -508,10 +551,10 @@ impl ExternalAgentConfigService { fn default_external_agent_home() -> PathBuf { if let Some(home) = std::env::var_os("HOME").or_else(|| std::env::var_os("USERPROFILE")) { - return PathBuf::from(home).join(".claude"); + return PathBuf::from(home).join(EXTERNAL_AGENT_DIR); } - PathBuf::from(".claude") + PathBuf::from(EXTERNAL_AGENT_DIR) } fn read_external_settings(path: &Path) -> io::Result> { @@ -525,12 +568,37 @@ fn read_external_settings(path: &Path) -> io::Result> { Ok(Some(settings)) } -fn extract_plugin_migration_details(settings: &JsonValue) -> Option { +fn extract_plugin_migration_details( + settings: &JsonValue, + source_root: &Path, + configured_plugin_ids: &HashSet, + configured_marketplace_plugins: &BTreeMap>, +) -> Option { + let loadable_marketplaces = collect_marketplace_import_sources(settings, source_root) + .into_iter() + .filter_map(|(marketplace_name, source)| { + parse_marketplace_source(&source.source, source.ref_name) + .ok() + .map(|_| marketplace_name) + }) + .collect::>(); let mut plugins = BTreeMap::new(); - for plugin_id in collect_enabled_plugins(settings) { + for plugin_id in collect_enabled_plugins(settings) + .into_iter() + .filter(|plugin_id| !configured_plugin_ids.contains(plugin_id)) + { let Ok(plugin_id) = PluginId::parse(&plugin_id) else { continue; }; + if let Some(installable_plugins) = + configured_marketplace_plugins.get(&plugin_id.marketplace_name) + { + if !installable_plugins.contains(&plugin_id.plugin_name) { + continue; + } + } else if !loadable_marketplaces.contains(&plugin_id.marketplace_name) { + continue; + } let plugin_group = plugins .entry(plugin_id.marketplace_name.clone()) .or_insert_with(|| PluginsMigration { @@ -579,8 +647,40 @@ fn collect_enabled_plugins(settings: &JsonValue) -> Vec { .collect() } +fn configured_marketplace_plugins( + config: &Config, + plugins_manager: &PluginsManager, +) -> io::Result>> { + let marketplaces = plugins_manager + .list_marketplaces_for_config(config, &[]) + .map_err(|err| { + invalid_data_error(format!("failed to list configured marketplaces: {err}")) + })?; + let mut marketplace_plugins = BTreeMap::new(); + for marketplace in marketplaces.marketplaces { + let plugins = marketplace + .plugins + .into_iter() + .filter(|plugin| { + plugin.policy.installation != MarketplacePluginInstallPolicy::NotAvailable + }) + .filter(|plugin| { + plugin + .policy + .products + .as_deref() + .is_none_or(|products| Product::Codex.matches_product_restriction(products)) + }) + .map(|plugin| plugin.name) + .collect::>(); + marketplace_plugins.insert(marketplace.name, plugins); + } + Ok(marketplace_plugins) +} + fn collect_marketplace_import_sources( settings: &JsonValue, + source_root: &Path, ) -> BTreeMap { let Some(extra_known_marketplaces) = settings .as_object() @@ -611,6 +711,7 @@ fn collect_marketplace_import_sources( if source.is_empty() { return None; } + let source = resolve_external_marketplace_source(&source, source_root); let ref_name = source_fields .get("ref") @@ -631,20 +732,16 @@ struct MarketplaceImportSource { ref_name: Option, } -fn read_marketplace_import_source( - cwd: Option<&Path>, - external_agent_home: &Path, - marketplace_name: &str, -) -> io::Result> { - let source_settings = cwd.map_or_else( - || external_agent_home.join("settings.json"), - |cwd| cwd.join(".claude").join("settings.json"), - ); - let Some(settings) = read_external_settings(&source_settings)? else { - return Ok(None); - }; +fn resolve_external_marketplace_source(source: &str, source_root: &Path) -> String { + if !looks_like_relative_local_path(source) { + return source.to_string(); + } - Ok(collect_marketplace_import_sources(&settings).remove(marketplace_name)) + source_root.join(source).display().to_string() +} + +fn looks_like_relative_local_path(source: &str) -> bool { + source.starts_with("./") || source.starts_with("../") || source == "." || source == ".." } fn find_repo_root(cwd: Option<&Path>) -> io::Result> { @@ -729,8 +826,10 @@ fn is_non_empty_text_file(path: &Path) -> io::Result { fn find_repo_agents_md_source(repo_root: &Path) -> io::Result> { for candidate in [ - repo_root.join("CLAUDE.md"), - repo_root.join(".claude").join("CLAUDE.md"), + repo_root.join(EXTERNAL_AGENT_CONFIG_MD), + repo_root + .join(EXTERNAL_AGENT_DIR) + .join(EXTERNAL_AGENT_CONFIG_MD), ] { if is_non_empty_text_file(&candidate)? { return Ok(Some(candidate)); @@ -779,7 +878,11 @@ fn rewrite_and_copy_text_file(source: &Path, target: &Path) -> io::Result<()> { } fn rewrite_external_agent_terms(content: &str) -> String { - let mut rewritten = replace_case_insensitive_with_boundaries(content, "claude.md", "AGENTS.md"); + let mut rewritten = replace_case_insensitive_with_boundaries( + content, + &EXTERNAL_AGENT_CONFIG_MD.to_ascii_lowercase(), + "AGENTS.md", + ); for from in [ "claude code", "claude-code", diff --git a/codex-rs/core/src/external_agent_config_tests.rs b/codex-rs/core/src/external_agent_config_tests.rs index f2afab0170..f01a40e3d2 100644 --- a/codex-rs/core/src/external_agent_config_tests.rs +++ b/codex-rs/core/src/external_agent_config_tests.rs @@ -5,13 +5,16 @@ use tempfile::TempDir; fn fixture_paths() -> (TempDir, PathBuf, PathBuf) { let root = TempDir::new().expect("create tempdir"); - let claude_home = root.path().join(".claude"); + let external_agent_home = root.path().join(".claude"); let codex_home = root.path().join(".codex"); - (root, claude_home, codex_home) + (root, external_agent_home, codex_home) } -fn service_for_paths(claude_home: PathBuf, codex_home: PathBuf) -> ExternalAgentConfigService { - ExternalAgentConfigService::new_for_test(codex_home, claude_home) +fn service_for_paths( + external_agent_home: PathBuf, + codex_home: PathBuf, +) -> ExternalAgentConfigService { + ExternalAgentConfigService::new_for_test(codex_home, external_agent_home) } fn github_plugin_details() -> MigrationDetails { @@ -23,26 +26,28 @@ fn github_plugin_details() -> MigrationDetails { } } -#[test] -fn detect_home_lists_config_skills_and_agents_md() { - let (_root, claude_home, codex_home) = fixture_paths(); +#[tokio::test] +async fn detect_home_lists_config_skills_and_agents_md() { + let (_root, external_agent_home, codex_home) = fixture_paths(); let agents_skills = codex_home .parent() .map(|parent| parent.join(".agents").join("skills")) .unwrap_or_else(|| PathBuf::from(".agents").join("skills")); - fs::create_dir_all(claude_home.join("skills").join("skill-a")).expect("create skills"); - fs::write(claude_home.join("CLAUDE.md"), "claude rules").expect("write claude md"); + fs::create_dir_all(external_agent_home.join("skills").join("skill-a")).expect("create skills"); + fs::write(external_agent_home.join("CLAUDE.md"), "claude rules") + .expect("write external agent md"); fs::write( - claude_home.join("settings.json"), + external_agent_home.join("settings.json"), r#"{"model":"claude","env":{"FOO":"bar"}}"#, ) .expect("write settings"); - let items = service_for_paths(claude_home.clone(), codex_home.clone()) + let items = service_for_paths(external_agent_home.clone(), codex_home.clone()) .detect(ExternalAgentConfigDetectOptions { include_home: true, cwds: None, }) + .await .expect("detect"); let expected = vec![ @@ -50,7 +55,7 @@ fn detect_home_lists_config_skills_and_agents_md() { item_type: ExternalAgentConfigMigrationItemType::Config, description: format!( "Migrate {} into {}", - claude_home.join("settings.json").display(), + external_agent_home.join("settings.json").display(), codex_home.join("config.toml").display() ), cwd: None, @@ -59,8 +64,8 @@ fn detect_home_lists_config_skills_and_agents_md() { ExternalAgentConfigMigrationItem { item_type: ExternalAgentConfigMigrationItemType::Skills, description: format!( - "Copy skill folders from {} to {}", - claude_home.join("skills").display(), + "Migrate skills from {} to {}", + external_agent_home.join("skills").display(), agents_skills.display() ), cwd: None, @@ -70,7 +75,7 @@ fn detect_home_lists_config_skills_and_agents_md() { item_type: ExternalAgentConfigMigrationItemType::AgentsMd, description: format!( "Import {} to {}", - claude_home.join("CLAUDE.md").display(), + external_agent_home.join("CLAUDE.md").display(), codex_home.join("AGENTS.md").display() ), cwd: None, @@ -81,8 +86,8 @@ fn detect_home_lists_config_skills_and_agents_md() { assert_eq!(items, expected); } -#[test] -fn detect_repo_lists_agents_md_for_each_cwd() { +#[tokio::test] +async fn detect_repo_lists_agents_md_for_each_cwd() { let root = TempDir::new().expect("create tempdir"); let repo_root = root.path().join("repo"); let nested = repo_root.join("nested").join("child"); @@ -95,6 +100,7 @@ fn detect_repo_lists_agents_md_for_each_cwd() { include_home: false, cwds: Some(vec![nested, repo_root.clone()]), }) + .await .expect("detect"); let expected = vec![ @@ -123,27 +129,110 @@ fn detect_repo_lists_agents_md_for_each_cwd() { assert_eq!(items, expected); } +#[tokio::test] +async fn detect_repo_still_reports_non_plugin_items_when_home_config_is_invalid() { + let root = TempDir::new().expect("create tempdir"); + let repo_root = root.path().join("repo"); + let codex_home = root.path().join(".codex"); + fs::create_dir_all(repo_root.join(".git")).expect("create git dir"); + fs::create_dir_all(repo_root.join(".claude").join("skills").join("skill-a")) + .expect("create repo skills"); + fs::create_dir_all(&codex_home).expect("create codex home"); + fs::write(codex_home.join("config.toml"), "this is not valid = [toml") + .expect("write invalid codex config"); + fs::write( + repo_root.join(".claude").join("settings.json"), + r#"{"env":{"FOO":"bar"}}"#, + ) + .expect("write settings"); + fs::write( + repo_root + .join(".claude") + .join("skills") + .join("skill-a") + .join("SKILL.md"), + "Use Claude Code and CLAUDE utilities.", + ) + .expect("write skill"); + fs::write( + repo_root.join(".claude").join("CLAUDE.md"), + "Claude code guidance", + ) + .expect("write agents"); + + let items = service_for_paths(root.path().join(".claude"), codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: false, + cwds: Some(vec![repo_root.clone()]), + }) + .await + .expect("detect"); + + assert_eq!( + items, + vec![ + ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::Config, + description: format!( + "Migrate {} into {}", + repo_root.join(".claude").join("settings.json").display(), + repo_root.join(".codex").join("config.toml").display() + ), + cwd: Some(repo_root.clone()), + details: None, + }, + ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::Skills, + description: format!( + "Migrate skills from {} to {}", + repo_root.join(".claude").join("skills").display(), + repo_root.join(".agents").join("skills").display() + ), + cwd: Some(repo_root.clone()), + details: None, + }, + ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::AgentsMd, + description: format!( + "Import {} to {}", + repo_root.join(".claude").join("CLAUDE.md").display(), + repo_root.join("AGENTS.md").display(), + ), + cwd: Some(repo_root), + details: None, + }, + ] + ); +} + #[tokio::test] async fn import_home_migrates_supported_config_fields_skills_and_agents_md() { - let (_root, claude_home, codex_home) = fixture_paths(); + let (_root, external_agent_home, codex_home) = fixture_paths(); let agents_skills = codex_home .parent() .map(|parent| parent.join(".agents").join("skills")) .unwrap_or_else(|| PathBuf::from(".agents").join("skills")); - fs::create_dir_all(claude_home.join("skills").join("skill-a")).expect("create skills"); + fs::create_dir_all(external_agent_home.join("skills").join("skill-a")).expect("create skills"); fs::write( - claude_home.join("settings.json"), + external_agent_home.join("settings.json"), r#"{"model":"claude","permissions":{"ask":["git push"]},"env":{"FOO":"bar","CI":false,"MAX_RETRIES":3,"MY_TEAM":"codex","IGNORED":null,"LIST":["a","b"],"MAP":{"x":1}},"sandbox":{"enabled":true,"network":{"allowLocalBinding":true}}}"#, ) .expect("write settings"); fs::write( - claude_home.join("skills").join("skill-a").join("SKILL.md"), + external_agent_home + .join("skills") + .join("skill-a") + .join("SKILL.md"), "Use Claude Code and CLAUDE utilities.", ) .expect("write skill"); - fs::write(claude_home.join("CLAUDE.md"), "Claude code guidance").expect("write agents"); + fs::write( + external_agent_home.join("CLAUDE.md"), + "Claude code guidance", + ) + .expect("write agents"); - service_for_paths(claude_home, codex_home.clone()) + service_for_paths(external_agent_home, codex_home.clone()) .import(vec![ ExternalAgentConfigMigrationItem { item_type: ExternalAgentConfigMigrationItemType::AgentsMd, @@ -185,15 +274,15 @@ async fn import_home_migrates_supported_config_fields_skills_and_agents_md() { #[tokio::test] async fn import_home_skips_empty_config_migration() { - let (_root, claude_home, codex_home) = fixture_paths(); - fs::create_dir_all(&claude_home).expect("create claude home"); + let (_root, external_agent_home, codex_home) = fixture_paths(); + fs::create_dir_all(&external_agent_home).expect("create external agent home"); fs::write( - claude_home.join("settings.json"), + external_agent_home.join("settings.json"), r#"{"model":"claude","sandbox":{"enabled":false}}"#, ) .expect("write settings"); - service_for_paths(claude_home, codex_home.clone()) + service_for_paths(external_agent_home, codex_home.clone()) .import(vec![ExternalAgentConfigMigrationItem { item_type: ExternalAgentConfigMigrationItemType::Config, description: String::new(), @@ -206,13 +295,13 @@ async fn import_home_skips_empty_config_migration() { assert!(!codex_home.join("config.toml").exists()); } -#[test] -fn detect_home_skips_config_when_target_already_has_supported_fields() { - let (_root, claude_home, codex_home) = fixture_paths(); - fs::create_dir_all(&claude_home).expect("create claude home"); +#[tokio::test] +async fn detect_home_skips_config_when_target_already_has_supported_fields() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + fs::create_dir_all(&external_agent_home).expect("create external agent home"); fs::create_dir_all(&codex_home).expect("create codex home"); fs::write( - claude_home.join("settings.json"), + external_agent_home.join("settings.json"), r#"{"env":{"FOO":"bar"},"sandbox":{"enabled":true}}"#, ) .expect("write settings"); @@ -230,31 +319,33 @@ fn detect_home_skips_config_when_target_already_has_supported_fields() { ) .expect("write config"); - let items = service_for_paths(claude_home, codex_home) + let items = service_for_paths(external_agent_home, codex_home) .detect(ExternalAgentConfigDetectOptions { include_home: true, cwds: None, }) + .await .expect("detect"); assert_eq!(items, Vec::::new()); } -#[test] -fn detect_home_skips_skills_when_all_skill_directories_exist() { - let (_root, claude_home, codex_home) = fixture_paths(); +#[tokio::test] +async fn detect_home_skips_skills_when_all_skill_directories_exist() { + let (_root, external_agent_home, codex_home) = fixture_paths(); let agents_skills = codex_home .parent() .map(|parent| parent.join(".agents").join("skills")) .unwrap_or_else(|| PathBuf::from(".agents").join("skills")); - fs::create_dir_all(claude_home.join("skills").join("skill-a")).expect("create source"); + fs::create_dir_all(external_agent_home.join("skills").join("skill-a")).expect("create source"); fs::create_dir_all(agents_skills.join("skill-a")).expect("create target"); - let items = service_for_paths(claude_home, codex_home) + let items = service_for_paths(external_agent_home, codex_home) .detect(ExternalAgentConfigDetectOptions { include_home: true, cwds: None, }) + .await .expect("detect"); assert_eq!(items, Vec::::new()); @@ -332,8 +423,8 @@ async fn import_repo_agents_md_overwrites_empty_targets() { ); } -#[test] -fn detect_repo_prefers_non_empty_dot_claude_agents_source() { +#[tokio::test] +async fn detect_repo_prefers_non_empty_external_agent_agents_source() { let root = TempDir::new().expect("create tempdir"); let repo_root = root.path().join("repo"); fs::create_dir_all(repo_root.join(".git")).expect("create git"); @@ -350,6 +441,7 @@ fn detect_repo_prefers_non_empty_dot_claude_agents_source() { include_home: false, cwds: Some(vec![repo_root.clone()]), }) + .await .expect("detect"); assert_eq!( @@ -368,7 +460,7 @@ fn detect_repo_prefers_non_empty_dot_claude_agents_source() { } #[tokio::test] -async fn import_repo_uses_non_empty_dot_claude_agents_source() { +async fn import_repo_uses_non_empty_external_agent_agents_source() { let root = TempDir::new().expect("create tempdir"); let repo_root = root.path().join("repo"); fs::create_dir_all(repo_root.join(".git")).expect("create git"); @@ -407,27 +499,33 @@ fn migration_metric_tags_for_skills_include_skills_count() { ); } -#[test] -fn detect_home_lists_enabled_plugins_from_settings() { - let (_root, claude_home, codex_home) = fixture_paths(); - fs::create_dir_all(&claude_home).expect("create claude home"); +#[tokio::test] +async fn detect_home_lists_enabled_plugins_from_settings() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + fs::create_dir_all(&external_agent_home).expect("create external agent home"); fs::write( - claude_home.join("settings.json"), + external_agent_home.join("settings.json"), r#"{ "enabledPlugins": { "formatter@acme-tools": true, "deployer@acme-tools": true, "analyzer@security-plugins": false + }, + "extraKnownMarketplaces": { + "acme-tools": { + "source": "acme-corp/claude-plugins" + } } }"#, ) .expect("write settings"); - let items = service_for_paths(claude_home.clone(), codex_home) + let items = service_for_paths(external_agent_home.clone(), codex_home) .detect(ExternalAgentConfigDetectOptions { include_home: true, cwds: None, }) + .await .expect("detect"); assert_eq!( @@ -436,7 +534,7 @@ fn detect_home_lists_enabled_plugins_from_settings() { item_type: ExternalAgentConfigMigrationItemType::Plugins, description: format!( "Import enabled plugins from {}", - claude_home.join("settings.json").display() + external_agent_home.join("settings.json").display() ), cwd: None, details: Some(MigrationDetails { @@ -450,10 +548,155 @@ fn detect_home_lists_enabled_plugins_from_settings() { } #[tokio::test] -async fn import_plugins_requires_details() { - let (_root, claude_home, codex_home) = fixture_paths(); +async fn detect_repo_skips_plugins_that_are_already_configured_in_codex() { + let root = TempDir::new().expect("create tempdir"); + let external_agent_home = root.path().join(".claude"); + let codex_home = root.path().join(".codex"); + let repo_root = root.path().join("repo"); + fs::create_dir_all(repo_root.join(".git")).expect("create git dir"); + fs::create_dir_all(repo_root.join(".claude")).expect("create repo external agent dir"); + fs::create_dir_all(&codex_home).expect("create codex home"); + fs::write( + repo_root.join(".claude").join("settings.json"), + r#"{ + "enabledPlugins": { + "formatter@acme-tools": true, + "deployer@acme-tools": true + }, + "extraKnownMarketplaces": { + "acme-tools": { + "source": "acme-corp/claude-plugins" + } + } + }"#, + ) + .expect("write repo settings"); + fs::write( + codex_home.join("config.toml"), + r#" +[plugins."formatter@acme-tools"] +enabled = true +"#, + ) + .expect("write codex config"); - let err = service_for_paths(claude_home, codex_home) + let items = service_for_paths(external_agent_home, codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: false, + cwds: Some(vec![repo_root.clone()]), + }) + .await + .expect("detect"); + + assert_eq!( + items, + vec![ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::Plugins, + description: format!( + "Import enabled plugins from {}", + repo_root.join(".claude").join("settings.json").display() + ), + cwd: Some(repo_root), + details: Some(MigrationDetails { + plugins: vec![PluginsMigration { + marketplace_name: "acme-tools".to_string(), + plugin_names: vec!["deployer".to_string()], + }], + }), + }] + ); +} + +#[tokio::test] +async fn detect_repo_skips_plugins_that_are_disabled_in_codex() { + let root = TempDir::new().expect("create tempdir"); + let external_agent_home = root.path().join(".claude"); + let codex_home = root.path().join(".codex"); + let repo_root = root.path().join("repo"); + fs::create_dir_all(repo_root.join(".git")).expect("create git dir"); + fs::create_dir_all(repo_root.join(".claude")).expect("create repo external agent dir"); + fs::create_dir_all(&codex_home).expect("create codex home"); + fs::write( + repo_root.join(".claude").join("settings.json"), + r#"{ + "enabledPlugins": { + "formatter@acme-tools": true + }, + "extraKnownMarketplaces": { + "acme-tools": { + "source": "acme-corp/claude-plugins" + } + } + }"#, + ) + .expect("write repo settings"); + fs::write( + codex_home.join("config.toml"), + r#" +[plugins."formatter@acme-tools"] +enabled = false +"#, + ) + .expect("write codex config"); + + let items = service_for_paths(external_agent_home, codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: false, + cwds: Some(vec![repo_root]), + }) + .await + .expect("detect"); + + assert_eq!(items, Vec::::new()); +} + +#[tokio::test] +async fn detect_repo_skips_plugins_without_explicit_enabled_in_codex() { + let root = TempDir::new().expect("create tempdir"); + let external_agent_home = root.path().join(".claude"); + let codex_home = root.path().join(".codex"); + let repo_root = root.path().join("repo"); + fs::create_dir_all(repo_root.join(".git")).expect("create git dir"); + fs::create_dir_all(repo_root.join(".claude")).expect("create repo external agent dir"); + fs::create_dir_all(&codex_home).expect("create codex home"); + fs::write( + repo_root.join(".claude").join("settings.json"), + r#"{ + "enabledPlugins": { + "formatter@acme-tools": true + }, + "extraKnownMarketplaces": { + "acme-tools": { + "source": "acme-corp/claude-plugins" + } + } + }"#, + ) + .expect("write repo settings"); + fs::write( + codex_home.join("config.toml"), + r#" +[plugins."formatter@acme-tools"] +"#, + ) + .expect("write codex config"); + + let items = service_for_paths(external_agent_home, codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: false, + cwds: Some(vec![repo_root]), + }) + .await + .expect("detect"); + + assert_eq!(items, Vec::::new()); +} + +#[tokio::test] +async fn import_plugins_requires_details() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + + let err = service_for_paths(external_agent_home, codex_home) .import_plugins(/*cwd*/ None, /*details*/ None) .await .expect_err("expected missing details error"); @@ -463,11 +706,251 @@ async fn import_plugins_requires_details() { } #[tokio::test] -async fn import_plugins_requires_source_marketplace_details() { - let (_root, claude_home, codex_home) = fixture_paths(); - fs::create_dir_all(&claude_home).expect("create claude home"); +async fn detect_repo_does_not_skip_plugins_only_configured_in_project_codex() { + let root = TempDir::new().expect("create tempdir"); + let external_agent_home = root.path().join(".claude"); + let codex_home = root.path().join(".codex"); + let repo_root = root.path().join("repo"); + fs::create_dir_all(repo_root.join(".git")).expect("create git dir"); + fs::create_dir_all(repo_root.join(".claude")).expect("create repo external agent dir"); + fs::create_dir_all(repo_root.join(".codex")).expect("create repo codex dir"); + fs::create_dir_all(&codex_home).expect("create codex home"); fs::write( - claude_home.join("settings.json"), + repo_root.join(".claude").join("settings.json"), + r#"{ + "enabledPlugins": { + "formatter@acme-tools": true + }, + "extraKnownMarketplaces": { + "acme-tools": { + "source": "acme-corp/claude-plugins" + } + } + }"#, + ) + .expect("write repo settings"); + fs::write( + repo_root.join(".codex").join("config.toml"), + r#" +[plugins."formatter@acme-tools"] +enabled = true +"#, + ) + .expect("write project codex config"); + + let items = service_for_paths(external_agent_home, codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: false, + cwds: Some(vec![repo_root.clone()]), + }) + .await + .expect("detect"); + + assert_eq!( + items, + vec![ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::Plugins, + description: format!( + "Import enabled plugins from {}", + repo_root.join(".claude").join("settings.json").display() + ), + cwd: Some(repo_root), + details: Some(MigrationDetails { + plugins: vec![PluginsMigration { + marketplace_name: "acme-tools".to_string(), + plugin_names: vec!["formatter".to_string()], + }], + }), + }] + ); +} + +#[tokio::test] +async fn detect_home_skips_plugins_without_marketplace_source() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + fs::create_dir_all(&external_agent_home).expect("create external agent home"); + fs::write( + external_agent_home.join("settings.json"), + r#"{ + "enabledPlugins": { + "formatter@acme-tools": true + } + }"#, + ) + .expect("write settings"); + + let items = service_for_paths(external_agent_home, codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: true, + cwds: None, + }) + .await + .expect("detect"); + + assert_eq!(items, Vec::::new()); +} + +#[tokio::test] +async fn detect_home_skips_plugins_with_invalid_marketplace_source() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + fs::create_dir_all(&external_agent_home).expect("create external agent home"); + fs::write( + external_agent_home.join("settings.json"), + r#"{ + "enabledPlugins": { + "formatter@acme-tools": true + }, + "extraKnownMarketplaces": { + "acme-tools": { + "source": "github" + } + } + }"#, + ) + .expect("write settings"); + + let items = service_for_paths(external_agent_home, codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: true, + cwds: None, + }) + .await + .expect("detect"); + + assert_eq!(items, Vec::::new()); +} + +#[tokio::test] +async fn detect_repo_filters_plugins_against_installed_marketplace() { + let root = TempDir::new().expect("create tempdir"); + let external_agent_home = root.path().join(".claude"); + let codex_home = root.path().join(".codex"); + let repo_root = root.path().join("repo"); + let marketplace_root = codex_home.join(".tmp").join("marketplaces").join("debug"); + fs::create_dir_all(repo_root.join(".git")).expect("create git dir"); + fs::create_dir_all(repo_root.join(".claude")).expect("create repo external agent dir"); + fs::create_dir_all(marketplace_root.join(".agents").join("plugins")) + .expect("create marketplace manifest dir"); + fs::create_dir_all( + marketplace_root + .join("plugins") + .join("sample") + .join(".codex-plugin"), + ) + .expect("create sample plugin"); + fs::create_dir_all( + marketplace_root + .join("plugins") + .join("available") + .join(".codex-plugin"), + ) + .expect("create available plugin"); + fs::write( + repo_root.join(".claude").join("settings.json"), + r#"{ + "enabledPlugins": { + "sample@debug": true, + "available@debug": true, + "missing@debug": true + }, + "extraKnownMarketplaces": { + "debug": { + "source": "owner/debug-marketplace" + } + } + }"#, + ) + .expect("write repo settings"); + fs::write( + codex_home.join("config.toml"), + r#" +[marketplaces.debug] +source_type = "git" +source = "owner/debug-marketplace" +"#, + ) + .expect("write codex config"); + fs::write( + marketplace_root + .join(".agents") + .join("plugins") + .join("marketplace.json"), + r#"{ + "name": "debug", + "plugins": [ + { + "name": "sample", + "source": { + "source": "local", + "path": "./plugins/sample" + }, + "policy": { + "installation": "NOT_AVAILABLE" + } + }, + { + "name": "available", + "source": { + "source": "local", + "path": "./plugins/available" + } + } + ] +}"#, + ) + .expect("write marketplace manifest"); + fs::write( + marketplace_root + .join("plugins") + .join("sample") + .join(".codex-plugin") + .join("plugin.json"), + r#"{"name":"sample"}"#, + ) + .expect("write sample plugin manifest"); + fs::write( + marketplace_root + .join("plugins") + .join("available") + .join(".codex-plugin") + .join("plugin.json"), + r#"{"name":"available"}"#, + ) + .expect("write available plugin manifest"); + + let items = service_for_paths(external_agent_home, codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: false, + cwds: Some(vec![repo_root.clone()]), + }) + .await + .expect("detect"); + + assert_eq!( + items, + vec![ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::Plugins, + description: format!( + "Import enabled plugins from {}", + repo_root.join(".claude").join("settings.json").display() + ), + cwd: Some(repo_root), + details: Some(MigrationDetails { + plugins: vec![PluginsMigration { + marketplace_name: "debug".to_string(), + plugin_names: vec!["available".to_string()], + }], + }), + }] + ); +} + +#[tokio::test] +async fn import_plugins_requires_source_marketplace_details() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + fs::create_dir_all(&external_agent_home).expect("create external agent home"); + fs::write( + external_agent_home.join("settings.json"), r#"{ "enabledPlugins": { "formatter@acme-tools": true @@ -482,7 +965,7 @@ async fn import_plugins_requires_source_marketplace_details() { ) .expect("write settings"); - let outcome = service_for_paths(claude_home, codex_home) + let outcome = service_for_paths(external_agent_home, codex_home) .import_plugins( /*cwd*/ None, Some(MigrationDetails { @@ -508,10 +991,10 @@ async fn import_plugins_requires_source_marketplace_details() { #[tokio::test] async fn import_plugins_defers_marketplace_source_validation_to_add_marketplace() { - let (_root, claude_home, codex_home) = fixture_paths(); - fs::create_dir_all(&claude_home).expect("create claude home"); + let (_root, external_agent_home, codex_home) = fixture_paths(); + fs::create_dir_all(&external_agent_home).expect("create external agent home"); fs::write( - claude_home.join("settings.json"), + external_agent_home.join("settings.json"), r#"{ "enabledPlugins": { "formatter@acme-tools": true @@ -526,7 +1009,7 @@ async fn import_plugins_defers_marketplace_source_validation_to_add_marketplace( ) .expect("write settings"); - let outcome = service_for_paths(claude_home, codex_home) + let outcome = service_for_paths(external_agent_home, codex_home) .import_plugins(/*cwd*/ None, Some(github_plugin_details())) .await .expect("import plugins"); @@ -542,18 +1025,396 @@ async fn import_plugins_defers_marketplace_source_validation_to_add_marketplace( ); } +#[tokio::test] +async fn import_plugins_supports_external_agent_plugin_marketplace_layout() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + let marketplace_root = external_agent_home.join("my-marketplace"); + let plugin_root = marketplace_root.join("plugins").join("cloudflare"); + fs::create_dir_all(marketplace_root.join(".claude-plugin")) + .expect("create marketplace manifest dir"); + fs::create_dir_all(plugin_root.join(".codex-plugin")).expect("create plugin manifest dir"); + fs::create_dir_all(&codex_home).expect("create codex home"); + + fs::write( + external_agent_home.join("settings.json"), + serde_json::to_string_pretty(&serde_json::json!({ + "enabledPlugins": { + "cloudflare@my-plugins": true + }, + "extraKnownMarketplaces": { + "my-plugins": { + "source": "local", + "path": marketplace_root + } + } + })) + .expect("serialize settings"), + ) + .expect("write settings"); + fs::write( + marketplace_root + .join(".claude-plugin") + .join("marketplace.json"), + r#"{ + "name": "my-plugins", + "plugins": [ + { + "name": "cloudflare", + "source": "./plugins/cloudflare" + } + ] + }"#, + ) + .expect("write marketplace manifest"); + fs::write( + plugin_root.join(".codex-plugin").join("plugin.json"), + r#"{"name":"cloudflare","version":"0.1.0"}"#, + ) + .expect("write plugin manifest"); + + let outcome = service_for_paths(external_agent_home, codex_home.clone()) + .import_plugins( + /*cwd*/ None, + Some(MigrationDetails { + plugins: vec![PluginsMigration { + marketplace_name: "my-plugins".to_string(), + plugin_names: vec!["cloudflare".to_string()], + }], + }), + ) + .await + .expect("import plugins"); + + assert_eq!( + outcome, + PluginImportOutcome { + succeeded_marketplaces: vec!["my-plugins".to_string()], + succeeded_plugin_ids: vec!["cloudflare@my-plugins".to_string()], + failed_marketplaces: Vec::new(), + failed_plugin_ids: Vec::new(), + } + ); + let config = fs::read_to_string(codex_home.join("config.toml")).expect("read config"); + assert!(config.contains(r#"[plugins."cloudflare@my-plugins"]"#)); + assert!(config.contains("enabled = true")); +} + +#[tokio::test] +async fn detect_home_supports_relative_external_agent_plugin_marketplace_path() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + let marketplace_root = external_agent_home.join("my-marketplace"); + let plugin_root = marketplace_root.join("plugins").join("cloudflare"); + fs::create_dir_all(marketplace_root.join(".claude-plugin")) + .expect("create marketplace manifest dir"); + fs::create_dir_all(plugin_root.join(".codex-plugin")).expect("create plugin manifest dir"); + fs::create_dir_all(&codex_home).expect("create codex home"); + + fs::write( + external_agent_home.join("settings.json"), + r#"{ + "enabledPlugins": { + "cloudflare@my-plugins": true + }, + "extraKnownMarketplaces": { + "my-plugins": { + "source": "directory", + "path": "./my-marketplace" + } + } + }"#, + ) + .expect("write settings"); + fs::write( + marketplace_root + .join(".claude-plugin") + .join("marketplace.json"), + r#"{ + "name": "my-plugins", + "plugins": [ + { + "name": "cloudflare", + "source": "./plugins/cloudflare" + } + ] + }"#, + ) + .expect("write marketplace manifest"); + fs::write( + plugin_root.join(".codex-plugin").join("plugin.json"), + r#"{"name":"cloudflare","version":"0.1.0"}"#, + ) + .expect("write plugin manifest"); + + let items = service_for_paths(external_agent_home.clone(), codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: true, + cwds: None, + }) + .await + .expect("detect"); + + assert_eq!( + items, + vec![ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::Plugins, + description: format!( + "Import enabled plugins from {}", + external_agent_home.join("settings.json").display() + ), + cwd: None, + details: Some(MigrationDetails { + plugins: vec![PluginsMigration { + marketplace_name: "my-plugins".to_string(), + plugin_names: vec!["cloudflare".to_string()], + }], + }), + }] + ); +} + +#[tokio::test] +async fn import_plugins_supports_relative_external_agent_plugin_marketplace_path() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + let marketplace_root = external_agent_home.join("my-marketplace"); + let plugin_root = marketplace_root.join("plugins").join("cloudflare"); + fs::create_dir_all(marketplace_root.join(".claude-plugin")) + .expect("create marketplace manifest dir"); + fs::create_dir_all(plugin_root.join(".codex-plugin")).expect("create plugin manifest dir"); + fs::create_dir_all(&codex_home).expect("create codex home"); + + fs::write( + external_agent_home.join("settings.json"), + r#"{ + "enabledPlugins": { + "cloudflare@my-plugins": true + }, + "extraKnownMarketplaces": { + "my-plugins": { + "source": "directory", + "path": "./my-marketplace" + } + } + }"#, + ) + .expect("write settings"); + fs::write( + marketplace_root + .join(".claude-plugin") + .join("marketplace.json"), + r#"{ + "name": "my-plugins", + "plugins": [ + { + "name": "cloudflare", + "source": "./plugins/cloudflare" + } + ] + }"#, + ) + .expect("write marketplace manifest"); + fs::write( + plugin_root.join(".codex-plugin").join("plugin.json"), + r#"{"name":"cloudflare","version":"0.1.0"}"#, + ) + .expect("write plugin manifest"); + + let outcome = service_for_paths(external_agent_home, codex_home.clone()) + .import_plugins( + /*cwd*/ None, + Some(MigrationDetails { + plugins: vec![PluginsMigration { + marketplace_name: "my-plugins".to_string(), + plugin_names: vec!["cloudflare".to_string()], + }], + }), + ) + .await + .expect("import plugins"); + + assert_eq!( + outcome, + PluginImportOutcome { + succeeded_marketplaces: vec!["my-plugins".to_string()], + succeeded_plugin_ids: vec!["cloudflare@my-plugins".to_string()], + failed_marketplaces: Vec::new(), + failed_plugin_ids: Vec::new(), + } + ); + let config = fs::read_to_string(codex_home.join("config.toml")).expect("read config"); + assert!(config.contains(r#"[plugins."cloudflare@my-plugins"]"#)); + assert!(config.contains("enabled = true")); +} + +#[tokio::test] +async fn detect_repo_supports_project_relative_external_agent_plugin_marketplace_path() { + let root = TempDir::new().expect("create tempdir"); + let external_agent_home = root.path().join(".claude"); + let codex_home = root.path().join(".codex"); + let repo_root = root.path().join("repo"); + let marketplace_root = repo_root.join("my-marketplace"); + let plugin_root = marketplace_root.join("plugins").join("cloudflare"); + fs::create_dir_all(repo_root.join(".git")).expect("create git dir"); + fs::create_dir_all(repo_root.join(".claude")).expect("create repo external agent dir"); + fs::create_dir_all(marketplace_root.join(".claude-plugin")) + .expect("create marketplace manifest dir"); + fs::create_dir_all(plugin_root.join(".codex-plugin")).expect("create plugin manifest dir"); + fs::create_dir_all(&codex_home).expect("create codex home"); + + fs::write( + repo_root.join(".claude").join("settings.json"), + r#"{ + "enabledPlugins": { + "cloudflare@my-plugins": true + }, + "extraKnownMarketplaces": { + "my-plugins": { + "source": "directory", + "path": "./my-marketplace" + } + } + }"#, + ) + .expect("write settings"); + fs::write( + marketplace_root + .join(".claude-plugin") + .join("marketplace.json"), + r#"{ + "name": "my-plugins", + "plugins": [ + { + "name": "cloudflare", + "source": "./plugins/cloudflare" + } + ] + }"#, + ) + .expect("write marketplace manifest"); + fs::write( + plugin_root.join(".codex-plugin").join("plugin.json"), + r#"{"name":"cloudflare","version":"0.1.0"}"#, + ) + .expect("write plugin manifest"); + + let items = service_for_paths(external_agent_home, codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: false, + cwds: Some(vec![repo_root.clone()]), + }) + .await + .expect("detect"); + + assert_eq!( + items, + vec![ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::Plugins, + description: format!( + "Import enabled plugins from {}", + repo_root.join(".claude").join("settings.json").display() + ), + cwd: Some(repo_root), + details: Some(MigrationDetails { + plugins: vec![PluginsMigration { + marketplace_name: "my-plugins".to_string(), + plugin_names: vec!["cloudflare".to_string()], + }], + }), + }] + ); +} + +#[tokio::test] +async fn import_plugins_supports_project_relative_external_agent_plugin_marketplace_path() { + let root = TempDir::new().expect("create tempdir"); + let external_agent_home = root.path().join(".claude"); + let codex_home = root.path().join(".codex"); + let repo_root = root.path().join("repo"); + let marketplace_root = repo_root.join("my-marketplace"); + let plugin_root = marketplace_root.join("plugins").join("cloudflare"); + fs::create_dir_all(repo_root.join(".git")).expect("create git dir"); + fs::create_dir_all(repo_root.join(".claude")).expect("create repo external agent dir"); + fs::create_dir_all(marketplace_root.join(".claude-plugin")) + .expect("create marketplace manifest dir"); + fs::create_dir_all(plugin_root.join(".codex-plugin")).expect("create plugin manifest dir"); + fs::create_dir_all(&codex_home).expect("create codex home"); + + fs::write( + repo_root.join(".claude").join("settings.json"), + r#"{ + "enabledPlugins": { + "cloudflare@my-plugins": true + }, + "extraKnownMarketplaces": { + "my-plugins": { + "source": "directory", + "path": "./my-marketplace" + } + } + }"#, + ) + .expect("write settings"); + fs::write( + marketplace_root + .join(".claude-plugin") + .join("marketplace.json"), + r#"{ + "name": "my-plugins", + "plugins": [ + { + "name": "cloudflare", + "source": "./plugins/cloudflare" + } + ] + }"#, + ) + .expect("write marketplace manifest"); + fs::write( + plugin_root.join(".codex-plugin").join("plugin.json"), + r#"{"name":"cloudflare","version":"0.1.0"}"#, + ) + .expect("write plugin manifest"); + + let outcome = service_for_paths(external_agent_home, codex_home.clone()) + .import_plugins( + Some(repo_root.as_path()), + Some(MigrationDetails { + plugins: vec![PluginsMigration { + marketplace_name: "my-plugins".to_string(), + plugin_names: vec!["cloudflare".to_string()], + }], + }), + ) + .await + .expect("import plugins"); + + assert_eq!( + outcome, + PluginImportOutcome { + succeeded_marketplaces: vec!["my-plugins".to_string()], + succeeded_plugin_ids: vec!["cloudflare@my-plugins".to_string()], + failed_marketplaces: Vec::new(), + failed_plugin_ids: Vec::new(), + } + ); + let config = fs::read_to_string(codex_home.join("config.toml")).expect("read config"); + assert!(config.contains(r#"[plugins."cloudflare@my-plugins"]"#)); + assert!(config.contains("enabled = true")); +} + #[test] fn import_skills_returns_only_new_skill_directory_count() { - let (_root, claude_home, codex_home) = fixture_paths(); + let (_root, external_agent_home, codex_home) = fixture_paths(); let agents_skills = codex_home .parent() .map(|parent| parent.join(".agents").join("skills")) .unwrap_or_else(|| PathBuf::from(".agents").join("skills")); - fs::create_dir_all(claude_home.join("skills").join("skill-a")).expect("create source a"); - fs::create_dir_all(claude_home.join("skills").join("skill-b")).expect("create source b"); + fs::create_dir_all(external_agent_home.join("skills").join("skill-a")) + .expect("create source a"); + fs::create_dir_all(external_agent_home.join("skills").join("skill-b")) + .expect("create source b"); fs::create_dir_all(agents_skills.join("skill-a")).expect("create existing target"); - let copied_count = service_for_paths(claude_home, codex_home) + let copied_count = service_for_paths(external_agent_home, codex_home) .import_skills(/*cwd*/ None) .expect("import skills"); diff --git a/codex-rs/core/src/plugins/manager.rs b/codex-rs/core/src/plugins/manager.rs index 50c62d5823..3449ecffdf 100644 --- a/codex-rs/core/src/plugins/manager.rs +++ b/codex-rs/core/src/plugins/manager.rs @@ -1346,7 +1346,7 @@ impl PluginUninstallError { } } -fn configured_plugins_from_stack( +pub(crate) fn configured_plugins_from_stack( config_layer_stack: &ConfigLayerStack, ) -> HashMap { // Plugin entries remain persisted user config only. diff --git a/codex-rs/core/src/plugins/marketplace_add.rs b/codex-rs/core/src/plugins/marketplace_add.rs index b8d1b34d41..38fc04c199 100644 --- a/codex-rs/core/src/plugins/marketplace_add.rs +++ b/codex-rs/core/src/plugins/marketplace_add.rs @@ -20,7 +20,7 @@ use metadata::find_marketplace_root_by_name; use metadata::installed_marketplace_root_for_source; use metadata::record_added_marketplace_entry; use source::MarketplaceSource; -use source::parse_marketplace_source; +pub(crate) use source::parse_marketplace_source; use source::stage_marketplace_source; use source::validate_marketplace_source_root; diff --git a/codex-rs/core/src/plugins/marketplace_add/source.rs b/codex-rs/core/src/plugins/marketplace_add/source.rs index 9a0b666c60..98bdc2b8bb 100644 --- a/codex-rs/core/src/plugins/marketplace_add/source.rs +++ b/codex-rs/core/src/plugins/marketplace_add/source.rs @@ -5,7 +5,7 @@ use std::path::Path; use std::path::PathBuf; #[derive(Debug, Clone, PartialEq, Eq)] -pub(super) enum MarketplaceSource { +pub(crate) enum MarketplaceSource { Git { url: String, ref_name: Option, @@ -15,7 +15,7 @@ pub(super) enum MarketplaceSource { }, } -pub(super) fn parse_marketplace_source( +pub(crate) fn parse_marketplace_source( source: &str, explicit_ref: Option, ) -> Result { diff --git a/codex-rs/core/src/plugins/mod.rs b/codex-rs/core/src/plugins/mod.rs index 57997eb134..2e86643f34 100644 --- a/codex-rs/core/src/plugins/mod.rs +++ b/codex-rs/core/src/plugins/mod.rs @@ -22,6 +22,7 @@ pub use codex_plugin::validate_plugin_segment; pub type LoadedPlugin = codex_plugin::LoadedPlugin; pub type PluginLoadOutcome = codex_plugin::PluginLoadOutcome; +pub(crate) use codex_core_plugins::marketplace::find_marketplace_manifest_path; pub(crate) use discoverable::list_tool_suggest_discoverable_plugins; pub(crate) use injection::build_plugin_injections; pub use installed_marketplaces::INSTALLED_MARKETPLACES_DIR; @@ -40,10 +41,12 @@ pub use manager::PluginRemoteSyncError; pub use manager::PluginUninstallError; pub use manager::PluginsManager; pub use manager::RemotePluginSyncResult; +pub(crate) use manager::configured_plugins_from_stack; pub use marketplace_add::MarketplaceAddError; pub use marketplace_add::MarketplaceAddOutcome; pub use marketplace_add::MarketplaceAddRequest; pub use marketplace_add::add_marketplace; +pub(crate) use marketplace_add::parse_marketplace_source; pub(crate) use render::render_explicit_plugin_instructions; pub(crate) use render::render_plugins_section; pub(crate) use startup_sync::curated_plugins_repo_path; From faf48489f3ade607701de064300ee127b1fb71d8 Mon Sep 17 00:00:00 2001 From: xli-oai Date: Thu, 16 Apr 2026 10:36:34 -0700 Subject: [PATCH 81/84] Auto-upgrade configured marketplaces (#17425) ## Summary - Add best-effort auto-upgrade for user-configured Git marketplaces recorded in `config.toml`. - Track the last activated Git revision with `last_revision` so unchanged marketplace sources skip clone work. - Trigger the upgrade from plugin startup and `plugin/list`, while preserving existing fail-open plugin behavior with warning logs rather than new user-visible errors. ## Details - Remote configured marketplaces use `git ls-remote` to compare the source/ref against the recorded revision. - Upgrades clone into a staging directory, validate that `.agents/plugins/marketplace.json` exists and that the manifest name matches the configured marketplace key, then atomically activate the new root. - Local `.agents/plugins/marketplace.json` marketplaces remain live filesystem state and are not auto-pulled. - Existing non-curated plugin cache refresh is kicked after successful marketplace root upgrades. ## Validation - `just write-config-schema` - `cargo test -p codex-core marketplace_upgrade` - `cargo check -p codex-cli -p codex-app-server` - `just fix -p codex-core` Did not run the complete `cargo test` suite because the repo instructions require asking before a full core workspace run. --- codex-rs/Cargo.lock | 1 + .../app-server/src/codex_message_processor.rs | 2 +- codex-rs/cli/src/marketplace_cmd.rs | 81 ++++- codex-rs/config/src/marketplace_edit.rs | 4 + codex-rs/config/src/types.rs | 3 + codex-rs/core-plugins/Cargo.toml | 1 + codex-rs/core-plugins/src/lib.rs | 1 + codex-rs/core-plugins/src/loader.rs | 33 +- .../core-plugins/src/marketplace_upgrade.rs | 298 ++++++++++++++++++ .../src/marketplace_upgrade/activation.rs | 167 ++++++++++ .../src/marketplace_upgrade/git.rs | 238 ++++++++++++++ codex-rs/core/config.schema.json | 5 + codex-rs/core/src/plugins/manager.rs | 209 ++++++++++-- codex-rs/core/src/plugins/manager_tests.rs | 64 ++++ .../src/plugins/marketplace_add/metadata.rs | 1 + .../src/plugins/marketplace_add/source.rs | 18 ++ codex-rs/core/src/plugins/mod.rs | 2 + 17 files changed, 1094 insertions(+), 34 deletions(-) create mode 100644 codex-rs/core-plugins/src/marketplace_upgrade.rs create mode 100644 codex-rs/core-plugins/src/marketplace_upgrade/activation.rs create mode 100644 codex-rs/core-plugins/src/marketplace_upgrade/git.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index a6d1ec2e47..a499a511ca 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2019,6 +2019,7 @@ dependencies = [ name = "codex-core-plugins" version = "0.0.0" dependencies = [ + "chrono", "codex-app-server-protocol", "codex-config", "codex-core-skills", diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 4d12df89fa..25892e40eb 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -6178,7 +6178,7 @@ impl CodexMessageProcessor { force_remote_sync, } = params; let roots = cwds.unwrap_or_default(); - plugins_manager.maybe_start_non_curated_plugin_cache_refresh_for_roots(&roots); + plugins_manager.maybe_start_non_curated_plugin_cache_refresh(&roots); let mut config = match self.load_latest_config(/*fallback_cwd*/ None).await { Ok(config) => config, diff --git a/codex-rs/cli/src/marketplace_cmd.rs b/codex-rs/cli/src/marketplace_cmd.rs index cde50a2bf4..ce1f99390a 100644 --- a/codex-rs/cli/src/marketplace_cmd.rs +++ b/codex-rs/cli/src/marketplace_cmd.rs @@ -1,8 +1,12 @@ use anyhow::Context; use anyhow::Result; +use anyhow::bail; use clap::Parser; +use codex_core::config::Config; use codex_core::config::find_codex_home; use codex_core::plugins::MarketplaceAddRequest; +use codex_core::plugins::PluginMarketplaceUpgradeOutcome; +use codex_core::plugins::PluginsManager; use codex_core::plugins::add_marketplace; use codex_utils_cli::CliConfigOverrides; @@ -17,8 +21,8 @@ pub struct MarketplaceCli { #[derive(Debug, clap::Subcommand)] enum MarketplaceSubcommand { - /// Add a remote marketplace repository. Add(AddMarketplaceArgs), + Upgrade(UpgradeMarketplaceArgs), } #[derive(Debug, Parser)] @@ -27,11 +31,9 @@ struct AddMarketplaceArgs { /// or local marketplace root directories. source: String, - /// Git ref to check out. Overrides any @ref or #ref suffix in SOURCE. #[arg(long = "ref", value_name = "REF")] ref_name: Option, - /// Sparse-checkout path to use while cloning git sources. Repeat to include multiple paths. #[arg( long = "sparse", value_name = "PATH", @@ -40,6 +42,11 @@ struct AddMarketplaceArgs { sparse_paths: Vec, } +#[derive(Debug, Parser)] +struct UpgradeMarketplaceArgs { + marketplace_name: Option, +} + impl MarketplaceCli { pub async fn run(self) -> Result<()> { let MarketplaceCli { @@ -47,14 +54,13 @@ impl MarketplaceCli { subcommand, } = self; - // Validate overrides now. This command writes to CODEX_HOME only; marketplace discovery - // happens from that cache root after the next plugin/list or app-server start. - config_overrides + let overrides = config_overrides .parse_overrides() .map_err(anyhow::Error::msg)?; match subcommand { MarketplaceSubcommand::Add(args) => run_add(args).await?, + MarketplaceSubcommand::Upgrade(args) => run_upgrade(overrides, args).await?, } Ok(()) @@ -98,6 +104,60 @@ async fn run_add(args: AddMarketplaceArgs) -> Result<()> { Ok(()) } +async fn run_upgrade( + overrides: Vec<(String, toml::Value)>, + args: UpgradeMarketplaceArgs, +) -> Result<()> { + let UpgradeMarketplaceArgs { marketplace_name } = args; + let config = Config::load_with_cli_overrides(overrides) + .await + .context("failed to load configuration")?; + let codex_home = find_codex_home().context("failed to resolve CODEX_HOME")?; + let manager = PluginsManager::new(codex_home.to_path_buf()); + let outcome = manager + .upgrade_configured_marketplaces_for_config(&config, marketplace_name.as_deref()) + .map_err(anyhow::Error::msg)?; + print_upgrade_outcome(&outcome, marketplace_name.as_deref()) +} + +fn print_upgrade_outcome( + outcome: &PluginMarketplaceUpgradeOutcome, + marketplace_name: Option<&str>, +) -> Result<()> { + for error in &outcome.errors { + eprintln!( + "Failed to upgrade marketplace `{}`: {}", + error.marketplace_name, error.message + ); + } + if !outcome.all_succeeded() { + bail!("{} upgrade failure(s) occurred.", outcome.errors.len()); + } + + let selection_label = marketplace_name.unwrap_or("all configured Git marketplaces"); + if outcome.selected_marketplaces.is_empty() { + println!("No configured Git marketplaces to upgrade."); + } else if outcome.upgraded_roots.is_empty() { + if marketplace_name.is_some() { + println!("Marketplace `{selection_label}` is already up to date."); + } else { + println!("All configured Git marketplaces are already up to date."); + } + } else if marketplace_name.is_some() { + println!("Upgraded marketplace `{selection_label}` to the latest configured revision."); + for root in &outcome.upgraded_roots { + println!("Installed marketplace root: {}", root.display()); + } + } else { + println!("Upgraded {} marketplace(s).", outcome.upgraded_roots.len()); + for root in &outcome.upgraded_roots { + println!("Installed marketplace root: {}", root.display()); + } + } + + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -132,4 +192,13 @@ mod tests { vec!["plugins/foo", "skills/bar"] ); } + + #[test] + fn upgrade_subcommand_parses_optional_marketplace_name() { + let upgrade_all = UpgradeMarketplaceArgs::try_parse_from(["upgrade"]).unwrap(); + assert_eq!(upgrade_all.marketplace_name, None); + + let upgrade_one = UpgradeMarketplaceArgs::try_parse_from(["upgrade", "debug"]).unwrap(); + assert_eq!(upgrade_one.marketplace_name.as_deref(), Some("debug")); + } } diff --git a/codex-rs/config/src/marketplace_edit.rs b/codex-rs/config/src/marketplace_edit.rs index 33cdd8e163..e20a75a02e 100644 --- a/codex-rs/config/src/marketplace_edit.rs +++ b/codex-rs/config/src/marketplace_edit.rs @@ -12,6 +12,7 @@ use crate::CONFIG_TOML_FILE; pub struct MarketplaceConfigUpdate<'a> { pub last_updated: &'a str, + pub last_revision: Option<&'a str>, pub source_type: &'a str, pub source: &'a str, pub ref_name: Option<&'a str>, @@ -63,6 +64,9 @@ fn upsert_marketplace( let mut entry = TomlTable::new(); entry.set_implicit(false); entry["last_updated"] = value(update.last_updated.to_string()); + if let Some(last_revision) = update.last_revision { + entry["last_revision"] = value(last_revision.to_string()); + } entry["source_type"] = value(update.source_type.to_string()); entry["source"] = value(update.source.to_string()); if let Some(ref_name) = update.ref_name { diff --git a/codex-rs/config/src/types.rs b/codex-rs/config/src/types.rs index 15c1a6fad5..b3576cb3c1 100644 --- a/codex-rs/config/src/types.rs +++ b/codex-rs/config/src/types.rs @@ -626,6 +626,9 @@ pub struct MarketplaceConfig { /// Last time Codex successfully added or refreshed this marketplace. #[serde(default)] pub last_updated: Option, + /// Git revision Codex last successfully activated for this marketplace. + #[serde(default)] + pub last_revision: Option, /// Source kind used to install this marketplace. #[serde(default)] pub source_type: Option, diff --git a/codex-rs/core-plugins/Cargo.toml b/codex-rs/core-plugins/Cargo.toml index 02e121f0e6..0372d9a14a 100644 --- a/codex-rs/core-plugins/Cargo.toml +++ b/codex-rs/core-plugins/Cargo.toml @@ -23,6 +23,7 @@ codex-plugin = { workspace = true } codex-protocol = { workspace = true } codex-utils-absolute-path = { workspace = true } codex-utils-plugins = { workspace = true } +chrono = { workspace = true } dirs = { workspace = true } reqwest = { workspace = true } serde = { workspace = true, features = ["derive"] } diff --git a/codex-rs/core-plugins/src/lib.rs b/codex-rs/core-plugins/src/lib.rs index e075158cb1..82ff4df3c7 100644 --- a/codex-rs/core-plugins/src/lib.rs +++ b/codex-rs/core-plugins/src/lib.rs @@ -1,6 +1,7 @@ pub mod loader; pub mod manifest; pub mod marketplace; +pub mod marketplace_upgrade; pub mod remote; pub mod store; pub mod toggles; diff --git a/codex-rs/core-plugins/src/loader.rs b/codex-rs/core-plugins/src/loader.rs index 206add5b53..ff3b8fc494 100644 --- a/codex-rs/core-plugins/src/loader.rs +++ b/codex-rs/core-plugins/src/loader.rs @@ -41,6 +41,12 @@ const DEFAULT_APP_CONFIG_FILE: &str = ".app.json"; const OPENAI_CURATED_MARKETPLACE_NAME: &str = "openai-curated"; const CONFIG_TOML_FILE: &str = "config.toml"; +#[derive(Clone, Copy, PartialEq, Eq)] +enum NonCuratedCacheRefreshMode { + IfVersionChanged, + ForceReinstall, +} + pub fn log_plugin_load_errors(outcome: &PluginLoadOutcome) { for plugin in outcome .plugins() @@ -180,6 +186,29 @@ pub fn refresh_curated_plugin_cache( pub fn refresh_non_curated_plugin_cache( codex_home: &Path, additional_roots: &[AbsolutePathBuf], +) -> Result { + refresh_non_curated_plugin_cache_with_mode( + codex_home, + additional_roots, + NonCuratedCacheRefreshMode::IfVersionChanged, + ) +} + +pub fn refresh_non_curated_plugin_cache_force_reinstall( + codex_home: &Path, + additional_roots: &[AbsolutePathBuf], +) -> Result { + refresh_non_curated_plugin_cache_with_mode( + codex_home, + additional_roots, + NonCuratedCacheRefreshMode::ForceReinstall, + ) +} + +fn refresh_non_curated_plugin_cache_with_mode( + codex_home: &Path, + additional_roots: &[AbsolutePathBuf], + mode: NonCuratedCacheRefreshMode, ) -> Result { let configured_non_curated_plugin_ids = non_curated_plugin_ids_from_config_keys(configured_plugins_from_codex_home( @@ -248,7 +277,9 @@ pub fn refresh_non_curated_plugin_cache( continue; }; - if store.active_plugin_version(&plugin_id).as_deref() == Some(plugin_version.as_str()) { + if mode == NonCuratedCacheRefreshMode::IfVersionChanged + && store.active_plugin_version(&plugin_id).as_deref() == Some(plugin_version.as_str()) + { continue; } diff --git a/codex-rs/core-plugins/src/marketplace_upgrade.rs b/codex-rs/core-plugins/src/marketplace_upgrade.rs new file mode 100644 index 0000000000..81474cf66f --- /dev/null +++ b/codex-rs/core-plugins/src/marketplace_upgrade.rs @@ -0,0 +1,298 @@ +mod activation; +mod git; + +use self::activation::activate_marketplace_root; +use self::activation::installed_marketplace_metadata_matches; +use self::activation::write_installed_marketplace_metadata; +use self::git::clone_git_source; +use self::git::git_remote_revision; +use crate::marketplace::validate_marketplace_root; +use codex_config::CONFIG_TOML_FILE; +use codex_config::ConfigLayerStack; +use codex_config::MarketplaceConfigUpdate; +use codex_config::record_user_marketplace; +use codex_config::types::MarketplaceConfig; +use codex_config::types::MarketplaceSourceType; +use codex_plugin::validate_plugin_segment; +use codex_utils_absolute_path::AbsolutePathBuf; +use std::collections::HashMap; +use std::path::Path; +use std::path::PathBuf; +use std::time::Duration; +use tracing::warn; + +const INSTALLED_MARKETPLACES_DIR: &str = ".tmp/marketplaces"; +const MARKETPLACE_UPGRADE_GIT_TIMEOUT: Duration = Duration::from_secs(30); + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ConfiguredMarketplaceUpgradeError { + pub marketplace_name: String, + pub message: String, +} + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct ConfiguredMarketplaceUpgradeOutcome { + pub selected_marketplaces: Vec, + pub upgraded_roots: Vec, + pub errors: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ConfiguredGitMarketplace { + name: String, + source: String, + ref_name: Option, + sparse_paths: Vec, + last_revision: Option, +} + +impl ConfiguredMarketplaceUpgradeOutcome { + pub fn all_succeeded(&self) -> bool { + self.errors.is_empty() + } +} + +pub fn configured_git_marketplace_names(config_layer_stack: &ConfigLayerStack) -> Vec { + let mut names = configured_git_marketplaces(config_layer_stack) + .into_iter() + .map(|marketplace| marketplace.name) + .collect::>(); + names.sort_unstable(); + names +} + +pub fn upgrade_configured_git_marketplaces( + codex_home: &Path, + config_layer_stack: &ConfigLayerStack, + marketplace_name: Option<&str>, +) -> ConfiguredMarketplaceUpgradeOutcome { + let marketplaces = configured_git_marketplaces(config_layer_stack) + .into_iter() + .filter(|marketplace| marketplace_name.is_none_or(|name| marketplace.name.as_str() == name)) + .collect::>(); + if marketplaces.is_empty() { + return ConfiguredMarketplaceUpgradeOutcome::default(); + } + + let install_root = marketplace_install_root(codex_home); + let selected_marketplaces = marketplaces + .iter() + .map(|marketplace| marketplace.name.clone()) + .collect(); + let mut upgraded_roots = Vec::new(); + let mut errors = Vec::new(); + for marketplace in marketplaces { + match upgrade_configured_git_marketplace(codex_home, &install_root, &marketplace) { + Ok(Some(upgraded_root)) => upgraded_roots.push(upgraded_root), + Ok(None) => {} + Err(err) => { + errors.push(ConfiguredMarketplaceUpgradeError { + marketplace_name: marketplace.name, + message: err, + }); + } + } + } + + ConfiguredMarketplaceUpgradeOutcome { + selected_marketplaces, + upgraded_roots, + errors, + } +} + +fn marketplace_install_root(codex_home: &Path) -> PathBuf { + codex_home.join(INSTALLED_MARKETPLACES_DIR) +} + +fn configured_git_marketplaces( + config_layer_stack: &ConfigLayerStack, +) -> Vec { + let Some(user_layer) = config_layer_stack.get_user_layer() else { + return Vec::new(); + }; + let Some(marketplaces_value) = user_layer.config.get("marketplaces") else { + return Vec::new(); + }; + let marketplaces = match marketplaces_value + .clone() + .try_into::>() + { + Ok(marketplaces) => marketplaces, + Err(err) => { + warn!("invalid marketplaces config while preparing auto-upgrade: {err}"); + return Vec::new(); + } + }; + + let mut configured = marketplaces + .into_iter() + .filter_map(|(name, marketplace)| configured_git_marketplace_from_config(name, marketplace)) + .collect::>(); + configured.sort_unstable_by(|left, right| left.name.cmp(&right.name)); + configured +} + +fn configured_git_marketplace_from_config( + name: String, + marketplace: MarketplaceConfig, +) -> Option { + let MarketplaceConfig { + last_updated: _, + last_revision, + source_type, + source, + ref_name, + sparse_paths, + } = marketplace; + if source_type != Some(MarketplaceSourceType::Git) { + return None; + } + let Some(source) = source else { + warn!( + marketplace = name, + "ignoring configured Git marketplace without source" + ); + return None; + }; + Some(ConfiguredGitMarketplace { + name, + source, + ref_name, + sparse_paths: sparse_paths.unwrap_or_default(), + last_revision, + }) +} + +fn upgrade_configured_git_marketplace( + codex_home: &Path, + install_root: &Path, + marketplace: &ConfiguredGitMarketplace, +) -> Result, String> { + validate_plugin_segment(&marketplace.name, "marketplace name")?; + let remote_revision = git_remote_revision( + &marketplace.source, + marketplace.ref_name.as_deref(), + MARKETPLACE_UPGRADE_GIT_TIMEOUT, + )?; + let destination = install_root.join(&marketplace.name); + if destination + .join(".agents/plugins/marketplace.json") + .is_file() + && marketplace.last_revision.as_deref() == Some(remote_revision.as_str()) + && installed_marketplace_metadata_matches(&destination, marketplace, &remote_revision) + { + return Ok(None); + } + + let staging_parent = install_root.join(".staging"); + std::fs::create_dir_all(&staging_parent).map_err(|err| { + format!( + "failed to create marketplace upgrade staging directory {}: {err}", + staging_parent.display() + ) + })?; + let staged_dir = tempfile::Builder::new() + .prefix("marketplace-upgrade-") + .tempdir_in(&staging_parent) + .map_err(|err| { + format!( + "failed to create temporary marketplace upgrade directory in {}: {err}", + staging_parent.display() + ) + })?; + + let activated_revision = clone_git_source( + &marketplace.source, + marketplace.ref_name.as_deref(), + &marketplace.sparse_paths, + staged_dir.path(), + MARKETPLACE_UPGRADE_GIT_TIMEOUT, + )?; + let marketplace_name = validate_marketplace_root(staged_dir.path()) + .map_err(|err| format!("failed to validate upgraded marketplace root: {err}"))?; + if marketplace_name != marketplace.name { + return Err(format!( + "upgraded marketplace name `{marketplace_name}` does not match configured marketplace `{}`", + marketplace.name + )); + } + write_installed_marketplace_metadata(staged_dir.path(), marketplace, &activated_revision)?; + + let last_updated = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true); + let update = MarketplaceConfigUpdate { + last_updated: &last_updated, + last_revision: Some(&activated_revision), + source_type: "git", + source: &marketplace.source, + ref_name: marketplace.ref_name.as_deref(), + sparse_paths: &marketplace.sparse_paths, + }; + activate_marketplace_root(&destination, staged_dir, || { + ensure_configured_git_marketplace_unchanged(codex_home, marketplace)?; + record_user_marketplace(codex_home, &marketplace.name, &update).map_err(|err| { + format!( + "failed to record upgraded marketplace `{}` in user config.toml: {err}", + marketplace.name + ) + }) + })?; + + AbsolutePathBuf::try_from(destination) + .map(Some) + .map_err(|err| format!("upgraded marketplace path is not absolute: {err}")) +} +fn ensure_configured_git_marketplace_unchanged( + codex_home: &Path, + expected: &ConfiguredGitMarketplace, +) -> Result<(), String> { + let current = read_configured_git_marketplace(codex_home, &expected.name)?; + match current { + Some(current) if current == *expected => Ok(()), + Some(_) => Err(format!( + "configured marketplace `{}` changed while auto-upgrade was in flight", + expected.name + )), + None => Err(format!( + "configured marketplace `{}` was removed or is no longer a Git marketplace", + expected.name + )), + } +} + +fn read_configured_git_marketplace( + codex_home: &Path, + marketplace_name: &str, +) -> Result, String> { + let config_path = codex_home.join(CONFIG_TOML_FILE); + let raw_config = match std::fs::read_to_string(&config_path) { + Ok(raw_config) => raw_config, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(err) => { + return Err(format!( + "failed to read user config {} while checking marketplace auto-upgrade: {err}", + config_path.display() + )); + } + }; + let config: toml::Value = toml::from_str(&raw_config).map_err(|err| { + format!( + "failed to parse user config {} while checking marketplace auto-upgrade: {err}", + config_path.display() + ) + })?; + let Some(marketplaces_value) = config.get("marketplaces") else { + return Ok(None); + }; + let mut marketplaces = marketplaces_value + .clone() + .try_into::>() + .map_err(|err| format!("invalid marketplaces config while checking auto-upgrade: {err}"))?; + let Some(marketplace) = marketplaces.remove(marketplace_name) else { + return Ok(None); + }; + Ok(configured_git_marketplace_from_config( + marketplace_name.to_string(), + marketplace, + )) +} diff --git a/codex-rs/core-plugins/src/marketplace_upgrade/activation.rs b/codex-rs/core-plugins/src/marketplace_upgrade/activation.rs new file mode 100644 index 0000000000..366b35fb43 --- /dev/null +++ b/codex-rs/core-plugins/src/marketplace_upgrade/activation.rs @@ -0,0 +1,167 @@ +use super::ConfiguredGitMarketplace; +use codex_config::types::MarketplaceSourceType; +use serde::Deserialize; +use serde::Serialize; +use std::path::Path; +use std::path::PathBuf; +use tempfile::TempDir; +use tracing::warn; + +const MARKETPLACE_INSTALL_METADATA_FILE: &str = ".codex-marketplace-install.json"; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +struct InstalledMarketplaceMetadata { + source_type: MarketplaceSourceType, + source: String, + ref_name: Option, + sparse_paths: Vec, + revision: String, +} + +pub(super) fn installed_marketplace_metadata_matches( + root: &Path, + marketplace: &ConfiguredGitMarketplace, + revision: &str, +) -> bool { + let metadata = match std::fs::read_to_string(installed_marketplace_metadata_path(root)) { + Ok(metadata) => metadata, + Err(_) => return false, + }; + let metadata = match serde_json::from_str::(&metadata) { + Ok(metadata) => metadata, + Err(err) => { + warn!( + marketplace = marketplace.name, + error = %err, + "failed to parse activated marketplace metadata" + ); + return false; + } + }; + metadata == installed_marketplace_metadata(marketplace, revision) +} + +pub(super) fn write_installed_marketplace_metadata( + root: &Path, + marketplace: &ConfiguredGitMarketplace, + revision: &str, +) -> Result<(), String> { + let metadata = installed_marketplace_metadata(marketplace, revision); + let contents = serde_json::to_string_pretty(&metadata) + .map_err(|err| format!("failed to serialize activated marketplace metadata: {err}"))?; + std::fs::write(installed_marketplace_metadata_path(root), contents) + .map_err(|err| format!("failed to write activated marketplace metadata: {err}")) +} + +pub(super) fn activate_marketplace_root( + destination: &Path, + staged_dir: TempDir, + after_activate: impl FnOnce() -> Result<(), String>, +) -> Result<(), String> { + let staged_root = staged_dir.path(); + let Some(parent) = destination.parent() else { + return Err(format!( + "failed to determine marketplace install parent for {}", + destination.display() + )); + }; + std::fs::create_dir_all(parent).map_err(|err| { + format!( + "failed to create marketplace install parent {}: {err}", + parent.display() + ) + })?; + + if destination.exists() { + let backup_dir = tempfile::Builder::new() + .prefix("marketplace-backup-") + .tempdir_in(parent) + .map_err(|err| { + format!( + "failed to create marketplace backup directory in {}: {err}", + parent.display() + ) + })?; + let backup_root = backup_dir.path().join("root"); + std::fs::rename(destination, &backup_root).map_err(|err| { + format!( + "failed to move previous marketplace root out of the way at {}: {err}", + destination.display() + ) + })?; + + if let Err(err) = std::fs::rename(staged_root, destination) { + let rollback_result = std::fs::rename(&backup_root, destination); + return match rollback_result { + Ok(()) => Err(format!( + "failed to activate upgraded marketplace at {}: {err}", + destination.display() + )), + Err(rollback_err) => { + let backup_path = backup_dir.keep().join("root"); + Err(format!( + "failed to activate upgraded marketplace at {}: {err}; failed to restore previous marketplace root (left at {}): {rollback_err}", + destination.display(), + backup_path.display() + )) + } + }; + } + + if let Err(err) = after_activate() { + let remove_result = std::fs::remove_dir_all(destination); + let rollback_result = + remove_result.and_then(|()| std::fs::rename(&backup_root, destination)); + return match rollback_result { + Ok(()) => Err(err), + Err(rollback_err) => { + let backup_path = backup_dir.keep().join("root"); + Err(format!( + "{err}; failed to restore previous marketplace root at {} (left at {}): {rollback_err}", + destination.display(), + backup_path.display() + )) + } + }; + } + + return Ok(()); + } + + std::fs::rename(staged_root, destination).map_err(|err| { + format!( + "failed to activate upgraded marketplace at {}: {err}", + destination.display() + ) + })?; + if let Err(err) = after_activate() { + let remove_result = std::fs::remove_dir_all(destination); + return match remove_result { + Ok(()) => Err(err), + Err(remove_err) => Err(format!( + "{err}; failed to remove newly activated marketplace root at {}: {remove_err}", + destination.display() + )), + }; + } + + Ok(()) +} + +fn installed_marketplace_metadata( + marketplace: &ConfiguredGitMarketplace, + revision: &str, +) -> InstalledMarketplaceMetadata { + InstalledMarketplaceMetadata { + source_type: MarketplaceSourceType::Git, + source: marketplace.source.clone(), + ref_name: marketplace.ref_name.clone(), + sparse_paths: marketplace.sparse_paths.clone(), + revision: revision.to_string(), + } +} + +fn installed_marketplace_metadata_path(root: &Path) -> PathBuf { + root.join(MARKETPLACE_INSTALL_METADATA_FILE) +} diff --git a/codex-rs/core-plugins/src/marketplace_upgrade/git.rs b/codex-rs/core-plugins/src/marketplace_upgrade/git.rs new file mode 100644 index 0000000000..80a7c68f68 --- /dev/null +++ b/codex-rs/core-plugins/src/marketplace_upgrade/git.rs @@ -0,0 +1,238 @@ +use std::path::Path; +use std::process::Command; +use std::process::Output; +use std::process::Stdio; +use std::time::Duration; + +pub(super) fn git_remote_revision( + source: &str, + ref_name: Option<&str>, + timeout: Duration, +) -> Result { + if let Some(ref_name) = ref_name + && is_full_git_sha(ref_name) + { + return Ok(ref_name.to_string()); + } + + let ref_name = ref_name.unwrap_or("HEAD"); + let output = run_git_command_with_timeout( + git_command().arg("ls-remote").arg(source).arg(ref_name), + "git ls-remote marketplace source", + timeout, + )?; + ensure_git_success(&output, "git ls-remote marketplace source")?; + + let stdout = String::from_utf8_lossy(&output.stdout); + let Some(first_line) = stdout.lines().next() else { + return Err("git ls-remote returned empty output for marketplace source".to_string()); + }; + let Some((revision, _)) = first_line.split_once('\t') else { + return Err(format!( + "unexpected git ls-remote output for marketplace source: {first_line}" + )); + }; + let revision = revision.trim(); + if revision.is_empty() { + return Err("git ls-remote returned empty revision for marketplace source".to_string()); + } + Ok(revision.to_string()) +} + +pub(super) fn clone_git_source( + source: &str, + ref_name: Option<&str>, + sparse_paths: &[String], + destination: &Path, + timeout: Duration, +) -> Result { + if sparse_paths.is_empty() { + let output = run_git_command_with_timeout( + git_command().arg("clone").arg(source).arg(destination), + "git clone marketplace source", + timeout, + )?; + ensure_git_success(&output, "git clone marketplace source")?; + if let Some(ref_name) = ref_name { + let output = run_git_command_with_timeout( + git_command() + .arg("-C") + .arg(destination) + .arg("checkout") + .arg(ref_name), + "git checkout marketplace ref", + timeout, + )?; + ensure_git_success(&output, "git checkout marketplace ref")?; + } + return git_worktree_revision(destination, timeout); + } + + let output = run_git_command_with_timeout( + git_command() + .arg("clone") + .arg("--filter=blob:none") + .arg("--no-checkout") + .arg(source) + .arg(destination), + "git clone marketplace source", + timeout, + )?; + ensure_git_success(&output, "git clone marketplace source")?; + + let mut sparse_checkout = git_command(); + sparse_checkout + .arg("-C") + .arg(destination) + .arg("sparse-checkout") + .arg("set") + .args(sparse_paths); + let output = run_git_command_with_timeout( + &mut sparse_checkout, + "git sparse-checkout marketplace source", + timeout, + )?; + ensure_git_success(&output, "git sparse-checkout marketplace source")?; + + let output = run_git_command_with_timeout( + git_command() + .arg("-C") + .arg(destination) + .arg("checkout") + .arg(ref_name.unwrap_or("HEAD")), + "git checkout marketplace ref", + timeout, + )?; + ensure_git_success(&output, "git checkout marketplace ref")?; + git_worktree_revision(destination, timeout) +} + +fn git_worktree_revision(destination: &Path, timeout: Duration) -> Result { + let output = run_git_command_with_timeout( + git_command() + .arg("-C") + .arg(destination) + .arg("rev-parse") + .arg("HEAD"), + "git rev-parse marketplace revision", + timeout, + )?; + ensure_git_success(&output, "git rev-parse marketplace revision")?; + + let revision = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if revision.is_empty() { + Err("git rev-parse returned empty revision for marketplace source".to_string()) + } else { + Ok(revision) + } +} + +fn is_full_git_sha(value: &str) -> bool { + value.len() == 40 && value.chars().all(|ch| ch.is_ascii_hexdigit()) +} + +fn git_command() -> Command { + let mut command = Command::new("git"); + command + .env("GIT_OPTIONAL_LOCKS", "0") + .env("GIT_TERMINAL_PROMPT", "0"); + command +} + +fn run_git_command_with_timeout( + command: &mut Command, + context: &str, + timeout: Duration, +) -> Result { + let mut child = command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|err| format!("failed to run {context}: {err}"))?; + let start = std::time::Instant::now(); + loop { + match child.try_wait() { + Ok(Some(_)) => { + return child + .wait_with_output() + .map_err(|err| format!("failed to wait for {context}: {err}")); + } + Ok(None) => {} + Err(err) => return Err(format!("failed to poll {context}: {err}")), + } + + if start.elapsed() >= timeout { + let _ = child.kill(); + let output = child + .wait_with_output() + .map_err(|err| format!("failed to wait for {context} after timeout: {err}"))?; + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + return if stderr.is_empty() { + Err(format!("{context} timed out after {}s", timeout.as_secs())) + } else { + Err(format!( + "{context} timed out after {}s: {stderr}", + timeout.as_secs() + )) + }; + } + + std::thread::sleep(Duration::from_millis(100)); + } +} + +fn ensure_git_success(output: &Output, context: &str) -> Result<(), String> { + if output.status.success() { + return Ok(()); + } + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + if stderr.is_empty() { + Err(format!("{context} failed with status {}", output.status)) + } else { + Err(format!( + "{context} failed with status {}: {stderr}", + output.status + )) + } +} + +#[cfg(test)] +mod tests { + use super::git_command; + use super::is_full_git_sha; + use std::ffi::OsStr; + + #[test] + fn full_git_sha_ref_is_already_a_remote_revision() { + assert!(is_full_git_sha("0123456789abcdef0123456789abcdef01234567")); + assert!(!is_full_git_sha("main")); + assert!(!is_full_git_sha("0123456")); + } + + #[test] + fn git_command_uses_path_lookup_with_stable_noninteractive_env() { + let command = git_command(); + + assert_eq!(command.get_program(), OsStr::new("git")); + assert_eq!( + command_env(&command, "GIT_OPTIONAL_LOCKS"), + Some(Some(OsStr::new("0"))) + ); + assert_eq!( + command_env(&command, "GIT_TERMINAL_PROMPT"), + Some(Some(OsStr::new("0"))) + ); + assert_eq!(command_env(&command, "PATH"), None); + } + + fn command_env<'a>( + command: &'a std::process::Command, + name: &str, + ) -> Option> { + command + .get_envs() + .find(|(key, _)| key == &OsStr::new(name)) + .map(|(_, value)| value) + } +} diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index 2e381cd3f4..9c7eb853a9 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -779,6 +779,11 @@ "MarketplaceConfig": { "additionalProperties": false, "properties": { + "last_revision": { + "default": null, + "description": "Git revision Codex last successfully activated for this marketplace.", + "type": "string" + }, "last_updated": { "default": null, "description": "Last time Codex successfully added or refreshed this marketplace.", diff --git a/codex-rs/core/src/plugins/manager.rs b/codex-rs/core/src/plugins/manager.rs index 3449ecffdf..7a5991b03a 100644 --- a/codex-rs/core/src/plugins/manager.rs +++ b/codex-rs/core/src/plugins/manager.rs @@ -25,6 +25,7 @@ use codex_core_plugins::loader::log_plugin_load_errors; use codex_core_plugins::loader::plugin_telemetry_metadata_from_root; use codex_core_plugins::loader::refresh_curated_plugin_cache; use codex_core_plugins::loader::refresh_non_curated_plugin_cache; +use codex_core_plugins::loader::refresh_non_curated_plugin_cache_force_reinstall; use codex_core_plugins::manifest::PluginManifestInterface; use codex_core_plugins::manifest::load_plugin_manifest; use codex_core_plugins::marketplace::MarketplaceError; @@ -37,6 +38,10 @@ use codex_core_plugins::marketplace::ResolvedMarketplacePlugin; use codex_core_plugins::marketplace::list_marketplaces; use codex_core_plugins::marketplace::load_marketplace; use codex_core_plugins::marketplace::resolve_marketplace_plugin; +use codex_core_plugins::marketplace_upgrade::ConfiguredMarketplaceUpgradeError; +use codex_core_plugins::marketplace_upgrade::ConfiguredMarketplaceUpgradeOutcome; +use codex_core_plugins::marketplace_upgrade::configured_git_marketplace_names; +use codex_core_plugins::marketplace_upgrade::upgrade_configured_git_marketplaces; use codex_core_plugins::remote::RemotePluginFetchError; use codex_core_plugins::remote::RemotePluginMutationError; use codex_core_plugins::remote::RemotePluginServiceConfig; @@ -88,10 +93,27 @@ struct CachedFeaturedPluginIds { featured_plugin_ids: Vec, } +#[derive(Clone, PartialEq, Eq)] +struct NonCuratedCacheRefreshRequest { + roots: Vec, + mode: NonCuratedCacheRefreshMode, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum NonCuratedCacheRefreshMode { + IfVersionChanged, + ForceReinstall, +} + #[derive(Default)] struct NonCuratedCacheRefreshState { - requested_roots: Option>, - last_refreshed_roots: Option>, + requested: Option, + last_refreshed: Option, + in_flight: bool, +} + +#[derive(Default)] +struct ConfiguredMarketplaceUpgradeState { in_flight: bool, } @@ -314,6 +336,7 @@ pub struct PluginsManager { codex_home: PathBuf, store: PluginStore, featured_plugin_ids_cache: RwLock>, + configured_marketplace_upgrade_state: RwLock, non_curated_cache_refresh_state: RwLock, cached_enabled_outcome: RwLock>, remote_sync_lock: Mutex<()>, @@ -341,6 +364,9 @@ impl PluginsManager { codex_home: codex_home.clone(), store: PluginStore::new(codex_home), featured_plugin_ids_cache: RwLock::new(None), + configured_marketplace_upgrade_state: RwLock::new( + ConfiguredMarketplaceUpgradeState::default(), + ), non_curated_cache_refresh_state: RwLock::new(NonCuratedCacheRefreshState::default()), cached_enabled_outcome: RwLock::new(None), remote_sync_lock: Mutex::new(()), @@ -1074,6 +1100,57 @@ impl PluginsManager { ) { if config.features.enabled(Feature::Plugins) { self.start_curated_repo_sync(); + let should_spawn_marketplace_auto_upgrade = { + let mut state = match self.configured_marketplace_upgrade_state.write() { + Ok(state) => state, + Err(err) => err.into_inner(), + }; + if state.in_flight { + false + } else { + state.in_flight = true; + true + } + }; + if should_spawn_marketplace_auto_upgrade { + let manager = Arc::clone(self); + let config = config.clone(); + if let Err(err) = std::thread::Builder::new() + .name("plugins-marketplace-auto-upgrade".to_string()) + .spawn(move || { + let outcome = manager.upgrade_configured_marketplaces_for_config( + &config, /*marketplace_name*/ None, + ); + match outcome { + Ok(outcome) => { + for error in outcome.errors { + warn!( + marketplace = error.marketplace_name, + error = %error.message, + "failed to auto-upgrade configured marketplace" + ); + } + } + Err(err) => { + warn!("failed to auto-upgrade configured marketplaces: {err}"); + } + } + + let mut state = match manager.configured_marketplace_upgrade_state.write() { + Ok(state) => state, + Err(err) => err.into_inner(), + }; + state.in_flight = false; + }) + { + let mut state = match self.configured_marketplace_upgrade_state.write() { + Ok(state) => state, + Err(err) => err.into_inner(), + }; + state.in_flight = false; + warn!("failed to start configured marketplace auto-upgrade task: {err}"); + } + } start_startup_remote_plugin_sync_once( Arc::clone(self), self.codex_home.clone(), @@ -1098,9 +1175,66 @@ impl PluginsManager { } } - pub fn maybe_start_non_curated_plugin_cache_refresh_for_roots( + pub fn upgrade_configured_marketplaces_for_config( + &self, + config: &Config, + marketplace_name: Option<&str>, + ) -> Result { + if let Some(marketplace_name) = marketplace_name + && !configured_git_marketplace_names(&config.config_layer_stack) + .iter() + .any(|name| name == marketplace_name) + { + return Err(format!( + "marketplace `{marketplace_name}` is not configured as a Git marketplace" + )); + } + + let mut outcome = upgrade_configured_git_marketplaces( + self.codex_home.as_path(), + &config.config_layer_stack, + marketplace_name, + ); + if !outcome.upgraded_roots.is_empty() { + match refresh_non_curated_plugin_cache_force_reinstall( + self.codex_home.as_path(), + &outcome.upgraded_roots, + ) { + Ok(cache_refreshed) => { + if cache_refreshed { + self.clear_cache(); + } + } + Err(err) => { + self.clear_cache(); + outcome.errors.push(ConfiguredMarketplaceUpgradeError { + marketplace_name: marketplace_name + .unwrap_or("all configured marketplaces") + .to_string(), + message: format!( + "failed to refresh installed plugin cache after marketplace upgrade: {err}" + ), + }); + } + } + } + Ok(outcome) + } + + pub fn maybe_start_non_curated_plugin_cache_refresh( self: &Arc, roots: &[AbsolutePathBuf], + ) { + self.schedule_non_curated_plugin_cache_refresh( + roots, + NonCuratedCacheRefreshMode::IfVersionChanged, + ); + } + + fn schedule_non_curated_plugin_cache_refresh( + self: &Arc, + roots: &[AbsolutePathBuf], + mode: NonCuratedCacheRefreshMode, ) { let mut roots = roots.to_vec(); roots.sort_unstable(); @@ -1108,6 +1242,7 @@ impl PluginsManager { if roots.is_empty() { return; } + let request = NonCuratedCacheRefreshRequest { roots, mode }; let should_spawn = { let mut state = match self.non_curated_cache_refresh_state.write() { @@ -1115,13 +1250,25 @@ impl PluginsManager { Err(err) => err.into_inner(), }; // Collapse repeated plugin/list requests onto one worker and only queue another pass - // when the requested roots set actually changes. - if state.requested_roots.as_ref() == Some(&roots) - || (!state.in_flight && state.last_refreshed_roots.as_ref() == Some(&roots)) + // when the requested roots set actually changes. Forced reinstall requests are not + // deduped against the last completed pass because the same marketplace root path can + // point at newly activated files after an auto-upgrade. + if state.requested.as_ref() == Some(&request) + || (mode == NonCuratedCacheRefreshMode::IfVersionChanged + && !state.in_flight + && state.last_refreshed.as_ref() == Some(&request)) { return; } - state.requested_roots = Some(roots); + if mode == NonCuratedCacheRefreshMode::IfVersionChanged + && state.requested.as_ref().is_some_and(|requested| { + requested.mode == NonCuratedCacheRefreshMode::ForceReinstall + && requested.roots == request.roots + }) + { + return; + } + state.requested = Some(request); if state.in_flight { false } else { @@ -1143,7 +1290,7 @@ impl PluginsManager { Err(err) => err.into_inner(), }; state.in_flight = false; - state.requested_roots = None; + state.requested = None; warn!("failed to start non-curated plugin cache refresh task: {err}"); } } @@ -1192,15 +1339,15 @@ impl PluginsManager { fn run_non_curated_plugin_cache_refresh_loop(self: Arc) { loop { - let roots = { + let request = { let state = match self.non_curated_cache_refresh_state.read() { Ok(state) => state, Err(err) => err.into_inner(), }; - state.requested_roots.clone() + state.requested.clone() }; - let Some(roots) = roots else { + let Some(request) = request else { let mut state = match self.non_curated_cache_refresh_state.write() { Ok(state) => state, Err(err) => err.into_inner(), @@ -1209,30 +1356,40 @@ impl PluginsManager { return; }; - let refreshed = - match refresh_non_curated_plugin_cache(self.codex_home.as_path(), &roots) { - Ok(cache_refreshed) => { - if cache_refreshed { - self.clear_cache(); - } - true - } - Err(err) => { + let refresh_result = match request.mode { + NonCuratedCacheRefreshMode::IfVersionChanged => { + refresh_non_curated_plugin_cache(self.codex_home.as_path(), &request.roots) + } + NonCuratedCacheRefreshMode::ForceReinstall => { + refresh_non_curated_plugin_cache_force_reinstall( + self.codex_home.as_path(), + &request.roots, + ) + } + }; + let refreshed = match refresh_result { + Ok(cache_refreshed) => { + if cache_refreshed { self.clear_cache(); - warn!("failed to refresh non-curated plugin cache: {err}"); - false } - }; + true + } + Err(err) => { + self.clear_cache(); + warn!("failed to refresh non-curated plugin cache: {err}"); + false + } + }; let mut state = match self.non_curated_cache_refresh_state.write() { Ok(state) => state, Err(err) => err.into_inner(), }; if refreshed { - state.last_refreshed_roots = Some(roots.clone()); + state.last_refreshed = Some(request.clone()); } - if state.requested_roots.as_ref() == Some(&roots) { - state.requested_roots = None; + if state.requested.as_ref() == Some(&request) { + state.requested = None; state.in_flight = false; return; } diff --git a/codex-rs/core/src/plugins/manager_tests.rs b/codex-rs/core/src/plugins/manager_tests.rs index e12f52c82b..c82e1d174c 100644 --- a/codex-rs/core/src/plugins/manager_tests.rs +++ b/codex-rs/core/src/plugins/manager_tests.rs @@ -15,6 +15,8 @@ use crate::plugins::test_support::write_openai_curated_marketplace; use codex_app_server_protocol::ConfigLayerSource; use codex_config::McpServerConfig; use codex_config::types::McpServerTransportConfig; +use codex_core_plugins::loader::refresh_non_curated_plugin_cache; +use codex_core_plugins::loader::refresh_non_curated_plugin_cache_force_reinstall; use codex_core_plugins::marketplace::MarketplacePluginInstallPolicy; use codex_login::CodexAuth; use codex_protocol::protocol::Product; @@ -2676,6 +2678,68 @@ enabled = true ); } +#[test] +fn refresh_non_curated_plugin_cache_force_reinstalls_current_local_version() { + let tmp = tempfile::tempdir().unwrap(); + let repo_root = tmp.path().join("repo"); + fs::create_dir_all(repo_root.join(".git")).unwrap(); + fs::create_dir_all(repo_root.join(".agents/plugins")).unwrap(); + write_plugin(&repo_root, "sample-plugin", "sample-plugin"); + fs::write(repo_root.join("sample-plugin/skills/SKILL.md"), "new skill").unwrap(); + write_file( + &repo_root.join(".agents/plugins/marketplace.json"), + r#"{ + "name": "debug", + "plugins": [ + { + "name": "sample-plugin", + "source": { + "source": "local", + "path": "./sample-plugin" + } + } + ] +}"#, + ); + write_plugin( + &tmp.path().join("plugins/cache/debug"), + "sample-plugin/local", + "sample-plugin", + ); + fs::write( + tmp.path() + .join("plugins/cache/debug/sample-plugin/local/skills/SKILL.md"), + "old skill", + ) + .unwrap(); + write_file( + &tmp.path().join(CONFIG_TOML_FILE), + r#"[features] +plugins = true + +[plugins."sample-plugin@debug"] +enabled = true +"#, + ); + + assert!( + refresh_non_curated_plugin_cache_force_reinstall( + tmp.path(), + &[AbsolutePathBuf::try_from(repo_root).unwrap()], + ) + .expect("cache refresh should reinstall unchanged local version") + ); + + assert_eq!( + fs::read_to_string( + tmp.path() + .join("plugins/cache/debug/sample-plugin/local/skills/SKILL.md") + ) + .unwrap(), + "new skill" + ); +} + #[test] fn refresh_non_curated_plugin_cache_ignores_invalid_unconfigured_plugin_versions() { let tmp = tempfile::tempdir().unwrap(); diff --git a/codex-rs/core/src/plugins/marketplace_add/metadata.rs b/codex-rs/core/src/plugins/marketplace_add/metadata.rs index f21ed0faa2..ccded11c87 100644 --- a/codex-rs/core/src/plugins/marketplace_add/metadata.rs +++ b/codex-rs/core/src/plugins/marketplace_add/metadata.rs @@ -38,6 +38,7 @@ pub(super) fn record_added_marketplace_entry( let timestamp = utc_timestamp_now()?; let update = MarketplaceConfigUpdate { last_updated: ×tamp, + last_revision: None, source_type: install_metadata.config_source_type(), source: &source, ref_name: install_metadata.ref_name(), diff --git a/codex-rs/core/src/plugins/marketplace_add/source.rs b/codex-rs/core/src/plugins/marketplace_add/source.rs index 98bdc2b8bb..e723c8865a 100644 --- a/codex-rs/core/src/plugins/marketplace_add/source.rs +++ b/codex-rs/core/src/plugins/marketplace_add/source.rs @@ -125,6 +125,7 @@ fn normalize_git_url(url: &str) -> String { fn looks_like_local_path(source: &str) -> bool { Path::new(source).is_absolute() + || looks_like_windows_absolute_path(source) || source.starts_with("./") || source.starts_with(".\\") || source.starts_with("../") @@ -134,6 +135,15 @@ fn looks_like_local_path(source: &str) -> bool { || source == ".." } +fn looks_like_windows_absolute_path(source: &str) -> bool { + let bytes = source.as_bytes(); + bytes.len() >= 3 + && bytes[0].is_ascii_alphabetic() + && bytes[1] == b':' + && matches!(bytes[2], b'\\' | b'/') + || source.starts_with(r"\\") +} + fn resolve_local_source_path(source: &str) -> Result { let path = expand_tilde_path(source); let path = if path.is_absolute() { @@ -312,6 +322,14 @@ mod tests { assert!(path.is_absolute()); } + #[test] + fn windows_absolute_paths_look_like_local_paths_on_every_host() { + assert!(looks_like_local_path(r"C:\Users\alice\marketplace")); + assert!(looks_like_local_path("C:/Users/alice/marketplace")); + assert!(looks_like_local_path(r"\\server\share\marketplace")); + assert!(!looks_like_local_path(r"C:relative\path")); + } + #[test] fn local_file_source_is_rejected() { let tempdir = TempDir::new().unwrap(); diff --git a/codex-rs/core/src/plugins/mod.rs b/codex-rs/core/src/plugins/mod.rs index 2e86643f34..16cc8d08c4 100644 --- a/codex-rs/core/src/plugins/mod.rs +++ b/codex-rs/core/src/plugins/mod.rs @@ -11,6 +11,8 @@ mod startup_sync; #[cfg(test)] pub(crate) mod test_support; +pub use codex_core_plugins::marketplace_upgrade::ConfiguredMarketplaceUpgradeError as PluginMarketplaceUpgradeError; +pub use codex_core_plugins::marketplace_upgrade::ConfiguredMarketplaceUpgradeOutcome as PluginMarketplaceUpgradeOutcome; pub use codex_plugin::AppConnectorId; pub use codex_plugin::EffectiveSkillRoots; pub use codex_plugin::PluginCapabilitySummary; From ab97c9aaad7cb29a5d23c7485318e0ad6b20400c Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Thu, 16 Apr 2026 10:51:33 -0700 Subject: [PATCH 82/84] Refactor AGENTS.md discovery into AgentsMdManager (#18035) Encapsulate Agents MD processing a bit and drop user_instructions_path from config. --- codex-rs/app-server-client/src/lib.rs | 5 +- .../app-server/src/codex_message_processor.rs | 14 +- codex-rs/core/src/agents_md.rs | 367 ++++++++++++++++++ ...roject_doc_tests.rs => agents_md_tests.rs} | 92 +++-- codex-rs/core/src/codex.rs | 6 +- codex-rs/core/src/config/config_tests.rs | 36 +- codex-rs/core/src/config/mod.rs | 39 +- codex-rs/core/src/lib.rs | 9 +- codex-rs/core/src/project_doc.rs | 326 ---------------- codex-rs/tui/src/chatwidget.rs | 2 +- codex-rs/tui/src/chatwidget/slash_dispatch.rs | 4 +- .../src/chatwidget/tests/slash_commands.rs | 4 +- codex-rs/tui/src/status/helpers.rs | 14 +- 13 files changed, 464 insertions(+), 454 deletions(-) create mode 100644 codex-rs/core/src/agents_md.rs rename codex-rs/core/src/{project_doc_tests.rs => agents_md_tests.rs} (90%) delete mode 100644 codex-rs/core/src/project_doc.rs diff --git a/codex-rs/app-server-client/src/lib.rs b/codex-rs/app-server-client/src/lib.rs index 9818b2f61b..71a8784e0a 100644 --- a/codex-rs/app-server-client/src/lib.rs +++ b/codex-rs/app-server-client/src/lib.rs @@ -65,9 +65,9 @@ pub use crate::remote::RemoteAppServerConnectArgs; /// while legacy startup/config paths are migrated to RPCs. pub mod legacy_core { pub use codex_core::Cursor; - pub use codex_core::DEFAULT_PROJECT_DOC_FILENAME; + pub use codex_core::DEFAULT_AGENTS_MD_FILENAME; pub use codex_core::INTERACTIVE_SESSION_SOURCES; - pub use codex_core::LOCAL_PROJECT_DOC_FILENAME; + pub use codex_core::LOCAL_AGENTS_MD_FILENAME; pub use codex_core::McpManager; pub use codex_core::PLUGIN_TEXT_MENTION_SIGIL; pub use codex_core::RolloutRecorder; @@ -77,7 +77,6 @@ pub mod legacy_core { pub use codex_core::ThreadsPage; pub use codex_core::append_message_history_entry; pub use codex_core::check_execpolicy_for_warnings; - pub use codex_core::discover_project_doc_paths; pub use codex_core::find_thread_meta_by_name_str; pub use codex_core::find_thread_name_by_id; pub use codex_core::find_thread_names_by_ids; diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 25892e40eb..410f22ece8 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -617,17 +617,9 @@ pub(crate) struct CodexMessageProcessorArgs { impl CodexMessageProcessor { async fn instruction_sources_from_config(config: &Config) -> Vec { - let mut paths: Vec = - config.user_instructions_path.iter().cloned().collect(); - match codex_core::discover_project_doc_paths(config, LOCAL_FS.as_ref()).await { - Ok(project_doc_paths) => { - paths.extend(project_doc_paths); - } - Err(err) => { - tracing::warn!(error = %err, "failed to discover project docs for thread response"); - } - } - paths + codex_core::AgentsMdManager::new(config) + .instruction_sources(LOCAL_FS.as_ref()) + .await } pub(crate) fn handle_config_mutation(&self) { diff --git a/codex-rs/core/src/agents_md.rs b/codex-rs/core/src/agents_md.rs new file mode 100644 index 0000000000..a1a883e839 --- /dev/null +++ b/codex-rs/core/src/agents_md.rs @@ -0,0 +1,367 @@ +//! AGENTS.md discovery and user instruction assembly. +//! +//! Project-level documentation is primarily stored in files named `AGENTS.md`. +//! Additional fallback filenames can be configured via `project_doc_fallback_filenames`. +//! We include the concatenation of all files found along the path from the +//! project root to the current working directory as follows: +//! +//! 1. Determine the project root by walking upwards from the current working +//! directory until a configured `project_root_markers` entry is found. +//! When `project_root_markers` is unset, the default marker list is used +//! (`.git`). If no marker is found, only the current working directory is +//! considered. An empty marker list disables parent traversal. +//! 2. Collect every `AGENTS.md` found from the project root down to the +//! current working directory (inclusive) and concatenate their contents in +//! that order. +//! 3. We do **not** walk past the project root. + +use crate::config::Config; +use crate::config_loader::ConfigLayerStackOrdering; +use crate::config_loader::default_project_root_markers; +use crate::config_loader::merge_toml_values; +use crate::config_loader::project_root_markers_from_config; +use codex_app_server_protocol::ConfigLayerSource; +use codex_exec_server::Environment; +use codex_exec_server::ExecutorFileSystem; +use codex_features::Feature; +use codex_utils_absolute_path::AbsolutePathBuf; +use dunce::canonicalize as normalize_path; +use std::io; +use toml::Value as TomlValue; +use tracing::error; + +pub(crate) const HIERARCHICAL_AGENTS_MESSAGE: &str = + include_str!("../hierarchical_agents_message.md"); + +/// Default filename scanned for AGENTS.md instructions. +pub const DEFAULT_AGENTS_MD_FILENAME: &str = "AGENTS.md"; +/// Preferred local override for AGENTS.md instructions. +pub const LOCAL_AGENTS_MD_FILENAME: &str = "AGENTS.override.md"; + +/// When both `Config::instructions` and AGENTS.md docs are present, they will +/// be concatenated with the following separator. +const AGENTS_MD_SEPARATOR: &str = "\n\n--- project-doc ---\n\n"; + +fn render_js_repl_instructions(config: &Config) -> Option { + if !config.features.enabled(Feature::JsRepl) { + return None; + } + + let mut section = String::from("## JavaScript REPL (Node)\n"); + section.push_str( + "- Use `js_repl` for Node-backed JavaScript with top-level await in a persistent kernel.\n", + ); + section.push_str("- `js_repl` is a freeform/custom tool. Direct `js_repl` calls must send raw JavaScript tool input (optionally with first-line `// codex-js-repl: timeout_ms=15000`). Do not wrap code in JSON (for example `{\"code\":\"...\"}`), quotes, or markdown code fences.\n"); + section.push_str( + "- Helpers: `codex.cwd`, `codex.homeDir`, `codex.tmpDir`, `codex.tool(name, args?)`, and `codex.emitImage(imageLike)`.\n", + ); + section.push_str("- `codex.tool` executes a normal tool call and resolves to the raw tool output object. Use it for shell and non-shell tools alike. Nested tool outputs stay inside JavaScript unless you emit them explicitly.\n"); + section.push_str("- `codex.emitImage(...)` adds one image to the outer `js_repl` function output each time you call it, so you can call it multiple times to emit multiple images. It accepts a data URL, a single `input_image` item, an object like `{ bytes, mimeType }`, or a raw tool response object with exactly one image and no text. It rejects mixed text-and-image content.\n"); + section.push_str("- `codex.tool(...)` and `codex.emitImage(...)` keep stable helper identities across cells. Saved references and persisted objects can reuse them in later cells, but async callbacks that fire after a cell finishes still fail because no exec is active.\n"); + section.push_str("- Request full-resolution image processing with `detail: \"original\"` only when the `view_image` tool schema includes a `detail` argument. The same availability applies to `codex.emitImage(...)`: if `view_image.detail` is present, you may also pass `detail: \"original\"` there. Use this when high-fidelity image perception or precise localization is needed, especially for CUA agents.\n"); + section.push_str("- Raw MCP image blocks can request the same behavior by returning `_meta: { \"codex/imageDetail\": \"original\" }` on the image content item.\n"); + section.push_str("- Example of sharing an in-memory Playwright screenshot: `await codex.emitImage({ bytes: await page.screenshot({ type: \"jpeg\", quality: 85 }), mimeType: \"image/jpeg\", detail: \"original\" })`.\n"); + section.push_str("- Example of sharing a local image tool result: `await codex.emitImage(codex.tool(\"view_image\", { path: \"/absolute/path\", detail: \"original\" }))`.\n"); + section.push_str("- When encoding an image to send with `codex.emitImage(...)` or `view_image`, prefer JPEG at about 85 quality when lossy compression is acceptable; use PNG when transparency or lossless detail matters. Smaller uploads are faster and less likely to hit size limits.\n"); + section.push_str("- Top-level bindings persist across cells. If a cell throws, prior bindings remain available and bindings that finished initializing before the throw often remain usable in later cells. For code you plan to reuse across cells, prefer declaring or assigning it in direct top-level statements before operations that might throw. If you hit `SyntaxError: Identifier 'x' has already been declared`, first reuse the existing binding, reassign a previously declared `let`, or pick a new descriptive name. Use `{ ... }` only for a short temporary block when you specifically need local scratch names; do not wrap an entire cell in block scope if you want those names reusable later. Reset the kernel with `js_repl_reset` only when you need a clean state.\n"); + section.push_str("- Top-level static import declarations (for example `import x from \"./file.js\"`) are currently unsupported in `js_repl`; use dynamic imports with `await import(\"pkg\")`, `await import(\"./file.js\")`, or `await import(\"/abs/path/file.mjs\")` instead. Imported local files must be ESM `.js`/`.mjs` files and run in the same REPL VM context. Bare package imports always resolve from REPL-global search roots (`CODEX_JS_REPL_NODE_MODULE_DIRS`, then cwd), not relative to the imported file location. Local files may statically import only other local relative/absolute/`file://` `.js`/`.mjs` files; package and builtin imports from local files must stay dynamic. `import.meta.resolve()` returns importable strings such as `file://...`, bare package names, and `node:...` specifiers. Local file modules reload between execs, while top-level bindings persist until `js_repl_reset`.\n"); + + if config.features.enabled(Feature::JsReplToolsOnly) { + section.push_str("- Do not call tools directly; use `js_repl` + `codex.tool(...)` for all tool calls, including shell commands.\n"); + section + .push_str("- MCP tools (if any) can also be called by name via `codex.tool(...)`.\n"); + } + + section.push_str("- Avoid direct access to `process.stdout` / `process.stderr` / `process.stdin`; it can corrupt the JSON line protocol. Use `console.log`, `codex.tool(...)`, and `codex.emitImage(...)`."); + + Some(section) +} + +/// Resolves AGENTS.md files into model-visible user instructions and source +/// paths. +pub struct AgentsMdManager<'a> { + config: &'a Config, +} + +pub(crate) struct LoadedAgentsMd { + pub(crate) contents: String, + pub(crate) path: AbsolutePathBuf, +} + +impl<'a> AgentsMdManager<'a> { + pub fn new(config: &'a Config) -> Self { + Self { config } + } + + pub(crate) fn load_global_instructions( + codex_dir: Option<&AbsolutePathBuf>, + ) -> Option { + let base = codex_dir?; + for candidate in [LOCAL_AGENTS_MD_FILENAME, DEFAULT_AGENTS_MD_FILENAME] { + let path = base.join(candidate); + if let Ok(contents) = std::fs::read_to_string(&path) { + let trimmed = contents.trim(); + if !trimmed.is_empty() { + return Some(LoadedAgentsMd { + contents: trimmed.to_string(), + path, + }); + } + } + } + None + } + + /// Combines configured user instructions and AGENTS.md content into a + /// single model-visible instruction string. + pub(crate) async fn user_instructions( + &self, + environment: Option<&Environment>, + ) -> Option { + let fs = environment?.get_filesystem(); + self.user_instructions_with_fs(fs.as_ref()).await + } + + pub(crate) async fn user_instructions_with_fs( + &self, + fs: &dyn ExecutorFileSystem, + ) -> Option { + let agents_md_docs = self.read_agents_md(fs).await; + + let mut output = String::new(); + + if let Some(instructions) = self.config.user_instructions.clone() { + output.push_str(&instructions); + } + + match agents_md_docs { + Ok(Some(docs)) => { + if !output.is_empty() { + output.push_str(AGENTS_MD_SEPARATOR); + } + output.push_str(&docs); + } + Ok(None) => {} + Err(e) => { + error!("error trying to find AGENTS.md docs: {e:#}"); + } + }; + + if let Some(js_repl_section) = render_js_repl_instructions(self.config) { + if !output.is_empty() { + output.push_str("\n\n"); + } + output.push_str(&js_repl_section); + } + + if self.config.features.enabled(Feature::ChildAgentsMd) { + if !output.is_empty() { + output.push_str("\n\n"); + } + output.push_str(HIERARCHICAL_AGENTS_MESSAGE); + } + + if !output.is_empty() { + Some(output) + } else { + None + } + } + + /// Returns all instruction source files included in the current config. + pub async fn instruction_sources(&self, fs: &dyn ExecutorFileSystem) -> Vec { + let mut paths = Self::load_global_instructions(Some(&self.config.codex_home)) + .map(|loaded| vec![loaded.path]) + .unwrap_or_default(); + match self.agents_md_paths(fs).await { + Ok(agents_md_paths) => paths.extend(agents_md_paths), + Err(err) => { + tracing::warn!(error = %err, "failed to discover AGENTS.md docs for instruction sources"); + } + } + paths + } + + /// Attempt to locate and load AGENTS.md documentation. + /// + /// On success returns `Ok(Some(contents))` where `contents` is the + /// concatenation of all discovered docs. If no documentation file is found + /// the function returns `Ok(None)`. Unexpected I/O failures bubble up as + /// `Err` so callers can decide how to handle them. + async fn read_agents_md(&self, fs: &dyn ExecutorFileSystem) -> io::Result> { + let max_total = self.config.project_doc_max_bytes; + + if max_total == 0 { + return Ok(None); + } + + let paths = self.agents_md_paths(fs).await?; + if paths.is_empty() { + return Ok(None); + } + + let mut remaining: u64 = max_total as u64; + let mut parts: Vec = Vec::new(); + + for p in paths { + if remaining == 0 { + break; + } + + match fs.get_metadata(&p, /*sandbox*/ None).await { + Ok(metadata) if !metadata.is_file => continue, + Ok(_) => {} + Err(err) if err.kind() == io::ErrorKind::NotFound => continue, + Err(err) => return Err(err), + } + + let mut data = match fs.read_file(&p, /*sandbox*/ None).await { + Ok(data) => data, + Err(err) if err.kind() == io::ErrorKind::NotFound => continue, + Err(err) => return Err(err), + }; + let size = data.len() as u64; + if size > remaining { + data.truncate(remaining as usize); + } + + if size > remaining { + tracing::warn!( + "Project doc `{}` exceeds remaining budget ({} bytes) - truncating.", + p.display(), + remaining, + ); + } + + let text = String::from_utf8_lossy(&data).to_string(); + if !text.trim().is_empty() { + parts.push(text); + remaining = remaining.saturating_sub(data.len() as u64); + } + } + + if parts.is_empty() { + Ok(None) + } else { + Ok(Some(parts.join("\n\n"))) + } + } + + /// Discover the list of AGENTS.md files using the same search rules as + /// `read_agents_md`, but return the file paths instead of concatenated + /// contents. The list is ordered from project root to the current working + /// directory (inclusive). Symlinks are allowed. When `project_doc_max_bytes` + /// is zero, returns an empty list. + async fn agents_md_paths( + &self, + fs: &dyn ExecutorFileSystem, + ) -> io::Result> { + if self.config.project_doc_max_bytes == 0 { + return Ok(Vec::new()); + } + + let mut dir = self.config.cwd.clone(); + if let Ok(canon) = normalize_path(&dir) { + dir = AbsolutePathBuf::try_from(canon)?; + } + + let mut merged = TomlValue::Table(toml::map::Map::new()); + for layer in self.config.config_layer_stack.get_layers( + ConfigLayerStackOrdering::LowestPrecedenceFirst, + /*include_disabled*/ false, + ) { + if matches!(layer.name, ConfigLayerSource::Project { .. }) { + continue; + } + merge_toml_values(&mut merged, &layer.config); + } + let project_root_markers = match project_root_markers_from_config(&merged) { + Ok(Some(markers)) => markers, + Ok(None) => default_project_root_markers(), + Err(err) => { + tracing::warn!("invalid project_root_markers: {err}"); + default_project_root_markers() + } + }; + let mut project_root = None; + if !project_root_markers.is_empty() { + for ancestor in dir.ancestors() { + for marker in &project_root_markers { + let marker_path = ancestor.join(marker); + let marker_exists = match fs.get_metadata(&marker_path, /*sandbox*/ None).await + { + Ok(_) => true, + Err(err) if err.kind() == io::ErrorKind::NotFound => false, + Err(err) => return Err(err), + }; + if marker_exists { + project_root = Some(ancestor.clone()); + break; + } + } + if project_root.is_some() { + break; + } + } + } + + let search_dirs: Vec = if let Some(root) = project_root { + let mut dirs = Vec::new(); + let mut cursor = dir.clone(); + loop { + dirs.push(cursor.clone()); + if cursor == root { + break; + } + let Some(parent) = cursor.parent() else { + break; + }; + cursor = parent; + } + dirs.reverse(); + dirs + } else { + vec![dir] + }; + + let mut found: Vec = Vec::new(); + let candidate_filenames = self.candidate_filenames(); + for d in search_dirs { + for name in &candidate_filenames { + let candidate = d.join(name); + match fs.get_metadata(&candidate, /*sandbox*/ None).await { + Ok(md) if md.is_file => { + found.push(candidate); + break; + } + Ok(_) => {} + Err(err) if err.kind() == io::ErrorKind::NotFound => continue, + Err(err) => return Err(err), + } + } + } + + Ok(found) + } + + fn candidate_filenames(&self) -> Vec<&str> { + let mut names: Vec<&str> = + Vec::with_capacity(2 + self.config.project_doc_fallback_filenames.len()); + names.push(LOCAL_AGENTS_MD_FILENAME); + names.push(DEFAULT_AGENTS_MD_FILENAME); + for candidate in &self.config.project_doc_fallback_filenames { + let candidate = candidate.as_str(); + if candidate.is_empty() { + continue; + } + if !names.contains(&candidate) { + names.push(candidate); + } + } + names + } +} + +#[cfg(test)] +#[path = "agents_md_tests.rs"] +mod tests; diff --git a/codex-rs/core/src/project_doc_tests.rs b/codex-rs/core/src/agents_md_tests.rs similarity index 90% rename from codex-rs/core/src/project_doc_tests.rs rename to codex-rs/core/src/agents_md_tests.rs index 31f73805f7..012724b43e 100644 --- a/codex-rs/core/src/project_doc_tests.rs +++ b/codex-rs/core/src/agents_md_tests.rs @@ -11,11 +11,15 @@ use std::path::PathBuf; use tempfile::TempDir; async fn get_user_instructions(config: &Config) -> Option { - super::get_user_instructions_with_fs(config, LOCAL_FS.as_ref()).await + AgentsMdManager::new(config) + .user_instructions_with_fs(LOCAL_FS.as_ref()) + .await } -async fn discover_project_doc_paths(config: &Config) -> std::io::Result> { - super::discover_project_doc_paths(config, LOCAL_FS.as_ref()).await +async fn agents_md_paths(config: &Config) -> std::io::Result> { + AgentsMdManager::new(config) + .agents_md_paths(LOCAL_FS.as_ref()) + .await } /// Helper that returns a `Config` pointing at `root` and using `limit` as @@ -101,7 +105,9 @@ async fn no_environment_returns_none() { let tmp = tempfile::tempdir().expect("tempdir"); let config = make_config(&tmp, /*limit*/ 4096, Some("user instructions")).await; - let res = super::get_user_instructions(&config, /*environment*/ None).await; + let res = AgentsMdManager::new(&config) + .user_instructions(/*environment*/ None) + .await; assert_eq!(res, None); } @@ -187,10 +193,9 @@ async fn zero_byte_limit_disables_discovery() { let tmp = tempfile::tempdir().expect("tempdir"); fs::write(tmp.path().join("AGENTS.md"), "something").unwrap(); - let discovery = - discover_project_doc_paths(&make_config(&tmp, /*limit*/ 0, /*instructions*/ None).await) - .await - .expect("discover paths"); + let discovery = agents_md_paths(&make_config(&tmp, /*limit*/ 0, /*instructions*/ None).await) + .await + .expect("discover paths"); assert_eq!(discovery, Vec::::new()); } @@ -228,10 +233,10 @@ async fn js_repl_tools_only_instructions_are_feature_gated() { assert_eq!(res, expected); } -/// When both system instructions *and* a project doc are present the two +/// When both system instructions and AGENTS.md docs are present the two /// should be concatenated with the separator. #[tokio::test] -async fn merges_existing_instructions_with_project_doc() { +async fn merges_existing_instructions_with_agents_md() { let tmp = tempfile::tempdir().expect("tempdir"); fs::write(tmp.path().join("AGENTS.md"), "proj doc").unwrap(); @@ -241,12 +246,12 @@ async fn merges_existing_instructions_with_project_doc() { .await .expect("should produce a combined instruction string"); - let expected = format!("{INSTRUCTIONS}{PROJECT_DOC_SEPARATOR}{}", "proj doc"); + let expected = format!("{INSTRUCTIONS}{AGENTS_MD_SEPARATOR}{}", "proj doc"); assert_eq!(res, expected); } -/// If there are existing system instructions but the project doc is +/// If there are existing system instructions but AGENTS.md docs are /// missing we expect the original instructions to be returned unchanged. #[tokio::test] async fn keeps_existing_instructions_when_doc_missing() { @@ -307,9 +312,7 @@ async fn project_root_markers_are_honored_for_agents_discovery() { .await; cfg.cwd = nested.abs(); - let discovery = discover_project_doc_paths(&cfg) - .await - .expect("discover paths"); + let discovery = agents_md_paths(&cfg).await.expect("discover paths"); let expected_parent = AbsolutePathBuf::try_from( dunce::canonicalize(root.path().join("AGENTS.md")).expect("canonical parent doc path"), ) @@ -326,12 +329,33 @@ async fn project_root_markers_are_honored_for_agents_discovery() { assert_eq!(res, "parent doc\n\nchild doc"); } +#[tokio::test] +async fn instruction_sources_include_global_before_agents_md_docs() { + let tmp = tempfile::tempdir().expect("tempdir"); + fs::write(tmp.path().join("AGENTS.md"), "project doc").unwrap(); + + let cfg = make_config(&tmp, /*limit*/ 4096, Some("global doc")).await; + let global_agents = cfg.codex_home.join(DEFAULT_AGENTS_MD_FILENAME); + fs::create_dir_all(&cfg.codex_home).unwrap(); + fs::write(&global_agents, "global doc").unwrap(); + + let sources = AgentsMdManager::new(&cfg) + .instruction_sources(LOCAL_FS.as_ref()) + .await; + let project_agents = AbsolutePathBuf::try_from( + dunce::canonicalize(cfg.cwd.join("AGENTS.md")).expect("canonical project doc path"), + ) + .expect("absolute project doc path"); + + assert_eq!(sources, vec![global_agents, project_agents]); +} + /// AGENTS.override.md is preferred over AGENTS.md when both are present. #[tokio::test] async fn agents_local_md_preferred() { let tmp = tempfile::tempdir().expect("tempdir"); - fs::write(tmp.path().join(DEFAULT_PROJECT_DOC_FILENAME), "versioned").unwrap(); - fs::write(tmp.path().join(LOCAL_PROJECT_DOC_FILENAME), "local").unwrap(); + fs::write(tmp.path().join(DEFAULT_AGENTS_MD_FILENAME), "versioned").unwrap(); + fs::write(tmp.path().join(LOCAL_AGENTS_MD_FILENAME), "local").unwrap(); let cfg = make_config(&tmp, /*limit*/ 4096, /*instructions*/ None).await; @@ -341,13 +365,11 @@ async fn agents_local_md_preferred() { assert_eq!(res, "local"); - let discovery = discover_project_doc_paths(&cfg) - .await - .expect("discover paths"); + let discovery = agents_md_paths(&cfg).await.expect("discover paths"); assert_eq!(discovery.len(), 1); assert_eq!( discovery[0].file_name().unwrap().to_string_lossy(), - LOCAL_PROJECT_DOC_FILENAME + LOCAL_AGENTS_MD_FILENAME ); } @@ -393,16 +415,14 @@ async fn agents_md_preferred_over_fallbacks() { assert_eq!(res, "primary"); - let discovery = discover_project_doc_paths(&cfg) - .await - .expect("discover paths"); + let discovery = agents_md_paths(&cfg).await.expect("discover paths"); assert_eq!(discovery.len(), 1); assert!( discovery[0] .file_name() .unwrap() .to_string_lossy() - .eq(DEFAULT_PROJECT_DOC_FILENAME) + .eq(DEFAULT_AGENTS_MD_FILENAME) ); } @@ -416,9 +436,7 @@ async fn agents_md_directory_is_ignored() { let res = get_user_instructions(&cfg).await; assert_eq!(res, None); - let discovery = discover_project_doc_paths(&cfg) - .await - .expect("discover paths"); + let discovery = agents_md_paths(&cfg).await.expect("discover paths"); assert_eq!(discovery, Vec::::new()); } @@ -441,17 +459,15 @@ async fn agents_md_special_file_is_ignored() { let res = get_user_instructions(&cfg).await; assert_eq!(res, None); - let discovery = discover_project_doc_paths(&cfg) - .await - .expect("discover paths"); + let discovery = agents_md_paths(&cfg).await.expect("discover paths"); assert_eq!(discovery, Vec::::new()); } #[tokio::test] async fn override_directory_falls_back_to_agents_md_file() { let tmp = tempfile::tempdir().expect("tempdir"); - fs::create_dir(tmp.path().join(LOCAL_PROJECT_DOC_FILENAME)).unwrap(); - fs::write(tmp.path().join(DEFAULT_PROJECT_DOC_FILENAME), "primary").unwrap(); + fs::create_dir(tmp.path().join(LOCAL_AGENTS_MD_FILENAME)).unwrap(); + fs::write(tmp.path().join(DEFAULT_AGENTS_MD_FILENAME), "primary").unwrap(); let cfg = make_config(&tmp, /*limit*/ 4096, /*instructions*/ None).await; @@ -460,21 +476,19 @@ async fn override_directory_falls_back_to_agents_md_file() { .expect("AGENTS.md should be used when override is a directory"); assert_eq!(res, "primary"); - let discovery = discover_project_doc_paths(&cfg) - .await - .expect("discover paths"); + let discovery = agents_md_paths(&cfg).await.expect("discover paths"); assert_eq!(discovery.len(), 1); assert_eq!( discovery[0] .file_name() .expect("file name") .to_string_lossy(), - DEFAULT_PROJECT_DOC_FILENAME + DEFAULT_AGENTS_MD_FILENAME ); } #[tokio::test] -async fn skills_are_not_appended_to_project_doc() { +async fn skills_are_not_appended_to_agents_md() { let tmp = tempfile::tempdir().expect("tempdir"); fs::write(tmp.path().join("AGENTS.md"), "base doc").unwrap(); @@ -504,7 +518,7 @@ async fn apps_feature_does_not_emit_user_instructions_by_itself() { } #[tokio::test] -async fn apps_feature_does_not_append_to_project_doc_user_instructions() { +async fn apps_feature_does_not_append_to_agents_md_user_instructions() { let tmp = tempfile::tempdir().expect("tempdir"); fs::write(tmp.path().join("AGENTS.md"), "base doc").unwrap(); diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 5f821eb6b4..50a4c25abd 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -265,6 +265,7 @@ use crate::SkillInjections; use crate::SkillLoadOutcome; use crate::SkillMetadata; use crate::SkillsManager; +use crate::agents_md::AgentsMdManager; use crate::build_skill_injections; use crate::collect_env_var_dependencies; use crate::collect_explicit_skill_mentions; @@ -293,7 +294,6 @@ use crate::network_policy_decision::execpolicy_network_rule_amendment; use crate::plugins::PluginsManager; use crate::plugins::build_plugin_injections; use crate::plugins::render_plugins_section; -use crate::project_doc::get_user_instructions; use crate::resolve_skill_dependencies_for_turn; use crate::rollout::RolloutRecorder; use crate::rollout::RolloutRecorderParams; @@ -556,7 +556,9 @@ impl Codex { config.startup_warnings.push(message); } - let user_instructions = get_user_instructions(&config, environment.as_deref()).await; + let user_instructions = AgentsMdManager::new(&config) + .user_instructions(environment.as_deref()) + .await; let exec_policy = if crate::guardian::is_guardian_reviewer_source(&session_source) { // Guardian review should rely on the built-in shell safety checks, diff --git a/codex-rs/core/src/config/config_tests.rs b/codex-rs/core/src/config/config_tests.rs index a7a5e88352..0001439816 100644 --- a/codex-rs/core/src/config/config_tests.rs +++ b/codex-rs/core/src/config/config_tests.rs @@ -1,3 +1,5 @@ +use crate::agents_md::DEFAULT_AGENTS_MD_FILENAME; +use crate::agents_md::LOCAL_AGENTS_MD_FILENAME; use crate::config::edit::ConfigEdit; use crate::config::edit::ConfigEditsBuilder; use crate::config::edit::apply_blocking; @@ -138,10 +140,12 @@ async fn load_config_normalizes_relative_cwd_override() -> std::io::Result<()> { } #[tokio::test] -async fn load_config_records_global_agents_path() -> std::io::Result<()> { +async fn load_config_loads_global_agents_instructions() -> std::io::Result<()> { let codex_home = tempdir()?; - let global_agents_path = codex_home.path().join(DEFAULT_PROJECT_DOC_FILENAME); - std::fs::write(&global_agents_path, "\n global instructions \n")?; + std::fs::write( + codex_home.path().join(DEFAULT_AGENTS_MD_FILENAME), + "\n global instructions \n", + )?; let config = Config::load_from_base_config_with_overrides( ConfigToml::default(), @@ -154,21 +158,17 @@ async fn load_config_records_global_agents_path() -> std::io::Result<()> { config.user_instructions.as_deref(), Some("global instructions") ); - assert_eq!( - config.user_instructions_path.as_deref(), - Some(global_agents_path.as_path()) - ); Ok(()) } #[tokio::test] -async fn load_config_records_preferred_global_agents_override_path() -> std::io::Result<()> { +async fn load_config_prefers_global_agents_override_instructions() -> std::io::Result<()> { let codex_home = tempdir()?; std::fs::write( - codex_home.path().join(DEFAULT_PROJECT_DOC_FILENAME), + codex_home.path().join(DEFAULT_AGENTS_MD_FILENAME), "global instructions", )?; - let global_agents_override_path = codex_home.path().join(LOCAL_PROJECT_DOC_FILENAME); + let global_agents_override_path = codex_home.path().join(LOCAL_AGENTS_MD_FILENAME); std::fs::write(&global_agents_override_path, "local override instructions")?; let config = Config::load_from_base_config_with_overrides( @@ -182,10 +182,6 @@ async fn load_config_records_preferred_global_agents_override_path() -> std::io: config.user_instructions.as_deref(), Some("local override instructions") ); - assert_eq!( - config.user_instructions_path.as_deref(), - Some(global_agents_override_path.as_path()) - ); Ok(()) } @@ -4721,7 +4717,6 @@ async fn test_precedence_fixture_with_o3_profile() -> std::io::Result<()> { approvals_reviewer: ApprovalsReviewer::User, enforce_residency: Constrained::allow_any(/*initial_value*/ None), user_instructions: None, - user_instructions_path: None, notify: None, cwd: fixture.cwd(), cli_auth_credentials_store_mode: Default::default(), @@ -4733,7 +4728,7 @@ async fn test_precedence_fixture_with_o3_profile() -> std::io::Result<()> { mcp_oauth_callback_port: None, mcp_oauth_callback_url: None, model_providers: fixture.model_provider_map.clone(), - project_doc_max_bytes: PROJECT_DOC_MAX_BYTES, + project_doc_max_bytes: AGENTS_MD_MAX_BYTES, project_doc_fallback_filenames: Vec::new(), tool_output_token_limit: None, agent_max_threads: DEFAULT_AGENT_MAX_THREADS, @@ -4872,7 +4867,6 @@ async fn test_precedence_fixture_with_gpt3_profile() -> std::io::Result<()> { approvals_reviewer: ApprovalsReviewer::User, enforce_residency: Constrained::allow_any(/*initial_value*/ None), user_instructions: None, - user_instructions_path: None, notify: None, cwd: fixture.cwd(), cli_auth_credentials_store_mode: Default::default(), @@ -4884,7 +4878,7 @@ async fn test_precedence_fixture_with_gpt3_profile() -> std::io::Result<()> { mcp_oauth_callback_port: None, mcp_oauth_callback_url: None, model_providers: fixture.model_provider_map.clone(), - project_doc_max_bytes: PROJECT_DOC_MAX_BYTES, + project_doc_max_bytes: AGENTS_MD_MAX_BYTES, project_doc_fallback_filenames: Vec::new(), tool_output_token_limit: None, agent_max_threads: DEFAULT_AGENT_MAX_THREADS, @@ -5021,7 +5015,6 @@ async fn test_precedence_fixture_with_zdr_profile() -> std::io::Result<()> { approvals_reviewer: ApprovalsReviewer::User, enforce_residency: Constrained::allow_any(/*initial_value*/ None), user_instructions: None, - user_instructions_path: None, notify: None, cwd: fixture.cwd(), cli_auth_credentials_store_mode: Default::default(), @@ -5033,7 +5026,7 @@ async fn test_precedence_fixture_with_zdr_profile() -> std::io::Result<()> { mcp_oauth_callback_port: None, mcp_oauth_callback_url: None, model_providers: fixture.model_provider_map.clone(), - project_doc_max_bytes: PROJECT_DOC_MAX_BYTES, + project_doc_max_bytes: AGENTS_MD_MAX_BYTES, project_doc_fallback_filenames: Vec::new(), tool_output_token_limit: None, agent_max_threads: DEFAULT_AGENT_MAX_THREADS, @@ -5155,7 +5148,6 @@ async fn test_precedence_fixture_with_gpt5_profile() -> std::io::Result<()> { approvals_reviewer: ApprovalsReviewer::User, enforce_residency: Constrained::allow_any(/*initial_value*/ None), user_instructions: None, - user_instructions_path: None, notify: None, cwd: fixture.cwd(), cli_auth_credentials_store_mode: Default::default(), @@ -5167,7 +5159,7 @@ async fn test_precedence_fixture_with_gpt5_profile() -> std::io::Result<()> { mcp_oauth_callback_port: None, mcp_oauth_callback_url: None, model_providers: fixture.model_provider_map.clone(), - project_doc_max_bytes: PROJECT_DOC_MAX_BYTES, + project_doc_max_bytes: AGENTS_MD_MAX_BYTES, project_doc_fallback_filenames: Vec::new(), tool_output_token_limit: None, agent_max_threads: DEFAULT_AGENT_MAX_THREADS, diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index 87e888e46e..c4217bdf2a 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -1,3 +1,4 @@ +use crate::agents_md::AgentsMdManager; use crate::config::edit::ConfigEdit; use crate::config::edit::ConfigEditsBuilder; use crate::config_loader::CloudRequirementsLoader; @@ -15,8 +16,6 @@ use crate::config_loader::load_config_layers_state; use crate::config_loader::project_trust_key; use crate::memories::memory_root; use crate::path_utils::normalize_for_native_workdir; -use crate::project_doc::DEFAULT_PROJECT_DOC_FILENAME; -use crate::project_doc::LOCAL_PROJECT_DOC_FILENAME; use crate::unified_exec::DEFAULT_MAX_BACKGROUND_TERMINAL_TIMEOUT_MS; use crate::unified_exec::MIN_EMPTY_YIELD_TIME_MS; use crate::windows_sandbox::WindowsSandboxLevelExt; @@ -120,7 +119,7 @@ pub use codex_git_utils::GhostSnapshotConfig; /// Maximum number of bytes of the documentation that will be embedded. Larger /// files are *silently truncated* to this size so we do not take up too much of /// the context window. -pub(crate) const PROJECT_DOC_MAX_BYTES: usize = 32 * 1024; // 32 KiB +pub(crate) const AGENTS_MD_MAX_BYTES: usize = 32 * 1024; // 32 KiB pub(crate) const DEFAULT_AGENT_MAX_THREADS: Option = Some(6); pub(crate) const DEFAULT_AGENT_MAX_DEPTH: i32 = 1; pub(crate) const DEFAULT_AGENT_JOB_MAX_RUNTIME_SECONDS: Option = None; @@ -272,9 +271,6 @@ pub struct Config { /// User-provided instructions from AGENTS.md. pub user_instructions: Option, - /// Path to the global AGENTS file loaded into `user_instructions`. - pub user_instructions_path: Option, - /// Base instructions override. pub base_instructions: Option, @@ -1451,10 +1447,8 @@ impl Config { network: network_requirements, } = config_layer_stack.requirements().clone(); - let (user_instructions, user_instructions_path) = - Self::load_instructions(Some(&codex_home)) - .map(|loaded| (Some(loaded.contents), Some(loaded.path))) - .unwrap_or((None, None)); + let user_instructions = AgentsMdManager::load_global_instructions(Some(&codex_home)) + .map(|loaded| loaded.contents); let mut startup_warnings = Vec::new(); // Destructure ConfigOverrides fully to ensure all overrides are applied. @@ -2046,7 +2040,6 @@ impl Config { enforce_residency: enforce_residency.value, notify: cfg.notify, user_instructions, - user_instructions_path, base_instructions, personality, developer_instructions, @@ -2071,7 +2064,7 @@ impl Config { mcp_oauth_callback_port: cfg.mcp_oauth_callback_port, mcp_oauth_callback_url: cfg.mcp_oauth_callback_url.clone(), model_providers, - project_doc_max_bytes: cfg.project_doc_max_bytes.unwrap_or(PROJECT_DOC_MAX_BYTES), + project_doc_max_bytes: cfg.project_doc_max_bytes.unwrap_or(AGENTS_MD_MAX_BYTES), project_doc_fallback_filenames: cfg .project_doc_fallback_filenames .unwrap_or_default() @@ -2222,23 +2215,6 @@ impl Config { .await } - fn load_instructions(codex_dir: Option<&AbsolutePathBuf>) -> Option { - let base = codex_dir?; - for candidate in [LOCAL_PROJECT_DOC_FILENAME, DEFAULT_PROJECT_DOC_FILENAME] { - let path = base.join(candidate); - if let Ok(contents) = std::fs::read_to_string(&path) { - let trimmed = contents.trim(); - if !trimmed.is_empty() { - return Some(LoadedUserInstructions { - contents: trimmed.to_string(), - path, - }); - } - } - } - None - } - /// If `path` is `Some`, attempts to read the file at the given path and /// returns its contents as a trimmed `String`. If the file is empty, or /// is `Some` but cannot be read, returns an `Err`. @@ -2310,11 +2286,6 @@ impl Config { } } -struct LoadedUserInstructions { - contents: String, - path: AbsolutePathBuf, -} - pub(crate) fn uses_deprecated_instructions_file(config_layer_stack: &ConfigLayerStack) -> bool { config_layer_stack .layers_high_to_low() diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 17c4e63b7c..50777ad157 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -130,11 +130,10 @@ pub type ConversationManager = ThreadManager; pub type NewConversation = NewThread; #[deprecated(note = "use CodexThread")] pub type CodexConversation = CodexThread; -pub(crate) mod project_doc; -pub use project_doc::DEFAULT_PROJECT_DOC_FILENAME; -pub use project_doc::LOCAL_PROJECT_DOC_FILENAME; -pub use project_doc::discover_project_doc_paths; -pub use project_doc::read_project_docs; +pub(crate) mod agents_md; +pub use agents_md::AgentsMdManager; +pub use agents_md::DEFAULT_AGENTS_MD_FILENAME; +pub use agents_md::LOCAL_AGENTS_MD_FILENAME; mod rollout; pub(crate) mod safety; mod session_rollout_init_error; diff --git a/codex-rs/core/src/project_doc.rs b/codex-rs/core/src/project_doc.rs deleted file mode 100644 index c4f70a6f55..0000000000 --- a/codex-rs/core/src/project_doc.rs +++ /dev/null @@ -1,326 +0,0 @@ -//! Project-level documentation discovery. -//! -//! Project-level documentation is primarily stored in files named `AGENTS.md`. -//! Additional fallback filenames can be configured via `project_doc_fallback_filenames`. -//! We include the concatenation of all files found along the path from the -//! project root to the current working directory as follows: -//! -//! 1. Determine the project root by walking upwards from the current working -//! directory until a configured `project_root_markers` entry is found. -//! When `project_root_markers` is unset, the default marker list is used -//! (`.git`). If no marker is found, only the current working directory is -//! considered. An empty marker list disables parent traversal. -//! 2. Collect every `AGENTS.md` found from the project root down to the -//! current working directory (inclusive) and concatenate their contents in -//! that order. -//! 3. We do **not** walk past the project root. - -use crate::config::Config; -use crate::config_loader::ConfigLayerStackOrdering; -use crate::config_loader::default_project_root_markers; -use crate::config_loader::merge_toml_values; -use crate::config_loader::project_root_markers_from_config; -use codex_app_server_protocol::ConfigLayerSource; -use codex_exec_server::Environment; -use codex_exec_server::ExecutorFileSystem; -use codex_features::Feature; -use codex_utils_absolute_path::AbsolutePathBuf; -use dunce::canonicalize as normalize_path; -use std::io; -use toml::Value as TomlValue; -use tracing::error; - -pub(crate) const HIERARCHICAL_AGENTS_MESSAGE: &str = - include_str!("../hierarchical_agents_message.md"); - -/// Default filename scanned for project-level docs. -pub const DEFAULT_PROJECT_DOC_FILENAME: &str = "AGENTS.md"; -/// Preferred local override for project-level docs. -pub const LOCAL_PROJECT_DOC_FILENAME: &str = "AGENTS.override.md"; - -/// When both `Config::instructions` and the project doc are present, they will -/// be concatenated with the following separator. -const PROJECT_DOC_SEPARATOR: &str = "\n\n--- project-doc ---\n\n"; - -fn render_js_repl_instructions(config: &Config) -> Option { - if !config.features.enabled(Feature::JsRepl) { - return None; - } - - let mut section = String::from("## JavaScript REPL (Node)\n"); - section.push_str( - "- Use `js_repl` for Node-backed JavaScript with top-level await in a persistent kernel.\n", - ); - section.push_str("- `js_repl` is a freeform/custom tool. Direct `js_repl` calls must send raw JavaScript tool input (optionally with first-line `// codex-js-repl: timeout_ms=15000`). Do not wrap code in JSON (for example `{\"code\":\"...\"}`), quotes, or markdown code fences.\n"); - section.push_str( - "- Helpers: `codex.cwd`, `codex.homeDir`, `codex.tmpDir`, `codex.tool(name, args?)`, and `codex.emitImage(imageLike)`.\n", - ); - section.push_str("- `codex.tool` executes a normal tool call and resolves to the raw tool output object. Use it for shell and non-shell tools alike. Nested tool outputs stay inside JavaScript unless you emit them explicitly.\n"); - section.push_str("- `codex.emitImage(...)` adds one image to the outer `js_repl` function output each time you call it, so you can call it multiple times to emit multiple images. It accepts a data URL, a single `input_image` item, an object like `{ bytes, mimeType }`, or a raw tool response object with exactly one image and no text. It rejects mixed text-and-image content.\n"); - section.push_str("- `codex.tool(...)` and `codex.emitImage(...)` keep stable helper identities across cells. Saved references and persisted objects can reuse them in later cells, but async callbacks that fire after a cell finishes still fail because no exec is active.\n"); - section.push_str("- Request full-resolution image processing with `detail: \"original\"` only when the `view_image` tool schema includes a `detail` argument. The same availability applies to `codex.emitImage(...)`: if `view_image.detail` is present, you may also pass `detail: \"original\"` there. Use this when high-fidelity image perception or precise localization is needed, especially for CUA agents.\n"); - section.push_str("- Raw MCP image blocks can request the same behavior by returning `_meta: { \"codex/imageDetail\": \"original\" }` on the image content item.\n"); - section.push_str("- Example of sharing an in-memory Playwright screenshot: `await codex.emitImage({ bytes: await page.screenshot({ type: \"jpeg\", quality: 85 }), mimeType: \"image/jpeg\", detail: \"original\" })`.\n"); - section.push_str("- Example of sharing a local image tool result: `await codex.emitImage(codex.tool(\"view_image\", { path: \"/absolute/path\", detail: \"original\" }))`.\n"); - section.push_str("- When encoding an image to send with `codex.emitImage(...)` or `view_image`, prefer JPEG at about 85 quality when lossy compression is acceptable; use PNG when transparency or lossless detail matters. Smaller uploads are faster and less likely to hit size limits.\n"); - section.push_str("- Top-level bindings persist across cells. If a cell throws, prior bindings remain available and bindings that finished initializing before the throw often remain usable in later cells. For code you plan to reuse across cells, prefer declaring or assigning it in direct top-level statements before operations that might throw. If you hit `SyntaxError: Identifier 'x' has already been declared`, first reuse the existing binding, reassign a previously declared `let`, or pick a new descriptive name. Use `{ ... }` only for a short temporary block when you specifically need local scratch names; do not wrap an entire cell in block scope if you want those names reusable later. Reset the kernel with `js_repl_reset` only when you need a clean state.\n"); - section.push_str("- Top-level static import declarations (for example `import x from \"./file.js\"`) are currently unsupported in `js_repl`; use dynamic imports with `await import(\"pkg\")`, `await import(\"./file.js\")`, or `await import(\"/abs/path/file.mjs\")` instead. Imported local files must be ESM `.js`/`.mjs` files and run in the same REPL VM context. Bare package imports always resolve from REPL-global search roots (`CODEX_JS_REPL_NODE_MODULE_DIRS`, then cwd), not relative to the imported file location. Local files may statically import only other local relative/absolute/`file://` `.js`/`.mjs` files; package and builtin imports from local files must stay dynamic. `import.meta.resolve()` returns importable strings such as `file://...`, bare package names, and `node:...` specifiers. Local file modules reload between execs, while top-level bindings persist until `js_repl_reset`.\n"); - - if config.features.enabled(Feature::JsReplToolsOnly) { - section.push_str("- Do not call tools directly; use `js_repl` + `codex.tool(...)` for all tool calls, including shell commands.\n"); - section - .push_str("- MCP tools (if any) can also be called by name via `codex.tool(...)`.\n"); - } - - section.push_str("- Avoid direct access to `process.stdout` / `process.stderr` / `process.stdin`; it can corrupt the JSON line protocol. Use `console.log`, `codex.tool(...)`, and `codex.emitImage(...)`."); - - Some(section) -} - -/// Combines `Config::instructions` and `AGENTS.md` (if present) into a single -/// string of instructions. -pub(crate) async fn get_user_instructions( - config: &Config, - environment: Option<&Environment>, -) -> Option { - let fs = environment?.get_filesystem(); - get_user_instructions_with_fs(config, fs.as_ref()).await -} - -pub(crate) async fn get_user_instructions_with_fs( - config: &Config, - fs: &dyn ExecutorFileSystem, -) -> Option { - let project_docs = read_project_docs_with_fs(config, fs).await; - - let mut output = String::new(); - - if let Some(instructions) = config.user_instructions.clone() { - output.push_str(&instructions); - } - - match project_docs { - Ok(Some(docs)) => { - if !output.is_empty() { - output.push_str(PROJECT_DOC_SEPARATOR); - } - output.push_str(&docs); - } - Ok(None) => {} - Err(e) => { - error!("error trying to find project doc: {e:#}"); - } - }; - - if let Some(js_repl_section) = render_js_repl_instructions(config) { - if !output.is_empty() { - output.push_str("\n\n"); - } - output.push_str(&js_repl_section); - } - - if config.features.enabled(Feature::ChildAgentsMd) { - if !output.is_empty() { - output.push_str("\n\n"); - } - output.push_str(HIERARCHICAL_AGENTS_MESSAGE); - } - - if !output.is_empty() { - Some(output) - } else { - None - } -} - -/// Attempt to locate and load the project documentation. -/// -/// On success returns `Ok(Some(contents))` where `contents` is the -/// concatenation of all discovered docs. If no documentation file is found the -/// function returns `Ok(None)`. Unexpected I/O failures bubble up as `Err` so -/// callers can decide how to handle them. -pub async fn read_project_docs( - config: &Config, - environment: &Environment, -) -> io::Result> { - let fs = environment.get_filesystem(); - read_project_docs_with_fs(config, fs.as_ref()).await -} - -async fn read_project_docs_with_fs( - config: &Config, - fs: &dyn ExecutorFileSystem, -) -> io::Result> { - let max_total = config.project_doc_max_bytes; - - if max_total == 0 { - return Ok(None); - } - - let paths = discover_project_doc_paths(config, fs).await?; - if paths.is_empty() { - return Ok(None); - } - - let mut remaining: u64 = max_total as u64; - let mut parts: Vec = Vec::new(); - - for p in paths { - if remaining == 0 { - break; - } - - match fs.get_metadata(&p, /*sandbox*/ None).await { - Ok(metadata) if !metadata.is_file => continue, - Ok(_) => {} - Err(err) if err.kind() == io::ErrorKind::NotFound => continue, - Err(err) => return Err(err), - } - - let mut data = match fs.read_file(&p, /*sandbox*/ None).await { - Ok(data) => data, - Err(err) if err.kind() == io::ErrorKind::NotFound => continue, - Err(err) => return Err(err), - }; - let size = data.len() as u64; - if size > remaining { - data.truncate(remaining as usize); - } - - if size > remaining { - tracing::warn!( - "Project doc `{}` exceeds remaining budget ({} bytes) - truncating.", - p.display(), - remaining, - ); - } - - let text = String::from_utf8_lossy(&data).to_string(); - if !text.trim().is_empty() { - parts.push(text); - remaining = remaining.saturating_sub(data.len() as u64); - } - } - - if parts.is_empty() { - Ok(None) - } else { - Ok(Some(parts.join("\n\n"))) - } -} - -/// Discover the list of AGENTS.md files using the same search rules as -/// `read_project_docs`, but return the file paths instead of concatenated -/// contents. The list is ordered from project root to the current working -/// directory (inclusive). Symlinks are allowed. When `project_doc_max_bytes` -/// is zero, returns an empty list. -pub async fn discover_project_doc_paths( - config: &Config, - fs: &dyn ExecutorFileSystem, -) -> io::Result> { - if config.project_doc_max_bytes == 0 { - return Ok(Vec::new()); - } - - let mut dir = config.cwd.clone(); - if let Ok(canon) = normalize_path(&dir) { - dir = AbsolutePathBuf::try_from(canon)?; - } - - let mut merged = TomlValue::Table(toml::map::Map::new()); - for layer in config.config_layer_stack.get_layers( - ConfigLayerStackOrdering::LowestPrecedenceFirst, - /*include_disabled*/ false, - ) { - if matches!(layer.name, ConfigLayerSource::Project { .. }) { - continue; - } - merge_toml_values(&mut merged, &layer.config); - } - let project_root_markers = match project_root_markers_from_config(&merged) { - Ok(Some(markers)) => markers, - Ok(None) => default_project_root_markers(), - Err(err) => { - tracing::warn!("invalid project_root_markers: {err}"); - default_project_root_markers() - } - }; - let mut project_root = None; - if !project_root_markers.is_empty() { - for ancestor in dir.ancestors() { - for marker in &project_root_markers { - let marker_path = ancestor.join(marker); - let marker_exists = match fs.get_metadata(&marker_path, /*sandbox*/ None).await { - Ok(_) => true, - Err(err) if err.kind() == io::ErrorKind::NotFound => false, - Err(err) => return Err(err), - }; - if marker_exists { - project_root = Some(ancestor.clone()); - break; - } - } - if project_root.is_some() { - break; - } - } - } - - let search_dirs: Vec = if let Some(root) = project_root { - let mut dirs = Vec::new(); - let mut cursor = dir.clone(); - loop { - dirs.push(cursor.clone()); - if cursor == root { - break; - } - let Some(parent) = cursor.parent() else { - break; - }; - cursor = parent; - } - dirs.reverse(); - dirs - } else { - vec![dir] - }; - - let mut found: Vec = Vec::new(); - let candidate_filenames = candidate_filenames(config); - for d in search_dirs { - for name in &candidate_filenames { - let candidate = d.join(name); - match fs.get_metadata(&candidate, /*sandbox*/ None).await { - Ok(md) if md.is_file => { - found.push(candidate); - break; - } - Ok(_) => {} - Err(err) if err.kind() == io::ErrorKind::NotFound => continue, - Err(err) => return Err(err), - } - } - } - - Ok(found) -} -fn candidate_filenames<'a>(config: &'a Config) -> Vec<&'a str> { - let mut names: Vec<&'a str> = - Vec::with_capacity(2 + config.project_doc_fallback_filenames.len()); - names.push(LOCAL_PROJECT_DOC_FILENAME); - names.push(DEFAULT_PROJECT_DOC_FILENAME); - for candidate in &config.project_doc_fallback_filenames { - let candidate = candidate.as_str(); - if candidate.is_empty() { - continue; - } - if !names.contains(&candidate) { - names.push(candidate); - } - } - names -} - -#[cfg(test)] -#[path = "project_doc_tests.rs"] -mod tests; diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 890489bc2c..05f1bb5ccc 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -55,7 +55,7 @@ use crate::bottom_pane::StatusLinePreviewData; use crate::bottom_pane::StatusLineSetupView; use crate::bottom_pane::TerminalTitleItem; use crate::bottom_pane::TerminalTitleSetupView; -use crate::legacy_core::DEFAULT_PROJECT_DOC_FILENAME; +use crate::legacy_core::DEFAULT_AGENTS_MD_FILENAME; use crate::legacy_core::config::Config; use crate::legacy_core::config::Constrained; use crate::legacy_core::config::ConstraintResult; diff --git a/codex-rs/tui/src/chatwidget/slash_dispatch.rs b/codex-rs/tui/src/chatwidget/slash_dispatch.rs index ee4d22e95f..2c5c83b380 100644 --- a/codex-rs/tui/src/chatwidget/slash_dispatch.rs +++ b/codex-rs/tui/src/chatwidget/slash_dispatch.rs @@ -92,10 +92,10 @@ impl ChatWidget { self.app_event_tx.send(AppEvent::ForkCurrentSession); } SlashCommand::Init => { - let init_target = self.config.cwd.join(DEFAULT_PROJECT_DOC_FILENAME); + let init_target = self.config.cwd.join(DEFAULT_AGENTS_MD_FILENAME); if init_target.exists() { let message = format!( - "{DEFAULT_PROJECT_DOC_FILENAME} already exists here. Skipping /init to avoid overwriting it." + "{DEFAULT_AGENTS_MD_FILENAME} already exists here. Skipping /init to avoid overwriting it." ); self.add_info_message(message, /*hint*/ None); return; diff --git a/codex-rs/tui/src/chatwidget/tests/slash_commands.rs b/codex-rs/tui/src/chatwidget/tests/slash_commands.rs index 0c6b9be8e6..c7caacc2eb 100644 --- a/codex-rs/tui/src/chatwidget/tests/slash_commands.rs +++ b/codex-rs/tui/src/chatwidget/tests/slash_commands.rs @@ -73,7 +73,7 @@ async fn ctrl_d_with_modal_open_does_not_quit() { async fn slash_init_skips_when_project_doc_exists() { let (mut chat, mut rx, mut op_rx) = make_chatwidget_manual(/*model_override*/ None).await; let tempdir = tempdir().unwrap(); - let existing_path = tempdir.path().join(DEFAULT_PROJECT_DOC_FILENAME); + let existing_path = tempdir.path().join(DEFAULT_AGENTS_MD_FILENAME); std::fs::write(&existing_path, "existing instructions").unwrap(); chat.config.cwd = tempdir.path().to_path_buf().abs(); @@ -88,7 +88,7 @@ async fn slash_init_skips_when_project_doc_exists() { assert_eq!(cells.len(), 1, "expected one info message"); let rendered = lines_to_single_string(&cells[0]); assert!( - rendered.contains(DEFAULT_PROJECT_DOC_FILENAME), + rendered.contains(DEFAULT_AGENTS_MD_FILENAME), "info message should mention the existing file: {rendered:?}" ); assert!( diff --git a/codex-rs/tui/src/status/helpers.rs b/codex-rs/tui/src/status/helpers.rs index 2ea56e0b5c..27292e1364 100644 --- a/codex-rs/tui/src/status/helpers.rs +++ b/codex-rs/tui/src/status/helpers.rs @@ -185,8 +185,8 @@ fn title_case(s: &str) -> String { #[cfg(test)] mod tests { use super::*; - use crate::legacy_core::DEFAULT_PROJECT_DOC_FILENAME; - use crate::legacy_core::LOCAL_PROJECT_DOC_FILENAME; + use crate::legacy_core::DEFAULT_AGENTS_MD_FILENAME; + use crate::legacy_core::LOCAL_AGENTS_MD_FILENAME; use crate::legacy_core::config::ConfigBuilder; use codex_utils_absolute_path::test_support::PathBufExt; use pretty_assertions::assert_eq; @@ -227,7 +227,7 @@ mod tests { async fn compose_agents_summary_includes_global_agents_path() { let codex_home = TempDir::new().expect("temp codex home"); let cwd = TempDir::new().expect("temp cwd"); - let global_agents_path = codex_home.path().join(DEFAULT_PROJECT_DOC_FILENAME); + let global_agents_path = codex_home.path().join(DEFAULT_AGENTS_MD_FILENAME); let config = test_config(&codex_home, &cwd).await; assert_eq!( @@ -240,7 +240,7 @@ mod tests { async fn compose_agents_summary_names_global_agents_override() { let codex_home = TempDir::new().expect("temp codex home"); let cwd = TempDir::new().expect("temp cwd"); - let override_path = codex_home.path().join(LOCAL_PROJECT_DOC_FILENAME); + let override_path = codex_home.path().join(LOCAL_AGENTS_MD_FILENAME); let config = test_config(&codex_home, &cwd).await; assert_eq!( @@ -253,8 +253,8 @@ mod tests { async fn compose_agents_summary_orders_global_before_project_agents() { let codex_home = TempDir::new().expect("temp codex home"); let cwd = TempDir::new().expect("temp cwd"); - let global_agents_path = codex_home.path().join(DEFAULT_PROJECT_DOC_FILENAME); - let project_agents_path = cwd.path().join(DEFAULT_PROJECT_DOC_FILENAME); + let global_agents_path = codex_home.path().join(DEFAULT_AGENTS_MD_FILENAME); + let project_agents_path = cwd.path().join(DEFAULT_AGENTS_MD_FILENAME); let config = test_config(&codex_home, &cwd).await; let summary = compose_agents_summary( @@ -270,7 +270,7 @@ mod tests { Some(format_directory_display(&global_agents_path, /*max_width*/ None).as_str()) ); let project_path = paths.next().expect("project agents path"); - assert!(project_path.ends_with(DEFAULT_PROJECT_DOC_FILENAME)); + assert!(project_path.ends_with(DEFAULT_AGENTS_MD_FILENAME)); assert_eq!(paths.next(), None); } } From 206dd13c329e4174090b8d77b4da088ffa930144 Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Thu, 16 Apr 2026 11:16:44 -0700 Subject: [PATCH 83/84] Move more connector logic into connectors crate (#18158) Reduce the size of core --- codex-rs/Cargo.lock | 2 + codex-rs/chatgpt/Cargo.toml | 1 + codex-rs/chatgpt/src/connectors.rs | 97 ++--- codex-rs/connectors/src/accessible.rs | 76 ++++ codex-rs/connectors/src/filter.rs | 68 ++++ codex-rs/connectors/src/lib.rs | 5 + codex-rs/connectors/src/merge.rs | 119 ++++++ codex-rs/connectors/src/metadata.rs | 27 ++ codex-rs/core/src/codex.rs | 18 +- codex-rs/core/src/codex_tests.rs | 2 +- codex-rs/core/src/connectors.rs | 348 ++---------------- codex-rs/core/src/connectors_tests.rs | 44 ++- codex-rs/core/src/plugins/injection.rs | 3 +- codex-rs/core/src/plugins/mentions.rs | 3 +- codex-rs/tui/Cargo.toml | 1 + codex-rs/tui/src/app_event.rs | 2 +- codex-rs/tui/src/bottom_pane/chat_composer.rs | 7 +- codex-rs/tui/src/chatwidget.rs | 19 +- codex-rs/tui/src/chatwidget/skills.rs | 7 +- .../chatwidget/tests/popups_and_settings.rs | 49 +-- 20 files changed, 462 insertions(+), 436 deletions(-) create mode 100644 codex-rs/connectors/src/accessible.rs create mode 100644 codex-rs/connectors/src/filter.rs create mode 100644 codex-rs/connectors/src/merge.rs create mode 100644 codex-rs/connectors/src/metadata.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index a499a511ca..296e51725c 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1649,6 +1649,7 @@ version = "0.0.0" dependencies = [ "anyhow", "clap", + "codex-app-server-protocol", "codex-config", "codex-connectors", "codex-core", @@ -2936,6 +2937,7 @@ dependencies = [ "codex-cli", "codex-cloud-requirements", "codex-config", + "codex-connectors", "codex-exec-server", "codex-features", "codex-feedback", diff --git a/codex-rs/chatgpt/Cargo.toml b/codex-rs/chatgpt/Cargo.toml index 381b4fc873..354449934a 100644 --- a/codex-rs/chatgpt/Cargo.toml +++ b/codex-rs/chatgpt/Cargo.toml @@ -10,6 +10,7 @@ workspace = true [dependencies] anyhow = { workspace = true } clap = { workspace = true, features = ["derive"] } +codex-app-server-protocol = { workspace = true } codex-connectors = { workspace = true } codex-config = { workspace = true } codex-core = { workspace = true } diff --git a/codex-rs/chatgpt/src/connectors.rs b/codex-rs/chatgpt/src/connectors.rs index f37609be31..5f6efbc124 100644 --- a/codex-rs/chatgpt/src/connectors.rs +++ b/codex-rs/chatgpt/src/connectors.rs @@ -1,7 +1,3 @@ -use codex_core::config::Config; -use codex_login::AuthManager; -use codex_login::CodexAuth; -use codex_login::token_data::TokenData; use std::collections::HashSet; use std::time::Duration; @@ -9,21 +5,24 @@ use crate::chatgpt_client::chatgpt_get_request_with_timeout; use crate::chatgpt_token::get_chatgpt_token_data; use crate::chatgpt_token::init_chatgpt_token_from_auth; +use codex_app_server_protocol::AppInfo; use codex_connectors::AllConnectorsCacheKey; use codex_connectors::DirectoryListResponse; - -pub use codex_core::connectors::AppInfo; -pub use codex_core::connectors::connector_display_label; -use codex_core::connectors::filter_disallowed_connectors; +use codex_connectors::filter::filter_disallowed_connectors; +use codex_connectors::merge::merge_connectors; +use codex_connectors::merge::merge_plugin_connectors; +use codex_core::config::Config; pub use codex_core::connectors::list_accessible_connectors_from_mcp_tools; pub use codex_core::connectors::list_accessible_connectors_from_mcp_tools_with_options; pub use codex_core::connectors::list_accessible_connectors_from_mcp_tools_with_options_and_status; pub use codex_core::connectors::list_cached_accessible_connectors_from_mcp_tools; -use codex_core::connectors::merge_connectors; -use codex_core::connectors::merge_plugin_apps; pub use codex_core::connectors::with_app_enabled_state; use codex_core::plugins::AppConnectorId; use codex_core::plugins::PluginsManager; +use codex_login::AuthManager; +use codex_login::CodexAuth; +use codex_login::default_client::originator; +use codex_login::token_data::TokenData; const DIRECTORY_CONNECTORS_TIMEOUT: Duration = Duration::from_secs(60); @@ -74,8 +73,17 @@ pub async fn list_cached_all_connectors(config: &Config) -> Option> let token_data = get_chatgpt_token_data()?; let cache_key = all_connectors_cache_key(config, &token_data); let connectors = codex_connectors::cached_all_connectors(&cache_key)?; - let connectors = merge_plugin_apps(connectors, plugin_apps_for_config(config).await); - Some(filter_disallowed_connectors(connectors)) + let connectors = merge_plugin_connectors( + connectors, + plugin_apps_for_config(config) + .await + .into_iter() + .map(|connector_id| connector_id.0), + ); + Some(filter_disallowed_connectors( + connectors, + originator().value.as_str(), + )) } pub async fn list_all_connectors_with_options( @@ -105,8 +113,17 @@ pub async fn list_all_connectors_with_options( }, ) .await?; - let connectors = merge_plugin_apps(connectors, plugin_apps_for_config(config).await); - Ok(filter_disallowed_connectors(connectors)) + let connectors = merge_plugin_connectors( + connectors, + plugin_apps_for_config(config) + .await + .into_iter() + .map(|connector_id| connector_id.0), + ); + Ok(filter_disallowed_connectors( + connectors, + originator().value.as_str(), + )) } fn all_connectors_cache_key(config: &Config, token_data: &TokenData) -> AllConnectorsCacheKey { @@ -134,7 +151,13 @@ pub fn connectors_for_plugin_apps( .map(|connector_id| connector_id.0.as_str()) .collect::>(); - filter_disallowed_connectors(merge_plugin_apps(connectors, plugin_apps.to_vec())) + let connectors = merge_plugin_connectors( + connectors, + plugin_apps + .iter() + .map(|connector_id| connector_id.0.clone()), + ); + filter_disallowed_connectors(connectors, originator().value.as_str()) .into_iter() .filter(|connector| plugin_app_ids.contains(connector.id.as_str())) .collect() @@ -158,13 +181,13 @@ pub fn merge_connectors_with_accessible( accessible_connectors }; let merged = merge_connectors(connectors, accessible_connectors); - filter_disallowed_connectors(merged) + filter_disallowed_connectors(merged, originator().value.as_str()) } #[cfg(test)] mod tests { use super::*; - use codex_core::connectors::connector_install_url; + use codex_connectors::metadata::connector_install_url; use codex_core::plugins::AppConnectorId; use pretty_assertions::assert_eq; @@ -186,46 +209,6 @@ mod tests { } } - #[test] - fn allows_asdk_connectors() { - let filtered = filter_disallowed_connectors(vec![app("asdk_app_hidden"), app("alpha")]); - assert_eq!(filtered, vec![app("asdk_app_hidden"), app("alpha")]); - } - - #[test] - fn allows_whitelisted_asdk_connectors() { - let filtered = filter_disallowed_connectors(vec![ - app("asdk_app_69781557cc1481919cf5e9824fa2e792"), - app("beta"), - ]); - assert_eq!( - filtered, - vec![ - app("asdk_app_69781557cc1481919cf5e9824fa2e792"), - app("beta") - ] - ); - } - - #[test] - fn filters_openai_prefixed_connectors() { - let filtered = filter_disallowed_connectors(vec![ - app("connector_openai_foo"), - app("connector_openai_bar"), - app("gamma"), - ]); - assert_eq!(filtered, vec![app("gamma")]); - } - - #[test] - fn filters_disallowed_connector_ids() { - let filtered = filter_disallowed_connectors(vec![ - app("asdk_app_6938a94a61d881918ef32cb999ff937c"), - app("delta"), - ]); - assert_eq!(filtered, vec![app("delta")]); - } - fn merged_app(id: &str, is_accessible: bool) -> AppInfo { AppInfo { id: id.to_string(), diff --git a/codex-rs/connectors/src/accessible.rs b/codex-rs/connectors/src/accessible.rs new file mode 100644 index 0000000000..c44f8d8a38 --- /dev/null +++ b/codex-rs/connectors/src/accessible.rs @@ -0,0 +1,76 @@ +use std::collections::BTreeSet; +use std::collections::HashMap; + +use crate::metadata::connector_install_url; +use crate::normalize_connector_value; +use codex_app_server_protocol::AppInfo; + +pub struct AccessibleConnectorTool { + pub connector_id: String, + pub connector_name: Option, + pub connector_description: Option, + pub plugin_display_names: Vec, +} + +pub fn collect_accessible_connectors(tools: I) -> Vec +where + I: IntoIterator, +{ + let mut connectors: HashMap)> = HashMap::new(); + for tool in tools { + let connector_id = tool.connector_id; + let connector_name = normalize_connector_value(tool.connector_name.as_deref()) + .unwrap_or_else(|| connector_id.clone()); + let connector_description = + normalize_connector_value(tool.connector_description.as_deref()); + if let Some((existing, existing_plugin_display_names)) = connectors.get_mut(&connector_id) { + if existing.name == connector_id && connector_name != connector_id { + existing.name = connector_name; + } + if existing.description.is_none() && connector_description.is_some() { + existing.description = connector_description; + } + existing_plugin_display_names.extend(tool.plugin_display_names); + } else { + connectors.insert( + connector_id.clone(), + ( + AppInfo { + id: connector_id.clone(), + name: connector_name, + description: connector_description, + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + branding: None, + app_metadata: None, + labels: None, + install_url: None, + is_accessible: true, + is_enabled: true, + plugin_display_names: Vec::new(), + }, + tool.plugin_display_names + .into_iter() + .collect::>(), + ), + ); + } + } + let mut accessible: Vec = connectors + .into_values() + .map(|(mut connector, plugin_display_names)| { + connector.plugin_display_names = plugin_display_names.into_iter().collect(); + connector.install_url = Some(connector_install_url(&connector.name, &connector.id)); + connector + }) + .collect(); + accessible.sort_by(|left, right| { + right + .is_accessible + .cmp(&left.is_accessible) + .then_with(|| left.name.cmp(&right.name)) + .then_with(|| left.id.cmp(&right.id)) + }); + accessible +} diff --git a/codex-rs/connectors/src/filter.rs b/codex-rs/connectors/src/filter.rs new file mode 100644 index 0000000000..82c334f82d --- /dev/null +++ b/codex-rs/connectors/src/filter.rs @@ -0,0 +1,68 @@ +use std::collections::HashSet; + +use codex_app_server_protocol::AppInfo; + +pub fn filter_tool_suggest_discoverable_connectors( + directory_connectors: Vec, + accessible_connectors: &[AppInfo], + discoverable_connector_ids: &HashSet, + originator_value: &str, +) -> Vec { + let accessible_connector_ids: HashSet<&str> = accessible_connectors + .iter() + .filter(|connector| connector.is_accessible) + .map(|connector| connector.id.as_str()) + .collect(); + + let mut connectors = filter_disallowed_connectors(directory_connectors, originator_value) + .into_iter() + .filter(|connector| !accessible_connector_ids.contains(connector.id.as_str())) + .filter(|connector| discoverable_connector_ids.contains(connector.id.as_str())) + .collect::>(); + connectors.sort_by(|left, right| { + left.name + .cmp(&right.name) + .then_with(|| left.id.cmp(&right.id)) + }); + connectors +} + +const DISALLOWED_CONNECTOR_IDS: &[&str] = &[ + "asdk_app_6938a94a61d881918ef32cb999ff937c", + "connector_2b0a9009c9c64bf9933a3dae3f2b1254", + "connector_3f8d1a79f27c4c7ba1a897ab13bf37dc", + "connector_68de829bf7648191acd70a907364c67c", + "connector_68e004f14af881919eb50893d3d9f523", + "connector_69272cb413a081919685ec3c88d1744e", +]; +const FIRST_PARTY_CHAT_DISALLOWED_CONNECTOR_IDS: &[&str] = + &["connector_0f9c9d4592e54d0a9a12b3f44a1e2010"]; +const DISALLOWED_CONNECTOR_PREFIX: &str = "connector_openai_"; + +pub fn filter_disallowed_connectors( + connectors: Vec, + originator_value: &str, +) -> Vec { + let first_party_chat_originator = is_first_party_chat_originator(originator_value); + connectors + .into_iter() + .filter(|connector| { + is_connector_id_allowed(connector.id.as_str(), first_party_chat_originator) + }) + .collect() +} + +fn is_first_party_chat_originator(originator_value: &str) -> bool { + originator_value == "codex_atlas" || originator_value == "codex_chatgpt_desktop" +} + +fn is_connector_id_allowed(connector_id: &str, first_party_chat_originator: bool) -> bool { + let disallowed_connector_ids = if first_party_chat_originator { + FIRST_PARTY_CHAT_DISALLOWED_CONNECTOR_IDS + } else { + DISALLOWED_CONNECTOR_IDS + }; + + !connector_id.starts_with(DISALLOWED_CONNECTOR_PREFIX) + && !disallowed_connector_ids.contains(&connector_id) +} diff --git a/codex-rs/connectors/src/lib.rs b/codex-rs/connectors/src/lib.rs index a74ece5a35..d7c5fb405c 100644 --- a/codex-rs/connectors/src/lib.rs +++ b/codex-rs/connectors/src/lib.rs @@ -10,6 +10,11 @@ use codex_app_server_protocol::AppInfo; use codex_app_server_protocol::AppMetadata; use serde::Deserialize; +pub mod accessible; +pub mod filter; +pub mod merge; +pub mod metadata; + pub const CONNECTORS_CACHE_TTL: Duration = Duration::from_secs(3600); #[derive(Clone, Debug, PartialEq, Eq)] diff --git a/codex-rs/connectors/src/merge.rs b/codex-rs/connectors/src/merge.rs new file mode 100644 index 0000000000..b41ee63add --- /dev/null +++ b/codex-rs/connectors/src/merge.rs @@ -0,0 +1,119 @@ +use std::collections::HashMap; +use std::collections::HashSet; + +use crate::metadata::connector_install_url; +use crate::metadata::sort_connectors_by_accessibility_and_name; +use codex_app_server_protocol::AppInfo; + +pub fn merge_connectors( + connectors: Vec, + accessible_connectors: Vec, +) -> Vec { + let mut merged: HashMap = connectors + .into_iter() + .map(|mut connector| { + connector.is_accessible = false; + (connector.id.clone(), connector) + }) + .collect(); + + for mut connector in accessible_connectors { + connector.is_accessible = true; + let connector_id = connector.id.clone(); + if let Some(existing) = merged.get_mut(&connector_id) { + existing.is_accessible = true; + if existing.name == existing.id && connector.name != connector.id { + existing.name = connector.name; + } + if existing.description.is_none() && connector.description.is_some() { + existing.description = connector.description; + } + if existing.logo_url.is_none() && connector.logo_url.is_some() { + existing.logo_url = connector.logo_url; + } + if existing.logo_url_dark.is_none() && connector.logo_url_dark.is_some() { + existing.logo_url_dark = connector.logo_url_dark; + } + if existing.distribution_channel.is_none() && connector.distribution_channel.is_some() { + existing.distribution_channel = connector.distribution_channel; + } + existing + .plugin_display_names + .extend(connector.plugin_display_names); + } else { + merged.insert(connector_id, connector); + } + } + + let mut merged = merged.into_values().collect::>(); + for connector in &mut merged { + if connector.install_url.is_none() { + connector.install_url = Some(connector_install_url(&connector.name, &connector.id)); + } + connector.plugin_display_names.sort_unstable(); + connector.plugin_display_names.dedup(); + } + sort_connectors_by_accessibility_and_name(&mut merged); + merged +} + +pub fn merge_plugin_connectors(connectors: Vec, plugin_app_ids: I) -> Vec +where + I: IntoIterator, +{ + let mut merged = connectors; + let mut connector_ids = merged + .iter() + .map(|connector| connector.id.clone()) + .collect::>(); + + for connector_id in plugin_app_ids { + if connector_ids.insert(connector_id.clone()) { + merged.push(plugin_connector_to_app_info(connector_id)); + } + } + + sort_connectors_by_accessibility_and_name(&mut merged); + merged +} + +pub fn merge_plugin_connectors_with_accessible( + plugin_app_ids: I, + accessible_connectors: Vec, +) -> Vec +where + I: IntoIterator, +{ + let accessible_connector_ids: HashSet<&str> = accessible_connectors + .iter() + .map(|connector| connector.id.as_str()) + .collect(); + let plugin_connectors = plugin_app_ids + .into_iter() + .filter(|connector_id| accessible_connector_ids.contains(connector_id.as_str())) + .map(plugin_connector_to_app_info) + .collect::>(); + merge_connectors(plugin_connectors, accessible_connectors) +} + +pub fn plugin_connector_to_app_info(connector_id: String) -> AppInfo { + // Leave the placeholder name as the connector id so merge_connectors() can + // replace it with canonical app metadata from directory fetches or + // connector_name values from codex_apps tool discovery. + let name = connector_id.clone(); + AppInfo { + id: connector_id.clone(), + name: name.clone(), + description: None, + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + branding: None, + app_metadata: None, + labels: None, + install_url: Some(connector_install_url(&name, &connector_id)), + is_accessible: false, + is_enabled: true, + plugin_display_names: Vec::new(), + } +} diff --git a/codex-rs/connectors/src/metadata.rs b/codex-rs/connectors/src/metadata.rs new file mode 100644 index 0000000000..64becb766d --- /dev/null +++ b/codex-rs/connectors/src/metadata.rs @@ -0,0 +1,27 @@ +use codex_app_server_protocol::AppInfo; + +pub fn connector_display_label(connector: &AppInfo) -> String { + connector.name.clone() +} + +pub fn connector_mention_slug(connector: &AppInfo) -> String { + crate::connector_name_slug(&connector_display_label(connector)) +} + +pub fn connector_install_url(name: &str, connector_id: &str) -> String { + crate::connector_install_url(name, connector_id) +} + +pub fn sanitize_name(name: &str) -> String { + crate::connector_name_slug(name).replace("-", "_") +} + +pub(crate) fn sort_connectors_by_accessibility_and_name(connectors: &mut [AppInfo]) { + connectors.sort_by(|left, right| { + right + .is_accessible + .cmp(&left.is_accessible) + .then_with(|| left.name.cmp(&right.name)) + .then_with(|| left.id.cmp(&right.id)) + }); +} diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 50a4c25abd..5274d88664 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -6267,8 +6267,11 @@ pub(crate) async fn run_turn( HashMap::new() }; let available_connectors = if turn_context.apps_enabled() { - let connectors = connectors::merge_plugin_apps_with_accessible( - loaded_plugins.effective_apps(), + let connectors = codex_connectors::merge::merge_plugin_connectors_with_accessible( + loaded_plugins + .effective_apps() + .into_iter() + .map(|connector_id| connector_id.0), connectors::accessible_connectors_from_mcp_tools(&mcp_tools), ); connectors::with_app_enabled_state(connectors, &turn_context.config) @@ -6914,7 +6917,7 @@ fn collect_explicit_app_ids_from_skill_items( let connector_slug_counts = build_connector_slug_counts(connectors); for connector in connectors { - let slug = connectors::connector_mention_slug(connector); + let slug = codex_connectors::metadata::connector_mention_slug(connector); let connector_count = connector_slug_counts.get(&slug).copied().unwrap_or(0); let skill_count = skill_name_counts_lower.get(&slug).copied().unwrap_or(0); if connector_count == 1 && skill_count == 0 && mention_names_lower.contains(&slug) { @@ -6989,7 +6992,7 @@ fn connector_inserted_in_messages( return true; } - let mention_slug = connectors::connector_mention_slug(connector); + let mention_slug = codex_connectors::metadata::connector_mention_slug(connector); let connector_count = connector_slug_counts .get(&mention_slug) .copied() @@ -7217,8 +7220,11 @@ pub(crate) async fn built_tools( connectors::with_app_enabled_state(connectors.clone(), &turn_context.config) }); let connectors = if apps_enabled { - let connectors = connectors::merge_plugin_apps_with_accessible( - loaded_plugins.effective_apps(), + let connectors = codex_connectors::merge::merge_plugin_connectors_with_accessible( + loaded_plugins + .effective_apps() + .into_iter() + .map(|connector_id| connector_id.0), accessible_connectors.clone().unwrap_or_default(), ); Some(connectors::with_app_enabled_state( diff --git a/codex-rs/core/src/codex_tests.rs b/codex-rs/core/src/codex_tests.rs index 6f4276b077..81ebd6c9f8 100644 --- a/codex-rs/core/src/codex_tests.rs +++ b/codex-rs/core/src/codex_tests.rs @@ -413,7 +413,7 @@ fn make_mcp_tool( ) -> ToolInfo { let tool_namespace = if server_name == CODEX_APPS_MCP_SERVER_NAME { connector_name - .map(crate::connectors::sanitize_name) + .map(codex_connectors::metadata::sanitize_name) .map(|connector_name| format!("mcp__{server_name}__{connector_name}")) .unwrap_or_else(|| server_name.to_string()) } else { diff --git a/codex-rs/core/src/connectors.rs b/codex-rs/core/src/connectors.rs index 77603adcf1..103ce251fc 100644 --- a/codex-rs/core/src/connectors.rs +++ b/codex-rs/core/src/connectors.rs @@ -1,4 +1,3 @@ -use std::collections::BTreeSet; use std::collections::HashMap; use std::collections::HashSet; use std::sync::Arc; @@ -26,7 +25,6 @@ use crate::codex::INITIAL_SUBMIT_ID; use crate::config::Config; use crate::config_loader::AppsRequirementsToml; use crate::mcp::McpManager; -use crate::plugins::AppConnectorId; use crate::plugins::PluginsManager; use crate::plugins::list_tool_suggest_discoverable_plugins; use codex_config::types::AppToolApproval; @@ -36,7 +34,6 @@ use codex_features::Feature; use codex_login::AuthManager; use codex_login::CodexAuth; use codex_login::default_client::create_client; -use codex_login::default_client::is_first_party_chat_originator; use codex_login::default_client::originator; use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; use codex_mcp::McpConnectionManager; @@ -46,7 +43,6 @@ use codex_mcp::codex_apps_tools_cache_key; use codex_mcp::compute_auth_statuses; use codex_mcp::with_codex_apps_mcp; -pub use codex_connectors::CONNECTORS_CACHE_TTL; const CONNECTORS_READY_TIMEOUT_ON_EMPTY_TOOLS: Duration = Duration::from_secs(30); const DIRECTORY_CONNECTORS_TIMEOUT: Duration = Duration::from_secs(60); @@ -122,13 +118,15 @@ pub(crate) async fn list_tool_suggest_discoverable_tools_with_auth( let directory_connectors = list_directory_connectors_for_tool_suggest_with_auth(config, auth).await?; let connector_ids = tool_suggest_connector_ids(config).await; - let discoverable_connectors = filter_tool_suggest_discoverable_connectors( - directory_connectors, - accessible_connectors, - &connector_ids, - ) - .into_iter() - .map(DiscoverableTool::from); + let discoverable_connectors = + codex_connectors::filter::filter_tool_suggest_discoverable_connectors( + directory_connectors, + accessible_connectors, + &connector_ids, + originator().value.as_str(), + ) + .into_iter() + .map(DiscoverableTool::from); let discoverable_plugins = list_tool_suggest_discoverable_plugins(config) .await? .into_iter() @@ -151,7 +149,12 @@ pub async fn list_cached_accessible_connectors_from_mcp_tools( return Some(Vec::new()); } let cache_key = accessible_connectors_cache_key(config, auth.as_ref()); - read_cached_accessible_connectors(&cache_key).map(filter_disallowed_connectors) + read_cached_accessible_connectors(&cache_key).map(|connectors| { + codex_connectors::filter::filter_disallowed_connectors( + connectors, + originator().value.as_str(), + ) + }) } pub(crate) fn refresh_accessible_connectors_cache_from_mcp_tools( @@ -164,8 +167,10 @@ pub(crate) fn refresh_accessible_connectors_cache_from_mcp_tools( } let cache_key = accessible_connectors_cache_key(config, auth); - let accessible_connectors = - filter_disallowed_connectors(accessible_connectors_from_mcp_tools(mcp_tools)); + let accessible_connectors = codex_connectors::filter::filter_disallowed_connectors( + accessible_connectors_from_mcp_tools(mcp_tools), + originator().value.as_str(), + ); write_cached_accessible_connectors(cache_key, &accessible_connectors); } @@ -202,7 +207,10 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_options_and_status( let tool_plugin_provenance = mcp_manager.tool_plugin_provenance(config).await; if !force_refetch && let Some(cached_connectors) = read_cached_accessible_connectors(&cache_key) { - let cached_connectors = filter_disallowed_connectors(cached_connectors); + let cached_connectors = codex_connectors::filter::filter_disallowed_connectors( + cached_connectors, + originator().value.as_str(), + ); let cached_connectors = with_app_plugin_sources(cached_connectors, &tool_plugin_provenance); return Ok(AccessibleConnectorsStatus { connectors: cached_connectors, @@ -293,8 +301,10 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_options_and_status( cancel_token.cancel(); } - let accessible_connectors = - filter_disallowed_connectors(accessible_connectors_from_mcp_tools(&tools)); + let accessible_connectors = codex_connectors::filter::filter_disallowed_connectors( + accessible_connectors_from_mcp_tools(&tools), + originator().value.as_str(), + ); if codex_apps_ready || !accessible_connectors.is_empty() { write_cached_accessible_connectors(cache_key, &accessible_connectors); } @@ -357,35 +367,11 @@ fn write_cached_accessible_connectors( .unwrap_or_else(std::sync::PoisonError::into_inner); *cache_guard = Some(CachedAccessibleConnectors { key: cache_key, - expires_at: Instant::now() + CONNECTORS_CACHE_TTL, + expires_at: Instant::now() + codex_connectors::CONNECTORS_CACHE_TTL, connectors: connectors.to_vec(), }); } -fn filter_tool_suggest_discoverable_connectors( - directory_connectors: Vec, - accessible_connectors: &[AppInfo], - discoverable_connector_ids: &HashSet, -) -> Vec { - let accessible_connector_ids: HashSet<&str> = accessible_connectors - .iter() - .filter(|connector| connector.is_accessible) - .map(|connector| connector.id.as_str()) - .collect(); - - let mut connectors = filter_disallowed_connectors(directory_connectors) - .into_iter() - .filter(|connector| !accessible_connector_ids.contains(connector.id.as_str())) - .filter(|connector| discoverable_connector_ids.contains(connector.id.as_str())) - .collect::>(); - connectors.sort_by(|left, right| { - left.name - .cmp(&right.name) - .then_with(|| left.id.cmp(&right.id)) - }); - connectors -} - async fn tool_suggest_connector_ids(config: &Config) -> HashSet { let mut connector_ids = PluginsManager::new(config.codex_home.to_path_buf()) .plugins_for_config(config) @@ -493,14 +479,6 @@ async fn chatgpt_get_request_with_token( } } -pub fn connector_display_label(connector: &AppInfo) -> String { - format_connector_label(&connector.name, &connector.id) -} - -pub fn connector_mention_slug(connector: &AppInfo) -> String { - sanitize_slug(&connector_display_label(connector)) -} - pub(crate) fn accessible_connectors_from_mcp_tools( mcp_tools: &HashMap, ) -> Vec { @@ -511,114 +489,14 @@ pub(crate) fn accessible_connectors_from_mcp_tools( return None; } let connector_id = tool.connector_id.as_deref()?; - Some(( - connector_id.to_string(), - normalize_connector_value(tool.connector_name.as_deref()), - normalize_connector_value(tool.connector_description.as_deref()), - tool.plugin_display_names.clone(), - )) - }); - collect_accessible_connectors(tools) -} - -pub fn merge_connectors( - connectors: Vec, - accessible_connectors: Vec, -) -> Vec { - let mut merged: HashMap = connectors - .into_iter() - .map(|mut connector| { - connector.is_accessible = false; - (connector.id.clone(), connector) + Some(codex_connectors::accessible::AccessibleConnectorTool { + connector_id: connector_id.to_string(), + connector_name: tool.connector_name.clone(), + connector_description: tool.connector_description.clone(), + plugin_display_names: tool.plugin_display_names.clone(), }) - .collect(); - - for mut connector in accessible_connectors { - connector.is_accessible = true; - let connector_id = connector.id.clone(); - if let Some(existing) = merged.get_mut(&connector_id) { - existing.is_accessible = true; - if existing.name == existing.id && connector.name != connector.id { - existing.name = connector.name; - } - if existing.description.is_none() && connector.description.is_some() { - existing.description = connector.description; - } - if existing.logo_url.is_none() && connector.logo_url.is_some() { - existing.logo_url = connector.logo_url; - } - if existing.logo_url_dark.is_none() && connector.logo_url_dark.is_some() { - existing.logo_url_dark = connector.logo_url_dark; - } - if existing.distribution_channel.is_none() && connector.distribution_channel.is_some() { - existing.distribution_channel = connector.distribution_channel; - } - existing - .plugin_display_names - .extend(connector.plugin_display_names); - } else { - merged.insert(connector_id, connector); - } - } - - let mut merged = merged.into_values().collect::>(); - for connector in &mut merged { - if connector.install_url.is_none() { - connector.install_url = Some(connector_install_url(&connector.name, &connector.id)); - } - connector.plugin_display_names.sort_unstable(); - connector.plugin_display_names.dedup(); - } - merged.sort_by(|left, right| { - right - .is_accessible - .cmp(&left.is_accessible) - .then_with(|| left.name.cmp(&right.name)) - .then_with(|| left.id.cmp(&right.id)) }); - merged -} - -pub fn merge_plugin_apps( - connectors: Vec, - plugin_apps: Vec, -) -> Vec { - let mut merged = connectors; - let mut connector_ids = merged - .iter() - .map(|connector| connector.id.clone()) - .collect::>(); - - for connector_id in plugin_apps { - if connector_ids.insert(connector_id.0.clone()) { - merged.push(plugin_app_to_app_info(connector_id)); - } - } - - merged.sort_by(|left, right| { - right - .is_accessible - .cmp(&left.is_accessible) - .then_with(|| left.name.cmp(&right.name)) - .then_with(|| left.id.cmp(&right.id)) - }); - merged -} - -pub fn merge_plugin_apps_with_accessible( - plugin_apps: Vec, - accessible_connectors: Vec, -) -> Vec { - let accessible_connector_ids: HashSet<&str> = accessible_connectors - .iter() - .map(|connector| connector.id.as_str()) - .collect(); - let plugin_connectors = plugin_apps - .into_iter() - .filter(|connector_id| accessible_connector_ids.contains(connector_id.0.as_str())) - .map(plugin_app_to_app_info) - .collect::>(); - merge_connectors(plugin_connectors, accessible_connectors) + codex_connectors::accessible::collect_accessible_connectors(tools) } pub fn with_app_enabled_state(mut connectors: Vec, config: &Config) -> Vec { @@ -691,45 +569,6 @@ pub(crate) fn codex_app_tool_is_enabled(config: &Config, tool_info: &ToolInfo) - .enabled } -const DISALLOWED_CONNECTOR_IDS: &[&str] = &[ - "asdk_app_6938a94a61d881918ef32cb999ff937c", - "connector_2b0a9009c9c64bf9933a3dae3f2b1254", - "connector_3f8d1a79f27c4c7ba1a897ab13bf37dc", - "connector_68de829bf7648191acd70a907364c67c", - "connector_68e004f14af881919eb50893d3d9f523", - "connector_69272cb413a081919685ec3c88d1744e", -]; -const FIRST_PARTY_CHAT_DISALLOWED_CONNECTOR_IDS: &[&str] = - &["connector_0f9c9d4592e54d0a9a12b3f44a1e2010"]; -const DISALLOWED_CONNECTOR_PREFIX: &str = "connector_openai_"; - -pub fn filter_disallowed_connectors(connectors: Vec) -> Vec { - filter_disallowed_connectors_for_originator(connectors, originator().value.as_str()) -} - -fn filter_disallowed_connectors_for_originator( - connectors: Vec, - originator_value: &str, -) -> Vec { - connectors - .into_iter() - .filter(|connector| { - is_connector_id_allowed_for_originator(connector.id.as_str(), originator_value) - }) - .collect() -} - -fn is_connector_id_allowed_for_originator(connector_id: &str, originator_value: &str) -> bool { - let disallowed_connector_ids = if is_first_party_chat_originator(originator_value) { - FIRST_PARTY_CHAT_DISALLOWED_CONNECTOR_IDS - } else { - DISALLOWED_CONNECTOR_IDS - }; - - !connector_id.starts_with(DISALLOWED_CONNECTOR_PREFIX) - && !disallowed_connector_ids.contains(&connector_id) -} - fn read_apps_config(config: &Config) -> Option { let apps_config = read_user_apps_config(config); let had_apps_config = apps_config.is_some(); @@ -851,125 +690,6 @@ fn app_tool_policy_from_apps_config( AppToolPolicy { enabled, approval } } -fn collect_accessible_connectors(tools: I) -> Vec -where - I: IntoIterator, Option, Vec)>, -{ - let mut connectors: HashMap)> = HashMap::new(); - for (connector_id, connector_name, connector_description, plugin_display_names) in tools { - let connector_name = connector_name.unwrap_or_else(|| connector_id.clone()); - if let Some((existing, existing_plugin_display_names)) = connectors.get_mut(&connector_id) { - if existing.name == connector_id && connector_name != connector_id { - existing.name = connector_name; - } - if existing.description.is_none() && connector_description.is_some() { - existing.description = connector_description; - } - existing_plugin_display_names.extend(plugin_display_names); - } else { - connectors.insert( - connector_id.clone(), - ( - AppInfo { - id: connector_id.clone(), - name: connector_name, - description: connector_description, - logo_url: None, - logo_url_dark: None, - distribution_channel: None, - branding: None, - app_metadata: None, - labels: None, - install_url: None, - is_accessible: true, - is_enabled: true, - plugin_display_names: Vec::new(), - }, - plugin_display_names - .into_iter() - .collect::>(), - ), - ); - } - } - let mut accessible: Vec = connectors - .into_values() - .map(|(mut connector, plugin_display_names)| { - connector.plugin_display_names = plugin_display_names.into_iter().collect(); - connector.install_url = Some(connector_install_url(&connector.name, &connector.id)); - connector - }) - .collect(); - accessible.sort_by(|left, right| { - right - .is_accessible - .cmp(&left.is_accessible) - .then_with(|| left.name.cmp(&right.name)) - .then_with(|| left.id.cmp(&right.id)) - }); - accessible -} - -fn plugin_app_to_app_info(connector_id: AppConnectorId) -> AppInfo { - // Leave the placeholder name as the connector id so merge_connectors() can - // replace it with canonical app metadata from directory fetches or - // connector_name values from codex_apps tool discovery. - let connector_id = connector_id.0; - let name = connector_id.clone(); - AppInfo { - id: connector_id.clone(), - name: name.clone(), - description: None, - logo_url: None, - logo_url_dark: None, - distribution_channel: None, - branding: None, - app_metadata: None, - labels: None, - install_url: Some(connector_install_url(&name, &connector_id)), - is_accessible: false, - is_enabled: true, - plugin_display_names: Vec::new(), - } -} - -fn normalize_connector_value(value: Option<&str>) -> Option { - value - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_string) -} - -pub fn connector_install_url(name: &str, connector_id: &str) -> String { - let slug = sanitize_slug(name); - format!("https://chatgpt.com/apps/{slug}/{connector_id}") -} - -pub fn sanitize_name(name: &str) -> String { - sanitize_slug(name).replace("-", "_") -} - -fn sanitize_slug(name: &str) -> String { - let mut normalized = String::with_capacity(name.len()); - for character in name.chars() { - if character.is_ascii_alphanumeric() { - normalized.push(character.to_ascii_lowercase()); - } else { - normalized.push('-'); - } - } - let normalized = normalized.trim_matches('-'); - if normalized.is_empty() { - "app".to_string() - } else { - normalized.to_string() - } -} - -fn format_connector_label(name: &str, _id: &str) -> String { - name.to_string() -} - #[cfg(test)] #[path = "connectors_tests.rs"] mod tests; diff --git a/codex-rs/core/src/connectors_tests.rs b/codex-rs/core/src/connectors_tests.rs index c9f77d0469..e462c7939e 100644 --- a/codex-rs/core/src/connectors_tests.rs +++ b/codex-rs/core/src/connectors_tests.rs @@ -11,6 +11,13 @@ use codex_config::types::AppConfig; use codex_config::types::AppToolConfig; use codex_config::types::AppToolsConfig; use codex_config::types::AppsDefaultConfig; +use codex_connectors::filter::filter_disallowed_connectors; +use codex_connectors::filter::filter_tool_suggest_discoverable_connectors; +use codex_connectors::merge::merge_connectors; +use codex_connectors::merge::plugin_connector_to_app_info; +use codex_connectors::metadata::connector_install_url; +use codex_connectors::metadata::connector_mention_slug; +use codex_connectors::metadata::sanitize_name; use codex_features::Feature; use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; use codex_mcp::ToolInfo; @@ -138,7 +145,7 @@ fn with_accessible_connectors_cache_cleared(f: impl FnOnce() -> R) -> R { #[test] fn merge_connectors_replaces_plugin_placeholder_name_with_accessible_name() { - let plugin = plugin_app_to_app_info(AppConnectorId("calendar".to_string())); + let plugin = plugin_connector_to_app_info("calendar".to_string()); let accessible = google_calendar_accessible_connector(&[]); let merged = merge_connectors(vec![plugin], vec![accessible]); @@ -281,7 +288,7 @@ async fn refresh_accessible_connectors_cache_from_mcp_tools_writes_latest_instal #[test] fn merge_connectors_unions_and_dedupes_plugin_display_names() { - let mut plugin = plugin_app_to_app_info(AppConnectorId("calendar".to_string())); + let mut plugin = plugin_connector_to_app_info("calendar".to_string()); plugin.plugin_display_names = plugin_names(&["sample", "alpha", "sample"]); let accessible = google_calendar_accessible_connector(&["beta", "alpha"]); @@ -975,33 +982,40 @@ fn app_tool_policy_matches_prefix_stripped_tool_name_for_tool_config() { #[test] fn filter_disallowed_connectors_allows_non_disallowed_connectors() { - let filtered = filter_disallowed_connectors(vec![app("asdk_app_hidden"), app("alpha")]); + let filtered = + filter_disallowed_connectors(vec![app("asdk_app_hidden"), app("alpha")], "codex_cli"); assert_eq!(filtered, vec![app("asdk_app_hidden"), app("alpha")]); } #[test] fn filter_disallowed_connectors_filters_openai_prefix() { - let filtered = filter_disallowed_connectors(vec![ - app("connector_openai_foo"), - app("connector_openai_bar"), - app("gamma"), - ]); + let filtered = filter_disallowed_connectors( + vec![ + app("connector_openai_foo"), + app("connector_openai_bar"), + app("gamma"), + ], + "codex_cli", + ); assert_eq!(filtered, vec![app("gamma")]); } #[test] fn filter_disallowed_connectors_filters_disallowed_connector_ids() { - let filtered = filter_disallowed_connectors(vec![ - app("asdk_app_6938a94a61d881918ef32cb999ff937c"), - app("connector_3f8d1a79f27c4c7ba1a897ab13bf37dc"), - app("delta"), - ]); + let filtered = filter_disallowed_connectors( + vec![ + app("asdk_app_6938a94a61d881918ef32cb999ff937c"), + app("connector_3f8d1a79f27c4c7ba1a897ab13bf37dc"), + app("delta"), + ], + "codex_cli", + ); assert_eq!(filtered, vec![app("delta")]); } #[test] fn first_party_chat_originator_filters_target_and_openai_prefixed_connectors() { - let filtered = filter_disallowed_connectors_for_originator( + let filtered = filter_disallowed_connectors( vec![ app("connector_openai_foo"), app("asdk_app_6938a94a61d881918ef32cb999ff937c"), @@ -1064,6 +1078,7 @@ fn filter_tool_suggest_discoverable_connectors_keeps_only_plugin_backed_uninstal "connector_2128aebfecb84f64a069897515042a44".to_string(), "connector_68df038e0ba48191908c8434991bbac2".to_string(), ]), + "codex_cli", ); assert_eq!( @@ -1103,6 +1118,7 @@ fn filter_tool_suggest_discoverable_connectors_excludes_accessible_apps_even_whe "connector_2128aebfecb84f64a069897515042a44".to_string(), "connector_68df038e0ba48191908c8434991bbac2".to_string(), ]), + "codex_cli", ); assert_eq!(filtered, Vec::::new()); diff --git a/codex-rs/core/src/plugins/injection.rs b/codex-rs/core/src/plugins/injection.rs index 3e5bb6ecb5..2f6ec26260 100644 --- a/codex-rs/core/src/plugins/injection.rs +++ b/codex-rs/core/src/plugins/injection.rs @@ -1,6 +1,7 @@ use std::collections::BTreeSet; use std::collections::HashMap; +use codex_connectors::metadata::connector_display_label; use codex_protocol::models::DeveloperInstructions; use codex_protocol::models::ResponseItem; @@ -46,7 +47,7 @@ pub(crate) fn build_plugin_injections( .iter() .any(|plugin_name| plugin_name == &plugin.display_name) }) - .map(connectors::connector_display_label) + .map(connector_display_label) .collect::>() .into_iter() .collect::>(); diff --git a/codex-rs/core/src/plugins/mentions.rs b/codex-rs/core/src/plugins/mentions.rs index 5e11b9b91e..a5e94345cb 100644 --- a/codex-rs/core/src/plugins/mentions.rs +++ b/codex-rs/core/src/plugins/mentions.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use std::collections::HashSet; +use codex_connectors::metadata::connector_mention_slug; use codex_protocol::user_input::UserInput; use crate::connectors; @@ -108,7 +109,7 @@ pub(crate) fn build_connector_slug_counts( ) -> HashMap { let mut counts: HashMap = HashMap::new(); for connector in connectors { - let slug = connectors::connector_mention_slug(connector); + let slug = connector_mention_slug(connector); *counts.entry(slug).or_insert(0) += 1; } counts diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml index 0f29fdeae6..65f35b665a 100644 --- a/codex-rs/tui/Cargo.toml +++ b/codex-rs/tui/Cargo.toml @@ -33,6 +33,7 @@ codex-install-context = { workspace = true } codex-chatgpt = { workspace = true } codex-cloud-requirements = { workspace = true } codex-config = { workspace = true } +codex-connectors = { workspace = true } codex-exec-server = { workspace = true } codex-features = { workspace = true } codex-feedback = { workspace = true } diff --git a/codex-rs/tui/src/app_event.rs b/codex-rs/tui/src/app_event.rs index a3b31aa988..934baac231 100644 --- a/codex-rs/tui/src/app_event.rs +++ b/codex-rs/tui/src/app_event.rs @@ -10,13 +10,13 @@ use std::path::PathBuf; +use codex_app_server_protocol::AppInfo; use codex_app_server_protocol::McpServerStatus; use codex_app_server_protocol::PluginInstallResponse; use codex_app_server_protocol::PluginListResponse; use codex_app_server_protocol::PluginReadParams; use codex_app_server_protocol::PluginReadResponse; use codex_app_server_protocol::PluginUninstallResponse; -use codex_chatgpt::connectors::AppInfo; use codex_file_search::FileMatch; use codex_protocol::ThreadId; use codex_protocol::openai_models::ModelPreset; diff --git a/codex-rs/tui/src/bottom_pane/chat_composer.rs b/codex-rs/tui/src/bottom_pane/chat_composer.rs index b1c3129ff6..5b39a0f01d 100644 --- a/codex-rs/tui/src/bottom_pane/chat_composer.rs +++ b/codex-rs/tui/src/bottom_pane/chat_composer.rs @@ -209,8 +209,7 @@ use crate::legacy_core::plugins::PluginCapabilitySummary; use crate::legacy_core::skills::model::SkillMetadata; use crate::tui::FrameRequester; use crate::ui_consts::LIVE_PREFIX_COLS; -use codex_chatgpt::connectors; -use codex_chatgpt::connectors::AppInfo; +use codex_app_server_protocol::AppInfo; use codex_file_search::FileMatch; use std::cell::RefCell; use std::collections::HashMap; @@ -3417,9 +3416,9 @@ impl ChatComposer { if !connector.is_accessible || !connector.is_enabled { continue; } - let display_name = connectors::connector_display_label(connector); + let display_name = codex_connectors::metadata::connector_display_label(connector); let description = Some(Self::connector_brief_description(connector)); - let slug = crate::legacy_core::connectors::connector_mention_slug(connector); + let slug = codex_connectors::metadata::connector_mention_slug(connector); let search_terms = vec![display_name.clone(), connector.id.clone(), slug.clone()]; let connector_id = connector.id.as_str(); mentions.push(MentionItem { diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 05f1bb5ccc..f2697d9072 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -79,6 +79,7 @@ use crate::terminal_title::clear_terminal_title; use crate::terminal_title::set_terminal_title; use crate::text_formatting::proper_join; use crate::version::CODEX_CLI_VERSION; +use codex_app_server_protocol::AppInfo; use codex_app_server_protocol::AppSummary; use codex_app_server_protocol::CodexErrorInfo as AppServerCodexErrorInfo; use codex_app_server_protocol::CollabAgentState as AppServerCollabAgentState; @@ -5525,7 +5526,7 @@ impl ChatWidget { let app_mentions = find_app_mentions(&mentions, apps, &skill_names_lower); for app in app_mentions { - let slug = crate::legacy_core::connectors::connector_mention_slug(&app); + let slug = codex_connectors::metadata::connector_mention_slug(&app); if bound_names.contains(&slug) || !selected_app_ids.insert(app.id.clone()) { continue; } @@ -9701,7 +9702,7 @@ impl ChatWidget { self.config.features.enabled(Feature::Apps) && self.has_chatgpt_account } - fn connectors_for_mentions(&self) -> Option<&[connectors::AppInfo]> { + fn connectors_for_mentions(&self) -> Option<&[AppInfo]> { if !self.connectors_enabled() { return None; } @@ -9888,7 +9889,7 @@ impl ChatWidget { } } - fn open_connectors_popup(&mut self, connectors: &[connectors::AppInfo]) { + fn open_connectors_popup(&mut self, connectors: &[AppInfo]) { self.bottom_pane.show_selection_view( self.connectors_popup_params(connectors, /*selected_connector_id*/ None), ); @@ -9914,7 +9915,7 @@ impl ChatWidget { fn connectors_popup_params( &self, - connectors: &[connectors::AppInfo], + connectors: &[AppInfo], selected_connector_id: Option<&str>, ) -> SelectionViewParams { let total = connectors.len(); @@ -9937,7 +9938,7 @@ impl ChatWidget { }); let mut items: Vec = Vec::with_capacity(connectors.len()); for connector in connectors { - let connector_label = connectors::connector_display_label(connector); + let connector_label = codex_connectors::metadata::connector_display_label(connector); let connector_title = connector_label.clone(); let link_description = Self::connector_description(connector); let description = Self::connector_brief_description(connector); @@ -10011,7 +10012,7 @@ impl ChatWidget { } } - fn refresh_connectors_popup_if_open(&mut self, connectors: &[connectors::AppInfo]) { + fn refresh_connectors_popup_if_open(&mut self, connectors: &[AppInfo]) { let selected_connector_id = if let (Some(selected_index), ConnectorsCacheState::Ready(snapshot)) = ( self.bottom_pane @@ -10039,7 +10040,7 @@ impl ChatWidget { ]) } - fn connector_brief_description(connector: &connectors::AppInfo) -> String { + fn connector_brief_description(connector: &AppInfo) -> String { let status_label = Self::connector_status_label(connector); match Self::connector_description(connector) { Some(description) => format!("{status_label} ยท {description}"), @@ -10047,7 +10048,7 @@ impl ChatWidget { } } - fn connector_status_label(connector: &connectors::AppInfo) -> &'static str { + fn connector_status_label(connector: &AppInfo) -> &'static str { if connector.is_accessible { if connector.is_enabled { "Installed" @@ -10059,7 +10060,7 @@ impl ChatWidget { } } - fn connector_description(connector: &connectors::AppInfo) -> Option { + fn connector_description(connector: &AppInfo) -> Option { connector .description .as_deref() diff --git a/codex-rs/tui/src/chatwidget/skills.rs b/codex-rs/tui/src/chatwidget/skills.rs index 0bcfb3b3bd..9bbde1aaec 100644 --- a/codex-rs/tui/src/chatwidget/skills.rs +++ b/codex-rs/tui/src/chatwidget/skills.rs @@ -9,14 +9,13 @@ use crate::bottom_pane::SkillsToggleItem; use crate::bottom_pane::SkillsToggleView; use crate::bottom_pane::popup_consts::standard_popup_hint_line; use crate::legacy_core::TOOL_MENTION_SIGIL; -use crate::legacy_core::connectors::connector_mention_slug; use crate::legacy_core::skills::model::SkillDependencies; use crate::legacy_core::skills::model::SkillInterface; use crate::legacy_core::skills::model::SkillMetadata; use crate::legacy_core::skills::model::SkillToolDependency; use crate::skills_helpers::skill_description; use crate::skills_helpers::skill_display_name; -use codex_chatgpt::connectors::AppInfo; +use codex_app_server_protocol::AppInfo; use codex_protocol::parse_command::ParsedCommand; use codex_protocol::protocol::ListSkillsResponseEvent; use codex_protocol::protocol::SkillMetadata as ProtocolSkillMetadata; @@ -296,12 +295,12 @@ pub(crate) fn find_app_mentions( let mut slug_counts: HashMap = HashMap::new(); for app in apps.iter().filter(|app| app.is_enabled) { - let slug = connector_mention_slug(app); + let slug = codex_connectors::metadata::connector_mention_slug(app); *slug_counts.entry(slug).or_insert(0) += 1; } for app in apps.iter().filter(|app| app.is_enabled) { - let slug = connector_mention_slug(app); + let slug = codex_connectors::metadata::connector_mention_slug(app); let slug_count = slug_counts.get(&slug).copied().unwrap_or(0); if mentions.names.contains(&slug) && !explicit_names.contains(&slug) diff --git a/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs b/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs index 375c37314b..10e3451d03 100644 --- a/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs +++ b/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs @@ -1,4 +1,5 @@ use super::*; +use codex_app_server_protocol::AppInfo; use pretty_assertions::assert_eq; #[tokio::test] @@ -511,7 +512,7 @@ async fn apps_popup_stays_loading_until_final_snapshot_updates() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: notion_id.to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -545,7 +546,7 @@ async fn apps_popup_stays_loading_until_final_snapshot_updates() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { connectors: vec![ - codex_chatgpt::connectors::AppInfo { + AppInfo { id: notion_id.to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -560,7 +561,7 @@ async fn apps_popup_stays_loading_until_final_snapshot_updates() { is_enabled: true, plugin_display_names: Vec::new(), }, - codex_chatgpt::connectors::AppInfo { + AppInfo { id: linear_id.to_string(), name: "Linear".to_string(), description: Some("Project tracking".to_string()), @@ -604,7 +605,7 @@ async fn apps_refresh_failure_keeps_existing_full_snapshot() { let linear_id = "unit_test_apps_refresh_failure_connector_2"; let full_connectors = vec![ - codex_chatgpt::connectors::AppInfo { + AppInfo { id: notion_id.to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -619,7 +620,7 @@ async fn apps_refresh_failure_keeps_existing_full_snapshot() { is_enabled: true, plugin_display_names: Vec::new(), }, - codex_chatgpt::connectors::AppInfo { + AppInfo { id: linear_id.to_string(), name: "Linear".to_string(), description: Some("Project tracking".to_string()), @@ -644,7 +645,7 @@ async fn apps_refresh_failure_keeps_existing_full_snapshot() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: notion_id.to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -693,7 +694,7 @@ async fn apps_popup_preserves_selected_app_across_refresh() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { connectors: vec![ - codex_chatgpt::connectors::AppInfo { + AppInfo { id: "notion".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -708,7 +709,7 @@ async fn apps_popup_preserves_selected_app_across_refresh() { is_enabled: true, plugin_display_names: Vec::new(), }, - codex_chatgpt::connectors::AppInfo { + AppInfo { id: "slack".to_string(), name: "Slack".to_string(), description: Some("Team chat".to_string()), @@ -739,7 +740,7 @@ async fn apps_popup_preserves_selected_app_across_refresh() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { connectors: vec![ - codex_chatgpt::connectors::AppInfo { + AppInfo { id: "airtable".to_string(), name: "Airtable".to_string(), description: Some("Spreadsheets".to_string()), @@ -754,7 +755,7 @@ async fn apps_popup_preserves_selected_app_across_refresh() { is_enabled: true, plugin_display_names: Vec::new(), }, - codex_chatgpt::connectors::AppInfo { + AppInfo { id: "notion".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -769,7 +770,7 @@ async fn apps_popup_preserves_selected_app_across_refresh() { is_enabled: true, plugin_display_names: Vec::new(), }, - codex_chatgpt::connectors::AppInfo { + AppInfo { id: "slack".to_string(), name: "Slack".to_string(), description: Some("Team chat".to_string()), @@ -812,7 +813,7 @@ async fn apps_refresh_failure_with_cached_snapshot_triggers_pending_force_refetc chat.connectors_prefetch_in_flight = true; chat.connectors_force_refetch_pending = true; - let full_connectors = vec![codex_chatgpt::connectors::AppInfo { + let full_connectors = vec![AppInfo { id: "unit_test_apps_refresh_failure_pending_connector".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -855,7 +856,7 @@ async fn apps_popup_keeps_existing_full_snapshot_while_partial_refresh_loads() { chat.bottom_pane.set_connectors_enabled(/*enabled*/ true); let full_connectors = vec![ - codex_chatgpt::connectors::AppInfo { + AppInfo { id: "unit_test_connector_1".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -870,7 +871,7 @@ async fn apps_popup_keeps_existing_full_snapshot_while_partial_refresh_loads() { is_enabled: true, plugin_display_names: Vec::new(), }, - codex_chatgpt::connectors::AppInfo { + AppInfo { id: "unit_test_connector_2".to_string(), name: "Linear".to_string(), description: Some("Project tracking".to_string()), @@ -897,7 +898,7 @@ async fn apps_popup_keeps_existing_full_snapshot_while_partial_refresh_loads() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { connectors: vec![ - codex_chatgpt::connectors::AppInfo { + AppInfo { id: "unit_test_connector_1".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -912,7 +913,7 @@ async fn apps_popup_keeps_existing_full_snapshot_while_partial_refresh_loads() { is_enabled: true, plugin_display_names: Vec::new(), }, - codex_chatgpt::connectors::AppInfo { + AppInfo { id: "connector_openai_hidden".to_string(), name: "Hidden OpenAI".to_string(), description: Some("Should be filtered".to_string()), @@ -960,7 +961,7 @@ async fn apps_refresh_failure_without_full_snapshot_falls_back_to_installed_apps chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: "unit_test_apps_refresh_failure_fallback_connector".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -1019,7 +1020,7 @@ async fn apps_popup_shows_disabled_status_for_installed_but_disabled_apps() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: "connector_1".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -1073,7 +1074,7 @@ async fn apps_initial_load_applies_enabled_state_from_config() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: "connector_1".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -1139,7 +1140,7 @@ async fn apps_initial_load_applies_enabled_state_from_requirements_with_user_ove chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: "connector_1".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -1203,7 +1204,7 @@ async fn apps_initial_load_applies_enabled_state_from_requirements_without_user_ chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: "connector_1".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -1252,7 +1253,7 @@ async fn apps_refresh_preserves_toggled_enabled_state() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: "connector_1".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -1274,7 +1275,7 @@ async fn apps_refresh_preserves_toggled_enabled_state() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: "connector_1".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -1323,7 +1324,7 @@ async fn apps_popup_for_not_installed_app_uses_install_only_selected_description chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: "connector_2".to_string(), name: "Linear".to_string(), description: Some("Project tracking".to_string()), From 71174574adb09a90ebd83e2acfe284a39aaca2cf Mon Sep 17 00:00:00 2001 From: Matthew Zeng Date: Thu, 16 Apr 2026 11:18:07 -0700 Subject: [PATCH 84/84] Add server-level approval defaults for custom MCP servers (#17843) ## Summary - Add `default_tools_approval_mode` support for custom MCP server configs, matching the existing `codex_apps` behavior - Apply approval precedence as per-tool override, then server default, then `auto` - Update config serialization, CLI display, schema generation, docs, and tests ## Testing - `cargo check -p codex-config` - `cargo check -p codex-core` - `just write-config-schema` - `just fmt` - `cargo test -p codex-config` - Targeted `codex-core` tests for config parsing, config writes, and MCP approval precedence - `just fix -p codex-config -p codex-core` --- codex-rs/cli/src/mcp_cmd.rs | 10 +++++ codex-rs/codex-mcp/src/mcp/mod.rs | 1 + codex-rs/codex-mcp/src/mcp/mod_tests.rs | 2 + .../codex-mcp/src/mcp/skill_dependencies.rs | 2 + .../src/mcp/skill_dependencies_tests.rs | 2 + .../src/mcp_connection_manager_tests.rs | 2 + codex-rs/config/src/mcp_edit.rs | 7 +++ codex-rs/config/src/mcp_edit_tests.rs | 2 + codex-rs/config/src/mcp_types.rs | 8 ++++ codex-rs/config/src/mcp_types_tests.rs | 33 ++++++++++++++ codex-rs/core/config.schema.json | 8 ++++ codex-rs/core/src/config/config_tests.rs | 33 +++++++++++--- codex-rs/core/src/config/edit.rs | 7 +++ codex-rs/core/src/config/edit_tests.rs | 8 ++++ codex-rs/core/src/config/service_tests.rs | 43 +++++++++++++++++++ codex-rs/core/src/mcp_skill_dependencies.rs | 2 + codex-rs/core/src/mcp_tool_call.rs | 11 +++-- codex-rs/core/src/mcp_tool_call_tests.rs | 37 ++++++++++++++++ codex-rs/core/src/plugins/manager_tests.rs | 4 ++ codex-rs/core/tests/suite/code_mode.rs | 1 + codex-rs/core/tests/suite/js_repl.rs | 1 + codex-rs/core/tests/suite/rmcp_client.rs | 1 + codex-rs/core/tests/suite/search_tool.rs | 1 + codex-rs/core/tests/suite/sqlite_state.rs | 1 + codex-rs/core/tests/suite/tools.rs | 1 + codex-rs/core/tests/suite/truncation.rs | 3 ++ docs/config.md | 12 ++++-- 27 files changed, 231 insertions(+), 12 deletions(-) diff --git a/codex-rs/cli/src/mcp_cmd.rs b/codex-rs/cli/src/mcp_cmd.rs index 4ea084b83f..d413f72ddc 100644 --- a/codex-rs/cli/src/mcp_cmd.rs +++ b/codex-rs/cli/src/mcp_cmd.rs @@ -6,6 +6,7 @@ use anyhow::Result; use anyhow::anyhow; use anyhow::bail; use clap::ArgGroup; +use codex_config::types::AppToolApproval; use codex_config::types::McpServerConfig; use codex_config::types::McpServerTransportConfig; use codex_core::McpManager; @@ -304,6 +305,7 @@ async fn run_add(config_overrides: &CliConfigOverrides, add_args: AddArgs) -> Re disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -879,6 +881,14 @@ async fn run_get(config_overrides: &CliConfigOverrides, get_args: GetArgs) -> Re if let Some(timeout) = server.tool_timeout_sec { println!(" tool_timeout_sec: {}", timeout.as_secs_f64()); } + if let Some(approval_mode) = server.default_tools_approval_mode { + let approval_mode = match approval_mode { + AppToolApproval::Auto => "auto", + AppToolApproval::Prompt => "prompt", + AppToolApproval::Approve => "approve", + }; + println!(" default_tools_approval_mode: {approval_mode}"); + } println!(" remove: codex mcp remove {}", get_args.name); Ok(()) diff --git a/codex-rs/codex-mcp/src/mcp/mod.rs b/codex-rs/codex-mcp/src/mcp/mod.rs index db90bfe429..1dc9db0789 100644 --- a/codex-rs/codex-mcp/src/mcp/mod.rs +++ b/codex-rs/codex-mcp/src/mcp/mod.rs @@ -276,6 +276,7 @@ fn codex_apps_mcp_server_config(config: &McpConfig, auth: Option<&CodexAuth>) -> disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(30)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/codex-mcp/src/mcp/mod_tests.rs b/codex-rs/codex-mcp/src/mcp/mod_tests.rs index c1d39f2a7c..8db52a9d83 100644 --- a/codex-rs/codex-mcp/src/mcp/mod_tests.rs +++ b/codex-rs/codex-mcp/src/mcp/mod_tests.rs @@ -200,6 +200,7 @@ async fn effective_mcp_servers_preserve_user_servers_and_add_codex_apps() { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -223,6 +224,7 @@ async fn effective_mcp_servers_preserve_user_servers_and_add_codex_apps() { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/codex-mcp/src/mcp/skill_dependencies.rs b/codex-rs/codex-mcp/src/mcp/skill_dependencies.rs index ab72e25958..f785fe4bde 100644 --- a/codex-rs/codex-mcp/src/mcp/skill_dependencies.rs +++ b/codex-rs/codex-mcp/src/mcp/skill_dependencies.rs @@ -126,6 +126,7 @@ fn mcp_dependency_to_server_config( disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -154,6 +155,7 @@ fn mcp_dependency_to_server_config( disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/codex-mcp/src/mcp/skill_dependencies_tests.rs b/codex-rs/codex-mcp/src/mcp/skill_dependencies_tests.rs index b562bf9e2c..2d8390d15e 100644 --- a/codex-rs/codex-mcp/src/mcp/skill_dependencies_tests.rs +++ b/codex-rs/codex-mcp/src/mcp/skill_dependencies_tests.rs @@ -46,6 +46,7 @@ fn collect_missing_respects_canonical_installed_key() { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -98,6 +99,7 @@ fn collect_missing_dedupes_by_canonical_key_but_preserves_original_name() { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs b/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs index 6b65d455a8..cf2889ccde 100644 --- a/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs +++ b/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs @@ -799,6 +799,7 @@ fn mcp_init_error_display_prompts_for_github_pat() { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -850,6 +851,7 @@ fn mcp_init_error_display_reports_generic_errors() { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/config/src/mcp_edit.rs b/codex-rs/config/src/mcp_edit.rs index d315d23f8e..c4bb38c543 100644 --- a/codex-rs/config/src/mcp_edit.rs +++ b/codex-rs/config/src/mcp_edit.rs @@ -189,6 +189,13 @@ fn serialize_mcp_server(config: &McpServerConfig) -> TomlItem { if let Some(timeout) = config.tool_timeout_sec { entry["tool_timeout_sec"] = value(timeout.as_secs_f64()); } + if let Some(approval_mode) = config.default_tools_approval_mode { + entry["default_tools_approval_mode"] = value(match approval_mode { + AppToolApproval::Auto => "auto", + AppToolApproval::Prompt => "prompt", + AppToolApproval::Approve => "approve", + }); + } if let Some(enabled_tools) = &config.enabled_tools && !enabled_tools.is_empty() { diff --git a/codex-rs/config/src/mcp_edit_tests.rs b/codex-rs/config/src/mcp_edit_tests.rs index 5f60c2612a..cfcd73c3e5 100644 --- a/codex-rs/config/src/mcp_edit_tests.rs +++ b/codex-rs/config/src/mcp_edit_tests.rs @@ -29,6 +29,7 @@ async fn replace_mcp_servers_serializes_per_tool_approval_overrides() -> anyhow: disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: Some(AppToolApproval::Auto), enabled_tools: None, disabled_tools: None, scopes: None, @@ -62,6 +63,7 @@ async fn replace_mcp_servers_serializes_per_tool_approval_overrides() -> anyhow: r#"[mcp_servers.docs] command = "docs-server" supports_parallel_tool_calls = true +default_tools_approval_mode = "auto" [mcp_servers.docs.tools] diff --git a/codex-rs/config/src/mcp_types.rs b/codex-rs/config/src/mcp_types.rs index 8aece35c0d..75b68c3f94 100644 --- a/codex-rs/config/src/mcp_types.rs +++ b/codex-rs/config/src/mcp_types.rs @@ -93,6 +93,10 @@ pub struct McpServerConfig { #[serde(default, with = "option_duration_secs")] pub tool_timeout_sec: Option, + /// Approval mode for tools in this server unless a tool override exists. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub default_tools_approval_mode: Option, + /// Explicit allow-list of tools exposed from this server. When set, only these tools will be registered. #[serde(default, skip_serializing_if = "Option::is_none")] pub enabled_tools: Option>, @@ -158,6 +162,8 @@ pub struct RawMcpServerConfig { #[serde(default)] pub supports_parallel_tool_calls: Option, #[serde(default)] + pub default_tools_approval_mode: Option, + #[serde(default)] pub enabled_tools: Option>, #[serde(default)] pub disabled_tools: Option>, @@ -194,6 +200,7 @@ impl TryFrom for McpServerConfig { enabled, required, supports_parallel_tool_calls, + default_tools_approval_mode, enabled_tools, disabled_tools, scopes, @@ -260,6 +267,7 @@ impl TryFrom for McpServerConfig { required: required.unwrap_or_default(), supports_parallel_tool_calls: supports_parallel_tool_calls.unwrap_or_default(), disabled_reason: None, + default_tools_approval_mode, enabled_tools, disabled_tools, scopes, diff --git a/codex-rs/config/src/mcp_types_tests.rs b/codex-rs/config/src/mcp_types_tests.rs index 500ea56f62..dff4a6bbf5 100644 --- a/codex-rs/config/src/mcp_types_tests.rs +++ b/codex-rs/config/src/mcp_types_tests.rs @@ -258,6 +258,38 @@ fn deserialize_server_config_with_parallel_tool_calls() { assert!(cfg.supports_parallel_tool_calls); } +#[test] +fn deserialize_server_config_with_default_tool_approval_mode() { + let cfg: McpServerConfig = toml::from_str( + r#" + command = "echo" + default_tools_approval_mode = "approve" + + [tools.search] + approval_mode = "prompt" + "#, + ) + .expect("should deserialize default tool approval mode"); + + assert_eq!( + cfg.default_tools_approval_mode, + Some(AppToolApproval::Approve) + ); + assert_eq!( + cfg.tools.get("search"), + Some(&McpServerToolConfig { + approval_mode: Some(AppToolApproval::Prompt), + }) + ); + + let serialized = toml::to_string(&cfg).expect("should serialize MCP config"); + assert!(serialized.contains("default_tools_approval_mode = \"approve\"")); + + let round_tripped: McpServerConfig = + toml::from_str(&serialized).expect("should deserialize serialized MCP config"); + assert_eq!(round_tripped, cfg); +} + #[test] fn serialize_round_trips_server_config_with_parallel_tool_calls() { let cfg: McpServerConfig = toml::from_str( @@ -304,6 +336,7 @@ fn deserialize_ignores_unknown_server_fields() { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index 9c7eb853a9..966555856b 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -1463,6 +1463,14 @@ "default": null, "type": "string" }, + "default_tools_approval_mode": { + "allOf": [ + { + "$ref": "#/definitions/AppToolApproval" + } + ], + "default": null + }, "disabled_tools": { "default": null, "items": { diff --git a/codex-rs/core/src/config/config_tests.rs b/codex-rs/core/src/config/config_tests.rs index 0001439816..34002b129a 100644 --- a/codex-rs/core/src/config/config_tests.rs +++ b/codex-rs/core/src/config/config_tests.rs @@ -90,6 +90,7 @@ fn stdio_mcp(command: &str) -> McpServerConfig { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -113,6 +114,7 @@ fn http_mcp(url: &str) -> McpServerConfig { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2130,6 +2132,7 @@ async fn replace_mcp_servers_round_trips_entries() -> anyhow::Result<()> { disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(3)), tool_timeout_sec: Some(Duration::from_secs(5)), + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2243,23 +2246,28 @@ fn mcp_servers_toml_parses_per_tool_approval_overrides() { [mcp_servers.docs] command = "docs-server" name = "Docs" +default_tools_approval_mode = "prompt" [mcp_servers.docs.tools.search] approval_mode = "approve" "#, ) .expect("TOML deserialization should succeed"); - let tool = config + let server = config .mcp_servers .get("docs") - .and_then(|server| server.tools.get("search")) - .expect("docs/search tool config exists"); + .expect("docs server config exists"); assert_eq!( - tool, - &McpServerToolConfig { + server.default_tools_approval_mode, + Some(AppToolApproval::Prompt) + ); + + assert_eq!( + server.tools.get("search"), + Some(&McpServerToolConfig { approval_mode: Some(AppToolApproval::Approve), - } + }) ); } @@ -2380,6 +2388,7 @@ async fn replace_mcp_servers_serializes_env_sorted() -> anyhow::Result<()> { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2455,6 +2464,7 @@ async fn replace_mcp_servers_serializes_env_vars() -> anyhow::Result<()> { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2510,6 +2520,7 @@ async fn replace_mcp_servers_serializes_cwd() -> anyhow::Result<()> { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2563,6 +2574,7 @@ async fn replace_mcp_servers_streamable_http_serializes_bearer_token() -> anyhow disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(2)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2632,6 +2644,7 @@ async fn replace_mcp_servers_streamable_http_serializes_custom_headers() -> anyh disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(2)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2713,6 +2726,7 @@ async fn replace_mcp_servers_streamable_http_removes_optional_sections() -> anyh disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(2)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2747,6 +2761,7 @@ async fn replace_mcp_servers_streamable_http_removes_optional_sections() -> anyh disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2816,6 +2831,7 @@ async fn replace_mcp_servers_streamable_http_isolates_headers_between_servers() disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(2)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2840,6 +2856,7 @@ async fn replace_mcp_servers_streamable_http_isolates_headers_between_servers() disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2927,6 +2944,7 @@ async fn replace_mcp_servers_serializes_disabled_flag() -> anyhow::Result<()> { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2976,6 +2994,7 @@ async fn replace_mcp_servers_serializes_required_flag() -> anyhow::Result<()> { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -3025,6 +3044,7 @@ async fn replace_mcp_servers_serializes_tool_filters() -> anyhow::Result<()> { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: Some(vec!["allowed".to_string()]), disabled_tools: Some(vec!["blocked".to_string()]), scopes: None, @@ -3078,6 +3098,7 @@ async fn replace_mcp_servers_streamable_http_serializes_oauth_resource() -> anyh disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/core/src/config/edit.rs b/codex-rs/core/src/config/edit.rs index 6f4231f908..5adc0f998c 100644 --- a/codex-rs/core/src/config/edit.rs +++ b/codex-rs/core/src/config/edit.rs @@ -237,6 +237,13 @@ mod document_helpers { if let Some(timeout) = config.tool_timeout_sec { entry["tool_timeout_sec"] = value(timeout.as_secs_f64()); } + if let Some(approval_mode) = config.default_tools_approval_mode { + entry["default_tools_approval_mode"] = value(match approval_mode { + AppToolApproval::Auto => "auto", + AppToolApproval::Prompt => "prompt", + AppToolApproval::Approve => "approve", + }); + } if let Some(enabled_tools) = &config.enabled_tools && !enabled_tools.is_empty() { diff --git a/codex-rs/core/src/config/edit_tests.rs b/codex-rs/core/src/config/edit_tests.rs index 46680d5434..4f340d89b3 100644 --- a/codex-rs/core/src/config/edit_tests.rs +++ b/codex-rs/core/src/config/edit_tests.rs @@ -582,6 +582,7 @@ fn blocking_replace_mcp_servers_round_trips() { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: Some(vec!["one".to_string(), "two".to_string()]), disabled_tools: None, scopes: None, @@ -610,6 +611,7 @@ fn blocking_replace_mcp_servers_round_trips() { disabled_reason: None, startup_timeout_sec: Some(std::time::Duration::from_secs(5)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: Some(vec!["forbidden".to_string()]), scopes: None, @@ -675,6 +677,7 @@ fn blocking_replace_mcp_servers_serializes_tool_approval_overrides() { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: Some(AppToolApproval::Prompt), enabled_tools: None, disabled_tools: None, scopes: None, @@ -699,6 +702,7 @@ fn blocking_replace_mcp_servers_serializes_tool_approval_overrides() { let expected = "\ [mcp_servers.docs] command = \"docs-server\" +default_tools_approval_mode = \"prompt\" [mcp_servers.docs.tools.search] approval_mode = \"approve\" @@ -737,6 +741,7 @@ foo = { command = "cmd" } disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -790,6 +795,7 @@ foo = { command = "cmd" } # keep me disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -842,6 +848,7 @@ foo = { command = "cmd", args = ["--flag"] } # keep me disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -895,6 +902,7 @@ foo = { command = "cmd" } disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/core/src/config/service_tests.rs b/codex-rs/core/src/config/service_tests.rs index ae62326605..faaffb34df 100644 --- a/codex-rs/core/src/config/service_tests.rs +++ b/codex-rs/core/src/config/service_tests.rs @@ -164,6 +164,49 @@ async fn write_value_supports_nested_app_paths() -> Result<()> { Ok(()) } +#[tokio::test] +async fn write_value_supports_custom_mcp_server_default_tool_approval_mode() -> Result<()> { + let tmp = tempdir().expect("tempdir"); + std::fs::write( + tmp.path().join(CONFIG_TOML_FILE), + "[mcp_servers.docs]\ncommand = \"docs-server\"\n", + )?; + + let service = ConfigService::without_managed_config_for_tests(tmp.path().to_path_buf()); + service + .write_value(ConfigValueWriteParams { + file_path: Some(tmp.path().join(CONFIG_TOML_FILE).display().to_string()), + key_path: "mcp_servers.docs.default_tools_approval_mode".to_string(), + value: serde_json::json!("approve"), + merge_strategy: MergeStrategy::Replace, + expected_version: None, + }) + .await + .expect("write mcp server default_tools_approval_mode succeeds"); + + let contents = std::fs::read_to_string(tmp.path().join(CONFIG_TOML_FILE))?; + assert!(contents.contains("default_tools_approval_mode = \"approve\"")); + + let read = service + .read(ConfigReadParams { + include_layers: false, + cwd: None, + }) + .await + .expect("config read succeeds"); + + assert_eq!( + read.config + .additional + .get("mcp_servers") + .and_then(|servers| servers.get("docs")) + .and_then(|docs| docs.get("default_tools_approval_mode")), + Some(&serde_json::json!("approve")) + ); + + Ok(()) +} + #[tokio::test] async fn read_includes_origins_and_layers() { let tmp = tempdir().expect("tempdir"); diff --git a/codex-rs/core/src/mcp_skill_dependencies.rs b/codex-rs/core/src/mcp_skill_dependencies.rs index c87b02730c..c12fe28cb2 100644 --- a/codex-rs/core/src/mcp_skill_dependencies.rs +++ b/codex-rs/core/src/mcp_skill_dependencies.rs @@ -371,6 +371,7 @@ fn mcp_dependency_to_server_config( disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -399,6 +400,7 @@ fn mcp_dependency_to_server_config( disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/core/src/mcp_tool_call.rs b/codex-rs/core/src/mcp_tool_call.rs index 064585f447..b707d51890 100644 --- a/codex-rs/core/src/mcp_tool_call.rs +++ b/codex-rs/core/src/mcp_tool_call.rs @@ -676,9 +676,14 @@ fn custom_mcp_tool_approval_mode( .and_then(|value| { HashMap::::deserialize(value).ok() }) - .and_then(|servers| servers.get(server).cloned()) - .and_then(|server| server.tools.get(tool_name).cloned()) - .and_then(|tool| tool.approval_mode) + .and_then(|servers| { + let server_config = servers.get(server)?; + server_config + .tools + .get(tool_name) + .and_then(|tool| tool.approval_mode) + .or(server_config.default_tools_approval_mode) + }) .unwrap_or_default() } diff --git a/codex-rs/core/src/mcp_tool_call_tests.rs b/codex-rs/core/src/mcp_tool_call_tests.rs index 53e838908d..a85233c773 100644 --- a/codex-rs/core/src/mcp_tool_call_tests.rs +++ b/codex-rs/core/src/mcp_tool_call_tests.rs @@ -1146,6 +1146,43 @@ async fn persist_custom_mcp_tool_approval_writes_tool_override() { assert!(contents.contains("[mcp_servers.docs.tools.search]")); } +#[tokio::test] +async fn custom_mcp_tool_approval_mode_uses_server_default_with_tool_override() { + let tmp = tempdir().expect("tempdir"); + std::fs::write( + tmp.path().join(CONFIG_TOML_FILE), + r#" +[mcp_servers.docs] +command = "docs-server" +default_tools_approval_mode = "approve" + +[mcp_servers.docs.tools.search] +approval_mode = "prompt" +"#, + ) + .expect("seed config"); + let config = ConfigBuilder::default() + .codex_home(tmp.path().to_path_buf()) + .build() + .await + .expect("load config"); + let (_session, mut turn_context) = make_session_and_context().await; + turn_context.config = Arc::new(config); + + assert_eq!( + custom_mcp_tool_approval_mode(&turn_context, "docs", "read"), + AppToolApproval::Approve + ); + assert_eq!( + custom_mcp_tool_approval_mode(&turn_context, "docs", "search"), + AppToolApproval::Prompt + ); + assert_eq!( + custom_mcp_tool_approval_mode(&turn_context, "unknown", "search"), + AppToolApproval::Auto + ); +} + #[tokio::test] async fn maybe_persist_mcp_tool_approval_reloads_session_config() { let (session, turn_context) = make_session_and_context().await; diff --git a/codex-rs/core/src/plugins/manager_tests.rs b/codex-rs/core/src/plugins/manager_tests.rs index c82e1d174c..2b2f486586 100644 --- a/codex-rs/core/src/plugins/manager_tests.rs +++ b/codex-rs/core/src/plugins/manager_tests.rs @@ -183,6 +183,7 @@ async fn load_plugins_loads_default_skills_and_mcp_servers() { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -520,6 +521,7 @@ async fn load_plugins_uses_manifest_configured_component_paths() { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -630,6 +632,7 @@ async fn load_plugins_ignores_manifest_component_paths_without_dot_slash() { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -788,6 +791,7 @@ fn capability_index_filters_inactive_and_zero_capability_plugins() { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index a428b32ba2..474d221d27 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -239,6 +239,7 @@ async fn run_code_mode_turn_with_rmcp_config( disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(10)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/core/tests/suite/js_repl.rs b/codex-rs/core/tests/suite/js_repl.rs index 9c5eaef191..450aa08f0b 100644 --- a/codex-rs/core/tests/suite/js_repl.rs +++ b/codex-rs/core/tests/suite/js_repl.rs @@ -629,6 +629,7 @@ async fn js_repl_can_invoke_mcp_tools_by_display_name() -> Result<()> { disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(10)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/core/tests/suite/rmcp_client.rs b/codex-rs/core/tests/suite/rmcp_client.rs index 607dc7bf9c..8c7e7133a3 100644 --- a/codex-rs/core/tests/suite/rmcp_client.rs +++ b/codex-rs/core/tests/suite/rmcp_client.rs @@ -151,6 +151,7 @@ fn insert_mcp_server( disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(10)), tool_timeout_sec: options.tool_timeout_sec, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/core/tests/suite/search_tool.rs b/codex-rs/core/tests/suite/search_tool.rs index eff7bfb427..41f8e623c9 100644 --- a/codex-rs/core/tests/suite/search_tool.rs +++ b/codex-rs/core/tests/suite/search_tool.rs @@ -703,6 +703,7 @@ async fn tool_search_indexes_only_enabled_non_app_mcp_tools() -> Result<()> { disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(10)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: Some(vec!["echo".to_string(), "image".to_string()]), disabled_tools: Some(vec!["image".to_string()]), scopes: None, diff --git a/codex-rs/core/tests/suite/sqlite_state.rs b/codex-rs/core/tests/suite/sqlite_state.rs index acdc1e9d8c..271f7bfc90 100644 --- a/codex-rs/core/tests/suite/sqlite_state.rs +++ b/codex-rs/core/tests/suite/sqlite_state.rs @@ -378,6 +378,7 @@ async fn mcp_call_marks_thread_memory_mode_polluted_when_configured() -> Result< disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(10)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/core/tests/suite/tools.rs b/codex-rs/core/tests/suite/tools.rs index d0bb138188..bc286ce5ea 100644 --- a/codex-rs/core/tests/suite/tools.rs +++ b/codex-rs/core/tests/suite/tools.rs @@ -162,6 +162,7 @@ async fn historical_unavailable_mcp_call_is_exposed_as_placeholder_tool() -> Res disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(10)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/core/tests/suite/truncation.rs b/codex-rs/core/tests/suite/truncation.rs index 33c74633f3..81c22e1b6c 100644 --- a/codex-rs/core/tests/suite/truncation.rs +++ b/codex-rs/core/tests/suite/truncation.rs @@ -382,6 +382,7 @@ async fn mcp_tool_call_output_exceeds_limit_truncated_for_model() -> Result<()> disabled_reason: None, startup_timeout_sec: Some(std::time::Duration::from_secs(10)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -480,6 +481,7 @@ async fn mcp_image_output_preserves_image_and_no_text_summary() -> Result<()> { disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(10)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -761,6 +763,7 @@ async fn mcp_tool_call_output_not_truncated_with_custom_limit() -> Result<()> { disabled_reason: None, startup_timeout_sec: Some(std::time::Duration::from_secs(10)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/docs/config.md b/docs/config.md index c314ce2283..d583122e01 100644 --- a/docs/config.md +++ b/docs/config.md @@ -28,12 +28,18 @@ resources, review those read/write race conditions before enabling this setting. ## MCP tool approvals -Codex stores per-tool approval overrides for custom MCP servers under -`mcp_servers` in `~/.codex/config.toml`: +Codex stores approval defaults and per-tool overrides for custom MCP servers +under `mcp_servers` in `~/.codex/config.toml`. Set +`default_tools_approval_mode` on the server to apply a default to every tool, +and use per-tool `approval_mode` entries for exceptions: ```toml +[mcp_servers.docs] +command = "docs-server" +default_tools_approval_mode = "approve" + [mcp_servers.docs.tools.search] -approval_mode = "approve" +approval_mode = "prompt" ``` ## Apps (Connectors)