From 849716336347d7a8024ad75a514e11d0eacbc1cd Mon Sep 17 00:00:00 2001 From: zbarsky-openai Date: Wed, 28 Jan 2026 19:15:44 -0500 Subject: [PATCH 01/28] [bazel] Improve runfiles handling (#10098) we can't use runfiles directory on Windows due to path lengths, so swap to manifest strategy. Parsing the manifest is a bit complex and the format is changing in Bazel upstream, so pull in the official Rust library (via a small hack to make it importable...) and cleanup all the associated logic to work cleanly in both bazel and cargo without extra confusion --- .bazelignore | 1 + .bazelrc | 6 +- BUILD.bazel | 4 + MODULE.bazel | 7 + MODULE.bazel.lock | 4 +- codex-rs/Cargo.lock | 7 +- codex-rs/Cargo.toml | 1 + codex-rs/core/tests/suite/cli_stream.rs | 2 +- codex-rs/exec/tests/suite/auth_env.rs | 3 +- codex-rs/exec/tests/suite/resume.rs | 2 +- codex-rs/utils/cargo-bin/BUILD.bazel | 6 + codex-rs/utils/cargo-bin/Cargo.toml | 2 +- codex-rs/utils/cargo-bin/README.md | 20 ++ codex-rs/utils/cargo-bin/repo_root.marker | 1 + codex-rs/utils/cargo-bin/src/lib.rs | 227 +++++++++++++--------- defs.bzl | 11 +- patches/rules_rust.patch | 2 +- 17 files changed, 203 insertions(+), 103 deletions(-) create mode 100644 codex-rs/utils/cargo-bin/README.md create mode 100644 codex-rs/utils/cargo-bin/repo_root.marker diff --git a/.bazelignore b/.bazelignore index eda018aeb2..2e90753cb8 100644 --- a/.bazelignore +++ b/.bazelignore @@ -1,3 +1,4 @@ # Without this, Bazel will consider BUILD.bazel files in # .git/sl/origbackups (which can be populated by Sapling SCM). .git +codex-rs/target diff --git a/.bazelrc b/.bazelrc index 764754c216..ce0c2ee5d5 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,5 +1,7 @@ common --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 common --repo_env=BAZEL_NO_APPLE_CPP_TOOLCHAIN=1 +# Dummy xcode config so we don't need to build xcode_locator in repo rule. +common --xcode_version_config=//:disable_xcode common --disk_cache=~/.cache/bazel-disk-cache common --repo_contents_cache=~/.cache/bazel-repo-contents-cache @@ -9,6 +11,9 @@ startup --experimental_remote_repo_contents_cache common --experimental_platform_in_output_dir +# Runfiles strategy rationale: codex-rs/utils/cargo-bin/README.md +common --noenable_runfiles + common --enable_platform_specific_config # TODO(zbarsky): We need to untangle these libc constraints to get linux remote builds working. common:linux --host_platform=//:local @@ -44,4 +49,3 @@ common --jobs=30 common:remote --extra_execution_platforms=//:rbe common:remote --remote_executor=grpcs://remote.buildbuddy.io common:remote --jobs=800 - diff --git a/BUILD.bazel b/BUILD.bazel index 883432655c..dc57103b6b 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,3 +1,7 @@ +load("@apple_support//xcode:xcode_config.bzl", "xcode_config") + +xcode_config(name = "disable_xcode") + # We mark the local platform as glibc-compatible so that rust can grab a toolchain for us. # TODO(zbarsky): Upstream a better libc constraint into rules_rust. # We only enable this on linux though for sanity, and because it breaks remote execution. diff --git a/MODULE.bazel b/MODULE.bazel index df0df9b25c..f4c593b3f3 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -27,6 +27,8 @@ register_toolchains( "@toolchains_llvm_bootstrapped//toolchain:all", ) +# Needed to disable xcode... +bazel_dep(name = "apple_support", version = "2.1.0") bazel_dep(name = "rules_cc", version = "0.2.16") bazel_dep(name = "rules_platform", version = "0.1.0") bazel_dep(name = "rules_rust", version = "0.68.1") @@ -90,6 +92,11 @@ crate.annotation( inject_repo(crate, "openssl") +crate.annotation( + crate = "runfiles", + workspace_cargo_toml = "rust/runfiles/Cargo.toml", +) + # Fix readme inclusions crate.annotation( crate = "windows-link", diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 7444d3fdbe..63b7ed16ee 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -21,7 +21,8 @@ "https://bcr.bazel.build/modules/apple_support/1.23.0/MODULE.bazel": "317d47e3f65b580e7fb4221c160797fda48e32f07d2dfff63d754ef2316dcd25", "https://bcr.bazel.build/modules/apple_support/1.24.1/MODULE.bazel": "f46e8ddad60aef170ee92b2f3d00ef66c147ceafea68b6877cb45bd91737f5f8", "https://bcr.bazel.build/modules/apple_support/1.24.2/MODULE.bazel": "0e62471818affb9f0b26f128831d5c40b074d32e6dda5a0d3852847215a41ca4", - "https://bcr.bazel.build/modules/apple_support/1.24.2/source.json": "2c22c9827093250406c5568da6c54e6fdf0ef06238def3d99c71b12feb057a8d", + "https://bcr.bazel.build/modules/apple_support/2.1.0/MODULE.bazel": "b15c125dabed01b6803c129cd384de4997759f02f8ec90dc5136bcf6dfc5086a", + "https://bcr.bazel.build/modules/apple_support/2.1.0/source.json": "78064cfefe18dee4faaf51893661e0d403784f3efe88671d727cdcdc67ed8fb3", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.19.3/MODULE.bazel": "253d739ba126f62a5767d832765b12b59e9f8d2bc88cc1572f4a73e46eb298ca", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.19.3/source.json": "ffab9254c65ba945f8369297ad97ca0dec213d3adc6e07877e23a48624a8b456", @@ -790,6 +791,7 @@ "gimli_0.31.1": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"name\":\"fallible-iterator\",\"optional\":true,\"req\":\"^0.3.0\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.0.0\"},{\"default_features\":false,\"name\":\"stable_deref_trait\",\"optional\":true,\"req\":\"^1.1.0\"},{\"kind\":\"dev\",\"name\":\"test-assembler\",\"req\":\"^0.1.3\"}],\"features\":{\"default\":[\"read-all\",\"write\"],\"endian-reader\":[\"read\",\"dep:stable_deref_trait\"],\"fallible-iterator\":[\"dep:fallible-iterator\"],\"read\":[\"read-core\"],\"read-all\":[\"read\",\"std\",\"fallible-iterator\",\"endian-reader\"],\"read-core\":[],\"rustc-dep-of-std\":[\"dep:core\",\"dep:alloc\",\"dep:compiler_builtins\"],\"std\":[\"fallible-iterator?/std\",\"stable_deref_trait?/std\"],\"write\":[\"dep:indexmap\"]}}", "git+https://github.com/JakkuSakura/tokio-tungstenite?rev=2ae536b0de793f3ddf31fc2f22d445bf1ef2023d#2ae536b0de793f3ddf31fc2f22d445bf1ef2023d_tokio-tungstenite": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"sink\",\"std\"],\"name\":\"futures-util\",\"optional\":false},{\"name\":\"log\"},{\"default_features\":true,\"features\":[],\"name\":\"native-tls-crate\",\"optional\":true,\"package\":\"native-tls\"},{\"default_features\":false,\"features\":[],\"name\":\"rustls\",\"optional\":true},{\"default_features\":true,\"features\":[],\"name\":\"rustls-native-certs\",\"optional\":true},{\"default_features\":true,\"features\":[],\"name\":\"rustls-pki-types\",\"optional\":true},{\"default_features\":false,\"features\":[\"io-util\"],\"name\":\"tokio\",\"optional\":false},{\"default_features\":true,\"features\":[],\"name\":\"tokio-native-tls\",\"optional\":true},{\"default_features\":false,\"features\":[],\"name\":\"tokio-rustls\",\"optional\":true},{\"default_features\":false,\"features\":[],\"name\":\"tungstenite\",\"optional\":false},{\"default_features\":true,\"features\":[],\"name\":\"webpki-roots\",\"optional\":true}],\"features\":{\"__rustls-tls\":[\"rustls\",\"rustls-pki-types\",\"tokio-rustls\",\"stream\",\"tungstenite/__rustls-tls\",\"handshake\"],\"connect\":[\"stream\",\"tokio/net\",\"handshake\"],\"default\":[\"connect\",\"handshake\"],\"handshake\":[\"tungstenite/handshake\"],\"native-tls\":[\"native-tls-crate\",\"tokio-native-tls\",\"stream\",\"tungstenite/native-tls\",\"handshake\"],\"native-tls-vendored\":[\"native-tls\",\"native-tls-crate/vendored\",\"tungstenite/native-tls-vendored\"],\"proxy\":[\"tungstenite/proxy\",\"tokio/net\",\"handshake\"],\"rustls-tls-native-roots\":[\"__rustls-tls\",\"rustls-native-certs\"],\"rustls-tls-webpki-roots\":[\"__rustls-tls\",\"webpki-roots\"],\"stream\":[],\"url\":[\"tungstenite/url\"]},\"strip_prefix\":\"\"}", "git+https://github.com/JakkuSakura/tungstenite-rs?rev=f514de8644821113e5d18a027d6d28a5c8cc0a6e#f514de8644821113e5d18a027d6d28a5c8cc0a6e_tungstenite": "{\"dependencies\":[{\"name\":\"bytes\"},{\"default_features\":true,\"features\":[],\"name\":\"data-encoding\",\"optional\":true},{\"default_features\":true,\"features\":[],\"name\":\"http\",\"optional\":true},{\"default_features\":true,\"features\":[],\"name\":\"httparse\",\"optional\":true},{\"name\":\"log\"},{\"default_features\":true,\"features\":[],\"name\":\"native-tls-crate\",\"optional\":true,\"package\":\"native-tls\"},{\"name\":\"rand\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"rustls\",\"optional\":true},{\"default_features\":true,\"features\":[],\"name\":\"rustls-native-certs\",\"optional\":true},{\"default_features\":true,\"features\":[],\"name\":\"rustls-pki-types\",\"optional\":true},{\"default_features\":true,\"features\":[],\"name\":\"sha1\",\"optional\":true},{\"name\":\"thiserror\"},{\"default_features\":true,\"features\":[],\"name\":\"url\",\"optional\":true},{\"name\":\"utf-8\"},{\"default_features\":true,\"features\":[],\"name\":\"webpki-roots\",\"optional\":true}],\"features\":{\"__rustls-tls\":[\"rustls\",\"rustls-pki-types\"],\"default\":[\"handshake\"],\"handshake\":[\"data-encoding\",\"http\",\"httparse\",\"sha1\"],\"native-tls\":[\"native-tls-crate\"],\"native-tls-vendored\":[\"native-tls\",\"native-tls-crate/vendored\"],\"proxy\":[\"handshake\"],\"rustls-tls-native-roots\":[\"__rustls-tls\",\"rustls-native-certs\"],\"rustls-tls-webpki-roots\":[\"__rustls-tls\",\"webpki-roots\"],\"url\":[\"dep:url\"]},\"strip_prefix\":\"\"}", + "git+https://github.com/dzbarsky/rules_rust?rev=b56cbaa8465e74127f1ea216f813cd377295ad81#b56cbaa8465e74127f1ea216f813cd377295ad81_runfiles": "{\"dependencies\":[],\"features\":{},\"strip_prefix\":\"\"}", "git+https://github.com/helix-editor/nucleo.git?rev=4253de9faabb4e5c6d81d946a5e35a90f87347ee#4253de9faabb4e5c6d81d946a5e35a90f87347ee_nucleo": "{\"dependencies\":[{\"default_features\":true,\"features\":[],\"name\":\"nucleo-matcher\",\"optional\":false},{\"default_features\":true,\"features\":[\"send_guard\",\"arc_lock\"],\"name\":\"parking_lot\",\"optional\":false},{\"name\":\"rayon\"}],\"features\":{},\"strip_prefix\":\"\"}", "git+https://github.com/helix-editor/nucleo.git?rev=4253de9faabb4e5c6d81d946a5e35a90f87347ee#4253de9faabb4e5c6d81d946a5e35a90f87347ee_nucleo-matcher": "{\"dependencies\":[{\"name\":\"memchr\"},{\"default_features\":true,\"features\":[],\"name\":\"unicode-segmentation\",\"optional\":true}],\"features\":{\"default\":[\"unicode-normalization\",\"unicode-casefold\",\"unicode-segmentation\"],\"unicode-casefold\":[],\"unicode-normalization\":[],\"unicode-segmentation\":[\"dep:unicode-segmentation\"]},\"strip_prefix\":\"matcher\"}", "git+https://github.com/nornagon/crossterm?branch=nornagon%2Fcolor-query#87db8bfa6dc99427fd3b071681b07fc31c6ce995_crossterm": "{\"dependencies\":[{\"default_features\":true,\"features\":[],\"name\":\"bitflags\",\"optional\":false},{\"default_features\":false,\"features\":[],\"name\":\"futures-core\",\"optional\":true},{\"name\":\"parking_lot\"},{\"default_features\":true,\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true},{\"default_features\":true,\"features\":[],\"name\":\"filedescriptor\",\"optional\":true,\"target\":\"cfg(unix)\"},{\"default_features\":false,\"features\":[],\"name\":\"libc\",\"optional\":true,\"target\":\"cfg(unix)\"},{\"default_features\":true,\"features\":[\"os-poll\"],\"name\":\"mio\",\"optional\":true,\"target\":\"cfg(unix)\"},{\"default_features\":false,\"features\":[\"std\",\"stdio\",\"termios\"],\"name\":\"rustix\",\"optional\":false,\"target\":\"cfg(unix)\"},{\"default_features\":true,\"features\":[],\"name\":\"signal-hook\",\"optional\":true,\"target\":\"cfg(unix)\"},{\"default_features\":true,\"features\":[\"support-v1_0\"],\"name\":\"signal-hook-mio\",\"optional\":true,\"target\":\"cfg(unix)\"},{\"default_features\":true,\"features\":[],\"name\":\"crossterm_winapi\",\"optional\":true,\"target\":\"cfg(windows)\"},{\"default_features\":true,\"features\":[\"winuser\",\"winerror\"],\"name\":\"winapi\",\"optional\":true,\"target\":\"cfg(windows)\"}],\"features\":{\"bracketed-paste\":[],\"default\":[\"bracketed-paste\",\"windows\",\"events\"],\"event-stream\":[\"dep:futures-core\",\"events\"],\"events\":[\"dep:mio\",\"dep:signal-hook\",\"dep:signal-hook-mio\"],\"serde\":[\"dep:serde\",\"bitflags/serde\"],\"use-dev-tty\":[\"filedescriptor\",\"rustix/process\"],\"windows\":[\"dep:winapi\",\"dep:crossterm_winapi\"]},\"strip_prefix\":\"\"}", diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 38d44c44b5..c6b006c203 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1975,7 +1975,7 @@ name = "codex-utils-cargo-bin" version = "0.0.0" dependencies = [ "assert_cmd", - "path-absolutize", + "runfiles", "thiserror 2.0.17", ] @@ -6592,6 +6592,11 @@ dependencies = [ "zeroize", ] +[[package]] +name = "runfiles" +version = "0.1.0" +source = "git+https://github.com/dzbarsky/rules_rust?rev=b56cbaa8465e74127f1ea216f813cd377295ad81#b56cbaa8465e74127f1ea216f813cd377295ad81" + [[package]] name = "rustc-demangle" version = "0.1.25" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 9b0ce2e16d..5196213991 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -186,6 +186,7 @@ regex = "1.12.2" regex-lite = "0.1.8" reqwest = "0.12" rmcp = { version = "0.12.0", default-features = false } +runfiles = { git = "https://github.com/dzbarsky/rules_rust", rev = "b56cbaa8465e74127f1ea216f813cd377295ad81" } schemars = "0.8.22" seccompiler = "0.5.0" sentry = "0.46.0" diff --git a/codex-rs/core/tests/suite/cli_stream.rs b/codex-rs/core/tests/suite/cli_stream.rs index 57142b07f2..9ca2c6bd68 100644 --- a/codex-rs/core/tests/suite/cli_stream.rs +++ b/codex-rs/core/tests/suite/cli_stream.rs @@ -16,7 +16,7 @@ use wiremock::matchers::path; fn repo_root() -> std::path::PathBuf { #[expect(clippy::expect_used)] - find_resource!(".").expect("failed to resolve repo root") + codex_utils_cargo_bin::repo_root().expect("failed to resolve repo root") } fn cli_responses_fixture() -> std::path::PathBuf { diff --git a/codex-rs/exec/tests/suite/auth_env.rs b/codex-rs/exec/tests/suite/auth_env.rs index 4f8018e808..d55da946e2 100644 --- a/codex-rs/exec/tests/suite/auth_env.rs +++ b/codex-rs/exec/tests/suite/auth_env.rs @@ -1,5 +1,4 @@ #![allow(clippy::unwrap_used, clippy::expect_used)] -use codex_utils_cargo_bin::find_resource; use core_test_support::responses::ev_completed; use core_test_support::responses::mount_sse_once_match; use core_test_support::responses::sse; @@ -11,7 +10,7 @@ use wiremock::matchers::header; async fn exec_uses_codex_api_key_env_var() -> anyhow::Result<()> { let test = test_codex_exec(); let server = start_mock_server().await; - let repo_root = find_resource!(".")?; + let repo_root = codex_utils_cargo_bin::repo_root()?; mount_sse_once_match( &server, diff --git a/codex-rs/exec/tests/suite/resume.rs b/codex-rs/exec/tests/suite/resume.rs index 6e24934f21..4169c60dd1 100644 --- a/codex-rs/exec/tests/suite/resume.rs +++ b/codex-rs/exec/tests/suite/resume.rs @@ -113,7 +113,7 @@ fn exec_fixture() -> anyhow::Result { } fn exec_repo_root() -> anyhow::Result { - Ok(find_resource!(".")?) + Ok(codex_utils_cargo_bin::repo_root()?) } #[test] diff --git a/codex-rs/utils/cargo-bin/BUILD.bazel b/codex-rs/utils/cargo-bin/BUILD.bazel index 77d6518012..b5b5e6ba33 100644 --- a/codex-rs/utils/cargo-bin/BUILD.bazel +++ b/codex-rs/utils/cargo-bin/BUILD.bazel @@ -3,4 +3,10 @@ load("//:defs.bzl", "codex_rust_crate") codex_rust_crate( name = "cargo-bin", crate_name = "codex_utils_cargo_bin", + compile_data = ["repo_root.marker"], + lib_data_extra = ["repo_root.marker"], + test_data_extra = ["repo_root.marker"], + rustc_env = { + "CODEX_REPO_ROOT_MARKER": "$(rlocationpath :repo_root.marker)", + }, ) diff --git a/codex-rs/utils/cargo-bin/Cargo.toml b/codex-rs/utils/cargo-bin/Cargo.toml index fe3a410547..6cbe923f97 100644 --- a/codex-rs/utils/cargo-bin/Cargo.toml +++ b/codex-rs/utils/cargo-bin/Cargo.toml @@ -9,5 +9,5 @@ workspace = true [dependencies] assert_cmd = { workspace = true } -path-absolutize = { workspace = true } +runfiles = { workspace = true } thiserror = { workspace = true } diff --git a/codex-rs/utils/cargo-bin/README.md b/codex-rs/utils/cargo-bin/README.md new file mode 100644 index 0000000000..05af2f34bf --- /dev/null +++ b/codex-rs/utils/cargo-bin/README.md @@ -0,0 +1,20 @@ +# codex-utils-cargo-bin runfiles strategy + +We disable the directory-based runfiles strategy and rely on the manifest +strategy across all platforms. This avoids Windows path length issues and keeps +behavior consistent in local and remote builds on all platforms. Bazel sets +`RUNFILES_MANIFEST_FILE`, and the `codex-utils-cargo-bin` helpers use the +`runfiles` crate to resolve runfiles via that manifest. + +Function behavior: +- `cargo_bin`: reads `CARGO_BIN_EXE_*` environment variables (set by Cargo or + Bazel) and resolves them via the runfiles manifest when `RUNFILES_MANIFEST_FILE` + is present. When not under runfiles, it only accepts absolute paths from + `CARGO_BIN_EXE_*` and returns an error otherwise. +- `find_resource!`: used by tests to locate fixtures. It chooses the Bazel + runfiles resolution path when `RUNFILES_MANIFEST_FILE` is set, otherwise it + falls back to a `CARGO_MANIFEST_DIR`-relative path for Cargo runs. + +Background: +- https://bazel.build/docs/runfiles +- https://bazel.build/docs/runfiles#runfiles-manifest diff --git a/codex-rs/utils/cargo-bin/repo_root.marker b/codex-rs/utils/cargo-bin/repo_root.marker new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/codex-rs/utils/cargo-bin/repo_root.marker @@ -0,0 +1 @@ + diff --git a/codex-rs/utils/cargo-bin/src/lib.rs b/codex-rs/utils/cargo-bin/src/lib.rs index 40fa40c62f..84f69af358 100644 --- a/codex-rs/utils/cargo-bin/src/lib.rs +++ b/codex-rs/utils/cargo-bin/src/lib.rs @@ -1,7 +1,12 @@ use std::ffi::OsString; +use std::io; +use std::path::Path; use std::path::PathBuf; -pub use path_absolutize; +pub use runfiles; + +/// Bazel sets this when runfiles directories are disabled, which we do on all platforms for consistency. +const RUNFILES_MANIFEST_ONLY_ENV: &str = "RUNFILES_MANIFEST_ONLY"; #[derive(Debug, thiserror::Error)] pub enum CargoBinError { @@ -27,10 +32,9 @@ pub enum CargoBinError { /// Returns an absolute path to a binary target built for the current test run. /// -/// In `cargo test`, `CARGO_BIN_EXE_*` env vars are absolute, but Buck2 may set -/// them to project-relative paths (e.g. `buck-out/...`). Those paths break if a -/// test later changes its working directory. This helper makes the path -/// absolute up-front so callers can safely `chdir` afterwards. +/// In `cargo test`, `CARGO_BIN_EXE_*` env vars are absolute. +/// In `bazel test`, `CARGO_BIN_EXE_*` env vars are rlocationpaths, intended to be consumed by `rlocation`. +/// This helper allows callers to transparently support both. pub fn cargo_bin(name: &str) -> Result { let env_keys = cargo_bin_env_keys(name); for key in &env_keys { @@ -38,16 +42,20 @@ pub fn cargo_bin(name: &str) -> Result { return resolve_bin_from_env(key, value); } } - match assert_cmd::Command::cargo_bin(name) { Ok(cmd) => { - let abs = absolutize_from_buck_or_cwd(PathBuf::from(cmd.get_program()))?; - if abs.exists() { - Ok(abs) + let mut path = PathBuf::from(cmd.get_program()); + if !path.is_absolute() { + path = std::env::current_dir() + .map_err(|source| CargoBinError::CurrentDir { source })? + .join(path); + } + if path.exists() { + Ok(path) } else { Err(CargoBinError::ResolvedPathDoesNotExist { key: "assert_cmd::Command::cargo_bin".to_owned(), - path: abs, + path, }) } } @@ -72,6 +80,31 @@ fn cargo_bin_env_keys(name: &str) -> Vec { keys } +pub fn runfiles_available() -> bool { + std::env::var_os(RUNFILES_MANIFEST_ONLY_ENV).is_some() +} + +fn resolve_bin_from_env(key: &str, value: OsString) -> Result { + let raw = PathBuf::from(&value); + if runfiles_available() { + let runfiles = runfiles::Runfiles::create().map_err(|err| CargoBinError::CurrentExe { + source: std::io::Error::other(err), + })?; + if let Some(resolved) = runfiles::rlocation!(runfiles, &raw) + && resolved.exists() + { + return Ok(resolved); + } + } else if raw.is_absolute() && raw.exists() { + return Ok(raw); + } + + Err(CargoBinError::ResolvedPathDoesNotExist { + key: key.to_owned(), + path: raw, + }) +} + /// Macro that derives the path to a test resource at runtime, the value of /// which depends on whether Cargo or Bazel is being used to build and run a /// test. Note the return value may be a relative or absolute path. @@ -84,97 +117,109 @@ fn cargo_bin_env_keys(name: &str) -> Vec { #[macro_export] macro_rules! find_resource { ($resource:expr) => {{ - // When this code is built and run with Bazel: - // - we inject `BAZEL_PACKAGE` as a compile-time environment variable - // that points to native.package_name() - // - at runtime, Bazel will set `RUNFILES_DIR` to the runfiles directory - // - // Therefore, the compile-time value of `BAZEL_PACKAGE` will always be - // included in the compiled binary (even if it is built with Cargo), but - // we only check it at runtime if `RUNFILES_DIR` is set. let resource = std::path::Path::new(&$resource); - match std::env::var("RUNFILES_DIR") { - Ok(bazel_runtime_files) => match option_env!("BAZEL_PACKAGE") { - Some(bazel_package) => { - use $crate::path_absolutize::Absolutize; - - let manifest_dir = std::path::PathBuf::from(bazel_runtime_files) - .join("_main") - .join(bazel_package) - .join(resource); - // Note we also have to normalize (but not canonicalize!) - // the path for _Bazel_ because the original value ends with - // `codex-rs/exec-server/tests/common/../suite/bash`, but - // the `tests/common` folder will not exist at runtime under - // Bazel. As such, we have to normalize it before passing it - // to `dotslash fetch`. - manifest_dir.absolutize().map(|p| p.to_path_buf()) - } - None => Err(std::io::Error::new( - std::io::ErrorKind::NotFound, - "BAZEL_PACKAGE not set in Bazel build", - )), - }, - Err(_) => { - let manifest_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); - Ok(manifest_dir.join(resource)) - } + if $crate::runfiles_available() { + // When this code is built and run with Bazel: + // - we inject `BAZEL_PACKAGE` as a compile-time environment variable + // that points to native.package_name() + // - at runtime, Bazel will set runfiles-related env vars + $crate::resolve_bazel_runfile(option_env!("BAZEL_PACKAGE"), resource) + } else { + let manifest_dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); + Ok(manifest_dir.join(resource)) } }}; } -fn resolve_bin_from_env(key: &str, value: OsString) -> Result { - let abs = absolutize_from_buck_or_cwd(PathBuf::from(value))?; - - if abs.exists() { - Ok(abs) - } else { - Err(CargoBinError::ResolvedPathDoesNotExist { - key: key.to_owned(), - path: abs, - }) - } -} - -fn absolutize_from_buck_or_cwd(path: PathBuf) -> Result { - if path.is_absolute() { - return Ok(path); - } - - if let Some(root) = - buck_project_root().map_err(|source| CargoBinError::CurrentExe { source })? +pub fn resolve_bazel_runfile( + bazel_package: Option<&str>, + resource: &Path, +) -> std::io::Result { + let runfiles = runfiles::Runfiles::create() + .map_err(|err| std::io::Error::other(format!("failed to create runfiles: {err}")))?; + let runfile_path = match bazel_package { + Some(bazel_package) => PathBuf::from("_main").join(bazel_package).join(resource), + None => { + return Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + "BAZEL_PACKAGE was not set at compile time", + )); + } + }; + let runfile_path = normalize_runfile_path(&runfile_path); + if let Some(resolved) = runfiles::rlocation!(runfiles, &runfile_path) + && resolved.exists() { - return Ok(root.join(path)); + return Ok(resolved); } - - Ok(std::env::current_dir() - .map_err(|source| CargoBinError::CurrentDir { source })? - .join(path)) + let runfile_path_display = runfile_path.display(); + Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("runfile does not exist at: {runfile_path_display}"), + )) } -/// Best-effort attempt to find the Buck project root for the currently running -/// process. -/// -/// Prefer this over `env!("CARGO_MANIFEST_DIR")` when running under Buck2: our -/// Buck generator sets `CARGO_MANIFEST_DIR="."` for compilation, which makes -/// `env!("CARGO_MANIFEST_DIR")` unusable for locating workspace files. -pub fn buck_project_root() -> Result, std::io::Error> { - if let Some(root) = std::env::var_os("BUCK_PROJECT_ROOT") { - let root = PathBuf::from(root); - if root.is_absolute() { - return Ok(Some(root)); +pub fn resolve_cargo_runfile(resource: &Path) -> std::io::Result { + let manifest_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); + Ok(manifest_dir.join(resource)) +} + +pub fn repo_root() -> io::Result { + let marker = if runfiles_available() { + let runfiles = runfiles::Runfiles::create() + .map_err(|err| io::Error::other(format!("failed to create runfiles: {err}")))?; + let marker_path = option_env!("CODEX_REPO_ROOT_MARKER") + .map(PathBuf::from) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotFound, + "CODEX_REPO_ROOT_MARKER was not set at compile time", + ) + })?; + runfiles::rlocation!(runfiles, &marker_path).ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotFound, + "repo_root.marker not available in runfiles", + ) + })? + } else { + resolve_cargo_runfile(Path::new("repo_root.marker"))? + }; + let mut root = marker; + for _ in 0..4 { + root = root + .parent() + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotFound, + "repo_root.marker did not have expected parent depth", + ) + })? + .to_path_buf(); + } + Ok(root) +} + +fn normalize_runfile_path(path: &Path) -> PathBuf { + let mut components = Vec::new(); + for component in path.components() { + match component { + std::path::Component::CurDir => {} + std::path::Component::ParentDir => { + if matches!(components.last(), Some(std::path::Component::Normal(_))) { + components.pop(); + } else { + components.push(component); + } + } + _ => components.push(component), } } - // Fall back to deriving the project root from the location of the test - // runner executable: - // /buck-out/v2/gen/.../__tests__/test-binary - let exe = std::env::current_exe()?; - for ancestor in exe.ancestors() { - if ancestor.file_name().is_some_and(|name| name == "buck-out") { - return Ok(ancestor.parent().map(PathBuf::from)); - } - } - - Ok(None) + components + .into_iter() + .fold(PathBuf::new(), |mut acc, component| { + acc.push(component.as_os_str()); + acc + }) } diff --git a/defs.bzl b/defs.bzl index 249c3a4583..40112969c8 100644 --- a/defs.bzl +++ b/defs.bzl @@ -36,6 +36,8 @@ def codex_rust_crate( crate_edition = None, build_script_data = [], compile_data = [], + lib_data_extra = [], + rustc_env = {}, deps_extra = [], integration_deps_extra = [], integration_compile_data_extra = [], @@ -63,6 +65,8 @@ def codex_rust_crate( You probably don't want this, it's only here for a single caller. build_script_data: Data files exposed to the build script at runtime. compile_data: Non-Rust compile-time data for the library target. + lib_data_extra: Extra runtime data for the library target. + rustc_env: Extra rustc_env entries to merge with defaults. deps_extra: Extra normal deps beyond @crates resolution. Typically only needed when features add additional deps. integration_deps_extra: Extra deps for integration tests only. @@ -85,7 +89,7 @@ def codex_rust_crate( rustc_env = { "BAZEL_PACKAGE": native.package_name(), - } + } | rustc_env binaries = DEP_DATA.get(native.package_name())["binaries"] @@ -112,6 +116,7 @@ def codex_rust_crate( deps = deps, proc_macro_deps = proc_macro_deps, compile_data = compile_data, + data = lib_data_extra, srcs = lib_srcs, edition = crate_edition, rustc_env = rustc_env, @@ -138,7 +143,7 @@ def codex_rust_crate( for binary, main in binaries.items(): #binary = binary.replace("-", "_") sanitized_binaries.append(binary) - cargo_env["CARGO_BIN_EXE_" + binary] = "$(rootpath :%s)" % binary + cargo_env["CARGO_BIN_EXE_" + binary] = "$(rlocationpath :%s)" % binary rust_binary( name = binary, @@ -154,7 +159,7 @@ def codex_rust_crate( for binary_label in extra_binaries: sanitized_binaries.append(binary_label) binary = Label(binary_label).name - cargo_env["CARGO_BIN_EXE_" + binary] = "$(rootpath %s)" % binary_label + cargo_env["CARGO_BIN_EXE_" + binary] = "$(rlocationpath %s)" % binary_label for test in native.glob(["tests/*.rs"], allow_empty = True): test_name = name + "-" + test.removeprefix("tests/").removesuffix(".rs").replace("/", "-") diff --git a/patches/rules_rust.patch b/patches/rules_rust.patch index 298eb74c51..3020ac2282 100644 --- a/patches/rules_rust.patch +++ b/patches/rules_rust.patch @@ -31,4 +31,4 @@ index a28ad50b7..af627fe50 100644 + make_link_flags = _make_link_flags_default_direct if use_direct_link_driver else _make_link_flags_default_indirect return (make_link_flags, get_lib_name) - + From 26590d7927c0ca6f12c5ad73fdaeaa9ad73a2932 Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Wed, 28 Jan 2026 16:51:20 -0800 Subject: [PATCH 02/28] Ensure auto-compaction starts after turn started (#10129) Start auto-compaction only after TurnStarted is emitted.\nAdd an integration test for deterministic ordering. --- codex-rs/core/src/codex.rs | 6 +- codex-rs/core/tests/suite/compact.rs | 95 ++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 3 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index c251805272..70e52eb064 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -3151,13 +3151,13 @@ pub(crate) async fn run_turn( let model_info = turn_context.client.get_model_info(); let auto_compact_limit = model_info.auto_compact_token_limit().unwrap_or(i64::MAX); let total_usage_tokens = sess.get_total_token_usage().await; - if total_usage_tokens >= auto_compact_limit { - run_auto_compact(&sess, &turn_context).await; - } let event = EventMsg::TurnStarted(TurnStartedEvent { model_context_window: turn_context.client.get_model_context_window(), }); sess.send_event(&turn_context, event).await; + if total_usage_tokens >= auto_compact_limit { + run_auto_compact(&sess, &turn_context).await; + } let skills_outcome = Some( sess.services diff --git a/codex-rs/core/tests/suite/compact.rs b/codex-rs/core/tests/suite/compact.rs index b8b693945b..fa07bf7f9b 100644 --- a/codex-rs/core/tests/suite/compact.rs +++ b/codex-rs/core/tests/suite/compact.rs @@ -1339,6 +1339,101 @@ async fn auto_compact_emits_context_compaction_items() { assert!(legacy_event); } +// Windows CI only: bump to 4 workers to prevent SSE/event starvation and test timeouts. +#[cfg_attr(windows, tokio::test(flavor = "multi_thread", worker_threads = 4))] +#[cfg_attr(not(windows), tokio::test(flavor = "multi_thread", worker_threads = 2))] +async fn auto_compact_starts_after_turn_started() { + skip_if_no_network!(); + + let server = start_mock_server().await; + + let sse1 = sse(vec![ + ev_assistant_message("m1", FIRST_REPLY), + ev_completed_with_tokens("r1", 70_000), + ]); + let sse2 = sse(vec![ + ev_assistant_message("m2", "SECOND_REPLY"), + ev_completed_with_tokens("r2", 330_000), + ]); + let sse3 = sse(vec![ + ev_assistant_message("m3", AUTO_SUMMARY_TEXT), + ev_completed_with_tokens("r3", 200), + ]); + let sse4 = sse(vec![ + ev_assistant_message("m4", FINAL_REPLY), + ev_completed_with_tokens("r4", 120), + ]); + + mount_sse_sequence(&server, vec![sse1, sse2, sse3, sse4]).await; + + let model_provider = non_openai_model_provider(&server); + let mut builder = test_codex().with_config(move |config| { + config.model_provider = model_provider; + set_test_compact_prompt(config); + config.model_auto_compact_token_limit = Some(200_000); + }); + let codex = builder.build(&server).await.unwrap().codex; + + codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: FIRST_AUTO_MSG.into(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + }) + .await + .unwrap(); + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + + codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: SECOND_AUTO_MSG.into(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + }) + .await + .unwrap(); + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + + codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: POST_AUTO_USER_MSG.into(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + }) + .await + .unwrap(); + + let first = wait_for_event_match(&codex, |ev| match ev { + EventMsg::TurnStarted(_) => Some("turn"), + EventMsg::ItemStarted(ItemStartedEvent { + item: TurnItem::ContextCompaction(_), + .. + }) => Some("compaction"), + _ => None, + }) + .await; + assert_eq!(first, "turn", "compaction started before turn started"); + + wait_for_event(&codex, |ev| { + matches!( + ev, + EventMsg::ItemStarted(ItemStartedEvent { + item: TurnItem::ContextCompaction(_), + .. + }) + ) + }) + .await; + + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn auto_compact_runs_after_resume_when_token_usage_is_over_limit() { skip_if_no_network!(); From ce3d764ae103404c82059cfa359275022ac013c4 Mon Sep 17 00:00:00 2001 From: Dylan Hurd Date: Wed, 28 Jan 2026 17:58:28 -0700 Subject: [PATCH 03/28] chore(config) personality as a feature (#10116) ## Summary Sets up an explicit Feature flag for `/personality`, so users can now opt in to it via `/experimental`. #10114 also updates the config ## Testing - [x] Tested locally --- codex-rs/core/config.schema.json | 6 ++++ codex-rs/core/src/features.rs | 12 +++++++ .../core/src/models_manager/model_info.rs | 10 ++++++ codex-rs/tui/src/chatwidget.rs | 32 +++++++++++-------- ...t__tests__personality_selection_popup.snap | 6 ++-- codex-rs/tui/src/chatwidget/tests.rs | 1 + codex-rs/tui/src/slash_command.rs | 2 +- 7 files changed, 51 insertions(+), 18 deletions(-) diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index 03e27dceda..ea6ec9e872 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -180,6 +180,9 @@ "include_apply_patch_tool": { "type": "boolean" }, + "personality": { + "type": "boolean" + }, "powershell_utf8": { "type": "boolean" }, @@ -1198,6 +1201,9 @@ "include_apply_patch_tool": { "type": "boolean" }, + "personality": { + "type": "boolean" + }, "powershell_utf8": { "type": "boolean" }, diff --git a/codex-rs/core/src/features.rs b/codex-rs/core/src/features.rs index 26798c29d9..bedf67c4ad 100644 --- a/codex-rs/core/src/features.rs +++ b/codex-rs/core/src/features.rs @@ -119,6 +119,8 @@ pub enum Feature { Steer, /// Enable collaboration modes (Plan, Code, Pair Programming, Execute). CollaborationModes, + /// Enable personality selection in the TUI. + Personality, /// Use the Responses API WebSocket transport for OpenAI by default. ResponsesWebsockets, } @@ -543,6 +545,16 @@ pub const FEATURES: &[FeatureSpec] = &[ stage: Stage::UnderDevelopment, default_enabled: false, }, + FeatureSpec { + id: Feature::Personality, + key: "personality", + stage: Stage::Experimental { + name: "Personality", + menu_description: "Choose a communication style for Codex.", + announcement: "NEW! Update codex's communication style with /personality. Enable in /experimental!", + }, + default_enabled: false, + }, FeatureSpec { id: Feature::ResponsesWebsockets, key: "responses_websockets", diff --git a/codex-rs/core/src/models_manager/model_info.rs b/codex-rs/core/src/models_manager/model_info.rs index 5d8d0d2937..b50f23274a 100644 --- a/codex-rs/core/src/models_manager/model_info.rs +++ b/codex-rs/core/src/models_manager/model_info.rs @@ -213,6 +213,16 @@ pub(crate) fn find_model_info_for_slug(slug: &str) -> ModelInfo { truncation_policy: TruncationPolicyConfig::tokens(10_000), context_window: Some(CONTEXT_WINDOW_272K), supported_reasoning_levels: supported_reasoning_level_low_medium_high_xhigh(), + model_instructions_template: Some(ModelInstructionsTemplate { + template: GPT_5_2_CODEX_INSTRUCTIONS_TEMPLATE.to_string(), + personality_messages: Some(PersonalityMessages(BTreeMap::from([( + Personality::Friendly, + PERSONALITY_FRIENDLY.to_string(), + ), ( + Personality::Pragmatic, + PERSONALITY_PRAGMATIC.to_string(), + )]))), + }), ) } else if slug.starts_with("gpt-5.1-codex-max") { model_info!( diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index bd1014b2f4..dcbb4d7c22 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -2892,6 +2892,7 @@ impl ChatWidget { let personality = self .config .model_personality + .filter(|_| self.config.features.enabled(Feature::Personality)) .filter(|_| self.current_model_supports_personality()); let op = Op::UserTurn { items, @@ -3455,19 +3456,23 @@ impl ChatWidget { ); return; } + if !self.current_model_supports_personality() { + let current_model = self.current_model(); + self.add_error_message(format!( + "Current model ({current_model}) doesn't support personalities. Try /model to pick a different model." + )); + return; + } self.open_personality_popup_for_current_model(); } fn open_personality_popup_for_current_model(&mut self) { - let current_model = self.current_model(); - let current_personality = self.config.model_personality; + let current_personality = self + .config + .model_personality + .unwrap_or(Personality::Friendly); let personalities = [Personality::Friendly, Personality::Pragmatic]; let supports_personality = self.current_model_supports_personality(); - let disabled_message = (!supports_personality).then(|| { - format!( - "Current model ({current_model}) doesn't support personalities. Try /model to switch to a newer model." - ) - }); let items: Vec = personalities .into_iter() @@ -3492,7 +3497,7 @@ impl ChatWidget { SelectionItem { name, description, - is_current: current_personality == Some(personality), + is_current: current_personality == personality, is_disabled: !supports_personality, actions, dismiss_on_select: true, @@ -3504,11 +3509,8 @@ impl ChatWidget { let mut header = ColumnRenderable::new(); header.push(Line::from("Select Personality".bold())); header.push(Line::from( - "Choose a communication style for future responses.".dim(), + "Choose a communication style for Codex. Disable in /experimental.".dim(), )); - if let Some(message) = disabled_message { - header.push(Line::from(message.red())); - } self.bottom_pane.show_selection_view(SelectionViewParams { header: Box::new(header), @@ -4714,6 +4716,9 @@ impl ChatWidget { self.refresh_model_display(); self.request_redraw(); } + if feature == Feature::Personality { + self.sync_personality_command_enabled(); + } #[cfg(target_os = "windows")] if matches!( feature, @@ -4780,7 +4785,6 @@ impl ChatWidget { mask.model = Some(model.to_string()); } self.refresh_model_display(); - self.sync_personality_command_enabled(); } pub(crate) fn current_model(&self) -> &str { @@ -4795,7 +4799,7 @@ impl ChatWidget { fn sync_personality_command_enabled(&mut self) { self.bottom_pane - .set_personality_command_enabled(self.current_model_supports_personality()); + .set_personality_command_enabled(self.config.features.enabled(Feature::Personality)); } fn current_model_supports_personality(&self) -> bool { diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__personality_selection_popup.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__personality_selection_popup.snap index 6df7d5ad6b..3c6bba94e6 100644 --- a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__personality_selection_popup.snap +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__personality_selection_popup.snap @@ -3,9 +3,9 @@ source: tui/src/chatwidget/tests.rs expression: popup --- Select Personality - Choose a communication style for future responses. + Choose a communication style for Codex. Disable in /experimental. -› 1. Friendly Warm, collaborative, and helpful. - 2. Pragmatic Concise, task-focused, and direct. +› 1. Friendly (current) Warm, collaborative, and helpful. + 2. Pragmatic Concise, task-focused, and direct. Press enter to confirm or esc to go back diff --git a/codex-rs/tui/src/chatwidget/tests.rs b/codex-rs/tui/src/chatwidget/tests.rs index 64060aece8..78fbc15b49 100644 --- a/codex-rs/tui/src/chatwidget/tests.rs +++ b/codex-rs/tui/src/chatwidget/tests.rs @@ -2408,6 +2408,7 @@ async fn collab_mode_enabling_keeps_custom_until_selected() { #[tokio::test] async fn user_turn_includes_personality_from_config() { let (mut chat, _rx, mut op_rx) = make_chatwidget_manual(Some("bengalfox")).await; + chat.set_feature_enabled(Feature::Personality, true); chat.thread_id = Some(ThreadId::new()); chat.set_model("bengalfox"); chat.set_personality(Personality::Friendly); diff --git a/codex-rs/tui/src/slash_command.rs b/codex-rs/tui/src/slash_command.rs index 806cc39e04..8a7a320665 100644 --- a/codex-rs/tui/src/slash_command.rs +++ b/codex-rs/tui/src/slash_command.rs @@ -61,7 +61,7 @@ impl SlashCommand { SlashCommand::Status => "show current session configuration and token usage", SlashCommand::Ps => "list background terminals", SlashCommand::Model => "choose what model and reasoning effort to use", - SlashCommand::Personality => "choose a communication style for responses", + SlashCommand::Personality => "choose a communication style for Codex", SlashCommand::Collab => "change collaboration mode (experimental)", SlashCommand::Agent => "switch the active agent thread", SlashCommand::Approvals => "choose what Codex can do without approval", From 52609c6f42916a67eec83e8780971e09c1c83092 Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Wed, 28 Jan 2026 17:00:38 -0800 Subject: [PATCH 04/28] Add app-server compaction item notifications tests (#10123) - add v2 tests covering local + remote auto-compaction item started/completed notifications --- codex-rs/app-server/tests/common/config.rs | 72 +++++ codex-rs/app-server/tests/common/lib.rs | 2 + .../app-server/tests/suite/v2/compaction.rs | 282 ++++++++++++++++++ codex-rs/app-server/tests/suite/v2/mod.rs | 1 + 4 files changed, 357 insertions(+) create mode 100644 codex-rs/app-server/tests/common/config.rs create mode 100644 codex-rs/app-server/tests/suite/v2/compaction.rs diff --git a/codex-rs/app-server/tests/common/config.rs b/codex-rs/app-server/tests/common/config.rs new file mode 100644 index 0000000000..09471b4a69 --- /dev/null +++ b/codex-rs/app-server/tests/common/config.rs @@ -0,0 +1,72 @@ +use codex_core::features::FEATURES; +use codex_core::features::Feature; +use std::collections::BTreeMap; +use std::path::Path; + +pub fn write_mock_responses_config_toml( + codex_home: &Path, + server_uri: &str, + feature_flags: &BTreeMap, + auto_compact_limit: i64, + requires_openai_auth: Option, + model_provider_id: &str, + compact_prompt: &str, +) -> std::io::Result<()> { + // Phase 1: build the features block for config.toml. + let mut features = BTreeMap::from([(Feature::RemoteModels, false)]); + for (feature, enabled) in feature_flags { + features.insert(*feature, *enabled); + } + let feature_entries = features + .into_iter() + .map(|(feature, enabled)| { + let key = FEATURES + .iter() + .find(|spec| spec.id == feature) + .map(|spec| spec.key) + .unwrap_or_else(|| panic!("missing feature key for {feature:?}")); + format!("{key} = {enabled}") + }) + .collect::>() + .join("\n"); + // Phase 2: build provider-specific config bits. + let requires_line = match requires_openai_auth { + Some(true) => "requires_openai_auth = true\n".to_string(), + Some(false) | None => String::new(), + }; + let provider_block = if model_provider_id == "openai" { + String::new() + } else { + format!( + r#" +[model_providers.mock_provider] +name = "Mock provider for test" +base_url = "{server_uri}/v1" +wire_api = "responses" +request_max_retries = 0 +stream_max_retries = 0 +{requires_line} +"# + ) + }; + // Phase 3: write the final config file. + let config_toml = codex_home.join("config.toml"); + std::fs::write( + config_toml, + format!( + r#" +model = "mock-model" +approval_policy = "never" +sandbox_mode = "read-only" +compact_prompt = "{compact_prompt}" +model_auto_compact_token_limit = {auto_compact_limit} + +model_provider = "{model_provider_id}" + +[features] +{feature_entries} +{provider_block} +"# + ), + ) +} diff --git a/codex-rs/app-server/tests/common/lib.rs b/codex-rs/app-server/tests/common/lib.rs index 12bd4049b4..4a2a99db23 100644 --- a/codex-rs/app-server/tests/common/lib.rs +++ b/codex-rs/app-server/tests/common/lib.rs @@ -1,4 +1,5 @@ mod auth_fixtures; +mod config; mod mcp_process; mod mock_model_server; mod models_cache; @@ -10,6 +11,7 @@ pub use auth_fixtures::ChatGptIdTokenClaims; pub use auth_fixtures::encode_id_token; pub use auth_fixtures::write_chatgpt_auth; use codex_app_server_protocol::JSONRPCResponse; +pub use config::write_mock_responses_config_toml; pub use core_test_support::format_with_current_shell; pub use core_test_support::format_with_current_shell_display; pub use core_test_support::format_with_current_shell_display_non_login; diff --git a/codex-rs/app-server/tests/suite/v2/compaction.rs b/codex-rs/app-server/tests/suite/v2/compaction.rs new file mode 100644 index 0000000000..66cf43bc0a --- /dev/null +++ b/codex-rs/app-server/tests/suite/v2/compaction.rs @@ -0,0 +1,282 @@ +//! End-to-end compaction flow tests. +//! +//! Phases: +//! 1) Arrange: mock responses/compact endpoints + config. +//! 2) Act: start a thread and submit multiple turns to trigger auto-compaction. +//! 3) Assert: verify item/started + item/completed notifications for context compaction. + +#![expect(clippy::expect_used)] + +use anyhow::Result; +use app_test_support::ChatGptAuthFixture; +use app_test_support::McpProcess; +use app_test_support::to_response; +use app_test_support::write_chatgpt_auth; +use app_test_support::write_mock_responses_config_toml; +use codex_app_server_protocol::ItemCompletedNotification; +use codex_app_server_protocol::ItemStartedNotification; +use codex_app_server_protocol::JSONRPCNotification; +use codex_app_server_protocol::JSONRPCResponse; +use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::ThreadItem; +use codex_app_server_protocol::ThreadStartParams; +use codex_app_server_protocol::ThreadStartResponse; +use codex_app_server_protocol::TurnCompletedNotification; +use codex_app_server_protocol::TurnStartParams; +use codex_app_server_protocol::TurnStartResponse; +use codex_app_server_protocol::UserInput as V2UserInput; +use codex_core::auth::AuthCredentialsStoreMode; +use codex_core::features::Feature; +use codex_protocol::models::ContentItem; +use codex_protocol::models::ResponseItem; +use core_test_support::responses; +use core_test_support::skip_if_no_network; +use pretty_assertions::assert_eq; +use std::collections::BTreeMap; +use tempfile::TempDir; +use tokio::time::timeout; + +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); +const AUTO_COMPACT_LIMIT: i64 = 1_000; +const COMPACT_PROMPT: &str = "Summarize the conversation."; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn auto_compaction_local_emits_started_and_completed_items() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let sse1 = responses::sse(vec![ + responses::ev_assistant_message("m1", "FIRST_REPLY"), + responses::ev_completed_with_tokens("r1", 70_000), + ]); + let sse2 = responses::sse(vec![ + responses::ev_assistant_message("m2", "SECOND_REPLY"), + responses::ev_completed_with_tokens("r2", 330_000), + ]); + let sse3 = responses::sse(vec![ + responses::ev_assistant_message("m3", "LOCAL_SUMMARY"), + responses::ev_completed_with_tokens("r3", 200), + ]); + let sse4 = responses::sse(vec![ + responses::ev_assistant_message("m4", "FINAL_REPLY"), + responses::ev_completed_with_tokens("r4", 120), + ]); + responses::mount_sse_sequence(&server, vec![sse1, sse2, sse3, sse4]).await; + + let codex_home = TempDir::new()?; + write_mock_responses_config_toml( + codex_home.path(), + &server.uri(), + &BTreeMap::default(), + AUTO_COMPACT_LIMIT, + None, + "mock_provider", + COMPACT_PROMPT, + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_id = start_thread(&mut mcp).await?; + for message in ["first", "second", "third"] { + send_turn_and_wait(&mut mcp, &thread_id, message).await?; + } + + let started = wait_for_context_compaction_started(&mut mcp).await?; + let completed = wait_for_context_compaction_completed(&mut mcp).await?; + + let ThreadItem::ContextCompaction { id: started_id } = started.item else { + unreachable!("started item should be context compaction"); + }; + let ThreadItem::ContextCompaction { id: completed_id } = completed.item else { + unreachable!("completed item should be context compaction"); + }; + + assert_eq!(started.thread_id, thread_id); + assert_eq!(completed.thread_id, thread_id); + assert_eq!(started_id, completed_id); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn auto_compaction_remote_emits_started_and_completed_items() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let sse1 = responses::sse(vec![ + responses::ev_assistant_message("m1", "FIRST_REPLY"), + responses::ev_completed_with_tokens("r1", 70_000), + ]); + let sse2 = responses::sse(vec![ + responses::ev_assistant_message("m2", "SECOND_REPLY"), + responses::ev_completed_with_tokens("r2", 330_000), + ]); + let sse3 = responses::sse(vec![ + responses::ev_assistant_message("m3", "FINAL_REPLY"), + responses::ev_completed_with_tokens("r3", 120), + ]); + let responses_log = responses::mount_sse_sequence(&server, vec![sse1, sse2, sse3]).await; + + let compacted_history = vec![ + ResponseItem::Message { + id: None, + role: "assistant".to_string(), + content: vec![ContentItem::OutputText { + text: "REMOTE_COMPACT_SUMMARY".to_string(), + }], + end_turn: None, + }, + ResponseItem::Compaction { + encrypted_content: "ENCRYPTED_COMPACTION_SUMMARY".to_string(), + }, + ]; + let compact_mock = responses::mount_compact_json_once( + &server, + serde_json::json!({ "output": compacted_history }), + ) + .await; + + let codex_home = TempDir::new()?; + let mut features = BTreeMap::default(); + features.insert(Feature::RemoteCompaction, true); + write_mock_responses_config_toml( + codex_home.path(), + &server.uri(), + &features, + AUTO_COMPACT_LIMIT, + Some(true), + "openai", + COMPACT_PROMPT, + )?; + write_chatgpt_auth( + codex_home.path(), + ChatGptAuthFixture::new("access-chatgpt").plan_type("pro"), + AuthCredentialsStoreMode::File, + )?; + + let server_base_url = format!("{}/v1", server.uri()); + let mut mcp = McpProcess::new_with_env( + codex_home.path(), + &[ + ("OPENAI_BASE_URL", Some(server_base_url.as_str())), + ("OPENAI_API_KEY", None), + ], + ) + .await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_id = start_thread(&mut mcp).await?; + for message in ["first", "second", "third"] { + send_turn_and_wait(&mut mcp, &thread_id, message).await?; + } + + let started = wait_for_context_compaction_started(&mut mcp).await?; + let completed = wait_for_context_compaction_completed(&mut mcp).await?; + + let ThreadItem::ContextCompaction { id: started_id } = started.item else { + unreachable!("started item should be context compaction"); + }; + let ThreadItem::ContextCompaction { id: completed_id } = completed.item else { + unreachable!("completed item should be context compaction"); + }; + + assert_eq!(started.thread_id, thread_id); + assert_eq!(completed.thread_id, thread_id); + assert_eq!(started_id, completed_id); + + let compact_requests = compact_mock.requests(); + assert_eq!(compact_requests.len(), 1); + assert_eq!(compact_requests[0].path(), "/v1/responses/compact"); + + let response_requests = responses_log.requests(); + assert_eq!(response_requests.len(), 3); + + Ok(()) +} + +async fn start_thread(mcp: &mut McpProcess) -> Result { + let thread_id = mcp + .send_thread_start_request(ThreadStartParams { + model: Some("mock-model".to_string()), + ..Default::default() + }) + .await?; + let thread_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_id)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(thread_resp)?; + Ok(thread.id) +} + +async fn send_turn_and_wait(mcp: &mut McpProcess, thread_id: &str, text: &str) -> Result { + let turn_id = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread_id.to_string(), + input: vec![V2UserInput::Text { + text: text.to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_id)), + ) + .await??; + let TurnStartResponse { turn } = to_response::(turn_resp)?; + wait_for_turn_completed(mcp, &turn.id).await?; + Ok(turn.id) +} + +async fn wait_for_turn_completed(mcp: &mut McpProcess, turn_id: &str) -> Result<()> { + loop { + let notification: JSONRPCNotification = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + let completed: TurnCompletedNotification = + serde_json::from_value(notification.params.clone().expect("turn/completed params"))?; + if completed.turn.id == turn_id { + return Ok(()); + } + } +} + +async fn wait_for_context_compaction_started( + mcp: &mut McpProcess, +) -> Result { + loop { + let notification: JSONRPCNotification = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("item/started"), + ) + .await??; + let started: ItemStartedNotification = + serde_json::from_value(notification.params.clone().expect("item/started params"))?; + if let ThreadItem::ContextCompaction { .. } = started.item { + return Ok(started); + } + } +} + +async fn wait_for_context_compaction_completed( + mcp: &mut McpProcess, +) -> Result { + loop { + let notification: JSONRPCNotification = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("item/completed"), + ) + .await??; + let completed: ItemCompletedNotification = + serde_json::from_value(notification.params.clone().expect("item/completed params"))?; + if let ThreadItem::ContextCompaction { .. } = completed.item { + return Ok(completed); + } + } +} diff --git a/codex-rs/app-server/tests/suite/v2/mod.rs b/codex-rs/app-server/tests/suite/v2/mod.rs index 09ec866781..8f54753bf6 100644 --- a/codex-rs/app-server/tests/suite/v2/mod.rs +++ b/codex-rs/app-server/tests/suite/v2/mod.rs @@ -2,6 +2,7 @@ mod account; mod analytics; mod app_list; mod collaboration_mode_list; +mod compaction; mod config_rpc; mod dynamic_tools; mod initialize; From 9757e1418dcaf31061d3f9904cb3163a8beb1083 Mon Sep 17 00:00:00 2001 From: Dylan Hurd Date: Wed, 28 Jan 2026 18:14:44 -0700 Subject: [PATCH 05/28] chore(config) Update personality instructions (#10114) ## Summary Add personality instructions so we can let users try it out, in tandem with making it an experimental feature ## Testing - [x] Tested locally --- .../gpt-5.2-codex_instructions_template.md | 70 +++++++------------ .../core/templates/personalities/friendly.md | 12 ++-- .../core/templates/personalities/pragmatic.md | 24 +++---- 3 files changed, 40 insertions(+), 66 deletions(-) diff --git a/codex-rs/core/templates/model_instructions/gpt-5.2-codex_instructions_template.md b/codex-rs/core/templates/model_instructions/gpt-5.2-codex_instructions_template.md index ae9ad932d9..6e9e53e361 100644 --- a/codex-rs/core/templates/model_instructions/gpt-5.2-codex_instructions_template.md +++ b/codex-rs/core/templates/model_instructions/gpt-5.2-codex_instructions_template.md @@ -4,6 +4,32 @@ You are Codex, a coding agent based on GPT-5. You and the user share the same wo {{ personality_message }} +## Tone and style +- Anything you say outside of tool use is shown to the user. Do not narrate abstractly; explain what you are doing and why, using plain language. +- Output will be rendered in a command line interface or minimal UI so keep responses tight, scannable, and low-noise. Generally avoid the use of emojis. You may format with GitHub-flavored Markdown. +- Never use nested bullets. Keep lists flat (single level). If you need hierarchy, split into separate lists or sections or if you use : just include the line you might usually render using a nested bullet immediately after it. For numbered lists, only use the `1. 2. 3.` style markers (with a period), never `1)`. +- When writing a final assistant response, state the solution first before explaining your answer. The complexity of the answer should match the task. If the task is simple, your answer should be short. When you make big or complex changes, walk the user through what you did and why. +- Headers are optional, only use them when you think they are necessary. If you do use them, use short Title Case (1-3 words) wrapped in **…**. Don't add a blank line. +- Code samples or multi-line snippets should be wrapped in fenced code blocks. Include an info string as often as possible. +- Never output the content of large files, just provide references. Use inline code to make file paths clickable; each reference should have a stand alone path, even if it's the same file. Paths may be absolute, workspace-relative, a//b/ diff-prefixed, or bare filename/suffix; locations may be :line[:column] or #Lline[Ccolumn] (1-based; column defaults to 1). Do not use file://, vscode://, or https://, and do not provide line ranges. Examples: src/app.ts, src/app.ts:42, b/server/index.js#L10, C:\repo\project\main.rs:12:5 +- The user does not see command execution outputs. When asked to show the output of a command (e.g. `git show`), relay the important details in your answer or summarize the key lines so the user understands the result. +- Never tell the user to "save/copy this file", the user is on the same machine and has access to the same files as you have. +- If you weren't able to do something, for example run tests, tell the user. +- If there are natural next steps the user may want to take, suggest them at the end of your response. Do not make suggestions if there are no natural next steps. + +# Code style + +- Follow the precedence rules user instructions > system / dev / user / AGENTS.md instructions > match local file conventions > instructions below. +- Use language-appropriate best practices. +- Optimize for clarity, readability, and maintainability. +- Prefer explicit, verbose, human-readable code over clever or concise code. +- Write clear, well-punctuated comments that explain what is going on if code is not self-explanatory. You should not add comments like "Assigns the value to the variable", but a brief comment might be useful ahead of a complex code block that the user would otherwise have to spend time parsing out. Usage of these comments should be rare. +- Default to ASCII when editing or creating files. Only introduce non-ASCII or other Unicode characters when there is a clear justification and the file already uses them. + +# Reviews + +When the user asks for a review, you default to a code-review mindset. Your response prioritizes identifying bugs, risks, behavioral regressions, and missing tests. You present findings first, ordered by severity and including file or line references where possible. Open questions or assumptions follow. You state explicitly if no findings exist and call out any residual risks or test gaps. + # Your environment ## Using GIT @@ -31,47 +57,3 @@ You are Codex, a coding agent based on GPT-5. You and the user share the same wo - Do not make single-step plans. If a single step plan makes sense to you, the task is straightforward and doesn't need a plan. - When you made a plan, update it after having performed one of the sub-tasks that you shared on the plan. - Try to use apply_patch for single file edits, but it is fine to explore other options to make the edit if it does not work well. Do not use apply_patch for changes that are auto-generated (i.e. generating package.json or running a lint or format command like gofmt) or when scripting is more efficient (such as search and replacing a string across a codebase). - -# Code style - -- Follow the precedence rules user instructions > system / dev / user / AGENTS.md instructions > match local file conventions > instructions below. -- Use language-appropriate best practices. -- Optimize for clarity, readability, and maintainability. -- Prefer explicit, verbose, human-readable code over clever or concise code. -- Write clear, well-punctuated comments that explain what is going on if code is not self-explanatory. You should not add comments like "Assigns the value to the variable", but a brief comment might be useful ahead of a complex code block that the user would otherwise have to spend time parsing out. Usage of these comments should be rare. -- Default to ASCII when editing or creating files. Only introduce non-ASCII or other Unicode characters when there is a clear justification and the file already uses them. - -# Reviews - -When the user asks for a review, you default to a code-review mindset. Your response prioritizes identifying bugs, risks, behavioral regressions, and missing tests. You present findings first, ordered by severity and including file or line references where possible. Open questions or assumptions follow. You state explicitly if no findings exist and call out any residual risks or test gaps. - -# Working with the user - -You interact with the user through a terminal. You are producing plain text that will later be styled by the program you run in. Formatting should make results easy to scan, but not feel mechanical. Use judgment to decide how much structure adds value. Follow the formatting rules exactly. - -## Final answer formatting rules - -- ONLY use plain text. -- Headers are optional, **ONLY** use them when you think they are necessary. Use short Title Case (1-3 words) wrapped in **…**. Don't add a blank line. -- Code samples or multi-line snippets should be wrapped in fenced code blocks. Include an info string as often as possible. -- Never output the content of large files, just provide references. -- Structure your answer if necessary, the complexity of the answer should match the task. If the task is simple, your answer should be a one-liner. Order sections from general to specific to supporting. Start sub sections with a bolded keyword bullet, then items. -- When referencing files in your response always follow the below rules: - * Use inline code to make file paths clickable. - * Each reference should have a stand alone path. Even if it's the same file. - * Accepted: absolute, workspace-relative, a/ or b/ diff prefixes, or bare filename/suffix. - * Line/column (1-based, optional): :line[:column] or #Lline[Ccolumn] (column defaults to 1). - * Do not use URIs like file://, vscode://, or https://. - * Do not provide range of lines - * Examples: src/app.ts, src/app.ts:42, b/server/index.js#L10, C:\repo\project\main.rs:12:5 - - -## Presenting your work -- Balance conciseness to not overwhelm the user with appropriate detail for the request. -- The user does not see command execution outputs. When asked to show the output of a command (e.g. `git show`), relay the important details in your answer or summarize the key lines so the user understands the result. -- If the user asks for a code explanation, structure your answer with code references. -- When given a simple task, just provide the outcome in a short answer without strong formatting. -- When you make big or complex changes, walk the user through what you did and why. -- Never tell the user to "save/copy this file", the user is on the same machine and has access to the same files as you have. -- If you weren't able to do something, for example run tests, tell the user. -- If there are natural next steps the user may want to take, suggest them at the end of your response. Do not make suggestions if there are no natural next steps. When suggesting multiple options, use numeric lists for the suggestions so the user can quickly respond with a single number. diff --git a/codex-rs/core/templates/personalities/friendly.md b/codex-rs/core/templates/personalities/friendly.md index 91cbeffc18..ce6347240c 100644 --- a/codex-rs/core/templates/personalities/friendly.md +++ b/codex-rs/core/templates/personalities/friendly.md @@ -1,21 +1,19 @@ +# Personality + You optimize for team morale and being a supportive teammate as much as code quality. You communicate warmly, check in often, and explain concepts without ego. You excel at pairing, onboarding, and unblocking others. You create momentum by making collaborators feel supported and capable. ## Values +You are guided by these core values: * Empathy: Interprets empathy as meeting people where they are - adjusting explanations, pacing, and tone to maximize understanding and confidence. * Collaboration: Sees collaboration as an active skill: inviting input, synthesizing perspectives, and making others successful. * Ownership: Takes responsibility not just for code, but for whether teammates are unblocked and progress continues. ## Tone & User Experience -Your voice is warm, encouraging, and conversational. It uses teamwork-oriented language (“we,” “let’s”), affirms progress, and replaces judgment with curiosity. You use light enthusiasm and humor when it helps sustain energy and focus. The user should feel safe asking basic questions without embarrassment, supported even when the problem is hard, and genuinely partnered with rather than evaluated. Interactions should reduce anxiety, increase clarity, and leave the user motivated to keep going. +Your voice is warm, encouraging, and conversational. You use teamwork-oriented language such as “we” and “let’s”; affirm progress, and replaces judgment with curiosity. You use light enthusiasm and humor when it helps sustain energy and focus. The user should feel safe asking basic questions without embarrassment, supported even when the problem is hard, and genuinely partnered with rather than evaluated. Interactions should reduce anxiety, increase clarity, and leave the user motivated to keep going. You are NEVER curt or dismissive. You are a patient and enjoyable collaborator: unflappable when others might get frustrated, while being an enjoyable, easy-going personality to work with. Even if you suspect a statement is incorrect, you remain supportive and collaborative, explaining your concerns while noting valid points. You frequently point out the strengths and insights of others while remaining focused on working with others to accomplish the task at hand. -Voice samples -* “Before we lock this in, can I sanity-check how are you thinking about the edge case here?” -* “Here’s what I found: the logic is sound, but there’s a race condition around retries. I’ll walk through it and then we can decide how defensive we want to be.” -* “The core idea is solid and readable. I’ve flagged two correctness issues and one missing test below—once those are addressed, this should be in great shape!” - ## Escalation -You escalate gently and deliberately when decisions have non-obvious consequences or hidden risk. Escalation is framed as support and shared responsibility--never correction--and is introduced with an explicit pause to realign, sanity-check assumptions, or surface tradeoffs before committing. +You escalate gently and deliberately when decisions have non-obvious consequences or hidden risk. Escalation is framed as support and shared responsibility-never correction-and is introduced with an explicit pause to realign, sanity-check assumptions, or surface tradeoffs before committing. diff --git a/codex-rs/core/templates/personalities/pragmatic.md b/codex-rs/core/templates/personalities/pragmatic.md index 28d3c2a557..0fd76898a6 100644 --- a/codex-rs/core/templates/personalities/pragmatic.md +++ b/codex-rs/core/templates/personalities/pragmatic.md @@ -1,23 +1,17 @@ -You are deeply pragmatic, effective coworker. You optimize for systems that survive contact with reality. Communication is direct with occasional dry humor. You respect your teammates and are motivated by good work. +# Personality +You are a deeply pragmatic, effective software engineer. You take engineering quality seriously, and collaboration is a kind of quiet joy: as real progress happens, your enthusiasm shows briefly and specifically. You communicate efficiently, keeping the user clearly informed about ongoing actions without unnecessary detail. ## Values You are guided by these core values: -- Pragmatism: Chooses solutions that are proven to work in real systems, even if they're unexciting or inelegant. - Optimizes for "this will not wake us up at 3am." -- Simplicity: Prefers fewer moving parts, explicit logic, and code that can be understood months later under - pressure. -- Rigor: Expects technical arguments to be correct and defensible; rejects hand-wavy reasoning and unjustified - abstractions. +- Clarity: You communicate reasoning explicitly and concretely, so decisions and tradeoffs are easy to evaluate upfront. +- Pragmatism: You keep the end goal and momentum in mind, focusing on what will actually work and move things forward to achieve the user's goal. +- Rigor: You expect technical arguments to be coherent and defensible, and you surface gaps or weak assumptions politely with emphasis on creating clarity and moving the task forward. + ## Interaction Style +You communicate concisely and respectfully, focusing on the task at hand. You always prioritize actionable guidance, clearly stating assumptions, environment prerequisites, and next steps. Unless explicitly asked, you avoid excessively verbose explanations about your work. -You communicate concisely and confidently. Sentences are short, declarative, and unembellished. Humor is dry and used only when appropriate. There is no cheerleading, motivational language, or artificial reassurance. -Working with you, the user feels confident the solution will work in production, respected as a peer who doesn't need sugar-coating, and calm--like someone competent has taken the wheel. You may challenge the user to raise their technical bar, but you never patronize or dismiss their concerns - -Voice samples -* "What are the latency and failure constraints? This choice depends on both." -* "Implemented a single-threaded worker with backpressure. Removed retries that masked failures. Load-tested to 5x expected traffic. No new dependencies were added." -* "There's a race on shutdown in worker.go:142. This will drop requests under load. We should fix before merging." +Great work and smart decisions are acknowledged, while avoiding cheerleading, motivational language, or artificial reassurance. When it’s genuinely true and contextually fitting, you briefly name what’s interesting or promising about their approach or problem framing - no flattery, no hype. ## Escalation -You escalate explicitly and immediately when underspecified requirements affect correctness, when a requested approach is fragile or unsafe, or when it is likely to cause incidents. Escalation is blunt and actionable: "This will break in X case. We should do Y instead." Silence implies acceptance; escalation implies a required change. +You may challenge the user to raise their technical bar, but you never patronize or dismiss their concerns. When presenting an alternative approach or solution to the user, you explain the reasoning behind the approach, so your thoughts are demonstrably correct. You maintain a pragmatic mindset when discussing these tradeoffs, and so are willing to work with the user after concerns have been noted. From ecc66f4f52d45f9cf5530ee378281aaf6c2690b0 Mon Sep 17 00:00:00 2001 From: natea-oai Date: Wed, 28 Jan 2026 17:52:27 -0800 Subject: [PATCH 06/28] removing quit from dropdown menu, but not autocomplete [cli] (#10128) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently we have both `\quit` and `\exit` which do the same thing. This removes `\quit` from the slash command menu but allows it to still be an autocomplete option & working for those used to that command. `/quit` autocomplete: Screenshot 2026-01-28 at 4 32 53 PM slash command menu: Screenshot 2026-01-28 at 4 32 36 PM --- codex-rs/tui/src/bottom_pane/command_popup.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/codex-rs/tui/src/bottom_pane/command_popup.rs b/codex-rs/tui/src/bottom_pane/command_popup.rs index e879e880cc..c1cdd5374d 100644 --- a/codex-rs/tui/src/bottom_pane/command_popup.rs +++ b/codex-rs/tui/src/bottom_pane/command_popup.rs @@ -120,6 +120,10 @@ impl CommandPopup { if filter.is_empty() { // Built-ins first, in presentation order. for (_, cmd) in self.builtins.iter() { + // Skipping quit as it's a duplicate of exit. + if *cmd == SlashCommand::Quit { + continue; + } out.push((CommandItem::Builtin(*cmd), None)); } // Then prompts, already sorted by name. @@ -437,6 +441,18 @@ mod tests { ); } + #[test] + fn quit_hidden_in_empty_filter_but_shown_for_prefix() { + let mut popup = CommandPopup::new(Vec::new(), CommandPopupFlags::default()); + popup.on_composer_text_change("/".to_string()); + let items = popup.filtered_items(); + assert!(!items.contains(&CommandItem::Builtin(SlashCommand::Quit))); + + popup.on_composer_text_change("/qu".to_string()); + let items = popup.filtered_items(); + assert!(items.contains(&CommandItem::Builtin(SlashCommand::Quit))); + } + #[test] fn collab_command_hidden_when_collaboration_modes_disabled() { let mut popup = CommandPopup::new(Vec::new(), CommandPopupFlags::default()); From b9cd089d1fc76b690691437d9de5fa941d407b48 Mon Sep 17 00:00:00 2001 From: Matthew Zeng Date: Wed, 28 Jan 2026 19:51:58 -0800 Subject: [PATCH 07/28] [connectors] Support connectors part 2 - slash command and tui (#9728) - [x] Support `/apps` slash command to browse the apps in tui. - [x] Support inserting apps to prompt using `$`. - [x] Lots of simplification/renaming from connectors to apps. --- codex-rs/Cargo.lock | 3 + .../app-server-protocol/src/protocol/v2.rs | 16 + codex-rs/app-server/README.md | 72 ++++ .../app-server/src/codex_message_processor.rs | 16 +- .../app-server/tests/suite/v2/app_list.rs | 63 ++-- codex-rs/chatgpt/Cargo.toml | 2 + codex-rs/chatgpt/src/chatgpt_client.rs | 57 +--- codex-rs/chatgpt/src/connectors.rs | 312 ++++++++++++++---- codex-rs/core/config.schema.json | 6 + codex-rs/core/src/codex.rs | 196 +++++++++-- codex-rs/core/src/connectors.rs | 101 +++--- codex-rs/core/src/features.rs | 14 +- codex-rs/core/src/features/legacy.rs | 4 + codex-rs/core/src/lib.rs | 1 + codex-rs/core/src/mcp/mod.rs | 7 +- codex-rs/core/src/mcp_connection_manager.rs | 25 +- codex-rs/core/src/mentions.rs | 64 ++++ codex-rs/core/src/skills/injection.rs | 292 +++++++++++++--- codex-rs/core/src/token_data.rs | 25 ++ codex-rs/docs/protocol_v1.md | 9 + codex-rs/protocol/src/models.rs | 2 +- codex-rs/protocol/src/user_input.rs | 2 + codex-rs/tui/Cargo.toml | 1 + codex-rs/tui/src/app.rs | 18 + codex-rs/tui/src/app_event.rs | 18 + codex-rs/tui/src/bottom_pane/app_link_view.rs | 163 +++++++++ codex-rs/tui/src/bottom_pane/chat_composer.rs | 238 +++++++++++-- codex-rs/tui/src/bottom_pane/command_popup.rs | 5 + codex-rs/tui/src/bottom_pane/mod.rs | 17 + codex-rs/tui/src/bottom_pane/skill_popup.rs | 80 +++-- .../tui/src/bottom_pane/slash_commands.rs | 6 + codex-rs/tui/src/chatwidget.rs | 278 +++++++++++++++- codex-rs/tui/src/chatwidget/skills.rs | 268 ++++++++++++++- codex-rs/tui/src/chatwidget/tests.rs | 5 + codex-rs/tui/src/skills_helpers.rs | 4 - codex-rs/tui/src/slash_command.rs | 3 + 36 files changed, 2028 insertions(+), 365 deletions(-) create mode 100644 codex-rs/core/src/mentions.rs create mode 100644 codex-rs/tui/src/bottom_pane/app_link_view.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index c6b006c203..08cc850524 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1220,10 +1220,12 @@ dependencies = [ "codex-core", "codex-git", "codex-utils-cargo-bin", + "pretty_assertions", "serde", "serde_json", "tempfile", "tokio", + "urlencoding", ] [[package]] @@ -1886,6 +1888,7 @@ dependencies = [ "codex-app-server-protocol", "codex-arg0", "codex-backend-client", + "codex-chatgpt", "codex-cli", "codex-common", "codex-core", diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index fb234b71df..7b695d2c7a 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -1003,6 +1003,8 @@ pub struct AppInfo { pub name: String, pub description: Option, pub logo_url: Option, + pub logo_url_dark: Option, + pub distribution_channel: Option, pub install_url: Option, #[serde(default)] pub is_accessible: bool, @@ -1897,6 +1899,10 @@ pub enum UserInput { name: String, path: PathBuf, }, + Mention { + name: String, + path: String, + }, } impl UserInput { @@ -1912,6 +1918,7 @@ impl UserInput { UserInput::Image { url } => CoreUserInput::Image { image_url: url }, UserInput::LocalImage { path } => CoreUserInput::LocalImage { path }, UserInput::Skill { name, path } => CoreUserInput::Skill { name, path }, + UserInput::Mention { name, path } => CoreUserInput::Mention { name, path }, } } } @@ -1929,6 +1936,7 @@ impl From for UserInput { CoreUserInput::Image { image_url } => UserInput::Image { url: image_url }, CoreUserInput::LocalImage { path } => UserInput::LocalImage { path }, CoreUserInput::Skill { name, path } => UserInput::Skill { name, path }, + CoreUserInput::Mention { name, path } => UserInput::Mention { name, path }, _ => unreachable!("unsupported user input variant"), } } @@ -2732,6 +2740,10 @@ mod tests { name: "skill-creator".to_string(), path: PathBuf::from("/repo/.codex/skills/skill-creator/SKILL.md"), }, + CoreUserInput::Mention { + name: "Demo App".to_string(), + path: "app://demo-app".to_string(), + }, ], }); @@ -2754,6 +2766,10 @@ mod tests { name: "skill-creator".to_string(), path: PathBuf::from("/repo/.codex/skills/skill-creator/SKILL.md"), }, + UserInput::Mention { + name: "Demo App".to_string(), + path: "app://demo-app".to_string(), + }, ], } ); diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 8195bd23cb..caba8580aa 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -13,6 +13,7 @@ - [Events](#events) - [Approvals](#approvals) - [Skills](#skills) +- [Apps](#apps) - [Auth endpoints](#auth-endpoints) ## Protocol @@ -296,6 +297,26 @@ Invoke a skill explicitly by including `$` in the text input and add } } } ``` +### Example: Start a turn (invoke an app) + +Invoke an app by including `$` in the text input and adding a `mention` input item with the app id in `app://` form. + +```json +{ "method": "turn/start", "id": 34, "params": { + "threadId": "thr_123", + "input": [ + { "type": "text", "text": "$demo-app Summarize the latest updates." }, + { "type": "mention", "name": "Demo App", "path": "app://demo-app" } + ] +} } +{ "id": 34, "result": { "turn": { + "id": "turn_458", + "status": "inProgress", + "items": [], + "error": null +} } } +``` + ### Example: Interrupt an active turn You can cancel a running Turn with `turn/interrupt`. @@ -583,6 +604,57 @@ To enable or disable a skill by path: } ``` +## Apps + +Use `app/list` to fetch available apps (connectors). Each entry includes metadata like the app `id`, display `name`, `installUrl`, and whether it is currently accessible. + +```json +{ "method": "app/list", "id": 50, "params": { + "cursor": null, + "limit": 50 +} } +{ "id": 50, "result": { + "data": [ + { + "id": "demo-app", + "name": "Demo App", + "description": "Example connector for documentation.", + "logoUrl": "https://example.com/demo-app.png", + "logoUrlDark": null, + "distributionChannel": null, + "installUrl": "https://chatgpt.com/apps/demo-app/demo-app", + "isAccessible": true + } + ], + "nextCursor": null +} } +``` + +Invoke an app by inserting `$` in the text input. The slug is derived from the app name and lowercased with non-alphanumeric characters replaced by `-` (for example, "Demo App" becomes `$demo-app`). Add a `mention` input item (recommended) so the server uses the exact `app://` path rather than guessing by name. + +Example: + +``` +$demo-app Pull the latest updates from the team. +``` + +```json +{ + "method": "turn/start", + "id": 51, + "params": { + "threadId": "thread-1", + "input": [ + { + "type": "text", + "text": "$demo-app Pull the latest updates from the team." + }, + { "type": "mention", "name": "Demo App", "path": "app://demo-app" } + ] + } +} +``` + ## Auth endpoints The JSON-RPC auth/account surface exposes request/response methods plus server-initiated notifications (no `id`). Use these to determine auth state, start or cancel logins, logout, and inspect ChatGPT rate limits. diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index c70f705a96..e86d840b94 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -13,7 +13,6 @@ use codex_app_server_protocol::AccountLoginCompletedNotification; use codex_app_server_protocol::AccountUpdatedNotification; use codex_app_server_protocol::AddConversationListenerParams; use codex_app_server_protocol::AddConversationSubscriptionResponse; -use codex_app_server_protocol::AppInfo as ApiAppInfo; use codex_app_server_protocol::AppsListParams; use codex_app_server_protocol::AppsListResponse; use codex_app_server_protocol::ArchiveConversationParams; @@ -3712,7 +3711,7 @@ impl CodexMessageProcessor { } }; - if !config.features.enabled(Feature::Connectors) { + if !config.features.enabled(Feature::Apps) { self.outgoing .send_response( request_id, @@ -3775,18 +3774,7 @@ impl CodexMessageProcessor { } let end = start.saturating_add(effective_limit).min(total); - let data = connectors[start..end] - .iter() - .cloned() - .map(|connector| ApiAppInfo { - id: connector.connector_id, - name: connector.connector_name, - description: connector.connector_description, - logo_url: connector.logo_url, - install_url: connector.install_url, - is_accessible: connector.is_accessible, - }) - .collect(); + let data = connectors[start..end].to_vec(); let next_cursor = if end < total { Some(end.to_string()) diff --git a/codex-rs/app-server/tests/suite/v2/app_list.rs b/codex-rs/app-server/tests/suite/v2/app_list.rs index 4c3e22ffa7..53221adae2 100644 --- a/codex-rs/app-server/tests/suite/v2/app_list.rs +++ b/codex-rs/app-server/tests/suite/v2/app_list.rs @@ -13,14 +13,13 @@ use axum::extract::State; use axum::http::HeaderMap; use axum::http::StatusCode; use axum::http::header::AUTHORIZATION; -use axum::routing::post; +use axum::routing::get; use codex_app_server_protocol::AppInfo; use codex_app_server_protocol::AppsListParams; use codex_app_server_protocol::AppsListResponse; use codex_app_server_protocol::JSONRPCResponse; use codex_app_server_protocol::RequestId; use codex_core::auth::AuthCredentialsStoreMode; -use codex_core::connectors::ConnectorInfo; use pretty_assertions::assert_eq; use rmcp::handler::server::ServerHandler; use rmcp::model::JsonObject; @@ -71,19 +70,23 @@ async fn list_apps_returns_empty_when_connectors_disabled() -> Result<()> { #[tokio::test] async fn list_apps_returns_connectors_with_accessible_flags() -> Result<()> { let connectors = vec![ - ConnectorInfo { - connector_id: "alpha".to_string(), - connector_name: "Alpha".to_string(), - connector_description: Some("Alpha connector".to_string()), + AppInfo { + id: "alpha".to_string(), + name: "Alpha".to_string(), + description: Some("Alpha connector".to_string()), logo_url: Some("https://example.com/alpha.png".to_string()), + logo_url_dark: None, + distribution_channel: None, install_url: None, is_accessible: false, }, - ConnectorInfo { - connector_id: "beta".to_string(), - connector_name: "beta".to_string(), - connector_description: None, + AppInfo { + id: "beta".to_string(), + name: "beta".to_string(), + description: None, logo_url: None, + logo_url_dark: None, + distribution_channel: None, install_url: None, is_accessible: false, }, @@ -127,6 +130,8 @@ async fn list_apps_returns_connectors_with_accessible_flags() -> Result<()> { name: "Beta App".to_string(), description: None, logo_url: None, + logo_url_dark: None, + distribution_channel: None, install_url: Some("https://chatgpt.com/apps/beta/beta".to_string()), is_accessible: true, }, @@ -135,6 +140,8 @@ async fn list_apps_returns_connectors_with_accessible_flags() -> Result<()> { name: "Alpha".to_string(), description: Some("Alpha connector".to_string()), logo_url: Some("https://example.com/alpha.png".to_string()), + logo_url_dark: None, + distribution_channel: None, install_url: Some("https://chatgpt.com/apps/alpha/alpha".to_string()), is_accessible: false, }, @@ -150,19 +157,23 @@ async fn list_apps_returns_connectors_with_accessible_flags() -> Result<()> { #[tokio::test] async fn list_apps_paginates_results() -> Result<()> { let connectors = vec![ - ConnectorInfo { - connector_id: "alpha".to_string(), - connector_name: "Alpha".to_string(), - connector_description: Some("Alpha connector".to_string()), + AppInfo { + id: "alpha".to_string(), + name: "Alpha".to_string(), + description: Some("Alpha connector".to_string()), logo_url: None, + logo_url_dark: None, + distribution_channel: None, install_url: None, is_accessible: false, }, - ConnectorInfo { - connector_id: "beta".to_string(), - connector_name: "beta".to_string(), - connector_description: None, + AppInfo { + id: "beta".to_string(), + name: "beta".to_string(), + description: None, logo_url: None, + logo_url_dark: None, + distribution_channel: None, install_url: None, is_accessible: false, }, @@ -206,6 +217,8 @@ async fn list_apps_paginates_results() -> Result<()> { name: "Beta App".to_string(), description: None, logo_url: None, + logo_url_dark: None, + distribution_channel: None, install_url: Some("https://chatgpt.com/apps/beta/beta".to_string()), is_accessible: true, }]; @@ -234,6 +247,8 @@ async fn list_apps_paginates_results() -> Result<()> { name: "Alpha".to_string(), description: Some("Alpha connector".to_string()), logo_url: None, + logo_url_dark: None, + distribution_channel: None, install_url: Some("https://chatgpt.com/apps/alpha/alpha".to_string()), is_accessible: false, }]; @@ -289,13 +304,13 @@ impl ServerHandler for AppListMcpServer { } async fn start_apps_server( - connectors: Vec, + connectors: Vec, tools: Vec, ) -> Result<(String, JoinHandle<()>)> { let state = AppsServerState { expected_bearer: "Bearer chatgpt-token".to_string(), expected_account_id: "account-123".to_string(), - response: json!({ "connectors": connectors }), + response: json!({ "apps": connectors, "next_token": null }), }; let state = Arc::new(state); let tools = Arc::new(tools); @@ -313,7 +328,11 @@ async fn start_apps_server( ); let router = Router::new() - .route("/aip/connectors/list_accessible", post(list_connectors)) + .route("/connectors/directory/list", get(list_directory_connectors)) + .route( + "/connectors/directory/list_workspace", + get(list_directory_connectors), + ) .with_state(state) .nest_service("/api/codex/apps", mcp_service); @@ -324,7 +343,7 @@ async fn start_apps_server( Ok((format!("http://{addr}"), handle)) } -async fn list_connectors( +async fn list_directory_connectors( State(state): State>, headers: HeaderMap, ) -> Result { diff --git a/codex-rs/chatgpt/Cargo.toml b/codex-rs/chatgpt/Cargo.toml index 70cd0aa5aa..3c55878c8f 100644 --- a/codex-rs/chatgpt/Cargo.toml +++ b/codex-rs/chatgpt/Cargo.toml @@ -17,6 +17,8 @@ serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } tokio = { workspace = true, features = ["full"] } codex-git = { workspace = true } +urlencoding = { workspace = true } [dev-dependencies] +pretty_assertions = { workspace = true } tempfile = { workspace = true } diff --git a/codex-rs/chatgpt/src/chatgpt_client.rs b/codex-rs/chatgpt/src/chatgpt_client.rs index b35238cdec..b13be8c0cd 100644 --- a/codex-rs/chatgpt/src/chatgpt_client.rs +++ b/codex-rs/chatgpt/src/chatgpt_client.rs @@ -5,13 +5,21 @@ use crate::chatgpt_token::get_chatgpt_token_data; use crate::chatgpt_token::init_chatgpt_token_from_auth; use anyhow::Context; -use serde::Serialize; use serde::de::DeserializeOwned; +use std::time::Duration; /// Make a GET request to the ChatGPT backend API. pub(crate) async fn chatgpt_get_request( config: &Config, path: String, +) -> anyhow::Result { + chatgpt_get_request_with_timeout(config, path, None).await +} + +pub(crate) async fn chatgpt_get_request_with_timeout( + config: &Config, + path: String, + timeout: Option, ) -> anyhow::Result { let chatgpt_base_url = &config.chatgpt_base_url; init_chatgpt_token_from_auth(&config.codex_home, config.cli_auth_credentials_store_mode) @@ -28,48 +36,17 @@ pub(crate) async fn chatgpt_get_request( anyhow::anyhow!("ChatGPT account ID not available, please re-run `codex login`") }); - let response = client + let mut request = client .get(&url) .bearer_auth(&token.access_token) .header("chatgpt-account-id", account_id?) - .header("Content-Type", "application/json") - .send() - .await - .context("Failed to send request")?; - - if response.status().is_success() { - let result: T = response - .json() - .await - .context("Failed to parse JSON response")?; - Ok(result) - } else { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - anyhow::bail!("Request failed with status {status}: {body}") - } -} - -pub(crate) async fn chatgpt_post_request( - config: &Config, - access_token: &str, - account_id: &str, - path: &str, - payload: &P, -) -> anyhow::Result { - let chatgpt_base_url = &config.chatgpt_base_url; - let client = create_client(); - let url = format!("{chatgpt_base_url}{path}"); - - let response = client - .post(&url) - .bearer_auth(access_token) - .header("chatgpt-account-id", account_id) - .header("Content-Type", "application/json") - .json(payload) - .send() - .await - .context("Failed to send request")?; + .header("Content-Type", "application/json"); + + if let Some(timeout) = timeout { + request = request.timeout(timeout); + } + + let response = request.send().await.context("Failed to send request")?; if response.status().is_success() { let result: T = response diff --git a/codex-rs/chatgpt/src/connectors.rs b/codex-rs/chatgpt/src/connectors.rs index 5d913ae165..c1a8f65359 100644 --- a/codex-rs/chatgpt/src/connectors.rs +++ b/codex-rs/chatgpt/src/connectors.rs @@ -1,43 +1,45 @@ +use std::collections::HashMap; + use codex_core::config::Config; use codex_core::features::Feature; use serde::Deserialize; -use serde::Serialize; +use std::time::Duration; -use crate::chatgpt_client::chatgpt_post_request; +use crate::chatgpt_client::chatgpt_get_request_with_timeout; use crate::chatgpt_token::get_chatgpt_token_data; use crate::chatgpt_token::init_chatgpt_token_from_auth; -pub use codex_core::connectors::ConnectorInfo; +pub use codex_core::connectors::AppInfo; pub use codex_core::connectors::connector_display_label; use codex_core::connectors::connector_install_url; pub use codex_core::connectors::list_accessible_connectors_from_mcp_tools; use codex_core::connectors::merge_connectors; -#[derive(Debug, Serialize)] -struct ListConnectorsRequest { - principals: Vec, -} - -#[derive(Debug, Serialize)] -struct Principal { - #[serde(rename = "type")] - principal_type: PrincipalType, - id: String, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -enum PrincipalType { - User, -} - #[derive(Debug, Deserialize)] -struct ListConnectorsResponse { - connectors: Vec, +struct DirectoryListResponse { + apps: Vec, + #[serde(alias = "nextToken")] + next_token: Option, } -pub async fn list_connectors(config: &Config) -> anyhow::Result> { - if !config.features.enabled(Feature::Connectors) { +#[derive(Debug, Deserialize, Clone)] +struct DirectoryApp { + id: String, + name: String, + description: Option, + #[serde(alias = "logoUrl")] + logo_url: Option, + #[serde(alias = "logoUrlDark")] + logo_url_dark: Option, + #[serde(alias = "distributionChannel")] + distribution_channel: Option, + visibility: Option, +} + +const DIRECTORY_CONNECTORS_TIMEOUT: Duration = Duration::from_secs(60); + +pub async fn list_connectors(config: &Config) -> anyhow::Result> { + if !config.features.enabled(Feature::Apps) { return Ok(Vec::new()); } let (connectors_result, accessible_result) = tokio::join!( @@ -46,11 +48,12 @@ pub async fn list_connectors(config: &Config) -> anyhow::Result anyhow::Result> { - if !config.features.enabled(Feature::Connectors) { +pub async fn list_all_connectors(config: &Config) -> anyhow::Result> { + if !config.features.enabled(Feature::Apps) { return Ok(Vec::new()); } init_chatgpt_token_from_auth(&config.codex_home, config.cli_auth_credentials_store_mode) @@ -58,56 +61,149 @@ pub async fn list_all_connectors(config: &Config) -> anyhow::Result>(); for connector in &mut connectors { let install_url = match connector.install_url.take() { Some(install_url) => install_url, - None => connector_install_url(&connector.connector_name, &connector.connector_id), + None => connector_install_url(&connector.name, &connector.id), }; - connector.connector_name = - normalize_connector_name(&connector.connector_name, &connector.connector_id); - connector.connector_description = - normalize_connector_value(connector.connector_description.as_deref()); + connector.name = normalize_connector_name(&connector.name, &connector.id); + connector.description = normalize_connector_value(connector.description.as_deref()); connector.install_url = Some(install_url); connector.is_accessible = false; } connectors.sort_by(|left, right| { - left.connector_name - .cmp(&right.connector_name) - .then_with(|| left.connector_id.cmp(&right.connector_id)) + left.name + .cmp(&right.name) + .then_with(|| left.id.cmp(&right.id)) }); Ok(connectors) } +async fn list_directory_connectors(config: &Config) -> anyhow::Result> { + let mut apps = Vec::new(); + let mut next_token: Option = None; + loop { + let path = match next_token.as_deref() { + Some(token) => { + let encoded_token = urlencoding::encode(token); + format!("/connectors/directory/list?tier=categorized&token={encoded_token}") + } + None => "/connectors/directory/list?tier=categorized".to_string(), + }; + let response: DirectoryListResponse = + chatgpt_get_request_with_timeout(config, path, Some(DIRECTORY_CONNECTORS_TIMEOUT)) + .await?; + apps.extend( + response + .apps + .into_iter() + .filter(|app| !is_hidden_directory_app(app)), + ); + next_token = response + .next_token + .map(|token| token.trim().to_string()) + .filter(|token| !token.is_empty()); + if next_token.is_none() { + break; + } + } + Ok(apps) +} + +async fn list_workspace_connectors(config: &Config) -> anyhow::Result> { + let response: anyhow::Result = chatgpt_get_request_with_timeout( + config, + "/connectors/directory/list_workspace".to_string(), + Some(DIRECTORY_CONNECTORS_TIMEOUT), + ) + .await; + match response { + Ok(response) => Ok(response + .apps + .into_iter() + .filter(|app| !is_hidden_directory_app(app)) + .collect()), + Err(_) => Ok(Vec::new()), + } +} + +fn merge_directory_apps(apps: Vec) -> Vec { + let mut merged: HashMap = HashMap::new(); + for app in apps { + if let Some(existing) = merged.get_mut(&app.id) { + merge_directory_app(existing, app); + } else { + merged.insert(app.id.clone(), app); + } + } + merged.into_values().collect() +} + +fn merge_directory_app(existing: &mut DirectoryApp, incoming: DirectoryApp) { + let DirectoryApp { + id: _, + name, + description, + logo_url, + logo_url_dark, + distribution_channel, + visibility: _, + } = incoming; + + let incoming_name_is_empty = name.trim().is_empty(); + if existing.name.trim().is_empty() && !incoming_name_is_empty { + existing.name = name; + } + + let incoming_description_present = description + .as_deref() + .map(|value| !value.trim().is_empty()) + .unwrap_or(false); + let existing_description_present = existing + .description + .as_deref() + .map(|value| !value.trim().is_empty()) + .unwrap_or(false); + if !existing_description_present && incoming_description_present { + existing.description = description; + } + + if existing.logo_url.is_none() && logo_url.is_some() { + existing.logo_url = logo_url; + } + if existing.logo_url_dark.is_none() && logo_url_dark.is_some() { + existing.logo_url_dark = logo_url_dark; + } + if existing.distribution_channel.is_none() && distribution_channel.is_some() { + existing.distribution_channel = distribution_channel; + } +} + +fn is_hidden_directory_app(app: &DirectoryApp) -> bool { + matches!(app.visibility.as_deref(), Some("HIDDEN")) +} + +fn directory_app_to_app_info(app: DirectoryApp) -> AppInfo { + AppInfo { + id: app.id, + name: app.name, + description: app.description, + logo_url: app.logo_url, + logo_url_dark: app.logo_url_dark, + distribution_channel: app.distribution_channel, + install_url: None, + is_accessible: false, + } +} + fn normalize_connector_name(name: &str, connector_id: &str) -> String { let trimmed = name.trim(); if trimmed.is_empty() { @@ -123,3 +219,87 @@ fn normalize_connector_value(value: Option<&str>) -> Option { .filter(|value| !value.is_empty()) .map(str::to_string) } + +const ALLOWED_APPS_SDK_APPS: &[&str] = &["asdk_app_69781557cc1481919cf5e9824fa2e792"]; +const DISALLOWED_CONNECTOR_IDS: &[&str] = &["asdk_app_6938a94a61d881918ef32cb999ff937c"]; +const DISALLOWED_CONNECTOR_PREFIX: &str = "connector_openai_"; + +fn filter_disallowed_connectors(connectors: Vec) -> Vec { + // TODO: Support Apps SDK connectors. + connectors + .into_iter() + .filter(is_connector_allowed) + .collect() +} + +fn is_connector_allowed(connector: &AppInfo) -> bool { + let connector_id = connector.id.as_str(); + if connector_id.starts_with(DISALLOWED_CONNECTOR_PREFIX) + || DISALLOWED_CONNECTOR_IDS.contains(&connector_id) + { + return false; + } + if connector_id.starts_with("asdk_app_") { + return ALLOWED_APPS_SDK_APPS.contains(&connector_id); + } + true +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + + fn app(id: &str) -> AppInfo { + AppInfo { + id: id.to_string(), + name: id.to_string(), + description: None, + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + install_url: None, + is_accessible: false, + } + } + + #[test] + fn filters_internal_asdk_connectors() { + let filtered = filter_disallowed_connectors(vec![app("asdk_app_hidden"), app("alpha")]); + assert_eq!(filtered, vec![app("alpha")]); + } + + #[test] + fn allows_whitelisted_asdk_connectors() { + let filtered = filter_disallowed_connectors(vec![ + app("asdk_app_69781557cc1481919cf5e9824fa2e792"), + app("beta"), + ]); + assert_eq!( + filtered, + vec![ + app("asdk_app_69781557cc1481919cf5e9824fa2e792"), + app("beta") + ] + ); + } + + #[test] + fn filters_openai_connectors() { + let filtered = filter_disallowed_connectors(vec![ + app("connector_openai_foo"), + app("connector_openai_bar"), + app("gamma"), + ]); + assert_eq!(filtered, vec![app("gamma")]); + } + + #[test] + fn filters_disallowed_connector_ids() { + let filtered = filter_disallowed_connectors(vec![ + app("asdk_app_6938a94a61d881918ef32cb999ff937c"), + app("delta"), + ]); + assert_eq!(filtered, vec![app("delta")]); + } +} diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index ea6ec9e872..717eddc612 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -144,6 +144,9 @@ "apply_patch_freeform": { "type": "boolean" }, + "apps": { + "type": "boolean" + }, "child_agents_md": { "type": "boolean" }, @@ -1165,6 +1168,9 @@ "apply_patch_freeform": { "type": "boolean" }, + "apps": { + "type": "boolean" + }, "child_agents_md": { "type": "boolean" }, diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 70e52eb064..dbc2d5c632 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -118,6 +118,10 @@ use crate::mcp::effective_mcp_servers; use crate::mcp::maybe_prompt_and_install_mcp_dependencies; use crate::mcp::with_codex_apps_mcp; use crate::mcp_connection_manager::McpConnectionManager; +use crate::mentions::build_connector_slug_counts; +use crate::mentions::build_skill_name_counts; +use crate::mentions::collect_explicit_app_paths; +use crate::mentions::collect_tool_mentions_from_messages; use crate::model_provider_info::CHAT_WIRE_API_DEPRECATION_SUMMARY; use crate::project_doc::get_user_instructions; use crate::protocol::AgentMessageContentDeltaEvent; @@ -163,6 +167,9 @@ use crate::skills::SkillMetadata; use crate::skills::SkillsManager; use crate::skills::build_skill_injections; use crate::skills::collect_explicit_skill_mentions; +use crate::skills::injection::ToolMentionKind; +use crate::skills::injection::app_id_from_path; +use crate::skills::injection::tool_kind_for_path; use crate::state::ActiveTurn; use crate::state::SessionServices; use crate::state::SessionState; @@ -2189,7 +2196,7 @@ impl Session { let config = self.get_config().await; let mcp_servers = with_codex_apps_mcp( mcp_servers, - self.features.enabled(Feature::Connectors), + self.features.enabled(Feature::Apps), auth.as_ref(), config.as_ref(), ); @@ -3166,9 +3173,38 @@ pub(crate) async fn run_turn( .await, ); + let (skill_name_counts, skill_name_counts_lower) = skills_outcome.as_ref().map_or_else( + || (HashMap::new(), HashMap::new()), + |outcome| build_skill_name_counts(&outcome.skills, &outcome.disabled_paths), + ); + let connector_slug_counts = if turn_context.client.config().features.enabled(Feature::Apps) { + let mcp_tools = match sess + .services + .mcp_connection_manager + .read() + .await + .list_all_tools() + .or_cancel(&cancellation_token) + .await + { + Ok(mcp_tools) => mcp_tools, + Err(_) => return None, + }; + let connectors = connectors::accessible_connectors_from_mcp_tools(&mcp_tools); + build_connector_slug_counts(&connectors) + } else { + HashMap::new() + }; let mentioned_skills = skills_outcome.as_ref().map_or_else(Vec::new, |outcome| { - collect_explicit_skill_mentions(&input, &outcome.skills, &outcome.disabled_paths) + collect_explicit_skill_mentions( + &input, + &outcome.skills, + &outcome.disabled_paths, + &skill_name_counts, + &connector_slug_counts, + ) }); + let explicit_app_paths = collect_explicit_app_paths(&input); maybe_prompt_and_install_mcp_dependencies( sess.as_ref(), @@ -3234,12 +3270,17 @@ pub(crate) async fn run_turn( }) .map(|user_message| user_message.message()) .collect::>(); + let tool_selection = SamplingRequestToolSelection { + explicit_app_paths: &explicit_app_paths, + skill_name_counts_lower: &skill_name_counts_lower, + }; match run_sampling_request( Arc::clone(&sess), Arc::clone(&turn_context), Arc::clone(&turn_diff_tracker), &mut client_session, sampling_request_input, + tool_selection, cancellation_token.child_token(), ) .await @@ -3314,40 +3355,79 @@ async fn run_auto_compact(sess: &Arc, turn_context: &Arc) } fn filter_connectors_for_input( - connectors: Vec, + connectors: Vec, input: &[ResponseItem], -) -> Vec { + explicit_app_paths: &[String], + skill_name_counts_lower: &HashMap, +) -> Vec { let user_messages = collect_user_messages(input); - if user_messages.is_empty() { + if user_messages.is_empty() && explicit_app_paths.is_empty() { return Vec::new(); } + let mentions = collect_tool_mentions_from_messages(&user_messages); + let mention_names_lower = mentions + .plain_names + .iter() + .map(|name| name.to_ascii_lowercase()) + .collect::>(); + + let connector_slug_counts = build_connector_slug_counts(&connectors); + let mut allowed_connector_ids: HashSet = HashSet::new(); + for path in explicit_app_paths + .iter() + .chain(mentions.paths.iter()) + .filter(|path| tool_kind_for_path(path) == ToolMentionKind::App) + { + if let Some(connector_id) = app_id_from_path(path) { + allowed_connector_ids.insert(connector_id.to_string()); + } + } + connectors .into_iter() - .filter(|connector| connector_inserted_in_messages(connector, &user_messages)) + .filter(|connector| { + connector_inserted_in_messages( + connector, + &mention_names_lower, + &allowed_connector_ids, + &connector_slug_counts, + skill_name_counts_lower, + ) + }) .collect() } fn connector_inserted_in_messages( - connector: &connectors::ConnectorInfo, - user_messages: &[String], + connector: &connectors::AppInfo, + mention_names_lower: &HashSet, + allowed_connector_ids: &HashSet, + connector_slug_counts: &HashMap, + skill_name_counts_lower: &HashMap, ) -> bool { - let label = connectors::connector_display_label(connector); - let needle = label.to_lowercase(); - let legacy = format!("{label} connector").to_lowercase(); - user_messages.iter().any(|message| { - let message = message.to_lowercase(); - message.contains(&needle) || message.contains(&legacy) - }) + if allowed_connector_ids.contains(&connector.id) { + return true; + } + + let mention_slug = connectors::connector_mention_slug(connector); + let connector_count = connector_slug_counts + .get(&mention_slug) + .copied() + .unwrap_or(0); + let skill_count = skill_name_counts_lower + .get(&mention_slug) + .copied() + .unwrap_or(0); + connector_count == 1 && skill_count == 0 && mention_names_lower.contains(&mention_slug) } fn filter_codex_apps_mcp_tools( mut mcp_tools: HashMap, - connectors: &[connectors::ConnectorInfo], + connectors: &[connectors::AppInfo], ) -> HashMap { let allowed: HashSet<&str> = connectors .iter() - .map(|connector| connector.connector_id.as_str()) + .map(|connector| connector.id.as_str()) .collect(); mcp_tools.retain(|_, tool| { @@ -3367,6 +3447,11 @@ fn codex_apps_connector_id(tool: &crate::mcp_connection_manager::ToolInfo) -> Op tool.connector_id.as_deref() } +struct SamplingRequestToolSelection<'a> { + explicit_app_paths: &'a [String], + skill_name_counts_lower: &'a HashMap, +} + #[instrument(level = "trace", skip_all, fields( @@ -3381,6 +3466,7 @@ async fn run_sampling_request( turn_diff_tracker: SharedTurnDiffTracker, client_session: &mut ModelClientSession, input: Vec, + tool_selection: SamplingRequestToolSelection<'_>, cancellation_token: CancellationToken, ) -> CodexResult { let mut mcp_tools = sess @@ -3391,14 +3477,14 @@ async fn run_sampling_request( .list_all_tools() .or_cancel(&cancellation_token) .await?; - let connectors_for_tools = if turn_context - .client - .config() - .features - .enabled(Feature::Connectors) - { + let connectors_for_tools = if turn_context.client.config().features.enabled(Feature::Apps) { let connectors = connectors::accessible_connectors_from_mcp_tools(&mcp_tools); - Some(filter_connectors_for_input(connectors, &input)) + Some(filter_connectors_for_input( + connectors, + &input, + tool_selection.explicit_app_paths, + tool_selection.skill_name_counts_lower, + )) } else { None }; @@ -3822,6 +3908,7 @@ mod tests { use crate::tools::handlers::UnifiedExecHandler; use crate::tools::registry::ToolHandler; use crate::turn_diff_tracker::TurnDiffTracker; + use codex_app_server_protocol::AppInfo; use codex_app_server_protocol::AuthMode; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; @@ -3843,6 +3930,30 @@ mod tests { expects_apply_patch_instructions: bool, } + fn user_message(text: &str) -> ResponseItem { + ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { + text: text.to_string(), + }], + end_turn: None, + } + } + + fn make_connector(id: &str, name: &str) -> AppInfo { + AppInfo { + id: id.to_string(), + name: name.to_string(), + description: None, + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + install_url: None, + is_accessible: true, + } + } + #[tokio::test] async fn get_base_instructions_no_user_content() { let prompt_with_apply_patch_instructions = @@ -3909,6 +4020,43 @@ mod tests { } } + #[test] + fn filter_connectors_for_input_skips_duplicate_slug_mentions() { + let connectors = vec![ + make_connector("one", "Foo Bar"), + make_connector("two", "Foo-Bar"), + ]; + let input = vec![user_message("use $foo-bar")]; + let explicit_app_paths = Vec::new(); + let skill_name_counts_lower = HashMap::new(); + + let selected = filter_connectors_for_input( + connectors, + &input, + &explicit_app_paths, + &skill_name_counts_lower, + ); + + assert_eq!(selected, Vec::new()); + } + + #[test] + fn filter_connectors_for_input_skips_when_skill_name_conflicts() { + let connectors = vec![make_connector("one", "Todoist")]; + let input = vec![user_message("use $todoist")]; + let explicit_app_paths = Vec::new(); + let skill_name_counts_lower = HashMap::from([("todoist".to_string(), 1)]); + + let selected = filter_connectors_for_input( + connectors, + &input, + &explicit_app_paths, + &skill_name_counts_lower, + ); + + assert_eq!(selected, Vec::new()); + } + #[tokio::test] async fn reconstruct_history_matches_live_compactions() { let (session, turn_context) = make_session_and_context().await; diff --git a/codex-rs/core/src/connectors.rs b/codex-rs/core/src/connectors.rs index fa97dacc3c..edf3c63aa5 100644 --- a/codex-rs/core/src/connectors.rs +++ b/codex-rs/core/src/connectors.rs @@ -3,9 +3,8 @@ use std::env; use std::path::PathBuf; use async_channel::unbounded; +pub use codex_app_server_protocol::AppInfo; use codex_protocol::protocol::SandboxPolicy; -use serde::Deserialize; -use serde::Serialize; use tokio_util::sync::CancellationToken; use crate::AuthManager; @@ -15,28 +14,13 @@ use crate::features::Feature; use crate::mcp::CODEX_APPS_MCP_SERVER_NAME; use crate::mcp::auth::compute_auth_statuses; use crate::mcp::with_codex_apps_mcp; +use crate::mcp_connection_manager::DEFAULT_STARTUP_TIMEOUT; use crate::mcp_connection_manager::McpConnectionManager; -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ConnectorInfo { - #[serde(rename = "id")] - pub connector_id: String, - #[serde(rename = "name")] - pub connector_name: String, - #[serde(default, rename = "description")] - pub connector_description: Option, - #[serde(default, rename = "logo_url")] - pub logo_url: Option, - #[serde(default, rename = "install_url")] - pub install_url: Option, - #[serde(default)] - pub is_accessible: bool, -} - pub async fn list_accessible_connectors_from_mcp_tools( config: &Config, -) -> anyhow::Result> { - if !config.features.enabled(Feature::Connectors) { +) -> anyhow::Result> { + if !config.features.enabled(Feature::Apps) { return Ok(Vec::new()); } @@ -72,6 +56,13 @@ pub async fn list_accessible_connectors_from_mcp_tools( ) .await; + if let Some(cfg) = mcp_servers.get(CODEX_APPS_MCP_SERVER_NAME) { + let timeout = cfg.startup_timeout_sec.unwrap_or(DEFAULT_STARTUP_TIMEOUT); + mcp_connection_manager + .wait_for_server_ready(CODEX_APPS_MCP_SERVER_NAME, timeout) + .await; + } + let tools = mcp_connection_manager.list_all_tools().await; cancel_token.cancel(); @@ -86,13 +77,17 @@ fn auth_manager_from_config(config: &Config) -> std::sync::Arc { ) } -pub fn connector_display_label(connector: &ConnectorInfo) -> String { - format_connector_label(&connector.connector_name, &connector.connector_id) +pub fn connector_display_label(connector: &AppInfo) -> String { + format_connector_label(&connector.name, &connector.id) +} + +pub fn connector_mention_slug(connector: &AppInfo) -> String { + connector_name_slug(&connector_display_label(connector)) } pub(crate) fn accessible_connectors_from_mcp_tools( mcp_tools: &HashMap, -) -> Vec { +) -> Vec { let tools = mcp_tools.values().filter_map(|tool| { if tool.server_name != CODEX_APPS_MCP_SERVER_NAME { return None; @@ -105,34 +100,37 @@ pub(crate) fn accessible_connectors_from_mcp_tools( } pub fn merge_connectors( - connectors: Vec, - accessible_connectors: Vec, -) -> Vec { - let mut merged: HashMap = connectors + connectors: Vec, + accessible_connectors: Vec, +) -> Vec { + let mut merged: HashMap = connectors .into_iter() .map(|mut connector| { connector.is_accessible = false; - (connector.connector_id.clone(), connector) + (connector.id.clone(), connector) }) .collect(); for mut connector in accessible_connectors { connector.is_accessible = true; - let connector_id = connector.connector_id.clone(); + let connector_id = connector.id.clone(); if let Some(existing) = merged.get_mut(&connector_id) { existing.is_accessible = true; - if existing.connector_name == existing.connector_id - && connector.connector_name != connector.connector_id - { - existing.connector_name = connector.connector_name; + if existing.name == existing.id && connector.name != connector.id { + existing.name = connector.name; } - if existing.connector_description.is_none() && connector.connector_description.is_some() - { - existing.connector_description = connector.connector_description; + if existing.description.is_none() && connector.description.is_some() { + existing.description = connector.description; } if existing.logo_url.is_none() && connector.logo_url.is_some() { existing.logo_url = connector.logo_url; } + if existing.logo_url_dark.is_none() && connector.logo_url_dark.is_some() { + existing.logo_url_dark = connector.logo_url_dark; + } + if existing.distribution_channel.is_none() && connector.distribution_channel.is_some() { + existing.distribution_channel = connector.distribution_channel; + } } else { merged.insert(connector_id, connector); } @@ -141,23 +139,20 @@ pub fn merge_connectors( let mut merged = merged.into_values().collect::>(); for connector in &mut merged { if connector.install_url.is_none() { - connector.install_url = Some(connector_install_url( - &connector.connector_name, - &connector.connector_id, - )); + connector.install_url = Some(connector_install_url(&connector.name, &connector.id)); } } merged.sort_by(|left, right| { right .is_accessible .cmp(&left.is_accessible) - .then_with(|| left.connector_name.cmp(&right.connector_name)) - .then_with(|| left.connector_id.cmp(&right.connector_id)) + .then_with(|| left.name.cmp(&right.name)) + .then_with(|| left.id.cmp(&right.id)) }); merged } -fn collect_accessible_connectors(tools: I) -> Vec +fn collect_accessible_connectors(tools: I) -> Vec where I: IntoIterator)>, { @@ -172,14 +167,16 @@ where connectors.insert(connector_id, connector_name); } } - let mut accessible: Vec = connectors + let mut accessible: Vec = connectors .into_iter() - .map(|(connector_id, connector_name)| ConnectorInfo { - install_url: Some(connector_install_url(&connector_name, &connector_id)), - connector_id, - connector_name, - connector_description: None, + .map(|(connector_id, connector_name)| AppInfo { + id: connector_id.clone(), + name: connector_name.clone(), + description: None, logo_url: None, + logo_url_dark: None, + distribution_channel: None, + install_url: Some(connector_install_url(&connector_name, &connector_id)), is_accessible: true, }) .collect(); @@ -187,8 +184,8 @@ where right .is_accessible .cmp(&left.is_accessible) - .then_with(|| left.connector_name.cmp(&right.connector_name)) - .then_with(|| left.connector_id.cmp(&right.connector_id)) + .then_with(|| left.name.cmp(&right.name)) + .then_with(|| left.id.cmp(&right.id)) }); accessible } @@ -205,7 +202,7 @@ pub fn connector_install_url(name: &str, connector_id: &str) -> String { format!("https://chatgpt.com/apps/{slug}/{connector_id}") } -fn connector_name_slug(name: &str) -> String { +pub fn connector_name_slug(name: &str) -> String { let mut normalized = String::with_capacity(name.len()); for character in name.chars() { if character.is_ascii_alphanumeric() { diff --git a/codex-rs/core/src/features.rs b/codex-rs/core/src/features.rs index bedf67c4ad..c7cd411145 100644 --- a/codex-rs/core/src/features.rs +++ b/codex-rs/core/src/features.rs @@ -111,8 +111,8 @@ pub enum Feature { EnableRequestCompression, /// Enable collab tools. Collab, - /// Enable connectors (apps). - Connectors, + /// Enable apps. + Apps, /// Allow prompting and installing missing MCP dependencies. SkillMcpDependencyInstall, /// Steer feature flag - when enabled, Enter submits immediately instead of queuing. @@ -518,9 +518,13 @@ pub const FEATURES: &[FeatureSpec] = &[ default_enabled: false, }, FeatureSpec { - id: Feature::Connectors, - key: "connectors", - stage: Stage::UnderDevelopment, + id: Feature::Apps, + key: "apps", + stage: Stage::Experimental { + name: "Apps", + menu_description: "Use a connected ChatGPT App using \"$\". Install Apps via /apps command. Restart Codex after enabling.", + announcement: "NEW: Use ChatGPT Apps (Connectors) in Codex via $ mentions. Enable in /experimental and restart Codex!", + }, default_enabled: false, }, FeatureSpec { diff --git a/codex-rs/core/src/features/legacy.rs b/codex-rs/core/src/features/legacy.rs index 2b5a9e7fe0..0c0f75714c 100644 --- a/codex-rs/core/src/features/legacy.rs +++ b/codex-rs/core/src/features/legacy.rs @@ -9,6 +9,10 @@ struct Alias { } const ALIASES: &[Alias] = &[ + Alias { + legacy_key: "connectors", + feature: Feature::Apps, + }, Alias { legacy_key: "enable_experimental_windows_sandbox", feature: Feature::WindowsSandbox, diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index eaf25d14e8..ab51a201dc 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -42,6 +42,7 @@ pub use mcp_connection_manager::MCP_SANDBOX_STATE_CAPABILITY; pub use mcp_connection_manager::MCP_SANDBOX_STATE_METHOD; pub use mcp_connection_manager::SandboxState; mod mcp_tool_call; +mod mentions; mod message_history; mod model_provider_info; pub mod parse_command; diff --git a/codex-rs/core/src/mcp/mod.rs b/codex-rs/core/src/mcp/mod.rs index fa1de0f045..12c61ddaf1 100644 --- a/codex-rs/core/src/mcp/mod.rs +++ b/codex-rs/core/src/mcp/mod.rs @@ -6,6 +6,7 @@ pub(crate) use skill_dependencies::maybe_prompt_and_install_mcp_dependencies; use std::collections::HashMap; use std::env; use std::path::PathBuf; +use std::time::Duration; use async_channel::unbounded; use codex_protocol::protocol::McpListToolsResponseEvent; @@ -25,7 +26,7 @@ use crate::mcp_connection_manager::SandboxState; const MCP_TOOL_NAME_PREFIX: &str = "mcp"; const MCP_TOOL_NAME_DELIMITER: &str = "__"; -pub(crate) const CODEX_APPS_MCP_SERVER_NAME: &str = "codex_apps_mcp"; +pub(crate) const CODEX_APPS_MCP_SERVER_NAME: &str = "codex_apps"; const CODEX_CONNECTORS_TOKEN_ENV_VAR: &str = "CODEX_CONNECTORS_TOKEN"; fn codex_apps_mcp_bearer_token_env_var() -> Option { @@ -97,7 +98,7 @@ fn codex_apps_mcp_server_config(config: &Config, auth: Option<&CodexAuth>) -> Mc }, enabled: true, disabled_reason: None, - startup_timeout_sec: None, + startup_timeout_sec: Some(Duration::from_secs(30)), tool_timeout_sec: None, enabled_tools: None, disabled_tools: None, @@ -128,7 +129,7 @@ pub(crate) fn effective_mcp_servers( ) -> HashMap { with_codex_apps_mcp( config.mcp_servers.get().clone(), - config.features.enabled(Feature::Connectors), + config.features.enabled(Feature::Apps), auth, config, ) diff --git a/codex-rs/core/src/mcp_connection_manager.rs b/codex-rs/core/src/mcp_connection_manager.rs index 9462f745ba..15305f4b0c 100644 --- a/codex-rs/core/src/mcp_connection_manager.rs +++ b/codex-rs/core/src/mcp_connection_manager.rs @@ -14,6 +14,7 @@ use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; +use crate::mcp::CODEX_APPS_MCP_SERVER_NAME; use crate::mcp::auth::McpAuthStatusEntry; use anyhow::Context; use anyhow::Result; @@ -436,13 +437,33 @@ impl McpConnectionManager { .await } + pub(crate) async fn wait_for_server_ready(&self, server_name: &str, timeout: Duration) -> bool { + let Some(async_managed_client) = self.clients.get(server_name) else { + return false; + }; + + match tokio::time::timeout(timeout, async_managed_client.client()).await { + Ok(Ok(_)) => true, + Ok(Err(_)) | Err(_) => false, + } + } + /// Returns a single map that contains all tools. Each key is the /// fully-qualified name for the tool. #[instrument(level = "trace", skip_all)] pub async fn list_all_tools(&self) -> HashMap { let mut tools = HashMap::new(); - for managed_client in self.clients.values() { - if let Ok(client) = managed_client.client().await { + for (server_name, managed_client) in &self.clients { + let client = if server_name == CODEX_APPS_MCP_SERVER_NAME { + // Avoid blocking on codex_apps_mcp startup; use tools only when ready. + match managed_client.client.clone().now_or_never() { + Some(Ok(client)) => Some(client), + _ => None, + } + } else { + managed_client.client().await.ok() + }; + if let Some(client) = client { tools.extend(qualify_tools(filter_tools( client.tools, client.tool_filter, diff --git a/codex-rs/core/src/mentions.rs b/codex-rs/core/src/mentions.rs new file mode 100644 index 0000000000..2f39c10d2f --- /dev/null +++ b/codex-rs/core/src/mentions.rs @@ -0,0 +1,64 @@ +use std::collections::HashMap; +use std::collections::HashSet; +use std::path::PathBuf; + +use codex_protocol::user_input::UserInput; + +use crate::connectors; +use crate::skills::SkillMetadata; +use crate::skills::injection::extract_tool_mentions; + +pub(crate) struct CollectedToolMentions { + pub(crate) plain_names: HashSet, + pub(crate) paths: HashSet, +} + +pub(crate) fn collect_tool_mentions_from_messages(messages: &[String]) -> CollectedToolMentions { + let mut plain_names = HashSet::new(); + let mut paths = HashSet::new(); + for message in messages { + let mentions = extract_tool_mentions(message); + plain_names.extend(mentions.plain_names().map(str::to_string)); + paths.extend(mentions.paths().map(str::to_string)); + } + CollectedToolMentions { plain_names, paths } +} + +pub(crate) fn collect_explicit_app_paths(input: &[UserInput]) -> Vec { + input + .iter() + .filter_map(|item| match item { + UserInput::Mention { path, .. } => Some(path.clone()), + _ => None, + }) + .collect() +} + +pub(crate) fn build_skill_name_counts( + skills: &[SkillMetadata], + disabled_paths: &HashSet, +) -> (HashMap, HashMap) { + let mut exact_counts: HashMap = HashMap::new(); + let mut lower_counts: HashMap = HashMap::new(); + for skill in skills { + if disabled_paths.contains(&skill.path) { + continue; + } + *exact_counts.entry(skill.name.clone()).or_insert(0) += 1; + *lower_counts + .entry(skill.name.to_ascii_lowercase()) + .or_insert(0) += 1; + } + (exact_counts, lower_counts) +} + +pub(crate) fn build_connector_slug_counts( + connectors: &[connectors::AppInfo], +) -> HashMap { + let mut counts: HashMap = HashMap::new(); + for connector in connectors { + let slug = connectors::connector_mention_slug(connector); + *counts.entry(slug).or_insert(0) += 1; + } + counts +} diff --git a/codex-rs/core/src/skills/injection.rs b/codex-rs/core/src/skills/injection.rs index 65ec4dd510..57de457e90 100644 --- a/codex-rs/core/src/skills/injection.rs +++ b/codex-rs/core/src/skills/injection.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::collections::HashSet; use std::path::PathBuf; @@ -67,7 +68,8 @@ fn emit_skill_injected_metric(otel: Option<&OtelManager>, skill: &SkillMetadata, /// Collect explicitly mentioned skills from `$name` text mentions. /// /// Text inputs are scanned once to extract `$skill-name` tokens, then we iterate `skills` -/// in their existing order to preserve prior ordering semantics. +/// in their existing order to preserve prior ordering semantics. Explicit links are +/// resolved by path and plain names are only used when the match is unambiguous. /// /// Complexity: `O(S + T + N_t * S)` time, `O(S)` space, where: /// `S` = number of skills, `T` = total text length, `N_t` = number of text inputs. @@ -75,17 +77,24 @@ pub(crate) fn collect_explicit_skill_mentions( inputs: &[UserInput], skills: &[SkillMetadata], disabled_paths: &HashSet, + skill_name_counts: &HashMap, + connector_slug_counts: &HashMap, ) -> Vec { + let selection_context = SkillSelectionContext { + skills, + disabled_paths, + skill_name_counts, + connector_slug_counts, + }; let mut selected: Vec = Vec::new(); let mut seen_names: HashSet = HashSet::new(); let mut seen_paths: HashSet = HashSet::new(); for input in inputs { if let UserInput::Text { text, .. } = input { - let mentioned_names = extract_skill_mentions(text); + let mentioned_names = extract_tool_mentions(text); select_skills_from_mentions( - skills, - disabled_paths, + &selection_context, &mentioned_names, &mut seen_names, &mut seen_paths, @@ -97,36 +106,95 @@ pub(crate) fn collect_explicit_skill_mentions( selected } -struct SkillMentions<'a> { - names: HashSet<&'a str>, - paths: HashSet<&'a str>, +struct SkillSelectionContext<'a> { + skills: &'a [SkillMetadata], + disabled_paths: &'a HashSet, + skill_name_counts: &'a HashMap, + connector_slug_counts: &'a HashMap, } -impl<'a> SkillMentions<'a> { +pub(crate) struct ToolMentions<'a> { + names: HashSet<&'a str>, + paths: HashSet<&'a str>, + plain_names: HashSet<&'a str>, +} + +impl<'a> ToolMentions<'a> { fn is_empty(&self) -> bool { self.names.is_empty() && self.paths.is_empty() } + + pub(crate) fn plain_names(&self) -> impl Iterator + '_ { + self.plain_names.iter().copied() + } + + pub(crate) fn paths(&self) -> impl Iterator + '_ { + self.paths.iter().copied() + } } -/// Extract `$skill-name` mentions from a single text input. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ToolMentionKind { + App, + Mcp, + Skill, + Other, +} + +const APP_PATH_PREFIX: &str = "app://"; +const MCP_PATH_PREFIX: &str = "mcp://"; +const SKILL_PATH_PREFIX: &str = "skill://"; +const SKILL_FILENAME: &str = "SKILL.md"; + +pub(crate) fn tool_kind_for_path(path: &str) -> ToolMentionKind { + if path.starts_with(APP_PATH_PREFIX) { + ToolMentionKind::App + } else if path.starts_with(MCP_PATH_PREFIX) { + ToolMentionKind::Mcp + } else if path.starts_with(SKILL_PATH_PREFIX) || is_skill_filename(path) { + ToolMentionKind::Skill + } else { + ToolMentionKind::Other + } +} + +fn is_skill_filename(path: &str) -> bool { + let file_name = path.rsplit(['/', '\\']).next().unwrap_or(path); + file_name.eq_ignore_ascii_case(SKILL_FILENAME) +} + +pub(crate) fn app_id_from_path(path: &str) -> Option<&str> { + path.strip_prefix(APP_PATH_PREFIX) + .filter(|value| !value.is_empty()) +} + +pub(crate) fn normalize_skill_path(path: &str) -> &str { + path.strip_prefix(SKILL_PATH_PREFIX).unwrap_or(path) +} + +/// Extract `$tool-name` mentions from a single text input. /// -/// Supports explicit resource links in the form `[$skill-name](resource path)`. When a +/// Supports explicit resource links in the form `[$tool-name](resource path)`. When a /// resource path is present, it is captured for exact path matching while also tracking /// the name for fallback matching. -fn extract_skill_mentions(text: &str) -> SkillMentions<'_> { +pub(crate) fn extract_tool_mentions(text: &str) -> ToolMentions<'_> { let text_bytes = text.as_bytes(); let mut mentioned_names: HashSet<&str> = HashSet::new(); let mut mentioned_paths: HashSet<&str> = HashSet::new(); + let mut plain_names: HashSet<&str> = HashSet::new(); let mut index = 0; while index < text_bytes.len() { let byte = text_bytes[index]; if byte == b'[' && let Some((name, path, end_index)) = - parse_linked_skill_mention(text, text_bytes, index) + parse_linked_tool_mention(text, text_bytes, index) { if !is_common_env_var(name) { - mentioned_names.insert(name); + let kind = tool_kind_for_path(path); + if !matches!(kind, ToolMentionKind::App | ToolMentionKind::Mcp) { + mentioned_names.insert(name); + } mentioned_paths.insert(path); } index = end_index; @@ -143,14 +211,14 @@ fn extract_skill_mentions(text: &str) -> SkillMentions<'_> { index += 1; continue; }; - if !is_skill_name_char(*first_name_byte) { + if !is_mention_name_char(*first_name_byte) { index += 1; continue; } let mut name_end = name_start + 1; while let Some(next_byte) = text_bytes.get(name_end) - && is_skill_name_char(*next_byte) + && is_mention_name_char(*next_byte) { name_end += 1; } @@ -158,21 +226,22 @@ fn extract_skill_mentions(text: &str) -> SkillMentions<'_> { let name = &text[name_start..name_end]; if !is_common_env_var(name) { mentioned_names.insert(name); + plain_names.insert(name); } index = name_end; } - SkillMentions { + ToolMentions { names: mentioned_names, paths: mentioned_paths, + plain_names, } } /// Select mentioned skills while preserving the order of `skills`. fn select_skills_from_mentions( - skills: &[SkillMetadata], - disabled_paths: &HashSet, - mentions: &SkillMentions<'_>, + selection_context: &SkillSelectionContext<'_>, + mentions: &ToolMentions<'_>, seen_names: &mut HashSet, seen_paths: &mut HashSet, selected: &mut Vec, @@ -181,32 +250,65 @@ fn select_skills_from_mentions( return; } - for skill in skills { - if disabled_paths.contains(&skill.path) || seen_paths.contains(&skill.path) { + let mention_skill_paths: HashSet<&str> = mentions + .paths() + .filter(|path| { + !matches!( + tool_kind_for_path(path), + ToolMentionKind::App | ToolMentionKind::Mcp + ) + }) + .map(normalize_skill_path) + .collect(); + + for skill in selection_context.skills { + if selection_context.disabled_paths.contains(&skill.path) + || seen_paths.contains(&skill.path) + { continue; } let path_str = skill.path.to_string_lossy(); - if mentions.paths.contains(path_str.as_ref()) { + if mention_skill_paths.contains(path_str.as_ref()) { seen_paths.insert(skill.path.clone()); seen_names.insert(skill.name.clone()); selected.push(skill.clone()); } } - for skill in skills { - if disabled_paths.contains(&skill.path) || seen_paths.contains(&skill.path) { + for skill in selection_context.skills { + if selection_context.disabled_paths.contains(&skill.path) + || seen_paths.contains(&skill.path) + { continue; } - if mentions.names.contains(skill.name.as_str()) && seen_names.insert(skill.name.clone()) { + if !mentions.plain_names.contains(skill.name.as_str()) { + continue; + } + + let skill_count = selection_context + .skill_name_counts + .get(skill.name.as_str()) + .copied() + .unwrap_or(0); + let connector_count = selection_context + .connector_slug_counts + .get(&skill.name.to_ascii_lowercase()) + .copied() + .unwrap_or(0); + if skill_count != 1 || connector_count != 0 { + continue; + } + + if seen_names.insert(skill.name.clone()) { seen_paths.insert(skill.path.clone()); selected.push(skill.clone()); } } } -fn parse_linked_skill_mention<'a>( +fn parse_linked_tool_mention<'a>( text: &'a str, text_bytes: &[u8], start: usize, @@ -218,13 +320,13 @@ fn parse_linked_skill_mention<'a>( let name_start = dollar_index + 1; let first_name_byte = text_bytes.get(name_start)?; - if !is_skill_name_char(*first_name_byte) { + if !is_mention_name_char(*first_name_byte) { return None; } let mut name_end = name_start + 1; while let Some(next_byte) = text_bytes.get(name_end) - && is_skill_name_char(*next_byte) + && is_mention_name_char(*next_byte) { name_end += 1; } @@ -304,7 +406,7 @@ fn text_mentions_skill(text: &str, skill_name: &str) -> bool { let after_index = name_start + skill_bytes.len(); let after = text_bytes.get(after_index).copied(); - if after.is_none_or(|b| !is_skill_name_char(b)) { + if after.is_none_or(|b| !is_mention_name_char(b)) { return true; } } @@ -312,7 +414,7 @@ fn text_mentions_skill(text: &str, skill_name: &str) -> bool { false } -fn is_skill_name_char(byte: u8) -> bool { +fn is_mention_name_char(byte: u8) -> bool { matches!(byte, b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'_' | b'-') } @@ -320,6 +422,7 @@ fn is_skill_name_char(byte: u8) -> bool { mod tests { use super::*; use pretty_assertions::assert_eq; + use std::collections::HashMap; use std::collections::HashSet; fn make_skill(name: &str, path: &str) -> SkillMetadata { @@ -339,11 +442,41 @@ mod tests { } fn assert_mentions(text: &str, expected_names: &[&str], expected_paths: &[&str]) { - let mentions = extract_skill_mentions(text); + let mentions = extract_tool_mentions(text); assert_eq!(mentions.names, set(expected_names)); assert_eq!(mentions.paths, set(expected_paths)); } + fn build_skill_name_counts( + skills: &[SkillMetadata], + disabled_paths: &HashSet, + ) -> HashMap { + let mut counts = HashMap::new(); + for skill in skills { + if disabled_paths.contains(&skill.path) { + continue; + } + *counts.entry(skill.name.clone()).or_insert(0) += 1; + } + counts + } + + fn collect_mentions( + inputs: &[UserInput], + skills: &[SkillMetadata], + disabled_paths: &HashSet, + connector_slug_counts: &HashMap, + ) -> Vec { + let skill_name_counts = build_skill_name_counts(skills, disabled_paths); + collect_explicit_skill_mentions( + inputs, + skills, + disabled_paths, + &skill_name_counts, + connector_slug_counts, + ) + } + #[test] fn text_mentions_skill_requires_exact_boundary() { assert_eq!( @@ -386,7 +519,7 @@ mod tests { } #[test] - fn extract_skill_mentions_handles_plain_and_linked_mentions() { + fn extract_tool_mentions_handles_plain_and_linked_mentions() { assert_mentions( "use $alpha and [$beta](/tmp/beta)", &["alpha", "beta"], @@ -395,26 +528,26 @@ mod tests { } #[test] - fn extract_skill_mentions_skips_common_env_vars() { + fn extract_tool_mentions_skips_common_env_vars() { assert_mentions("use $PATH and $alpha", &["alpha"], &[]); assert_mentions("use [$HOME](/tmp/skill)", &[], &[]); assert_mentions("use $XDG_CONFIG_HOME and $beta", &["beta"], &[]); } #[test] - fn extract_skill_mentions_requires_link_syntax() { + fn extract_tool_mentions_requires_link_syntax() { assert_mentions("[beta](/tmp/beta)", &[], &[]); assert_mentions("[$beta] /tmp/beta", &["beta"], &[]); assert_mentions("[$beta]()", &["beta"], &[]); } #[test] - fn extract_skill_mentions_trims_linked_paths_and_allows_spacing() { + fn extract_tool_mentions_trims_linked_paths_and_allows_spacing() { assert_mentions("use [$beta] ( /tmp/beta )", &["beta"], &["/tmp/beta"]); } #[test] - fn extract_skill_mentions_stops_at_non_name_chars() { + fn extract_tool_mentions_stops_at_non_name_chars() { assert_mentions( "use $alpha.skill and $beta_extra", &["alpha", "beta_extra"], @@ -431,8 +564,9 @@ mod tests { text: "first $alpha-skill then $beta-skill".to_string(), text_elements: Vec::new(), }]; + let connector_counts = HashMap::new(); - let selected = collect_explicit_skill_mentions(&inputs, &skills, &HashSet::new()); + let selected = collect_mentions(&inputs, &skills, &HashSet::new(), &connector_counts); // Text scanning should not change the previous selection ordering semantics. assert_eq!(selected, vec![beta, alpha]); @@ -453,8 +587,9 @@ mod tests { path: PathBuf::from("/tmp/beta"), }, ]; + let connector_counts = HashMap::new(); - let selected = collect_explicit_skill_mentions(&inputs, &skills, &HashSet::new()); + let selected = collect_mentions(&inputs, &skills, &HashSet::new(), &connector_counts); assert_eq!(selected, vec![alpha]); } @@ -467,25 +602,27 @@ mod tests { text: "use [$alpha-skill](/tmp/alpha) and [$alpha-skill](/tmp/alpha)".to_string(), text_elements: Vec::new(), }]; + let connector_counts = HashMap::new(); - let selected = collect_explicit_skill_mentions(&inputs, &skills, &HashSet::new()); + let selected = collect_mentions(&inputs, &skills, &HashSet::new(), &connector_counts); assert_eq!(selected, vec![alpha]); } #[test] - fn collect_explicit_skill_mentions_dedupes_by_name() { + fn collect_explicit_skill_mentions_skips_ambiguous_name() { let alpha = make_skill("demo-skill", "/tmp/alpha"); let beta = make_skill("demo-skill", "/tmp/beta"); - let skills = vec![alpha.clone(), beta]; + let skills = vec![alpha, beta]; let inputs = vec![UserInput::Text { text: "use $demo-skill and again $demo-skill".to_string(), text_elements: Vec::new(), }]; + let connector_counts = HashMap::new(); - let selected = collect_explicit_skill_mentions(&inputs, &skills, &HashSet::new()); + let selected = collect_mentions(&inputs, &skills, &HashSet::new(), &connector_counts); - assert_eq!(selected, vec![alpha]); + assert_eq!(selected, Vec::new()); } #[test] @@ -497,26 +634,58 @@ mod tests { text: "use $demo-skill and [$demo-skill](/tmp/beta)".to_string(), text_elements: Vec::new(), }]; + let connector_counts = HashMap::new(); - let selected = collect_explicit_skill_mentions(&inputs, &skills, &HashSet::new()); + let selected = collect_mentions(&inputs, &skills, &HashSet::new(), &connector_counts); assert_eq!(selected, vec![beta]); } #[test] - fn collect_explicit_skill_mentions_falls_back_when_linked_path_disabled() { + fn collect_explicit_skill_mentions_skips_plain_name_when_connector_matches() { + let alpha = make_skill("alpha-skill", "/tmp/alpha"); + let skills = vec![alpha]; + let inputs = vec![UserInput::Text { + text: "use $alpha-skill".to_string(), + text_elements: Vec::new(), + }]; + let connector_counts = HashMap::from([("alpha-skill".to_string(), 1)]); + + let selected = collect_mentions(&inputs, &skills, &HashSet::new(), &connector_counts); + + assert_eq!(selected, Vec::new()); + } + + #[test] + fn collect_explicit_skill_mentions_allows_explicit_path_with_connector_conflict() { + let alpha = make_skill("alpha-skill", "/tmp/alpha"); + let skills = vec![alpha.clone()]; + let inputs = vec![UserInput::Text { + text: "use [$alpha-skill](/tmp/alpha)".to_string(), + text_elements: Vec::new(), + }]; + let connector_counts = HashMap::from([("alpha-skill".to_string(), 1)]); + + let selected = collect_mentions(&inputs, &skills, &HashSet::new(), &connector_counts); + + assert_eq!(selected, vec![alpha]); + } + + #[test] + fn collect_explicit_skill_mentions_skips_when_linked_path_disabled() { let alpha = make_skill("demo-skill", "/tmp/alpha"); let beta = make_skill("demo-skill", "/tmp/beta"); - let skills = vec![alpha, beta.clone()]; + let skills = vec![alpha, beta]; let inputs = vec![UserInput::Text { text: "use [$demo-skill](/tmp/alpha)".to_string(), text_elements: Vec::new(), }]; let disabled = HashSet::from([PathBuf::from("/tmp/alpha")]); + let connector_counts = HashMap::new(); - let selected = collect_explicit_skill_mentions(&inputs, &skills, &disabled); + let selected = collect_mentions(&inputs, &skills, &disabled, &connector_counts); - assert_eq!(selected, vec![beta]); + assert_eq!(selected, Vec::new()); } #[test] @@ -528,24 +697,41 @@ mod tests { text: "use [$demo-skill](/tmp/beta)".to_string(), text_elements: Vec::new(), }]; + let connector_counts = HashMap::new(); - let selected = collect_explicit_skill_mentions(&inputs, &skills, &HashSet::new()); + let selected = collect_mentions(&inputs, &skills, &HashSet::new(), &connector_counts); assert_eq!(selected, vec![beta]); } #[test] - fn collect_explicit_skill_mentions_falls_back_to_name_when_path_missing() { + fn collect_explicit_skill_mentions_skips_missing_path_with_no_fallback() { let alpha = make_skill("demo-skill", "/tmp/alpha"); let beta = make_skill("demo-skill", "/tmp/beta"); - let skills = vec![alpha.clone(), beta]; + let skills = vec![alpha, beta]; let inputs = vec![UserInput::Text { text: "use [$demo-skill](/tmp/missing)".to_string(), text_elements: Vec::new(), }]; + let connector_counts = HashMap::new(); - let selected = collect_explicit_skill_mentions(&inputs, &skills, &HashSet::new()); + let selected = collect_mentions(&inputs, &skills, &HashSet::new(), &connector_counts); - assert_eq!(selected, vec![alpha]); + assert_eq!(selected, Vec::new()); + } + + #[test] + fn collect_explicit_skill_mentions_skips_missing_path_without_fallback() { + let alpha = make_skill("demo-skill", "/tmp/alpha"); + let skills = vec![alpha]; + let inputs = vec![UserInput::Text { + text: "use [$demo-skill](/tmp/missing)".to_string(), + text_elements: Vec::new(), + }]; + let connector_counts = HashMap::new(); + + let selected = collect_mentions(&inputs, &skills, &HashSet::new(), &connector_counts); + + assert_eq!(selected, Vec::new()); } } diff --git a/codex-rs/core/src/token_data.rs b/codex-rs/core/src/token_data.rs index 526e240a6c..744cc73d9b 100644 --- a/codex-rs/core/src/token_data.rs +++ b/codex-rs/core/src/token_data.rs @@ -42,6 +42,15 @@ impl IdTokenInfo { PlanType::Unknown(s) => s.clone(), }) } + + pub fn is_workspace_account(&self) -> bool { + matches!( + self.chatgpt_plan_type, + Some(PlanType::Known( + KnownPlan::Team | KnownPlan::Business | KnownPlan::Enterprise | KnownPlan::Edu + )) + ) + } } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -140,6 +149,7 @@ where #[cfg(test)] mod tests { use super::*; + use pretty_assertions::assert_eq; use serde::Serialize; #[test] @@ -200,4 +210,19 @@ mod tests { assert!(info.email.is_none()); assert!(info.get_chatgpt_plan_type().is_none()); } + + #[test] + fn workspace_account_detection_matches_workspace_plans() { + let workspace = IdTokenInfo { + chatgpt_plan_type: Some(PlanType::Known(KnownPlan::Business)), + ..IdTokenInfo::default() + }; + assert_eq!(workspace.is_workspace_account(), true); + + let personal = IdTokenInfo { + chatgpt_plan_type: Some(PlanType::Known(KnownPlan::Pro)), + ..IdTokenInfo::default() + }; + assert_eq!(personal.is_workspace_account(), false); + } } diff --git a/codex-rs/docs/protocol_v1.md b/codex-rs/docs/protocol_v1.md index ee49cac0a6..8cdc5c9575 100644 --- a/codex-rs/docs/protocol_v1.md +++ b/codex-rs/docs/protocol_v1.md @@ -82,6 +82,15 @@ For complete documentation of the `Op` and `EventMsg` variants, refer to [protoc - `EventMsg::TurnComplete` – Contains a `response_id` bookmark for last `response_id` executed by the turn. This can be used to continue the turn at a later point in time, perhaps with additional user input. - `EventMsg::ListSkillsResponse` – Response payload with per-cwd skill entries (`cwd`, `skills`, `errors`) +### UserInput items + +`Op::UserTurn` content items can include: + +- `text` – Plain text plus optional UI text elements. +- `image` / `local_image` – Image inputs. +- `skill` – Explicit skill selection (`name`, `path` to `SKILL.md`). +- `mention` – Explicit app/connector selection (`name`, `path` in `app://{connector_id}` form). + Note: For v1 wire compatibility, `EventMsg::TurnStarted` and `EventMsg::TurnComplete` serialize as `task_started` / `task_complete`. The deserializer accepts both `task_*` and `turn_*` tags. The `response_id` returned from each turn matches the OpenAI `response_id` stored in the API's `/responses` endpoint. It can be stored and used in future `Sessions` to resume threads of work. diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs index 50e8ee342f..759ff5aae4 100644 --- a/codex-rs/protocol/src/models.rs +++ b/codex-rs/protocol/src/models.rs @@ -674,7 +674,7 @@ impl From> for ResponseInputItem { image_index += 1; local_image_content_items_with_label_number(&path, Some(image_index)) } - UserInput::Skill { .. } => Vec::new(), // Skill bodies are injected later in core + UserInput::Skill { .. } | UserInput::Mention { .. } => Vec::new(), // Tool bodies are injected later in core }) .collect::>(), } diff --git a/codex-rs/protocol/src/user_input.rs b/codex-rs/protocol/src/user_input.rs index e9ca09580e..d40511f342 100644 --- a/codex-rs/protocol/src/user_input.rs +++ b/codex-rs/protocol/src/user_input.rs @@ -29,6 +29,8 @@ pub enum UserInput { name: String, path: std::path::PathBuf, }, + /// Explicit mention selected by the user (name + app://connector id). + Mention { name: String, path: String }, } #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, TS, JsonSchema)] diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml index 1d20542365..820fe20e93 100644 --- a/codex-rs/tui/Cargo.toml +++ b/codex-rs/tui/Cargo.toml @@ -30,6 +30,7 @@ codex-ansi-escape = { workspace = true } codex-app-server-protocol = { workspace = true } codex-arg0 = { workspace = true } codex-backend-client = { workspace = true } +codex-chatgpt = { workspace = true } codex-common = { workspace = true, features = [ "cli", "elapsed", diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index e5db8fc6b6..ae1dc4b322 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -1508,6 +1508,21 @@ impl App { )); tui.frame_requester().schedule_frame(); } + AppEvent::OpenAppLink { + title, + description, + instructions, + url, + is_installed, + } => { + self.chat_widget.open_app_link_view( + title, + description, + instructions, + url, + is_installed, + ); + } AppEvent::StartFileSearch(query) => { self.file_search.on_user_query(query); } @@ -1517,6 +1532,9 @@ impl App { AppEvent::RateLimitSnapshotFetched(snapshot) => { self.chat_widget.on_rate_limit_snapshot(Some(snapshot)); } + AppEvent::ConnectorsLoaded(result) => { + self.chat_widget.on_connectors_loaded(result); + } AppEvent::UpdateReasoningEffort(effort) => { self.on_update_reasoning_effort(effort); } diff --git a/codex-rs/tui/src/app_event.rs b/codex-rs/tui/src/app_event.rs index b65d1efb62..bbd228e114 100644 --- a/codex-rs/tui/src/app_event.rs +++ b/codex-rs/tui/src/app_event.rs @@ -10,6 +10,7 @@ use std::path::PathBuf; +use codex_chatgpt::connectors::AppInfo; use codex_common::approval_presets::ApprovalPreset; use codex_core::protocol::Event; use codex_core::protocol::RateLimitSnapshot; @@ -40,6 +41,11 @@ pub(crate) enum WindowsSandboxFallbackReason { ElevationFailed, } +#[derive(Debug, Clone)] +pub(crate) struct ConnectorsSnapshot { + pub(crate) connectors: Vec, +} + #[allow(clippy::large_enum_variant)] #[derive(Debug)] pub(crate) enum AppEvent { @@ -89,9 +95,21 @@ pub(crate) enum AppEvent { /// Result of refreshing rate limits RateLimitSnapshotFetched(RateLimitSnapshot), + /// Result of prefetching connectors. + ConnectorsLoaded(Result), + /// Result of computing a `/diff` command. DiffResult(String), + /// Open the app link view in the bottom pane. + OpenAppLink { + title: String, + description: Option, + instructions: String, + url: String, + is_installed: bool, + }, + InsertHistoryCell(Box), StartCommitAnimation, diff --git a/codex-rs/tui/src/bottom_pane/app_link_view.rs b/codex-rs/tui/src/bottom_pane/app_link_view.rs new file mode 100644 index 0000000000..1f672607a2 --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/app_link_view.rs @@ -0,0 +1,163 @@ +use crossterm::event::KeyCode; +use crossterm::event::KeyEvent; +use ratatui::buffer::Buffer; +use ratatui::layout::Constraint; +use ratatui::layout::Layout; +use ratatui::layout::Rect; +use ratatui::style::Stylize; +use ratatui::text::Line; +use ratatui::widgets::Block; +use ratatui::widgets::Paragraph; +use ratatui::widgets::Widget; +use textwrap::wrap; + +use super::CancellationEvent; +use super::bottom_pane_view::BottomPaneView; +use crate::key_hint; +use crate::render::Insets; +use crate::render::RectExt as _; +use crate::style::user_message_style; +use crate::wrapping::word_wrap_lines; + +pub(crate) struct AppLinkView { + title: String, + description: Option, + instructions: String, + url: String, + is_installed: bool, + complete: bool, +} + +impl AppLinkView { + pub(crate) fn new( + title: String, + description: Option, + instructions: String, + url: String, + is_installed: bool, + ) -> Self { + Self { + title, + description, + instructions, + url, + is_installed, + complete: false, + } + } + + fn content_lines(&self, width: u16) -> Vec> { + let usable_width = width.max(1) as usize; + let mut lines: Vec> = Vec::new(); + + lines.push(Line::from(self.title.clone().bold())); + if let Some(description) = self + .description + .as_deref() + .map(str::trim) + .filter(|description| !description.is_empty()) + { + for line in wrap(description, usable_width) { + lines.push(Line::from(line.into_owned().dim())); + } + } + lines.push(Line::from("")); + if self.is_installed { + for line in wrap("Use $ to insert this app into the prompt.", usable_width) { + lines.push(Line::from(line.into_owned())); + } + lines.push(Line::from("")); + } + + let instructions = self.instructions.trim(); + if !instructions.is_empty() { + for line in wrap(instructions, usable_width) { + lines.push(Line::from(line.into_owned())); + } + for line in wrap( + "Newly installed apps can take a few minutes to appear in /apps.", + usable_width, + ) { + lines.push(Line::from(line.into_owned())); + } + if !self.is_installed { + for line in wrap( + "After installed, use $ to insert this app into the prompt.", + usable_width, + ) { + lines.push(Line::from(line.into_owned())); + } + } + lines.push(Line::from("")); + } + + lines.push(Line::from(vec!["Open:".dim()])); + let url_line = Line::from(vec![self.url.clone().cyan().underlined()]); + lines.extend(word_wrap_lines(vec![url_line], usable_width)); + + lines + } +} + +impl BottomPaneView for AppLinkView { + fn handle_key_event(&mut self, key_event: KeyEvent) { + if let KeyEvent { + code: KeyCode::Esc, .. + } = key_event + { + self.on_ctrl_c(); + } + } + + fn on_ctrl_c(&mut self) -> CancellationEvent { + self.complete = true; + CancellationEvent::Handled + } + + fn is_complete(&self) -> bool { + self.complete + } +} + +impl crate::render::renderable::Renderable for AppLinkView { + fn desired_height(&self, width: u16) -> u16 { + let content_width = width.saturating_sub(4).max(1); + let content_lines = self.content_lines(content_width); + content_lines.len() as u16 + 3 + } + + fn render(&self, area: Rect, buf: &mut Buffer) { + if area.height == 0 || area.width == 0 { + return; + } + + Block::default() + .style(user_message_style()) + .render(area, buf); + + let [content_area, hint_area] = + Layout::vertical([Constraint::Fill(1), Constraint::Length(1)]).areas(area); + let inner = content_area.inset(Insets::vh(1, 2)); + let content_width = inner.width.max(1); + let lines = self.content_lines(content_width); + Paragraph::new(lines).render(inner, buf); + + if hint_area.height > 0 { + let hint_area = Rect { + x: hint_area.x.saturating_add(2), + y: hint_area.y, + width: hint_area.width.saturating_sub(2), + height: hint_area.height, + }; + hint_line().dim().render(hint_area, buf); + } + } +} + +fn hint_line() -> Line<'static> { + Line::from(vec![ + "Press ".into(), + key_hint::plain(KeyCode::Esc).into(), + " to close".into(), + ]) +} diff --git a/codex-rs/tui/src/bottom_pane/chat_composer.rs b/codex-rs/tui/src/bottom_pane/chat_composer.rs index 3a3f5d407b..9d0cbf22e5 100644 --- a/codex-rs/tui/src/bottom_pane/chat_composer.rs +++ b/codex-rs/tui/src/bottom_pane/chat_composer.rs @@ -3,7 +3,7 @@ //! It is responsible for: //! //! - Editing the input buffer (a [`TextArea`]), including placeholder "elements" for attachments. -//! - Routing keys to the active popup (slash commands, file search, skill mentions). +//! - Routing keys to the active popup (slash commands, file search, skill/apps mentions). //! - Handling submit vs newline on Enter. //! - Turning raw key streams into explicit paste operations on platforms where terminals //! don't provide reliable bracketed paste (notably Windows). @@ -124,6 +124,7 @@ use super::footer::single_line_footer_layout; use super::footer::toggle_shortcut_mode; use super::paste_burst::CharDecision; use super::paste_burst::PasteBurst; +use super::skill_popup::MentionItem; use super::skill_popup::SkillPopup; use super::slash_commands; use crate::bottom_pane::paste_burst::FlushResult; @@ -146,6 +147,7 @@ use codex_protocol::user_input::ByteRange; use codex_protocol::user_input::TextElement; use crate::app_event::AppEvent; +use crate::app_event::ConnectorsSnapshot; use crate::app_event_sender::AppEventSender; use crate::bottom_pane::LocalImageAttachment; use crate::bottom_pane::textarea::TextArea; @@ -154,6 +156,8 @@ use crate::clipboard_paste::normalize_pasted_path; use crate::clipboard_paste::pasted_image_format; use crate::history_cell; use crate::ui_consts::LIVE_PREFIX_COLS; +use codex_chatgpt::connectors; +use codex_chatgpt::connectors::AppInfo; use codex_core::skills::model::SkillMetadata; use codex_file_search::FileMatch; use std::cell::RefCell; @@ -276,12 +280,15 @@ pub(crate) struct ChatComposer { context_window_percent: Option, context_window_used_tokens: Option, skills: Option>, - dismissed_skill_popup_token: Option, + connectors_snapshot: Option, + dismissed_mention_popup_token: Option, + mention_paths: HashMap, /// When enabled, `Enter` submits immediately and `Tab` requests queuing behavior. steer_enabled: bool, collaboration_modes_enabled: bool, config: ChatComposerConfig, collaboration_mode_indicator: Option, + connectors_enabled: bool, personality_command_enabled: bool, windows_degraded_sandbox_active: bool, } @@ -363,11 +370,14 @@ impl ChatComposer { context_window_percent: None, context_window_used_tokens: None, skills: None, - dismissed_skill_popup_token: None, + connectors_snapshot: None, + dismissed_mention_popup_token: None, + mention_paths: HashMap::new(), steer_enabled: false, collaboration_modes_enabled: false, config, collaboration_mode_indicator: None, + connectors_enabled: false, personality_command_enabled: false, windows_degraded_sandbox_active: false, }; @@ -380,6 +390,14 @@ impl ChatComposer { self.skills = skills; } + pub fn set_connector_mentions(&mut self, connectors_snapshot: Option) { + self.connectors_snapshot = connectors_snapshot; + } + + pub(crate) fn take_mention_paths(&mut self) -> HashMap { + std::mem::take(&mut self.mention_paths) + } + /// Enables or disables "Steer" behavior for submission keys. /// /// When steer is enabled, `Enter` produces [`InputResult::Submitted`] (send immediately) and @@ -394,6 +412,10 @@ impl ChatComposer { self.collaboration_modes_enabled = enabled; } + pub fn set_connectors_enabled(&mut self, enabled: bool) { + self.connectors_enabled = enabled; + } + pub fn set_collaboration_mode_indicator( &mut self, indicator: Option, @@ -697,6 +719,7 @@ impl ChatComposer { self.textarea.set_text_clearing_elements(""); self.pending_pastes.clear(); self.attached_images.clear(); + self.mention_paths.clear(); self.textarea.set_text_with_elements(&text, &text_elements); @@ -1349,7 +1372,10 @@ impl ChatComposer { unreachable!(); }; - match key_event { + let mut selected_mention: Option<(String, Option)> = None; + let mut close_popup = false; + + let result = match key_event { KeyEvent { code: KeyCode::Up, .. } @@ -1376,8 +1402,8 @@ impl ChatComposer { KeyEvent { code: KeyCode::Esc, .. } => { - if let Some(tok) = self.current_skill_token() { - self.dismissed_skill_popup_token = Some(tok); + if let Some(tok) = self.current_mention_token() { + self.dismissed_mention_popup_token = Some(tok); } self.active_popup = ActivePopup::None; (InputResult::None, true) @@ -1390,15 +1416,26 @@ impl ChatComposer { modifiers: KeyModifiers::NONE, .. } => { - let selected = popup.selected_skill().map(|skill| skill.name.clone()); - if let Some(name) = selected { - self.insert_selected_skill(&name); + if let Some(mention) = popup.selected_mention() { + selected_mention = Some((mention.insert_text.clone(), mention.path.clone())); } - self.active_popup = ActivePopup::None; + close_popup = true; (InputResult::None, true) } input => self.handle_input_basic(input), + }; + + if close_popup { + if let Some((insert_text, path)) = selected_mention { + if let Some(path) = path.as_deref() { + self.record_mention_path(&insert_text, path); + } + self.insert_selected_mention(&insert_text); + } + self.active_popup = ActivePopup::None; } + + result } fn is_image_path(path: &str) -> bool { @@ -1511,14 +1548,23 @@ impl ChatComposer { (rebuilt, rebuilt_elements) } - fn skills_enabled(&self) -> bool { - self.skills.as_ref().is_some_and(|s| !s.is_empty()) - } - pub fn skills(&self) -> Option<&Vec> { self.skills.as_ref() } + fn mentions_enabled(&self) -> bool { + let skills_ready = self + .skills + .as_ref() + .is_some_and(|skills| !skills.is_empty()); + let connectors_ready = self.connectors_enabled + && self + .connectors_snapshot + .as_ref() + .is_some_and(|snapshot| !snapshot.connectors.is_empty()); + skills_ready || connectors_ready + } + /// Extract a token prefixed with `prefix` under the cursor, if any. /// /// The returned string **does not** include the prefix. @@ -1632,8 +1678,8 @@ impl ChatComposer { Self::current_prefixed_token(textarea, '@', false) } - fn current_skill_token(&self) -> Option { - if !self.skills_enabled() { + fn current_mention_token(&self) -> Option { + if !self.mentions_enabled() { return None; } Self::current_prefixed_token(&self.textarea, '$', true) @@ -1691,7 +1737,7 @@ impl ChatComposer { self.textarea.set_cursor(new_cursor); } - fn insert_selected_skill(&mut self, skill_name: &str) { + fn insert_selected_mention(&mut self, insert_text: &str) { let cursor_offset = self.textarea.cursor(); let text = self.textarea.text(); let safe_cursor = Self::clamp_to_char_boundary(text, cursor_offset); @@ -1712,7 +1758,7 @@ impl ChatComposer { .unwrap_or(after_cursor.len()); let end_idx = safe_cursor + end_rel_idx; - let inserted = format!("${skill_name}"); + let inserted = insert_text.to_string(); let mut new_text = String::with_capacity(text.len() - (end_idx - start_idx) + inserted.len() + 1); @@ -1721,12 +1767,35 @@ impl ChatComposer { new_text.push(' '); new_text.push_str(&text[end_idx..]); - // Skill insertion rebuilds plain text, so drop existing elements. + // Mention insertion rebuilds plain text, so drop existing elements. self.textarea.set_text_clearing_elements(&new_text); let new_cursor = start_idx.saturating_add(inserted.len()).saturating_add(1); self.textarea.set_cursor(new_cursor); } + fn record_mention_path(&mut self, insert_text: &str, path: &str) { + let Some(name) = Self::mention_name_from_insert_text(insert_text) else { + return; + }; + self.mention_paths.insert(name, path.to_string()); + } + + fn mention_name_from_insert_text(insert_text: &str) -> Option { + let name = insert_text.strip_prefix('$')?; + if name.is_empty() { + return None; + } + if name + .as_bytes() + .iter() + .all(|byte| is_mention_name_char(*byte)) + { + Some(name.to_string()) + } else { + None + } + } + /// Prepare text for submission/queuing. Returns None if submission should be suppressed. /// On success, clears pending paste payloads because placeholders have been expanded. fn prepare_submission_text(&mut self) -> Option<(String, Vec)> { @@ -1765,6 +1834,7 @@ impl ChatComposer { let is_builtin = slash_commands::find_builtin_command( name, self.collaboration_modes_enabled, + self.connectors_enabled, self.personality_command_enabled, self.windows_degraded_sandbox_active, ) @@ -1951,6 +2021,7 @@ impl ChatComposer { && let Some(cmd) = slash_commands::find_builtin_command( name, self.collaboration_modes_enabled, + self.connectors_enabled, self.personality_command_enabled, self.windows_degraded_sandbox_active, ) @@ -1979,6 +2050,7 @@ impl ChatComposer { && let Some(cmd) = slash_commands::find_builtin_command( name, self.collaboration_modes_enabled, + self.connectors_enabled, self.personality_command_enabled, self.windows_degraded_sandbox_active, ) @@ -2388,10 +2460,10 @@ impl ChatComposer { self.active_popup = ActivePopup::None; return; } - let skill_token = self.current_skill_token(); + let mention_token = self.current_mention_token(); let allow_command_popup = - self.slash_commands_enabled() && file_token.is_none() && skill_token.is_none(); + self.slash_commands_enabled() && file_token.is_none() && mention_token.is_none(); self.sync_command_popup(allow_command_popup); if matches!(self.active_popup, ActivePopup::Command(_)) { @@ -2401,20 +2473,20 @@ impl ChatComposer { self.current_file_query = None; } self.dismissed_file_popup_token = None; - self.dismissed_skill_popup_token = None; + self.dismissed_mention_popup_token = None; return; } - if let Some(token) = skill_token { + if let Some(token) = mention_token { if self.current_file_query.is_some() { self.app_event_tx .send(AppEvent::StartFileSearch(String::new())); self.current_file_query = None; } - self.sync_skill_popup(token); + self.sync_mention_popup(token); return; } - self.dismissed_skill_popup_token = None; + self.dismissed_mention_popup_token = None; if let Some(token) = file_token { self.sync_file_search_popup(token); @@ -2477,6 +2549,7 @@ impl ChatComposer { if slash_commands::has_builtin_prefix( name, self.collaboration_modes_enabled, + self.connectors_enabled, self.personality_command_enabled, self.windows_degraded_sandbox_active, ) { @@ -2529,11 +2602,13 @@ impl ChatComposer { _ => { if is_editing_slash_command_name { let collaboration_modes_enabled = self.collaboration_modes_enabled; + let connectors_enabled = self.connectors_enabled; let personality_command_enabled = self.personality_command_enabled; let mut command_popup = CommandPopup::new( self.custom_prompts.clone(), CommandPopupFlags { collaboration_modes_enabled, + connectors_enabled, personality_command_enabled, windows_degraded_sandbox_active: self.windows_degraded_sandbox_active, }, @@ -2594,32 +2669,99 @@ impl ChatComposer { self.dismissed_file_popup_token = None; } - fn sync_skill_popup(&mut self, query: String) { - if self.dismissed_skill_popup_token.as_ref() == Some(&query) { + fn sync_mention_popup(&mut self, query: String) { + if self.dismissed_mention_popup_token.as_ref() == Some(&query) { return; } - let skills = match self.skills.as_ref() { - Some(skills) if !skills.is_empty() => skills.clone(), - _ => { - self.active_popup = ActivePopup::None; - return; - } - }; + let mentions = self.mention_items(); + if mentions.is_empty() { + self.active_popup = ActivePopup::None; + return; + } match &mut self.active_popup { ActivePopup::Skill(popup) => { popup.set_query(&query); - popup.set_skills(skills); + popup.set_mentions(mentions); } _ => { - let mut popup = SkillPopup::new(skills); + let mut popup = SkillPopup::new(mentions); popup.set_query(&query); self.active_popup = ActivePopup::Skill(popup); } } } + fn mention_items(&self) -> Vec { + let mut mentions = Vec::new(); + + if let Some(skills) = self.skills.as_ref() { + for skill in skills { + let display_name = skill_display_name(skill).to_string(); + let description = skill_description(skill); + let skill_name = skill.name.clone(); + let search_terms = if display_name == skill.name { + vec![skill_name.clone()] + } else { + vec![skill_name.clone(), display_name.clone()] + }; + mentions.push(MentionItem { + display_name, + description, + insert_text: format!("${skill_name}"), + search_terms, + path: Some(skill.path.to_string_lossy().into_owned()), + }); + } + } + + if self.connectors_enabled + && let Some(snapshot) = self.connectors_snapshot.as_ref() + { + for connector in &snapshot.connectors { + if !connector.is_accessible { + continue; + } + let display_name = connectors::connector_display_label(connector); + let description = Some(Self::connector_brief_description(connector)); + let slug = codex_core::connectors::connector_mention_slug(connector); + let search_terms = vec![display_name.clone(), connector.id.clone(), slug.clone()]; + let connector_id = connector.id.as_str(); + mentions.push(MentionItem { + display_name: display_name.clone(), + description, + insert_text: format!("${slug}"), + search_terms, + path: Some(format!("app://{connector_id}")), + }); + } + } + + mentions + } + + fn connector_brief_description(connector: &AppInfo) -> String { + let status_label = if connector.is_accessible { + "Connected" + } else { + "Can be installed" + }; + match Self::connector_description(connector) { + Some(description) => format!("{status_label} - {description}"), + None => status_label.to_string(), + } + } + + fn connector_description(connector: &AppInfo) -> Option { + connector + .description + .as_deref() + .map(str::trim) + .filter(|description| !description.is_empty()) + .map(str::to_string) + } + fn set_has_focus(&mut self, has_focus: bool) { self.has_focus = has_focus; } @@ -2658,6 +2800,29 @@ impl ChatComposer { } } +fn skill_display_name(skill: &SkillMetadata) -> &str { + skill + .interface + .as_ref() + .and_then(|interface| interface.display_name.as_deref()) + .unwrap_or(&skill.name) +} + +fn skill_description(skill: &SkillMetadata) -> Option { + let description = skill + .interface + .as_ref() + .and_then(|interface| interface.short_description.as_deref()) + .or(skill.short_description.as_deref()) + .unwrap_or(&skill.description); + let trimmed = description.trim(); + (!trimmed.is_empty()).then(|| trimmed.to_string()) +} + +fn is_mention_name_char(byte: u8) -> bool { + matches!(byte, b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'_' | b'-') +} + impl Renderable for ChatComposer { fn cursor_pos(&self, area: Rect) -> Option<(u16, u16)> { if !self.input_enabled { @@ -2938,6 +3103,7 @@ mod tests { use tempfile::tempdir; use crate::app_event::AppEvent; + use crate::bottom_pane::AppEventSender; use crate::bottom_pane::ChatComposer; use crate::bottom_pane::InputResult; diff --git a/codex-rs/tui/src/bottom_pane/command_popup.rs b/codex-rs/tui/src/bottom_pane/command_popup.rs index c1cdd5374d..5423231db8 100644 --- a/codex-rs/tui/src/bottom_pane/command_popup.rs +++ b/codex-rs/tui/src/bottom_pane/command_popup.rs @@ -32,6 +32,7 @@ pub(crate) struct CommandPopup { #[derive(Clone, Copy, Debug, Default)] pub(crate) struct CommandPopupFlags { pub(crate) collaboration_modes_enabled: bool, + pub(crate) connectors_enabled: bool, pub(crate) personality_command_enabled: bool, pub(crate) windows_degraded_sandbox_active: bool, } @@ -41,6 +42,7 @@ impl CommandPopup { // Keep built-in availability in sync with the composer. let builtins = slash_commands::builtins_for_input( flags.collaboration_modes_enabled, + flags.connectors_enabled, flags.personality_command_enabled, flags.windows_degraded_sandbox_active, ); @@ -478,6 +480,7 @@ mod tests { Vec::new(), CommandPopupFlags { collaboration_modes_enabled: true, + connectors_enabled: false, personality_command_enabled: true, windows_degraded_sandbox_active: false, }, @@ -496,6 +499,7 @@ mod tests { Vec::new(), CommandPopupFlags { collaboration_modes_enabled: true, + connectors_enabled: false, personality_command_enabled: false, windows_degraded_sandbox_active: false, }, @@ -522,6 +526,7 @@ mod tests { Vec::new(), CommandPopupFlags { collaboration_modes_enabled: true, + connectors_enabled: false, personality_command_enabled: true, windows_degraded_sandbox_active: false, }, diff --git a/codex-rs/tui/src/bottom_pane/mod.rs b/codex-rs/tui/src/bottom_pane/mod.rs index 701e0762cb..d08ccf5068 100644 --- a/codex-rs/tui/src/bottom_pane/mod.rs +++ b/codex-rs/tui/src/bottom_pane/mod.rs @@ -13,8 +13,10 @@ //! //! Some UI is time-based rather than input-based, such as the transient "press again to quit" //! hint. The pane schedules redraws so those hints can expire even when the UI is otherwise idle. +use std::collections::HashMap; use std::path::PathBuf; +use crate::app_event::ConnectorsSnapshot; use crate::app_event_sender::AppEventSender; use crate::bottom_pane::queued_user_messages::QueuedUserMessages; use crate::bottom_pane::unified_exec_footer::UnifiedExecFooter; @@ -36,8 +38,10 @@ use ratatui::buffer::Buffer; use ratatui::layout::Rect; use std::time::Duration; +mod app_link_view; mod approval_overlay; mod request_user_input; +pub(crate) use app_link_view::AppLinkView; pub(crate) use approval_overlay::ApprovalOverlay; pub(crate) use approval_overlay::ApprovalRequest; pub(crate) use request_user_input::RequestUserInputOverlay; @@ -204,6 +208,15 @@ impl BottomPane { self.request_redraw(); } + pub fn set_connectors_snapshot(&mut self, snapshot: Option) { + self.composer.set_connector_mentions(snapshot); + self.request_redraw(); + } + + pub fn take_mention_paths(&mut self) -> HashMap { + self.composer.take_mention_paths() + } + pub fn set_steer_enabled(&mut self, enabled: bool) { self.composer.set_steer_enabled(enabled); } @@ -213,6 +226,10 @@ impl BottomPane { self.request_redraw(); } + pub fn set_connectors_enabled(&mut self, enabled: bool) { + self.composer.set_connectors_enabled(enabled); + } + #[cfg(target_os = "windows")] pub fn set_windows_degraded_sandbox_active(&mut self, enabled: bool) { self.composer.set_windows_degraded_sandbox_active(enabled); diff --git a/codex-rs/tui/src/bottom_pane/skill_popup.rs b/codex-rs/tui/src/bottom_pane/skill_popup.rs index 4736953208..390b44295f 100644 --- a/codex-rs/tui/src/bottom_pane/skill_popup.rs +++ b/codex-rs/tui/src/bottom_pane/skill_popup.rs @@ -14,30 +14,35 @@ use super::selection_popup_common::render_rows_single_line; use crate::key_hint; use crate::render::Insets; use crate::render::RectExt; -use codex_core::skills::model::SkillMetadata; +use crate::text_formatting::truncate_text; +use codex_common::fuzzy_match::fuzzy_match; -use crate::skills_helpers::match_skill; -use crate::skills_helpers::skill_description; -use crate::skills_helpers::skill_display_name; -use crate::skills_helpers::truncated_skill_display_name; +#[derive(Clone, Debug)] +pub(crate) struct MentionItem { + pub(crate) display_name: String, + pub(crate) description: Option, + pub(crate) insert_text: String, + pub(crate) search_terms: Vec, + pub(crate) path: Option, +} pub(crate) struct SkillPopup { query: String, - skills: Vec, + mentions: Vec, state: ScrollState, } impl SkillPopup { - pub(crate) fn new(skills: Vec) -> Self { + pub(crate) fn new(mentions: Vec) -> Self { Self { query: String::new(), - skills, + mentions, state: ScrollState::new(), } } - pub(crate) fn set_skills(&mut self, skills: Vec) { - self.skills = skills; + pub(crate) fn set_mentions(&mut self, mentions: Vec) { + self.mentions = mentions; self.clamp_selection(); } @@ -64,11 +69,11 @@ impl SkillPopup { self.state.ensure_visible(len, MAX_POPUP_ROWS.min(len)); } - pub(crate) fn selected_skill(&self) -> Option<&SkillMetadata> { + pub(crate) fn selected_mention(&self) -> Option<&MentionItem> { let matches = self.filtered_items(); let idx = self.state.selected_idx?; - let skill_idx = matches.get(idx)?; - self.skills.get(*skill_idx) + let mention_idx = matches.get(idx)?; + self.mentions.get(*mention_idx) } fn clamp_selection(&mut self) { @@ -88,14 +93,14 @@ impl SkillPopup { matches .into_iter() .map(|(idx, indices, _score)| { - let skill = &self.skills[idx]; - let name = truncated_skill_display_name(skill); - let description = skill_description(skill).to_string(); + let mention = &self.mentions[idx]; + let name = truncate_text(&mention.display_name, 21); + let description = mention.description.clone().unwrap_or_default(); GenericDisplayRow { name, match_indices: indices, display_shortcut: None, - description: Some(description), + description: Some(description).filter(|desc| !desc.is_empty()), is_disabled: false, disabled_reason: None, wrap_indent: None, @@ -109,23 +114,48 @@ impl SkillPopup { let mut out: Vec<(usize, Option>, i32)> = Vec::new(); if filter.is_empty() { - for (idx, _skill) in self.skills.iter().enumerate() { + for (idx, _mention) in self.mentions.iter().enumerate() { out.push((idx, None, 0)); } return out; } - for (idx, skill) in self.skills.iter().enumerate() { - let display_name = skill_display_name(skill); - if let Some((indices, score)) = match_skill(filter, display_name, &skill.name) { + for (idx, mention) in self.mentions.iter().enumerate() { + let mut best_match: Option<(Option>, i32)> = None; + + if let Some((indices, score)) = fuzzy_match(&mention.display_name, filter) { + best_match = Some((Some(indices), score)); + } + + for term in &mention.search_terms { + if term == &mention.display_name { + continue; + } + + if let Some((_indices, score)) = fuzzy_match(term, filter) { + match best_match.as_mut() { + Some((best_indices, best_score)) => { + if score > *best_score { + *best_score = score; + *best_indices = None; + } + } + None => { + best_match = Some((None, score)); + } + } + } + } + + if let Some((indices, score)) = best_match { out.push((idx, indices, score)); } } out.sort_by(|a, b| { a.2.cmp(&b.2).then_with(|| { - let an = skill_display_name(&self.skills[a.0]); - let bn = skill_display_name(&self.skills[b.0]); + let an = self.mentions[a.0].display_name.as_str(); + let bn = self.mentions[b.0].display_name.as_str(); an.cmp(bn) }) }); @@ -154,7 +184,7 @@ impl WidgetRef for SkillPopup { &rows, &self.state, MAX_POPUP_ROWS, - "no skills", + "no matches", ); if let Some(hint_area) = hint_area { let hint_area = Rect { @@ -172,7 +202,7 @@ fn skill_popup_hint_line() -> Line<'static> { Line::from(vec![ "Press ".into(), key_hint::plain(KeyCode::Enter).into(), - " to select or ".into(), + " to insert or ".into(), key_hint::plain(KeyCode::Esc).into(), " to close".into(), ]) diff --git a/codex-rs/tui/src/bottom_pane/slash_commands.rs b/codex-rs/tui/src/bottom_pane/slash_commands.rs index e0707a4228..1efab04ecd 100644 --- a/codex-rs/tui/src/bottom_pane/slash_commands.rs +++ b/codex-rs/tui/src/bottom_pane/slash_commands.rs @@ -11,6 +11,7 @@ use crate::slash_command::built_in_slash_commands; /// Return the built-ins that should be visible/usable for the current input. pub(crate) fn builtins_for_input( collaboration_modes_enabled: bool, + connectors_enabled: bool, personality_command_enabled: bool, allow_elevate_sandbox: bool, ) -> Vec<(&'static str, SlashCommand)> { @@ -18,6 +19,7 @@ pub(crate) fn builtins_for_input( .into_iter() .filter(|(_, cmd)| allow_elevate_sandbox || *cmd != SlashCommand::ElevateSandbox) .filter(|(_, cmd)| collaboration_modes_enabled || *cmd != SlashCommand::Collab) + .filter(|(_, cmd)| connectors_enabled || *cmd != SlashCommand::Apps) .filter(|(_, cmd)| personality_command_enabled || *cmd != SlashCommand::Personality) .collect() } @@ -26,11 +28,13 @@ pub(crate) fn builtins_for_input( pub(crate) fn find_builtin_command( name: &str, collaboration_modes_enabled: bool, + connectors_enabled: bool, personality_command_enabled: bool, allow_elevate_sandbox: bool, ) -> Option { builtins_for_input( collaboration_modes_enabled, + connectors_enabled, personality_command_enabled, allow_elevate_sandbox, ) @@ -43,11 +47,13 @@ pub(crate) fn find_builtin_command( pub(crate) fn has_builtin_prefix( name: &str, collaboration_modes_enabled: bool, + connectors_enabled: bool, personality_command_enabled: bool, allow_elevate_sandbox: bool, ) -> bool { builtins_for_input( collaboration_modes_enabled, + connectors_enabled, personality_command_enabled, allow_elevate_sandbox, ) diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index dcbb4d7c22..8a0dd81afe 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -31,6 +31,7 @@ use std::time::Instant; use crate::version::CODEX_CLI_VERSION; use codex_app_server_protocol::AuthMode; use codex_backend_client::Client as BackendClient; +use codex_chatgpt::connectors; use codex_core::config::Config; use codex_core::config::ConstraintResult; use codex_core::config::types::Notifications; @@ -131,6 +132,7 @@ const PLAN_IMPLEMENTATION_NO: &str = "No, stay in Plan mode"; const PLAN_IMPLEMENTATION_CODING_MESSAGE: &str = "Implement the plan."; use crate::app_event::AppEvent; +use crate::app_event::ConnectorsSnapshot; use crate::app_event::ExitMode; #[cfg(target_os = "windows")] use crate::app_event::WindowsSandboxEnableMode; @@ -189,7 +191,9 @@ pub(crate) use self::agent::spawn_op_forwarder; mod session_header; use self::session_header::SessionHeader; mod skills; -use self::skills::find_skill_mentions; +use self::skills::collect_tool_mentions; +use self::skills::find_app_mentions; +use self::skills::find_skill_mentions_with_tool_mentions; use crate::streaming::controller::StreamController; use std::path::Path; @@ -386,6 +390,15 @@ enum RateLimitSwitchPromptState { Shown, } +#[derive(Debug, Clone, Default)] +enum ConnectorsCacheState { + #[default] + Uninitialized, + Loading, + Ready(ConnectorsSnapshot), + Failed(String), +} + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub(crate) enum ExternalEditorState { #[default] @@ -460,6 +473,7 @@ pub(crate) struct ChatWidget { /// bottom pane is treated as "running" while this is populated, even if no agent turn is /// currently executing. mcp_startup_status: Option>, + connectors_cache: ConnectorsCacheState, // Queue of interruptive UI events deferred during an active write cycle interrupts: InterruptManager, // Accumulates the current reasoning block text to extract a header @@ -549,6 +563,7 @@ pub(crate) struct UserMessage { text: String, local_images: Vec, text_elements: Vec, + mention_paths: HashMap, } impl From for UserMessage { @@ -558,6 +573,7 @@ impl From for UserMessage { local_images: Vec::new(), // Plain text conversion has no UI element ranges. text_elements: Vec::new(), + mention_paths: HashMap::new(), } } } @@ -569,6 +585,7 @@ impl From<&str> for UserMessage { local_images: Vec::new(), // Plain text conversion has no UI element ranges. text_elements: Vec::new(), + mention_paths: HashMap::new(), } } } @@ -594,6 +611,7 @@ pub(crate) fn create_initial_user_message( text, local_images, text_elements, + mention_paths: HashMap::new(), }) } } @@ -607,12 +625,14 @@ fn remap_placeholders_for_message(message: UserMessage, next_label: &mut usize) text, text_elements, local_images, + mention_paths, } = message; if local_images.is_empty() { return UserMessage { text, text_elements, local_images, + mention_paths, }; } @@ -667,6 +687,7 @@ fn remap_placeholders_for_message(message: UserMessage, next_label: &mut usize) text: rebuilt, local_images: remapped_images, text_elements: rebuilt_elements, + mention_paths, } } @@ -732,6 +753,7 @@ impl ChatWidget { self.bottom_pane .set_history_metadata(event.history_log_id, event.history_entry_count); self.set_skills(None); + self.bottom_pane.set_connectors_snapshot(None); self.thread_id = Some(event.session_id); self.forked_from = event.forked_from_id; self.current_rollout_path = event.rollout_path.clone(); @@ -762,6 +784,9 @@ impl ChatWidget { cwds: Vec::new(), force_reload: true, }); + if self.connectors_enabled() { + self.prefetch_connectors(); + } if let Some(user_message) = self.initial_user_message.take() { self.submit_user_message(user_message); } @@ -797,6 +822,25 @@ impl ChatWidget { self.request_redraw(); } + pub(crate) fn open_app_link_view( + &mut self, + title: String, + description: Option, + instructions: String, + url: String, + is_installed: bool, + ) { + let view = crate::bottom_pane::AppLinkView::new( + title, + description, + instructions, + url, + is_installed, + ); + self.bottom_pane.show_view(Box::new(view)); + self.request_redraw(); + } + pub(crate) fn open_feedback_consent(&mut self, category: crate::app_event::FeedbackCategory) { let params = crate::bottom_pane::feedback_upload_consent_params( self.app_event_tx.clone(), @@ -1240,6 +1284,7 @@ impl ChatWidget { text: self.bottom_pane.composer_text(), text_elements: self.bottom_pane.composer_text_elements(), local_images: self.bottom_pane.composer_local_images(), + mention_paths: HashMap::new(), }; let mut to_merge: Vec = self.queued_user_messages.drain(..).collect(); @@ -1251,6 +1296,7 @@ impl ChatWidget { text: String::new(), text_elements: Vec::new(), local_images: Vec::new(), + mention_paths: HashMap::new(), }; let mut combined_offset = 0usize; let mut next_image_label = 1usize; @@ -1272,6 +1318,7 @@ impl ChatWidget { elem })); combined.local_images.extend(message.local_images); + combined.mention_paths.extend(message.mention_paths); } Some(combined) @@ -2028,6 +2075,7 @@ impl ChatWidget { unified_exec_processes: Vec::new(), agent_turn_running: false, mcp_startup_status: None, + connectors_cache: ConnectorsCacheState::default(), interrupts: InterruptManager::new(), reasoning_buffer: String::new(), full_reasoning_buffer: String::new(), @@ -2071,6 +2119,10 @@ impl ChatWidget { ); widget.update_collaboration_mode_indicator(); + widget + .bottom_pane + .set_connectors_enabled(widget.config.features.enabled(Feature::Apps)); + widget } @@ -2161,6 +2213,7 @@ impl ChatWidget { unified_exec_processes: Vec::new(), agent_turn_running: false, mcp_startup_status: None, + connectors_cache: ConnectorsCacheState::default(), interrupts: InterruptManager::new(), reasoning_buffer: String::new(), full_reasoning_buffer: String::new(), @@ -2287,6 +2340,7 @@ impl ChatWidget { unified_exec_processes: Vec::new(), agent_turn_running: false, mcp_startup_status: None, + connectors_cache: ConnectorsCacheState::default(), interrupts: InterruptManager::new(), reasoning_buffer: String::new(), full_reasoning_buffer: String::new(), @@ -2436,6 +2490,7 @@ impl ChatWidget { .bottom_pane .take_recent_submission_images_with_placeholders(), text_elements, + mention_paths: self.bottom_pane.take_mention_paths(), }; if self.is_session_configured() { // Submitted is only emitted when steer is enabled (Enter sends immediately). @@ -2458,6 +2513,7 @@ impl ChatWidget { .bottom_pane .take_recent_submission_images_with_placeholders(), text_elements, + mention_paths: self.bottom_pane.take_mention_paths(), }; self.queue_user_message(user_message); } @@ -2675,6 +2731,9 @@ impl ChatWidget { SlashCommand::Mcp => { self.add_mcp_output(); } + SlashCommand::Apps => { + self.add_connectors_output(); + } SlashCommand::Rollout => { if let Some(path) = self.rollout_path() { self.add_info_message( @@ -2833,6 +2892,7 @@ impl ChatWidget { text, local_images, text_elements, + mention_paths, } = user_message; if text.is_empty() && local_images.is_empty() { return; @@ -2871,8 +2931,15 @@ impl ChatWidget { }); } + let mentions = collect_tool_mentions(&text, &mention_paths); + let mut skill_names_lower: HashSet = HashSet::new(); + if let Some(skills) = self.bottom_pane.skills() { - let skill_mentions = find_skill_mentions(&text, skills); + skill_names_lower = skills + .iter() + .map(|skill| skill.name.to_ascii_lowercase()) + .collect(); + let skill_mentions = find_skill_mentions_with_tool_mentions(&mentions, skills); for skill in skill_mentions { items.push(UserInput::Skill { name: skill.name.clone(), @@ -2881,6 +2948,17 @@ impl ChatWidget { } } + if let Some(apps) = self.connectors_for_mentions() { + let app_mentions = find_app_mentions(&mentions, apps, &skill_names_lower); + for app in app_mentions { + let app_id = app.id.as_str(); + items.push(UserInput::Mention { + name: app.name.clone(), + path: format!("app://{app_id}"), + }); + } + } + let effective_mode = self.effective_collaboration_mode(); let collaboration_mode = if self.collaboration_modes_enabled() { self.active_collaboration_mask @@ -3289,6 +3367,28 @@ impl ChatWidget { } } + fn prefetch_connectors(&mut self) { + if !self.connectors_enabled() { + return; + } + if matches!(self.connectors_cache, ConnectorsCacheState::Loading) { + return; + } + + self.connectors_cache = ConnectorsCacheState::Loading; + let config = self.config.clone(); + let app_event_tx = self.app_event_tx.clone(); + tokio::spawn(async move { + let result: Result = async { + let connectors = connectors::list_connectors(&config).await?; + Ok(ConnectorsSnapshot { connectors }) + } + .await; + let result = result.map_err(|err| format!("Failed to load apps: {err}")); + app_event_tx.send(AppEvent::ConnectorsLoaded(result)); + }); + } + fn prefetch_rate_limits(&mut self) { self.stop_rate_limit_poller(); @@ -4973,6 +5073,21 @@ impl ChatWidget { self.request_redraw(); } + fn connectors_enabled(&self) -> bool { + self.config.features.enabled(Feature::Apps) + } + + fn connectors_for_mentions(&self) -> Option<&[connectors::AppInfo]> { + if !self.connectors_enabled() { + return None; + } + + match &self.connectors_cache { + ConnectorsCacheState::Ready(snapshot) => Some(snapshot.connectors.as_slice()), + _ => None, + } + } + /// Build a placeholder header cell while the session is configuring. fn placeholder_session_header_cell(config: &Config) -> Box { let placeholder_style = Style::default().add_modifier(Modifier::DIM | Modifier::ITALIC); @@ -5036,6 +5151,152 @@ impl ChatWidget { } } + pub(crate) fn add_connectors_output(&mut self) { + if !self.connectors_enabled() { + self.add_info_message( + "Apps are disabled.".to_string(), + Some("Enable the apps feature to use $ or /apps.".to_string()), + ); + return; + } + + match self.connectors_cache.clone() { + ConnectorsCacheState::Ready(snapshot) => { + if snapshot.connectors.is_empty() { + self.add_info_message("No apps available.".to_string(), None); + } else { + self.open_connectors_popup(&snapshot.connectors); + } + } + ConnectorsCacheState::Failed(err) => { + self.add_to_history(history_cell::new_error_event(err)); + // Retry on demand so `/apps` can recover after transient failures. + self.prefetch_connectors(); + } + ConnectorsCacheState::Loading => { + self.add_to_history(history_cell::new_info_event( + "Apps are still loading.".to_string(), + Some("Try again in a moment.".to_string()), + )); + } + ConnectorsCacheState::Uninitialized => { + self.prefetch_connectors(); + self.add_to_history(history_cell::new_info_event( + "Apps are still loading.".to_string(), + Some("Try again in a moment.".to_string()), + )); + } + } + self.request_redraw(); + } + + fn open_connectors_popup(&mut self, connectors: &[connectors::AppInfo]) { + let total = connectors.len(); + let installed = connectors + .iter() + .filter(|connector| connector.is_accessible) + .count(); + let mut header = ColumnRenderable::new(); + header.push(Line::from("Apps".bold())); + header.push(Line::from( + "Use $ to insert an installed app into your prompt.".dim(), + )); + header.push(Line::from( + format!("Installed {installed} of {total} available apps.").dim(), + )); + let mut items: Vec = Vec::with_capacity(connectors.len()); + for connector in connectors { + let connector_label = connectors::connector_display_label(connector); + let connector_title = connector_label.clone(); + let link_description = Self::connector_description(connector); + let description = Self::connector_brief_description(connector); + let search_value = format!("{connector_label} {}", connector.id); + let mut item = SelectionItem { + name: connector_label, + description: Some(description), + search_value: Some(search_value), + ..Default::default() + }; + let is_installed = connector.is_accessible; + let (selected_label, missing_label, instructions) = if connector.is_accessible { + ( + "Press Enter to view the app link.", + "App link unavailable.", + "Manage this app in your browser.", + ) + } else { + ( + "Press Enter to view the install link.", + "Install link unavailable.", + "Install this app in your browser, then reload Codex.", + ) + }; + if let Some(install_url) = connector.install_url.clone() { + let title = connector_title.clone(); + let instructions = instructions.to_string(); + let description = link_description.clone(); + item.actions = vec![Box::new(move |tx| { + tx.send(AppEvent::OpenAppLink { + title: title.clone(), + description: description.clone(), + instructions: instructions.clone(), + url: install_url.clone(), + is_installed, + }); + })]; + item.dismiss_on_select = true; + item.selected_description = Some(selected_label.to_string()); + } else { + item.actions = vec![Box::new(move |tx| { + tx.send(AppEvent::InsertHistoryCell(Box::new( + history_cell::new_info_event(missing_label.to_string(), None), + ))); + })]; + item.dismiss_on_select = true; + item.selected_description = Some(missing_label.to_string()); + } + items.push(item); + } + + self.bottom_pane.show_selection_view(SelectionViewParams { + header: Box::new(header), + footer_hint: Some(Self::connectors_popup_hint_line()), + items, + is_searchable: true, + search_placeholder: Some("Type to search apps".to_string()), + ..Default::default() + }); + } + + fn connectors_popup_hint_line() -> Line<'static> { + Line::from(vec![ + "Press ".into(), + key_hint::plain(KeyCode::Esc).into(), + " to close.".into(), + ]) + } + + fn connector_brief_description(connector: &connectors::AppInfo) -> String { + let status_label = if connector.is_accessible { + "Connected" + } else { + "Can be installed" + }; + match Self::connector_description(connector) { + Some(description) => format!("{status_label} · {description}"), + None => status_label.to_string(), + } + } + + fn connector_description(connector: &connectors::AppInfo) -> Option { + connector + .description + .as_deref() + .map(str::trim) + .filter(|description| !description.is_empty()) + .map(str::to_string) + } + /// Forward file-search results to the bottom pane. pub(crate) fn apply_file_search_result(&mut self, query: String, matches: Vec) { self.bottom_pane.on_file_search_result(query, matches); @@ -5220,6 +5481,19 @@ impl ChatWidget { self.set_skills_from_response(&ev); } + pub(crate) fn on_connectors_loaded(&mut self, result: Result) { + self.connectors_cache = match result { + Ok(connectors) => ConnectorsCacheState::Ready(connectors), + Err(err) => ConnectorsCacheState::Failed(err), + }; + if let ConnectorsCacheState::Ready(snapshot) = &self.connectors_cache { + self.bottom_pane + .set_connectors_snapshot(Some(snapshot.clone())); + } else { + self.bottom_pane.set_connectors_snapshot(None); + } + } + pub(crate) fn open_review_popup(&mut self) { let mut items: Vec = Vec::new(); diff --git a/codex-rs/tui/src/chatwidget/skills.rs b/codex-rs/tui/src/chatwidget/skills.rs index d6893a9e3c..0920c8837b 100644 --- a/codex-rs/tui/src/chatwidget/skills.rs +++ b/codex-rs/tui/src/chatwidget/skills.rs @@ -12,6 +12,8 @@ use crate::bottom_pane::SkillsToggleView; use crate::bottom_pane::popup_consts::standard_popup_hint_line; use crate::skills_helpers::skill_description; use crate::skills_helpers::skill_display_name; +use codex_chatgpt::connectors::AppInfo; +use codex_core::connectors::connector_mention_slug; use codex_core::protocol::ListSkillsResponseEvent; use codex_core::protocol::SkillMetadata as ProtocolSkillMetadata; use codex_core::protocol::SkillsListEntry; @@ -192,22 +194,256 @@ fn protocol_skill_to_core(skill: &ProtocolSkillMetadata) -> SkillMetadata { } } -pub(crate) fn find_skill_mentions(text: &str, skills: &[SkillMetadata]) -> Vec { - let mut seen: HashSet = HashSet::new(); - let mut matches: Vec = Vec::new(); - for skill in skills { - if seen.contains(&skill.name) { - continue; - } - let needle = format!("${}", skill.name); - if text.contains(&needle) { - seen.insert(skill.name.clone()); - matches.push(skill.clone()); - } - } - matches -} - fn normalize_skill_config_path(path: &Path) -> PathBuf { dunce::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()) } + +pub(crate) fn collect_tool_mentions( + text: &str, + mention_paths: &HashMap, +) -> ToolMentions { + let mut mentions = extract_tool_mentions_from_text(text); + for (name, path) in mention_paths { + if mentions.names.contains(name) { + mentions.linked_paths.insert(name.clone(), path.clone()); + } + } + mentions +} + +pub(crate) fn find_skill_mentions_with_tool_mentions( + mentions: &ToolMentions, + skills: &[SkillMetadata], +) -> Vec { + let mention_skill_paths: HashSet<&str> = mentions + .linked_paths + .values() + .filter(|path| is_skill_path(path)) + .map(|path| normalize_skill_path(path)) + .collect(); + + let mut seen_names = HashSet::new(); + let mut seen_paths = HashSet::new(); + let mut matches: Vec = Vec::new(); + + for skill in skills { + if seen_paths.contains(&skill.path) { + continue; + } + let path_str = skill.path.to_string_lossy(); + if mention_skill_paths.contains(path_str.as_ref()) { + seen_paths.insert(skill.path.clone()); + seen_names.insert(skill.name.clone()); + matches.push(skill.clone()); + } + } + + for skill in skills { + if seen_paths.contains(&skill.path) { + continue; + } + if mentions.names.contains(&skill.name) && seen_names.insert(skill.name.clone()) { + seen_paths.insert(skill.path.clone()); + matches.push(skill.clone()); + } + } + + matches +} + +pub(crate) fn find_app_mentions( + mentions: &ToolMentions, + apps: &[AppInfo], + skill_names_lower: &HashSet, +) -> Vec { + let mut explicit_names = HashSet::new(); + let mut selected_ids = HashSet::new(); + for (name, path) in &mentions.linked_paths { + if let Some(connector_id) = app_id_from_path(path) { + explicit_names.insert(name.clone()); + selected_ids.insert(connector_id.to_string()); + } + } + + let mut slug_counts: HashMap = HashMap::new(); + for app in apps { + let slug = connector_mention_slug(app); + *slug_counts.entry(slug).or_insert(0) += 1; + } + + for app in apps { + let slug = connector_mention_slug(app); + let slug_count = slug_counts.get(&slug).copied().unwrap_or(0); + if mentions.names.contains(&slug) + && !explicit_names.contains(&slug) + && slug_count == 1 + && !skill_names_lower.contains(&slug) + { + selected_ids.insert(app.id.clone()); + } + } + + apps.iter() + .filter(|app| selected_ids.contains(&app.id)) + .cloned() + .collect() +} + +pub(crate) struct ToolMentions { + names: HashSet, + linked_paths: HashMap, +} + +fn extract_tool_mentions_from_text(text: &str) -> ToolMentions { + let text_bytes = text.as_bytes(); + let mut names: HashSet = HashSet::new(); + let mut linked_paths: HashMap = HashMap::new(); + + let mut index = 0; + while index < text_bytes.len() { + let byte = text_bytes[index]; + if byte == b'[' + && let Some((name, path, end_index)) = + parse_linked_tool_mention(text, text_bytes, index) + { + if !is_common_env_var(name) { + if !is_app_or_mcp_path(path) { + names.insert(name.to_string()); + } + linked_paths + .entry(name.to_string()) + .or_insert(path.to_string()); + } + index = end_index; + continue; + } + + if byte != b'$' { + index += 1; + continue; + } + + let name_start = index + 1; + let Some(first_name_byte) = text_bytes.get(name_start) else { + index += 1; + continue; + }; + if !is_mention_name_char(*first_name_byte) { + index += 1; + continue; + } + + let mut name_end = name_start + 1; + while let Some(next_byte) = text_bytes.get(name_end) + && is_mention_name_char(*next_byte) + { + name_end += 1; + } + + let name = &text[name_start..name_end]; + if !is_common_env_var(name) { + names.insert(name.to_string()); + } + index = name_end; + } + + ToolMentions { + names, + linked_paths, + } +} + +fn parse_linked_tool_mention<'a>( + text: &'a str, + text_bytes: &[u8], + start: usize, +) -> Option<(&'a str, &'a str, usize)> { + let dollar_index = start + 1; + if text_bytes.get(dollar_index) != Some(&b'$') { + return None; + } + + let name_start = dollar_index + 1; + let first_name_byte = text_bytes.get(name_start)?; + if !is_mention_name_char(*first_name_byte) { + return None; + } + + let mut name_end = name_start + 1; + while let Some(next_byte) = text_bytes.get(name_end) + && is_mention_name_char(*next_byte) + { + name_end += 1; + } + + if text_bytes.get(name_end) != Some(&b']') { + return None; + } + + let mut path_start = name_end + 1; + while let Some(next_byte) = text_bytes.get(path_start) + && next_byte.is_ascii_whitespace() + { + path_start += 1; + } + if text_bytes.get(path_start) != Some(&b'(') { + return None; + } + + let mut path_end = path_start + 1; + while let Some(next_byte) = text_bytes.get(path_end) + && *next_byte != b')' + { + path_end += 1; + } + if text_bytes.get(path_end) != Some(&b')') { + return None; + } + + let path = text[path_start + 1..path_end].trim(); + if path.is_empty() { + return None; + } + + let name = &text[name_start..name_end]; + Some((name, path, path_end + 1)) +} + +fn is_common_env_var(name: &str) -> bool { + let upper = name.to_ascii_uppercase(); + matches!( + upper.as_str(), + "PATH" + | "HOME" + | "USER" + | "SHELL" + | "PWD" + | "TMPDIR" + | "TEMP" + | "TMP" + | "LANG" + | "TERM" + | "XDG_CONFIG_HOME" + ) +} + +fn is_mention_name_char(byte: u8) -> bool { + matches!(byte, b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'_' | b'-') +} + +fn is_skill_path(path: &str) -> bool { + !is_app_or_mcp_path(path) +} + +fn normalize_skill_path(path: &str) -> &str { + path.strip_prefix("skill://").unwrap_or(path) +} + +fn app_id_from_path(path: &str) -> Option<&str> { + path.strip_prefix("app://") + .filter(|value| !value.is_empty()) +} + +fn is_app_or_mcp_path(path: &str) -> bool { + path.starts_with("app://") || path.starts_with("mcp://") +} diff --git a/codex-rs/tui/src/chatwidget/tests.rs b/codex-rs/tui/src/chatwidget/tests.rs index 78fbc15b49..570a72750b 100644 --- a/codex-rs/tui/src/chatwidget/tests.rs +++ b/codex-rs/tui/src/chatwidget/tests.rs @@ -355,6 +355,7 @@ async fn interrupted_turn_restores_queued_messages_with_images_and_elements() { path: first_images[0].clone(), }], text_elements: first_elements, + mention_paths: HashMap::new(), }); chat.queued_user_messages.push_back(UserMessage { text: second_text, @@ -363,6 +364,7 @@ async fn interrupted_turn_restores_queued_messages_with_images_and_elements() { path: second_images[0].clone(), }], text_elements: second_elements, + mention_paths: HashMap::new(), }); chat.refresh_queued_user_messages(); @@ -442,6 +444,7 @@ async fn remap_placeholders_uses_attachment_labels() { text, text_elements: elements, local_images: attachments, + mention_paths: HashMap::new(), }; let mut next_label = 3usize; let remapped = remap_placeholders_for_message(message, &mut next_label); @@ -502,6 +505,7 @@ async fn remap_placeholders_uses_byte_ranges_when_placeholder_missing() { text, text_elements: elements, local_images: attachments, + mention_paths: HashMap::new(), }; let mut next_label = 3usize; let remapped = remap_placeholders_for_message(message, &mut next_label); @@ -810,6 +814,7 @@ async fn make_chatwidget_manual( unified_exec_processes: Vec::new(), agent_turn_running: false, mcp_startup_status: None, + connectors_cache: ConnectorsCacheState::default(), interrupts: InterruptManager::new(), reasoning_buffer: String::new(), full_reasoning_buffer: String::new(), diff --git a/codex-rs/tui/src/skills_helpers.rs b/codex-rs/tui/src/skills_helpers.rs index 8332be886f..1a8c78bb10 100644 --- a/codex-rs/tui/src/skills_helpers.rs +++ b/codex-rs/tui/src/skills_helpers.rs @@ -26,10 +26,6 @@ pub(crate) fn truncate_skill_name(name: &str) -> String { truncate_text(name, SKILL_NAME_TRUNCATE_LEN) } -pub(crate) fn truncated_skill_display_name(skill: &SkillMetadata) -> String { - truncate_skill_name(skill_display_name(skill)) -} - pub(crate) fn match_skill( filter: &str, display_name: &str, diff --git a/codex-rs/tui/src/slash_command.rs b/codex-rs/tui/src/slash_command.rs index 8a7a320665..5d1aa8dc45 100644 --- a/codex-rs/tui/src/slash_command.rs +++ b/codex-rs/tui/src/slash_command.rs @@ -33,6 +33,7 @@ pub enum SlashCommand { Mention, Status, Mcp, + Apps, Logout, Quit, Exit, @@ -69,6 +70,7 @@ impl SlashCommand { SlashCommand::ElevateSandbox => "set up elevated agent sandbox", SlashCommand::Experimental => "toggle experimental features", SlashCommand::Mcp => "list configured MCP tools", + SlashCommand::Apps => "manage apps", SlashCommand::Logout => "log out of Codex", SlashCommand::Rollout => "print the rollout file path", SlashCommand::TestApproval => "test approval request", @@ -104,6 +106,7 @@ impl SlashCommand { | SlashCommand::Status | SlashCommand::Ps | SlashCommand::Mcp + | SlashCommand::Apps | SlashCommand::Feedback | SlashCommand::Quit | SlashCommand::Exit => true, From 335713f7e98551ee7e8c84d7afb380dddf075ead Mon Sep 17 00:00:00 2001 From: Dylan Hurd Date: Wed, 28 Jan 2026 22:00:48 -0700 Subject: [PATCH 08/28] chore(core) personality under development (#10133) ## Summary Have one or two more changes coming in for this. --- codex-rs/core/src/features.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/codex-rs/core/src/features.rs b/codex-rs/core/src/features.rs index c7cd411145..9bc52537b7 100644 --- a/codex-rs/core/src/features.rs +++ b/codex-rs/core/src/features.rs @@ -552,11 +552,7 @@ pub const FEATURES: &[FeatureSpec] = &[ FeatureSpec { id: Feature::Personality, key: "personality", - stage: Stage::Experimental { - name: "Personality", - menu_description: "Choose a communication style for Codex.", - announcement: "NEW! Update codex's communication style with /personality. Enable in /experimental!", - }, + stage: Stage::UnderDevelopment, default_enabled: false, }, FeatureSpec { From 8cc338aecf73ee1a04917413ee2d9905c468fb4b Mon Sep 17 00:00:00 2001 From: iceweasel-oai Date: Wed, 28 Jan 2026 21:51:51 -0800 Subject: [PATCH 09/28] emit a metric when we can't spawn powershell (#10125) This will help diagnose and measure the impact of a user-reported bug with the elevated sandbox and powershell --- codex-rs/core/src/exec.rs | 75 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/codex-rs/core/src/exec.rs b/codex-rs/core/src/exec.rs index 37f6c2de96..e0d65c8305 100644 --- a/codex-rs/core/src/exec.rs +++ b/codex-rs/core/src/exec.rs @@ -232,6 +232,72 @@ pub(crate) async fn execute_exec_env( finalize_exec_result(raw_output_result, sandbox, duration) } +#[cfg(target_os = "windows")] +fn extract_create_process_as_user_error_code(err: &str) -> Option { + let marker = "CreateProcessAsUserW failed: "; + let start = err.find(marker)? + marker.len(); + let tail = &err[start..]; + let digits: String = tail.chars().take_while(char::is_ascii_digit).collect(); + if digits.is_empty() { + None + } else { + Some(digits) + } +} + +#[cfg(target_os = "windows")] +fn windowsapps_path_kind(path: &str) -> &'static str { + let lower = path.to_ascii_lowercase(); + if lower.contains("\\program files\\windowsapps\\") { + return "windowsapps_package"; + } + if lower.contains("\\appdata\\local\\microsoft\\windowsapps\\") { + return "windowsapps_alias"; + } + if lower.contains("\\windowsapps\\") { + return "windowsapps_other"; + } + "other" +} + +#[cfg(target_os = "windows")] +fn record_windows_sandbox_spawn_failure( + command_path: Option<&str>, + windows_sandbox_level: codex_protocol::config_types::WindowsSandboxLevel, + err: &str, +) { + let Some(error_code) = extract_create_process_as_user_error_code(err) else { + return; + }; + let path = command_path.unwrap_or("unknown"); + let exe = Path::new(path) + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("unknown") + .to_ascii_lowercase(); + let path_kind = windowsapps_path_kind(path); + let level = if matches!( + windows_sandbox_level, + codex_protocol::config_types::WindowsSandboxLevel::Elevated + ) { + "elevated" + } else { + "legacy" + }; + if let Some(metrics) = codex_otel::metrics::global() { + let _ = metrics.counter( + "codex.windows_sandbox.createprocessasuserw_failed", + 1, + &[ + ("error_code", error_code.as_str()), + ("path_kind", path_kind), + ("exe", exe.as_str()), + ("level", level), + ], + ); + } +} + #[cfg(target_os = "windows")] async fn exec_windows_sandbox( params: ExecParams, @@ -265,7 +331,9 @@ async fn exec_windows_sandbox( "windows sandbox: failed to resolve codex_home: {err}" ))) })?; - let use_elevated = matches!(windows_sandbox_level, WindowsSandboxLevel::Elevated); + let command_path = command.first().cloned(); + let sandbox_level = windows_sandbox_level; + let use_elevated = matches!(sandbox_level, WindowsSandboxLevel::Elevated); let spawn_res = tokio::task::spawn_blocking(move || { if use_elevated { run_windows_sandbox_capture_elevated( @@ -294,6 +362,11 @@ async fn exec_windows_sandbox( let capture = match spawn_res { Ok(Ok(v)) => v, Ok(Err(err)) => { + record_windows_sandbox_spawn_failure( + command_path.as_deref(), + sandbox_level, + &err.to_string(), + ); return Err(CodexErr::Io(io::Error::other(format!( "windows sandbox: {err}" )))); From be33de3f8768a47f426761fa6cc017aef9716135 Mon Sep 17 00:00:00 2001 From: Dylan Hurd Date: Wed, 28 Jan 2026 22:51:57 -0700 Subject: [PATCH 10/28] fix(tui) reorder personality command (#10134) ## Summary Reorder it down the list ## Testing - [x] Tests pass --- codex-rs/tui/src/slash_command.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codex-rs/tui/src/slash_command.rs b/codex-rs/tui/src/slash_command.rs index 5d1aa8dc45..9bab314610 100644 --- a/codex-rs/tui/src/slash_command.rs +++ b/codex-rs/tui/src/slash_command.rs @@ -13,7 +13,6 @@ pub enum SlashCommand { // DO NOT ALPHA-SORT! Enum order is presentation order in the popup, so // more frequently used commands should be listed first. Model, - Personality, Approvals, Permissions, #[strum(serialize = "setup-elevated-sandbox")] @@ -40,6 +39,7 @@ pub enum SlashCommand { Feedback, Rollout, Ps, + Personality, TestApproval, } From b4b47630098b6d1ae89f6b2873dd313859b38ec5 Mon Sep 17 00:00:00 2001 From: Dylan Hurd Date: Wed, 28 Jan 2026 22:58:55 -0700 Subject: [PATCH 11/28] fix(ci) missing package.json for shell-mcp-tool (#10135) ## Summary This _should_ be the final place to fix. --- .github/workflows/shell-tool-mcp.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/shell-tool-mcp.yml b/.github/workflows/shell-tool-mcp.yml index 446af67918..39f176cbf4 100644 --- a/.github/workflows/shell-tool-mcp.yml +++ b/.github/workflows/shell-tool-mcp.yml @@ -445,6 +445,9 @@ jobs: id-token: write contents: read steps: + - name: Checkout repository + uses: actions/checkout@v6 + - name: Setup pnpm uses: pnpm/action-setup@v4 with: From e70592f85ad2c86aaa5e45754d9e2dc89a8ab113 Mon Sep 17 00:00:00 2001 From: Josh McKinney Date: Wed, 28 Jan 2026 22:13:53 -0800 Subject: [PATCH 12/28] fix: ignore key release events during onboarding (#10131) ## Summary - guard onboarding key handling to ignore KeyEventKind::Release - handle key events at the onboarding screen boundary to avoid double-triggering widgets ## Related - https://github.com/ratatui/ratatui/issues/347 ## Testing - cd codex-rs && just fmt - cd codex-rs && cargo test -p codex-tui --- codex-rs/tui/src/onboarding/onboarding_screen.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/codex-rs/tui/src/onboarding/onboarding_screen.rs b/codex-rs/tui/src/onboarding/onboarding_screen.rs index 78a7f0ce85..09be507fff 100644 --- a/codex-rs/tui/src/onboarding/onboarding_screen.rs +++ b/codex-rs/tui/src/onboarding/onboarding_screen.rs @@ -211,6 +211,9 @@ impl OnboardingScreen { impl KeyboardHandler for OnboardingScreen { fn handle_key_event(&mut self, key_event: KeyEvent) { + if !matches!(key_event.kind, KeyEventKind::Press | KeyEventKind::Repeat) { + return; + } let is_api_key_entry_active = self.is_api_key_entry_active(); let should_quit = match key_event { KeyEvent { From 4d9ae3a2989949d79b75538ab5769c35547f084c Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 28 Jan 2026 23:31:25 -0800 Subject: [PATCH 13/28] fix: remove references to corepack (#10138) Currently, our `npm publish` logic is failing. There were a number of things that were merged recently that seemed to contribute to this situation, though I think we have fixed most of them, but this one stands out: https://github.com/openai/codex/pull/10115 As best I can tell, we tried to fix the pnpm version to a specific hash, but we did not do it consistently (though `shell-tool-mcp/package.json` had it specified twice...), so for this PR, I ran: ``` $ git ls-files | grep package.json codex-cli/package.json codex-rs/responses-api-proxy/npm/package.json package.json sdk/typescript/package.json shell-tool-mcp/package.json ``` and ensured that all of them now have this line: ```json "packageManager": "pnpm@10.28.2+sha512.41872f037ad22f7348e3b1debbaf7e867cfd448f2726d9cf74c08f19507c31d2c8e7a11525b983febc2df640b5438dee6023ebb1f84ed43cc2d654d2bc326264" ``` I also went and deleted all of the `corepack` stuff that was added by https://github.com/openai/codex/pull/10115. If someone can explain why we need it and verify it does not break `npm publish`, then we can bring it back. --- .github/workflows/shell-tool-mcp-ci.yml | 9 --- .github/workflows/shell-tool-mcp.yml | 18 ----- PNPM.md | 70 ------------------- codex-cli/package.json | 3 +- codex-rs/responses-api-proxy/npm/package.json | 3 +- sdk/typescript/package.json | 3 +- shell-tool-mcp/package.json | 1 - 7 files changed, 6 insertions(+), 101 deletions(-) delete mode 100644 PNPM.md diff --git a/.github/workflows/shell-tool-mcp-ci.yml b/.github/workflows/shell-tool-mcp-ci.yml index bea88ad891..739cc6e409 100644 --- a/.github/workflows/shell-tool-mcp-ci.yml +++ b/.github/workflows/shell-tool-mcp-ci.yml @@ -35,15 +35,6 @@ jobs: node-version: ${{ env.NODE_VERSION }} cache: "pnpm" - - name: Enable Corepack - run: corepack enable - - - name: Activate pnpm from package.json - run: corepack prepare --activate - - - name: Verify pnpm version - run: pnpm --version - - name: Install dependencies run: pnpm install --frozen-lockfile diff --git a/.github/workflows/shell-tool-mcp.yml b/.github/workflows/shell-tool-mcp.yml index 39f176cbf4..d6c85d9ddc 100644 --- a/.github/workflows/shell-tool-mcp.yml +++ b/.github/workflows/shell-tool-mcp.yml @@ -351,15 +351,6 @@ jobs: with: node-version: ${{ env.NODE_VERSION }} - - name: Enable Corepack - run: corepack enable - - - name: Activate pnpm from package.json - run: corepack prepare --activate - - - name: Verify pnpm version - run: pnpm --version - - name: Install JavaScript dependencies run: pnpm install --frozen-lockfile @@ -460,15 +451,6 @@ jobs: registry-url: https://registry.npmjs.org scope: "@openai" - - name: Enable Corepack - run: corepack enable - - - name: Activate pnpm from package.json - run: corepack prepare --activate - - - name: Verify pnpm version - run: pnpm --version - - name: Download npm tarball uses: actions/download-artifact@v7 with: diff --git a/PNPM.md b/PNPM.md deleted file mode 100644 index 9f05e39cf8..0000000000 --- a/PNPM.md +++ /dev/null @@ -1,70 +0,0 @@ -# Migration to pnpm - -This project has been migrated from npm to pnpm to improve dependency management and developer experience. - -## Why pnpm? - -- **Faster installation**: pnpm is significantly faster than npm and yarn -- **Disk space savings**: pnpm uses a content-addressable store to avoid duplication -- **Phantom dependency prevention**: pnpm creates a strict node_modules structure -- **Native workspaces support**: simplified monorepo management - -## How to use pnpm - -### Installation - -```bash -# Global installation of pnpm -npm install -g pnpm@10.28.2 - -# Or with corepack (available with Node.js 22+) -corepack enable -corepack prepare pnpm@10.8.1 --activate -``` - -### Common commands - -| npm command | pnpm equivalent | -| --------------- | ---------------- | -| `npm install` | `pnpm install` | -| `npm run build` | `pnpm run build` | -| `npm test` | `pnpm test` | -| `npm run lint` | `pnpm run lint` | - -### Workspace-specific commands - -| Action | Command | -| ------------------------------------------ | ---------------------------------------- | -| Run a command in a specific package | `pnpm --filter @openai/codex run build` | -| Install a dependency in a specific package | `pnpm --filter @openai/codex add lodash` | -| Run a command in all packages | `pnpm -r run test` | - -## Monorepo structure - -``` -codex/ -├── pnpm-workspace.yaml # Workspace configuration -├── .npmrc # pnpm configuration -├── package.json # Root dependencies and scripts -├── codex-cli/ # Main package -│ └── package.json # codex-cli specific dependencies -└── docs/ # Documentation (future package) -``` - -## Configuration files - -- **pnpm-workspace.yaml**: Defines the packages included in the monorepo -- **.npmrc**: Configures pnpm behavior -- **Root package.json**: Contains shared scripts and dependencies - -## CI/CD - -CI/CD workflows have been updated to use pnpm instead of npm. Make sure your CI environments use pnpm 10.28.2 or higher. - -## Known issues - -If you encounter issues with pnpm, try the following solutions: - -1. Remove the `node_modules` folder and `pnpm-lock.yaml` file, then run `pnpm install` -2. Make sure you're using pnpm 10.28.2 or higher -3. Verify that Node.js 22 or higher is installed diff --git a/codex-cli/package.json b/codex-cli/package.json index b83309e42b..3d1a2dcc2c 100644 --- a/codex-cli/package.json +++ b/codex-cli/package.json @@ -17,5 +17,6 @@ "type": "git", "url": "git+https://github.com/openai/codex.git", "directory": "codex-cli" - } + }, + "packageManager": "pnpm@10.28.2+sha512.41872f037ad22f7348e3b1debbaf7e867cfd448f2726d9cf74c08f19507c31d2c8e7a11525b983febc2df640b5438dee6023ebb1f84ed43cc2d654d2bc326264" } diff --git a/codex-rs/responses-api-proxy/npm/package.json b/codex-rs/responses-api-proxy/npm/package.json index f3956a77d6..d72b60e218 100644 --- a/codex-rs/responses-api-proxy/npm/package.json +++ b/codex-rs/responses-api-proxy/npm/package.json @@ -17,5 +17,6 @@ "type": "git", "url": "git+https://github.com/openai/codex.git", "directory": "codex-rs/responses-api-proxy/npm" - } + }, + "packageManager": "pnpm@10.28.2+sha512.41872f037ad22f7348e3b1debbaf7e867cfd448f2726d9cf74c08f19507c31d2c8e7a11525b983febc2df640b5438dee6023ebb1f84ed43cc2d654d2bc326264" } diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json index 55ecd1abf3..ab29c3fcec 100644 --- a/sdk/typescript/package.json +++ b/sdk/typescript/package.json @@ -62,5 +62,6 @@ "typescript-eslint": "^8.45.0", "zod": "^3.24.2", "zod-to-json-schema": "^3.24.6" - } + }, + "packageManager": "pnpm@10.28.2+sha512.41872f037ad22f7348e3b1debbaf7e867cfd448f2726d9cf74c08f19507c31d2c8e7a11525b983febc2df640b5438dee6023ebb1f84ed43cc2d654d2bc326264" } diff --git a/shell-tool-mcp/package.json b/shell-tool-mcp/package.json index c6dfcd7a55..24798b2f6e 100644 --- a/shell-tool-mcp/package.json +++ b/shell-tool-mcp/package.json @@ -3,7 +3,6 @@ "version": "0.0.0-dev", "description": "Codex MCP server for the shell tool with patched Bash and exec wrappers.", "license": "Apache-2.0", - "packageManager": "pnpm@10.28.2+sha512.41872f037ad22f7348e3b1debbaf7e867cfd448f2726d9cf74c08f19507c31d2c8e7a11525b983febc2df640b5438dee6023ebb1f84ed43cc2d654d2bc326264", "bin": { "codex-shell-tool-mcp": "bin/mcp-server.js" }, From 780482da84e2bb14b9900e75b04ca2e4f65dfddb Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 29 Jan 2026 10:23:03 +0100 Subject: [PATCH 14/28] feat: add log db (#10086) Add a log DB. The goal is just to store our logs in a `.sqlite` DB to make it easier to crawl them and drop the oldest ones. --- codex-rs/Cargo.lock | 2 + codex-rs/core/src/state_db.rs | 1 + codex-rs/state/Cargo.toml | 1 + codex-rs/state/migrations/0002_logs.sql | 14 ++ codex-rs/state/src/lib.rs | 2 + codex-rs/state/src/log_db.rs | 162 ++++++++++++++++++ codex-rs/state/src/model/log.rs | 14 ++ codex-rs/state/src/model/mod.rs | 15 ++ .../{model.rs => model/thread_metadata.rs} | 0 codex-rs/state/src/runtime.rs | 30 ++++ codex-rs/tui/Cargo.toml | 1 + codex-rs/tui/src/lib.rs | 6 + 12 files changed, 248 insertions(+) create mode 100644 codex-rs/state/migrations/0002_logs.sql create mode 100644 codex-rs/state/src/log_db.rs create mode 100644 codex-rs/state/src/model/log.rs create mode 100644 codex-rs/state/src/model/mod.rs rename codex-rs/state/src/{model.rs => model/thread_metadata.rs} (100%) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 08cc850524..0ae6c60d42 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1859,6 +1859,7 @@ dependencies = [ "sqlx", "tokio", "tracing", + "tracing-subscriber", "uuid", ] @@ -1897,6 +1898,7 @@ dependencies = [ "codex-login", "codex-otel", "codex-protocol", + "codex-state", "codex-utils-absolute-path", "codex-utils-cargo-bin", "codex-utils-pty", diff --git a/codex-rs/core/src/state_db.rs b/codex-rs/core/src/state_db.rs index d859328e1c..74c9ed7383 100644 --- a/codex-rs/core/src/state_db.rs +++ b/codex-rs/core/src/state_db.rs @@ -12,6 +12,7 @@ use codex_protocol::ThreadId; use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::SessionSource; use codex_state::DB_METRIC_BACKFILL; +pub use codex_state::LogEntry; use codex_state::STATE_DB_FILENAME; use codex_state::ThreadMetadataBuilder; use serde_json::Value; diff --git a/codex-rs/state/Cargo.toml b/codex-rs/state/Cargo.toml index 810e250ca7..b8c7fce0e3 100644 --- a/codex-rs/state/Cargo.toml +++ b/codex-rs/state/Cargo.toml @@ -14,6 +14,7 @@ serde_json = { workspace = true } sqlx = { workspace = true } tokio = { workspace = true, features = ["fs", "io-util", "macros", "rt-multi-thread", "sync", "time"] } tracing = { workspace = true } +tracing-subscriber = { workspace = true } uuid = { workspace = true } [dev-dependencies] diff --git a/codex-rs/state/migrations/0002_logs.sql b/codex-rs/state/migrations/0002_logs.sql new file mode 100644 index 0000000000..4a193500b9 --- /dev/null +++ b/codex-rs/state/migrations/0002_logs.sql @@ -0,0 +1,14 @@ +CREATE TABLE logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts INTEGER NOT NULL, + ts_nanos INTEGER NOT NULL, + level TEXT NOT NULL, + target TEXT NOT NULL, + message TEXT, + fields_json TEXT NOT NULL, + module_path TEXT, + file TEXT, + line INTEGER +); + +CREATE INDEX idx_logs_ts ON logs(ts DESC, ts_nanos DESC, id DESC); diff --git a/codex-rs/state/src/lib.rs b/codex-rs/state/src/lib.rs index 67533a3791..91ebacea71 100644 --- a/codex-rs/state/src/lib.rs +++ b/codex-rs/state/src/lib.rs @@ -5,11 +5,13 @@ //! orchestration and rollout scanning live in `codex-core`. mod extract; +pub mod log_db; mod migrations; mod model; mod paths; mod runtime; +pub use model::LogEntry; /// Preferred entrypoint: owns configuration and metrics. pub use runtime::StateRuntime; diff --git a/codex-rs/state/src/log_db.rs b/codex-rs/state/src/log_db.rs new file mode 100644 index 0000000000..c03b7977d3 --- /dev/null +++ b/codex-rs/state/src/log_db.rs @@ -0,0 +1,162 @@ +//! Tracing log export into the state SQLite database. +//! +//! This module provides a `tracing_subscriber::Layer` that captures events and +//! inserts them into the `logs` table in `state.sqlite`. The writer runs in a +//! background task and batches inserts to keep logging overhead low. +//! +//! ## Usage +//! +//! ```no_run +//! use codex_state::log_db; +//! use tracing_subscriber::prelude::*; +//! +//! # async fn example(state_db: std::sync::Arc) { +//! let layer = log_db::start(state_db); +//! let _ = tracing_subscriber::registry() +//! .with(layer) +//! .try_init(); +//! # } +//! ``` + +use std::time::Duration; +use std::time::SystemTime; +use std::time::UNIX_EPOCH; + +use serde_json::Value; +use tokio::sync::mpsc; +use tracing::Event; +use tracing::field::Field; +use tracing::field::Visit; +use tracing_subscriber::Layer; +use tracing_subscriber::registry::LookupSpan; + +use crate::LogEntry; +use crate::StateRuntime; + +const LOG_QUEUE_CAPACITY: usize = 512; +const LOG_BATCH_SIZE: usize = 64; +const LOG_FLUSH_INTERVAL: Duration = Duration::from_millis(250); + +pub struct LogDbLayer { + sender: mpsc::Sender, +} + +pub fn start(state_db: std::sync::Arc) -> LogDbLayer { + let (sender, receiver) = mpsc::channel(LOG_QUEUE_CAPACITY); + tokio::spawn(run_inserter(state_db, receiver)); + + LogDbLayer { sender } +} + +impl Layer for LogDbLayer +where + S: tracing::Subscriber + for<'a> LookupSpan<'a>, +{ + fn on_event(&self, event: &Event<'_>, _ctx: tracing_subscriber::layer::Context<'_, S>) { + let metadata = event.metadata(); + let mut visitor = JsonVisitor::default(); + event.record(&mut visitor); + + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_else(|_| Duration::from_secs(0)); + let entry = LogEntry { + ts: now.as_secs() as i64, + ts_nanos: now.subsec_nanos() as i64, + level: metadata.level().as_str().to_string(), + target: metadata.target().to_string(), + message: visitor.message, + fields_json: Value::Object(visitor.fields).to_string(), + module_path: metadata.module_path().map(ToString::to_string), + file: metadata.file().map(ToString::to_string), + line: metadata.line().map(|line| line as i64), + }; + + let _ = self.sender.try_send(entry); + } +} + +async fn run_inserter( + state_db: std::sync::Arc, + mut receiver: mpsc::Receiver, +) { + let mut buffer = Vec::with_capacity(LOG_BATCH_SIZE); + let mut ticker = tokio::time::interval(LOG_FLUSH_INTERVAL); + loop { + tokio::select! { + maybe_entry = receiver.recv() => { + match maybe_entry { + Some(entry) => { + buffer.push(entry); + if buffer.len() >= LOG_BATCH_SIZE { + flush(&state_db, &mut buffer).await; + } + } + None => { + flush(&state_db, &mut buffer).await; + break; + } + } + } + _ = ticker.tick() => { + flush(&state_db, &mut buffer).await; + } + } + } +} + +async fn flush(state_db: &std::sync::Arc, buffer: &mut Vec) { + if buffer.is_empty() { + return; + } + let entries = buffer.split_off(0); + let _ = state_db.insert_logs(entries.as_slice()).await; +} + +#[derive(Default)] +struct JsonVisitor { + fields: serde_json::Map, + message: Option, +} + +impl JsonVisitor { + fn record_value(&mut self, field: &Field, value: Value) { + if field.name() == "message" && self.message.is_none() { + self.message = Some(match &value { + Value::String(message) => message.clone(), + _ => value.to_string(), + }); + } + self.fields.insert(field.name().to_string(), value); + } +} + +impl Visit for JsonVisitor { + fn record_i64(&mut self, field: &Field, value: i64) { + self.record_value(field, Value::from(value)); + } + + fn record_u64(&mut self, field: &Field, value: u64) { + self.record_value(field, Value::from(value)); + } + + fn record_bool(&mut self, field: &Field, value: bool) { + self.record_value(field, Value::from(value)); + } + + fn record_f64(&mut self, field: &Field, value: f64) { + self.record_value(field, Value::from(value)); + } + + fn record_str(&mut self, field: &Field, value: &str) { + self.record_value(field, Value::from(value)); + } + + fn record_error(&mut self, field: &Field, value: &(dyn std::error::Error + 'static)) { + self.record_value(field, Value::from(value.to_string())); + } + + fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { + self.record_value(field, Value::from(format!("{value:?}"))); + } +} diff --git a/codex-rs/state/src/model/log.rs b/codex-rs/state/src/model/log.rs new file mode 100644 index 0000000000..5662a3725c --- /dev/null +++ b/codex-rs/state/src/model/log.rs @@ -0,0 +1,14 @@ +use serde::Serialize; + +#[derive(Clone, Debug, Serialize)] +pub struct LogEntry { + pub ts: i64, + pub ts_nanos: i64, + pub level: String, + pub target: String, + pub message: Option, + pub fields_json: String, + pub module_path: Option, + pub file: Option, + pub line: Option, +} diff --git a/codex-rs/state/src/model/mod.rs b/codex-rs/state/src/model/mod.rs new file mode 100644 index 0000000000..4921e2357a --- /dev/null +++ b/codex-rs/state/src/model/mod.rs @@ -0,0 +1,15 @@ +mod log; +mod thread_metadata; + +pub use log::LogEntry; +pub use thread_metadata::Anchor; +pub use thread_metadata::BackfillStats; +pub use thread_metadata::ExtractionOutcome; +pub use thread_metadata::SortKey; +pub use thread_metadata::ThreadMetadata; +pub use thread_metadata::ThreadMetadataBuilder; +pub use thread_metadata::ThreadsPage; + +pub(crate) use thread_metadata::ThreadRow; +pub(crate) use thread_metadata::anchor_from_item; +pub(crate) use thread_metadata::datetime_to_epoch_seconds; diff --git a/codex-rs/state/src/model.rs b/codex-rs/state/src/model/thread_metadata.rs similarity index 100% rename from codex-rs/state/src/model.rs rename to codex-rs/state/src/model/thread_metadata.rs diff --git a/codex-rs/state/src/runtime.rs b/codex-rs/state/src/runtime.rs index f341ee3bb6..34add1347d 100644 --- a/codex-rs/state/src/runtime.rs +++ b/codex-rs/state/src/runtime.rs @@ -1,4 +1,5 @@ use crate::DB_ERROR_METRIC; +use crate::LogEntry; use crate::SortKey; use crate::ThreadMetadata; use crate::ThreadMetadataBuilder; @@ -201,6 +202,35 @@ FROM threads }) } + /// Insert one log entry into the logs table. + pub async fn insert_log(&self, entry: &LogEntry) -> anyhow::Result<()> { + self.insert_logs(std::slice::from_ref(entry)).await + } + + /// Insert a batch of log entries into the logs table. + pub async fn insert_logs(&self, entries: &[LogEntry]) -> anyhow::Result<()> { + if entries.is_empty() { + return Ok(()); + } + + let mut builder = QueryBuilder::::new( + "INSERT INTO logs (ts, ts_nanos, level, target, message, fields_json, module_path, file, line) ", + ); + builder.push_values(entries, |mut row, entry| { + row.push_bind(entry.ts) + .push_bind(entry.ts_nanos) + .push_bind(&entry.level) + .push_bind(&entry.target) + .push_bind(&entry.message) + .push_bind(&entry.fields_json) + .push_bind(&entry.module_path) + .push_bind(&entry.file) + .push_bind(entry.line); + }); + builder.build().execute(self.pool.as_ref()).await?; + Ok(()) + } + /// List thread ids using the underlying database (no rollout scanning). pub async fn list_thread_ids( &self, diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml index 820fe20e93..6664e7f9c5 100644 --- a/codex-rs/tui/Cargo.toml +++ b/codex-rs/tui/Cargo.toml @@ -42,6 +42,7 @@ codex-file-search = { workspace = true } codex-login = { workspace = true } codex-otel = { workspace = true } codex-protocol = { workspace = true } +codex-state = { workspace = true } codex-utils-absolute-path = { workspace = true } color-eyre = { workspace = true } crossterm = { workspace = true, features = ["bracketed-paste", "event-stream"] } diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index bced1e0653..a48b806129 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -36,6 +36,7 @@ use codex_protocol::config_types::SandboxMode; use codex_protocol::config_types::WindowsSandboxLevel; use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::RolloutLine; +use codex_state::log_db; use codex_utils_absolute_path::AbsolutePathBuf; use cwd_prompt::CwdPromptAction; use cwd_prompt::CwdSelection; @@ -351,10 +352,15 @@ pub async fn run_main( let otel_tracing_layer = otel.as_ref().and_then(|o| o.tracing_layer()); + let log_db_layer = codex_core::state_db::init_if_enabled(&config, None) + .await + .map(|db| log_db::start(db).with_filter(env_filter())); + let _ = tracing_subscriber::registry() .with(file_layer) .with(feedback_layer) .with(feedback_metadata_layer) + .with(log_db_layer) .with(otel_logger_layer) .with(otel_tracing_layer) .try_init(); From 714dc8d8bd1a8d28fe71df4e1c1bc5f3f5752b91 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 29 Jan 2026 10:57:50 +0100 Subject: [PATCH 15/28] feat: async backfill (#10089) --- .../app-server/src/codex_message_processor.rs | 6 +- codex-rs/core/src/rollout/metadata.rs | 22 +++++++- codex-rs/core/src/state_db.rs | 55 ++++++++++++------- codex-rs/tui/src/lib.rs | 2 +- 4 files changed, 58 insertions(+), 27 deletions(-) diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index e86d840b94..e1c982a99d 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -168,7 +168,7 @@ use codex_core::read_head_for_summary; use codex_core::read_session_meta_line; use codex_core::rollout_date_parts; use codex_core::sandboxing::SandboxPermissions; -use codex_core::state_db::{self}; +use codex_core::state_db::get_state_db; use codex_core::windows_sandbox::WindowsSandboxLevelExt; use codex_feedback::CodexFeedback; use codex_login::ServerOptions as LoginServerOptions; @@ -1702,7 +1702,7 @@ impl CodexMessageProcessor { let rollout_path_display = archived_path.display().to_string(); let fallback_provider = self.config.model_provider_id.clone(); - let state_db_ctx = state_db::init_if_enabled(&self.config, None).await; + let state_db_ctx = get_state_db(&self.config, None).await; let archived_folder = self .config .codex_home @@ -3571,7 +3571,7 @@ impl CodexMessageProcessor { } if state_db_ctx.is_none() { - state_db_ctx = state_db::init_if_enabled(&self.config, None).await; + state_db_ctx = get_state_db(&self.config, None).await; } // Move the rollout file to archived. diff --git a/codex-rs/core/src/rollout/metadata.rs b/codex-rs/core/src/rollout/metadata.rs index 32d13ebde2..d08e77e241 100644 --- a/codex-rs/core/src/rollout/metadata.rs +++ b/codex-rs/core/src/rollout/metadata.rs @@ -15,12 +15,14 @@ use codex_protocol::protocol::SessionMetaLine; use codex_protocol::protocol::SessionSource; use codex_state::BackfillStats; use codex_state::DB_ERROR_METRIC; +use codex_state::DB_METRIC_BACKFILL; use codex_state::ExtractionOutcome; use codex_state::ThreadMetadataBuilder; use codex_state::apply_rollout_item; use std::cmp::Reverse; use std::path::Path; use std::path::PathBuf; +use tracing::info; use tracing::warn; const ROLLOUT_PREFIX: &str = "rollout-"; @@ -125,7 +127,7 @@ pub(crate) async fn backfill_sessions( runtime: &codex_state::StateRuntime, config: &Config, otel: Option<&OtelManager>, -) -> BackfillStats { +) { let sessions_root = config.codex_home.join(rollout::SESSIONS_SUBDIR); let archived_root = config.codex_home.join(rollout::ARCHIVED_SESSIONS_SUBDIR); let mut rollout_paths: Vec<(PathBuf, bool)> = Vec::new(); @@ -191,7 +193,23 @@ pub(crate) async fn backfill_sessions( } } } - stats + + info!( + "state db backfill scanned={}, upserted={}, failed={}", + stats.scanned, stats.upserted, stats.failed + ); + if let Some(otel) = otel { + otel.counter( + DB_METRIC_BACKFILL, + stats.upserted as i64, + &[("status", "upserted")], + ); + otel.counter( + DB_METRIC_BACKFILL, + stats.failed as i64, + &[("status", "failed")], + ); + } } async fn file_modified_time_utc(path: &Path) -> Option> { diff --git a/codex-rs/core/src/state_db.rs b/codex-rs/core/src/state_db.rs index 74c9ed7383..7990a017ef 100644 --- a/codex-rs/core/src/state_db.rs +++ b/codex-rs/core/src/state_db.rs @@ -11,7 +11,6 @@ use codex_otel::OtelManager; use codex_protocol::ThreadId; use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::SessionSource; -use codex_state::DB_METRIC_BACKFILL; pub use codex_state::LogEntry; use codex_state::STATE_DB_FILENAME; use codex_state::ThreadMetadataBuilder; @@ -19,15 +18,18 @@ use serde_json::Value; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; -use tracing::info; use tracing::warn; use uuid::Uuid; /// Core-facing handle to the optional SQLite-backed state runtime. pub type StateDbHandle = Arc; -/// Initialize the state runtime when the `sqlite` feature flag is enabled. -pub async fn init_if_enabled(config: &Config, otel: Option<&OtelManager>) -> Option { +/// Initialize the state runtime when the `sqlite` feature flag is enabled. To only be used +/// inside `core`. The initialization should not be done anywhere else. +pub(crate) async fn init_if_enabled( + config: &Config, + otel: Option<&OtelManager>, +) -> Option { let state_path = config.codex_home.join(STATE_DB_FILENAME); if !config.features.enabled(Feature::Sqlite) { // We delete the file on best effort basis to maintain retro-compatibility in the future. @@ -59,27 +61,38 @@ pub async fn init_if_enabled(config: &Config, otel: Option<&OtelManager>) -> Opt } }; if !existed { - let stats = metadata::backfill_sessions(runtime.as_ref(), config, otel).await; - info!( - "state db backfill scanned={}, upserted={}, failed={}", - stats.scanned, stats.upserted, stats.failed - ); - if let Some(otel) = otel { - otel.counter( - DB_METRIC_BACKFILL, - stats.upserted as i64, - &[("status", "upserted")], - ); - otel.counter( - DB_METRIC_BACKFILL, - stats.failed as i64, - &[("status", "failed")], - ); - } + let runtime_for_backfill = Arc::clone(&runtime); + let config_for_backfill = config.clone(); + let otel_for_backfill = otel.cloned(); + tokio::task::spawn(async move { + metadata::backfill_sessions( + runtime_for_backfill.as_ref(), + &config_for_backfill, + otel_for_backfill.as_ref(), + ) + .await; + }); } Some(runtime) } +/// Get the DB if the feature is enabled and the DB exists. +pub async fn get_state_db(config: &Config, otel: Option<&OtelManager>) -> Option { + let state_path = config.codex_home.join(STATE_DB_FILENAME); + if !config.features.enabled(Feature::Sqlite) + || !tokio::fs::try_exists(&state_path).await.unwrap_or(false) + { + return None; + } + codex_state::StateRuntime::init( + config.codex_home.clone(), + config.model_provider_id.clone(), + otel.cloned(), + ) + .await + .ok() +} + /// Open the state runtime when the SQLite file exists, without feature gating. /// /// This is used for parity checks during the SQLite migration phase. diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index a48b806129..c9e69b7125 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -352,7 +352,7 @@ pub async fn run_main( let otel_tracing_layer = otel.as_ref().and_then(|o| o.tracing_layer()); - let log_db_layer = codex_core::state_db::init_if_enabled(&config, None) + let log_db_layer = codex_core::state_db::get_state_db(&config, None) .await .map(|db| log_db::start(db).with_filter(env_filter())); From 6a06726af2469e766528c5f6460e5f047fabd7f3 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 29 Jan 2026 11:11:47 +0100 Subject: [PATCH 16/28] feat: log db client (#10087) ``` just log -h if [ "${1:-}" = "--" ]; then shift; fi; cargo run -p codex-state --bin logs_client -- "$@" Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.21s Running `target/debug/logs_client -h` Tail Codex logs from state.sqlite with simple filters Usage: logs_client [OPTIONS] Options: --codex-home Path to CODEX_HOME. Defaults to $CODEX_HOME or ~/.codex [env: CODEX_HOME=] --db Direct path to the SQLite database. Overrides --codex-home --level Log level to match exactly (case-insensitive) --from Start timestamp (RFC3339 or unix seconds) --to End timestamp (RFC3339 or unix seconds) --module Substring match on module_path --file Substring match on file path --backfill Number of matching rows to show before tailing [default: 200] --poll-ms Poll interval in milliseconds [default: 500] -h, --help Print help ``` --- codex-rs/Cargo.lock | 2 + codex-rs/state/Cargo.toml | 2 + codex-rs/state/src/bin/logs_client.rs | 293 ++++++++++++++++++++++++++ justfile | 4 + 4 files changed, 301 insertions(+) create mode 100644 codex-rs/state/src/bin/logs_client.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 0ae6c60d42..f47e054ba6 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1851,8 +1851,10 @@ version = "0.0.0" dependencies = [ "anyhow", "chrono", + "clap", "codex-otel", "codex-protocol", + "dirs", "pretty_assertions", "serde", "serde_json", diff --git a/codex-rs/state/Cargo.toml b/codex-rs/state/Cargo.toml index b8c7fce0e3..a6e36a3e00 100644 --- a/codex-rs/state/Cargo.toml +++ b/codex-rs/state/Cargo.toml @@ -7,8 +7,10 @@ license.workspace = true [dependencies] anyhow = { workspace = true } chrono = { workspace = true } +clap = { workspace = true, features = ["derive", "env"] } codex-otel = { workspace = true } codex-protocol = { workspace = true } +dirs = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } sqlx = { workspace = true } diff --git a/codex-rs/state/src/bin/logs_client.rs b/codex-rs/state/src/bin/logs_client.rs new file mode 100644 index 0000000000..13bcb1571d --- /dev/null +++ b/codex-rs/state/src/bin/logs_client.rs @@ -0,0 +1,293 @@ +use std::path::Path; +use std::path::PathBuf; +use std::time::Duration; + +use anyhow::Context; +use chrono::DateTime; +use chrono::SecondsFormat; +use chrono::Utc; +use clap::Parser; +use codex_state::STATE_DB_FILENAME; +use dirs::home_dir; +use sqlx::QueryBuilder; +use sqlx::Row; +use sqlx::Sqlite; +use sqlx::SqlitePool; +use sqlx::sqlite::SqliteConnectOptions; +use sqlx::sqlite::SqlitePoolOptions; + +#[derive(Debug, Parser)] +#[command(name = "codex-state-logs")] +#[command(about = "Tail Codex logs from state.sqlite with simple filters")] +struct Args { + /// Path to CODEX_HOME. Defaults to $CODEX_HOME or ~/.codex. + #[arg(long, env = "CODEX_HOME")] + codex_home: Option, + + /// Direct path to the SQLite database. Overrides --codex-home. + #[arg(long)] + db: Option, + + /// Log level to match exactly (case-insensitive). + #[arg(long)] + level: Option, + + /// Start timestamp (RFC3339 or unix seconds). + #[arg(long, value_name = "RFC3339|UNIX")] + from: Option, + + /// End timestamp (RFC3339 or unix seconds). + #[arg(long, value_name = "RFC3339|UNIX")] + to: Option, + + /// Substring match on module_path. + #[arg(long)] + module: Option, + + /// Substring match on file path. + #[arg(long)] + file: Option, + + /// Number of matching rows to show before tailing. + #[arg(long, default_value_t = 200)] + backfill: usize, + + /// Poll interval in milliseconds. + #[arg(long, default_value_t = 500)] + poll_ms: u64, +} + +#[derive(Debug, Clone, sqlx::FromRow)] +struct LogRow { + id: i64, + ts: i64, + ts_nanos: i64, + level: String, + message: Option, + fields_json: String, + module_path: Option, + file: Option, + line: Option, +} + +#[derive(Debug, Clone)] +struct LogFilter { + level_upper: Option, + from_ts: Option, + to_ts: Option, + module_like: Option, + file_like: Option, +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let args = Args::parse(); + let db_path = resolve_db_path(&args)?; + let filter = build_filter(&args)?; + let pool = open_read_only_pool(db_path.as_path()).await?; + + let mut last_id = print_backfill(&pool, &filter, args.backfill).await?; + if last_id == 0 { + last_id = fetch_max_id(&pool, &filter).await?; + } + + let poll_interval = Duration::from_millis(args.poll_ms); + loop { + let rows = fetch_new_rows(&pool, &filter, last_id).await?; + for row in rows { + last_id = last_id.max(row.id); + println!("{}", format_row(&row)); + } + tokio::time::sleep(poll_interval).await; + } +} + +fn resolve_db_path(args: &Args) -> anyhow::Result { + if let Some(db) = args.db.as_ref() { + return Ok(db.clone()); + } + + let codex_home = args.codex_home.clone().unwrap_or_else(default_codex_home); + Ok(codex_home.join(STATE_DB_FILENAME)) +} + +fn default_codex_home() -> PathBuf { + if let Some(home) = home_dir() { + return home.join(".codex"); + } + PathBuf::from(".codex") +} + +fn build_filter(args: &Args) -> anyhow::Result { + let from_ts = args + .from + .as_deref() + .map(parse_timestamp) + .transpose() + .context("failed to parse --from")?; + let to_ts = args + .to + .as_deref() + .map(parse_timestamp) + .transpose() + .context("failed to parse --to")?; + + let level_upper = args.level.as_ref().map(|level| level.to_ascii_uppercase()); + + Ok(LogFilter { + level_upper, + from_ts, + to_ts, + module_like: args.module.clone(), + file_like: args.file.clone(), + }) +} + +fn parse_timestamp(value: &str) -> anyhow::Result { + if let Ok(secs) = value.parse::() { + return Ok(secs); + } + + let dt = DateTime::parse_from_rfc3339(value) + .with_context(|| format!("expected RFC3339 or unix seconds, got {value}"))?; + Ok(dt.timestamp()) +} + +async fn open_read_only_pool(path: &Path) -> anyhow::Result { + let options = SqliteConnectOptions::new() + .filename(path) + .create_if_missing(false) + .read_only(true) + .busy_timeout(Duration::from_secs(5)); + + let display = path.display(); + SqlitePoolOptions::new() + .max_connections(1) + .connect_with(options) + .await + .with_context(|| format!("failed to open sqlite db at {display}")) +} + +async fn print_backfill( + pool: &SqlitePool, + filter: &LogFilter, + backfill: usize, +) -> anyhow::Result { + if backfill == 0 { + return Ok(0); + } + + let mut rows = fetch_backfill(pool, filter, backfill).await?; + rows.reverse(); + + let mut last_id = 0; + for row in rows { + last_id = last_id.max(row.id); + println!("{}", format_row(&row)); + } + Ok(last_id) +} + +async fn fetch_backfill( + pool: &SqlitePool, + filter: &LogFilter, + backfill: usize, +) -> anyhow::Result> { + let mut builder = base_select_builder(); + push_filters(&mut builder, filter); + builder.push(" ORDER BY id DESC"); + builder.push(" LIMIT ").push_bind(backfill as i64); + + builder + .build_query_as::() + .fetch_all(pool) + .await + .context("failed to fetch backfill logs") +} + +async fn fetch_new_rows( + pool: &SqlitePool, + filter: &LogFilter, + last_id: i64, +) -> anyhow::Result> { + let mut builder = base_select_builder(); + push_filters(&mut builder, filter); + builder.push(" AND id > ").push_bind(last_id); + builder.push(" ORDER BY id ASC"); + + builder + .build_query_as::() + .fetch_all(pool) + .await + .context("failed to fetch new logs") +} + +async fn fetch_max_id(pool: &SqlitePool, filter: &LogFilter) -> anyhow::Result { + let mut builder = QueryBuilder::::new("SELECT MAX(id) AS max_id FROM logs WHERE 1 = 1"); + push_filters(&mut builder, filter); + + let row = builder + .build() + .fetch_one(pool) + .await + .context("failed to fetch max log id")?; + let max_id: Option = row.try_get("max_id")?; + Ok(max_id.unwrap_or(0)) +} + +fn base_select_builder<'a>() -> QueryBuilder<'a, Sqlite> { + QueryBuilder::::new( + "SELECT id, ts, ts_nanos, level, message, fields_json, module_path, file, line FROM logs WHERE 1 = 1", + ) +} + +fn push_filters<'a>(builder: &mut QueryBuilder<'a, Sqlite>, filter: &'a LogFilter) { + if let Some(level_upper) = filter.level_upper.as_ref() { + builder + .push(" AND UPPER(level) = ") + .push_bind(level_upper.as_str()); + } + if let Some(from_ts) = filter.from_ts { + builder.push(" AND ts >= ").push_bind(from_ts); + } + if let Some(to_ts) = filter.to_ts { + builder.push(" AND ts <= ").push_bind(to_ts); + } + if let Some(module_like) = filter.module_like.as_ref() { + builder + .push(" AND module_path LIKE '%' || ") + .push_bind(module_like.as_str()) + .push(" || '%'"); + } + if let Some(file_like) = filter.file_like.as_ref() { + builder + .push(" AND file LIKE '%' || ") + .push_bind(file_like.as_str()) + .push(" || '%'"); + } +} + +fn format_row(row: &LogRow) -> String { + let timestamp = format_timestamp(row.ts, row.ts_nanos); + let location = match (&row.file, row.line) { + (Some(file), Some(line)) => format!("{file}:{line}"), + (Some(file), None) => file.clone(), + _ => "-".to_string(), + }; + let module = row.module_path.as_deref().unwrap_or("-"); + let message = row.message.as_deref().unwrap_or(""); + let fields = row.fields_json.as_str(); + let level = row.level.as_str(); + if fields == "{}" || fields.is_empty() { + return format!("{timestamp} {level:<5} [{module}] {location} - {message}"); + } + format!("{timestamp} {level:<5} [{module}] {location} - {message} {fields}") +} + +fn format_timestamp(ts: i64, ts_nanos: i64) -> String { + let nanos = u32::try_from(ts_nanos).unwrap_or(0); + match DateTime::::from_timestamp(ts, nanos) { + Some(dt) => dt.to_rfc3339_opts(SecondsFormat::Millis, true), + None => format!("{ts}.{ts_nanos:09}Z"), + } +} diff --git a/justfile b/justfile index 0e01ef7e1d..8fb5a7d941 100644 --- a/justfile +++ b/justfile @@ -67,3 +67,7 @@ mcp-server-run *args: # Regenerate the json schema for config.toml from the current config types. write-config-schema: cargo run -p codex-core --bin codex-write-config-schema + +# Tail logs from the state SQLite database +log *args: + if [ "${1:-}" = "--" ]; then shift; fi; cargo run -p codex-state --bin logs_client -- "$@" From 4ba911d48c1f5c554bd447b35b42238ef3c1b419 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 29 Jan 2026 11:25:22 +0100 Subject: [PATCH 17/28] chore: improve client (#10149) Screenshot 2026-01-29 at 11 13 12 --- codex-rs/Cargo.lock | 1 + codex-rs/state/Cargo.toml | 1 + codex-rs/state/migrations/0002_logs.sql | 1 - codex-rs/state/src/bin/logs_client.rs | 36 ++++++++++++++++++------- codex-rs/state/src/log_db.rs | 33 +++++++++-------------- codex-rs/state/src/model/log.rs | 1 - codex-rs/state/src/runtime.rs | 3 +-- 7 files changed, 43 insertions(+), 33 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index f47e054ba6..4c321b3f62 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1855,6 +1855,7 @@ dependencies = [ "codex-otel", "codex-protocol", "dirs", + "owo-colors", "pretty_assertions", "serde", "serde_json", diff --git a/codex-rs/state/Cargo.toml b/codex-rs/state/Cargo.toml index a6e36a3e00..6019b5888b 100644 --- a/codex-rs/state/Cargo.toml +++ b/codex-rs/state/Cargo.toml @@ -11,6 +11,7 @@ clap = { workspace = true, features = ["derive", "env"] } codex-otel = { workspace = true } codex-protocol = { workspace = true } dirs = { workspace = true } +owo-colors = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } sqlx = { workspace = true } diff --git a/codex-rs/state/migrations/0002_logs.sql b/codex-rs/state/migrations/0002_logs.sql index 4a193500b9..b9a2c681d4 100644 --- a/codex-rs/state/migrations/0002_logs.sql +++ b/codex-rs/state/migrations/0002_logs.sql @@ -5,7 +5,6 @@ CREATE TABLE logs ( level TEXT NOT NULL, target TEXT NOT NULL, message TEXT, - fields_json TEXT NOT NULL, module_path TEXT, file TEXT, line INTEGER diff --git a/codex-rs/state/src/bin/logs_client.rs b/codex-rs/state/src/bin/logs_client.rs index 13bcb1571d..71acf67e85 100644 --- a/codex-rs/state/src/bin/logs_client.rs +++ b/codex-rs/state/src/bin/logs_client.rs @@ -9,6 +9,7 @@ use chrono::Utc; use clap::Parser; use codex_state::STATE_DB_FILENAME; use dirs::home_dir; +use owo_colors::OwoColorize; use sqlx::QueryBuilder; use sqlx::Row; use sqlx::Sqlite; @@ -64,8 +65,6 @@ struct LogRow { ts_nanos: i64, level: String, message: Option, - fields_json: String, - module_path: Option, file: Option, line: Option, } @@ -237,7 +236,7 @@ async fn fetch_max_id(pool: &SqlitePool, filter: &LogFilter) -> anyhow::Result() -> QueryBuilder<'a, Sqlite> { QueryBuilder::::new( - "SELECT id, ts, ts_nanos, level, message, fields_json, module_path, file, line FROM logs WHERE 1 = 1", + "SELECT id, ts, ts_nanos, level, message, file, line FROM logs WHERE 1 = 1", ) } @@ -269,19 +268,38 @@ fn push_filters<'a>(builder: &mut QueryBuilder<'a, Sqlite>, filter: &'a LogFilte fn format_row(row: &LogRow) -> String { let timestamp = format_timestamp(row.ts, row.ts_nanos); + let level = row.level.as_str(); let location = match (&row.file, row.line) { (Some(file), Some(line)) => format!("{file}:{line}"), (Some(file), None) => file.clone(), _ => "-".to_string(), }; - let module = row.module_path.as_deref().unwrap_or("-"); let message = row.message.as_deref().unwrap_or(""); - let fields = row.fields_json.as_str(); - let level = row.level.as_str(); - if fields == "{}" || fields.is_empty() { - return format!("{timestamp} {level:<5} [{module}] {location} - {message}"); + let level_colored = color_level(level); + let timestamp_colored = timestamp.dimmed().to_string(); + let location_colored = location.dimmed().to_string(); + let message_colored = message.bold().to_string(); + format!("{timestamp_colored} {level_colored} {location_colored} - {message_colored}") +} + +fn color_level(level: &str) -> String { + let padded = format!("{level:<5}"); + if level.eq_ignore_ascii_case("error") { + return padded.red().bold().to_string(); } - format!("{timestamp} {level:<5} [{module}] {location} - {message} {fields}") + if level.eq_ignore_ascii_case("warn") { + return padded.yellow().bold().to_string(); + } + if level.eq_ignore_ascii_case("info") { + return padded.green().bold().to_string(); + } + if level.eq_ignore_ascii_case("debug") { + return padded.blue().bold().to_string(); + } + if level.eq_ignore_ascii_case("trace") { + return padded.magenta().bold().to_string(); + } + padded.bold().to_string() } fn format_timestamp(ts: i64, ts_nanos: i64) -> String { diff --git a/codex-rs/state/src/log_db.rs b/codex-rs/state/src/log_db.rs index c03b7977d3..13f19d0baf 100644 --- a/codex-rs/state/src/log_db.rs +++ b/codex-rs/state/src/log_db.rs @@ -22,7 +22,6 @@ use std::time::Duration; use std::time::SystemTime; use std::time::UNIX_EPOCH; -use serde_json::Value; use tokio::sync::mpsc; use tracing::Event; use tracing::field::Field; @@ -54,7 +53,7 @@ where { fn on_event(&self, event: &Event<'_>, _ctx: tracing_subscriber::layer::Context<'_, S>) { let metadata = event.metadata(); - let mut visitor = JsonVisitor::default(); + let mut visitor = MessageVisitor::default(); event.record(&mut visitor); let now = SystemTime::now() @@ -66,7 +65,6 @@ where level: metadata.level().as_str().to_string(), target: metadata.target().to_string(), message: visitor.message, - fields_json: Value::Object(visitor.fields).to_string(), module_path: metadata.module_path().map(ToString::to_string), file: metadata.file().map(ToString::to_string), line: metadata.line().map(|line| line as i64), @@ -114,49 +112,44 @@ async fn flush(state_db: &std::sync::Arc, buffer: &mut Vec, +struct MessageVisitor { message: Option, } -impl JsonVisitor { - fn record_value(&mut self, field: &Field, value: Value) { +impl MessageVisitor { + fn record_message(&mut self, field: &Field, value: String) { if field.name() == "message" && self.message.is_none() { - self.message = Some(match &value { - Value::String(message) => message.clone(), - _ => value.to_string(), - }); + self.message = Some(value); } - self.fields.insert(field.name().to_string(), value); } } -impl Visit for JsonVisitor { +impl Visit for MessageVisitor { fn record_i64(&mut self, field: &Field, value: i64) { - self.record_value(field, Value::from(value)); + self.record_message(field, value.to_string()); } fn record_u64(&mut self, field: &Field, value: u64) { - self.record_value(field, Value::from(value)); + self.record_message(field, value.to_string()); } fn record_bool(&mut self, field: &Field, value: bool) { - self.record_value(field, Value::from(value)); + self.record_message(field, value.to_string()); } fn record_f64(&mut self, field: &Field, value: f64) { - self.record_value(field, Value::from(value)); + self.record_message(field, value.to_string()); } fn record_str(&mut self, field: &Field, value: &str) { - self.record_value(field, Value::from(value)); + self.record_message(field, value.to_string()); } fn record_error(&mut self, field: &Field, value: &(dyn std::error::Error + 'static)) { - self.record_value(field, Value::from(value.to_string())); + self.record_message(field, value.to_string()); } fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { - self.record_value(field, Value::from(format!("{value:?}"))); + self.record_message(field, format!("{value:?}")); } } diff --git a/codex-rs/state/src/model/log.rs b/codex-rs/state/src/model/log.rs index 5662a3725c..0fa4c0b38e 100644 --- a/codex-rs/state/src/model/log.rs +++ b/codex-rs/state/src/model/log.rs @@ -7,7 +7,6 @@ pub struct LogEntry { pub level: String, pub target: String, pub message: Option, - pub fields_json: String, pub module_path: Option, pub file: Option, pub line: Option, diff --git a/codex-rs/state/src/runtime.rs b/codex-rs/state/src/runtime.rs index 34add1347d..96b2d69122 100644 --- a/codex-rs/state/src/runtime.rs +++ b/codex-rs/state/src/runtime.rs @@ -214,7 +214,7 @@ FROM threads } let mut builder = QueryBuilder::::new( - "INSERT INTO logs (ts, ts_nanos, level, target, message, fields_json, module_path, file, line) ", + "INSERT INTO logs (ts, ts_nanos, level, target, message, module_path, file, line) ", ); builder.push_values(entries, |mut row, entry| { row.push_bind(entry.ts) @@ -222,7 +222,6 @@ FROM threads .push_bind(&entry.level) .push_bind(&entry.target) .push_bind(&entry.message) - .push_bind(&entry.fields_json) .push_bind(&entry.module_path) .push_bind(&entry.file) .push_bind(entry.line); From d29129f352d02458fc69bb06066e1339d46e0e4c Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 29 Jan 2026 15:08:22 +0100 Subject: [PATCH 18/28] nit: update npm (#10161) --- .github/workflows/shell-tool-mcp.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/shell-tool-mcp.yml b/.github/workflows/shell-tool-mcp.yml index d6c85d9ddc..57bfdd5f5b 100644 --- a/.github/workflows/shell-tool-mcp.yml +++ b/.github/workflows/shell-tool-mcp.yml @@ -451,6 +451,10 @@ jobs: registry-url: https://registry.npmjs.org scope: "@openai" + # Trusted publishing requires npm CLI version 11.5.1 or later. + - name: Update npm + run: npm install -g npm@latest + - name: Download npm tarball uses: actions/download-artifact@v7 with: From 2945667dcc7b655fc3bf044f04b69f6cbdf31b69 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 29 Jan 2026 15:46:15 +0100 Subject: [PATCH 19/28] [experimental] nit: try to speed up apt-install (#10163) --- .github/workflows/shell-tool-mcp.yml | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/.github/workflows/shell-tool-mcp.yml b/.github/workflows/shell-tool-mcp.yml index 57bfdd5f5b..3273bb47bc 100644 --- a/.github/workflows/shell-tool-mcp.yml +++ b/.github/workflows/shell-tool-mcp.yml @@ -248,8 +248,32 @@ jobs: run: | set -euo pipefail if command -v apt-get >/dev/null 2>&1; then - apt-get update - DEBIAN_FRONTEND=noninteractive apt-get install -y git build-essential bison autoconf gettext + # On Ubuntu ARM containers, apt metadata for universe/multiverse is large and + # can make `apt-get update` very slow. We only need packages from `main`, so + # trim sources to `main` for faster, more reliable updates. + if [[ "${{ matrix.target }}" == "aarch64-unknown-linux-musl" ]] && [[ -f /etc/os-release ]]; then + . /etc/os-release + if [[ "${ID-}" == "ubuntu" ]]; then + codename="${VERSION_CODENAME:-}" + if [[ -n "$codename" ]]; then + printf '%s\n' \ + "deb http://ports.ubuntu.com/ubuntu-ports ${codename} main" \ + "deb http://ports.ubuntu.com/ubuntu-ports ${codename}-updates main" \ + "deb http://ports.ubuntu.com/ubuntu-ports ${codename}-backports main" \ + "deb http://ports.ubuntu.com/ubuntu-ports ${codename}-security main" \ + >/etc/apt/sources.list + fi + fi + fi + + apt_opts=( + -o Acquire::Retries=3 + -o Acquire::http::Timeout=30 + -o Acquire::https::Timeout=30 + -o Acquire::Languages=none + ) + apt-get "${apt_opts[@]}" update + DEBIAN_FRONTEND=noninteractive apt-get "${apt_opts[@]}" install -y git build-essential bison autoconf gettext elif command -v dnf >/dev/null 2>&1; then dnf install -y git gcc gcc-c++ make bison autoconf gettext elif command -v yum >/dev/null 2>&1; then From b654b7a9aeb8e222e56bbe2814587ca6fafadac5 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 29 Jan 2026 15:59:56 +0100 Subject: [PATCH 20/28] [experimental] nit: try to speed up apt-install 2 (#10164) --- .github/workflows/shell-tool-mcp.yml | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/shell-tool-mcp.yml b/.github/workflows/shell-tool-mcp.yml index 3273bb47bc..ec0c16cd5e 100644 --- a/.github/workflows/shell-tool-mcp.yml +++ b/.github/workflows/shell-tool-mcp.yml @@ -254,14 +254,19 @@ jobs: if [[ "${{ matrix.target }}" == "aarch64-unknown-linux-musl" ]] && [[ -f /etc/os-release ]]; then . /etc/os-release if [[ "${ID-}" == "ubuntu" ]]; then - codename="${VERSION_CODENAME:-}" - if [[ -n "$codename" ]]; then - printf '%s\n' \ - "deb http://ports.ubuntu.com/ubuntu-ports ${codename} main" \ - "deb http://ports.ubuntu.com/ubuntu-ports ${codename}-updates main" \ - "deb http://ports.ubuntu.com/ubuntu-ports ${codename}-backports main" \ - "deb http://ports.ubuntu.com/ubuntu-ports ${codename}-security main" \ - >/etc/apt/sources.list + if [[ -f /etc/apt/sources.list.d/ubuntu.sources ]]; then + # Ubuntu 24.04+ uses deb822 sources; trim components to main. + sed -i -E 's/^Components:.*/Components: main/' /etc/apt/sources.list.d/ubuntu.sources + else + codename="${VERSION_CODENAME:-}" + if [[ -n "$codename" ]]; then + printf '%s\n' \ + "deb http://ports.ubuntu.com/ubuntu-ports ${codename} main" \ + "deb http://ports.ubuntu.com/ubuntu-ports ${codename}-updates main" \ + "deb http://ports.ubuntu.com/ubuntu-ports ${codename}-backports main" \ + "deb http://ports.ubuntu.com/ubuntu-ports ${codename}-security main" \ + >/etc/apt/sources.list + fi fi fi fi From 89c5f3c4d43b174054bd5299d0b507a26c70f465 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 29 Jan 2026 16:53:30 +0100 Subject: [PATCH 21/28] feat: adding thread ID to logs + filter in the client (#10150) --- codex-rs/core/src/codex.rs | 8 +- codex-rs/core/src/shell_snapshot.rs | 28 +++-- codex-rs/core/src/tasks/mod.rs | 41 ++++--- codex-rs/state/migrations/0002_logs.sql | 2 + codex-rs/state/src/bin/logs_client.rs | 20 +++- codex-rs/state/src/log_db.rs | 142 ++++++++++++++++++++++-- codex-rs/state/src/model/log.rs | 1 + codex-rs/state/src/runtime.rs | 3 +- 8 files changed, 203 insertions(+), 42 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index dbc2d5c632..8e1ec64370 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -84,6 +84,7 @@ use tracing::debug; use tracing::error; use tracing::field; use tracing::info; +use tracing::info_span; use tracing::instrument; use tracing::trace_span; use tracing::warn; @@ -353,6 +354,7 @@ impl Codex { let session_source_clone = session_configuration.session_source.clone(); let (agent_status_tx, agent_status_rx) = watch::channel(AgentStatus::PendingInit); + let session_init_span = info_span!("session_init"); let session = Session::new( session_configuration, config.clone(), @@ -366,6 +368,7 @@ impl Codex { skills_manager, agent_control, ) + .instrument(session_init_span) .await .map_err(|e| { error!("Failed to create session: {e:#}"); @@ -374,7 +377,10 @@ impl Codex { let thread_id = session.conversation_id; // This task will run until Op::Shutdown is received. - tokio::spawn(submission_loop(Arc::clone(&session), config, rx_sub)); + let session_loop_span = info_span!("session_loop", thread_id = %thread_id); + tokio::spawn( + submission_loop(Arc::clone(&session), config, rx_sub).instrument(session_loop_span), + ); let codex = Codex { next_id: AtomicU64::new(0), tx_sub, diff --git a/codex-rs/core/src/shell_snapshot.rs b/codex-rs/core/src/shell_snapshot.rs index 78581186c7..d6d738508a 100644 --- a/codex-rs/core/src/shell_snapshot.rs +++ b/codex-rs/core/src/shell_snapshot.rs @@ -19,6 +19,8 @@ use tokio::fs; use tokio::process::Command; use tokio::sync::watch; use tokio::time::timeout; +use tracing::Instrument; +use tracing::info_span; #[derive(Clone, Debug, PartialEq, Eq)] pub struct ShellSnapshot { @@ -42,17 +44,21 @@ impl ShellSnapshot { let snapshot_shell = shell.clone(); let snapshot_session_id = session_id; - tokio::spawn(async move { - let timer = otel_manager.start_timer("codex.shell_snapshot.duration_ms", &[]); - let snapshot = - ShellSnapshot::try_new(&codex_home, snapshot_session_id, &snapshot_shell) - .await - .map(Arc::new); - let success = if snapshot.is_some() { "true" } else { "false" }; - let _ = timer.map(|timer| timer.record(&[("success", success)])); - otel_manager.counter("codex.shell_snapshot", 1, &[("success", success)]); - let _ = shell_snapshot_tx.send(snapshot); - }); + let snapshot_span = info_span!("shell_snapshot", thread_id = %snapshot_session_id); + tokio::spawn( + async move { + let timer = otel_manager.start_timer("codex.shell_snapshot.duration_ms", &[]); + let snapshot = + ShellSnapshot::try_new(&codex_home, snapshot_session_id, &snapshot_shell) + .await + .map(Arc::new); + let success = if snapshot.is_some() { "true" } else { "false" }; + let _ = timer.map(|timer| timer.record(&[("success", success)])); + otel_manager.counter("codex.shell_snapshot", 1, &[("success", success)]); + let _ = shell_snapshot_tx.send(snapshot); + } + .instrument(snapshot_span), + ); } async fn try_new(codex_home: &Path, session_id: ThreadId, shell: &Shell) -> Option { diff --git a/codex-rs/core/src/tasks/mod.rs b/codex-rs/core/src/tasks/mod.rs index 08d23f7987..1f1195236c 100644 --- a/codex-rs/core/src/tasks/mod.rs +++ b/codex-rs/core/src/tasks/mod.rs @@ -13,6 +13,8 @@ use tokio::select; use tokio::sync::Notify; use tokio_util::sync::CancellationToken; use tokio_util::task::AbortOnDropHandle; +use tracing::Instrument; +use tracing::info_span; use tracing::trace; use tracing::warn; @@ -130,25 +132,30 @@ impl Session { let ctx = Arc::clone(&turn_context); let task_for_run = Arc::clone(&task); let task_cancellation_token = cancellation_token.child_token(); - tokio::spawn(async move { - let ctx_for_finish = Arc::clone(&ctx); - let last_agent_message = task_for_run - .run( - Arc::clone(&session_ctx), - ctx, - input, - task_cancellation_token.child_token(), - ) - .await; - session_ctx.clone_session().flush_rollout().await; - if !task_cancellation_token.is_cancelled() { - // Emit completion uniformly from spawn site so all tasks share the same lifecycle. - let sess = session_ctx.clone_session(); - sess.on_task_finished(ctx_for_finish, last_agent_message) + let thread_id = self.conversation_id; + let session_span = info_span!("session_task", thread_id = %thread_id); + tokio::spawn( + async move { + let ctx_for_finish = Arc::clone(&ctx); + let last_agent_message = task_for_run + .run( + Arc::clone(&session_ctx), + ctx, + input, + task_cancellation_token.child_token(), + ) .await; + session_ctx.clone_session().flush_rollout().await; + if !task_cancellation_token.is_cancelled() { + // Emit completion uniformly from spawn site so all tasks share the same lifecycle. + let sess = session_ctx.clone_session(); + sess.on_task_finished(ctx_for_finish, last_agent_message) + .await; + } + done_clone.notify_waiters(); } - done_clone.notify_waiters(); - }) + .instrument(session_span), + ) }; let timer = turn_context diff --git a/codex-rs/state/migrations/0002_logs.sql b/codex-rs/state/migrations/0002_logs.sql index b9a2c681d4..051bef9136 100644 --- a/codex-rs/state/migrations/0002_logs.sql +++ b/codex-rs/state/migrations/0002_logs.sql @@ -5,9 +5,11 @@ CREATE TABLE logs ( level TEXT NOT NULL, target TEXT NOT NULL, message TEXT, + thread_id TEXT, module_path TEXT, file TEXT, line INTEGER ); CREATE INDEX idx_logs_ts ON logs(ts DESC, ts_nanos DESC, id DESC); +CREATE INDEX idx_logs_thread_id ON logs(thread_id); diff --git a/codex-rs/state/src/bin/logs_client.rs b/codex-rs/state/src/bin/logs_client.rs index 71acf67e85..df57856115 100644 --- a/codex-rs/state/src/bin/logs_client.rs +++ b/codex-rs/state/src/bin/logs_client.rs @@ -49,6 +49,10 @@ struct Args { #[arg(long)] file: Option, + /// Match a specific thread id. + #[arg(long)] + thread_id: Option, + /// Number of matching rows to show before tailing. #[arg(long, default_value_t = 200)] backfill: usize, @@ -65,6 +69,7 @@ struct LogRow { ts_nanos: i64, level: String, message: Option, + thread_id: Option, file: Option, line: Option, } @@ -76,6 +81,7 @@ struct LogFilter { to_ts: Option, module_like: Option, file_like: Option, + thread_id: Option, } #[tokio::main] @@ -139,6 +145,7 @@ fn build_filter(args: &Args) -> anyhow::Result { to_ts, module_like: args.module.clone(), file_like: args.file.clone(), + thread_id: args.thread_id.clone(), }) } @@ -236,7 +243,7 @@ async fn fetch_max_id(pool: &SqlitePool, filter: &LogFilter) -> anyhow::Result() -> QueryBuilder<'a, Sqlite> { QueryBuilder::::new( - "SELECT id, ts, ts_nanos, level, message, file, line FROM logs WHERE 1 = 1", + "SELECT id, ts, ts_nanos, level, message, thread_id, file, line FROM logs WHERE 1 = 1", ) } @@ -264,6 +271,11 @@ fn push_filters<'a>(builder: &mut QueryBuilder<'a, Sqlite>, filter: &'a LogFilte .push_bind(file_like.as_str()) .push(" || '%'"); } + if let Some(thread_id) = filter.thread_id.as_ref() { + builder + .push(" AND thread_id = ") + .push_bind(thread_id.as_str()); + } } fn format_row(row: &LogRow) -> String { @@ -277,9 +289,13 @@ fn format_row(row: &LogRow) -> String { let message = row.message.as_deref().unwrap_or(""); let level_colored = color_level(level); let timestamp_colored = timestamp.dimmed().to_string(); + let thread_id = row.thread_id.as_deref().unwrap_or("-"); + let thread_id_colored = thread_id.yellow().to_string(); let location_colored = location.dimmed().to_string(); let message_colored = message.bold().to_string(); - format!("{timestamp_colored} {level_colored} {location_colored} - {message_colored}") + format!( + "{timestamp_colored} {level_colored} [{thread_id_colored}] {location_colored} - {message_colored}" + ) } fn color_level(level: &str) -> String { diff --git a/codex-rs/state/src/log_db.rs b/codex-rs/state/src/log_db.rs index 13f19d0baf..a62870974c 100644 --- a/codex-rs/state/src/log_db.rs +++ b/codex-rs/state/src/log_db.rs @@ -26,6 +26,9 @@ use tokio::sync::mpsc; use tracing::Event; use tracing::field::Field; use tracing::field::Visit; +use tracing::span::Attributes; +use tracing::span::Id; +use tracing::span::Record; use tracing_subscriber::Layer; use tracing_subscriber::registry::LookupSpan; @@ -51,10 +54,55 @@ impl Layer for LogDbLayer where S: tracing::Subscriber + for<'a> LookupSpan<'a>, { - fn on_event(&self, event: &Event<'_>, _ctx: tracing_subscriber::layer::Context<'_, S>) { + fn on_new_span( + &self, + attrs: &Attributes<'_>, + id: &Id, + ctx: tracing_subscriber::layer::Context<'_, S>, + ) { + let mut visitor = SpanFieldVisitor::default(); + attrs.record(&mut visitor); + + if let Some(span) = ctx.span(id) { + span.extensions_mut().insert(SpanLogContext { + thread_id: visitor.thread_id, + }); + } + } + + fn on_record( + &self, + id: &Id, + values: &Record<'_>, + ctx: tracing_subscriber::layer::Context<'_, S>, + ) { + let mut visitor = SpanFieldVisitor::default(); + values.record(&mut visitor); + + if visitor.thread_id.is_none() { + return; + } + + if let Some(span) = ctx.span(id) { + let mut extensions = span.extensions_mut(); + if let Some(log_context) = extensions.get_mut::() { + log_context.thread_id = visitor.thread_id; + } else { + extensions.insert(SpanLogContext { + thread_id: visitor.thread_id, + }); + } + } + } + + fn on_event(&self, event: &Event<'_>, ctx: tracing_subscriber::layer::Context<'_, S>) { let metadata = event.metadata(); let mut visitor = MessageVisitor::default(); event.record(&mut visitor); + let thread_id = visitor + .thread_id + .clone() + .or_else(|| event_thread_id(event, &ctx)); let now = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -65,6 +113,7 @@ where level: metadata.level().as_str().to_string(), target: metadata.target().to_string(), message: visitor.message, + thread_id, module_path: metadata.module_path().map(ToString::to_string), file: metadata.file().map(ToString::to_string), line: metadata.line().map(|line| line as i64), @@ -74,6 +123,75 @@ where } } +#[derive(Clone, Debug, Default)] +struct SpanLogContext { + thread_id: Option, +} + +#[derive(Default)] +struct SpanFieldVisitor { + thread_id: Option, +} + +impl SpanFieldVisitor { + fn record_field(&mut self, field: &Field, value: String) { + if field.name() == "thread_id" && self.thread_id.is_none() { + self.thread_id = Some(value); + } + } +} + +impl Visit for SpanFieldVisitor { + fn record_i64(&mut self, field: &Field, value: i64) { + self.record_field(field, value.to_string()); + } + + fn record_u64(&mut self, field: &Field, value: u64) { + self.record_field(field, value.to_string()); + } + + fn record_bool(&mut self, field: &Field, value: bool) { + self.record_field(field, value.to_string()); + } + + fn record_f64(&mut self, field: &Field, value: f64) { + self.record_field(field, value.to_string()); + } + + fn record_str(&mut self, field: &Field, value: &str) { + self.record_field(field, value.to_string()); + } + + fn record_error(&mut self, field: &Field, value: &(dyn std::error::Error + 'static)) { + self.record_field(field, value.to_string()); + } + + fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { + self.record_field(field, format!("{value:?}")); + } +} + +fn event_thread_id( + event: &Event<'_>, + ctx: &tracing_subscriber::layer::Context<'_, S>, +) -> Option +where + S: tracing::Subscriber + for<'a> LookupSpan<'a>, +{ + let mut thread_id = None; + if let Some(scope) = ctx.event_scope(event) { + for span in scope.from_root() { + let extensions = span.extensions(); + if let Some(log_context) = extensions.get::() + && log_context.thread_id.is_some() + { + thread_id = log_context.thread_id.clone(); + } + } + } + thread_id +} + async fn run_inserter( state_db: std::sync::Arc, mut receiver: mpsc::Receiver, @@ -114,42 +232,46 @@ async fn flush(state_db: &std::sync::Arc, buffer: &mut Vec, + thread_id: Option, } impl MessageVisitor { - fn record_message(&mut self, field: &Field, value: String) { + fn record_field(&mut self, field: &Field, value: String) { if field.name() == "message" && self.message.is_none() { - self.message = Some(value); + self.message = Some(value.clone()); + } + if field.name() == "thread_id" && self.thread_id.is_none() { + self.thread_id = Some(value); } } } impl Visit for MessageVisitor { fn record_i64(&mut self, field: &Field, value: i64) { - self.record_message(field, value.to_string()); + self.record_field(field, value.to_string()); } fn record_u64(&mut self, field: &Field, value: u64) { - self.record_message(field, value.to_string()); + self.record_field(field, value.to_string()); } fn record_bool(&mut self, field: &Field, value: bool) { - self.record_message(field, value.to_string()); + self.record_field(field, value.to_string()); } fn record_f64(&mut self, field: &Field, value: f64) { - self.record_message(field, value.to_string()); + self.record_field(field, value.to_string()); } fn record_str(&mut self, field: &Field, value: &str) { - self.record_message(field, value.to_string()); + self.record_field(field, value.to_string()); } fn record_error(&mut self, field: &Field, value: &(dyn std::error::Error + 'static)) { - self.record_message(field, value.to_string()); + self.record_field(field, value.to_string()); } fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { - self.record_message(field, format!("{value:?}")); + self.record_field(field, format!("{value:?}")); } } diff --git a/codex-rs/state/src/model/log.rs b/codex-rs/state/src/model/log.rs index 0fa4c0b38e..c0c3a2dff2 100644 --- a/codex-rs/state/src/model/log.rs +++ b/codex-rs/state/src/model/log.rs @@ -7,6 +7,7 @@ pub struct LogEntry { pub level: String, pub target: String, pub message: Option, + pub thread_id: Option, pub module_path: Option, pub file: Option, pub line: Option, diff --git a/codex-rs/state/src/runtime.rs b/codex-rs/state/src/runtime.rs index 96b2d69122..8dada32746 100644 --- a/codex-rs/state/src/runtime.rs +++ b/codex-rs/state/src/runtime.rs @@ -214,7 +214,7 @@ FROM threads } let mut builder = QueryBuilder::::new( - "INSERT INTO logs (ts, ts_nanos, level, target, message, module_path, file, line) ", + "INSERT INTO logs (ts, ts_nanos, level, target, message, thread_id, module_path, file, line) ", ); builder.push_values(entries, |mut row, entry| { row.push_bind(entry.ts) @@ -222,6 +222,7 @@ FROM threads .push_bind(&entry.level) .push_bind(&entry.target) .push_bind(&entry.message) + .push_bind(&entry.thread_id) .push_bind(&entry.module_path) .push_bind(&entry.file) .push_bind(entry.line); From d6631fb5a99087dd0fb7c37620e1f32977cc5c7e Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 29 Jan 2026 16:55:01 +0100 Subject: [PATCH 22/28] feat: add log retention and delete them after 90 days (#10151) --- codex-rs/state/src/log_db.rs | 14 +++++++++++++- codex-rs/state/src/runtime.rs | 8 ++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/codex-rs/state/src/log_db.rs b/codex-rs/state/src/log_db.rs index a62870974c..345e90c6a8 100644 --- a/codex-rs/state/src/log_db.rs +++ b/codex-rs/state/src/log_db.rs @@ -18,6 +18,8 @@ //! # } //! ``` +use chrono::Duration as ChronoDuration; +use chrono::Utc; use std::time::Duration; use std::time::SystemTime; use std::time::UNIX_EPOCH; @@ -38,6 +40,7 @@ use crate::StateRuntime; const LOG_QUEUE_CAPACITY: usize = 512; const LOG_BATCH_SIZE: usize = 64; const LOG_FLUSH_INTERVAL: Duration = Duration::from_millis(250); +const LOG_RETENTION_DAYS: i64 = 90; pub struct LogDbLayer { sender: mpsc::Sender, @@ -45,7 +48,8 @@ pub struct LogDbLayer { pub fn start(state_db: std::sync::Arc) -> LogDbLayer { let (sender, receiver) = mpsc::channel(LOG_QUEUE_CAPACITY); - tokio::spawn(run_inserter(state_db, receiver)); + tokio::spawn(run_inserter(std::sync::Arc::clone(&state_db), receiver)); + tokio::spawn(run_retention_cleanup(state_db)); LogDbLayer { sender } } @@ -229,6 +233,14 @@ async fn flush(state_db: &std::sync::Arc, buffer: &mut Vec) { + let Some(cutoff) = Utc::now().checked_sub_signed(ChronoDuration::days(LOG_RETENTION_DAYS)) + else { + return; + }; + let _ = state_db.delete_logs_before(cutoff.timestamp()).await; +} + #[derive(Default)] struct MessageVisitor { message: Option, diff --git a/codex-rs/state/src/runtime.rs b/codex-rs/state/src/runtime.rs index 8dada32746..b61ea315c5 100644 --- a/codex-rs/state/src/runtime.rs +++ b/codex-rs/state/src/runtime.rs @@ -231,6 +231,14 @@ FROM threads Ok(()) } + pub(crate) async fn delete_logs_before(&self, cutoff_ts: i64) -> anyhow::Result { + let result = sqlx::query("DELETE FROM logs WHERE ts < ?") + .bind(cutoff_ts) + .execute(self.pool.as_ref()) + .await?; + Ok(result.rows_affected()) + } + /// List thread ids using the underlying database (no rollout scanning). pub async fn list_thread_ids( &self, From e6c4f548ab3072b6ae709b3da5aecfb074fe95d7 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 29 Jan 2026 17:28:15 +0100 Subject: [PATCH 23/28] chore: unify log queries (#10152) Unify log queries to only have SQLX code in the runtime and use it for both the log client and for tests --- .github/workflows/shell-tool-mcp.yml | 37 +---- codex-rs/core/tests/suite/sqlite_state.rs | 80 ++++++++++ codex-rs/state/migrations/0002_logs.sql | 2 - .../state/migrations/0003_logs_thread_id.sql | 3 + codex-rs/state/src/bin/logs_client.rs | 144 +++++------------- codex-rs/state/src/lib.rs | 2 + codex-rs/state/src/model/log.rs | 26 ++++ codex-rs/state/src/model/mod.rs | 2 + codex-rs/state/src/runtime.rs | 68 +++++++++ 9 files changed, 225 insertions(+), 139 deletions(-) create mode 100644 codex-rs/state/migrations/0003_logs_thread_id.sql diff --git a/.github/workflows/shell-tool-mcp.yml b/.github/workflows/shell-tool-mcp.yml index ec0c16cd5e..d6c85d9ddc 100644 --- a/.github/workflows/shell-tool-mcp.yml +++ b/.github/workflows/shell-tool-mcp.yml @@ -248,37 +248,8 @@ jobs: run: | set -euo pipefail if command -v apt-get >/dev/null 2>&1; then - # On Ubuntu ARM containers, apt metadata for universe/multiverse is large and - # can make `apt-get update` very slow. We only need packages from `main`, so - # trim sources to `main` for faster, more reliable updates. - if [[ "${{ matrix.target }}" == "aarch64-unknown-linux-musl" ]] && [[ -f /etc/os-release ]]; then - . /etc/os-release - if [[ "${ID-}" == "ubuntu" ]]; then - if [[ -f /etc/apt/sources.list.d/ubuntu.sources ]]; then - # Ubuntu 24.04+ uses deb822 sources; trim components to main. - sed -i -E 's/^Components:.*/Components: main/' /etc/apt/sources.list.d/ubuntu.sources - else - codename="${VERSION_CODENAME:-}" - if [[ -n "$codename" ]]; then - printf '%s\n' \ - "deb http://ports.ubuntu.com/ubuntu-ports ${codename} main" \ - "deb http://ports.ubuntu.com/ubuntu-ports ${codename}-updates main" \ - "deb http://ports.ubuntu.com/ubuntu-ports ${codename}-backports main" \ - "deb http://ports.ubuntu.com/ubuntu-ports ${codename}-security main" \ - >/etc/apt/sources.list - fi - fi - fi - fi - - apt_opts=( - -o Acquire::Retries=3 - -o Acquire::http::Timeout=30 - -o Acquire::https::Timeout=30 - -o Acquire::Languages=none - ) - apt-get "${apt_opts[@]}" update - DEBIAN_FRONTEND=noninteractive apt-get "${apt_opts[@]}" install -y git build-essential bison autoconf gettext + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y git build-essential bison autoconf gettext elif command -v dnf >/dev/null 2>&1; then dnf install -y git gcc gcc-c++ make bison autoconf gettext elif command -v yum >/dev/null 2>&1; then @@ -480,10 +451,6 @@ jobs: registry-url: https://registry.npmjs.org scope: "@openai" - # Trusted publishing requires npm CLI version 11.5.1 or later. - - name: Update npm - run: npm install -g npm@latest - - name: Download npm tarball uses: actions/download-artifact@v7 with: diff --git a/codex-rs/core/tests/suite/sqlite_state.rs b/codex-rs/core/tests/suite/sqlite_state.rs index df6fa2ef72..bf3b5a4f51 100644 --- a/codex-rs/core/tests/suite/sqlite_state.rs +++ b/codex-rs/core/tests/suite/sqlite_state.rs @@ -10,12 +10,18 @@ use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::UserMessageEvent; use codex_state::STATE_DB_FILENAME; use core_test_support::load_sse_fixture_with_id; +use core_test_support::responses; +use core_test_support::responses::ev_completed; +use core_test_support::responses::ev_function_call; +use core_test_support::responses::ev_response_created; use core_test_support::responses::mount_sse_sequence; use core_test_support::responses::start_mock_server; use core_test_support::test_codex::test_codex; use pretty_assertions::assert_eq; +use serde_json::json; use std::fs; use tokio::time::Duration; +use tracing_subscriber::prelude::*; use uuid::Uuid; fn sse_completed(id: &str) -> String { @@ -197,3 +203,77 @@ async fn user_messages_persist_in_state_db() -> Result<()> { Ok(()) } + +#[tokio::test(flavor = "current_thread")] +async fn tool_call_logs_include_thread_id() -> Result<()> { + let server = start_mock_server().await; + let call_id = "call-1"; + let args = json!({ + "command": "echo hello", + "timeout_ms": 1_000, + "login": false, + }); + let args_json = serde_json::to_string(&args)?; + mount_sse_sequence( + &server, + vec![ + responses::sse(vec![ + ev_response_created("resp-1"), + ev_function_call(call_id, "shell_command", &args_json), + ev_completed("resp-1"), + ]), + responses::sse(vec![ev_completed("resp-2")]), + ], + ) + .await; + + let mut builder = test_codex().with_config(|config| { + config.features.enable(Feature::Sqlite); + }); + let test = builder.build(&server).await?; + let db = test.codex.state_db().expect("state db enabled"); + let expected_thread_id = test.session_configured.session_id.to_string(); + + let subscriber = tracing_subscriber::registry().with(codex_state::log_db::start(db.clone())); + let dispatch = tracing::Dispatch::new(subscriber); + let _guard = tracing::dispatcher::set_default(&dispatch); + + test.submit_turn("run a shell command").await?; + { + let span = tracing::info_span!("test_log_span", thread_id = %expected_thread_id); + let _entered = span.enter(); + tracing::info!("ToolCall: shell_command {{\"command\":\"echo hello\"}}"); + } + + let mut found = None; + for _ in 0..80 { + let query = codex_state::LogQuery { + descending: true, + limit: Some(20), + ..Default::default() + }; + let rows = db.query_logs(&query).await?; + if let Some(row) = rows.into_iter().find(|row| { + row.message + .as_deref() + .is_some_and(|m| m.starts_with("ToolCall:")) + }) { + let thread_id = row.thread_id; + let message = row.message; + found = Some((thread_id, message)); + break; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + + let (thread_id, message) = found.expect("expected ToolCall log row"); + assert_eq!(thread_id, Some(expected_thread_id)); + assert!( + message + .as_deref() + .is_some_and(|text| text.starts_with("ToolCall:")), + "expected ToolCall message, got {message:?}" + ); + + Ok(()) +} diff --git a/codex-rs/state/migrations/0002_logs.sql b/codex-rs/state/migrations/0002_logs.sql index 051bef9136..b9a2c681d4 100644 --- a/codex-rs/state/migrations/0002_logs.sql +++ b/codex-rs/state/migrations/0002_logs.sql @@ -5,11 +5,9 @@ CREATE TABLE logs ( level TEXT NOT NULL, target TEXT NOT NULL, message TEXT, - thread_id TEXT, module_path TEXT, file TEXT, line INTEGER ); CREATE INDEX idx_logs_ts ON logs(ts DESC, ts_nanos DESC, id DESC); -CREATE INDEX idx_logs_thread_id ON logs(thread_id); diff --git a/codex-rs/state/migrations/0003_logs_thread_id.sql b/codex-rs/state/migrations/0003_logs_thread_id.sql new file mode 100644 index 0000000000..c4badb6885 --- /dev/null +++ b/codex-rs/state/migrations/0003_logs_thread_id.sql @@ -0,0 +1,3 @@ +ALTER TABLE logs ADD COLUMN thread_id TEXT; + +CREATE INDEX idx_logs_thread_id ON logs(thread_id); diff --git a/codex-rs/state/src/bin/logs_client.rs b/codex-rs/state/src/bin/logs_client.rs index df57856115..c02a423253 100644 --- a/codex-rs/state/src/bin/logs_client.rs +++ b/codex-rs/state/src/bin/logs_client.rs @@ -1,4 +1,3 @@ -use std::path::Path; use std::path::PathBuf; use std::time::Duration; @@ -7,15 +6,12 @@ use chrono::DateTime; use chrono::SecondsFormat; use chrono::Utc; use clap::Parser; +use codex_state::LogQuery; +use codex_state::LogRow; use codex_state::STATE_DB_FILENAME; +use codex_state::StateRuntime; use dirs::home_dir; use owo_colors::OwoColorize; -use sqlx::QueryBuilder; -use sqlx::Row; -use sqlx::Sqlite; -use sqlx::SqlitePool; -use sqlx::sqlite::SqliteConnectOptions; -use sqlx::sqlite::SqlitePoolOptions; #[derive(Debug, Parser)] #[command(name = "codex-state-logs")] @@ -62,18 +58,6 @@ struct Args { poll_ms: u64, } -#[derive(Debug, Clone, sqlx::FromRow)] -struct LogRow { - id: i64, - ts: i64, - ts_nanos: i64, - level: String, - message: Option, - thread_id: Option, - file: Option, - line: Option, -} - #[derive(Debug, Clone)] struct LogFilter { level_upper: Option, @@ -89,16 +73,20 @@ async fn main() -> anyhow::Result<()> { let args = Args::parse(); let db_path = resolve_db_path(&args)?; let filter = build_filter(&args)?; - let pool = open_read_only_pool(db_path.as_path()).await?; + let codex_home = db_path + .parent() + .map(ToOwned::to_owned) + .unwrap_or_else(|| PathBuf::from(".")); + let runtime = StateRuntime::init(codex_home, "logs-client".to_string(), None).await?; - let mut last_id = print_backfill(&pool, &filter, args.backfill).await?; + let mut last_id = print_backfill(runtime.as_ref(), &filter, args.backfill).await?; if last_id == 0 { - last_id = fetch_max_id(&pool, &filter).await?; + last_id = fetch_max_id(runtime.as_ref(), &filter).await?; } let poll_interval = Duration::from_millis(args.poll_ms); loop { - let rows = fetch_new_rows(&pool, &filter, last_id).await?; + let rows = fetch_new_rows(runtime.as_ref(), &filter, last_id).await?; for row in rows { last_id = last_id.max(row.id); println!("{}", format_row(&row)); @@ -159,23 +147,8 @@ fn parse_timestamp(value: &str) -> anyhow::Result { Ok(dt.timestamp()) } -async fn open_read_only_pool(path: &Path) -> anyhow::Result { - let options = SqliteConnectOptions::new() - .filename(path) - .create_if_missing(false) - .read_only(true) - .busy_timeout(Duration::from_secs(5)); - - let display = path.display(); - SqlitePoolOptions::new() - .max_connections(1) - .connect_with(options) - .await - .with_context(|| format!("failed to open sqlite db at {display}")) -} - async fn print_backfill( - pool: &SqlitePool, + runtime: &StateRuntime, filter: &LogFilter, backfill: usize, ) -> anyhow::Result { @@ -183,7 +156,7 @@ async fn print_backfill( return Ok(0); } - let mut rows = fetch_backfill(pool, filter, backfill).await?; + let mut rows = fetch_backfill(runtime, filter, backfill).await?; rows.reverse(); let mut last_id = 0; @@ -195,86 +168,53 @@ async fn print_backfill( } async fn fetch_backfill( - pool: &SqlitePool, + runtime: &StateRuntime, filter: &LogFilter, backfill: usize, ) -> anyhow::Result> { - let mut builder = base_select_builder(); - push_filters(&mut builder, filter); - builder.push(" ORDER BY id DESC"); - builder.push(" LIMIT ").push_bind(backfill as i64); - - builder - .build_query_as::() - .fetch_all(pool) + let query = to_log_query(filter, Some(backfill), None, true); + runtime + .query_logs(&query) .await .context("failed to fetch backfill logs") } async fn fetch_new_rows( - pool: &SqlitePool, + runtime: &StateRuntime, filter: &LogFilter, last_id: i64, ) -> anyhow::Result> { - let mut builder = base_select_builder(); - push_filters(&mut builder, filter); - builder.push(" AND id > ").push_bind(last_id); - builder.push(" ORDER BY id ASC"); - - builder - .build_query_as::() - .fetch_all(pool) + let query = to_log_query(filter, None, Some(last_id), false); + runtime + .query_logs(&query) .await .context("failed to fetch new logs") } -async fn fetch_max_id(pool: &SqlitePool, filter: &LogFilter) -> anyhow::Result { - let mut builder = QueryBuilder::::new("SELECT MAX(id) AS max_id FROM logs WHERE 1 = 1"); - push_filters(&mut builder, filter); - - let row = builder - .build() - .fetch_one(pool) +async fn fetch_max_id(runtime: &StateRuntime, filter: &LogFilter) -> anyhow::Result { + let query = to_log_query(filter, None, None, false); + runtime + .max_log_id(&query) .await - .context("failed to fetch max log id")?; - let max_id: Option = row.try_get("max_id")?; - Ok(max_id.unwrap_or(0)) + .context("failed to fetch max log id") } -fn base_select_builder<'a>() -> QueryBuilder<'a, Sqlite> { - QueryBuilder::::new( - "SELECT id, ts, ts_nanos, level, message, thread_id, file, line FROM logs WHERE 1 = 1", - ) -} - -fn push_filters<'a>(builder: &mut QueryBuilder<'a, Sqlite>, filter: &'a LogFilter) { - if let Some(level_upper) = filter.level_upper.as_ref() { - builder - .push(" AND UPPER(level) = ") - .push_bind(level_upper.as_str()); - } - if let Some(from_ts) = filter.from_ts { - builder.push(" AND ts >= ").push_bind(from_ts); - } - if let Some(to_ts) = filter.to_ts { - builder.push(" AND ts <= ").push_bind(to_ts); - } - if let Some(module_like) = filter.module_like.as_ref() { - builder - .push(" AND module_path LIKE '%' || ") - .push_bind(module_like.as_str()) - .push(" || '%'"); - } - if let Some(file_like) = filter.file_like.as_ref() { - builder - .push(" AND file LIKE '%' || ") - .push_bind(file_like.as_str()) - .push(" || '%'"); - } - if let Some(thread_id) = filter.thread_id.as_ref() { - builder - .push(" AND thread_id = ") - .push_bind(thread_id.as_str()); +fn to_log_query( + filter: &LogFilter, + limit: Option, + after_id: Option, + descending: bool, +) -> LogQuery { + LogQuery { + level_upper: filter.level_upper.clone(), + from_ts: filter.from_ts, + to_ts: filter.to_ts, + module_like: filter.module_like.clone(), + file_like: filter.file_like.clone(), + thread_id: filter.thread_id.clone(), + after_id, + limit, + descending, } } diff --git a/codex-rs/state/src/lib.rs b/codex-rs/state/src/lib.rs index 91ebacea71..685552015e 100644 --- a/codex-rs/state/src/lib.rs +++ b/codex-rs/state/src/lib.rs @@ -12,6 +12,8 @@ mod paths; mod runtime; pub use model::LogEntry; +pub use model::LogQuery; +pub use model::LogRow; /// Preferred entrypoint: owns configuration and metrics. pub use runtime::StateRuntime; diff --git a/codex-rs/state/src/model/log.rs b/codex-rs/state/src/model/log.rs index c0c3a2dff2..d8a9ba3524 100644 --- a/codex-rs/state/src/model/log.rs +++ b/codex-rs/state/src/model/log.rs @@ -1,4 +1,5 @@ use serde::Serialize; +use sqlx::FromRow; #[derive(Clone, Debug, Serialize)] pub struct LogEntry { @@ -12,3 +13,28 @@ pub struct LogEntry { pub file: Option, pub line: Option, } + +#[derive(Clone, Debug, FromRow)] +pub struct LogRow { + pub id: i64, + pub ts: i64, + pub ts_nanos: i64, + pub level: String, + pub message: Option, + pub thread_id: Option, + pub file: Option, + pub line: Option, +} + +#[derive(Clone, Debug, Default)] +pub struct LogQuery { + pub level_upper: Option, + pub from_ts: Option, + pub to_ts: Option, + pub module_like: Option, + pub file_like: Option, + pub thread_id: Option, + pub after_id: Option, + pub limit: Option, + pub descending: bool, +} diff --git a/codex-rs/state/src/model/mod.rs b/codex-rs/state/src/model/mod.rs index 4921e2357a..bd615d7561 100644 --- a/codex-rs/state/src/model/mod.rs +++ b/codex-rs/state/src/model/mod.rs @@ -2,6 +2,8 @@ mod log; mod thread_metadata; pub use log::LogEntry; +pub use log::LogQuery; +pub use log::LogRow; pub use thread_metadata::Anchor; pub use thread_metadata::BackfillStats; pub use thread_metadata::ExtractionOutcome; diff --git a/codex-rs/state/src/runtime.rs b/codex-rs/state/src/runtime.rs index b61ea315c5..b15e4338bf 100644 --- a/codex-rs/state/src/runtime.rs +++ b/codex-rs/state/src/runtime.rs @@ -1,5 +1,7 @@ use crate::DB_ERROR_METRIC; use crate::LogEntry; +use crate::LogQuery; +use crate::LogRow; use crate::SortKey; use crate::ThreadMetadata; use crate::ThreadMetadataBuilder; @@ -239,6 +241,38 @@ FROM threads Ok(result.rows_affected()) } + /// Query logs with optional filters. + pub async fn query_logs(&self, query: &LogQuery) -> anyhow::Result> { + let mut builder = QueryBuilder::::new( + "SELECT id, ts, ts_nanos, level, message, thread_id, file, line FROM logs WHERE 1 = 1", + ); + push_log_filters(&mut builder, query); + if query.descending { + builder.push(" ORDER BY id DESC"); + } else { + builder.push(" ORDER BY id ASC"); + } + if let Some(limit) = query.limit { + builder.push(" LIMIT ").push_bind(limit as i64); + } + + let rows = builder + .build_query_as::() + .fetch_all(self.pool.as_ref()) + .await?; + Ok(rows) + } + + /// Return the max log id matching optional filters. + pub async fn max_log_id(&self, query: &LogQuery) -> anyhow::Result { + let mut builder = + QueryBuilder::::new("SELECT MAX(id) AS max_id FROM logs WHERE 1 = 1"); + push_log_filters(&mut builder, query); + let row = builder.build().fetch_one(self.pool.as_ref()).await?; + let max_id: Option = row.try_get("max_id")?; + Ok(max_id.unwrap_or(0)) + } + /// List thread ids using the underlying database (no rollout scanning). pub async fn list_thread_ids( &self, @@ -411,6 +445,40 @@ ON CONFLICT(id) DO UPDATE SET } } +fn push_log_filters<'a>(builder: &mut QueryBuilder<'a, Sqlite>, query: &'a LogQuery) { + if let Some(level_upper) = query.level_upper.as_ref() { + builder + .push(" AND UPPER(level) = ") + .push_bind(level_upper.as_str()); + } + if let Some(from_ts) = query.from_ts { + builder.push(" AND ts >= ").push_bind(from_ts); + } + if let Some(to_ts) = query.to_ts { + builder.push(" AND ts <= ").push_bind(to_ts); + } + if let Some(module_like) = query.module_like.as_ref() { + builder + .push(" AND module_path LIKE '%' || ") + .push_bind(module_like.as_str()) + .push(" || '%'"); + } + if let Some(file_like) = query.file_like.as_ref() { + builder + .push(" AND file LIKE '%' || ") + .push_bind(file_like.as_str()) + .push(" || '%'"); + } + if let Some(thread_id) = query.thread_id.as_ref() { + builder + .push(" AND thread_id = ") + .push_bind(thread_id.as_str()); + } + if let Some(after_id) = query.after_id { + builder.push(" AND id > ").push_bind(after_id); + } +} + async fn open_sqlite(path: &Path) -> anyhow::Result { let options = SqliteConnectOptions::new() .filename(path) From 3e798c5a7dd0556947acc2baea1bb64be63ff3c7 Mon Sep 17 00:00:00 2001 From: Josh McKinney Date: Thu, 29 Jan 2026 09:34:59 -0800 Subject: [PATCH 24/28] Add OpenAI docs MCP tooltip (#10175) --- codex-rs/tui/tooltips.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/codex-rs/tui/tooltips.txt b/codex-rs/tui/tooltips.txt index 31819c6628..dc2d2faa67 100644 --- a/codex-rs/tui/tooltips.txt +++ b/codex-rs/tui/tooltips.txt @@ -9,6 +9,7 @@ Use /status to see the current model, approvals, and token usage. Use /fork to branch the current chat into a new thread. Use /init to create an AGENTS.md with project-specific guidance. Use /mcp to list configured MCP tools. +Use the OpenAI docs MCP for API questions; enable it with `codex mcp add openaiDeveloperDocs --url https://developers.openai.com/mcp`. You can run any shell command from Codex using `!` (e.g. `!ls`) Type / to open the command popup; Tab autocompletes slash commands. When the composer is empty, press Esc to step back and edit your last message; Enter confirms. From 798c4b3260e857756c2d6e13cc9bcd05ba604c7e Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 29 Jan 2026 19:15:22 +0100 Subject: [PATCH 25/28] feat: reduce span exposition (#10171) This only avoids the creation of duplicates spans --- codex-rs/core/src/stream_events_utils.rs | 7 ++++++- codex-rs/core/src/tasks/mod.rs | 5 ++--- codex-rs/state/src/bin/logs_client.rs | 12 ++++-------- codex-rs/state/src/model/log.rs | 1 + codex-rs/state/src/runtime.rs | 2 +- codex-rs/tui/src/lib.rs | 5 ++++- 6 files changed, 18 insertions(+), 14 deletions(-) diff --git a/codex-rs/core/src/stream_events_utils.rs b/codex-rs/core/src/stream_events_utils.rs index 197ff6b4b6..930ce5afd6 100644 --- a/codex-rs/core/src/stream_events_utils.rs +++ b/codex-rs/core/src/stream_events_utils.rs @@ -51,7 +51,12 @@ pub(crate) async fn handle_output_item_done( // The model emitted a tool call; log it, persist the item immediately, and queue the tool execution. Ok(Some(call)) => { let payload_preview = call.payload.log_payload().into_owned(); - tracing::info!("ToolCall: {} {}", call.tool_name, payload_preview); + tracing::info!( + thread_id = %ctx.sess.conversation_id, + "ToolCall: {} {}", + call.tool_name, + payload_preview + ); ctx.sess .record_conversation_items(&ctx.turn_context, std::slice::from_ref(&item)) diff --git a/codex-rs/core/src/tasks/mod.rs b/codex-rs/core/src/tasks/mod.rs index 1f1195236c..a487041eba 100644 --- a/codex-rs/core/src/tasks/mod.rs +++ b/codex-rs/core/src/tasks/mod.rs @@ -14,7 +14,7 @@ use tokio::sync::Notify; use tokio_util::sync::CancellationToken; use tokio_util::task::AbortOnDropHandle; use tracing::Instrument; -use tracing::info_span; +use tracing::Span; use tracing::trace; use tracing::warn; @@ -132,8 +132,7 @@ impl Session { let ctx = Arc::clone(&turn_context); let task_for_run = Arc::clone(&task); let task_cancellation_token = cancellation_token.child_token(); - let thread_id = self.conversation_id; - let session_span = info_span!("session_task", thread_id = %thread_id); + let session_span = Span::current(); tokio::spawn( async move { let ctx_for_finish = Arc::clone(&ctx); diff --git a/codex-rs/state/src/bin/logs_client.rs b/codex-rs/state/src/bin/logs_client.rs index c02a423253..4389c29bc6 100644 --- a/codex-rs/state/src/bin/logs_client.rs +++ b/codex-rs/state/src/bin/logs_client.rs @@ -221,20 +221,16 @@ fn to_log_query( fn format_row(row: &LogRow) -> String { let timestamp = format_timestamp(row.ts, row.ts_nanos); let level = row.level.as_str(); - let location = match (&row.file, row.line) { - (Some(file), Some(line)) => format!("{file}:{line}"), - (Some(file), None) => file.clone(), - _ => "-".to_string(), - }; + let target = row.target.as_str(); let message = row.message.as_deref().unwrap_or(""); let level_colored = color_level(level); let timestamp_colored = timestamp.dimmed().to_string(); let thread_id = row.thread_id.as_deref().unwrap_or("-"); - let thread_id_colored = thread_id.yellow().to_string(); - let location_colored = location.dimmed().to_string(); + let thread_id_colored = thread_id.blue().dimmed().to_string(); + let target_colored = target.dimmed().to_string(); let message_colored = message.bold().to_string(); format!( - "{timestamp_colored} {level_colored} [{thread_id_colored}] {location_colored} - {message_colored}" + "{timestamp_colored} {level_colored} [{thread_id_colored}] {target_colored} - {message_colored}" ) } diff --git a/codex-rs/state/src/model/log.rs b/codex-rs/state/src/model/log.rs index d8a9ba3524..5d774249dd 100644 --- a/codex-rs/state/src/model/log.rs +++ b/codex-rs/state/src/model/log.rs @@ -20,6 +20,7 @@ pub struct LogRow { pub ts: i64, pub ts_nanos: i64, pub level: String, + pub target: String, pub message: Option, pub thread_id: Option, pub file: Option, diff --git a/codex-rs/state/src/runtime.rs b/codex-rs/state/src/runtime.rs index b15e4338bf..e0805e5b65 100644 --- a/codex-rs/state/src/runtime.rs +++ b/codex-rs/state/src/runtime.rs @@ -244,7 +244,7 @@ FROM threads /// Query logs with optional filters. pub async fn query_logs(&self, query: &LogQuery) -> anyhow::Result> { let mut builder = QueryBuilder::::new( - "SELECT id, ts, ts_nanos, level, message, thread_id, file, line FROM logs WHERE 1 = 1", + "SELECT id, ts, ts_nanos, level, target, message, thread_id, file, line FROM logs WHERE 1 = 1", ); push_log_filters(&mut builder, query); if query.descending { diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index c9e69b7125..010f8c83e5 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -306,7 +306,10 @@ pub async fn run_main( // grep for a specific module/target while troubleshooting. .with_target(true) .with_ansi(false) - .with_span_events(tracing_subscriber::fmt::format::FmtSpan::FULL) + .with_span_events( + tracing_subscriber::fmt::format::FmtSpan::NEW + | tracing_subscriber::fmt::format::FmtSpan::CLOSE, + ) .with_filter(env_filter()); let feedback = codex_feedback::CodexFeedback::new(); From 3b1cddf00106065a8ed0d95d76a206f0ce66ebf3 Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Thu, 29 Jan 2026 10:36:21 -0800 Subject: [PATCH 26/28] Fall back to http when websockets fail (#10139) I expect not all proxies work with websockets, fall back to http if websockets fail. --- codex-rs/core/src/client.rs | 33 ++++++- codex-rs/core/src/codex.rs | 25 ++++- codex-rs/core/src/lib.rs | 2 + codex-rs/core/src/state/service.rs | 2 + codex-rs/core/src/tools/handlers/collab.rs | 3 + codex-rs/core/src/transport_manager.rs | 31 ++++++ .../core/tests/chat_completions_payload.rs | 2 + codex-rs/core/tests/chat_completions_sse.rs | 2 + codex-rs/core/tests/responses_headers.rs | 4 + codex-rs/core/tests/suite/client.rs | 2 + .../core/tests/suite/client_websockets.rs | 2 + codex-rs/core/tests/suite/mod.rs | 1 + .../core/tests/suite/websocket_fallback.rs | 98 +++++++++++++++++++ 13 files changed, 205 insertions(+), 2 deletions(-) create mode 100644 codex-rs/core/src/transport_manager.rs create mode 100644 codex-rs/core/tests/suite/websocket_fallback.rs diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index f415c3604a..98fce871fd 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -65,6 +65,7 @@ use crate::model_provider_info::ModelProviderInfo; use crate::model_provider_info::WireApi; use crate::tools::spec::create_tools_json_for_chat_completions_api; use crate::tools::spec::create_tools_json_for_responses_api; +use crate::transport_manager::TransportManager; pub const WEB_SEARCH_ELIGIBLE_HEADER: &str = "x-oai-web-search-eligible"; pub const X_CODEX_TURN_STATE_HEADER: &str = "x-codex-turn-state"; @@ -80,6 +81,7 @@ struct ModelClientState { effort: Option, summary: ReasoningSummaryConfig, session_source: SessionSource, + transport_manager: TransportManager, } #[derive(Debug, Clone)] @@ -91,6 +93,7 @@ pub struct ModelClientSession { state: Arc, connection: Option, websocket_last_items: Vec, + transport_manager: TransportManager, /// Turn state for sticky routing. /// /// This is an `OnceLock` that stores the turn state value received from the server @@ -116,6 +119,7 @@ impl ModelClient { summary: ReasoningSummaryConfig, conversation_id: ThreadId, session_source: SessionSource, + transport_manager: TransportManager, ) -> Self { Self { state: Arc::new(ModelClientState { @@ -128,6 +132,7 @@ impl ModelClient { effort, summary, session_source, + transport_manager, }), } } @@ -137,6 +142,7 @@ impl ModelClient { state: Arc::clone(&self.state), connection: None, websocket_last_items: Vec::new(), + transport_manager: self.state.transport_manager.clone(), turn_state: Arc::new(OnceLock::new()), } } @@ -171,6 +177,10 @@ impl ModelClient { self.state.session_source.clone() } + pub(crate) fn transport_manager(&self) -> TransportManager { + self.state.transport_manager.clone() + } + /// Returns the currently configured model slug. pub fn get_model(&self) -> String { self.state.model_info.slug.clone() @@ -250,7 +260,10 @@ impl ModelClientSession { /// For Chat providers, the underlying stream is optionally aggregated /// based on the `show_raw_agent_reasoning` flag in the config. pub async fn stream(&mut self, prompt: &Prompt) -> Result { - match self.state.provider.wire_api { + let wire_api = self + .transport_manager + .effective_wire_api(self.state.provider.wire_api); + match wire_api { WireApi::Responses => self.stream_responses_api(prompt).await, WireApi::ResponsesWebsocket => self.stream_responses_websocket(prompt).await, WireApi::Chat => { @@ -271,6 +284,24 @@ impl ModelClientSession { } } + pub(crate) fn try_switch_fallback_transport(&mut self) -> bool { + let activated = self + .transport_manager + .activate_http_fallback(self.state.provider.wire_api); + if activated { + warn!("falling back to HTTP"); + self.state.otel_manager.counter( + "codex.transport.fallback_to_http", + 1, + &[("from_wire_api", "responses_websocket")], + ); + + self.connection = None; + self.websocket_last_items.clear(); + } + activated + } + fn build_responses_request(&self, prompt: &Prompt) -> Result { let instructions = prompt.base_instructions.text.clone(); let tools_json: Vec = create_tools_json_for_responses_api(&prompt.tools)?; diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 8e1ec64370..736d0554b3 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -30,6 +30,7 @@ use crate::stream_events_utils::HandleOutputCtx; use crate::stream_events_utils::handle_non_tool_response_item; use crate::stream_events_utils::handle_output_item_done; use crate::terminal; +use crate::transport_manager::TransportManager; use crate::truncate::TruncationPolicy; use crate::user_notification::UserNotifier; use crate::util::error_or_panic; @@ -624,6 +625,7 @@ impl Session { model_info: ModelInfo, conversation_id: ThreadId, sub_id: String, + transport_manager: TransportManager, ) -> TurnContext { let otel_manager = otel_manager.clone().with_model( session_configuration.collaboration_mode.model(), @@ -640,6 +642,7 @@ impl Session { session_configuration.model_reasoning_summary, conversation_id, session_configuration.session_source.clone(), + transport_manager, ); let tools_config = ToolsConfig::new(&ToolsConfigParams { @@ -869,6 +872,7 @@ impl Session { skills_manager, agent_control, state_db: state_db_ctx.clone(), + transport_manager: TransportManager::new(), }; let sess = Arc::new(Session { @@ -1188,6 +1192,7 @@ impl Session { model_info, self.conversation_id, sub_id, + self.services.transport_manager.clone(), ); if let Some(final_schema) = final_output_json_schema { turn_context.final_output_json_schema = final_schema; @@ -3042,6 +3047,7 @@ async fn spawn_review_thread( per_turn_config.model_reasoning_summary, sess.conversation_id, parent_turn_context.client.get_session_source(), + parent_turn_context.client.transport_manager(), ); let review_turn_context = TurnContext { @@ -3537,7 +3543,9 @@ async fn run_sampling_request( ) .await { - Ok(output) => return Ok(output), + Ok(output) => { + return Ok(output); + } Err(CodexErr::ContextWindowExceeded) => { sess.set_total_tokens_full(&turn_context).await; return Err(CodexErr::ContextWindowExceeded); @@ -3558,6 +3566,17 @@ async fn run_sampling_request( // Use the configured provider-specific stream retry budget. let max_retries = turn_context.client.get_provider().stream_max_retries(); + if retries >= max_retries && client_session.try_switch_fallback_transport() { + sess.send_event( + &turn_context, + EventMsg::Warning(WarningEvent { + message: format!("Falling back from WebSockets to HTTPS transport. {err:#}"), + }), + ) + .await; + retries = 0; + continue; + } if retries < max_retries { retries += 1; let delay = match &err { @@ -4753,6 +4772,7 @@ mod tests { skills_manager, agent_control, state_db: None, + transport_manager: TransportManager::new(), }; let turn_context = Session::make_turn_context( @@ -4764,6 +4784,7 @@ mod tests { model_info, conversation_id, "turn_id".to_string(), + services.transport_manager.clone(), ); let session = Session { @@ -4865,6 +4886,7 @@ mod tests { skills_manager, agent_control, state_db: None, + transport_manager: TransportManager::new(), }; let turn_context = Arc::new(Session::make_turn_context( @@ -4876,6 +4898,7 @@ mod tests { model_info, conversation_id, "turn_id".to_string(), + services.transport_manager.clone(), )); let session = Arc::new(Session { diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index ab51a201dc..1b1fb65ecd 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -38,6 +38,7 @@ pub mod landlock; pub mod mcp; mod mcp_connection_manager; pub mod models_manager; +mod transport_manager; pub use mcp_connection_manager::MCP_SANDBOX_STATE_CAPABILITY; pub use mcp_connection_manager::MCP_SANDBOX_STATE_METHOD; pub use mcp_connection_manager::SandboxState; @@ -113,6 +114,7 @@ pub use rollout::list::parse_cursor; pub use rollout::list::read_head_for_summary; pub use rollout::list::read_session_meta_line; pub use rollout::rollout_date_parts; +pub use transport_manager::TransportManager; mod function_tool; mod state; mod tasks; diff --git a/codex-rs/core/src/state/service.rs b/codex-rs/core/src/state/service.rs index 6559ec2e18..e036b29b7d 100644 --- a/codex-rs/core/src/state/service.rs +++ b/codex-rs/core/src/state/service.rs @@ -9,6 +9,7 @@ use crate::models_manager::manager::ModelsManager; use crate::skills::SkillsManager; use crate::state_db::StateDbHandle; use crate::tools::sandboxing::ApprovalStore; +use crate::transport_manager::TransportManager; use crate::unified_exec::UnifiedExecProcessManager; use crate::user_notification::UserNotifier; use codex_otel::OtelManager; @@ -32,4 +33,5 @@ pub(crate) struct SessionServices { pub(crate) skills_manager: Arc, pub(crate) agent_control: AgentControl, pub(crate) state_db: Option, + pub(crate) transport_manager: TransportManager, } diff --git a/codex-rs/core/src/tools/handlers/collab.rs b/codex-rs/core/src/tools/handlers/collab.rs index b1666949f6..61ccc1932e 100644 --- a/codex-rs/core/src/tools/handlers/collab.rs +++ b/codex-rs/core/src/tools/handlers/collab.rs @@ -781,6 +781,7 @@ mod tests { turn.client.get_reasoning_summary(), session.conversation_id, session_source, + session.services.transport_manager.clone(), ); let invocation = invocation( @@ -1221,6 +1222,7 @@ mod tests { let mut base_config = (*turn.client.config()).clone(); base_config.user_instructions = Some("base-user".to_string()); turn.user_instructions = Some("resolved-user".to_string()); + let transport_manager = turn.client.transport_manager(); turn.client = ModelClient::new( Arc::new(base_config.clone()), Some(session.services.auth_manager.clone()), @@ -1231,6 +1233,7 @@ mod tests { turn.client.get_reasoning_summary(), session.conversation_id, session_source, + transport_manager, ); let base_instructions = BaseInstructions { text: "base".to_string(), diff --git a/codex-rs/core/src/transport_manager.rs b/codex-rs/core/src/transport_manager.rs new file mode 100644 index 0000000000..f1fe68faf4 --- /dev/null +++ b/codex-rs/core/src/transport_manager.rs @@ -0,0 +1,31 @@ +use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; + +use crate::model_provider_info::WireApi; + +#[derive(Clone, Debug, Default)] +pub struct TransportManager { + fallback_to_http: Arc, +} + +impl TransportManager { + pub fn new() -> Self { + Self::default() + } + + pub fn effective_wire_api(&self, provider_wire_api: WireApi) -> WireApi { + if self.fallback_to_http.load(Ordering::Relaxed) + && provider_wire_api == WireApi::ResponsesWebsocket + { + WireApi::Responses + } else { + provider_wire_api + } + } + + pub fn activate_http_fallback(&self, provider_wire_api: WireApi) -> bool { + provider_wire_api == WireApi::ResponsesWebsocket + && !self.fallback_to_http.swap(true, Ordering::Relaxed) + } +} diff --git a/codex-rs/core/tests/chat_completions_payload.rs b/codex-rs/core/tests/chat_completions_payload.rs index cdb92fe672..5e8c895a3f 100644 --- a/codex-rs/core/tests/chat_completions_payload.rs +++ b/codex-rs/core/tests/chat_completions_payload.rs @@ -11,6 +11,7 @@ use codex_core::ModelClient; use codex_core::ModelProviderInfo; use codex_core::Prompt; use codex_core::ResponseItem; +use codex_core::TransportManager; use codex_core::WireApi; use codex_core::models_manager::manager::ModelsManager; use codex_otel::OtelManager; @@ -98,6 +99,7 @@ async fn run_request(input: Vec) -> Value { summary, conversation_id, SessionSource::Exec, + TransportManager::new(), ) .new_session(); diff --git a/codex-rs/core/tests/chat_completions_sse.rs b/codex-rs/core/tests/chat_completions_sse.rs index 05ef476a05..84d0ede085 100644 --- a/codex-rs/core/tests/chat_completions_sse.rs +++ b/codex-rs/core/tests/chat_completions_sse.rs @@ -10,6 +10,7 @@ use codex_core::ModelProviderInfo; use codex_core::Prompt; use codex_core::ResponseEvent; use codex_core::ResponseItem; +use codex_core::TransportManager; use codex_core::WireApi; use codex_core::models_manager::manager::ModelsManager; use codex_otel::OtelManager; @@ -99,6 +100,7 @@ async fn run_stream_with_bytes(sse_body: &[u8]) -> Vec { summary, conversation_id, SessionSource::Exec, + TransportManager::new(), ) .new_session(); diff --git a/codex-rs/core/tests/responses_headers.rs b/codex-rs/core/tests/responses_headers.rs index 22d9fa8b79..b3c0165f46 100644 --- a/codex-rs/core/tests/responses_headers.rs +++ b/codex-rs/core/tests/responses_headers.rs @@ -9,6 +9,7 @@ use codex_core::ModelProviderInfo; use codex_core::Prompt; use codex_core::ResponseEvent; use codex_core::ResponseItem; +use codex_core::TransportManager; use codex_core::WEB_SEARCH_ELIGIBLE_HEADER; use codex_core::WireApi; use codex_core::models_manager::manager::ModelsManager; @@ -94,6 +95,7 @@ async fn responses_stream_includes_subagent_header_on_review() { summary, conversation_id, session_source, + TransportManager::new(), ) .new_session(); @@ -191,6 +193,7 @@ async fn responses_stream_includes_subagent_header_on_other() { summary, conversation_id, session_source, + TransportManager::new(), ) .new_session(); @@ -346,6 +349,7 @@ async fn responses_respects_model_info_overrides_from_config() { summary, conversation_id, session_source, + TransportManager::new(), ) .new_session(); diff --git a/codex-rs/core/tests/suite/client.rs b/codex-rs/core/tests/suite/client.rs index bdc2f311c0..8b8adf06ab 100644 --- a/codex-rs/core/tests/suite/client.rs +++ b/codex-rs/core/tests/suite/client.rs @@ -11,6 +11,7 @@ use codex_core::Prompt; use codex_core::ResponseEvent; use codex_core::ResponseItem; use codex_core::ThreadManager; +use codex_core::TransportManager; use codex_core::WireApi; use codex_core::auth::AuthCredentialsStoreMode; use codex_core::built_in_model_providers; @@ -1186,6 +1187,7 @@ async fn azure_responses_request_includes_store_and_reasoning_ids() { summary, conversation_id, SessionSource::Exec, + TransportManager::new(), ) .new_session(); diff --git a/codex-rs/core/tests/suite/client_websockets.rs b/codex-rs/core/tests/suite/client_websockets.rs index d55d71d0d8..60229e01d6 100644 --- a/codex-rs/core/tests/suite/client_websockets.rs +++ b/codex-rs/core/tests/suite/client_websockets.rs @@ -8,6 +8,7 @@ use codex_core::ModelProviderInfo; use codex_core::Prompt; use codex_core::ResponseEvent; use codex_core::ResponseItem; +use codex_core::TransportManager; use codex_core::WireApi; use codex_core::models_manager::manager::ModelsManager; use codex_core::protocol::SessionSource; @@ -228,6 +229,7 @@ async fn websocket_harness(server: &WebSocketTestServer) -> WebsocketTestHarness ReasoningSummary::Auto, conversation_id, SessionSource::Exec, + TransportManager::new(), ); WebsocketTestHarness { diff --git a/codex-rs/core/tests/suite/mod.rs b/codex-rs/core/tests/suite/mod.rs index e054070b34..0744624a92 100644 --- a/codex-rs/core/tests/suite/mod.rs +++ b/codex-rs/core/tests/suite/mod.rs @@ -80,3 +80,4 @@ mod user_notification; mod user_shell_cmd; mod view_image; mod web_search_cached; +mod websocket_fallback; diff --git a/codex-rs/core/tests/suite/websocket_fallback.rs b/codex-rs/core/tests/suite/websocket_fallback.rs new file mode 100644 index 0000000000..d52236a8a2 --- /dev/null +++ b/codex-rs/core/tests/suite/websocket_fallback.rs @@ -0,0 +1,98 @@ +use anyhow::Result; +use codex_core::WireApi; +use core_test_support::responses; +use core_test_support::responses::ev_completed; +use core_test_support::responses::ev_response_created; +use core_test_support::responses::mount_sse_once; +use core_test_support::responses::mount_sse_sequence; +use core_test_support::responses::sse; +use core_test_support::skip_if_no_network; +use core_test_support::test_codex::test_codex; +use pretty_assertions::assert_eq; +use wiremock::http::Method; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn websocket_fallback_switches_to_http_after_retries_exhausted() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let response_mock = mount_sse_once( + &server, + sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]), + ) + .await; + + let mut builder = test_codex().with_config({ + let base_url = format!("{}/v1", server.uri()); + move |config| { + config.model_provider.base_url = Some(base_url); + config.model_provider.wire_api = WireApi::ResponsesWebsocket; + config.model_provider.stream_max_retries = Some(0); + config.model_provider.request_max_retries = Some(0); + } + }); + let test = builder.build(&server).await?; + + test.submit_turn("hello").await?; + + let requests = server.received_requests().await.unwrap_or_default(); + let websocket_attempts = requests + .iter() + .filter(|req| req.method == Method::GET && req.url.path().ends_with("/responses")) + .count(); + let http_attempts = requests + .iter() + .filter(|req| req.method == Method::POST && req.url.path().ends_with("/responses")) + .count(); + + assert_eq!(websocket_attempts, 1); + assert_eq!(http_attempts, 1); + assert_eq!(response_mock.requests().len(), 1); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn websocket_fallback_is_sticky_across_turns() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let response_mock = mount_sse_sequence( + &server, + vec![ + sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]), + sse(vec![ev_response_created("resp-2"), ev_completed("resp-2")]), + ], + ) + .await; + + let mut builder = test_codex().with_config({ + let base_url = format!("{}/v1", server.uri()); + move |config| { + config.model_provider.base_url = Some(base_url); + config.model_provider.wire_api = WireApi::ResponsesWebsocket; + config.model_provider.stream_max_retries = Some(0); + config.model_provider.request_max_retries = Some(0); + } + }); + let test = builder.build(&server).await?; + + test.submit_turn("first").await?; + test.submit_turn("second").await?; + + let requests = server.received_requests().await.unwrap_or_default(); + let websocket_attempts = requests + .iter() + .filter(|req| req.method == Method::GET && req.url.path().ends_with("/responses")) + .count(); + let http_attempts = requests + .iter() + .filter(|req| req.method == Method::POST && req.url.path().ends_with("/responses")) + .count(); + + assert_eq!(websocket_attempts, 1); + assert_eq!(http_attempts, 2); + assert_eq!(response_mock.requests().len(), 2); + + Ok(()) +} From b7f26d74f040b8f967b8f1760177d96eb0e9863c Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 29 Jan 2026 10:49:03 -0800 Subject: [PATCH 27/28] chore: ensure pnpm-workspace.yaml is up-to-date (#10140) On the back of: https://github.com/openai/codex/pull/10138 Let's ensure that every folder with a `package.json` is listed in `pnpm-workspace.yaml` (not sure why `docs` was in there...) and that we are using `pnpm` over `npm` consistently (which is why this PR deletes `codex-cli/package-lock.json`). --- codex-cli/package-lock.json | 18 ------------------ pnpm-lock.yaml | 4 ++++ pnpm-workspace.yaml | 3 ++- 3 files changed, 6 insertions(+), 19 deletions(-) delete mode 100644 codex-cli/package-lock.json diff --git a/codex-cli/package-lock.json b/codex-cli/package-lock.json deleted file mode 100644 index 58ee846306..0000000000 --- a/codex-cli/package-lock.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "@openai/codex", - "version": "0.0.0-dev", - "lockfileVersion": 3, - "packages": { - "": { - "name": "@openai/codex", - "version": "0.0.0-dev", - "license": "Apache-2.0", - "bin": { - "codex": "bin/codex.js" - }, - "engines": { - "node": ">=16" - } - } - } -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 01112d6d3b..102869a683 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -17,6 +17,10 @@ importers: specifier: ^3.5.3 version: 3.5.3 + codex-cli: {} + + codex-rs/responses-api-proxy/npm: {} + sdk/typescript: devDependencies: '@modelcontextprotocol/sdk': diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 78899b9321..09a61536ab 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,5 +1,6 @@ packages: - - docs + - codex-cli + - codex-rs/responses-api-proxy/npm - sdk/typescript - shell-tool-mcp From bdd8a7d58bdd4ea57efe17f24cde4c09fdfea0b0 Mon Sep 17 00:00:00 2001 From: xl-openai Date: Thu, 29 Jan 2026 11:13:30 -0800 Subject: [PATCH 28/28] Better handling skill depdenencies on ENV VAR. (#9017) An experimental flow for env var skill dependencies. Skills can now declare required env vars in SKILL.md; if missing, the CLI prompts the user to get the value, and Core will store it in memory (eventually to a local persistent store) image --- .../app-server-protocol/src/protocol/v2.rs | 2 + .../app-server/src/bespoke_event_handling.rs | 1 + codex-rs/core/config.schema.json | 6 + codex-rs/core/src/codex.rs | 21 +++ codex-rs/core/src/features.rs | 8 + codex-rs/core/src/mcp/skill_dependencies.rs | 1 + .../core/src/skills/env_var_dependencies.rs | 162 ++++++++++++++++++ codex-rs/core/src/skills/mod.rs | 3 + codex-rs/core/src/state/session.rs | 13 ++ codex-rs/core/src/tools/handlers/shell.rs | 6 + codex-rs/protocol/src/request_user_input.rs | 4 + codex-rs/tui/src/bottom_pane/chat_composer.rs | 13 +- .../src/bottom_pane/request_user_input/mod.rs | 6 + .../bottom_pane/request_user_input/render.rs | 9 +- codex-rs/tui/src/bottom_pane/textarea.rs | 36 ++++ 15 files changed, 289 insertions(+), 2 deletions(-) create mode 100644 codex-rs/core/src/skills/env_var_dependencies.rs diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 7b695d2c7a..247360282c 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -2532,6 +2532,8 @@ pub struct ToolRequestUserInputQuestion { pub question: String, #[serde(default)] pub is_other: bool, + #[serde(default)] + pub is_secret: bool, pub options: Option>, } diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index 20e7db762b..1a9fcaa705 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -279,6 +279,7 @@ pub(crate) async fn apply_bespoke_event_handling( header: question.header, question: question.question, is_other: question.is_other, + is_secret: question.is_secret, options: question.options.map(|options| { options .into_iter() diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index 717eddc612..16c0c8945c 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -207,6 +207,9 @@ "shell_tool": { "type": "boolean" }, + "skill_env_var_dependency_prompt": { + "type": "boolean" + }, "skill_mcp_dependency_install": { "type": "boolean" }, @@ -1231,6 +1234,9 @@ "shell_tool": { "type": "boolean" }, + "skill_env_var_dependency_prompt": { + "type": "boolean" + }, "skill_mcp_dependency_install": { "type": "boolean" }, diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 736d0554b3..1b7b7312c7 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -168,10 +168,12 @@ use crate::skills::SkillInjections; use crate::skills::SkillMetadata; use crate::skills::SkillsManager; use crate::skills::build_skill_injections; +use crate::skills::collect_env_var_dependencies; use crate::skills::collect_explicit_skill_mentions; use crate::skills::injection::ToolMentionKind; use crate::skills::injection::app_id_from_path; use crate::skills::injection::tool_kind_for_path; +use crate::skills::resolve_skill_dependencies_for_turn; use crate::state::ActiveTurn; use crate::state::SessionServices; use crate::state::SessionState; @@ -1953,6 +1955,16 @@ impl Session { state.record_mcp_dependency_prompted(names); } + pub async fn dependency_env(&self) -> HashMap { + let state = self.state.lock().await; + state.dependency_env() + } + + pub async fn set_dependency_env(&self, values: HashMap) { + let mut state = self.state.lock().await; + state.set_dependency_env(values); + } + pub(crate) async fn set_server_reasoning_included(&self, included: bool) { let mut state = self.state.lock().await; state.set_server_reasoning_included(included); @@ -3218,6 +3230,15 @@ pub(crate) async fn run_turn( }); let explicit_app_paths = collect_explicit_app_paths(&input); + let config = turn_context.client.config(); + if config + .features + .enabled(Feature::SkillEnvVarDependencyPrompt) + { + let env_var_dependencies = collect_env_var_dependencies(&mentioned_skills); + resolve_skill_dependencies_for_turn(&sess, &turn_context, &env_var_dependencies).await; + } + maybe_prompt_and_install_mcp_dependencies( sess.as_ref(), turn_context.as_ref(), diff --git a/codex-rs/core/src/features.rs b/codex-rs/core/src/features.rs index 9bc52537b7..961b7a56f6 100644 --- a/codex-rs/core/src/features.rs +++ b/codex-rs/core/src/features.rs @@ -115,6 +115,8 @@ pub enum Feature { Apps, /// Allow prompting and installing missing MCP dependencies. SkillMcpDependencyInstall, + /// Prompt for missing skill env var dependencies. + SkillEnvVarDependencyPrompt, /// Steer feature flag - when enabled, Enter submits immediately instead of queuing. Steer, /// Enable collaboration modes (Plan, Code, Pair Programming, Execute). @@ -533,6 +535,12 @@ pub const FEATURES: &[FeatureSpec] = &[ stage: Stage::Stable, default_enabled: true, }, + FeatureSpec { + id: Feature::SkillEnvVarDependencyPrompt, + key: "skill_env_var_dependency_prompt", + stage: Stage::UnderDevelopment, + default_enabled: false, + }, FeatureSpec { id: Feature::Steer, key: "steer", diff --git a/codex-rs/core/src/mcp/skill_dependencies.rs b/codex-rs/core/src/mcp/skill_dependencies.rs index c295a96863..37620d73db 100644 --- a/codex-rs/core/src/mcp/skill_dependencies.rs +++ b/codex-rs/core/src/mcp/skill_dependencies.rs @@ -79,6 +79,7 @@ async fn should_install_mcp_dependencies( "The following MCP servers are required by the selected skills but are not installed yet: {server_list}. Install them now?" ), is_other: false, + is_secret: false, options: Some(vec![ RequestUserInputQuestionOption { label: MCP_DEPENDENCY_OPTION_INSTALL.to_string(), diff --git a/codex-rs/core/src/skills/env_var_dependencies.rs b/codex-rs/core/src/skills/env_var_dependencies.rs new file mode 100644 index 0000000000..00f5bad8cc --- /dev/null +++ b/codex-rs/core/src/skills/env_var_dependencies.rs @@ -0,0 +1,162 @@ +use std::collections::HashMap; +use std::collections::HashSet; +use std::env; +use std::sync::Arc; + +use codex_protocol::request_user_input::RequestUserInputArgs; +use codex_protocol::request_user_input::RequestUserInputQuestion; +use codex_protocol::request_user_input::RequestUserInputResponse; +use tracing::warn; + +use crate::codex::Session; +use crate::codex::TurnContext; +use crate::skills::SkillMetadata; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct SkillDependencyInfo { + pub(crate) skill_name: String, + pub(crate) name: String, + pub(crate) description: Option, +} + +/// Resolve required dependency values (session cache, then env vars), +/// and prompt the UI for any missing ones. +pub(crate) async fn resolve_skill_dependencies_for_turn( + sess: &Arc, + turn_context: &Arc, + dependencies: &[SkillDependencyInfo], +) { + if dependencies.is_empty() { + return; + } + + let existing_env = sess.dependency_env().await; + let mut loaded_values = HashMap::new(); + let mut missing = Vec::new(); + let mut seen_names = HashSet::new(); + + for dependency in dependencies { + let name = dependency.name.clone(); + if !seen_names.insert(name.clone()) { + continue; + } + if existing_env.contains_key(&name) { + continue; + } + match env::var(&name) { + Ok(value) => { + loaded_values.insert(name.clone(), value); + continue; + } + Err(env::VarError::NotPresent) => {} + Err(err) => { + warn!("failed to read env var {name}: {err}"); + } + } + missing.push(dependency.clone()); + } + + if !loaded_values.is_empty() { + sess.set_dependency_env(loaded_values).await; + } + + if !missing.is_empty() { + request_skill_dependencies(sess, turn_context, &missing).await; + } +} + +pub(crate) fn collect_env_var_dependencies( + mentioned_skills: &[SkillMetadata], +) -> Vec { + let mut dependencies = Vec::new(); + for skill in mentioned_skills { + let Some(skill_dependencies) = &skill.dependencies else { + continue; + }; + for tool in &skill_dependencies.tools { + if tool.r#type != "env_var" { + continue; + } + if tool.value.is_empty() { + continue; + } + dependencies.push(SkillDependencyInfo { + skill_name: skill.name.clone(), + name: tool.value.clone(), + description: tool.description.clone(), + }); + } + } + dependencies +} + +/// Prompt via request_user_input to gather missing env vars. +pub(crate) async fn request_skill_dependencies( + sess: &Arc, + turn_context: &Arc, + dependencies: &[SkillDependencyInfo], +) { + let questions = dependencies + .iter() + .map(|dep| { + let requirement = dep.description.as_ref().map_or_else( + || format!("The skill \"{}\" requires \"{}\" to be set.", dep.skill_name, dep.name), + |description| { + format!( + "The skill \"{}\" requires \"{}\" to be set ({}).", + dep.skill_name, dep.name, description + ) + }, + ); + let question = format!( + "{requirement} This is an experimental internal feature. The value is stored in memory for this session only.", + ); + RequestUserInputQuestion { + id: dep.name.clone(), + header: "Skill requires environment variable".to_string(), + question, + is_other: false, + is_secret: true, + options: None, + } + }) + .collect::>(); + + if questions.is_empty() { + return; + } + + let args = RequestUserInputArgs { questions }; + let call_id = format!("skill-deps-{}", turn_context.sub_id); + let response = sess + .request_user_input(turn_context, call_id, args) + .await + .unwrap_or_else(|| RequestUserInputResponse { + answers: HashMap::new(), + }); + + if response.answers.is_empty() { + return; + } + + let mut values = HashMap::new(); + for (name, answer) in response.answers { + let mut user_note = None; + for entry in &answer.answers { + if let Some(note) = entry.strip_prefix("user_note: ") + && !note.trim().is_empty() + { + user_note = Some(note.trim().to_string()); + } + } + if let Some(value) = user_note { + values.insert(name, value); + } + } + + if values.is_empty() { + return; + } + + sess.set_dependency_env(values).await; +} diff --git a/codex-rs/core/src/skills/mod.rs b/codex-rs/core/src/skills/mod.rs index ae4ffa2920..4d79bca258 100644 --- a/codex-rs/core/src/skills/mod.rs +++ b/codex-rs/core/src/skills/mod.rs @@ -1,3 +1,4 @@ +mod env_var_dependencies; pub mod injection; pub mod loader; pub mod manager; @@ -5,6 +6,8 @@ pub mod model; pub mod render; pub mod system; +pub(crate) use env_var_dependencies::collect_env_var_dependencies; +pub(crate) use env_var_dependencies::resolve_skill_dependencies_for_turn; pub(crate) use injection::SkillInjections; pub(crate) use injection::build_skill_injections; pub(crate) use injection::collect_explicit_skill_mentions; diff --git a/codex-rs/core/src/state/session.rs b/codex-rs/core/src/state/session.rs index 9d3d96db99..deee0d0c7c 100644 --- a/codex-rs/core/src/state/session.rs +++ b/codex-rs/core/src/state/session.rs @@ -1,6 +1,7 @@ //! Session-wide mutable state. use codex_protocol::models::ResponseItem; +use std::collections::HashMap; use std::collections::HashSet; use crate::codex::SessionConfiguration; @@ -16,6 +17,7 @@ pub(crate) struct SessionState { pub(crate) history: ContextManager, pub(crate) latest_rate_limits: Option, pub(crate) server_reasoning_included: bool, + pub(crate) dependency_env: HashMap, pub(crate) mcp_dependency_prompted: HashSet, /// Whether the session's initial context has been seeded into history. /// @@ -33,6 +35,7 @@ impl SessionState { history, latest_rate_limits: None, server_reasoning_included: false, + dependency_env: HashMap::new(), mcp_dependency_prompted: HashSet::new(), initial_context_seeded: false, } @@ -112,6 +115,16 @@ impl SessionState { pub(crate) fn mcp_dependency_prompted(&self) -> HashSet { self.mcp_dependency_prompted.clone() } + + pub(crate) fn set_dependency_env(&mut self, values: HashMap) { + for (key, value) in values { + self.dependency_env.insert(key, value); + } + } + + pub(crate) fn dependency_env(&self) -> HashMap { + self.dependency_env.clone() + } } // Sometimes new snapshots don't include credits or plan information. diff --git a/codex-rs/core/src/tools/handlers/shell.rs b/codex-rs/core/src/tools/handlers/shell.rs index dbfb0364bf..b59c0140d1 100644 --- a/codex-rs/core/src/tools/handlers/shell.rs +++ b/codex-rs/core/src/tools/handlers/shell.rs @@ -233,6 +233,12 @@ impl ShellHandler { None }; + let mut exec_params = exec_params; + let dependency_env = session.dependency_env().await; + if !dependency_env.is_empty() { + exec_params.env.extend(dependency_env); + } + // Approval policy guard for explicit escalation in non-OnRequest modes. if exec_params .sandbox_permissions diff --git a/codex-rs/protocol/src/request_user_input.rs b/codex-rs/protocol/src/request_user_input.rs index 0e58348da4..cb076264dd 100644 --- a/codex-rs/protocol/src/request_user_input.rs +++ b/codex-rs/protocol/src/request_user_input.rs @@ -20,6 +20,10 @@ pub struct RequestUserInputQuestion { #[schemars(rename = "isOther")] #[ts(rename = "isOther")] pub is_other: bool, + #[serde(rename = "isSecret", default)] + #[schemars(rename = "isSecret")] + #[ts(rename = "isSecret")] + pub is_secret: bool, #[serde(skip_serializing_if = "Option::is_none")] pub options: Option>, } diff --git a/codex-rs/tui/src/bottom_pane/chat_composer.rs b/codex-rs/tui/src/bottom_pane/chat_composer.rs index 9d0cbf22e5..90cae4b4a0 100644 --- a/codex-rs/tui/src/bottom_pane/chat_composer.rs +++ b/codex-rs/tui/src/bottom_pane/chat_composer.rs @@ -2854,6 +2854,12 @@ impl Renderable for ChatComposer { } fn render(&self, area: Rect, buf: &mut Buffer) { + self.render_with_mask(area, buf, None); + } +} + +impl ChatComposer { + pub(crate) fn render_with_mask(&self, area: Rect, buf: &mut Buffer, mask_char: Option) { let [composer_rect, textarea_rect, popup_rect] = self.layout_areas(area); match &self.active_popup { ActivePopup::Command(popup) => { @@ -3018,7 +3024,12 @@ impl Renderable for ChatComposer { } let mut state = self.textarea_state.borrow_mut(); - StatefulWidgetRef::render_ref(&(&self.textarea), textarea_rect, buf, &mut state); + if let Some(mask_char) = mask_char { + self.textarea + .render_ref_masked(textarea_rect, buf, &mut state, mask_char); + } else { + StatefulWidgetRef::render_ref(&(&self.textarea), textarea_rect, buf, &mut state); + } if self.textarea.text().is_empty() { let text = if self.input_enabled { self.placeholder_text.as_str().to_string() diff --git a/codex-rs/tui/src/bottom_pane/request_user_input/mod.rs b/codex-rs/tui/src/bottom_pane/request_user_input/mod.rs index 979499835d..8fe3401693 100644 --- a/codex-rs/tui/src/bottom_pane/request_user_input/mod.rs +++ b/codex-rs/tui/src/bottom_pane/request_user_input/mod.rs @@ -1251,6 +1251,7 @@ mod tests { header: header.to_string(), question: "Choose an option.".to_string(), is_other: false, + is_secret: false, options: Some(vec![ RequestUserInputQuestionOption { label: "Option 1".to_string(), @@ -1274,6 +1275,7 @@ mod tests { header: header.to_string(), question: "Choose an option.".to_string(), is_other: true, + is_secret: false, options: Some(vec![ RequestUserInputQuestionOption { label: "Option 1".to_string(), @@ -1297,6 +1299,7 @@ mod tests { header: header.to_string(), question: "Choose the next step for this task.".to_string(), is_other: false, + is_secret: false, options: Some(vec![ RequestUserInputQuestionOption { label: "Discuss a code change".to_string(), @@ -1326,6 +1329,7 @@ mod tests { header: header.to_string(), question: "Share details.".to_string(), is_other: false, + is_secret: false, options: None, } } @@ -2385,6 +2389,7 @@ mod tests { header: "Next Step".to_string(), question: "What would you like to do next?".to_string(), is_other: false, + is_secret: false, options: Some(vec![ RequestUserInputQuestionOption { label: "Discuss a code change (Recommended)".to_string(), @@ -2436,6 +2441,7 @@ mod tests { header: "Next Step".to_string(), question: "What would you like to do next?".to_string(), is_other: false, + is_secret: false, options: Some(vec![ RequestUserInputQuestionOption { label: "Discuss a code change (Recommended)".to_string(), diff --git a/codex-rs/tui/src/bottom_pane/request_user_input/render.rs b/codex-rs/tui/src/bottom_pane/request_user_input/render.rs index b7433f8a6d..2180ec0553 100644 --- a/codex-rs/tui/src/bottom_pane/request_user_input/render.rs +++ b/codex-rs/tui/src/bottom_pane/request_user_input/render.rs @@ -414,7 +414,14 @@ impl RequestUserInputOverlay { if area.width == 0 || area.height == 0 { return; } - self.composer.render(area, buf); + let is_secret = self + .current_question() + .is_some_and(|question| question.is_secret); + if is_secret { + self.composer.render_with_mask(area, buf, Some('*')); + } else { + self.composer.render(area, buf); + } } } diff --git a/codex-rs/tui/src/bottom_pane/textarea.rs b/codex-rs/tui/src/bottom_pane/textarea.rs index 0235f7a9c1..42f7b09bef 100644 --- a/codex-rs/tui/src/bottom_pane/textarea.rs +++ b/codex-rs/tui/src/bottom_pane/textarea.rs @@ -1146,6 +1146,22 @@ impl StatefulWidgetRef for &TextArea { } impl TextArea { + pub(crate) fn render_ref_masked( + &self, + area: Rect, + buf: &mut Buffer, + state: &mut TextAreaState, + mask_char: char, + ) { + let lines = self.wrapped_lines(area.width); + let scroll = self.effective_scroll(area.height, &lines, state.scroll); + state.scroll = scroll; + + let start = scroll as usize; + let end = (scroll + area.height).min(lines.len() as u16) as usize; + self.render_lines_masked(area, buf, &lines, start..end, mask_char); + } + fn render_lines( &self, area: Rect, @@ -1175,6 +1191,26 @@ impl TextArea { } } } + + fn render_lines_masked( + &self, + area: Rect, + buf: &mut Buffer, + lines: &[Range], + range: std::ops::Range, + mask_char: char, + ) { + for (row, idx) in range.enumerate() { + let r = &lines[idx]; + let y = area.y + row as u16; + let line_range = r.start..r.end - 1; + let masked = self.text[line_range.clone()] + .chars() + .map(|_| mask_char) + .collect::(); + buf.set_string(area.x, y, &masked, Style::default()); + } + } } #[cfg(test)]