openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-11-10 21:00:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into alexs/tui-migrate

Browse Source
This commit is contained in:
alexsong-oai
2026-04-15 14:35:25 -07:00
parent 6759e3266b e70ccdeaf7
commit 9521fa3497
201 changed files with 9396 additions and 2693 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/bazel.yml
+1 -1
View File
@@ -94,7 +94,7 @@ jobs:
"${bazel_wrapper_args[@]}" \
-- \
test \
--test_tag_filters=-argument-comment-lint \
--test_tag_filters=-argument-comment-lint,-flaky \
--test_verbose_timeout_warnings \
--build_metadata=COMMIT_SHA=${GITHUB_SHA} \
-- \
.github/workflows/rust-ci-full.yml
+1
View File
@@ -664,6 +664,7 @@ jobs:
export CODEX_TEST_REMOTE_ENV_CONTAINER_NAME=codex-remote-test-env
source "${GITHUB_WORKSPACE}/scripts/test-remote-env.sh"
echo "CODEX_TEST_REMOTE_ENV=${CODEX_TEST_REMOTE_ENV}" >> "$GITHUB_ENV"
echo "CODEX_TEST_REMOTE_EXEC_SERVER_URL=${CODEX_TEST_REMOTE_EXEC_SERVER_URL}" >> "$GITHUB_ENV"
- name: tests
id: test
MODULE.bazel.lock
Generated
+4 -2
View File
File diff suppressed because one or more lines are too long
SECURITY.md
+4
View File
@@ -11,3 +11,7 @@ Our security program is managed through Bugcrowd, and we ask that any validated
## Vulnerability Disclosure Program
Our Vulnerability Program Guidelines are defined on our [Bugcrowd program page](https://bugcrowd.com/engagements/openai).
## How to operate CODEX safely
For details on Codex security boundaries, including sandboxing, approvals, and network controls, see [Agent approvals & security](https://developers.openai.com/codex/agent-approvals-security).
announcement_tip.toml
+6 -12
View File
@@ -4,20 +4,14 @@
# version_regex matches against the CLI version (env!("CARGO_PKG_VERSION")); omit to apply to all versions.
# target_app specify which app should display the announcement (cli, vsce, ...).
[[announcements]]
content = "Welcome to Codex! Check out the new onboarding flow."
from_date = "2024-10-01"
to_date = "2024-10-15"
target_app = "cli"
# Test announcement only for local build version until 2026-01-10 excluded (past)
# Test announcement only for local build version until 2027-05-10 excluded
[[announcements]]
content = "This is a test announcement"
version_regex = "^0\\.0\\.0$"
to_date = "2026-05-10"
to_date = "2027-05-10"
[[announcements]]
content = "**BREAKING NEWS**: `gpt-5.3-codex` is out! Upgrade to `0.98.0` for a faster, smarter, more steerable agent."
from_date = "2026-02-01"
to_date = "2026-02-16"
version_regex = "^0\\.(?:[0-9]|[1-8][0-9]|9[0-7])\\."
content = "Update Required - This version will no longer be supported starting May 8th. Please upgrade to the latest version (https://github.com/openai/codex/releases/latest) using your preferred package manager."
# Matches 0.x.y versions from 0.0.y through 0.119.y; excludes 0.120.0 and newer.
version_regex = "^0\\.(?:[0-9]|[1-9][0-9]|1[01][0-9])\\."
to_date = "2026-05-08"
codex-rs/.cargo/audit.toml
+2 -1
View File
@@ -1,9 +1,10 @@
[advisories]
# Reviewed 2026-04-11. Keep this list in sync with ../deny.toml.
# Reviewed 2026-04-15. Keep this list in sync with ../deny.toml.
ignore = [
"RUSTSEC-2024-0388", # derivative 2.2.0 via starlark; upstream crate is unmaintained
"RUSTSEC-2025-0057", # fxhash 0.2.1 via starlark_map; upstream crate is unmaintained
"RUSTSEC-2024-0436", # paste 1.0.15 via starlark/ratatui; upstream crate is unmaintained
"RUSTSEC-2024-0320", # yaml-rust via syntect; remove when syntect drops or updates it
"RUSTSEC-2025-0141", # bincode via syntect; remove when syntect drops or updates it
"RUSTSEC-2026-0097", # rand 0.8.5 via age/codex-secrets and zbus/keyring; remove when transitive deps move to rand >=0.9.3
]
codex-rs/Cargo.lock
Generated
+67 -25
View File
@@ -1461,6 +1461,7 @@ dependencies = [
"codex-sandboxing",
"codex-shell-command",
"codex-state",
"codex-thread-store",
"codex-tools",
"codex-utils-absolute-path",
"codex-utils-cargo-bin",
@@ -1694,6 +1695,7 @@ dependencies = [
"codex-stdio-to-uds",
"codex-terminal-detection",
"codex-tui",
"codex-utils-absolute-path",
"codex-utils-cargo-bin",
"codex-utils-cli",
"codex-utils-path",
@@ -1728,7 +1730,7 @@ dependencies = [
"opentelemetry",
"opentelemetry_sdk",
"pretty_assertions",
"rand 0.9.2",
"rand 0.9.3",
"reqwest",
"rustls",
"rustls-native-certs",
@@ -1830,6 +1832,7 @@ name = "codex-code-mode"
version = "0.0.0"
dependencies = [
"async-trait",
"codex-protocol",
"deno_core_icudata",
"pretty_assertions",
"serde",
@@ -1905,6 +1908,7 @@ dependencies = [
"codex-api",
"codex-app-server-protocol",
"codex-apply-patch",
"codex-arg0",
"codex-async-utils",
"codex-code-mode",
"codex-config",
@@ -1935,6 +1939,7 @@ dependencies = [
"codex-state",
"codex-terminal-detection",
"codex-test-binary-support",
"codex-thread-store",
"codex-tools",
"codex-utils-absolute-path",
"codex-utils-cache",
@@ -1956,6 +1961,7 @@ dependencies = [
"ctor 0.6.3",
"dirs",
"dunce",
"ed25519-dalek",
"env-flags",
"eventsource-stream",
"futures",
@@ -1973,7 +1979,7 @@ dependencies = [
"opentelemetry_sdk",
"predicates",
"pretty_assertions",
"rand 0.9.2",
"rand 0.9.3",
"regex-lite",
"reqwest",
"rmcp",
@@ -2326,7 +2332,7 @@ dependencies = [
"once_cell",
"os_info",
"pretty_assertions",
"rand 0.9.2",
"rand 0.9.3",
"regex-lite",
"reqwest",
"serde",
@@ -2694,6 +2700,8 @@ dependencies = [
name = "codex-sandboxing"
version = "0.0.0"
dependencies = [
"anyhow",
"async-trait",
"codex-network-proxy",
"codex-protocol",
"codex-utils-absolute-path",
@@ -2702,6 +2710,7 @@ dependencies = [
"pretty_assertions",
"serde_json",
"tempfile",
"tokio",
"tracing",
"url",
"which 8.0.0",
@@ -2718,7 +2727,7 @@ dependencies = [
"codex-keyring-store",
"keyring",
"pretty_assertions",
"rand 0.9.2",
"rand 0.9.3",
"regex",
"schemars 0.8.22",
"serde",
@@ -2834,9 +2843,17 @@ version = "0.0.0"
dependencies = [
"async-trait",
"chrono",
"codex-git-utils",
"codex-protocol",
"codex-rollout",
"codex-state",
"pretty_assertions",
"serde",
"serde_json",
"tempfile",
"thiserror 2.0.18",
"tokio",
"uuid",
]
[[package]]
@@ -2917,7 +2934,7 @@ dependencies = [
"pathdiff",
"pretty_assertions",
"pulldown-cmark",
"rand 0.9.2",
"rand 0.9.3",
"ratatui",
"ratatui-macros",
"regex-lite",
@@ -3650,6 +3667,7 @@ dependencies = [
"cfg-if",
"cpufeatures",
"curve25519-dalek-derive",
"digest",
"fiat-crypto",
"rustc_version",
"subtle",
@@ -4126,7 +4144,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users 0.5.2",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -4230,6 +4248,30 @@ version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
[[package]]
name = "ed25519"
version = "2.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53"
dependencies = [
"pkcs8",
"signature",
]
[[package]]
name = "ed25519-dalek"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9"
dependencies = [
"curve25519-dalek",
"ed25519",
"serde",
"sha2",
"subtle",
"zeroize",
]
[[package]]
name = "either"
version = "1.15.0"
@@ -5159,7 +5201,7 @@ dependencies = [
"idna",
"ipnet",
"once_cell",
"rand 0.9.2",
"rand 0.9.3",
"ring",
"thiserror 2.0.18",
"tinyvec",
@@ -5181,7 +5223,7 @@ dependencies = [
"moka",
"once_cell",
"parking_lot",
"rand 0.9.2",
"rand 0.9.3",
"resolv-conf",
"smallvec",
"thiserror 2.0.18",
@@ -6728,7 +6770,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -7327,7 +7369,7 @@ dependencies = [
"futures-util",
"opentelemetry",
"percent-encoding",
"rand 0.9.2",
"rand 0.9.3",
"thiserror 2.0.18",
"tokio",
"tokio-stream",
@@ -7876,7 +7918,7 @@ checksum = "bee689443a2bd0a16ab0348b52ee43e3b2d1b1f931c8aa5c9f8de4c86fbe8c40"
dependencies = [
"bitflags 2.10.0",
"num-traits",
"rand 0.9.2",
"rand 0.9.3",
"rand_chacha 0.9.0",
"rand_xorshift",
"regex-syntax 0.8.8",
@@ -8047,7 +8089,7 @@ dependencies = [
"bytes",
"getrandom 0.3.4",
"lru-slab",
"rand 0.9.2",
"rand 0.9.3",
"ring",
"rustc-hash 2.1.1",
"rustls",
@@ -8179,7 +8221,7 @@ dependencies = [
"rama-http-types",
"rama-net",
"rama-utils",
"rand 0.9.2",
"rand 0.9.3",
"serde",
"serde_html_form",
"serde_json",
@@ -8249,7 +8291,7 @@ dependencies = [
"rama-macros",
"rama-net",
"rama-utils",
"rand 0.9.2",
"rand 0.9.3",
"serde",
"sha1",
]
@@ -8277,7 +8319,7 @@ dependencies = [
"rama-error",
"rama-macros",
"rama-utils",
"rand 0.9.2",
"rand 0.9.3",
"serde",
"serde_json",
"sync_wrapper",
@@ -8351,7 +8393,7 @@ dependencies = [
"rama-http-types",
"rama-net",
"rama-utils",
"rand 0.9.2",
"rand 0.9.3",
"tokio",
]
@@ -8431,9 +8473,9 @@ dependencies = [
[[package]]
name = "rand"
version = "0.9.2"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
checksum = "7ec095654a25171c2124e9e3393a930bddbffdc939556c914957a4c3e0a87166"
dependencies = [
"rand_chacha 0.9.0",
"rand_core 0.9.5",
@@ -8748,7 +8790,7 @@ dependencies = [
"pastey",
"pin-project-lite",
"process-wrap",
"rand 0.9.2",
"rand 0.9.3",
"reqwest",
"rmcp-macros",
"schemars 1.2.1",
@@ -8955,9 +8997,9 @@ dependencies = [
[[package]]
name = "rustls-webpki"
version = "0.103.10"
version = "0.103.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef"
checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06"
dependencies = [
"aws-lc-rs",
"ring",
@@ -9332,7 +9374,7 @@ version = "0.46.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26ab054c34b87f96c3e4701bea1888317cde30cc7e4a6136d2c48454ab96661c"
dependencies = [
"rand 0.9.2",
"rand 0.9.3",
"sentry-types",
"serde",
"serde_json",
@@ -9380,7 +9422,7 @@ checksum = "eecbd63e9d15a26a40675ed180d376fcb434635d2e33de1c24003f61e3e2230d"
dependencies = [
"debugid",
"hex",
"rand 0.9.2",
"rand 0.9.3",
"serde",
"serde_json",
"thiserror 2.0.18",
@@ -11165,7 +11207,7 @@ dependencies = [
"http 1.4.0",
"httparse",
"log",
"rand 0.9.2",
"rand 0.9.3",
"rustls",
"rustls-pki-types",
"sha1",
@@ -11852,7 +11894,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.48.0",
"windows-sys 0.61.2",
]
[[package]]
codex-rs/Cargo.toml
+2
View File
@@ -166,6 +166,7 @@ codex-state = { path = "state" }
codex-stdio-to-uds = { path = "stdio-to-uds" }
codex-terminal-detection = { path = "terminal-detection" }
codex-test-binary-support = { path = "test-binary-support" }
codex-thread-store = { path = "thread-store" }
codex-tools = { path = "tools" }
codex-tui = { path = "tui" }
codex-utils-absolute-path = { path = "utils/absolute-path" }
@@ -227,6 +228,7 @@ diffy = "0.4.2"
dirs = "6"
dotenvy = "0.15.7"
dunce = "1.0.4"
ed25519-dalek = { version = "2.2.0", features = ["pkcs8"] }
encoding_rs = "0.8.35"
env-flags = "0.1.1"
env_logger = "0.11.9"
codex-rs/app-server-client/src/lib.rs
+6
View File
@@ -28,6 +28,7 @@ use std::time::Duration;
pub use codex_app_server::in_process::DEFAULT_IN_PROCESS_CHANNEL_CAPACITY;
pub use codex_app_server::in_process::InProcessServerEvent;
use codex_app_server::in_process::InProcessStartArgs;
use codex_app_server::in_process::LogDbLayer;
use codex_app_server_protocol::ClientInfo;
use codex_app_server_protocol::ClientNotification;
use codex_app_server_protocol::ClientRequest;
@@ -354,6 +355,8 @@ pub struct InProcessClientStartArgs {
pub cloud_requirements: CloudRequirementsLoader,
/// Feedback sink used by app-server/core telemetry and logs.
pub feedback: CodexFeedback,
/// SQLite tracing layer used to flush recently emitted logs before feedback upload.
pub log_db: Option<LogDbLayer>,
/// Environment manager used by core execution and filesystem operations.
pub environment_manager: Arc<EnvironmentManager>,
/// Startup warnings emitted after initialize succeeds.
@@ -405,6 +408,7 @@ impl InProcessClientStartArgs {
loader_overrides: self.loader_overrides,
cloud_requirements: self.cloud_requirements,
feedback: self.feedback,
log_db: self.log_db,
environment_manager: self.environment_manager,
config_warnings: self.config_warnings,
session_source: self.session_source,
@@ -982,6 +986,7 @@ mod tests {
loader_overrides: LoaderOverrides::default(),
cloud_requirements: CloudRequirementsLoader::default(),
feedback: CodexFeedback::new(),
log_db: None,
environment_manager: Arc::new(EnvironmentManager::new(/*exec_server_url*/ None)),
config_warnings: Vec::new(),
session_source,
@@ -1994,6 +1999,7 @@ mod tests {
loader_overrides: LoaderOverrides::default(),
cloud_requirements: CloudRequirementsLoader::default(),
feedback: CodexFeedback::new(),
log_db: None,
environment_manager: environment_manager.clone(),
config_warnings: Vec::new(),
session_source: SessionSource::Exec,
codex-rs/app-server-protocol/src/protocol/common.rs
+5
View File
@@ -289,6 +289,11 @@ client_request_definitions! {
params: v2::ThreadMemoryModeSetParams,
response: v2::ThreadMemoryModeSetResponse,
},
#[experimental("memory/reset")]
MemoryReset => "memory/reset" {
params: #[ts(type = "undefined")] #[serde(skip_serializing_if = "Option::is_none")] Option<()>,
response: v2::MemoryResetResponse,
},
ThreadUnarchive => "thread/unarchive" {
params: v2::ThreadUnarchiveParams,
response: v2::ThreadUnarchiveResponse,
codex-rs/app-server-protocol/src/protocol/v2.rs
+5
View File
@@ -3114,6 +3114,11 @@ pub struct ThreadMemoryModeSetParams {
#[ts(export_to = "v2/")]
pub struct ThreadMemoryModeSetResponse {}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export_to = "v2/")]
pub struct MemoryResetResponse {}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export_to = "v2/")]
codex-rs/app-server/BUILD.bazel
+3
View File
@@ -3,6 +3,9 @@ load("//:defs.bzl", "codex_rust_crate")
codex_rust_crate(
name = "app-server",
crate_name = "codex_app_server",
integration_test_tags_extra_by_stem = {
"all": ["flaky"],
},
integration_test_timeout = "long",
test_tags = ["no-sandbox"],
)
codex-rs/app-server/Cargo.toml
+1
View File
@@ -54,6 +54,7 @@ codex-rmcp-client = { workspace = true }
codex-rollout = { workspace = true }
codex-sandboxing = { workspace = true }
codex-state = { workspace = true }
codex-thread-store = { workspace = true }
codex-tools = { workspace = true }
codex-utils-absolute-path = { workspace = true }
codex-utils-json-to-toml = { workspace = true }
codex-rs/app-server/README.md
+11
View File
@@ -41,8 +41,11 @@ Security note:
- Non-loopback websocket listeners currently allow unauthenticated connections by default during rollout. If you expose one remotely, configure websocket auth explicitly now.
- Supported auth modes are app-server flags:
- `--ws-auth capability-token --ws-token-file /absolute/path`
- `--ws-auth capability-token --ws-token-sha256 HEX`
- `--ws-auth signed-bearer-token --ws-shared-secret-file /absolute/path` for HMAC-signed JWT/JWS bearer tokens, with optional `--ws-issuer`, `--ws-audience`, `--ws-max-clock-skew-seconds`
- Clients present the credential as `Authorization: Bearer <token>` during the websocket handshake. Auth is enforced before JSON-RPC `initialize`.
- When starting `codex app-server` manually, prefer `--ws-token-file` over passing raw bearer tokens on the command line. Store a high-entropy token in a file readable only by your user, then have your client present that token in the websocket `Authorization` header.
- `--ws-token-sha256` is intended for clients that keep the raw token in a separate local secret store and only need the server to know the SHA-256 verifier. The hash may appear in process listings, but it is not sufficient to authenticate; clients still need the original raw token. Only use this mode with randomly generated high-entropy tokens, not passwords or other guessable values.
Tracing/log output:
@@ -141,6 +144,7 @@ Example with notification opt-out:
- `thread/read` — read a stored thread by id without resuming it; optionally include turns via `includeTurns`. The returned `thread` includes `status` (`ThreadStatus`), defaulting to `notLoaded` when the thread is not currently loaded.
- `thread/metadata/update` — patch stored thread metadata in sqlite; currently supports updating persisted `gitInfo` fields and returns the refreshed `thread`.
- `thread/memoryMode/set` — experimental; set a thread’s persisted memory eligibility to `"enabled"` or `"disabled"` for either a loaded thread or a stored rollout; returns `{}` on success.
- `memory/reset` — experimental; clear the current `CODEX_HOME/memories` directory and reset persisted memory stage data in sqlite while preserving existing thread memory modes; returns `{}` on success.
- `thread/status/changed` — notification emitted when a loaded thread’s status changes (`threadId` + new `status`).
- `thread/archive` — move a thread’s rollout file into the archived directory; returns `{}` on success and emits `thread/archived`.
- `thread/unsubscribe` — unsubscribe this connection from thread turn/item events. If this was the last subscriber, the server keeps the thread loaded and unloads it only after it has had no subscribers and no thread activity for 30 minutes, then emits `thread/closed`.
@@ -413,6 +417,13 @@ Experimental: use `thread/memoryMode/set` to change whether a thread remains eli
{ "id": 26, "result": {} }
```
Experimental: use `memory/reset` to clear local memory artifacts and sqlite-backed memory stage data for the current Codex home. This preserves existing thread memory modes; use `thread/memoryMode/set` separately when a thread's future memory eligibility should change.
```json
{ "method": "memory/reset", "id": 27 }
{ "id": 27, "result": {} }
```
### Example: Archive a thread
Use `thread/archive` to move the persisted rollout (stored as a JSONL file on disk) into the archived sessions directory.
codex-rs/app-server/src/codex_message_processor.rs
+206 -389
View File
@@ -92,6 +92,7 @@ use codex_app_server_protocol::McpServerStatus;
use codex_app_server_protocol::McpServerStatusDetail;
use codex_app_server_protocol::McpServerToolCallParams;
use codex_app_server_protocol::McpServerToolCallResponse;
use codex_app_server_protocol::MemoryResetResponse;
use codex_app_server_protocol::MockExperimentalMethodParams;
use codex_app_server_protocol::MockExperimentalMethodResponse;
use codex_app_server_protocol::ModelListParams;
@@ -202,7 +203,6 @@ use codex_chatgpt::connectors;
use codex_cloud_requirements::cloud_requirements_loader;
use codex_config::types::McpServerTransportConfig;
use codex_core::CodexThread;
use codex_core::Cursor as RolloutCursor;
use codex_core::ForkSnapshot;
use codex_core::NewThread;
use codex_core::RolloutRecorder;
@@ -210,7 +210,6 @@ use codex_core::SessionMeta;
use codex_core::SteerInputError;
use codex_core::ThreadConfigSnapshot;
use codex_core::ThreadManager;
use codex_core::ThreadSortKey as CoreThreadSortKey;
use codex_core::append_thread_name;
use codex_core::config::Config;
use codex_core::config::ConfigOverrides;
@@ -231,7 +230,6 @@ use codex_core::find_archived_thread_path_by_id_str;
use codex_core::find_thread_name_by_id;
use codex_core::find_thread_names_by_ids;
use codex_core::find_thread_path_by_id_str;
use codex_core::parse_cursor;
use codex_core::path_utils;
use codex_core::plugins::MarketplaceAddError;
use codex_core::plugins::MarketplaceError;
@@ -246,7 +244,6 @@ use codex_core::plugins::load_plugin_apps;
use codex_core::plugins::load_plugin_mcp_servers;
use codex_core::read_head_for_summary;
use codex_core::read_session_meta_line;
use codex_core::rollout_date_parts;
use codex_core::sandboxing::SandboxPermissions;
use codex_core::windows_sandbox::WindowsSandboxLevelExt;
use codex_core::windows_sandbox::WindowsSandboxSetupMode as CoreWindowsSandboxSetupMode;
@@ -321,15 +318,20 @@ use codex_state::StateRuntime;
use codex_state::ThreadMetadata;
use codex_state::ThreadMetadataBuilder;
use codex_state::log_db::LogDbLayer;
use codex_thread_store::ArchiveThreadParams as StoreArchiveThreadParams;
use codex_thread_store::ListThreadsParams as StoreListThreadsParams;
use codex_thread_store::LocalThreadStore;
use codex_thread_store::ReadThreadParams as StoreReadThreadParams;
use codex_thread_store::StoredThread;
use codex_thread_store::ThreadSortKey as StoreThreadSortKey;
use codex_thread_store::ThreadStore;
use codex_thread_store::ThreadStoreError;
use codex_utils_absolute_path::AbsolutePathBuf;
use codex_utils_json_to_toml::json_to_toml;
use codex_utils_pty::DEFAULT_OUTPUT_BYTES_CAP;
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::collections::HashSet;
use std::ffi::OsStr;
use std::fs::FileTimes;
use std::fs::OpenOptions;
use std::io::Error as IoError;
use std::path::Path;
use std::path::PathBuf;
@@ -339,7 +341,6 @@ use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;
use std::time::Duration;
use std::time::Instant;
use std::time::SystemTime;
use tokio::sync::Mutex;
use tokio::sync::broadcast;
use tokio::sync::oneshot;
@@ -443,6 +444,7 @@ pub(crate) struct CodexMessageProcessor {
analytics_events_client: AnalyticsEventsClient,
arg0_paths: Arg0DispatchPaths,
config: Arc<Config>,
thread_store: LocalThreadStore,
cli_overrides: Arc<RwLock<Vec<(String, TomlValue)>>>,
runtime_feature_enablement: Arc<RwLock<BTreeMap<String, bool>>>,
cloud_requirements: Arc<RwLock<CloudRequirementsLoader>>,
@@ -701,6 +703,7 @@ impl CodexMessageProcessor {
outgoing: outgoing.clone(),
analytics_events_client,
arg0_paths,
thread_store: LocalThreadStore::new(codex_rollout::RolloutConfig::from_view(&config)),
config,
cli_overrides,
runtime_feature_enablement,
@@ -908,6 +911,10 @@ impl CodexMessageProcessor {
self.thread_memory_mode_set(to_connection_request_id(request_id), params)
.await;
}
ClientRequest::MemoryReset { request_id, params } => {
self.memory_reset(to_connection_request_id(request_id), params)
.await;
}
ClientRequest::ThreadUnarchive { request_id, params } => {
self.thread_unarchive(to_connection_request_id(request_id), params)
.await;
@@ -2709,7 +2716,6 @@ impl CodexMessageProcessor {
}
async fn thread_archive(&self, request_id: ConnectionRequestId, params: ThreadArchiveParams) {
// TODO(jif) mostly rewrite this using sqlite after phase 1
let thread_id = match ThreadId::from_string(&params.thread_id) {
Ok(id) => id,
Err(err) => {
@@ -2723,32 +2729,28 @@ impl CodexMessageProcessor {
}
};
let rollout_path =
match find_thread_path_by_id_str(&self.config.codex_home, &thread_id.to_string()).await
{
Ok(Some(p)) => p,
Ok(None) => {
let error = JSONRPCErrorError {
code: INVALID_REQUEST_ERROR_CODE,
message: format!("no rollout found for thread id {thread_id}"),
data: None,
};
self.outgoing.send_error(request_id, error).await;
return;
}
Err(err) => {
let error = JSONRPCErrorError {
code: INVALID_REQUEST_ERROR_CODE,
message: format!("failed to locate thread id {thread_id}: {err}"),
data: None,
};
self.outgoing.send_error(request_id, error).await;
return;
}
};
let thread_id_str = thread_id.to_string();
match self.archive_thread_common(thread_id, &rollout_path).await {
if let Err(err) = self
.thread_store
.read_thread(StoreReadThreadParams {
thread_id,
include_archived: false,
include_history: false,
})
.await
{
self.outgoing
.send_error(request_id, thread_store_archive_error("archive", err))
.await;
return;
}
self.prepare_thread_for_archive(thread_id).await;
match self
.thread_store
.archive_thread(StoreArchiveThreadParams { thread_id })
.await
{
Ok(()) => {
let response = ThreadArchiveResponse {};
self.outgoing.send_response(request_id, response).await;
@@ -2760,7 +2762,9 @@ impl CodexMessageProcessor {
.await;
}
Err(err) => {
self.outgoing.send_error(request_id, err).await;
self.outgoing
.send_error(request_id, thread_store_archive_error("archive", err))
.await;
}
}
}
@@ -3046,6 +3050,86 @@ impl CodexMessageProcessor {
.await;
}
async fn memory_reset(&self, request_id: ConnectionRequestId, _params: Option<()>) {
let state_db = match StateRuntime::init(
self.config.sqlite_home.clone(),
self.config.model_provider_id.clone(),
)
.await
{
Ok(state_db) => state_db,
Err(err) => {
self.send_internal_error(
request_id,
format!("failed to open state db for memory reset: {err}"),
)
.await;
return;
}
};
if let Err(err) = state_db.clear_memory_data().await {
self.send_internal_error(
request_id,
format!("failed to clear memory rows in state db: {err}"),
)
.await;
return;
}
let memory_root = self.config.codex_home.join("memories");
let memory_extensions_root = self.config.codex_home.join("memories_extensions");
let clear_memory_root_result: std::io::Result<()> = async {
for directory in [memory_root.as_path(), memory_extensions_root.as_path()] {
match tokio::fs::symlink_metadata(directory).await {
Ok(metadata) if metadata.file_type().is_symlink() => {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
format!(
"refusing to clear symlinked memory root {}",
directory.display()
),
));
}
Ok(_) => {}
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {}
Err(err) => return Err(err),
}
tokio::fs::create_dir_all(directory).await?;
let mut entries = tokio::fs::read_dir(directory).await?;
while let Some(entry) = entries.next_entry().await? {
let path = entry.path();
let file_type = entry.file_type().await?;
if file_type.is_dir() {
tokio::fs::remove_dir_all(path).await?;
} else {
tokio::fs::remove_file(path).await?;
}
}
}
Ok(())
}
.await;
if let Err(err) = clear_memory_root_result {
self.send_internal_error(
request_id,
format!(
"failed to clear memory directory {}: {err}",
memory_root.display()
),
)
.await;
return;
}
self.outgoing
.send_response(request_id, MemoryResetResponse {})
.await;
}
async fn thread_metadata_update(
&self,
request_id: ConnectionRequestId,
@@ -3349,7 +3433,6 @@ impl CodexMessageProcessor {
request_id: ConnectionRequestId,
params: ThreadUnarchiveParams,
) {
// TODO(jif) mostly rewrite this using sqlite after phase 1
let thread_id = match ThreadId::from_string(&params.thread_id) {
Ok(id) => id,
Err(err) => {
@@ -3363,152 +3446,21 @@ impl CodexMessageProcessor {
}
};
let archived_path = match find_archived_thread_path_by_id_str(
&self.config.codex_home,
&thread_id.to_string(),
)
.await
{
Ok(Some(path)) => path,
Ok(None) => {
let error = JSONRPCErrorError {
code: INVALID_REQUEST_ERROR_CODE,
message: format!("no archived rollout found for thread id {thread_id}"),
data: None,
};
self.outgoing.send_error(request_id, error).await;
return;
}
Err(err) => {
let error = JSONRPCErrorError {
code: INVALID_REQUEST_ERROR_CODE,
message: format!("failed to locate archived thread id {thread_id}: {err}"),
data: None,
};
self.outgoing.send_error(request_id, error).await;
return;
}
};
let rollout_path_display = archived_path.display().to_string();
let fallback_provider = self.config.model_provider_id.clone();
let state_db_ctx = get_state_db(&self.config).await;
let archived_folder = self
.config
.codex_home
.join(codex_core::ARCHIVED_SESSIONS_SUBDIR);
let result: Result<Thread, JSONRPCErrorError> = async {
let canonical_archived_dir = tokio::fs::canonicalize(&archived_folder).await.map_err(
|err| JSONRPCErrorError {
code: INTERNAL_ERROR_CODE,
message: format!(
"failed to unarchive thread: unable to resolve archived directory: {err}"
),
data: None,
},
)?;
let canonical_rollout_path = tokio::fs::canonicalize(&archived_path).await;
let canonical_rollout_path = if let Ok(path) = canonical_rollout_path
&& path.starts_with(&canonical_archived_dir)
{
path
} else {
return Err(JSONRPCErrorError {
code: INVALID_REQUEST_ERROR_CODE,
message: format!(
"rollout path `{rollout_path_display}` must be in archived directory"
),
data: None,
});
};
let required_suffix = format!("{thread_id}.jsonl");
let Some(file_name) = canonical_rollout_path.file_name().map(OsStr::to_owned) else {
return Err(JSONRPCErrorError {
code: INVALID_REQUEST_ERROR_CODE,
message: format!("rollout path `{rollout_path_display}` missing file name"),
data: None,
});
};
if !file_name
.to_string_lossy()
.ends_with(required_suffix.as_str())
{
return Err(JSONRPCErrorError {
code: INVALID_REQUEST_ERROR_CODE,
message: format!(
"rollout path `{rollout_path_display}` does not match thread id {thread_id}"
),
data: None,
});
}
let Some((year, month, day)) = rollout_date_parts(&file_name) else {
return Err(JSONRPCErrorError {
code: INVALID_REQUEST_ERROR_CODE,
message: format!(
"rollout path `{rollout_path_display}` missing filename timestamp"
),
data: None,
});
};
let sessions_folder = self.config.codex_home.join(codex_core::SESSIONS_SUBDIR);
let dest_dir = sessions_folder.join(year).join(month).join(day);
let restored_path = dest_dir.join(&file_name);
tokio::fs::create_dir_all(&dest_dir)
.await
.map_err(|err| JSONRPCErrorError {
code: INTERNAL_ERROR_CODE,
message: format!("failed to unarchive thread: {err}"),
data: None,
})?;
tokio::fs::rename(&canonical_rollout_path, &restored_path)
.await
.map_err(|err| JSONRPCErrorError {
code: INTERNAL_ERROR_CODE,
message: format!("failed to unarchive thread: {err}"),
data: None,
})?;
tokio::task::spawn_blocking({
let restored_path = restored_path.clone();
move || -> std::io::Result<()> {
let times = FileTimes::new().set_modified(SystemTime::now());
OpenOptions::new()
.append(true)
.open(&restored_path)?
.set_times(times)?;
Ok(())
}
})
let result = self
.thread_store
.unarchive_thread(StoreArchiveThreadParams { thread_id })
.await
.map_err(|err| JSONRPCErrorError {
code: INTERNAL_ERROR_CODE,
message: format!("failed to update unarchived thread timestamp: {err}"),
data: None,
})?
.map_err(|err| JSONRPCErrorError {
code: INTERNAL_ERROR_CODE,
message: format!("failed to update unarchived thread timestamp: {err}"),
data: None,
})?;
if let Some(ctx) = state_db_ctx {
let _ = ctx
.mark_unarchived(thread_id, restored_path.as_path())
.await;
}
let summary =
read_summary_from_rollout(restored_path.as_path(), fallback_provider.as_str())
.await
.map_err(|err| JSONRPCErrorError {
.map_err(|err| thread_store_archive_error("unarchive", err))
.and_then(|stored_thread| {
summary_from_stored_thread(stored_thread, fallback_provider.as_str())
.map(|summary| summary_to_thread(summary, &self.config.cwd))
.ok_or_else(|| JSONRPCErrorError {
code: INTERNAL_ERROR_CODE,
message: format!("failed to read unarchived thread: {err}"),
message: format!("failed to read unarchived thread {thread_id}"),
data: None,
})?;
Ok(summary_to_thread(summary, &self.config.cwd))
}
.await;
})
});
match result {
Ok(mut thread) => {
@@ -3734,15 +3686,15 @@ impl CodexMessageProcessor {
.map(|value| value as usize)
.unwrap_or(THREAD_LIST_DEFAULT_LIMIT)
.clamp(1, THREAD_LIST_MAX_LIMIT);
let core_sort_key = match sort_key.unwrap_or(ThreadSortKey::CreatedAt) {
ThreadSortKey::CreatedAt => CoreThreadSortKey::CreatedAt,
ThreadSortKey::UpdatedAt => CoreThreadSortKey::UpdatedAt,
let store_sort_key = match sort_key.unwrap_or(ThreadSortKey::CreatedAt) {
ThreadSortKey::CreatedAt => StoreThreadSortKey::CreatedAt,
ThreadSortKey::UpdatedAt => StoreThreadSortKey::UpdatedAt,
};
let (summaries, next_cursor) = match self
.list_threads_common(
requested_page_size,
cursor,
core_sort_key,
store_sort_key,
ThreadListFilters {
model_providers,
source_kinds,
@@ -4986,7 +4938,7 @@ impl CodexMessageProcessor {
&self,
requested_page_size: usize,
cursor: Option<String>,
sort_key: CoreThreadSortKey,
sort_key: StoreThreadSortKey,
filters: ThreadListFilters,
) -> Result<(Vec<ConversationSummary>, Option<String>), JSONRPCErrorError> {
let ThreadListFilters {
@@ -4996,16 +4948,7 @@ impl CodexMessageProcessor {
cwd,
search_term,
} = filters;
let mut cursor_obj: Option<RolloutCursor> = match cursor.as_ref() {
Some(cursor_str) => {
Some(parse_cursor(cursor_str).ok_or_else(|| JSONRPCErrorError {
code: INVALID_REQUEST_ERROR_CODE,
message: format!("invalid cursor: {cursor_str}"),
data: None,
})?)
}
None => None,
};
let mut cursor_obj = cursor;
let mut last_cursor = cursor_obj.clone();
let mut remaining = requested_page_size;
let mut items = Vec::with_capacity(requested_page_size);
@@ -5024,54 +4967,26 @@ impl CodexMessageProcessor {
let fallback_provider = self.config.model_provider_id.clone();
let (allowed_sources_vec, source_kind_filter) = compute_source_filters(source_kinds);
let allowed_sources = allowed_sources_vec.as_slice();
let state_db_ctx = get_state_db(&self.config).await;
while remaining > 0 {
let page_size = remaining.min(THREAD_LIST_MAX_LIMIT);
let page = if archived {
RolloutRecorder::list_archived_threads(
&self.config,
let page = self
.thread_store
.list_threads(StoreListThreadsParams {
page_size,
cursor_obj.as_ref(),
cursor: cursor_obj.clone(),
sort_key,
allowed_sources,
model_provider_filter.as_deref(),
fallback_provider.as_str(),
search_term.as_deref(),
)
allowed_sources: allowed_sources.to_vec(),
model_providers: model_provider_filter.clone(),
archived,
search_term: search_term.clone(),
})
.await
.map_err(|err| JSONRPCErrorError {
code: INTERNAL_ERROR_CODE,
message: format!("failed to list threads: {err}"),
data: None,
})?
} else {
RolloutRecorder::list_threads(
&self.config,
page_size,
cursor_obj.as_ref(),
sort_key,
allowed_sources,
model_provider_filter.as_deref(),
fallback_provider.as_str(),
search_term.as_deref(),
)
.await
.map_err(|err| JSONRPCErrorError {
code: INTERNAL_ERROR_CODE,
message: format!("failed to list threads: {err}"),
data: None,
})?
};
.map_err(thread_store_list_error)?;
let mut filtered = Vec::with_capacity(page.items.len());
for it in page.items {
let Some(summary) = summary_from_thread_list_item(
it,
fallback_provider.as_str(),
state_db_ctx.as_ref(),
)
.await
let Some(summary) = summary_from_stored_thread(it, fallback_provider.as_str())
else {
continue;
};
@@ -5091,12 +5006,8 @@ impl CodexMessageProcessor {
items.extend(filtered);
remaining = requested_page_size.saturating_sub(items.len());
// Encode RolloutCursor into the JSON-RPC string form returned to clients.
let next_cursor_value = page.next_cursor.clone();
next_cursor = next_cursor_value
.as_ref()
.and_then(|cursor| serde_json::to_value(cursor).ok())
.and_then(|value| value.as_str().map(str::to_owned));
next_cursor = next_cursor_value.clone();
if remaining == 0 {
break;
}
@@ -5897,76 +5808,10 @@ impl CodexMessageProcessor {
.await;
}
async fn archive_thread_common(
&self,
thread_id: ThreadId,
rollout_path: &Path,
) -> Result<(), JSONRPCErrorError> {
// Verify rollout_path is under sessions dir.
let rollout_folder = self.config.codex_home.join(codex_core::SESSIONS_SUBDIR);
let canonical_sessions_dir = match tokio::fs::canonicalize(&rollout_folder).await {
Ok(path) => path,
Err(err) => {
return Err(JSONRPCErrorError {
code: INTERNAL_ERROR_CODE,
message: format!(
"failed to archive thread: unable to resolve sessions directory: {err}"
),
data: None,
});
}
};
let canonical_rollout_path = tokio::fs::canonicalize(rollout_path).await;
let canonical_rollout_path = if let Ok(path) = canonical_rollout_path
&& path.starts_with(&canonical_sessions_dir)
{
path
} else {
return Err(JSONRPCErrorError {
code: INVALID_REQUEST_ERROR_CODE,
message: format!(
"rollout path `{}` must be in sessions directory",
rollout_path.display()
),
data: None,
});
};
// Verify file name matches thread id.
let required_suffix = format!("{thread_id}.jsonl");
let Some(file_name) = canonical_rollout_path.file_name().map(OsStr::to_owned) else {
return Err(JSONRPCErrorError {
code: INVALID_REQUEST_ERROR_CODE,
message: format!(
"rollout path `{}` missing file name",
rollout_path.display()
),
data: None,
});
};
if !file_name
.to_string_lossy()
.ends_with(required_suffix.as_str())
{
return Err(JSONRPCErrorError {
code: INVALID_REQUEST_ERROR_CODE,
message: format!(
"rollout path `{}` does not match thread id {thread_id}",
rollout_path.display()
),
data: None,
});
}
let mut state_db_ctx = None;
async fn prepare_thread_for_archive(&self, thread_id: ThreadId) {
// If the thread is active, request shutdown and wait briefly.
let removed_conversation = self.thread_manager.remove_thread(&thread_id).await;
if let Some(conversation) = removed_conversation {
if let Some(ctx) = conversation.state_db() {
state_db_ctx = Some(ctx);
}
info!("thread {thread_id} was active; shutting down");
match Self::wait_for_thread_shutdown(&conversation).await {
ThreadShutdownResult::Complete => {}
@@ -5981,34 +5826,6 @@ impl CodexMessageProcessor {
}
}
self.finalize_thread_teardown(thread_id).await;
if state_db_ctx.is_none() {
state_db_ctx = get_state_db(&self.config).await;
}
// Move the rollout file to archived.
let result: std::io::Result<()> = async move {
let archive_folder = self
.config
.codex_home
.join(codex_core::ARCHIVED_SESSIONS_SUBDIR);
tokio::fs::create_dir_all(&archive_folder).await?;
let archived_path = archive_folder.join(&file_name);
tokio::fs::rename(&canonical_rollout_path, &archived_path).await?;
if let Some(ctx) = state_db_ctx {
let _ = ctx
.mark_archived(thread_id, archived_path.as_path(), Utc::now())
.await;
}
Ok(())
}
.await;
result.map_err(|err| JSONRPCErrorError {
code: INTERNAL_ERROR_CODE,
message: format!("failed to archive thread: {err}"),
data: None,
})
}
async fn apps_list(&self, request_id: ConnectionRequestId, params: AppsListParams) {
@@ -9336,67 +9153,67 @@ fn set_thread_name_from_title(thread: &mut Thread, title: String) {
thread.name = Some(title);
}
async fn summary_from_thread_list_item(
it: codex_core::ThreadItem,
fallback_provider: &str,
state_db_ctx: Option<&StateDbHandle>,
) -> Option<ConversationSummary> {
if let Some(thread_id) = it.thread_id {
let timestamp = it.created_at.clone();
let updated_at = it.updated_at.clone().or_else(|| timestamp.clone());
let model_provider = it
.model_provider
.clone()
.unwrap_or_else(|| fallback_provider.to_string());
let cwd = it.cwd?;
let cli_version = it.cli_version.unwrap_or_default();
let source = with_thread_spawn_agent_metadata(
it.source
.unwrap_or(codex_protocol::protocol::SessionSource::Unknown),
it.agent_nickname.clone(),
it.agent_role.clone(),
);
return Some(ConversationSummary {
conversation_id: thread_id,
path: it.path,
preview: it.first_user_message.unwrap_or_default(),
timestamp,
updated_at,
model_provider,
cwd,
cli_version,
source,
git_info: if it.git_sha.is_none()
&& it.git_branch.is_none()
&& it.git_origin_url.is_none()
{
None
} else {
Some(ConversationGitInfo {
sha: it.git_sha,
branch: it.git_branch,
origin_url: it.git_origin_url,
})
},
});
fn thread_store_list_error(err: ThreadStoreError) -> JSONRPCErrorError {
match err {
ThreadStoreError::InvalidRequest { message } => JSONRPCErrorError {
code: INVALID_REQUEST_ERROR_CODE,
message,
data: None,
},
err => JSONRPCErrorError {
code: INTERNAL_ERROR_CODE,
message: format!("failed to list threads: {err}"),
data: None,
},
}
if let Some(thread_id) = thread_id_from_rollout_path(it.path.as_path()) {
return read_summary_from_state_db_context_by_thread_id(state_db_ctx, thread_id).await;
}
None
}
fn thread_id_from_rollout_path(path: &Path) -> Option<ThreadId> {
let file_name = path.file_name()?.to_str()?;
let stem = file_name.strip_suffix(".jsonl")?;
if stem.len() < 37 {
return None;
fn thread_store_archive_error(operation: &str, err: ThreadStoreError) -> JSONRPCErrorError {
match err {
ThreadStoreError::InvalidRequest { message } => JSONRPCErrorError {
code: INVALID_REQUEST_ERROR_CODE,
message,
data: None,
},
err => JSONRPCErrorError {
code: INTERNAL_ERROR_CODE,
message: format!("failed to {operation} thread: {err}"),
data: None,
},
}
let uuid_start = stem.len().saturating_sub(36);
if !stem[..uuid_start].ends_with('-') {
return None;
}
ThreadId::from_string(&stem[uuid_start..]).ok()
}
fn summary_from_stored_thread(
thread: StoredThread,
fallback_provider: &str,
) -> Option<ConversationSummary> {
let path = thread.rollout_path?;
let source = with_thread_spawn_agent_metadata(
thread.source,
thread.agent_nickname.clone(),
thread.agent_role.clone(),
);
let git_info = thread.git_info.map(|git| ConversationGitInfo {
sha: git.commit_hash.map(|sha| sha.0),
branch: git.branch,
origin_url: git.repository_url,
});
Some(ConversationSummary {
conversation_id: thread.thread_id,
path,
preview: thread.first_user_message.unwrap_or(thread.preview),
timestamp: Some(thread.created_at.to_rfc3339_opts(SecondsFormat::Secs, true)),
updated_at: Some(thread.updated_at.to_rfc3339_opts(SecondsFormat::Secs, true)),
model_provider: if thread.model_provider.is_empty() {
fallback_provider.to_string()
} else {
thread.model_provider
},
cwd: thread.cwd,
cli_version: thread.cli_version,
source,
git_info,
})
}
#[allow(clippy::too_many_arguments)]
codex-rs/app-server/src/in_process.rs
+5 -1
View File
@@ -82,6 +82,7 @@ use codex_exec_server::EnvironmentManager;
use codex_feedback::CodexFeedback;
use codex_login::AuthManager;
use codex_protocol::protocol::SessionSource;
pub use codex_state::log_db::LogDbLayer;
use tokio::sync::mpsc;
use tokio::sync::oneshot;
use tokio::time::timeout;
@@ -117,6 +118,8 @@ pub struct InProcessStartArgs {
pub cloud_requirements: CloudRequirementsLoader,
/// Feedback sink used by app-server/core telemetry and logs.
pub feedback: CodexFeedback,
/// SQLite tracing layer used to flush recently emitted logs before feedback upload.
pub log_db: Option<LogDbLayer>,
/// Environment manager used by core execution and filesystem operations.
pub environment_manager: Arc<EnvironmentManager>,
/// Startup warnings emitted after initialize succeeds.
@@ -395,7 +398,7 @@ fn start_uninitialized(args: InProcessStartArgs) -> InProcessClientHandle {
loader_overrides: args.loader_overrides,
cloud_requirements: args.cloud_requirements,
feedback: args.feedback,
log_db: None,
log_db: args.log_db,
config_warnings: args.config_warnings,
session_source: args.session_source,
auth_manager,
@@ -728,6 +731,7 @@ mod tests {
loader_overrides: LoaderOverrides::default(),
cloud_requirements: CloudRequirementsLoader::default(),
feedback: CodexFeedback::new(),
log_db: None,
environment_manager: Arc::new(EnvironmentManager::new(/*exec_server_url*/ None)),
config_warnings: Vec::new(),
session_source,
codex-rs/app-server/src/transport/auth.rs
+157 -18
View File
@@ -34,6 +34,10 @@ pub struct AppServerWebsocketAuthArgs {
#[arg(long = "ws-token-file", value_name = "PATH")]
pub ws_token_file: Option<PathBuf>,
/// Hex-encoded SHA-256 digest of the capability token.
#[arg(long = "ws-token-sha256", value_name = "HEX")]
pub ws_token_sha256: Option<String>,
/// Absolute path to the shared secret file for signed JWT bearer tokens.
#[arg(long = "ws-shared-secret-file", value_name = "PATH")]
pub ws_shared_secret_file: Option<PathBuf>,
@@ -65,7 +69,7 @@ pub struct AppServerWebsocketAuthSettings {
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum AppServerWebsocketAuthConfig {
CapabilityToken {
token_file: AbsolutePathBuf,
source: AppServerWebsocketCapabilityTokenSource,
},
SignedBearerToken {
shared_secret_file: AbsolutePathBuf,
@@ -75,6 +79,12 @@ pub enum AppServerWebsocketAuthConfig {
},
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum AppServerWebsocketCapabilityTokenSource {
TokenFile { token_file: AbsolutePathBuf },
TokenSha256 { token_sha256: [u8; 32] },
}
#[derive(Clone, Debug, Default)]
pub(crate) struct WebsocketAuthPolicy {
pub(crate) mode: Option<WebsocketAuthMode>,
@@ -144,17 +154,34 @@ impl AppServerWebsocketAuthArgs {
"`--ws-shared-secret-file`, `--ws-issuer`, `--ws-audience`, and `--ws-max-clock-skew-seconds` require `--ws-auth signed-bearer-token`"
);
}
let token_file = self.ws_token_file.context(
"`--ws-token-file` is required when `--ws-auth capability-token` is set",
)?;
Some(AppServerWebsocketAuthConfig::CapabilityToken {
token_file: absolute_path_arg("--ws-token-file", token_file)?,
})
let source = match (self.ws_token_file, self.ws_token_sha256) {
(Some(_), Some(_)) => {
anyhow::bail!(
"`--ws-token-file` and `--ws-token-sha256` are mutually exclusive"
);
}
(Some(token_file), None) => {
AppServerWebsocketCapabilityTokenSource::TokenFile {
token_file: absolute_path_arg("--ws-token-file", token_file)?,
}
}
(None, Some(token_sha256)) => {
AppServerWebsocketCapabilityTokenSource::TokenSha256 {
token_sha256: sha256_digest_arg("--ws-token-sha256", &token_sha256)?,
}
}
(None, None) => {
anyhow::bail!(
"`--ws-token-file` or `--ws-token-sha256` is required when `--ws-auth capability-token` is set"
);
}
};
Some(AppServerWebsocketAuthConfig::CapabilityToken { source })
}
Some(WebsocketAuthCliMode::SignedBearerToken) => {
if self.ws_token_file.is_some() {
if self.ws_token_file.is_some() || self.ws_token_sha256.is_some() {
anyhow::bail!(
"`--ws-token-file` requires `--ws-auth capability-token`, not `signed-bearer-token`"
"`--ws-token-file` and `--ws-token-sha256` require `--ws-auth capability-token`, not `signed-bearer-token`"
);
}
let shared_secret_file = self.ws_shared_secret_file.context(
@@ -174,6 +201,7 @@ impl AppServerWebsocketAuthArgs {
}
None => {
if self.ws_token_file.is_some()
|| self.ws_token_sha256.is_some()
|| self.ws_shared_secret_file.is_some()
|| self.ws_issuer.is_some()
|| self.ws_audience.is_some()
@@ -195,12 +223,19 @@ pub(crate) fn policy_from_settings(
settings: &AppServerWebsocketAuthSettings,
) -> io::Result<WebsocketAuthPolicy> {
let mode = match settings.config.as_ref() {
Some(AppServerWebsocketAuthConfig::CapabilityToken { token_file }) => {
let token = read_trimmed_secret(token_file.as_ref())?;
Some(WebsocketAuthMode::CapabilityToken {
token_sha256: sha256_digest(token.as_bytes()),
})
}
Some(AppServerWebsocketAuthConfig::CapabilityToken { source }) => match source {
AppServerWebsocketCapabilityTokenSource::TokenFile { token_file } => {
let token = read_trimmed_secret(token_file.as_ref())?;
Some(WebsocketAuthMode::CapabilityToken {
token_sha256: sha256_digest(token.as_bytes()),
})
}
AppServerWebsocketCapabilityTokenSource::TokenSha256 { token_sha256 } => {
Some(WebsocketAuthMode::CapabilityToken {
token_sha256: *token_sha256,
})
}
},
Some(AppServerWebsocketAuthConfig::SignedBearerToken {
shared_secret_file,
issuer,
@@ -387,6 +422,30 @@ fn absolute_path_arg(flag_name: &str, path: PathBuf) -> anyhow::Result<AbsoluteP
AbsolutePathBuf::try_from(path).with_context(|| format!("{flag_name} must be an absolute path"))
}
fn sha256_digest_arg(flag_name: &str, value: &str) -> anyhow::Result<[u8; 32]> {
let trimmed = value.trim();
if trimmed.len() != 64 {
anyhow::bail!("{flag_name} must be a 64-character hex SHA-256 digest");
}
let mut digest = [0u8; 32];
for (index, pair) in trimmed.as_bytes().chunks_exact(2).enumerate() {
let high = hex_nibble(flag_name, pair[0])?;
let low = hex_nibble(flag_name, pair[1])?;
digest[index] = (high << 4) | low;
}
Ok(digest)
}
fn hex_nibble(flag_name: &str, byte: u8) -> anyhow::Result<u8> {
match byte {
b'0'..=b'9' => Ok(byte - b'0'),
b'a'..=b'f' => Ok(byte - b'a' + 10),
b'A'..=b'F' => Ok(byte - b'A' + 10),
_ => anyhow::bail!("{flag_name} must be a 64-character hex SHA-256 digest"),
}
}
fn sha256_digest(input: &[u8]) -> [u8; 32] {
let mut digest = [0u8; 32];
digest.copy_from_slice(&Sha256::digest(input));
@@ -403,6 +462,7 @@ fn unauthorized(message: &'static str) -> WebsocketAuthError {
#[cfg(test)]
mod tests {
use super::*;
use axum::http::HeaderValue;
use base64::Engine;
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
use hmac::Hmac;
@@ -443,19 +503,98 @@ mod tests {
}
#[test]
fn capability_token_args_require_token_file() {
fn capability_token_args_require_token_file_or_hash() {
let err = AppServerWebsocketAuthArgs {
ws_auth: Some(WebsocketAuthCliMode::CapabilityToken),
..Default::default()
}
.try_into_settings()
.expect_err("capability-token mode should require a token file");
.expect_err("capability-token mode should require a token source");
assert!(
err.to_string().contains("--ws-token-file"),
err.to_string().contains("--ws-token-file")
&& err.to_string().contains("--ws-token-sha256"),
"unexpected error: {err}"
);
}
#[test]
fn capability_token_args_accept_token_hash() {
let settings = AppServerWebsocketAuthArgs {
ws_auth: Some(WebsocketAuthCliMode::CapabilityToken),
ws_token_sha256: Some("ab".repeat(32)),
..Default::default()
}
.try_into_settings()
.expect("capability-token hash args should parse");
assert_eq!(
settings,
AppServerWebsocketAuthSettings {
config: Some(AppServerWebsocketAuthConfig::CapabilityToken {
source: AppServerWebsocketCapabilityTokenSource::TokenSha256 {
token_sha256: [0xab; 32],
},
}),
}
);
}
#[test]
fn capability_token_args_reject_multiple_token_sources() {
let err = AppServerWebsocketAuthArgs {
ws_auth: Some(WebsocketAuthCliMode::CapabilityToken),
ws_token_file: Some(PathBuf::from("/tmp/token")),
ws_token_sha256: Some("ab".repeat(32)),
..Default::default()
}
.try_into_settings()
.expect_err("capability-token mode should reject multiple token sources");
assert!(
err.to_string().contains("mutually exclusive"),
"unexpected error: {err}"
);
}
#[test]
fn capability_token_args_reject_malformed_token_hash() {
let err = AppServerWebsocketAuthArgs {
ws_auth: Some(WebsocketAuthCliMode::CapabilityToken),
ws_token_sha256: Some("not-a-sha256".to_string()),
..Default::default()
}
.try_into_settings()
.expect_err("capability-token mode should reject malformed token hashes");
assert!(
err.to_string().contains("64-character hex"),
"unexpected error: {err}"
);
}
#[test]
fn capability_token_hash_policy_authorizes_matching_bearer_token() {
let settings = AppServerWebsocketAuthSettings {
config: Some(AppServerWebsocketAuthConfig::CapabilityToken {
source: AppServerWebsocketCapabilityTokenSource::TokenSha256 {
token_sha256: sha256_digest(b"super-secret-token"),
},
}),
};
let policy = policy_from_settings(&settings).expect("hash policy should build");
let mut headers = HeaderMap::new();
headers.insert(
AUTHORIZATION,
HeaderValue::from_static("Bearer super-secret-token"),
);
authorize_upgrade(&headers, &policy).expect("matching token should authorize");
headers.insert(
AUTHORIZATION,
HeaderValue::from_static("Bearer wrong-token"),
);
let err = authorize_upgrade(&headers, &policy).expect_err("wrong token should fail");
assert_eq!(err.status_code(), StatusCode::UNAUTHORIZED);
}
#[test]
fn signed_bearer_args_require_mode_when_mode_specific_flags_are_set() {
let err = AppServerWebsocketAuthArgs {
codex-rs/app-server/src/transport/remote_control/protocol.rs
+3 -4
View File
@@ -48,9 +48,8 @@ pub enum ClientEvent {
message: JSONRPCMessage,
},
/// Backend-generated acknowledgement for all server envelopes addressed to
/// `client_id` whose envelope `seq_id` is less than or equal to this ack's
/// `seq_id`. This cursor is client-scoped, not stream-scoped, so receivers
/// must not use `stream_id` to partition acks.
/// `client_id` and `stream_id` whose envelope `seq_id` is less than or equal
/// to this ack's `seq_id`. This cursor is stream-scoped.
Ack,
Ping,
ClientClosed,
@@ -65,7 +64,7 @@ pub(crate) struct ClientEnvelope {
pub(crate) client_id: ClientId,
#[serde(rename = "stream_id", skip_serializing_if = "Option::is_none")]
pub(crate) stream_id: Option<StreamId>,
/// For `Ack`, this is the backend-generated per-client cursor over
/// For `Ack`, this is the backend-generated per-stream cursor over
/// `ServerEnvelope.seq_id`.
#[serde(rename = "seq_id", skip_serializing_if = "Option::is_none")]
pub(crate) seq_id: Option<u64>,
codex-rs/app-server/src/transport/remote_control/tests.rs
+24 -13
View File
@@ -5,6 +5,7 @@ use super::enroll::update_persisted_remote_control_enrollment;
use super::protocol::ClientEnvelope;
use super::protocol::ClientEvent;
use super::protocol::ClientId;
use super::protocol::StreamId;
use super::protocol::normalize_remote_control_url;
use super::websocket::REMOTE_CONTROL_PROTOCOL_VERSION;
use super::*;
@@ -96,6 +97,7 @@ fn remote_control_auth_dot_json(account_id: Option<&str>) -> AuthDotJson {
account_id: account_id.map(str::to_string),
}),
last_refresh: Some(chrono::Utc::now()),
agent_identity: None,
}
}
@@ -162,7 +164,7 @@ async fn remote_control_transport_manages_virtual_clients_and_routes_messages()
json!({
"type": "pong",
"client_id": "client-1",
"seq_id": 0,
"seq_id": 1,
"status": "unknown",
})
);
@@ -368,7 +370,7 @@ async fn remote_control_transport_manages_virtual_clients_and_routes_messages()
json!({
"type": "pong",
"client_id": "client-1",
"seq_id": 3,
"seq_id": 1,
"status": "unknown",
})
);
@@ -617,12 +619,13 @@ async fn remote_control_transport_clears_outgoing_buffer_when_backend_acks() {
))
.await
.expect("remote writer should accept outgoing message");
let (server_event, stream_id) = read_server_event_with_stream_id(&mut first_websocket).await;
assert_eq!(
read_server_event(&mut first_websocket).await,
server_event,
json!({
"type": "server_message",
"client_id": "client-1",
"seq_id": 0,
"seq_id": 1,
"message": {
"method": "configWarning",
"params": {
@@ -638,8 +641,8 @@ async fn remote_control_transport_clears_outgoing_buffer_when_backend_acks() {
ClientEnvelope {
event: ClientEvent::Ack,
client_id: client_id.clone(),
stream_id: None,
seq_id: Some(0),
stream_id: Some(stream_id),
seq_id: Some(1),
cursor: None,
},
)
@@ -853,7 +856,7 @@ async fn remote_control_http_mode_enrolls_before_connecting() {
json!({
"type": "server_message",
"client_id": backend_client_id.0.clone(),
"seq_id": 0,
"seq_id": 1,
"message": {
"id": 11,
"result": {
@@ -881,7 +884,7 @@ async fn remote_control_http_mode_enrolls_before_connecting() {
json!({
"type": "server_message",
"client_id": backend_client_id.0.clone(),
"seq_id": 1,
"seq_id": 2,
"message": {
"method": "configWarning",
"params": {
@@ -1355,6 +1358,12 @@ async fn send_client_event(
}
async fn read_server_event(websocket: &mut WebSocketStream<TcpStream>) -> serde_json::Value {
read_server_event_with_stream_id(websocket).await.0
}
async fn read_server_event_with_stream_id(
websocket: &mut WebSocketStream<TcpStream>,
) -> (serde_json::Value, StreamId) {
loop {
let frame = timeout(Duration::from_secs(5), websocket.next())
.await
@@ -1365,13 +1374,15 @@ async fn read_server_event(websocket: &mut WebSocketStream<TcpStream>) -> serde_
tungstenite::Message::Text(text) => {
let mut event: serde_json::Value =
serde_json::from_str(text.as_ref()).expect("server event should deserialize");
if let Some(stream_id) = event
let stream_id = event
.as_object_mut()
.and_then(|event| event.remove("stream_id"))
{
assert!(stream_id.is_string(), "stream_id should be a string");
}
return event;
.expect("stream_id should be present");
let stream_id = stream_id
.as_str()
.expect("stream_id should be a string")
.to_string();
return (event, StreamId(stream_id));
}
tungstenite::Message::Ping(payload) => {
websocket
codex-rs/app-server/src/transport/remote_control/websocket.rs
+191 -81
View File
@@ -14,6 +14,7 @@ use super::protocol::ClientEvent;
use super::protocol::ClientId;
use super::protocol::RemoteControlTarget;
use super::protocol::ServerEnvelope;
use super::protocol::StreamId;
use axum::http::HeaderValue;
use base64::Engine;
use codex_core::util::backoff;
@@ -25,8 +26,8 @@ use futures::SinkExt;
use futures::StreamExt;
use futures::stream::SplitSink;
use futures::stream::SplitStream;
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::collections::VecDeque;
use std::io;
use std::io::ErrorKind;
use std::sync::Arc;
@@ -57,10 +58,7 @@ const REMOTE_CONTROL_ACCOUNT_ID_RETRY_INTERVAL: std::time::Duration =
std::time::Duration::from_secs(1);
struct BoundedOutboundBuffer {
// Remote-control acks are generated by the backend at client scope, so
// retransmit retention is keyed by client_id only. stream_id stays on each
// envelope for routing, but it is not part of the ack cursor.
buffer_by_client: HashMap<ClientId, BTreeMap<u64, ServerEnvelope>>,
buffer_by_stream: HashMap<(ClientId, StreamId), VecDeque<ServerEnvelope>>,
used_tx: watch::Sender<usize>,
}
@@ -68,46 +66,50 @@ impl BoundedOutboundBuffer {
fn new() -> (Self, watch::Receiver<usize>) {
let (used_tx, used_rx) = watch::channel(0);
let buffer = Self {
buffer_by_client: HashMap::new(),
buffer_by_stream: HashMap::new(),
used_tx,
};
(buffer, used_rx)
}
fn insert(&mut self, server_envelope: &ServerEnvelope) {
self.buffer_by_client
.entry(server_envelope.client_id.clone())
self.buffer_by_stream
.entry((
server_envelope.client_id.clone(),
server_envelope.stream_id.clone(),
))
.or_default()
.insert(server_envelope.seq_id, server_envelope.clone());
.push_back(server_envelope.clone());
self.used_tx.send_modify(|used| *used += 1);
}
fn ack(&mut self, client_id: &ClientId, acked_seq_id: u64) {
let Some(buffer) = self.buffer_by_client.get_mut(client_id) else {
fn ack(&mut self, client_id: &ClientId, stream_id: &StreamId, acked_seq_id: u64) {
let key = (client_id.clone(), stream_id.clone());
let Some(buffer) = self.buffer_by_stream.get_mut(&key) else {
return;
};
while let Some(seq_id) = buffer.first_key_value().map(|(seq_id, _)| seq_id)
&& *seq_id <= acked_seq_id
while let Some(server_envelope) = buffer.front()
&& server_envelope.seq_id <= acked_seq_id
{
buffer.pop_first();
buffer.pop_front();
self.used_tx.send_modify(|used| *used -= 1);
}
if buffer.is_empty() {
self.buffer_by_client.remove(client_id);
self.buffer_by_stream.remove(&key);
}
}
fn server_envelopes(&self) -> impl Iterator<Item = &ServerEnvelope> {
self.buffer_by_client
self.buffer_by_stream
.values()
.flat_map(|buffer| buffer.values())
.flat_map(|buffer| buffer.iter())
}
}
struct WebsocketState {
outbound_buffer: BoundedOutboundBuffer,
subscribe_cursor: Option<String>,
next_seq_id: u64,
next_seq_id_by_stream: HashMap<(ClientId, StreamId), u64>,
}
pub(crate) struct RemoteControlWebsocket {
@@ -162,7 +164,7 @@ impl RemoteControlWebsocket {
state: Arc::new(Mutex::new(WebsocketState {
outbound_buffer,
subscribe_cursor: None,
next_seq_id: 0,
next_seq_id_by_stream: HashMap::new(),
})),
server_event_rx: Arc::new(Mutex::new(server_event_rx)),
used_rx,
@@ -445,10 +447,16 @@ impl RemoteControlWebsocket {
}
}
};
let (server_envelope, write_complete_tx) = {
let (payload, write_complete_tx) = {
let mut state = state.lock().await;
let seq_id = state.next_seq_id;
state.next_seq_id = state.next_seq_id.saturating_add(1);
let seq_key = (
queued_server_envelope.client_id.clone(),
queued_server_envelope.stream_id.clone(),
);
let seq_id = *state
.next_seq_id_by_stream
.entry(seq_key.clone())
.or_insert(1);
let server_envelope = ServerEnvelope {
event: queued_server_envelope.event,
@@ -456,17 +464,19 @@ impl RemoteControlWebsocket {
seq_id,
stream_id: queued_server_envelope.stream_id,
};
let payload = match serde_json::to_string(&server_envelope) {
Ok(payload) => payload,
Err(err) => {
error!("failed to serialize remote-control server event: {err}");
continue;
}
};
state
.next_seq_id_by_stream
.insert(seq_key, seq_id.saturating_add(1));
state.outbound_buffer.insert(&server_envelope);
(server_envelope, queued_server_envelope.write_complete_tx)
};
let payload = match serde_json::to_string(&server_envelope) {
Ok(payload) => payload,
Err(err) => {
error!("failed to serialize remote-control server event: {err}");
continue;
}
(payload, queued_server_envelope.write_complete_tx)
};
tokio::select! {
@@ -590,10 +600,13 @@ impl RemoteControlWebsocket {
}
if let ClientEvent::Ack = &client_envelope.event
&& let Some(acked_seq_id) = client_envelope.seq_id
&& let Some(stream_id) = client_envelope.stream_id.as_ref()
{
websocket_state
.outbound_buffer
.ack(&client_envelope.client_id, acked_seq_id);
websocket_state.outbound_buffer.ack(
&client_envelope.client_id,
stream_id,
acked_seq_id,
);
}
drop(websocket_state);
@@ -971,6 +984,7 @@ mod tests {
account_id: Some("account_id".to_string()),
}),
last_refresh: Some(Utc::now()),
agent_identity: None,
}
}
@@ -1219,7 +1233,7 @@ mod tests {
let state = Arc::new(Mutex::new(WebsocketState {
outbound_buffer,
subscribe_cursor: None,
next_seq_id: 0,
next_seq_id_by_stream: HashMap::new(),
}));
let (_server_event_tx, server_event_rx) = mpsc::channel(super::super::CHANNEL_CAPACITY);
let server_event_rx = Arc::new(Mutex::new(server_event_rx));
@@ -1247,6 +1261,83 @@ mod tests {
.expect("writer should stop cleanly");
}
#[tokio::test]
async fn run_server_writer_inner_assigns_contiguous_seq_ids_per_stream() {
let (client_stream, mut server_stream) = connected_websocket_pair().await;
let (websocket_writer, _websocket_reader) = client_stream.split();
let (outbound_buffer, used_rx) = BoundedOutboundBuffer::new();
let state = Arc::new(Mutex::new(WebsocketState {
outbound_buffer,
subscribe_cursor: None,
next_seq_id_by_stream: HashMap::new(),
}));
let (server_event_tx, server_event_rx) = mpsc::channel(super::super::CHANNEL_CAPACITY);
let server_event_rx = Arc::new(Mutex::new(server_event_rx));
let shutdown_token = CancellationToken::new();
let writer_task = tokio::spawn(RemoteControlWebsocket::run_server_writer_inner(
state,
server_event_rx,
used_rx,
websocket_writer,
Duration::from_secs(60),
shutdown_token.clone(),
));
let client_id = ClientId("client-1".to_string());
let first_stream = StreamId("stream-1".to_string());
let second_stream = StreamId("stream-2".to_string());
for stream_id in [&first_stream, &second_stream, &first_stream] {
server_event_tx
.send(super::super::QueuedServerEnvelope {
event: ServerEvent::Pong {
status: crate::transport::remote_control::protocol::PongStatus::Active,
},
client_id: client_id.clone(),
stream_id: stream_id.clone(),
write_complete_tx: None,
})
.await
.expect("server event should queue");
}
assert_eq!(
read_server_text_event(&mut server_stream).await,
serde_json::json!({
"type": "pong",
"client_id": "client-1",
"stream_id": "stream-1",
"seq_id": 1,
"status": "active",
})
);
assert_eq!(
read_server_text_event(&mut server_stream).await,
serde_json::json!({
"type": "pong",
"client_id": "client-1",
"stream_id": "stream-2",
"seq_id": 1,
"status": "active",
})
);
assert_eq!(
read_server_text_event(&mut server_stream).await,
serde_json::json!({
"type": "pong",
"client_id": "client-1",
"stream_id": "stream-1",
"seq_id": 2,
"status": "active",
})
);
shutdown_token.cancel();
writer_task
.await
.expect("writer task should join")
.expect("writer should stop cleanly");
}
#[tokio::test]
async fn run_websocket_reader_inner_times_out_without_pong_frames() {
let (client_stream, _server_stream) = connected_websocket_pair().await;
@@ -1255,7 +1346,7 @@ mod tests {
let state = Arc::new(Mutex::new(WebsocketState {
outbound_buffer,
subscribe_cursor: None,
next_seq_id: 0,
next_seq_id_by_stream: HashMap::new(),
}));
let (server_event_tx, _server_event_rx) = mpsc::channel(super::super::CHANNEL_CAPACITY);
let (transport_event_tx, _transport_event_rx) =
@@ -1286,70 +1377,32 @@ mod tests {
}
#[test]
fn outbound_buffer_acks_by_client_id_across_stream_ids() {
fn outbound_buffer_acks_by_stream_id() {
let (mut outbound_buffer, used_rx) = BoundedOutboundBuffer::new();
let client_1 = ClientId("client-1".to_string());
let client_2 = ClientId("client-2".to_string());
let stream_1 = StreamId("stream-1".to_string());
outbound_buffer.insert(&server_envelope(
&client_1,
"stream-1",
/*seq_id*/ 0,
/*seq_id*/ 1,
"first-client-old-stream",
));
outbound_buffer.insert(&server_envelope(
&client_2,
"stream-1",
/*seq_id*/ 1,
/*seq_id*/ 2,
"second-client",
));
outbound_buffer.insert(&server_envelope(
&client_1,
"stream-2",
/*seq_id*/ 2,
/*seq_id*/ 3,
"first-client-new-stream",
));
outbound_buffer.ack(&client_1, /*acked_seq_id*/ 2);
let mut retained = outbound_buffer
.server_envelopes()
.map(|server_envelope| {
(
server_envelope.client_id.0.as_str(),
server_envelope.stream_id.0.as_str(),
server_envelope.seq_id,
)
})
.collect::<Vec<_>>();
retained.sort_unstable();
assert_eq!(retained, vec![("client-2", "stream-1", 1)]);
assert_eq!(*used_rx.borrow(), 1);
}
#[test]
fn outbound_buffer_retains_unacked_messages_until_ack_advances() {
let (mut outbound_buffer, used_rx) = BoundedOutboundBuffer::new();
let client_1 = ClientId("client-1".to_string());
let client_2 = ClientId("client-2".to_string());
outbound_buffer.insert(&server_envelope(
&client_1,
"stream-1",
/*seq_id*/ 0,
"first-old",
));
outbound_buffer.insert(&server_envelope(
&client_1,
"stream-2",
/*seq_id*/ 1,
"first-new",
));
outbound_buffer.insert(&server_envelope(
&client_2, "stream-1", /*seq_id*/ 2, "second",
));
outbound_buffer.ack(&client_1, /*acked_seq_id*/ 0);
outbound_buffer.ack(&client_1, &stream_1, /*acked_seq_id*/ 3);
let mut retained = outbound_buffer
.server_envelopes()
@@ -1364,7 +1417,50 @@ mod tests {
retained.sort_unstable();
assert_eq!(
retained,
vec![("client-1", "stream-2", 1), ("client-2", "stream-1", 2)]
vec![("client-1", "stream-2", 3), ("client-2", "stream-1", 2)]
);
assert_eq!(*used_rx.borrow(), 2);
}
#[test]
fn outbound_buffer_retains_unacked_messages_until_ack_advances() {
let (mut outbound_buffer, used_rx) = BoundedOutboundBuffer::new();
let client_1 = ClientId("client-1".to_string());
let client_2 = ClientId("client-2".to_string());
let stream_1 = StreamId("stream-1".to_string());
outbound_buffer.insert(&server_envelope(
&client_1,
"stream-1",
/*seq_id*/ 1,
"first-old",
));
outbound_buffer.insert(&server_envelope(
&client_1,
"stream-2",
/*seq_id*/ 2,
"first-new",
));
outbound_buffer.insert(&server_envelope(
&client_2, "stream-1", /*seq_id*/ 3, "second",
));
outbound_buffer.ack(&client_1, &stream_1, /*acked_seq_id*/ 1);
let mut retained = outbound_buffer
.server_envelopes()
.map(|server_envelope| {
(
server_envelope.client_id.0.as_str(),
server_envelope.stream_id.0.as_str(),
server_envelope.seq_id,
)
})
.collect::<Vec<_>>();
retained.sort_unstable();
assert_eq!(
retained,
vec![("client-1", "stream-2", 2), ("client-2", "stream-1", 3)]
);
assert_eq!(*used_rx.borrow(), 2);
}
@@ -1449,6 +1545,20 @@ mod tests {
(client_stream, server_stream)
}
async fn read_server_text_event(
server_stream: &mut WebSocketStream<TcpStream>,
) -> serde_json::Value {
let message = timeout(Duration::from_secs(5), server_stream.next())
.await
.expect("server event should arrive in time")
.expect("server websocket should stay open")
.expect("server event should read");
let tungstenite::Message::Text(text) = message else {
panic!("expected text event, got {message:?}");
};
serde_json::from_str(text.as_ref()).expect("server event should deserialize")
}
async fn respond_with_status_and_headers(
mut stream: TcpStream,
status: &str,
codex-rs/app-server/tests/common/auth_fixtures.rs
+1
View File
@@ -163,6 +163,7 @@ pub fn write_chatgpt_auth(
openai_api_key: None,
tokens: Some(tokens),
last_refresh,
agent_identity: None,
};
save_auth(codex_home, &auth, cli_auth_credentials_store_mode).context("write auth.json")
codex-rs/app-server/tests/suite/v2/app_list.rs
+1
View File
@@ -117,6 +117,7 @@ async fn list_apps_returns_empty_with_api_key_auth() -> Result<()> {
openai_api_key: Some("test-api-key".to_string()),
tokens: None,
last_refresh: None,
agent_identity: None,
},
AuthCredentialsStoreMode::File,
)?;
codex-rs/app-server/tests/suite/v2/mcp_server_elicitation.rs
+5 -3
View File
@@ -66,8 +66,9 @@ use tokio::time::timeout;
const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
const CONNECTOR_ID: &str = "calendar";
const CONNECTOR_NAME: &str = "Calendar";
const TOOL_NAMESPACE: &str = "mcp__codex_apps__calendar";
const CALLABLE_TOOL_NAME: &str = "_confirm_action";
const TOOL_NAME: &str = "calendar_confirm_action";
const QUALIFIED_TOOL_NAME: &str = "mcp__codex_apps__calendar_confirm_action";
const TOOL_CALL_ID: &str = "call-calendar-confirm";
const ELICITATION_MESSAGE: &str = "Allow this request?";
@@ -85,9 +86,10 @@ async fn mcp_server_elicitation_round_trip() -> Result<()> {
]),
responses::sse(vec![
responses::ev_response_created("resp-1"),
responses::ev_function_call(
responses::ev_function_call_with_namespace(
TOOL_CALL_ID,
QUALIFIED_TOOL_NAME,
TOOL_NAMESPACE,
CALLABLE_TOOL_NAME,
&tool_call_arguments,
),
responses::ev_completed("resp-1"),
codex-rs/app-server/tests/suite/v2/memory_reset.rs
+145
View File
@@ -0,0 +1,145 @@
use anyhow::Result;
use app_test_support::McpProcess;
use app_test_support::to_response;
use chrono::Utc;
use codex_app_server_protocol::JSONRPCResponse;
use codex_app_server_protocol::MemoryResetResponse;
use codex_app_server_protocol::RequestId;
use codex_protocol::ThreadId;
use codex_protocol::protocol::SessionSource;
use codex_state::Stage1JobClaimOutcome;
use codex_state::StateRuntime;
use codex_state::ThreadMetadataBuilder;
use pretty_assertions::assert_eq;
use std::path::Path;
use std::sync::Arc;
use tempfile::TempDir;
use tokio::time::timeout;
use uuid::Uuid;
const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
#[tokio::test]
async fn memory_reset_clears_memory_files_and_rows_preserves_threads() -> Result<()> {
let codex_home = TempDir::new()?;
create_config_toml(codex_home.path())?;
let state_db = init_state_db(codex_home.path()).await?;
let memory_root = codex_home.path().join("memories");
tokio::fs::create_dir_all(memory_root.join("rollout_summaries")).await?;
tokio::fs::write(memory_root.join("MEMORY.md"), "stale memory\n").await?;
tokio::fs::write(
memory_root.join("rollout_summaries").join("stale.md"),
"stale rollout summary\n",
)
.await?;
let thread_id = seed_stage1_output(&state_db, codex_home.path()).await?;
let mut mcp = McpProcess::new(codex_home.path()).await?;
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
let request_id = mcp
.send_raw_request("memory/reset", /*params*/ None)
.await?;
let response: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(request_id)),
)
.await??;
let _: MemoryResetResponse = to_response::<MemoryResetResponse>(response)?;
let stage1_outputs = state_db.list_stage1_outputs_for_global(/*n*/ 10).await?;
assert_eq!(stage1_outputs, Vec::new());
assert_eq!(
state_db.get_thread_memory_mode(thread_id).await?.as_deref(),
Some("enabled")
);
let mut remaining_entries = tokio::fs::read_dir(&memory_root).await?;
assert!(
remaining_entries.next_entry().await?.is_none(),
"memory root should be empty after reset"
);
Ok(())
}
async fn seed_stage1_output(state_db: &Arc<StateRuntime>, codex_home: &Path) -> Result<ThreadId> {
let now = Utc::now();
let thread_id = ThreadId::from_string(&Uuid::new_v4().to_string())?;
let worker_id = ThreadId::from_string(&Uuid::new_v4().to_string())?;
let mut builder = ThreadMetadataBuilder::new(
thread_id,
codex_home.join("sessions").join("test.jsonl"),
now,
SessionSource::Cli,
);
builder.updated_at = Some(now);
builder.cwd = codex_home.to_path_buf();
let metadata = builder.build("mock_provider");
state_db.upsert_thread(&metadata).await?;
let claim = state_db
.try_claim_stage1_job(
thread_id,
worker_id,
now.timestamp(),
/*lease_seconds*/ 3600,
/*max_running_jobs*/ 64,
)
.await?;
let Stage1JobClaimOutcome::Claimed { ownership_token } = claim else {
anyhow::bail!("unexpected stage1 claim outcome: {claim:?}");
};
assert!(
state_db
.mark_stage1_job_succeeded(
thread_id,
ownership_token.as_str(),
now.timestamp(),
"raw memory",
"rollout summary",
/*rollout_slug*/ None,
)
.await?,
"stage1 success should be recorded"
);
state_db
.enqueue_global_consolidation(now.timestamp())
.await?;
Ok(thread_id)
}
async fn init_state_db(codex_home: &Path) -> Result<Arc<StateRuntime>> {
let state_db = StateRuntime::init(codex_home.to_path_buf(), "mock_provider".into()).await?;
state_db
.mark_backfill_complete(/*last_watermark*/ None)
.await?;
Ok(state_db)
}
fn create_config_toml(codex_home: &Path) -> std::io::Result<()> {
let config_toml = codex_home.join("config.toml");
std::fs::write(
config_toml,
r#"
model = "mock-model"
approval_policy = "never"
sandbox_mode = "read-only"
model_provider = "mock_provider"
suppress_unstable_features_warning = true
[features]
sqlite = true
[model_providers.mock_provider]
name = "Mock provider for test"
base_url = "http://127.0.0.1:9/v1"
wire_api = "responses"
request_max_retries = 0
stream_max_retries = 0
"#,
)
}
codex-rs/app-server/tests/suite/v2/mod.rs
+1
View File
@@ -20,6 +20,7 @@ mod mcp_resource;
mod mcp_server_elicitation;
mod mcp_server_status;
mod mcp_tool;
mod memory_reset;
mod model_list;
mod output_schema;
mod plan_item;
codex-rs/app-server/tests/suite/v2/thread_list.rs
+22 -3
View File
@@ -563,15 +563,34 @@ sqlite = true
/*git_info*/ None,
)?;
// `thread/list` only applies `search_term` on the sqlite path. In this test we
// create rollouts manually, so we must also create the sqlite DB and mark backfill
// complete; otherwise app-server will permanently use filesystem fallback.
// `thread/list` applies `search_term` on the sqlite fast path. This test creates
// rollouts manually, so mark the DB backfill complete and then run an unsearched
// list large enough to repair every rollout the searched list should find.
let state_db =
codex_state::StateRuntime::init(codex_home.path().to_path_buf(), "mock_provider".into())
.await?;
state_db
.mark_backfill_complete(/*last_watermark*/ None)
.await?;
let rollout_config = codex_rollout::RolloutConfig {
codex_home: codex_home.path().to_path_buf(),
sqlite_home: codex_home.path().to_path_buf(),
cwd: codex_home.path().to_path_buf(),
model_provider_id: "mock_provider".to_string(),
generate_memories: false,
};
let repaired_page = codex_core::RolloutRecorder::list_threads(
&rollout_config,
/*page_size*/ 10,
/*cursor*/ None,
codex_core::ThreadSortKey::CreatedAt,
&[],
/*model_providers*/ None,
"mock_provider",
/*search_term*/ None,
)
.await?;
assert_eq!(repaired_page.items.len(), 3);
let mut mcp = init_mcp(codex_home.path()).await?;
let request_id = mcp
codex-rs/app-server/tests/suite/v2/turn_interrupt.rs
+12 -3
View File
@@ -127,10 +127,17 @@ async fn turn_interrupt_aborts_running_turn() -> Result<()> {
#[tokio::test]
async fn turn_interrupt_resolves_pending_command_approval_request() -> Result<()> {
#[cfg(target_os = "windows")]
let shell_command = vec![
"powershell".to_string(),
"-Command".to_string(),
"Start-Sleep -Seconds 10".to_string(),
];
#[cfg(not(target_os = "windows"))]
let shell_command = vec![
"python3".to_string(),
"-c".to_string(),
"print(42)".to_string(),
"import time; time.sleep(10)".to_string(),
];
let tmp = TempDir::new()?;
@@ -143,7 +150,7 @@ async fn turn_interrupt_resolves_pending_command_approval_request() -> Result<()
shell_command.clone(),
Some(&working_directory),
Some(10_000),
"call_python_approval",
"call_sleep_approval",
)?])
.await;
create_config_toml(&codex_home, &server.uri(), "untrusted", "read-only")?;
@@ -172,6 +179,7 @@ async fn turn_interrupt_resolves_pending_command_approval_request() -> Result<()
text_elements: Vec::new(),
}],
cwd: Some(working_directory),
approval_policy: Some(codex_app_server_protocol::AskForApproval::UnlessTrusted),
..Default::default()
})
.await?;
@@ -190,7 +198,7 @@ async fn turn_interrupt_resolves_pending_command_approval_request() -> Result<()
let ServerRequest::CommandExecutionRequestApproval { request_id, params } = request else {
panic!("expected CommandExecutionRequestApproval request");
};
assert_eq!(params.item_id, "call_python_approval");
assert_eq!(params.item_id, "call_sleep_approval");
assert_eq!(params.thread_id, thread.id);
assert_eq!(params.turn_id, turn.id);
@@ -251,6 +259,7 @@ fn create_config_toml(
r#"
model = "mock-model"
approval_policy = "{approval_policy}"
approvals_reviewer = "user"
sandbox_mode = "{sandbox_mode}"
model_provider = "mock_provider"
codex-rs/cli/Cargo.toml
+1
View File
@@ -45,6 +45,7 @@ codex-state = { workspace = true }
codex-stdio-to-uds = { workspace = true }
codex-terminal-detection = { workspace = true }
codex-tui = { workspace = true }
codex-utils-absolute-path = { workspace = true }
codex-utils-path = { workspace = true }
libc = { workspace = true }
owo-colors = { workspace = true }
codex-rs/cli/src/debug_sandbox.rs
+18 -8
View File
@@ -18,7 +18,10 @@ use codex_protocol::config_types::SandboxMode;
use codex_protocol::permissions::NetworkSandboxPolicy;
use codex_sandboxing::landlock::create_linux_sandbox_command_args_for_policies;
#[cfg(target_os = "macos")]
use codex_sandboxing::seatbelt::create_seatbelt_command_args_for_policies;
use codex_sandboxing::seatbelt::CreateSeatbeltCommandArgsParams;
#[cfg(target_os = "macos")]
use codex_sandboxing::seatbelt::create_seatbelt_command_args;
use codex_utils_absolute_path::AbsolutePathBuf;
use codex_utils_cli::CliConfigOverrides;
use tokio::process::Child;
use tokio::process::Command as TokioCommand;
@@ -39,6 +42,7 @@ pub async fn run_command_under_seatbelt(
) -> anyhow::Result<()> {
let SeatbeltCommand {
full_auto,
allow_unix_sockets,
log_denials,
config_overrides,
command,
@@ -50,6 +54,7 @@ pub async fn run_command_under_seatbelt(
codex_linux_sandbox_exe,
SandboxType::Seatbelt,
log_denials,
&allow_unix_sockets,
)
.await
}
@@ -78,6 +83,7 @@ pub async fn run_command_under_landlock(
codex_linux_sandbox_exe,
SandboxType::Landlock,
/*log_denials*/ false,
&[],
)
.await
}
@@ -98,6 +104,7 @@ pub async fn run_command_under_windows(
codex_linux_sandbox_exe,
SandboxType::Windows,
/*log_denials*/ false,
&[],
)
.await
}
@@ -116,6 +123,8 @@ async fn run_command_under_sandbox(
codex_linux_sandbox_exe: Option<PathBuf>,
sandbox_type: SandboxType,
log_denials: bool,
#[cfg_attr(not(target_os = "macos"), allow(unused_variables))]
allow_unix_sockets: &[AbsolutePathBuf],
) -> anyhow::Result<()> {
let config = load_debug_sandbox_config(
config_overrides
@@ -252,14 +261,15 @@ async fn run_command_under_sandbox(
let mut child = match sandbox_type {
#[cfg(target_os = "macos")]
SandboxType::Seatbelt => {
let args = create_seatbelt_command_args_for_policies(
let args = create_seatbelt_command_args(CreateSeatbeltCommandArgsParams {
command,
&config.permissions.file_system_sandbox_policy,
config.permissions.network_sandbox_policy,
sandbox_policy_cwd.as_path(),
/*enforce_managed_network*/ false,
network.as_ref(),
);
file_system_sandbox_policy: &config.permissions.file_system_sandbox_policy,
network_sandbox_policy: config.permissions.network_sandbox_policy,
sandbox_policy_cwd: sandbox_policy_cwd.as_path(),
enforce_managed_network: false,
network: network.as_ref(),
extra_allow_unix_sockets: allow_unix_sockets,
});
let network_policy = config.permissions.network_sandbox_policy;
spawn_debug_sandbox_child(
PathBuf::from("/usr/bin/sandbox-exec"),
codex-rs/cli/src/lib.rs
+10
View File
@@ -3,6 +3,7 @@ mod exit_status;
pub(crate) mod login;
use clap::Parser;
use codex_utils_absolute_path::AbsolutePathBuf;
use codex_utils_cli::CliConfigOverrides;
pub use debug_sandbox::run_command_under_landlock;
@@ -22,6 +23,10 @@ pub struct SeatbeltCommand {
#[arg(long = "full-auto", default_value_t = false)]
pub full_auto: bool,
/// Allow the sandboxed command to bind/connect AF_UNIX sockets rooted at this path. Relative paths are resolved against the current directory. Repeat to allow multiple paths.
#[arg(long = "allow-unix-socket", value_parser = parse_allow_unix_socket_path)]
pub allow_unix_sockets: Vec<AbsolutePathBuf>,
/// While the command runs, capture macOS sandbox denials via `log stream` and print them after exit
#[arg(long = "log-denials", default_value_t = false)]
pub log_denials: bool,
@@ -34,6 +39,11 @@ pub struct SeatbeltCommand {
pub command: Vec<String>,
}
fn parse_allow_unix_socket_path(raw: &str) -> Result<AbsolutePathBuf, String> {
AbsolutePathBuf::relative_to_current_dir(raw)
.map_err(|err| format!("invalid path {raw}: {err}"))
}
#[derive(Debug, Parser)]
pub struct LandlockCommand {
/// Convenience alias for low-friction sandboxed automatic execution (network-disabled sandbox that can write to cwd and TMPDIR)
codex-rs/cli/src/main.rs
+1 -1
View File
@@ -1269,7 +1269,7 @@ async fn run_debug_clear_memories_command(
let state_db =
StateRuntime::init(config.sqlite_home.clone(), config.model_provider_id.clone())
.await?;
state_db.reset_memory_data_for_fresh_start().await?;
state_db.clear_memory_data().await?;
cleared_state_db = true;
}
codex-rs/cli/tests/debug_clear_memories.rs
-6
View File
@@ -125,12 +125,6 @@ INSERT INTO jobs (
.fetch_one(&pool)
.await?;
assert_eq!(memory_jobs_count, 0);
let memory_mode: String = sqlx::query_scalar("SELECT memory_mode FROM threads WHERE id = ?")
.bind(thread_id)
.fetch_one(&pool)
.await?;
assert_eq!(memory_mode, "disabled");
assert!(!memory_root.exists());
Ok(())
codex-rs/code-mode/Cargo.toml
+1
View File
@@ -14,6 +14,7 @@ workspace = true
[dependencies]
async-trait = { workspace = true }
codex-protocol = { workspace = true }
deno_core_icudata = { workspace = true }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
codex-rs/code-mode/src/description.rs
+30 -22
View File
@@ -1,3 +1,4 @@
use codex_protocol::ToolName;
use serde::Deserialize;
use serde::Serialize;
use serde_json::Value as JsonValue;
@@ -129,6 +130,7 @@ pub enum CodeModeToolKind {
#[derive(Clone, Debug, PartialEq)]
pub struct ToolDefinition {
pub name: String,
pub tool_name: ToolName,
pub description: String,
pub kind: CodeModeToolKind,
pub input_schema: Option<JsonValue>,
@@ -269,11 +271,15 @@ pub fn build_exec_tool_description(
for tool in enabled_tools {
let name = tool.name.as_str();
let nested_description = render_code_mode_sample_for_definition(tool);
let next_namespace = namespace_descriptions
.get(name)
let namespace_description = tool
.tool_name
.namespace
.as_ref()
.and_then(|namespace| namespace_descriptions.get(namespace));
let next_namespace = namespace_description
.map(|namespace_description| namespace_description.name.as_str());
if next_namespace != current_namespace {
if let Some(namespace_description) = namespace_descriptions.get(name) {
if let Some(namespace_description) = namespace_description {
let namespace_description_text = namespace_description.description.trim();
if !namespace_description_text.is_empty() {
nested_tool_sections.push(format!(
@@ -346,7 +352,7 @@ pub fn augment_tool_definition(mut definition: ToolDefinition) -> ToolDefinition
pub fn enabled_tool_metadata(definition: &ToolDefinition) -> EnabledToolMetadata {
EnabledToolMetadata {
tool_name: definition.name.clone(),
tool_name: definition.tool_name.clone(),
global_name: normalize_code_mode_identifier(&definition.name),
description: definition.description.clone(),
kind: definition.kind,
@@ -355,7 +361,7 @@ pub fn enabled_tool_metadata(definition: &ToolDefinition) -> EnabledToolMetadata
#[derive(Clone, Debug, Eq, PartialEq, Serialize)]
pub struct EnabledToolMetadata {
pub tool_name: String,
pub tool_name: ToolName,
pub global_name: String,
pub description: String,
pub kind: CodeModeToolKind,
@@ -706,6 +712,7 @@ mod tests {
use super::build_exec_tool_description;
use super::normalize_code_mode_identifier;
use super::parse_exec_source;
use codex_protocol::ToolName;
use pretty_assertions::assert_eq;
use serde_json::Value as JsonValue;
use serde_json::json;
@@ -770,6 +777,7 @@ mod tests {
fn augment_tool_definition_appends_typed_declaration() {
let definition = ToolDefinition {
name: "hidden_dynamic_tool".to_string(),
tool_name: ToolName::plain("hidden_dynamic_tool"),
description: "Test tool".to_string(),
kind: CodeModeToolKind::Function,
input_schema: Some(json!({
@@ -798,6 +806,7 @@ mod tests {
fn augment_tool_definition_includes_property_descriptions_as_comments() {
let definition = ToolDefinition {
name: "weather_tool".to_string(),
tool_name: ToolName::plain("weather_tool"),
description: "Weather tool".to_string(),
kind: CodeModeToolKind::Function,
input_schema: Some(json!({
@@ -846,6 +855,7 @@ mod tests {
let description = build_exec_tool_description(
&[ToolDefinition {
name: "foo".to_string(),
tool_name: ToolName::plain("foo"),
description: "bar".to_string(),
kind: CodeModeToolKind::Function,
input_schema: None,
@@ -870,26 +880,18 @@ bar"
#[test]
fn code_mode_only_description_groups_namespace_instructions_once() {
let namespace_descriptions = BTreeMap::from([
(
"mcp__sample__alpha".to_string(),
ToolNamespaceDescription {
name: "mcp__sample".to_string(),
description: "Shared namespace guidance.".to_string(),
},
),
(
"mcp__sample__beta".to_string(),
ToolNamespaceDescription {
name: "mcp__sample".to_string(),
description: "Shared namespace guidance.".to_string(),
},
),
]);
let namespace_descriptions = BTreeMap::from([(
"mcp__sample__".to_string(),
ToolNamespaceDescription {
name: "mcp__sample".to_string(),
description: "Shared namespace guidance.".to_string(),
},
)]);
let description = build_exec_tool_description(
&[
ToolDefinition {
name: "mcp__sample__alpha".to_string(),
tool_name: ToolName::namespaced("mcp__sample__", "alpha"),
description: "First tool".to_string(),
kind: CodeModeToolKind::Function,
input_schema: Some(json!({
@@ -905,6 +907,7 @@ bar"
},
ToolDefinition {
name: "mcp__sample__beta".to_string(),
tool_name: ToolName::namespaced("mcp__sample__", "beta"),
description: "Second tool".to_string(),
kind: CodeModeToolKind::Function,
input_schema: Some(json!({
@@ -935,7 +938,7 @@ bar"
#[test]
fn code_mode_only_description_omits_empty_namespace_sections() {
let namespace_descriptions = BTreeMap::from([(
"mcp__sample__alpha".to_string(),
"mcp__sample__".to_string(),
ToolNamespaceDescription {
name: "mcp__sample".to_string(),
description: String::new(),
@@ -944,6 +947,7 @@ bar"
let description = build_exec_tool_description(
&[ToolDefinition {
name: "mcp__sample__alpha".to_string(),
tool_name: ToolName::namespaced("mcp__sample__", "alpha"),
description: "First tool".to_string(),
kind: CodeModeToolKind::Function,
input_schema: Some(json!({
@@ -969,6 +973,7 @@ bar"
fn code_mode_only_description_renders_shared_mcp_types_once() {
let first_tool = augment_tool_definition(ToolDefinition {
name: "mcp__sample__alpha".to_string(),
tool_name: ToolName::namespaced("mcp__sample__", "alpha"),
description: "First tool".to_string(),
kind: CodeModeToolKind::Function,
input_schema: Some(json!({
@@ -1002,6 +1007,7 @@ bar"
});
let second_tool = augment_tool_definition(ToolDefinition {
name: "mcp__sample__beta".to_string(),
tool_name: ToolName::namespaced("mcp__sample__", "beta"),
description: "Second tool".to_string(),
kind: CodeModeToolKind::Function,
input_schema: Some(json!({
@@ -1038,6 +1044,7 @@ bar"
&[
ToolDefinition {
name: first_tool.name,
tool_name: first_tool.tool_name,
description: "First tool".to_string(),
kind: first_tool.kind,
input_schema: first_tool.input_schema,
@@ -1045,6 +1052,7 @@ bar"
},
ToolDefinition {
name: second_tool.name,
tool_name: second_tool.tool_name,
description: "Second tool".to_string(),
kind: second_tool.kind,
input_schema: second_tool.input_schema,
codex-rs/code-mode/src/runtime/callbacks.rs
+23 -1
View File
@@ -15,7 +15,13 @@ pub(super) fn tool_callback(
args: v8::FunctionCallbackArguments,
mut retval: v8::ReturnValue<v8::Value>,
) {
let tool_name = args.data().to_rust_string_lossy(scope);
let tool_index = match args.data().to_rust_string_lossy(scope).parse::<usize>() {
Ok(tool_index) => tool_index,
Err(_) => {
throw_type_error(scope, "invalid tool callback data");
return;
}
};
let input = if args.length() == 0 {
Ok(None)
} else {
@@ -36,6 +42,22 @@ pub(super) fn tool_callback(
let promise = resolver.get_promise(scope);
let resolver = v8::Global::new(scope, resolver);
let tool_name = {
let Some(state) = scope.get_slot::<RuntimeState>() else {
throw_type_error(scope, "runtime state unavailable");
return;
};
let Some(tool_name) = state
.enabled_tools
.get(tool_index)
.map(|tool| tool.tool_name.clone())
else {
throw_type_error(scope, "tool callback data is out of range");
return;
};
tool_name
};
let Some(state) = scope.get_slot_mut::<RuntimeState>() else {
throw_type_error(scope, "runtime state unavailable");
return;
codex-rs/code-mode/src/runtime/globals.rs
+4 -4
View File
@@ -53,10 +53,10 @@ fn build_tools_object<'s>(
.map(|state| state.enabled_tools.clone())
.unwrap_or_default();
for tool in enabled_tools {
for (tool_index, tool) in enabled_tools.iter().enumerate() {
let name = v8::String::new(scope, &tool.global_name)
.ok_or_else(|| "failed to allocate tool name".to_string())?;
let function = tool_function(scope, &tool.tool_name)?;
let function = tool_function(scope, tool_index)?;
tools.set(scope, name.into(), function.into());
}
Ok(tools)
@@ -116,9 +116,9 @@ where
fn tool_function<'s>(
scope: &mut v8::PinScope<'s, '_>,
tool_name: &str,
tool_index: usize,
) -> Result<v8::Local<'s, v8::Function>, String> {
let data = v8::String::new(scope, tool_name)
let data = v8::String::new(scope, &tool_index.to_string())
.ok_or_else(|| "failed to allocate tool callback data".to_string())?;
let template = v8::FunctionTemplate::builder(tool_callback)
.data(data.into())
codex-rs/code-mode/src/runtime/mod.rs
+3 -2
View File
@@ -9,6 +9,7 @@ use std::sync::OnceLock;
use std::sync::mpsc as std_mpsc;
use std::thread;
use codex_protocol::ToolName;
use serde_json::Value as JsonValue;
use tokio::sync::mpsc;
@@ -62,7 +63,7 @@ pub(crate) enum TurnMessage {
ToolCall {
cell_id: String,
id: String,
name: String,
name: ToolName,
input: Option<JsonValue>,
},
Notify {
@@ -87,7 +88,7 @@ pub(crate) enum RuntimeEvent {
YieldRequested,
ToolCall {
id: String,
name: String,
name: ToolName,
input: Option<JsonValue>,
},
Notify {
codex-rs/code-mode/src/service.rs
+2 -1
View File
@@ -5,6 +5,7 @@ use std::sync::atomic::Ordering;
use std::time::Duration;
use async_trait::async_trait;
use codex_protocol::ToolName;
use serde_json::Value as JsonValue;
use tokio::sync::Mutex;
use tokio::sync::mpsc;
@@ -26,7 +27,7 @@ use crate::runtime::spawn_runtime;
pub trait CodeModeTurnHost: Send + Sync {
async fn invoke_tool(
&self,
tool_name: String,
tool_name: ToolName,
input: Option<JsonValue>,
cancellation_token: CancellationToken,
) -> Result<JsonValue, String>;
codex-rs/codex-api/src/api_bridge.rs
+12 -6
View File
@@ -12,6 +12,7 @@ use codex_protocol::error::RetryLimitReachedError;
use codex_protocol::error::UnexpectedResponseError;
use codex_protocol::error::UsageLimitReachedError;
use http::HeaderMap;
use http::HeaderValue;
use serde::Deserialize;
use serde_json::Value;
@@ -200,11 +201,16 @@ impl CoreAuthProvider {
}
impl ApiAuthProvider for CoreAuthProvider {
fn bearer_token(&self) -> Option<String> {
self.token.clone()
}
fn account_id(&self) -> Option<String> {
self.account_id.clone()
fn add_auth_headers(&self, headers: &mut HeaderMap) {
if let Some(token) = self.token.as_ref()
&& let Ok(header) = HeaderValue::from_str(&format!("Bearer {token}"))
{
let _ = headers.insert(http::header::AUTHORIZATION, header);
}
if let Some(account_id) = self.account_id.as_ref()
&& let Ok(header) = HeaderValue::from_str(account_id)
{
let _ = headers.insert("ChatGPT-Account-ID", header);
}
}
}
codex-rs/codex-api/src/api_bridge_tests.rs
+21
View File
@@ -141,3 +141,24 @@ fn core_auth_provider_reports_when_auth_header_will_attach() {
assert!(auth.auth_header_attached());
assert_eq!(auth.auth_header_name(), Some("authorization"));
}
#[test]
fn core_auth_provider_adds_auth_headers() {
let auth = CoreAuthProvider::for_test(Some("access-token"), Some("workspace-123"));
let mut headers = HeaderMap::new();
crate::AuthProvider::add_auth_headers(&auth, &mut headers);
assert_eq!(
headers
.get(http::header::AUTHORIZATION)
.and_then(|value| value.to_str().ok()),
Some("Bearer access-token")
);
assert_eq!(
headers
.get("ChatGPT-Account-ID")
.and_then(|value| value.to_str().ok()),
Some("workspace-123")
);
}
codex-rs/codex-api/src/auth.rs
+2 -25
View File
@@ -1,33 +1,10 @@
use codex_client::Request;
use http::HeaderMap;
use http::HeaderValue;
/// Provides bearer and account identity information for API requests.
/// Adds authentication headers to API requests.
///
/// Implementations should be cheap and non-blocking; any asynchronous
/// refresh or I/O should be handled by higher layers before requests
/// reach this interface.
pub trait AuthProvider: Send + Sync {
fn bearer_token(&self) -> Option<String>;
fn account_id(&self) -> Option<String> {
None
}
}
pub(crate) fn add_auth_headers_to_header_map<A: AuthProvider>(auth: &A, headers: &mut HeaderMap) {
if let Some(token) = auth.bearer_token()
&& let Ok(header) = HeaderValue::from_str(&format!("Bearer {token}"))
{
let _ = headers.insert(http::header::AUTHORIZATION, header);
}
if let Some(account_id) = auth.account_id()
&& let Ok(header) = HeaderValue::from_str(&account_id)
{
let _ = headers.insert("ChatGPT-Account-ID", header);
}
}
pub(crate) fn add_auth_headers<A: AuthProvider>(auth: &A, mut req: Request) -> Request {
add_auth_headers_to_header_map(auth, &mut req.headers);
req
fn add_auth_headers(&self, headers: &mut HeaderMap);
}
codex-rs/codex-api/src/endpoint/compact.rs
+1 -3
View File
@@ -90,9 +90,7 @@ mod tests {
struct DummyAuth;
impl AuthProvider for DummyAuth {
fn bearer_token(&self) -> Option<String> {
None
}
fn add_auth_headers(&self, _headers: &mut HeaderMap) {}
}
#[test]
codex-rs/codex-api/src/endpoint/memories.rs
+1 -3
View File
@@ -103,9 +103,7 @@ mod tests {
struct DummyAuth;
impl AuthProvider for DummyAuth {
fn bearer_token(&self) -> Option<String> {
None
}
fn add_auth_headers(&self, _headers: &mut HeaderMap) {}
}
#[derive(Clone)]
codex-rs/codex-api/src/endpoint/models.rs
+1 -3
View File
@@ -132,9 +132,7 @@ mod tests {
struct DummyAuth;
impl AuthProvider for DummyAuth {
fn bearer_token(&self) -> Option<String> {
None
}
fn add_auth_headers(&self, _headers: &mut HeaderMap) {}
}
fn provider(base_url: &str) -> Provider {
codex-rs/codex-api/src/endpoint/realtime_call.rs
+5 -2
View File
@@ -284,8 +284,11 @@ mod tests {
struct DummyAuth;
impl AuthProvider for DummyAuth {
fn bearer_token(&self) -> Option<String> {
Some("test-token".to_string())
fn add_auth_headers(&self, headers: &mut HeaderMap) {
headers.insert(
http::header::AUTHORIZATION,
HeaderValue::from_static("Bearer test-token"),
);
}
}
codex-rs/codex-api/src/endpoint/responses_websocket.rs
+1 -2
View File
@@ -1,5 +1,4 @@
use crate::auth::AuthProvider;
use crate::auth::add_auth_headers_to_header_map;
use crate::common::ResponseEvent;
use crate::common::ResponseStream;
use crate::common::ResponsesWsRequest;
@@ -310,7 +309,7 @@ impl<A: AuthProvider> ResponsesWebsocketClient<A> {
let mut headers =
merge_request_headers(&self.provider.headers, extra_headers, default_headers);
add_auth_headers_to_header_map(&self.auth, &mut headers);
self.auth.add_auth_headers(&mut headers);
let (stream, server_reasoning_included, models_etag, server_model) =
connect_websocket(ws_url, headers, turn_state.clone()).await?;
codex-rs/codex-api/src/endpoint/session.rs
+2 -2
View File
@@ -1,5 +1,4 @@
use crate::auth::AuthProvider;
use crate::auth::add_auth_headers;
use crate::error::ApiError;
use crate::provider::Provider;
use crate::telemetry::run_with_request_telemetry;
@@ -56,7 +55,8 @@ impl<T: HttpTransport, A: AuthProvider> EndpointSession<T, A> {
if let Some(body) = body {
req.body = Some(RequestBody::Json(body.clone()));
}
add_auth_headers(&self.auth, req)
self.auth.add_auth_headers(&mut req.headers);
req
}
pub(crate) async fn execute(
codex-rs/codex-api/src/files.rs
+6 -9
View File
@@ -256,17 +256,14 @@ fn authorized_request(
method: reqwest::Method,
url: &str,
) -> reqwest::RequestBuilder {
let mut headers = http::HeaderMap::new();
auth.add_auth_headers(&mut headers);
let client = build_reqwest_client();
let mut request = client
client
.request(method, url)
.timeout(OPENAI_FILE_REQUEST_TIMEOUT);
if let Some(token) = auth.bearer_token() {
request = request.bearer_auth(token);
}
if let Some(account_id) = auth.account_id() {
request = request.header("chatgpt-account-id", account_id);
}
request
.timeout(OPENAI_FILE_REQUEST_TIMEOUT)
.headers(headers)
}
fn build_reqwest_client() -> reqwest::Client {
codex-rs/codex-api/src/lib.rs
+1 -1
View File
@@ -57,7 +57,7 @@ pub use crate::error::ApiError;
pub use crate::files::upload_local_file;
pub use crate::provider::Provider;
pub use crate::provider::RetryConfig;
pub use crate::provider::is_azure_responses_wire_base_url;
pub use crate::provider::is_azure_responses_provider;
pub use crate::requests::Compression;
pub use crate::sse::stream_from_fixture;
pub use crate::telemetry::SseTelemetry;
codex-rs/codex-api/src/provider.rs
+13 -14
View File
@@ -86,7 +86,7 @@ impl Provider {
}
pub fn is_azure_responses_endpoint(&self) -> bool {
is_azure_responses_wire_base_url(&self.name, Some(&self.base_url))
is_azure_responses_provider(&self.name, Some(&self.base_url))
}
pub fn websocket_url_for_path(&self, path: &str) -> Result<Url, url::ParseError> {
@@ -103,21 +103,20 @@ impl Provider {
}
}
pub fn is_azure_responses_wire_base_url(name: &str, base_url: Option<&str>) -> bool {
pub fn is_azure_responses_provider(name: &str, base_url: Option<&str>) -> bool {
if name.eq_ignore_ascii_case("azure") {
return true;
true
} else if let Some(base_url) = base_url {
matches_azure_responses_base_url(base_url)
} else {
false
}
let Some(base_url) = base_url else {
return false;
};
let base = base_url.to_ascii_lowercase();
base.contains("openai.azure.") || matches_azure_responses_base_url(&base)
}
fn matches_azure_responses_base_url(base_url: &str) -> bool {
const AZURE_MARKERS: [&str; 5] = [
let base_url = base_url.to_ascii_lowercase();
const AZURE_MARKERS: [&str; 6] = [
"openai.azure.",
"cognitiveservices.azure.",
"aoai.azure.",
"azure-api.",
@@ -144,12 +143,12 @@ mod tests {
for base_url in positive_cases {
assert!(
is_azure_responses_wire_base_url("test", Some(base_url)),
is_azure_responses_provider("test", Some(base_url)),
"expected {base_url} to be detected as Azure"
);
}
assert!(is_azure_responses_wire_base_url(
assert!(is_azure_responses_provider(
"Azure",
Some("https://example.com")
));
@@ -162,7 +161,7 @@ mod tests {
for base_url in negative_cases {
assert!(
!is_azure_responses_wire_base_url("test", Some(base_url)),
!is_azure_responses_provider("test", Some(base_url)),
"expected {base_url} not to be detected as Azure"
);
}
codex-rs/codex-api/tests/clients.rs
+9 -9
View File
@@ -91,9 +91,7 @@ impl HttpTransport for RecordingTransport {
struct NoAuth;
impl AuthProvider for NoAuth {
fn bearer_token(&self) -> Option<String> {
None
}
fn add_auth_headers(&self, _headers: &mut HeaderMap) {}
}
#[derive(Clone)]
@@ -112,12 +110,14 @@ impl StaticAuth {
}
impl AuthProvider for StaticAuth {
fn bearer_token(&self) -> Option<String> {
Some(self.token.clone())
}
fn account_id(&self) -> Option<String> {
Some(self.account_id.clone())
fn add_auth_headers(&self, headers: &mut HeaderMap) {
let token = &self.token;
if let Ok(header) = HeaderValue::from_str(&format!("Bearer {token}")) {
headers.insert(http::header::AUTHORIZATION, header);
}
if let Ok(header) = HeaderValue::from_str(&self.account_id) {
headers.insert("ChatGPT-Account-ID", header);
}
}
}
codex-rs/codex-api/tests/models_integration.rs
+1 -3
View File
@@ -24,9 +24,7 @@ use wiremock::matchers::path;
struct DummyAuth;
impl AuthProvider for DummyAuth {
fn bearer_token(&self) -> Option<String> {
None
}
fn add_auth_headers(&self, _headers: &mut HeaderMap) {}
}
fn provider(base_url: &str) -> Provider {
codex-rs/codex-api/tests/sse_end_to_end.rs
+1 -3
View File
@@ -53,9 +53,7 @@ impl HttpTransport for FixtureSseTransport {
struct NoAuth;
impl AuthProvider for NoAuth {
fn bearer_token(&self) -> Option<String> {
None
}
fn add_auth_headers(&self, _headers: &mut HeaderMap) {}
}
fn provider(name: &str) -> Provider {
codex-rs/codex-mcp/src/lib.rs
+1 -2
View File
@@ -36,8 +36,7 @@ pub use mcp::tool_plugin_provenance;
pub use mcp::with_codex_apps_mcp;
pub use mcp_connection_manager::CodexAppsToolsCacheKey;
pub use mcp_connection_manager::DEFAULT_STARTUP_TIMEOUT;
pub use mcp_connection_manager::MCP_SANDBOX_STATE_CAPABILITY;
pub use mcp_connection_manager::MCP_SANDBOX_STATE_METHOD;
pub use mcp_connection_manager::MCP_SANDBOX_STATE_META_CAPABILITY;
pub use mcp_connection_manager::McpConnectionManager;
pub use mcp_connection_manager::SandboxState;
pub use mcp_connection_manager::ToolInfo;
codex-rs/codex-mcp/src/mcp/mod.rs
+2 -18
View File
@@ -35,7 +35,6 @@ use codex_protocol::protocol::SandboxPolicy;
use serde_json::Value;
use crate::mcp_connection_manager::McpConnectionManager;
use crate::mcp_connection_manager::SandboxState;
use crate::mcp_connection_manager::codex_apps_tools_cache_key;
pub type McpManager = McpConnectionManager;
@@ -347,14 +346,6 @@ pub async fn collect_mcp_snapshot_with_detail(
let (tx_event, rx_event) = unbounded();
drop(rx_event);
// Use ReadOnly sandbox policy for MCP snapshot collection (safest default)
let sandbox_state = SandboxState {
sandbox_policy: SandboxPolicy::new_read_only_policy(),
codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(),
sandbox_cwd: env::current_dir().unwrap_or_else(|_| PathBuf::from("/")),
use_legacy_landlock: config.use_legacy_landlock,
};
let (mcp_connection_manager, cancel_token) = McpConnectionManager::new(
&mcp_servers,
config.mcp_oauth_credentials_store_mode,
@@ -362,7 +353,7 @@ pub async fn collect_mcp_snapshot_with_detail(
&config.approval_policy,
submit_id,
tx_event,
sandbox_state,
SandboxPolicy::new_read_only_policy(),
config.codex_home.clone(),
codex_apps_tools_cache_key(auth),
tool_plugin_provenance,
@@ -421,13 +412,6 @@ pub async fn collect_mcp_server_status_snapshot_with_detail(
let (tx_event, rx_event) = unbounded();
drop(rx_event);
let sandbox_state = SandboxState {
sandbox_policy: SandboxPolicy::new_read_only_policy(),
codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(),
sandbox_cwd: env::current_dir().unwrap_or_else(|_| PathBuf::from("/")),
use_legacy_landlock: config.use_legacy_landlock,
};
let (mcp_connection_manager, cancel_token) = McpConnectionManager::new(
&mcp_servers,
config.mcp_oauth_credentials_store_mode,
@@ -435,7 +419,7 @@ pub async fn collect_mcp_server_status_snapshot_with_detail(
&config.approval_policy,
submit_id,
tx_event,
sandbox_state,
SandboxPolicy::new_read_only_policy(),
config.codex_home.clone(),
codex_apps_tools_cache_key(auth),
tool_plugin_provenance,
codex-rs/codex-mcp/src/mcp_connection_manager.rs
+33 -84
View File
@@ -36,6 +36,7 @@ use codex_async_utils::CancelErr;
use codex_async_utils::OrCancelExt;
use codex_config::Constrained;
use codex_config::types::OAuthCredentialsStoreMode;
use codex_protocol::ToolName;
use codex_protocol::approvals::ElicitationRequest;
use codex_protocol::approvals::ElicitationRequestEvent;
use codex_protocol::mcp::CallToolResult;
@@ -155,6 +156,12 @@ pub struct ToolInfo {
pub connector_description: Option<String>,
}
impl ToolInfo {
pub fn canonical_tool_name(&self) -> ToolName {
ToolName::namespaced(self.callable_namespace.clone(), self.callable_name.clone())
}
}
const META_OPENAI_FILE_PARAMS: &str = "openai/fileParams";
pub fn declared_openai_file_input_param_names(
@@ -432,7 +439,7 @@ struct ManagedClient {
tool_filter: ToolFilter,
tool_timeout: Option<Duration>,
server_instructions: Option<String>,
server_supports_sandbox_state_capability: bool,
server_supports_sandbox_state_meta_capability: bool,
codex_apps_tools_cache_context: Option<CodexAppsToolsCacheContext>,
}
@@ -461,22 +468,6 @@ impl ManagedClient {
self.tools.clone()
}
/// Returns once the server has ack'd the sandbox state update.
async fn notify_sandbox_state_change(&self, sandbox_state: &SandboxState) -> Result<()> {
if !self.server_supports_sandbox_state_capability {
return Ok(());
}
let _response = self
.client
.send_custom_request(
MCP_SANDBOX_STATE_METHOD,
Some(serde_json::to_value(sandbox_state)?),
)
.await?;
Ok(())
}
}
#[derive(Clone)]
@@ -634,18 +625,11 @@ impl AsyncManagedClient {
};
tools.map(annotate_tools)
}
async fn notify_sandbox_state_change(&self, sandbox_state: &SandboxState) -> Result<()> {
let managed = self.client().await?;
managed.notify_sandbox_state_change(sandbox_state).await
}
}
pub const MCP_SANDBOX_STATE_CAPABILITY: &str = "codex/sandbox-state";
/// Custom MCP request to push sandbox state updates.
/// When used, the `params` field of the notification is [`SandboxState`].
pub const MCP_SANDBOX_STATE_METHOD: &str = "codex/sandbox-state/update";
/// MCP server capability indicating that Codex should include [`SandboxState`]
/// in tool-call request `_meta` under this key.
pub const MCP_SANDBOX_STATE_META_CAPABILITY: &str = "codex/sandbox-state-meta";
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
@@ -723,7 +707,7 @@ impl McpConnectionManager {
approval_policy: &Constrained<AskForApproval>,
submit_id: String,
tx_event: Sender<Event>,
initial_sandbox_state: SandboxState,
initial_sandbox_policy: SandboxPolicy,
codex_home: PathBuf,
codex_apps_tools_cache_key: CodexAppsToolsCacheKey,
tool_plugin_provenance: ToolPluginProvenance,
@@ -732,10 +716,8 @@ impl McpConnectionManager {
let mut clients = HashMap::new();
let mut server_origins = HashMap::new();
let mut join_set = JoinSet::new();
let elicitation_requests = ElicitationRequestManager::new(
approval_policy.value(),
initial_sandbox_state.sandbox_policy.clone(),
);
let elicitation_requests =
ElicitationRequestManager::new(approval_policy.value(), initial_sandbox_policy);
let tool_plugin_provenance = Arc::new(tool_plugin_provenance);
let startup_submit_id = submit_id.clone();
let mcp_servers = mcp_servers.clone();
@@ -775,25 +757,13 @@ impl McpConnectionManager {
let tx_event = tx_event.clone();
let submit_id = startup_submit_id.clone();
let auth_entry = auth_entries.get(&server_name).cloned();
let sandbox_state = initial_sandbox_state.clone();
join_set.spawn(async move {
let outcome = async_managed_client.client().await;
if cancel_token.is_cancelled() {
return (server_name, Err(StartupOutcomeError::Cancelled));
}
let status = match &outcome {
Ok(_) => {
// Send sandbox state notification immediately after Ready
if let Err(e) = async_managed_client
.notify_sandbox_state_change(&sandbox_state)
.await
{
warn!(
"Failed to notify sandbox state to MCP server {server_name}: {e:#}",
);
}
McpStartupStatus::Ready
}
Ok(_) => McpStartupStatus::Ready,
Err(error) => {
let error_str = mcp_init_error_display(
server_name.as_str(),
@@ -1142,6 +1112,16 @@ impl McpConnectionManager {
})
}
pub async fn server_supports_sandbox_state_meta_capability(
&self,
server: &str,
) -> Result<bool> {
Ok(self
.client_by_name(server)
.await?
.server_supports_sandbox_state_meta_capability)
}
/// List resources from the specified server.
pub async fn list_resources(
&self,
@@ -1191,42 +1171,11 @@ impl McpConnectionManager {
.with_context(|| format!("resources/read failed for `{server}` ({uri})"))
}
pub async fn resolve_tool_info(&self, name: &str, namespace: Option<&str>) -> Option<ToolInfo> {
let qualified_name = match namespace {
Some(namespace) if name.starts_with(namespace) => name.to_string(),
Some(namespace) => format!("{namespace}{name}"),
None => name.to_string(),
};
self.list_all_tools().await.get(&qualified_name).cloned()
}
pub async fn notify_sandbox_state_change(&self, sandbox_state: &SandboxState) -> Result<()> {
let mut join_set = JoinSet::new();
for async_managed_client in self.clients.values() {
let sandbox_state = sandbox_state.clone();
let async_managed_client = async_managed_client.clone();
join_set.spawn(async move {
async_managed_client
.notify_sandbox_state_change(&sandbox_state)
.await
});
}
while let Some(join_res) = join_set.join_next().await {
match join_res {
Ok(Ok(())) => {}
Ok(Err(err)) => {
warn!("Failed to notify sandbox state change to MCP server: {err:#}");
}
Err(err) => {
warn!("Task panic when notifying sandbox state change to MCP server: {err:#}");
}
}
}
Ok(())
pub async fn resolve_tool_info(&self, tool_name: &ToolName) -> Option<ToolInfo> {
let all_tools = self.list_all_tools().await;
all_tools
.into_values()
.find(|tool| tool.canonical_tool_name() == *tool_name)
}
}
@@ -1473,11 +1422,11 @@ async fn start_server_task(
.await
.map_err(StartupOutcomeError::from)?;
let server_supports_sandbox_state_capability = initialize_result
let server_supports_sandbox_state_meta_capability = initialize_result
.capabilities
.experimental
.as_ref()
.and_then(|exp| exp.get(MCP_SANDBOX_STATE_CAPABILITY))
.and_then(|exp| exp.get(MCP_SANDBOX_STATE_META_CAPABILITY))
.is_some();
let list_start = Instant::now();
let fetch_start = Instant::now();
@@ -1514,7 +1463,7 @@ async fn start_server_task(
tool_timeout: Some(tool_timeout),
tool_filter,
server_instructions: initialize_result.instructions,
server_supports_sandbox_state_capability,
server_supports_sandbox_state_meta_capability,
codex_apps_tools_cache_context,
};
codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs
+37
View File
@@ -1,4 +1,5 @@
use super::*;
use codex_protocol::ToolName;
use codex_protocol::protocol::GranularApprovalConfig;
use codex_protocol::protocol::McpAuthStatus;
use pretty_assertions::assert_eq;
@@ -646,6 +647,42 @@ async fn list_all_tools_uses_startup_snapshot_while_client_is_pending() {
assert_eq!(tool.callable_name, "calendar_create_event");
}
#[tokio::test]
async fn resolve_tool_info_accepts_canonical_namespaced_tool_names() {
let startup_tools = vec![create_test_tool("rmcp", "echo")];
let pending_client = futures::future::pending::<Result<ManagedClient, StartupOutcomeError>>()
.boxed()
.shared();
let approval_policy = Constrained::allow_any(AskForApproval::OnFailure);
let sandbox_policy = Constrained::allow_any(SandboxPolicy::new_read_only_policy());
let mut manager = McpConnectionManager::new_uninitialized(&approval_policy, &sandbox_policy);
manager.clients.insert(
"rmcp".to_string(),
AsyncManagedClient {
client: pending_client,
startup_snapshot: Some(startup_tools),
startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(false)),
tool_plugin_provenance: Arc::new(ToolPluginProvenance::default()),
},
);
let tool = manager
.resolve_tool_info(&ToolName::namespaced("mcp__rmcp__", "echo"))
.await
.expect("split MCP tool namespace and name should resolve");
let expected = ("rmcp", "mcp__rmcp__", "echo", "echo");
assert_eq!(
(
tool.server_name.as_str(),
tool.callable_namespace.as_str(),
tool.callable_name.as_str(),
tool.tool.name.as_ref(),
),
expected
);
}
#[tokio::test]
async fn list_all_tools_blocks_while_client_is_pending_without_startup_snapshot() {
let pending_client = futures::future::pending::<Result<ManagedClient, StartupOutcomeError>>()
codex-rs/core/BUILD.bazel
+55 -1
View File
@@ -1,4 +1,4 @@
load("//:defs.bzl", "codex_rust_crate")
load("//:defs.bzl", "codex_rust_crate", "workspace_root_test")
filegroup(
name = "model_availability_nux_fixtures",
@@ -31,6 +31,15 @@ codex_rust_crate(
"//codex-rs/apply-patch:apply_patch_tool_instructions.md",
"templates/realtime/backend_prompt.md",
],
integration_test_flaky_by_stem = {
"all": [
"suite::abort_tasks::interrupt_tool_records_history_entries",
"suite::apply_patch_cli::apply_patch_cli_rejects_invalid_hunk_header::applypatchmodeloutput_function_expects",
],
"responses_headers": [
"responses_stream_includes_turn_metadata_header_for_git_workspace_e2e",
],
},
integration_test_timeout = "long",
test_data_extra = [
"config.schema.json",
@@ -48,11 +57,56 @@ codex_rust_crate(
"//:AGENTS.md",
],
test_tags = ["no-sandbox"],
unit_test_args = [
"--skip",
"guardian::tests::guardian_parallel_reviews_fork_from_last_committed_trunk_history",
"--skip",
"agent::control::tests::completion_watcher_notifies_parent_when_child_is_missing",
],
unit_test_timeout = "long",
extra_binaries = [
"//codex-rs/linux-sandbox:codex-linux-sandbox",
"//codex-rs/rmcp-client:test_stdio_server",
"//codex-rs/rmcp-client:test_streamable_http_server",
"//codex-rs/responses-api-proxy:codex-responses-api-proxy",
"//codex-rs/cli:codex",
],
)
workspace_root_test(
name = "core-guardian-parallel-reviews-flaky-test",
env = {
"INSTA_WORKSPACE_ROOT": ".",
"INSTA_SNAPSHOT_PATH": "src",
},
test_bin = ":core-unit-tests-bin",
workspace_root_marker = "//codex-rs/utils/cargo-bin:repo_root.marker",
args = [
"--exact",
"guardian::tests::guardian_parallel_reviews_fork_from_last_committed_trunk_history",
],
tags = [
"flaky",
"no-sandbox",
],
timeout = "long",
)
workspace_root_test(
name = "core-completion-watcher-missing-child-flaky-test",
env = {
"INSTA_WORKSPACE_ROOT": ".",
"INSTA_SNAPSHOT_PATH": "src",
},
test_bin = ":core-unit-tests-bin",
workspace_root_marker = "//codex-rs/utils/cargo-bin:repo_root.marker",
args = [
"--exact",
"agent::control::tests::completion_watcher_notifies_parent_when_child_is_missing",
],
tags = [
"flaky",
"no-sandbox",
],
timeout = "long",
)
codex-rs/core/Cargo.toml
+3
View File
@@ -41,6 +41,7 @@ codex-login = { workspace = true }
codex-mcp = { workspace = true }
codex-model-provider-info = { workspace = true }
codex-models-manager = { workspace = true }
ed25519-dalek = { workspace = true }
codex-shell-command = { workspace = true }
codex-execpolicy = { workspace = true }
codex-git-utils = { workspace = true }
@@ -56,6 +57,7 @@ codex-rmcp-client = { workspace = true }
codex-sandboxing = { workspace = true }
codex-state = { workspace = true }
codex-terminal-detection = { workspace = true }
codex-thread-store = { workspace = true }
codex-tools = { workspace = true }
codex-utils-absolute-path = { workspace = true }
codex-utils-cache = { workspace = true }
@@ -143,6 +145,7 @@ codex-shell-escalation = { workspace = true }
[dev-dependencies]
assert_cmd = { workspace = true }
assert_matches = { workspace = true }
codex-arg0 = { workspace = true }
codex-otel = { workspace = true }
codex-test-binary-support = { workspace = true }
codex-utils-cargo-bin = { workspace = true }
codex-rs/core/config.schema.json
+6
View File
@@ -526,6 +526,9 @@
},
"web_search_request": {
"type": "boolean"
},
"workspace_dependencies": {
"type": "boolean"
}
},
"type": "object"
@@ -2417,6 +2420,9 @@
},
"web_search_request": {
"type": "boolean"
},
"workspace_dependencies": {
"type": "boolean"
}
},
"type": "object"
codex-rs/core/src/agent/control_tests.rs
+9 -27
View File
@@ -7,7 +7,6 @@ use crate::config::Config;
use crate::config::ConfigBuilder;
use crate::contextual_user_message::SUBAGENT_NOTIFICATION_OPEN_TAG;
use assert_matches::assert_matches;
use chrono::Utc;
use codex_features::Feature;
use codex_login::CodexAuth;
use codex_protocol::AgentPath;
@@ -24,6 +23,9 @@ use codex_protocol::protocol::TurnAbortReason;
use codex_protocol::protocol::TurnAbortedEvent;
use codex_protocol::protocol::TurnCompleteEvent;
use codex_protocol::protocol::TurnStartedEvent;
use codex_thread_store::ArchiveThreadParams;
use codex_thread_store::LocalThreadStore;
use codex_thread_store::ThreadStore;
use pretty_assertions::assert_eq;
use tempfile::TempDir;
use tokio::time::Duration;
@@ -1658,38 +1660,18 @@ async fn resume_agent_from_rollout_reads_archived_rollout_path() {
.await
.expect("child thread should exist");
persist_thread_for_tree_resume(&child_thread, "persist before archiving").await;
let rollout_path = child_thread
.rollout_path()
.expect("thread should have rollout path");
let state_db = child_thread
.state_db()
.expect("thread should have state db handle");
let _ = harness
.control
.shutdown_live_agent(child_thread_id)
.await
.expect("child shutdown should succeed");
let archived_root = harness
.config
.codex_home
.join(crate::ARCHIVED_SESSIONS_SUBDIR);
tokio::fs::create_dir_all(&archived_root)
let store = LocalThreadStore::new(codex_rollout::RolloutConfig::from_view(&harness.config));
store
.archive_thread(ArchiveThreadParams {
thread_id: child_thread_id,
})
.await
.expect("archived root should exist");
let archived_rollout_path = archived_root.join(
rollout_path
.file_name()
.expect("rollout file name should be present"),
);
tokio::fs::rename(&rollout_path, &archived_rollout_path)
.await
.expect("rollout should move to archived path");
state_db
.mark_archived(child_thread_id, archived_rollout_path.as_path(), Utc::now())
.await
.expect("state db archive update should succeed");
.expect("child thread should archive");
let resumed_thread_id = harness
.control
codex-rs/core/src/agent_identity.rs
+766
View File
@@ -0,0 +1,766 @@
use std::sync::Arc;
use std::time::Duration;
use anyhow::Context;
use anyhow::Result;
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
use chrono::SecondsFormat;
use chrono::Utc;
use codex_features::Feature;
use codex_login::AgentIdentityAuthRecord;
use codex_login::AuthManager;
use codex_login::CodexAuth;
use codex_login::default_client::create_client;
use codex_protocol::protocol::SessionSource;
use ed25519_dalek::SigningKey;
use ed25519_dalek::VerifyingKey;
use ed25519_dalek::pkcs8::DecodePrivateKey;
use ed25519_dalek::pkcs8::EncodePrivateKey;
use rand::TryRngCore;
use rand::rngs::OsRng;
use serde::Deserialize;
use serde::Serialize;
use tokio::sync::Mutex;
use tracing::debug;
use tracing::info;
use tracing::warn;
use crate::config::Config;
const AGENT_REGISTRATION_TIMEOUT: Duration = Duration::from_secs(15);
const AGENT_IDENTITY_BISCUIT_TIMEOUT: Duration = Duration::from_secs(15);
#[derive(Clone)]
pub(crate) struct AgentIdentityManager {
auth_manager: Arc<AuthManager>,
chatgpt_base_url: String,
feature_enabled: bool,
abom: AgentBillOfMaterials,
ensure_lock: Arc<Mutex<()>>,
}
impl std::fmt::Debug for AgentIdentityManager {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("AgentIdentityManager")
.field("chatgpt_base_url", &self.chatgpt_base_url)
.field("feature_enabled", &self.feature_enabled)
.field("abom", &self.abom)
.finish_non_exhaustive()
}
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub(crate) struct StoredAgentIdentity {
pub(crate) binding_id: String,
pub(crate) chatgpt_account_id: String,
pub(crate) chatgpt_user_id: Option<String>,
pub(crate) agent_runtime_id: String,
pub(crate) private_key_pkcs8_base64: String,
pub(crate) public_key_ssh: String,
pub(crate) registered_at: String,
pub(crate) abom: AgentBillOfMaterials,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub(crate) struct AgentBillOfMaterials {
pub(crate) agent_version: String,
pub(crate) agent_harness_id: String,
pub(crate) running_location: String,
}
#[derive(Debug, Serialize)]
struct RegisterAgentRequest {
abom: AgentBillOfMaterials,
agent_public_key: String,
capabilities: Vec<String>,
}
#[derive(Debug, Deserialize)]
struct RegisterAgentResponse {
agent_runtime_id: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct AgentIdentityBinding {
binding_id: String,
chatgpt_account_id: String,
chatgpt_user_id: Option<String>,
access_token: String,
}
struct GeneratedAgentKeyMaterial {
private_key_pkcs8_base64: String,
public_key_ssh: String,
}
impl AgentIdentityManager {
pub(crate) fn new(
config: &Config,
auth_manager: Arc<AuthManager>,
session_source: SessionSource,
) -> Self {
Self {
auth_manager,
chatgpt_base_url: config.chatgpt_base_url.clone(),
feature_enabled: config.features.enabled(Feature::UseAgentIdentity),
abom: build_abom(session_source),
ensure_lock: Arc::new(Mutex::new(())),
}
}
pub(crate) fn is_enabled(&self) -> bool {
self.feature_enabled
}
pub(crate) async fn ensure_registered_identity(&self) -> Result<Option<StoredAgentIdentity>> {
if !self.feature_enabled {
return Ok(None);
}
let Some(auth) = self.auth_manager.auth().await else {
debug!("skipping agent identity registration because no auth is available");
return Ok(None);
};
let Some(binding) =
AgentIdentityBinding::from_auth(&auth, self.auth_manager.forced_chatgpt_workspace_id())
else {
debug!("skipping agent identity registration because ChatGPT auth is unavailable");
return Ok(None);
};
let _guard = self.ensure_lock.lock().await;
if let Some(stored_identity) = self.load_stored_identity(&auth, &binding)? {
info!(
agent_runtime_id = %stored_identity.agent_runtime_id,
binding_id = %binding.binding_id,
"reusing stored agent identity"
);
return Ok(Some(stored_identity));
}
let stored_identity = self.register_agent_identity(&binding).await?;
self.store_identity(&auth, &stored_identity)?;
Ok(Some(stored_identity))
}
async fn register_agent_identity(
&self,
binding: &AgentIdentityBinding,
) -> Result<StoredAgentIdentity> {
let key_material = generate_agent_key_material()?;
let request_body = RegisterAgentRequest {
abom: self.abom.clone(),
agent_public_key: key_material.public_key_ssh.clone(),
capabilities: Vec::new(),
};
let url = agent_registration_url(&self.chatgpt_base_url);
let human_biscuit = self.mint_human_biscuit(binding, "POST", &url).await?;
let client = create_client();
let response = client
.post(&url)
.header("X-OpenAI-Authorization", human_biscuit)
.json(&request_body)
.timeout(AGENT_REGISTRATION_TIMEOUT)
.send()
.await
.with_context(|| {
format!("failed to send agent identity registration request to {url}")
})?;
if response.status().is_success() {
let response_body = response
.json::<RegisterAgentResponse>()
.await
.with_context(|| format!("failed to parse agent identity response from {url}"))?;
let stored_identity = StoredAgentIdentity {
binding_id: binding.binding_id.clone(),
chatgpt_account_id: binding.chatgpt_account_id.clone(),
chatgpt_user_id: binding.chatgpt_user_id.clone(),
agent_runtime_id: response_body.agent_runtime_id,
private_key_pkcs8_base64: key_material.private_key_pkcs8_base64,
public_key_ssh: key_material.public_key_ssh,
registered_at: Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true),
abom: self.abom.clone(),
};
info!(
agent_runtime_id = %stored_identity.agent_runtime_id,
binding_id = %binding.binding_id,
"registered agent identity"
);
return Ok(stored_identity);
}
let status = response.status();
let body = response.text().await.unwrap_or_default();
anyhow::bail!("agent identity registration failed with status {status} from {url}: {body}")
}
async fn mint_human_biscuit(
&self,
binding: &AgentIdentityBinding,
target_method: &str,
target_url: &str,
) -> Result<String> {
let url = agent_identity_biscuit_url(&self.chatgpt_base_url);
let request_id = agent_identity_request_id()?;
let client = create_client();
let response = client
.get(&url)
.bearer_auth(&binding.access_token)
.header("X-Request-Id", request_id.clone())
.header("X-Original-Method", target_method)
.header("X-Original-Url", target_url)
.timeout(AGENT_IDENTITY_BISCUIT_TIMEOUT)
.send()
.await
.with_context(|| format!("failed to send agent identity biscuit request to {url}"))?;
if response.status().is_success() {
let human_biscuit = response
.headers()
.get("x-openai-authorization")
.context("agent identity biscuit response did not include x-openai-authorization")?
.to_str()
.context("agent identity biscuit response header was not valid UTF-8")?
.to_string();
info!(
request_id = %request_id,
"minted human biscuit for agent identity registration"
);
return Ok(human_biscuit);
}
let status = response.status();
let body = response.text().await.unwrap_or_default();
anyhow::bail!(
"agent identity biscuit minting failed with status {status} from {url}: {body}"
)
}
fn load_stored_identity(
&self,
auth: &CodexAuth,
binding: &AgentIdentityBinding,
) -> Result<Option<StoredAgentIdentity>> {
let Some(record) = auth.get_agent_identity(&binding.chatgpt_account_id) else {
return Ok(None);
};
let stored_identity =
match StoredAgentIdentity::from_auth_record(binding, record, self.abom.clone()) {
Ok(stored_identity) => stored_identity,
Err(error) => {
warn!(
binding_id = %binding.binding_id,
error = %error,
"stored agent identity is invalid; deleting cached value"
);
auth.remove_agent_identity()?;
return Ok(None);
}
};
if !stored_identity.matches_binding(binding) {
warn!(
binding_id = %binding.binding_id,
"stored agent identity binding no longer matches current auth; deleting cached value"
);
auth.remove_agent_identity()?;
return Ok(None);
}
if let Err(error) = stored_identity.validate_key_material() {
warn!(
agent_runtime_id = %stored_identity.agent_runtime_id,
binding_id = %binding.binding_id,
error = %error,
"stored agent identity key material is invalid; deleting cached value"
);
auth.remove_agent_identity()?;
return Ok(None);
}
Ok(Some(stored_identity))
}
fn store_identity(
&self,
auth: &CodexAuth,
stored_identity: &StoredAgentIdentity,
) -> Result<()> {
auth.set_agent_identity(stored_identity.to_auth_record())?;
Ok(())
}
#[cfg(test)]
fn new_for_tests(
auth_manager: Arc<AuthManager>,
feature_enabled: bool,
chatgpt_base_url: String,
session_source: SessionSource,
) -> Self {
Self {
auth_manager,
chatgpt_base_url,
feature_enabled,
abom: build_abom(session_source),
ensure_lock: Arc::new(Mutex::new(())),
}
}
}
impl StoredAgentIdentity {
fn from_auth_record(
binding: &AgentIdentityBinding,
record: AgentIdentityAuthRecord,
abom: AgentBillOfMaterials,
) -> Result<Self> {
if record.workspace_id != binding.chatgpt_account_id {
anyhow::bail!(
"stored agent identity workspace {:?} does not match current workspace {:?}",
record.workspace_id,
binding.chatgpt_account_id
);
}
let signing_key = signing_key_from_private_key_pkcs8_base64(&record.agent_private_key)?;
Ok(Self {
binding_id: binding.binding_id.clone(),
chatgpt_account_id: binding.chatgpt_account_id.clone(),
chatgpt_user_id: record.chatgpt_user_id,
agent_runtime_id: record.agent_runtime_id,
private_key_pkcs8_base64: record.agent_private_key,
public_key_ssh: encode_ssh_ed25519_public_key(&signing_key.verifying_key()),
registered_at: record.registered_at,
abom,
})
}
fn to_auth_record(&self) -> AgentIdentityAuthRecord {
AgentIdentityAuthRecord {
workspace_id: self.chatgpt_account_id.clone(),
chatgpt_user_id: self.chatgpt_user_id.clone(),
agent_runtime_id: self.agent_runtime_id.clone(),
agent_private_key: self.private_key_pkcs8_base64.clone(),
registered_at: self.registered_at.clone(),
}
}
fn matches_binding(&self, binding: &AgentIdentityBinding) -> bool {
self.binding_id == binding.binding_id
&& self.chatgpt_account_id == binding.chatgpt_account_id
&& match binding.chatgpt_user_id.as_deref() {
Some(chatgpt_user_id) => self.chatgpt_user_id.as_deref() == Some(chatgpt_user_id),
None => true,
}
}
fn validate_key_material(&self) -> Result<()> {
let signing_key = self.signing_key()?;
let derived_public_key = encode_ssh_ed25519_public_key(&signing_key.verifying_key());
anyhow::ensure!(
self.public_key_ssh == derived_public_key,
"stored public key does not match the private key"
);
Ok(())
}
pub(crate) fn signing_key(&self) -> Result<SigningKey> {
signing_key_from_private_key_pkcs8_base64(&self.private_key_pkcs8_base64)
}
}
impl AgentIdentityBinding {
fn from_auth(auth: &CodexAuth, forced_workspace_id: Option<String>) -> Option<Self> {
if !auth.is_chatgpt_auth() {
return None;
}
let token_data = auth.get_token_data().ok()?;
let resolved_account_id =
forced_workspace_id
.filter(|value| !value.is_empty())
.or(token_data
.account_id
.clone()
.filter(|value| !value.is_empty()))?;
Some(Self {
binding_id: format!("chatgpt-account-{resolved_account_id}"),
chatgpt_account_id: resolved_account_id,
chatgpt_user_id: token_data
.id_token
.chatgpt_user_id
.filter(|value| !value.is_empty()),
access_token: token_data.access_token,
})
}
}
fn build_abom(session_source: SessionSource) -> AgentBillOfMaterials {
AgentBillOfMaterials {
agent_version: env!("CARGO_PKG_VERSION").to_string(),
agent_harness_id: match &session_source {
SessionSource::VSCode => "codex-app".to_string(),
SessionSource::Cli
| SessionSource::Exec
| SessionSource::Mcp
| SessionSource::Custom(_)
| SessionSource::SubAgent(_)
| SessionSource::Unknown => "codex-cli".to_string(),
},
running_location: format!("{}-{}", session_source, std::env::consts::OS),
}
}
fn generate_agent_key_material() -> Result<GeneratedAgentKeyMaterial> {
let mut secret_key_bytes = [0u8; 32];
OsRng
.try_fill_bytes(&mut secret_key_bytes)
.context("failed to generate agent identity private key bytes")?;
let signing_key = SigningKey::from_bytes(&secret_key_bytes);
let private_key_pkcs8 = signing_key
.to_pkcs8_der()
.context("failed to encode agent identity private key as PKCS#8")?;
Ok(GeneratedAgentKeyMaterial {
private_key_pkcs8_base64: BASE64_STANDARD.encode(private_key_pkcs8.as_bytes()),
public_key_ssh: encode_ssh_ed25519_public_key(&signing_key.verifying_key()),
})
}
fn encode_ssh_ed25519_public_key(verifying_key: &VerifyingKey) -> String {
let mut blob = Vec::with_capacity(4 + 11 + 4 + 32);
append_ssh_string(&mut blob, b"ssh-ed25519");
append_ssh_string(&mut blob, verifying_key.as_bytes());
format!("ssh-ed25519 {}", BASE64_STANDARD.encode(blob))
}
fn append_ssh_string(buf: &mut Vec<u8>, value: &[u8]) {
buf.extend_from_slice(&(value.len() as u32).to_be_bytes());
buf.extend_from_slice(value);
}
fn agent_registration_url(chatgpt_base_url: &str) -> String {
let trimmed = chatgpt_base_url.trim_end_matches('/');
format!("{trimmed}/v1/agent/register")
}
fn signing_key_from_private_key_pkcs8_base64(private_key_pkcs8_base64: &str) -> Result<SigningKey> {
let private_key = BASE64_STANDARD
.decode(private_key_pkcs8_base64)
.context("stored agent identity private key is not valid base64")?;
SigningKey::from_pkcs8_der(&private_key)
.context("stored agent identity private key is not valid PKCS#8")
}
fn agent_identity_biscuit_url(chatgpt_base_url: &str) -> String {
let trimmed = chatgpt_base_url.trim_end_matches('/');
format!("{trimmed}/authenticate_app_v2")
}
fn agent_identity_request_id() -> Result<String> {
let mut request_id_bytes = [0u8; 16];
OsRng
.try_fill_bytes(&mut request_id_bytes)
.context("failed to generate agent identity request id")?;
Ok(format!(
"codex-agent-identity-{}",
URL_SAFE_NO_PAD.encode(request_id_bytes)
))
}
#[cfg(test)]
mod tests {
use super::*;
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
use codex_app_server_protocol::AuthMode as ApiAuthMode;
use codex_login::AuthCredentialsStoreMode;
use codex_login::AuthDotJson;
use codex_login::save_auth;
use codex_login::token_data::IdTokenInfo;
use codex_login::token_data::TokenData;
use pretty_assertions::assert_eq;
use wiremock::Mock;
use wiremock::MockServer;
use wiremock::ResponseTemplate;
use wiremock::matchers::header;
use wiremock::matchers::method;
use wiremock::matchers::path;
#[tokio::test]
async fn ensure_registered_identity_skips_when_feature_is_disabled() {
let auth_manager =
AuthManager::from_auth_for_testing(make_chatgpt_auth("account-123", Some("user-123")));
let manager = AgentIdentityManager::new_for_tests(
auth_manager,
/*feature_enabled*/ false,
"https://chatgpt.com/backend-api/".to_string(),
SessionSource::Cli,
);
assert_eq!(manager.ensure_registered_identity().await.unwrap(), None);
}
#[tokio::test]
async fn ensure_registered_identity_skips_for_api_key_auth() {
let auth_manager = AuthManager::from_auth_for_testing(CodexAuth::from_api_key("test-key"));
let manager = AgentIdentityManager::new_for_tests(
auth_manager,
/*feature_enabled*/ true,
"https://chatgpt.com/backend-api/".to_string(),
SessionSource::Cli,
);
assert_eq!(manager.ensure_registered_identity().await.unwrap(), None);
}
#[tokio::test]
async fn ensure_registered_identity_registers_and_reuses_cached_identity() {
let server = MockServer::start().await;
let chatgpt_base_url = server.uri();
mount_human_biscuit(&server, &chatgpt_base_url).await;
Mock::given(method("POST"))
.and(path("/v1/agent/register"))
.and(header("x-openai-authorization", "human-biscuit"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
"agent_runtime_id": "agent_123",
})))
.expect(1)
.mount(&server)
.await;
let auth_manager =
AuthManager::from_auth_for_testing(make_chatgpt_auth("account-123", Some("user-123")));
let manager = AgentIdentityManager::new_for_tests(
auth_manager,
/*feature_enabled*/ true,
chatgpt_base_url,
SessionSource::Cli,
);
let first = manager
.ensure_registered_identity()
.await
.unwrap()
.expect("identity should be registered");
let second = manager
.ensure_registered_identity()
.await
.unwrap()
.expect("identity should be reused");
assert_eq!(first.agent_runtime_id, "agent_123");
assert_eq!(first, second);
assert_eq!(first.abom.agent_harness_id, "codex-cli");
assert_eq!(first.chatgpt_account_id, "account-123");
assert_eq!(first.chatgpt_user_id.as_deref(), Some("user-123"));
}
#[tokio::test]
async fn ensure_registered_identity_deletes_invalid_cached_identity_and_reregisters() {
let server = MockServer::start().await;
let chatgpt_base_url = server.uri();
mount_human_biscuit(&server, &chatgpt_base_url).await;
Mock::given(method("POST"))
.and(path("/v1/agent/register"))
.and(header("x-openai-authorization", "human-biscuit"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
"agent_runtime_id": "agent_456",
})))
.expect(1)
.mount(&server)
.await;
let auth = make_chatgpt_auth("account-123", Some("user-123"));
let auth_manager = AuthManager::from_auth_for_testing(auth.clone());
let manager = AgentIdentityManager::new_for_tests(
auth_manager,
/*feature_enabled*/ true,
chatgpt_base_url,
SessionSource::Cli,
);
let binding =
AgentIdentityBinding::from_auth(&auth, /*forced_workspace_id*/ None).expect("binding");
auth.set_agent_identity(AgentIdentityAuthRecord {
workspace_id: "account-123".to_string(),
chatgpt_user_id: Some("user-123".to_string()),
agent_runtime_id: "agent_invalid".to_string(),
agent_private_key: "not-valid-base64".to_string(),
registered_at: "2026-01-01T00:00:00Z".to_string(),
})
.expect("seed invalid identity");
let stored = manager
.ensure_registered_identity()
.await
.unwrap()
.expect("identity should be registered");
assert_eq!(stored.agent_runtime_id, "agent_456");
let persisted = auth
.get_agent_identity(&binding.chatgpt_account_id)
.expect("stored identity");
assert_eq!(persisted.agent_runtime_id, "agent_456");
}
#[tokio::test]
async fn ensure_registered_identity_deletes_different_user_identity_and_reregisters() {
let server = MockServer::start().await;
let chatgpt_base_url = server.uri();
mount_human_biscuit(&server, &chatgpt_base_url).await;
Mock::given(method("POST"))
.and(path("/v1/agent/register"))
.and(header("x-openai-authorization", "human-biscuit"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
"agent_runtime_id": "agent_new",
})))
.expect(1)
.mount(&server)
.await;
let auth = make_chatgpt_auth("account-123", Some("user-new"));
let stale_key = generate_agent_key_material().expect("key material");
auth.set_agent_identity(AgentIdentityAuthRecord {
workspace_id: "account-123".to_string(),
chatgpt_user_id: Some("user-old".to_string()),
agent_runtime_id: "agent_old".to_string(),
agent_private_key: stale_key.private_key_pkcs8_base64,
registered_at: "2026-01-01T00:00:00Z".to_string(),
})
.expect("seed stale identity");
let auth_manager = AuthManager::from_auth_for_testing(auth.clone());
let manager = AgentIdentityManager::new_for_tests(
auth_manager,
/*feature_enabled*/ true,
chatgpt_base_url,
SessionSource::Cli,
);
let stored = manager
.ensure_registered_identity()
.await
.unwrap()
.expect("identity should be registered");
assert_eq!(stored.agent_runtime_id, "agent_new");
assert_eq!(stored.chatgpt_user_id.as_deref(), Some("user-new"));
let persisted = auth
.get_agent_identity("account-123")
.expect("stored identity");
assert_eq!(persisted.agent_runtime_id, "agent_new");
assert_eq!(persisted.chatgpt_user_id.as_deref(), Some("user-new"));
}
#[tokio::test]
async fn ensure_registered_identity_uses_chatgpt_base_url() {
let server = MockServer::start().await;
let chatgpt_base_url = format!("{}/backend-api", server.uri());
mount_human_biscuit(&server, &chatgpt_base_url).await;
Mock::given(method("POST"))
.and(path("/backend-api/v1/agent/register"))
.and(header("x-openai-authorization", "human-biscuit"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
"agent_runtime_id": "agent_canonical",
})))
.expect(1)
.mount(&server)
.await;
let auth_manager =
AuthManager::from_auth_for_testing(make_chatgpt_auth("account-123", Some("user-123")));
let manager = AgentIdentityManager::new_for_tests(
auth_manager,
/*feature_enabled*/ true,
chatgpt_base_url,
SessionSource::Cli,
);
let stored = manager
.ensure_registered_identity()
.await
.unwrap()
.expect("identity should be registered");
assert_eq!(stored.agent_runtime_id, "agent_canonical");
}
async fn mount_human_biscuit(server: &MockServer, chatgpt_base_url: &str) {
let biscuit_url = agent_identity_biscuit_url(chatgpt_base_url);
let biscuit_path = reqwest::Url::parse(&biscuit_url)
.expect("biscuit URL parses")
.path()
.to_string();
let target_url = agent_registration_url(chatgpt_base_url);
Mock::given(method("GET"))
.and(path(biscuit_path))
.and(header("authorization", "Bearer access-token-account-123"))
.and(header("x-original-method", "POST"))
.and(header("x-original-url", target_url))
.respond_with(
ResponseTemplate::new(200).insert_header("x-openai-authorization", "human-biscuit"),
)
.expect(1)
.mount(server)
.await;
}
#[test]
fn encode_ssh_ed25519_public_key_matches_expected_wire_shape() {
let key_material = generate_agent_key_material().expect("key material");
let (_, encoded_blob) = key_material
.public_key_ssh
.split_once(' ')
.expect("public key contains scheme");
let decoded = BASE64_STANDARD.decode(encoded_blob).expect("base64");
assert_eq!(&decoded[..4], 11u32.to_be_bytes().as_slice());
assert_eq!(&decoded[4..15], b"ssh-ed25519");
assert_eq!(&decoded[15..19], 32u32.to_be_bytes().as_slice());
assert_eq!(decoded.len(), 51);
}
fn make_chatgpt_auth(account_id: &str, user_id: Option<&str>) -> CodexAuth {
let tempdir = tempfile::tempdir().expect("tempdir");
let auth_json = AuthDotJson {
auth_mode: Some(ApiAuthMode::Chatgpt),
openai_api_key: None,
tokens: Some(TokenData {
id_token: IdTokenInfo {
email: None,
chatgpt_plan_type: None,
chatgpt_user_id: user_id.map(ToOwned::to_owned),
chatgpt_account_id: Some(account_id.to_string()),
raw_jwt: fake_id_token(account_id, user_id),
},
access_token: format!("access-token-{account_id}"),
refresh_token: "refresh-token".to_string(),
account_id: Some(account_id.to_string()),
}),
last_refresh: Some(Utc::now()),
agent_identity: None,
};
save_auth(tempdir.path(), &auth_json, AuthCredentialsStoreMode::File).expect("save auth");
CodexAuth::from_auth_storage(tempdir.path(), AuthCredentialsStoreMode::File)
.expect("load auth")
.expect("auth")
}
fn fake_id_token(account_id: &str, user_id: Option<&str>) -> String {
let header = URL_SAFE_NO_PAD.encode(r#"{"alg":"none","typ":"JWT"}"#);
let payload = serde_json::json!({
"https://api.openai.com/auth": {
"chatgpt_user_id": user_id,
"chatgpt_account_id": account_id,
}
});
let payload = URL_SAFE_NO_PAD.encode(payload.to_string());
format!("{header}.{payload}.signature")
}
}
codex-rs/core/src/client.rs
+2 -11
View File
@@ -32,6 +32,7 @@ use std::sync::atomic::AtomicU64;
use std::sync::atomic::Ordering;
use codex_api::ApiError;
use codex_api::AuthProvider;
use codex_api::CompactClient as ApiCompactClient;
use codex_api::CompactionInput as ApiCompactionInput;
use codex_api::Compression;
@@ -83,7 +84,6 @@ use futures::StreamExt;
use http::HeaderMap as ApiHeaderMap;
use http::HeaderValue;
use http::StatusCode as HttpStatusCode;
use http::header::AUTHORIZATION;
use reqwest::StatusCode;
use std::time::Duration;
use std::time::Instant;
@@ -277,16 +277,7 @@ pub(crate) struct RealtimeWebrtcCallStart {
/// `api.openai.com` sideband path.
fn sideband_websocket_auth_headers(api_auth: &CoreAuthProvider) -> ApiHeaderMap {
let mut headers = ApiHeaderMap::new();
if let Some(token) = api_auth.token.as_ref()
&& let Ok(value) = HeaderValue::from_str(&format!("Bearer {token}"))
{
headers.insert(AUTHORIZATION, value);
}
if let Some(account_id) = api_auth.account_id.as_ref()
&& let Ok(value) = HeaderValue::from_str(account_id)
{
headers.insert("ChatGPT-Account-ID", value);
}
api_auth.add_auth_headers(&mut headers);
headers
}
codex-rs/core/src/codex.rs
+70 -47
View File
@@ -13,6 +13,7 @@ use crate::agent::Mailbox;
use crate::agent::MailboxReceiver;
use crate::agent::agent_status_from_event;
use crate::agent::status::is_final;
use crate::agent_identity::AgentIdentityManager;
use crate::apps::render_apps_section;
use crate::commit_attribution::commit_message_trailer_instruction;
use crate::compact;
@@ -77,7 +78,6 @@ use codex_login::CodexAuth;
use codex_login::auth_env_telemetry::collect_auth_env_telemetry;
use codex_login::default_client::originator;
use codex_mcp::McpConnectionManager;
use codex_mcp::SandboxState;
use codex_mcp::ToolInfo;
use codex_mcp::codex_apps_tools_cache_key;
#[cfg(test)]
@@ -91,6 +91,7 @@ use codex_otel::current_span_trace_id;
use codex_otel::current_span_w3c_trace_context;
use codex_otel::set_parent_from_w3c_trace_context;
use codex_protocol::ThreadId;
use codex_protocol::ToolName;
use codex_protocol::approvals::ElicitationRequestEvent;
use codex_protocol::approvals::ExecPolicyAmendment;
use codex_protocol::approvals::NetworkPolicyAmendment;
@@ -134,9 +135,11 @@ use codex_protocol::request_permissions::RequestPermissionsResponse;
use codex_protocol::request_user_input::RequestUserInputArgs;
use codex_protocol::request_user_input::RequestUserInputResponse;
use codex_rmcp_client::ElicitationResponse;
use codex_rollout::RolloutConfig;
use codex_rollout::state_db;
use codex_shell_command::parse_command::parse_command;
use codex_terminal_detection::user_agent;
use codex_thread_store::LocalThreadStore;
use codex_tools::filter_tool_suggest_discoverable_tools_for_client;
use codex_utils_output_truncation::TruncationPolicy;
use codex_utils_stream_parser::AssistantTextChunk;
@@ -1509,6 +1512,56 @@ impl Session {
});
}
fn start_agent_identity_registration(self: &Arc<Self>) {
if !self.services.agent_identity_manager.is_enabled() {
return;
}
let weak_sess = Arc::downgrade(self);
let mut auth_state_rx = self.services.auth_manager.subscribe_auth_state();
tokio::spawn(async move {
loop {
let Some(sess) = weak_sess.upgrade() else {
return;
};
match sess
.services
.agent_identity_manager
.ensure_registered_identity()
.await
{
Ok(Some(_)) => return,
Ok(None) => {
drop(sess);
if auth_state_rx.changed().await.is_err() {
return;
}
}
Err(error) => {
sess.fail_agent_identity_registration(error).await;
return;
}
}
}
});
}
async fn fail_agent_identity_registration(self: &Arc<Self>, error: anyhow::Error) {
warn!(error = %error, "agent identity registration failed");
let message = format!(
"Agent identity registration failed. Codex cannot continue while `features.use_agent_identity` is enabled: {error}"
);
self.send_event_raw(Event {
id: self.next_internal_sub_id(),
msg: EventMsg::Error(ErrorEvent {
message,
codex_error_info: Some(CodexErrorInfo::Other),
}),
})
.await;
handlers::shutdown(self, self.next_internal_sub_id()).await;
}
#[allow(clippy::too_many_arguments)]
fn make_turn_context(
conversation_id: ThreadId,
@@ -2054,6 +2107,11 @@ impl Session {
hooks,
rollout: Mutex::new(rollout_recorder),
user_shell: Arc::new(default_shell),
agent_identity_manager: Arc::new(AgentIdentityManager::new(
config.as_ref(),
Arc::clone(&auth_manager),
session_configuration.session_source.clone(),
)),
shell_snapshot_tx,
show_raw_agent_reasoning: config.show_raw_agent_reasoning,
exec_policy,
@@ -2070,6 +2128,7 @@ impl Session {
network_proxy,
network_approval: Arc::clone(&network_approval),
state_db: state_db_ctx.clone(),
thread_store: LocalThreadStore::new(RolloutConfig::from_view(config.as_ref())),
model_client: ModelClient::new(
Some(Arc::clone(&auth_manager)),
conversation_id,
@@ -2151,14 +2210,7 @@ impl Session {
// Start the watcher after SessionConfigured so it cannot emit earlier events.
sess.start_skills_watcher_listener();
// Construct sandbox_state before MCP startup so it can be sent to each
// MCP server immediately after it becomes ready (avoiding blocking).
let sandbox_state = SandboxState {
sandbox_policy: session_configuration.sandbox_policy.get().clone(),
codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(),
sandbox_cwd: session_configuration.cwd.to_path_buf(),
use_legacy_landlock: config.features.use_legacy_landlock(),
};
sess.start_agent_identity_registration();
let mut required_mcp_servers: Vec<String> = mcp_servers
.iter()
.filter(|(_, server)| server.enabled && server.required)
@@ -2180,7 +2232,7 @@ impl Session {
&session_configuration.approval_policy,
INITIAL_SUBMIT_ID.to_owned(),
tx_event.clone(),
sandbox_state,
session_configuration.sandbox_policy.get().clone(),
config.codex_home.to_path_buf(),
codex_apps_tools_cache_key(auth),
tool_plugin_provenance,
@@ -2612,12 +2664,16 @@ impl Session {
&session_source,
);
if sandbox_policy_changed {
self.refresh_managed_network_proxy_for_current_sandbox_policy()
.await;
}
Ok(self
.new_turn_from_configuration(
sub_id,
session_configuration,
updates.final_output_json_schema,
sandbox_policy_changed,
)
.await)
}
@@ -2627,7 +2683,6 @@ impl Session {
sub_id: String,
session_configuration: SessionConfiguration,
final_output_json_schema: Option<Option<Value>>,
sandbox_policy_changed: bool,
) -> Arc<TurnContext> {
let per_turn_config = Self::build_per_turn_config(&session_configuration);
{
@@ -2637,27 +2692,6 @@ impl Session {
.set_sandbox_policy(per_turn_config.permissions.sandbox_policy.get());
}
if sandbox_policy_changed {
self.refresh_managed_network_proxy_for_current_sandbox_policy()
.await;
let sandbox_state = SandboxState {
sandbox_policy: per_turn_config.permissions.sandbox_policy.get().clone(),
codex_linux_sandbox_exe: per_turn_config.codex_linux_sandbox_exe.clone(),
sandbox_cwd: per_turn_config.cwd.to_path_buf(),
use_legacy_landlock: per_turn_config.features.use_legacy_landlock(),
};
if let Err(e) = self
.services
.mcp_connection_manager
.read()
.await
.notify_sandbox_state_change(&sandbox_state)
.await
{
warn!("Failed to notify sandbox state change to MCP servers: {e:#}");
}
}
let model_info = self
.services
.models_manager
@@ -2806,7 +2840,6 @@ impl Session {
sub_id,
session_configuration,
/*final_output_json_schema*/ None,
/*sandbox_policy_changed*/ false,
)
.await
}
@@ -4453,16 +4486,12 @@ impl Session {
.await
}
pub(crate) async fn resolve_mcp_tool_info(
&self,
name: &str,
namespace: Option<&str>,
) -> Option<ToolInfo> {
pub(crate) async fn resolve_mcp_tool_info(&self, tool_name: &ToolName) -> Option<ToolInfo> {
self.services
.mcp_connection_manager
.read()
.await
.resolve_tool_info(name, namespace)
.resolve_tool_info(tool_name)
.await
}
@@ -4522,12 +4551,6 @@ impl Session {
.await;
let mcp_servers = with_codex_apps_mcp(mcp_servers, auth.as_ref(), &mcp_config);
let auth_statuses = compute_auth_statuses(mcp_servers.iter(), store_mode).await;
let sandbox_state = SandboxState {
sandbox_policy: turn_context.sandbox_policy.get().clone(),
codex_linux_sandbox_exe: turn_context.codex_linux_sandbox_exe.clone(),
sandbox_cwd: turn_context.cwd.to_path_buf(),
use_legacy_landlock: turn_context.features.use_legacy_landlock(),
};
{
let mut guard = self.services.mcp_startup_cancellation_token.lock().await;
guard.cancel();
@@ -4540,7 +4563,7 @@ impl Session {
&turn_context.config.permissions.approval_policy,
turn_context.sub_id.clone(),
self.get_tx_event(),
sandbox_state,
turn_context.sandbox_policy.get().clone(),
config.codex_home.to_path_buf(),
codex_apps_tools_cache_key(auth.as_ref()),
tool_plugin_provenance,
codex-rs/core/src/codex_tests.rs
+145 -3
View File
@@ -110,6 +110,7 @@ use opentelemetry::trace::TraceId;
use std::path::Path;
use std::time::Duration;
use tokio::time::sleep;
use tokio::time::timeout;
use tracing_opentelemetry::OpenTelemetrySpanExt;
use codex_protocol::mcp::CallToolResult as McpCallToolResult;
@@ -655,6 +656,89 @@ async fn managed_network_proxy_decider_survives_full_access_start() -> anyhow::R
Ok(())
}
#[tokio::test]
async fn new_turn_refreshes_managed_network_proxy_for_sandbox_change() -> anyhow::Result<()> {
let (mut session, _turn_context) = make_session_and_context().await;
let initial_policy = SandboxPolicy::new_workspace_write_policy();
let mut network_config = NetworkProxyConfig::default();
network_config
.network
.set_allowed_domains(vec!["evil.com".to_string()]);
let requirements = NetworkConstraints {
domains: Some(NetworkDomainPermissionsToml {
entries: std::collections::BTreeMap::from([(
"*.example.com".to_string(),
NetworkDomainPermissionToml::Allow,
)]),
}),
..Default::default()
};
let spec = crate::config::NetworkProxySpec::from_config_and_constraints(
network_config,
Some(requirements),
&initial_policy,
)?;
let (started_proxy, _) = Session::start_managed_network_proxy(
&spec,
&Policy::empty(),
&initial_policy,
/*network_policy_decider*/ None,
/*blocked_request_observer*/ None,
/*managed_network_requirements_enabled*/ false,
crate::config::NetworkProxyAuditMetadata::default(),
)
.await?;
assert_eq!(
started_proxy
.proxy()
.current_cfg()
.await?
.network
.allowed_domains(),
Some(vec!["*.example.com".to_string(), "evil.com".to_string()])
);
{
let mut state = session.state.lock().await;
let mut config = (*state.session_configuration.original_config_do_not_use).clone();
config.permissions.network = Some(spec);
config.permissions.sandbox_policy =
codex_config::Constrained::allow_any(initial_policy.clone());
state.session_configuration.original_config_do_not_use = Arc::new(config);
state.session_configuration.sandbox_policy =
codex_config::Constrained::allow_any(initial_policy);
}
session.services.network_proxy = Some(started_proxy);
session
.new_turn_with_sub_id(
"sandbox-policy-change".to_string(),
SessionSettingsUpdate {
sandbox_policy: Some(SandboxPolicy::DangerFullAccess),
..Default::default()
},
)
.await?;
let started_proxy = session
.services
.network_proxy
.as_ref()
.expect("managed network proxy should be present");
assert_eq!(
started_proxy
.proxy()
.current_cfg()
.await?
.network
.allowed_domains(),
Some(vec!["*.example.com".to_string()])
);
Ok(())
}
#[tokio::test]
async fn get_base_instructions_no_user_content() {
let prompt_with_apply_patch_instructions =
@@ -904,7 +988,7 @@ fn mcp_tool_exposure_searches_large_effective_tool_sets() {
}
#[test]
fn mcp_tool_exposure_directly_exposes_explicit_apps_in_large_search_sets() {
fn mcp_tool_exposure_directly_exposes_explicit_apps_without_deferred_overlap() {
let config = test_config();
let tools_config = tools_config_for_mcp_tool_exposure(/*search_tool*/ true);
let mut mcp_tools = numbered_mcp_tools(DIRECT_MCP_TOOL_EXPOSURE_THRESHOLD - 1);
@@ -935,13 +1019,19 @@ fn mcp_tool_exposure_directly_exposes_explicit_apps_in_large_search_sets() {
);
assert_eq!(
exposure.deferred_tools.as_ref().map(HashMap::len),
Some(DIRECT_MCP_TOOL_EXPOSURE_THRESHOLD)
Some(DIRECT_MCP_TOOL_EXPOSURE_THRESHOLD - 1)
);
let deferred_tools = exposure
.deferred_tools
.as_ref()
.expect("large tool sets should be discoverable through tool_search");
assert!(deferred_tools.contains_key("mcp__codex_apps__calendar_create_event"));
assert!(
tool_names
.iter()
.all(|direct_tool_name| !deferred_tools.contains_key(direct_tool_name)),
"direct tools should not also be deferred: {tool_names:?}"
);
assert!(!deferred_tools.contains_key("mcp__codex_apps__calendar_create_event"));
assert!(deferred_tools.contains_key("mcp__rmcp__tool_0"));
}
@@ -2808,6 +2898,11 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) {
}),
rollout: Mutex::new(None),
user_shell: Arc::new(default_user_shell()),
agent_identity_manager: Arc::new(crate::agent_identity::AgentIdentityManager::new(
config.as_ref(),
Arc::clone(&auth_manager),
session_configuration.session_source.clone(),
)),
shell_snapshot_tx: watch::channel(None).0,
show_raw_agent_reasoning: config.show_raw_agent_reasoning,
exec_policy,
@@ -2824,6 +2919,9 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) {
network_proxy: None,
network_approval: Arc::clone(&network_approval),
state_db: None,
thread_store: codex_thread_store::LocalThreadStore::new(
codex_rollout::RolloutConfig::from_view(config.as_ref()),
),
model_client: ModelClient::new(
Some(auth_manager.clone()),
conversation_id,
@@ -3660,6 +3758,11 @@ pub(crate) async fn make_session_and_context_with_dynamic_tools_and_rx(
}),
rollout: Mutex::new(None),
user_shell: Arc::new(default_user_shell()),
agent_identity_manager: Arc::new(crate::agent_identity::AgentIdentityManager::new(
config.as_ref(),
Arc::clone(&auth_manager),
session_configuration.session_source.clone(),
)),
shell_snapshot_tx: watch::channel(None).0,
show_raw_agent_reasoning: config.show_raw_agent_reasoning,
exec_policy,
@@ -3676,6 +3779,9 @@ pub(crate) async fn make_session_and_context_with_dynamic_tools_and_rx(
network_proxy: None,
network_approval: Arc::clone(&network_approval),
state_db: None,
thread_store: codex_thread_store::LocalThreadStore::new(
codex_rollout::RolloutConfig::from_view(config.as_ref()),
),
model_client: ModelClient::new(
Some(Arc::clone(&auth_manager)),
conversation_id,
@@ -3764,6 +3870,42 @@ pub(crate) async fn make_session_and_context_with_rx() -> (
make_session_and_context_with_dynamic_tools_and_rx(Vec::new()).await
}
#[tokio::test]
async fn fail_agent_identity_registration_emits_error_and_shutdown() {
let (session, _turn_context, rx_event) = make_session_and_context_with_rx().await;
session
.fail_agent_identity_registration(anyhow::anyhow!("registration exploded"))
.await;
let error_event = timeout(Duration::from_secs(1), rx_event.recv())
.await
.expect("error event should arrive")
.expect("error event should be readable");
match error_event.msg {
EventMsg::Error(ErrorEvent {
message,
codex_error_info,
}) => {
assert_eq!(
message,
"Agent identity registration failed. Codex cannot continue while `features.use_agent_identity` is enabled: registration exploded".to_string()
);
assert_eq!(codex_error_info, Some(CodexErrorInfo::Other));
}
other => panic!("expected error event, got {other:?}"),
}
let shutdown_event = timeout(Duration::from_secs(1), rx_event.recv())
.await
.expect("shutdown event should arrive")
.expect("shutdown event should be readable");
match shutdown_event.msg {
EventMsg::ShutdownComplete => {}
other => panic!("expected shutdown event, got {other:?}"),
}
}
#[tokio::test]
async fn refresh_mcp_servers_is_deferred_until_next_turn() {
let (session, turn_context) = make_session_and_context().await;
codex-rs/core/src/compact.rs
+1 -1
View File
@@ -58,7 +58,7 @@ pub(crate) enum InitialContextInjection {
}
pub(crate) fn should_use_remote_compact_task(provider: &ModelProviderInfo) -> bool {
provider.is_openai()
provider.supports_remote_compaction()
}
pub(crate) async fn run_inline_auto_compact_task(
codex-rs/core/src/compact_tests.rs
+26
View File
@@ -1,4 +1,6 @@
use super::*;
use codex_model_provider_info::ModelProviderInfo;
use codex_model_provider_info::WireApi;
use pretty_assertions::assert_eq;
async fn process_compacted_history_with_test_session(
@@ -185,6 +187,30 @@ fn build_token_limited_compacted_history_appends_summary_message() {
assert_eq!(summary, summary_text);
}
#[test]
fn should_use_remote_compact_task_for_azure_provider() {
let provider = ModelProviderInfo {
name: "Azure".into(),
base_url: Some("https://example.com/openai".into()),
env_key: Some("AZURE_OPENAI_API_KEY".into()),
env_key_instructions: None,
experimental_bearer_token: None,
auth: None,
wire_api: WireApi::Responses,
query_params: None,
http_headers: None,
env_http_headers: None,
request_max_retries: None,
stream_max_retries: None,
stream_idle_timeout_ms: None,
websocket_connect_timeout_ms: None,
requires_openai_auth: false,
supports_websockets: false,
};
assert!(should_use_remote_compact_task(&provider));
}
#[tokio::test]
async fn process_compacted_history_replaces_developer_messages() {
let compacted_history = vec![
codex-rs/core/src/connectors.rs
+1 -11
View File
@@ -1,8 +1,6 @@
use std::collections::BTreeSet;
use std::collections::HashMap;
use std::collections::HashSet;
use std::env;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::LazyLock;
use std::sync::Mutex as StdMutex;
@@ -42,7 +40,6 @@ use codex_login::default_client::is_first_party_chat_originator;
use codex_login::default_client::originator;
use codex_mcp::CODEX_APPS_MCP_SERVER_NAME;
use codex_mcp::McpConnectionManager;
use codex_mcp::SandboxState;
use codex_mcp::ToolInfo;
use codex_mcp::ToolPluginProvenance;
use codex_mcp::codex_apps_tools_cache_key;
@@ -228,13 +225,6 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_options_and_status(
let (tx_event, rx_event) = unbounded();
drop(rx_event);
let sandbox_state = SandboxState {
sandbox_policy: SandboxPolicy::new_read_only_policy(),
codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(),
sandbox_cwd: env::current_dir().unwrap_or_else(|_| PathBuf::from("/")),
use_legacy_landlock: config.features.use_legacy_landlock(),
};
let (mcp_connection_manager, cancel_token) = McpConnectionManager::new(
&mcp_servers,
config.mcp_oauth_credentials_store_mode,
@@ -242,7 +232,7 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_options_and_status(
&config.permissions.approval_policy,
INITIAL_SUBMIT_ID.to_owned(),
tx_event,
sandbox_state,
SandboxPolicy::new_read_only_policy(),
config.codex_home.to_path_buf(),
codex_apps_tools_cache_key(auth.as_ref()),
ToolPluginProvenance::default(),
codex-rs/core/src/exec.rs
+63 -36
View File
@@ -251,8 +251,24 @@ pub fn build_exec_request(
codex_linux_sandbox_exe: &Option<PathBuf>,
use_legacy_landlock: bool,
) -> Result<ExecRequest> {
let windows_sandbox_level = params.windows_sandbox_level;
let enforce_managed_network = params.network.is_some();
let ExecParams {
command,
cwd,
mut env,
expiration,
capture_policy,
network,
windows_sandbox_level,
windows_sandbox_private_desktop,
// TODO: Should arg0 be set on the ExecRequest that is returned?
arg0: _,
// These fields are related to approvals, so can be ignored here.
justification: _,
sandbox_permissions: _,
} = params;
let enforce_managed_network = network.is_some();
let sandbox_type = select_process_exec_tool_sandbox_type(
file_system_sandbox_policy,
network_sandbox_policy,
@@ -261,19 +277,6 @@ pub fn build_exec_request(
);
tracing::debug!("Sandbox type: {sandbox_type:?}");
let ExecParams {
command,
cwd,
mut env,
expiration,
capture_policy,
network,
sandbox_permissions: _,
windows_sandbox_level,
windows_sandbox_private_desktop,
justification: _,
arg0: _,
} = params;
if let Some(network) = network.as_ref() {
network.apply_to_env(&mut env);
}
@@ -357,7 +360,8 @@ pub(crate) async fn execute_exec_request(
windows_sandbox_level,
windows_sandbox_private_desktop,
sandbox_policy,
file_system_sandbox_policy,
// TODO(mbolin): Use file_system_sandbox_policy instead of sandbox_policy.
file_system_sandbox_policy: _,
network_sandbox_policy,
windows_sandbox_filesystem_overrides,
arg0,
@@ -378,21 +382,40 @@ pub(crate) async fn execute_exec_request(
};
let start = Instant::now();
let raw_output_result = exec(
let raw_output_result = get_raw_output_result(
params,
sandbox,
&sandbox_policy,
&file_system_sandbox_policy,
windows_sandbox_filesystem_overrides.as_ref(),
network_sandbox_policy,
stdout_stream,
after_spawn,
sandbox,
&sandbox_policy,
windows_sandbox_filesystem_overrides.as_ref(),
)
.await;
let duration = start.elapsed();
finalize_exec_result(raw_output_result, sandbox, duration)
}
async fn get_raw_output_result(
params: ExecParams,
network_sandbox_policy: NetworkSandboxPolicy,
stdout_stream: Option<StdoutStream>,
after_spawn: Option<Box<dyn FnOnce() + Send>>,
#[cfg_attr(not(windows), allow(unused_variables))] sandbox: SandboxType,
#[cfg_attr(not(windows), allow(unused_variables))] sandbox_policy: &SandboxPolicy,
#[cfg_attr(not(windows), allow(unused_variables))] windows_sandbox_filesystem_overrides: Option<
&WindowsSandboxFilesystemOverrides,
>,
) -> Result<RawExecToolCallOutput> {
#[cfg(target_os = "windows")]
if sandbox == SandboxType::WindowsRestrictedToken {
return exec_windows_sandbox(params, sandbox_policy, windows_sandbox_filesystem_overrides)
.await;
}
exec(params, network_sandbox_policy, stdout_stream, after_spawn).await
}
#[cfg(target_os = "windows")]
fn extract_create_process_as_user_error_code(err: &str) -> Option<String> {
let marker = "CreateProcessAsUserW failed: ";
@@ -799,26 +822,24 @@ fn aggregate_output(
}
}
#[allow(clippy::too_many_arguments)]
/// This is a general-purpose function for executing a command specified by
/// [ExecParams]. Events are reported via `stdout_stream`, if specified, and
/// `after_spawn` is invoked once the child process has been spawned, before
/// output consumption begins.
///
/// `network_sandbox_policy` is used to determine whether
/// CODEX_SANDBOX_NETWORK_DISABLED=1 is added to the environment of the spawned
/// process.
///
/// Note this command does not apply any sandboxing logic. The caller is
/// responsible for constructing [ExecParams::command] to include any sandboxing
/// wrapper args, as appropriate.
async fn exec(
params: ExecParams,
_sandbox: SandboxType,
_sandbox_policy: &SandboxPolicy,
_file_system_sandbox_policy: &FileSystemSandboxPolicy,
_windows_sandbox_filesystem_overrides: Option<&WindowsSandboxFilesystemOverrides>,
network_sandbox_policy: NetworkSandboxPolicy,
stdout_stream: Option<StdoutStream>,
after_spawn: Option<Box<dyn FnOnce() + Send>>,
) -> Result<RawExecToolCallOutput> {
#[cfg(target_os = "windows")]
if _sandbox == SandboxType::WindowsRestrictedToken {
return exec_windows_sandbox(
params,
_sandbox_policy,
_windows_sandbox_filesystem_overrides,
)
.await;
}
let ExecParams {
command,
cwd,
@@ -827,8 +848,14 @@ async fn exec(
arg0,
expiration,
capture_policy,
// If applicable, these fields should have been honored upstream of
// this exec call.
windows_sandbox_level: _,
..
windows_sandbox_private_desktop: _,
// These fields are related to approvals, so can be ignored here.
sandbox_permissions: _,
justification: _,
} = params;
if let Some(network) = network.as_ref() {
network.apply_to_env(&mut env);
codex-rs/core/src/exec_tests.rs
-12
View File
@@ -277,10 +277,6 @@ async fn exec_full_buffer_capture_ignores_expiration() -> Result<()> {
justification: None,
arg0: None,
},
SandboxType::None,
&SandboxPolicy::DangerFullAccess,
&FileSystemSandboxPolicy::unrestricted(),
/*windows_sandbox_filesystem_overrides*/ None,
NetworkSandboxPolicy::Enabled,
/*stdout_stream*/ None,
/*after_spawn*/ None,
@@ -317,10 +313,6 @@ async fn exec_full_buffer_capture_keeps_io_drain_timeout_when_descendant_holds_p
justification: None,
arg0: None,
},
SandboxType::None,
&SandboxPolicy::DangerFullAccess,
&FileSystemSandboxPolicy::unrestricted(),
/*windows_sandbox_filesystem_overrides*/ None,
NetworkSandboxPolicy::Enabled,
/*stdout_stream*/ None,
/*after_spawn*/ None,
@@ -931,10 +923,6 @@ async fn kill_child_process_group_kills_grandchildren_on_timeout() -> Result<()>
let output = exec(
params,
SandboxType::None,
&SandboxPolicy::new_read_only_policy(),
&FileSystemSandboxPolicy::from(&SandboxPolicy::new_read_only_policy()),
/*windows_sandbox_filesystem_overrides*/ None,
NetworkSandboxPolicy::Restricted,
/*stdout_stream*/ None,
/*after_spawn*/ None,
codex-rs/core/src/external_agent_config.rs
+10 -18
View File
@@ -3,8 +3,11 @@ use crate::plugins::PluginId;
use crate::plugins::PluginInstallRequest;
use crate::plugins::PluginsManager;
use crate::plugins::add_marketplace;
<<<<<<< HEAD
use crate::plugins::validate_marketplace_add_source;
use codex_utils_absolute_path::AbsolutePathBuf;
=======
>>>>>>> main
use serde_json::Value as JsonValue;
use std::collections::BTreeMap;
use std::collections::HashSet;
@@ -177,7 +180,7 @@ impl ExternalAgentConfigService {
|repo_root| repo_root.join(".codex").join("config.toml"),
);
if let Some(settings) = settings.as_ref() {
let migrated = build_config_from_external(&settings)?;
let migrated = build_config_from_external(settings)?;
if !is_empty_toml_table(&migrated) {
let mut should_include = true;
if target_config.exists() {
@@ -348,23 +351,12 @@ impl ExternalAgentConfigService {
let add_marketplace_outcome = add_marketplace(self.codex_home.clone(), request).await;
let marketplace_path = match add_marketplace_outcome {
Ok(add_marketplace_outcome) => {
match AbsolutePathBuf::try_from(
add_marketplace_outcome
.installed_root
.join(INSTALLED_MARKETPLACE_MANIFEST_RELATIVE_PATH),
) {
Ok(path) => {
outcome
.succeeded_marketplaces
.push(marketplace_name.clone());
path
}
Err(_) => {
outcome.failed_marketplaces.push(marketplace_name);
outcome.failed_plugin_ids.extend(plugin_ids);
continue;
}
}
outcome
.succeeded_marketplaces
.push(marketplace_name.clone());
add_marketplace_outcome
.installed_root
.join(INSTALLED_MARKETPLACE_MANIFEST_RELATIVE_PATH)
}
Err(_) => {
outcome.failed_marketplaces.push(marketplace_name);
codex-rs/core/src/guardian/review_session.rs
+20
View File
@@ -748,6 +748,7 @@ pub(crate) fn build_guardian_review_session_config(
for feature in [
Feature::SpawnCsv,
Feature::Collab,
Feature::CodexHooks,
Feature::WebSearchRequest,
Feature::WebSearchCached,
] {
@@ -854,6 +855,25 @@ mod tests {
);
}
#[test]
fn guardian_review_session_config_disables_hooks() {
let mut parent_config = crate::config::test_config();
parent_config
.features
.enable(Feature::CodexHooks)
.expect("enable hooks on parent config");
let guardian_config = build_guardian_review_session_config(
&parent_config,
/*live_network_config*/ None,
"active-model",
/*reasoning_effort*/ None,
)
.expect("guardian config");
assert!(!guardian_config.features.enabled(Feature::CodexHooks));
}
#[tokio::test(flavor = "current_thread")]
async fn run_before_review_deadline_times_out_before_future_completes() {
let outcome = run_before_review_deadline(
codex-rs/core/src/lib.rs
+1 -3
View File
@@ -5,6 +5,7 @@
// the TUI or the tracing stack).
#![deny(clippy::print_stdout, clippy::print_stderr)]
mod agent_identity;
mod apply_patch;
mod apps;
mod arc_monitor;
@@ -54,8 +55,6 @@ pub use network_proxy_loader::MtimeConfigReloader;
pub use network_proxy_loader::build_network_proxy_state;
pub use network_proxy_loader::build_network_proxy_state_and_reloader;
mod original_image_detail;
pub use codex_mcp::MCP_SANDBOX_STATE_CAPABILITY;
pub use codex_mcp::MCP_SANDBOX_STATE_METHOD;
pub use codex_mcp::SandboxState;
mod mcp_openai_file;
mod mcp_tool_call;
@@ -139,7 +138,6 @@ pub use project_doc::discover_project_doc_paths;
pub use project_doc::read_project_docs;
mod rollout;
pub(crate) mod safety;
pub mod seatbelt;
mod session_rollout_init_error;
pub mod shell;
pub(crate) mod shell_snapshot;
codex-rs/core/src/mcp_tool_call.rs
+51
View File
@@ -36,6 +36,7 @@ use codex_analytics::build_track_events_context;
use codex_config::types::AppToolApproval;
use codex_features::Feature;
use codex_mcp::CODEX_APPS_MCP_SERVER_NAME;
use codex_mcp::SandboxState;
use codex_mcp::declared_openai_file_input_param_names;
use codex_mcp::mcp_permission_prompt_is_auto_approved;
use codex_otel::sanitize_metric_tag_value;
@@ -466,6 +467,10 @@ async fn execute_mcp_tool_call(
metadata.and_then(|metadata| metadata.openai_file_input_params.as_deref()),
)
.await?;
let request_meta =
augment_mcp_tool_request_meta_with_sandbox_state(sess, turn_context, server, request_meta)
.await
.map_err(|e| format!("failed to build MCP tool request metadata: {e:#}"))?;
let result = sess
.call_tool(server, tool_name, rewritten_arguments, request_meta)
.await
@@ -479,6 +484,52 @@ async fn execute_mcp_tool_call(
)
}
async fn augment_mcp_tool_request_meta_with_sandbox_state(
sess: &Session,
turn_context: &TurnContext,
server: &str,
mut meta: Option<serde_json::Value>,
) -> anyhow::Result<Option<serde_json::Value>> {
let supports_sandbox_state_meta = sess
.services
.mcp_connection_manager
.read()
.await
.server_supports_sandbox_state_meta_capability(server)
.await
.unwrap_or(false);
if !supports_sandbox_state_meta {
return Ok(meta);
}
let sandbox_state = serde_json::to_value(SandboxState {
sandbox_policy: turn_context.sandbox_policy.get().clone(),
codex_linux_sandbox_exe: turn_context.codex_linux_sandbox_exe.clone(),
sandbox_cwd: turn_context.cwd.to_path_buf(),
use_legacy_landlock: turn_context.features.use_legacy_landlock(),
})?;
match meta.as_mut() {
Some(serde_json::Value::Object(map)) => {
map.insert(
codex_mcp::MCP_SANDBOX_STATE_META_CAPABILITY.to_string(),
sandbox_state,
);
}
Some(_) => {}
None => {
let mut map = serde_json::Map::new();
map.insert(
codex_mcp::MCP_SANDBOX_STATE_META_CAPABILITY.to_string(),
sandbox_state,
);
meta = Some(serde_json::Value::Object(map));
}
}
Ok(meta)
}
async fn maybe_mark_thread_memory_mode_polluted(sess: &Session, turn_context: &TurnContext) {
if !turn_context
.config
codex-rs/core/src/mcp_tool_exposure.rs
+5 -1
View File
@@ -41,9 +41,13 @@ pub(crate) fn build_mcp_tool_exposure(
let direct_tools =
filter_codex_apps_mcp_tools(all_mcp_tools, explicitly_enabled_connectors, config);
for direct_tool_name in direct_tools.keys() {
deferred_tools.remove(direct_tool_name);
}
McpToolExposure {
direct_tools,
deferred_tools: Some(deferred_tools),
deferred_tools: (!deferred_tools.is_empty()).then_some(deferred_tools),
}
}
codex-rs/core/src/memories/README.md
+8 -1
View File
@@ -84,7 +84,11 @@ What it does:
- `raw_memories.md` (merged raw memories, latest first)
- `rollout_summaries/` (one summary file per retained rollout)
- prunes stale rollout summaries that are no longer retained
- if there are no inputs, marks the job successful and exits
- finds old resource files from memory extensions under
`memories_extensions/<extension>/resources/` for extension directories that
have an `instructions.md`, using the memory module retention window
- if there are no Phase 1 inputs or old extension resources, marks the job
successful and exits
If there is input, it then:
@@ -92,10 +96,13 @@ If there is input, it then:
- builds the Phase 2 prompt with a diff of the current Phase 1 input
selection versus the last successful Phase 2 selection (`added`,
`retained`, `removed`)
- includes old extension resource paths in the prompt diff
- runs it with no approvals, no network, and local write access only
- disables collab for that agent (to prevent recursive delegation)
- watches the agent status and heartbeats the global job lease while it runs
- marks the phase-2 job success/failure in the state DB when the agent finishes
- prunes old extension resource files after the consolidation agent completes
and the successful Phase 2 job is recorded
Selection diff behavior:
codex-rs/core/src/memories/extensions.rs
+251
View File
@@ -0,0 +1,251 @@
use crate::memories::memory_extensions_root;
use chrono::DateTime;
use chrono::Duration;
use chrono::NaiveDateTime;
use chrono::Utc;
use std::path::Path;
use std::path::PathBuf;
use tracing::warn;
const FILENAME_TS_FORMAT: &str = "%Y-%m-%dT%H-%M-%S";
pub(super) const EXTENSION_RESOURCE_RETENTION_DAYS: i64 = 7;
#[derive(Debug, Clone, PartialEq, Eq)]
pub(super) struct RemovedExtensionResource {
pub(super) extension: String,
pub(super) resource_path: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(super) struct PendingExtensionResourceRemoval {
pub(super) removed: RemovedExtensionResource,
path: PathBuf,
}
pub(super) async fn find_old_extension_resources(
memory_root: &Path,
) -> Vec<PendingExtensionResourceRemoval> {
find_old_extension_resources_with_now(memory_root, Utc::now()).await
}
async fn find_old_extension_resources_with_now(
memory_root: &Path,
now: DateTime<Utc>,
) -> Vec<PendingExtensionResourceRemoval> {
let mut pending = Vec::new();
let cutoff = now - Duration::days(EXTENSION_RESOURCE_RETENTION_DAYS);
let extensions_root = memory_extensions_root(memory_root);
let mut extensions = match tokio::fs::read_dir(&extensions_root).await {
Ok(extensions) => extensions,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => return pending,
Err(err) => {
warn!(
"failed reading memory extensions root {}: {err}",
extensions_root.display()
);
return pending;
}
};
while let Ok(Some(extension_entry)) = extensions.next_entry().await {
let extension_path = extension_entry.path();
let Ok(file_type) = extension_entry.file_type().await else {
continue;
};
if !file_type.is_dir() {
continue;
}
let Some(extension) = extension_path
.file_name()
.and_then(|name| name.to_str())
.map(ToOwned::to_owned)
else {
continue;
};
if !tokio::fs::try_exists(extension_path.join("instructions.md"))
.await
.unwrap_or(false)
{
continue;
}
let resources_path = extension_path.join("resources");
let mut resources = match tokio::fs::read_dir(&resources_path).await {
Ok(resources) => resources,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
Err(err) => {
warn!(
"failed reading memory extension resources {}: {err}",
resources_path.display()
);
continue;
}
};
while let Ok(Some(resource_entry)) = resources.next_entry().await {
let resource_file_path = resource_entry.path();
let Ok(file_type) = resource_entry.file_type().await else {
continue;
};
if !file_type.is_file() {
continue;
}
let Some(file_name) = resource_file_path
.file_name()
.and_then(|name| name.to_str())
else {
continue;
};
if !file_name.ends_with(".md") {
continue;
}
let Some(resource_timestamp) = resource_timestamp(file_name) else {
continue;
};
if resource_timestamp > cutoff {
continue;
}
pending.push(PendingExtensionResourceRemoval {
removed: RemovedExtensionResource {
extension: extension.clone(),
resource_path: format!("resources/{file_name}"),
},
path: resource_file_path,
});
}
}
pending.sort_by(|left, right| {
left.removed
.extension
.cmp(&right.removed.extension)
.then_with(|| left.removed.resource_path.cmp(&right.removed.resource_path))
});
pending
}
pub(super) async fn remove_extension_resources(resources: &[PendingExtensionResourceRemoval]) {
for resource in resources {
if let Err(err) = tokio::fs::remove_file(&resource.path).await
&& err.kind() != std::io::ErrorKind::NotFound
{
warn!(
"failed pruning old memory extension resource {}: {err}",
resource.path.display()
);
}
}
}
fn resource_timestamp(file_name: &str) -> Option<DateTime<Utc>> {
let timestamp = file_name.get(..19)?;
let naive = NaiveDateTime::parse_from_str(timestamp, FILENAME_TS_FORMAT).ok()?;
Some(DateTime::from_naive_utc_and_offset(naive, Utc))
}
#[cfg(test)]
mod tests {
use super::*;
use pretty_assertions::assert_eq;
use tempfile::TempDir;
#[tokio::test]
async fn finds_only_old_resources_from_extensions_with_instructions() {
let codex_home = TempDir::new().expect("create temp codex home");
let memory_root = codex_home.path().join("memories");
let extensions_root = memory_extensions_root(&memory_root);
let telepathy_resources = extensions_root.join("telepathy/resources");
tokio::fs::create_dir_all(&telepathy_resources)
.await
.expect("create telepathy resources");
tokio::fs::write(
extensions_root.join("telepathy/instructions.md"),
"instructions",
)
.await
.expect("write telepathy instructions");
let now = DateTime::from_naive_utc_and_offset(
NaiveDateTime::parse_from_str("2026-04-14T12-00-00", FILENAME_TS_FORMAT)
.expect("parse now"),
Utc,
);
let old_file = telepathy_resources.join("2026-04-06T11-59-59-abcd-10min-old.md");
let exact_cutoff_file =
telepathy_resources.join("2026-04-07T12-00-00-abcd-10min-cutoff.md");
let recent_file = telepathy_resources.join("2026-04-08T12-00-00-abcd-10min-recent.md");
let invalid_file = telepathy_resources.join("not-a-timestamp.md");
for file in [&old_file, &exact_cutoff_file, &recent_file, &invalid_file] {
tokio::fs::write(file, "resource")
.await
.expect("write telepathy resource");
}
let ignored_resources = extensions_root.join("ignored/resources");
tokio::fs::create_dir_all(&ignored_resources)
.await
.expect("create ignored resources");
let ignored_old_file = ignored_resources.join("2026-04-06T11-59-59-abcd-10min-old.md");
tokio::fs::write(&ignored_old_file, "ignored")
.await
.expect("write ignored resource");
let pending = find_old_extension_resources_with_now(&memory_root, now).await;
assert_eq!(
pending
.iter()
.map(|resource| resource.removed.clone())
.collect::<Vec<_>>(),
vec![
RemovedExtensionResource {
extension: "telepathy".to_string(),
resource_path: "resources/2026-04-06T11-59-59-abcd-10min-old.md".to_string(),
},
RemovedExtensionResource {
extension: "telepathy".to_string(),
resource_path: "resources/2026-04-07T12-00-00-abcd-10min-cutoff.md".to_string(),
},
]
);
assert!(
tokio::fs::try_exists(&old_file)
.await
.expect("check old file before remove")
);
assert!(
tokio::fs::try_exists(&exact_cutoff_file)
.await
.expect("check cutoff file before remove")
);
remove_extension_resources(&pending).await;
assert!(
!tokio::fs::try_exists(&old_file)
.await
.expect("check old file")
);
assert!(
!tokio::fs::try_exists(&exact_cutoff_file)
.await
.expect("check cutoff file")
);
assert!(
tokio::fs::try_exists(&recent_file)
.await
.expect("check recent file")
);
assert!(
tokio::fs::try_exists(&invalid_file)
.await
.expect("check invalid file")
);
assert!(
tokio::fs::try_exists(&ignored_old_file)
.await
.expect("check ignored old file")
);
}
}
codex-rs/core/src/memories/mod.rs
+2
View File
@@ -30,6 +30,8 @@ mod artifacts {
pub(super) const RAW_MEMORIES_FILENAME: &str = "raw_memories.md";
}
mod extensions;
/// Phase 1 (startup extraction).
mod phase_one {
/// Default model used for phase 1.
codex-rs/core/src/memories/phase1.rs
+3 -2
View File
@@ -477,9 +477,10 @@ mod job {
}
})
.collect::<Vec<_>>();
serde_json::to_string(&filtered).map_err(|err| {
let serialized = serde_json::to_string(&filtered).map_err(|err| {
CodexErr::InvalidRequest(format!("failed to serialize rollout memory: {err}"))
})
})?;
Ok(redact_secrets(serialized))
}
fn sanitize_response_item_for_memories(item: &ResponseItem) -> Option<ResponseItem> {
codex-rs/core/src/memories/phase1_tests.rs
+21
View File
@@ -3,6 +3,8 @@ use super::JobResult;
use super::aggregate_stats;
use super::job::serialize_filtered_rollout_response_items;
use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::RolloutItem;
use codex_protocol::protocol::TokenUsage;
@@ -72,6 +74,25 @@ fn serializes_memory_rollout_with_agents_removed_but_environment_kept() {
);
}
#[test]
fn serializes_memory_rollout_redacts_secrets_before_prompt_upload() {
let serialized = serialize_filtered_rollout_response_items(&[RolloutItem::ResponseItem(
ResponseItem::FunctionCallOutput {
call_id: "call_123".to_string(),
output: FunctionCallOutputPayload {
body: FunctionCallOutputBody::Text(
r#"{"token":"sk-abcdefghijklmnopqrstuvwxyz123456"}"#.to_string(),
),
success: Some(true),
},
},
)])
.expect("serialize");
assert!(!serialized.contains("sk-abcdefghijklmnopqrstuvwxyz123456"));
assert!(serialized.contains("[REDACTED_SECRET]"));
}
#[test]
fn count_outcomes_sums_token_usage_across_all_jobs() {
let counts = aggregate_stats(vec![
codex-rs/core/src/memories/phase2.rs
+23 -9
View File
@@ -3,6 +3,9 @@ use crate::agent::status::is_final as is_final_agent_status;
use crate::codex::Session;
use crate::codex::emit_subagent_session_started;
use crate::config::Config;
use crate::memories::extensions::PendingExtensionResourceRemoval;
use crate::memories::extensions::find_old_extension_resources;
use crate::memories::extensions::remove_extension_resources;
use crate::memories::memory_root;
use crate::memories::metrics;
use crate::memories::phase_two;
@@ -112,7 +115,12 @@ pub(super) async fn run(session: &Arc<Session>, config: Arc<Config>) {
job::failed(session, db, &claim, "failed_rebuild_raw_memories").await;
return;
}
if raw_memories.is_empty() {
let pending_extension_resource_removals = find_old_extension_resources(&root).await;
let removed_extension_resources = pending_extension_resource_removals
.iter()
.map(|resource| resource.removed.clone())
.collect::<Vec<_>>();
if raw_memories.is_empty() && pending_extension_resource_removals.is_empty() {
// We check only after sync of the file system.
job::succeed(
session,
@@ -127,7 +135,7 @@ pub(super) async fn run(session: &Arc<Session>, config: Arc<Config>) {
}
// 5. Spawn the agent
let prompt = agent::get_prompt(config, &selection);
let prompt = agent::get_prompt(config, &selection, &removed_extension_resources);
let source = SessionSource::SubAgent(SubAgentSource::MemoryConsolidation);
let thread_id = match session
.services
@@ -170,6 +178,7 @@ pub(super) async fn run(session: &Arc<Session>, config: Arc<Config>) {
claim,
new_watermark,
raw_memories.clone(),
pending_extension_resource_removals,
thread_id,
phase_two_e2e_timer,
);
@@ -268,15 +277,15 @@ mod job {
completion_watermark: i64,
selected_outputs: &[codex_state::Stage1Output],
reason: &'static str,
) {
) -> bool {
session.services.session_telemetry.counter(
metrics::MEMORY_PHASE_TWO_JOBS,
/*inc*/ 1,
&[("status", reason)],
);
let _ = db
.mark_global_phase2_job_succeeded(&claim.token, completion_watermark, selected_outputs)
.await;
db.mark_global_phase2_job_succeeded(&claim.token, completion_watermark, selected_outputs)
.await
.unwrap_or(false)
}
}
@@ -328,9 +337,10 @@ mod agent {
pub(super) fn get_prompt(
config: Arc<Config>,
selection: &codex_state::Phase2InputSelection,
removed_extension_resources: &[crate::memories::extensions::RemovedExtensionResource],
) -> Vec<UserInput> {
let root = memory_root(&config.codex_home);
let prompt = build_consolidation_prompt(&root, selection);
let prompt = build_consolidation_prompt(&root, selection, removed_extension_resources);
vec![UserInput::Text {
text: prompt,
text_elements: vec![],
@@ -343,6 +353,7 @@ mod agent {
claim: Claim,
new_watermark: i64,
selected_outputs: Vec<codex_state::Stage1Output>,
pending_extension_resource_removals: Vec<PendingExtensionResourceRemoval>,
thread_id: ThreadId,
phase_two_e2e_timer: Option<codex_otel::Timer>,
) {
@@ -379,7 +390,7 @@ mod agent {
if let Some(token_usage) = agent_control.get_total_token_usage(thread_id).await {
emit_token_usage_metrics(&session, &token_usage);
}
job::succeed(
if job::succeed(
&session,
&db,
&claim,
@@ -387,7 +398,10 @@ mod agent {
&selected_outputs,
"succeeded",
)
.await;
.await
{
remove_extension_resources(&pending_extension_resource_removals).await;
}
} else {
job::failed(&session, &db, &claim, "failed_agent").await;
}
codex-rs/core/src/memories/prompts.rs
+36 -6
View File
@@ -1,3 +1,5 @@
use crate::memories::extensions::EXTENSION_RESOURCE_RETENTION_DAYS;
use crate::memories::extensions::RemovedExtensionResource;
use crate::memories::memory_extensions_root;
use crate::memories::memory_root;
use crate::memories::phase_one;
@@ -10,6 +12,7 @@ use codex_utils_absolute_path::AbsolutePathBuf;
use codex_utils_output_truncation::TruncationPolicy;
use codex_utils_output_truncation::truncate_text;
use codex_utils_template::Template;
use std::fmt::Write as _;
use std::path::Path;
use std::sync::LazyLock;
use tokio::fs;
@@ -62,8 +65,9 @@ Memory extensions (under {{ memory_extensions_root }}/):
source.
If the user has any memory extensions, you MUST read the instructions for each extension to
determine how to use the memory source. If it has no extension folders, continue with the standard
memory inputs only.
determine how to use the memory source. If the Phase 2 diff lists removed memory extension
resources, use that extension-specific deletion diff to remove stale memories derived only from
those resources. If it has no extension folders, continue with the standard memory inputs only.
"#;
const MEMORY_EXTENSIONS_PRIMARY_INPUTS: &str = r#"
@@ -73,12 +77,16 @@ Under `{{ memory_extensions_root }}/`:
- `<extension_name>/instructions.md`
- If extension folders exist, read each instructions.md first and follow it when interpreting
that extension's memory source.
If the Phase 2 diff lists removed memory extension resources, use that extension-specific deletion
diff to remove stale memories derived only from those resources.
"#;
/// Builds the consolidation subagent prompt for a specific memory root.
pub(super) fn build_consolidation_prompt(
memory_root: &Path,
selection: &Phase2InputSelection,
removed_extension_resources: &[RemovedExtensionResource],
) -> String {
let memory_extensions_root = memory_extensions_root(memory_root);
let memory_extensions_exist = memory_extensions_root.is_dir();
@@ -100,7 +108,8 @@ pub(super) fn build_consolidation_prompt(
} else {
String::new()
};
let phase2_input_selection = render_phase2_input_selection(selection);
let phase2_input_selection =
render_phase2_input_selection(selection, removed_extension_resources);
CONSOLIDATION_PROMPT_TEMPLATE
.render([
("memory_root", memory_root.as_str()),
@@ -131,7 +140,10 @@ fn render_memory_extensions_block(template: &Template, memory_extensions_root: &
})
}
fn render_phase2_input_selection(selection: &Phase2InputSelection) -> String {
fn render_phase2_input_selection(
selection: &Phase2InputSelection,
removed_extension_resources: &[RemovedExtensionResource],
) -> String {
let retained = selection.retained_thread_ids.len();
let added = selection.selected.len().saturating_sub(retained);
let selected = if selection.selected.is_empty() {
@@ -160,11 +172,29 @@ fn render_phase2_input_selection(selection: &Phase2InputSelection) -> String {
.join("\n")
};
format!(
let mut rendered = format!(
"- selected inputs this run: {}\n- newly added since the last successful Phase 2 run: {added}\n- retained from the last successful Phase 2 run: {retained}\n- removed from the last successful Phase 2 run: {}\n\nCurrent selected Phase 1 inputs:\n{selected}\n\nRemoved from the last successful Phase 2 selection:\n{removed}\n",
selection.selected.len(),
selection.removed.len(),
)
);
if !removed_extension_resources.is_empty() {
rendered.push_str("\nMemory extension resources removed by retention pruning:\n");
let _ = writeln!(
rendered,
"- retention window: {EXTENSION_RESOURCE_RETENTION_DAYS} days"
);
let mut current_extension = "";
for removed_resource in removed_extension_resources {
if removed_resource.extension != current_extension {
current_extension = &removed_resource.extension;
let _ = writeln!(rendered, "- extension: {current_extension}");
}
let _ = writeln!(rendered, " - {}", removed_resource.resource_path);
}
}
rendered
}
fn render_selected_input_line(item: &Stage1Output, retained: bool) -> String {
codex-rs/core/src/memories/prompts_tests.rs
+32
View File
@@ -1,5 +1,7 @@
use super::*;
use crate::memories::extensions::RemovedExtensionResource;
use codex_models_manager::model_info::model_info_from_slug;
use codex_state::Phase2InputSelection;
use core_test_support::PathExt;
use pretty_assertions::assert_eq;
use tempfile::tempdir;
@@ -54,6 +56,36 @@ fn build_stage_one_input_message_uses_default_limit_when_model_context_window_mi
assert!(message.contains(&expected_truncated));
}
#[test]
fn build_consolidation_prompt_includes_removed_extension_resources() {
let temp = tempdir().unwrap();
let memory_root = temp.path().join("memories");
std::fs::create_dir_all(temp.path().join("memories_extensions")).unwrap();
let removed_extension_resources = vec![
RemovedExtensionResource {
extension: "telepathy".to_string(),
resource_path: "resources/2026-04-06T11-59-59-abcd-10min-old.md".to_string(),
},
RemovedExtensionResource {
extension: "telepathy".to_string(),
resource_path: "resources/2026-04-07T12-00-00-abcd-10min-cutoff.md".to_string(),
},
];
let prompt = build_consolidation_prompt(
&memory_root,
&Phase2InputSelection::default(),
&removed_extension_resources,
);
assert!(prompt.contains("Memory extension resources removed by retention pruning:"));
assert!(prompt.contains("- retention window: 7 days"));
assert!(prompt.contains("- extension: telepathy"));
assert!(prompt.contains(" - resources/2026-04-06T11-59-59-abcd-10min-old.md"));
assert!(prompt.contains(" - resources/2026-04-07T12-00-00-abcd-10min-cutoff.md"));
assert!(prompt.contains("extension-specific deletion diff"));
}
#[tokio::test]
async fn build_memory_tool_developer_instructions_renders_embedded_template() {
let temp = tempdir().unwrap();
codex-rs/core/src/memories/tests.rs
+29
View File
@@ -424,6 +424,7 @@ mod phase2 {
use crate::memories::phase2;
use crate::memories::raw_memories_file;
use crate::memories::rollout_summaries_dir;
use chrono::Duration as ChronoDuration;
use chrono::Utc;
use codex_config::Constrained;
use codex_login::CodexAuth;
@@ -960,6 +961,28 @@ mod phase2 {
"stage-1 success should enqueue global consolidation"
);
let telepathy_resources = config
.codex_home
.join("memories_extensions/telepathy/resources");
tokio::fs::create_dir_all(&telepathy_resources)
.await
.expect("create telepathy resources");
tokio::fs::write(
config
.codex_home
.join("memories_extensions/telepathy/instructions.md"),
"instructions",
)
.await
.expect("write telepathy instructions");
let old_file = telepathy_resources.join(format!(
"{}-abcd-10min-old.md",
(Utc::now() - ChronoDuration::days(8)).format("%Y-%m-%dT%H-%M-%S")
));
tokio::fs::write(&old_file, "old resource")
.await
.expect("write old extension resource");
phase2::run(&session, Arc::clone(&config)).await;
let retry_claim = state_db
@@ -971,5 +994,11 @@ mod phase2 {
Phase2JobClaimOutcome::SkippedNotDirty,
"spawn failures should leave the job in retry backoff instead of running"
);
assert!(
tokio::fs::try_exists(&old_file)
.await
.expect("check old extension resource"),
"spawn failures should not prune extension resources before retry"
);
}
}
codex-rs/core/src/personality_migration.rs
+29 -57
View File
@@ -1,14 +1,10 @@
use crate::config::edit::ConfigEditsBuilder;
use crate::rollout::ARCHIVED_SESSIONS_SUBDIR;
use crate::rollout::SESSIONS_SUBDIR;
use crate::rollout::list::ThreadListConfig;
use crate::rollout::list::ThreadListLayout;
use crate::rollout::list::ThreadSortKey;
use crate::rollout::list::get_threads_in_root;
use codex_config::config_toml::ConfigToml;
use codex_protocol::config_types::Personality;
use codex_protocol::protocol::SessionSource;
use codex_rollout::state_db;
use codex_thread_store::ListThreadsParams;
use codex_thread_store::LocalThreadStore;
use codex_thread_store::ThreadSortKey;
use codex_thread_store::ThreadStore;
use std::io;
use std::path::Path;
use tokio::fs::OpenOptions;
@@ -64,57 +60,33 @@ pub async fn maybe_migrate_personality(
}
async fn has_recorded_sessions(codex_home: &Path, default_provider: &str) -> io::Result<bool> {
let allowed_sources: &[SessionSource] = &[];
let store = LocalThreadStore::new(codex_rollout::RolloutConfig {
codex_home: codex_home.to_path_buf(),
sqlite_home: codex_home.to_path_buf(),
cwd: codex_home.to_path_buf(),
model_provider_id: default_provider.to_string(),
generate_memories: false,
});
if has_threads(&store, /*archived*/ false).await? {
return Ok(true);
}
has_threads(&store, /*archived*/ true).await
}
if let Some(state_db_ctx) = state_db::open_if_present(codex_home, default_provider).await
&& let Some(ids) = state_db::list_thread_ids_db(
Some(state_db_ctx.as_ref()),
codex_home,
/*page_size*/ 1,
/*cursor*/ None,
ThreadSortKey::CreatedAt,
allowed_sources,
/*model_providers*/ None,
/*archived_only*/ false,
"personality_migration",
)
async fn has_threads(store: &LocalThreadStore, archived: bool) -> io::Result<bool> {
store
.list_threads(ListThreadsParams {
page_size: 1,
cursor: None,
sort_key: ThreadSortKey::CreatedAt,
allowed_sources: Vec::new(),
model_providers: None,
archived,
search_term: None,
})
.await
&& !ids.is_empty()
{
return Ok(true);
}
let sessions = get_threads_in_root(
codex_home.join(SESSIONS_SUBDIR),
/*page_size*/ 1,
/*cursor*/ None,
ThreadSortKey::CreatedAt,
ThreadListConfig {
allowed_sources,
model_providers: None,
default_provider,
layout: ThreadListLayout::NestedByDate,
},
)
.await?;
if !sessions.items.is_empty() {
return Ok(true);
}
let archived_sessions = get_threads_in_root(
codex_home.join(ARCHIVED_SESSIONS_SUBDIR),
/*page_size*/ 1,
/*cursor*/ None,
ThreadSortKey::CreatedAt,
ThreadListConfig {
allowed_sources,
model_providers: None,
default_provider,
layout: ThreadListLayout::Flat,
},
)
.await?;
Ok(!archived_sessions.items.is_empty())
.map(|page| !page.items.is_empty())
.map_err(io::Error::other)
}
async fn create_marker(marker_path: &Path) -> io::Result<()> {
codex-rs/core/src/personality_migration_tests.rs
+28
View File
@@ -7,6 +7,8 @@ use codex_protocol::protocol::SessionMeta;
use codex_protocol::protocol::SessionMetaLine;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::UserMessageEvent;
use codex_rollout::ARCHIVED_SESSIONS_SUBDIR;
use codex_rollout::SESSIONS_SUBDIR;
use pretty_assertions::assert_eq;
use tempfile::TempDir;
use tokio::io::AsyncWriteExt;
@@ -25,6 +27,16 @@ async fn write_session_with_user_event(codex_home: &Path) -> io::Result<()> {
.join("2025")
.join("01")
.join("01");
write_rollout_with_user_event(&dir, thread_id).await
}
async fn write_archived_session_with_user_event(codex_home: &Path) -> io::Result<()> {
let thread_id = ThreadId::new();
let dir = codex_home.join(ARCHIVED_SESSIONS_SUBDIR);
write_rollout_with_user_event(&dir, thread_id).await
}
async fn write_rollout_with_user_event(dir: &Path, thread_id: ThreadId) -> io::Result<()> {
tokio::fs::create_dir_all(&dir).await?;
let file_path = dir.join(format!("rollout-{TEST_TIMESTAMP}-{thread_id}.jsonl"));
let mut file = tokio::fs::File::create(&file_path).await?;
@@ -85,6 +97,22 @@ async fn applies_when_sessions_exist_and_no_personality() -> io::Result<()> {
Ok(())
}
#[tokio::test]
async fn applies_when_only_archived_sessions_exist_and_no_personality() -> io::Result<()> {
let temp = TempDir::new()?;
write_archived_session_with_user_event(temp.path()).await?;
let config_toml = ConfigToml::default();
let status = maybe_migrate_personality(temp.path(), &config_toml).await?;
assert_eq!(status, PersonalityMigrationStatus::Applied);
assert!(temp.path().join(PERSONALITY_MIGRATION_FILENAME).exists());
let persisted = read_config_toml(temp.path()).await?;
assert_eq!(persisted.personality, Some(Personality::Pragmatic));
Ok(())
}
#[tokio::test]
async fn skips_when_marker_exists() -> io::Result<()> {
let temp = TempDir::new()?;
codex-rs/core/src/plugins/installed_marketplaces.rs
+2 -3
View File
@@ -4,6 +4,7 @@ use std::path::Path;
use std::path::PathBuf;
use tracing::warn;
use super::marketplace::find_marketplace_manifest_path;
use super::validate_plugin_segment;
pub const INSTALLED_MARKETPLACES_DIR: &str = ".tmp/marketplaces";
@@ -50,9 +51,7 @@ pub(crate) fn installed_marketplace_roots_from_config(
marketplace,
&default_install_root,
)?;
path.join(".agents/plugins/marketplace.json")
.is_file()
.then_some(path)
find_marketplace_manifest_path(&path).map(|_| path)
})
.filter_map(|path| AbsolutePathBuf::try_from(path).ok())
.collect::<Vec<_>>();
codex-rs/core/src/plugins/marketplace.rs
+187 -119
View File
@@ -9,6 +9,8 @@ use codex_protocol::protocol::Product;
use codex_utils_absolute_path::AbsolutePathBuf;
use dirs::home_dir;
use serde::Deserialize;
use serde::Deserializer;
use serde_json::Value as JsonValue;
use std::fs;
use std::io;
use std::path::Component;
@@ -16,7 +18,10 @@ use std::path::Path;
use std::path::PathBuf;
use tracing::warn;
const MARKETPLACE_RELATIVE_PATH: &str = ".agents/plugins/marketplace.json";
const MARKETPLACE_MANIFEST_RELATIVE_PATHS: &[&str] = &[
".agents/plugins/marketplace.json",
".claude-plugin/marketplace.json",
];
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ResolvedMarketplacePlugin {
@@ -162,46 +167,50 @@ pub fn resolve_marketplace_plugin(
) -> Result<ResolvedMarketplacePlugin, MarketplaceError> {
let marketplace = load_raw_marketplace_manifest(marketplace_path)?;
let marketplace_name = marketplace.name;
let plugin = marketplace
.plugins
.into_iter()
.find(|plugin| plugin.name == plugin_name);
let Some(plugin) = plugin else {
return Err(MarketplaceError::PluginNotFound {
plugin_name: plugin_name.to_string(),
marketplace_name,
});
};
let RawMarketplaceManifestPlugin {
name,
source,
policy,
..
} = plugin;
let install_policy = policy.installation;
let product_allowed = match policy.products.as_deref() {
None => true,
Some([]) => false,
Some(products) => {
restriction_product.is_some_and(|product| product.matches_product_restriction(products))
let marketplace_name_for_not_found = marketplace_name.clone();
for plugin in marketplace.plugins {
if plugin.name != plugin_name {
continue;
}
};
if install_policy == MarketplacePluginInstallPolicy::NotAvailable || !product_allowed {
return Err(MarketplaceError::PluginNotAvailable {
plugin_name: name,
marketplace_name,
let RawMarketplaceManifestPlugin {
name,
source,
policy,
..
} = plugin;
let install_policy = policy.installation;
let product_allowed = match policy.products.as_deref() {
None => true,
Some([]) => false,
Some(products) => restriction_product
.is_some_and(|product| product.matches_product_restriction(products)),
};
if install_policy == MarketplacePluginInstallPolicy::NotAvailable || !product_allowed {
return Err(MarketplaceError::PluginNotAvailable {
plugin_name: name,
marketplace_name,
});
}
let Some(source_path) =
resolve_supported_plugin_source_path(marketplace_path, &name, source)
else {
continue;
};
return Ok(ResolvedMarketplacePlugin {
plugin_id: PluginId::new(name, marketplace_name).map_err(|err| match err {
PluginIdError::Invalid(message) => MarketplaceError::InvalidPlugin(message),
})?,
source_path,
auth_policy: policy.authentication,
});
}
let plugin_id = PluginId::new(name, marketplace_name).map_err(|err| match err {
PluginIdError::Invalid(message) => MarketplaceError::InvalidPlugin(message),
})?;
Ok(ResolvedMarketplacePlugin {
plugin_id,
source_path: resolve_plugin_source_path(marketplace_path, source)?,
auth_policy: policy.authentication,
Err(MarketplaceError::PluginNotFound {
plugin_name: plugin_name.to_string(),
marketplace_name: marketplace_name_for_not_found,
})
}
@@ -212,16 +221,50 @@ pub fn list_marketplaces(
}
pub fn validate_marketplace_root(root: &Path) -> Result<String, MarketplaceError> {
let path = AbsolutePathBuf::try_from(root.join(MARKETPLACE_RELATIVE_PATH)).map_err(|err| {
MarketplaceError::InvalidMarketplaceFile {
path: root.join(MARKETPLACE_RELATIVE_PATH),
message: format!("marketplace path must resolve to an absolute path: {err}"),
}
})?;
let Some(path) = find_marketplace_manifest_path(root) else {
return Err(MarketplaceError::InvalidMarketplaceFile {
path: root.to_path_buf(),
message: "marketplace root does not contain a supported manifest".to_string(),
});
};
let marketplace = load_marketplace(&path)?;
Ok(marketplace.name)
}
pub(crate) fn find_marketplace_manifest_path(root: &Path) -> Option<AbsolutePathBuf> {
MARKETPLACE_MANIFEST_RELATIVE_PATHS
.iter()
.find_map(|relative_path| {
let path = root.join(relative_path);
if !path.is_file() {
return None;
}
AbsolutePathBuf::try_from(path).ok()
})
}
fn invalid_marketplace_layout_error(path: &AbsolutePathBuf) -> MarketplaceError {
MarketplaceError::InvalidMarketplaceFile {
path: path.to_path_buf(),
message: "marketplace file is not in a supported location".to_string(),
}
}
fn marketplace_root_from_layout(marketplace_path: &Path, relative_path: &str) -> Option<PathBuf> {
let mut current = marketplace_path;
for component in Path::new(relative_path).components().rev() {
let expected = match component {
Component::Normal(expected) => expected,
_ => return None,
};
if current.file_name() != Some(expected) {
return None;
}
current = current.parent()?;
}
Some(current.to_path_buf())
}
pub(crate) fn load_marketplace(path: &AbsolutePathBuf) -> Result<Marketplace, MarketplaceError> {
let marketplace = load_raw_marketplace_manifest(path)?;
let mut plugins = Vec::new();
@@ -233,7 +276,9 @@ pub(crate) fn load_marketplace(path: &AbsolutePathBuf) -> Result<Marketplace, Ma
policy,
category,
} = plugin;
let source_path = resolve_plugin_source_path(path, source)?;
let Some(source_path) = resolve_supported_plugin_source_path(path, &name, source) else {
continue;
};
let source = MarketplacePluginSource::Local {
path: source_path.clone(),
};
@@ -298,30 +343,27 @@ fn discover_marketplace_paths_from_roots(
) -> Vec<AbsolutePathBuf> {
let mut paths = Vec::new();
if let Some(home) = home_dir {
let path = home.join(MARKETPLACE_RELATIVE_PATH);
if path.is_file()
&& let Ok(path) = AbsolutePathBuf::try_from(path)
{
paths.push(path);
}
if let Some(home) = home_dir
&& let Some(path) = find_marketplace_manifest_path(home)
{
paths.push(path);
}
for root in additional_roots {
// Curated marketplaces can now come from an HTTP-downloaded directory that is not a git
// checkout, so check the root directly before falling back to repo-root discovery.
let path = root.join(MARKETPLACE_RELATIVE_PATH);
if path.as_path().is_file() && !paths.contains(&path) {
if let Some(path) = find_marketplace_manifest_path(root.as_path())
&& !paths.contains(&path)
{
paths.push(path);
continue;
}
if let Some(repo_root) = get_git_repo_root(root.as_path())
&& let Ok(repo_root) = AbsolutePathBuf::try_from(repo_root)
&& let Some(path) = find_marketplace_manifest_path(repo_root.as_path())
&& !paths.contains(&path)
{
let path = repo_root.join(MARKETPLACE_RELATIVE_PATH);
if path.as_path().is_file() && !paths.contains(&path) {
paths.push(path);
}
paths.push(path);
}
}
@@ -346,80 +388,83 @@ fn load_raw_marketplace_manifest(
})
}
fn resolve_plugin_source_path(
fn resolve_supported_plugin_source_path(
marketplace_path: &AbsolutePathBuf,
plugin_name: &str,
source: RawMarketplaceManifestPluginSource,
) -> Result<AbsolutePathBuf, MarketplaceError> {
) -> Option<AbsolutePathBuf> {
match source {
RawMarketplaceManifestPluginSource::Local { path } => {
let Some(path) = path.strip_prefix("./") else {
return Err(MarketplaceError::InvalidMarketplaceFile {
path: marketplace_path.to_path_buf(),
message: "local plugin source path must start with `./`".to_string(),
});
};
if path.is_empty() {
return Err(MarketplaceError::InvalidMarketplaceFile {
path: marketplace_path.to_path_buf(),
message: "local plugin source path must not be empty".to_string(),
});
match resolve_local_plugin_source_path(marketplace_path, &path) {
Ok(path) => Some(path),
Err(err) => {
warn!(
path = %marketplace_path.display(),
plugin = plugin_name,
error = %err,
"skipping marketplace plugin that failed to resolve"
);
None
}
}
let relative_source_path = Path::new(path);
if relative_source_path
.components()
.any(|component| !matches!(component, Component::Normal(_)))
{
return Err(MarketplaceError::InvalidMarketplaceFile {
path: marketplace_path.to_path_buf(),
message: "local plugin source path must stay within the marketplace root"
.to_string(),
});
}
// `marketplace.json` lives under `<root>/.agents/plugins/`, but local plugin paths
// are resolved relative to `<root>`, not relative to the `plugins/` directory.
Ok(marketplace_root_dir(marketplace_path)?.join(relative_source_path))
}
RawMarketplaceManifestPluginSource::Unsupported => {
warn!(
path = %marketplace_path.display(),
plugin = plugin_name,
"skipping marketplace plugin with unsupported source"
);
None
}
}
}
fn resolve_local_plugin_source_path(
marketplace_path: &AbsolutePathBuf,
source_path: &str,
) -> Result<AbsolutePathBuf, MarketplaceError> {
let Some(source_path) = source_path.strip_prefix("./") else {
return Err(MarketplaceError::InvalidMarketplaceFile {
path: marketplace_path.to_path_buf(),
message: "local plugin source path must start with `./`".to_string(),
});
};
if source_path.is_empty() {
return Err(MarketplaceError::InvalidMarketplaceFile {
path: marketplace_path.to_path_buf(),
message: "local plugin source path must not be empty".to_string(),
});
}
let relative_source_path = Path::new(source_path);
if relative_source_path
.components()
.any(|component| !matches!(component, Component::Normal(_)))
{
return Err(MarketplaceError::InvalidMarketplaceFile {
path: marketplace_path.to_path_buf(),
message: "local plugin source path must stay within the marketplace root".to_string(),
});
}
// `marketplace.json` lives under a supported marketplace layout beneath `<root>`,
// but local plugin paths are resolved relative to `<root>`.
Ok(marketplace_root_dir(marketplace_path)?.join(relative_source_path))
}
fn marketplace_root_dir(
marketplace_path: &AbsolutePathBuf,
) -> Result<AbsolutePathBuf, MarketplaceError> {
let Some(plugins_dir) = marketplace_path.parent() else {
return Err(MarketplaceError::InvalidMarketplaceFile {
path: marketplace_path.to_path_buf(),
message: "marketplace file must live under `<root>/.agents/plugins/`".to_string(),
});
};
let Some(dot_agents_dir) = plugins_dir.parent() else {
return Err(MarketplaceError::InvalidMarketplaceFile {
path: marketplace_path.to_path_buf(),
message: "marketplace file must live under `<root>/.agents/plugins/`".to_string(),
});
};
let Some(marketplace_root) = dot_agents_dir.parent() else {
return Err(MarketplaceError::InvalidMarketplaceFile {
path: marketplace_path.to_path_buf(),
message: "marketplace file must live under `<root>/.agents/plugins/`".to_string(),
});
};
if plugins_dir.as_path().file_name().and_then(|s| s.to_str()) != Some("plugins")
|| dot_agents_dir
.as_path()
.file_name()
.and_then(|s| s.to_str())
!= Some(".agents")
{
return Err(MarketplaceError::InvalidMarketplaceFile {
path: marketplace_path.to_path_buf(),
message: "marketplace file must live under `<root>/.agents/plugins/`".to_string(),
});
for relative_path in MARKETPLACE_MANIFEST_RELATIVE_PATHS {
if let Some(marketplace_root) =
marketplace_root_from_layout(marketplace_path.as_path(), relative_path)
{
return AbsolutePathBuf::try_from(marketplace_root)
.map_err(|_| invalid_marketplace_layout_error(marketplace_path));
}
}
Ok(marketplace_root)
Err(invalid_marketplace_layout_error(marketplace_path))
}
#[derive(Debug, Deserialize)]
@@ -459,10 +504,33 @@ struct RawMarketplaceManifestPluginPolicy {
products: Option<Vec<Product>>,
}
#[derive(Debug, Deserialize)]
#[serde(tag = "source", rename_all = "lowercase")]
#[derive(Debug)]
enum RawMarketplaceManifestPluginSource {
Local { path: String },
// Mixed-source marketplaces should still contribute the local plugins we can load.
Unsupported,
}
impl<'de> Deserialize<'de> for RawMarketplaceManifestPluginSource {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let source = JsonValue::deserialize(deserializer)?;
Ok(match source {
JsonValue::String(path) => Self::Local { path },
JsonValue::Object(object) => match object.get("source").and_then(JsonValue::as_str) {
Some("local") => match object.get("path").and_then(JsonValue::as_str) {
Some(path) => Self::Local {
path: path.to_string(),
},
None => Self::Unsupported,
},
_ => Self::Unsupported,
},
_ => Self::Unsupported,
})
}
}
fn resolve_marketplace_interface(
codex-rs/core/src/plugins/marketplace_add.rs
+9 -3
View File
@@ -282,9 +282,15 @@ mod tests {
);
let config = fs::read_to_string(codex_home.path().join(codex_config::CONFIG_TOML_FILE))?;
assert!(config.contains("[marketplaces.debug]"));
assert!(config.contains("source_type = \"local\""));
assert!(config.contains(&format!("source = \"{expected_source}\"")));
let config: toml::Value = toml::from_str(&config)?;
assert_eq!(
config["marketplaces"]["debug"]["source_type"].as_str(),
Some("local")
);
assert_eq!(
config["marketplaces"]["debug"]["source"].as_str(),
Some(expected_source.as_str())
);
Ok(())
}
codex-rs/core/src/plugins/marketplace_add/metadata.rs
+1 -9
View File
@@ -296,19 +296,11 @@ mod tests {
r#"{"name":"debug","plugins":[]}"#,
)
.unwrap();
fs::write(
codex_home.path().join(CONFIG_TOML_FILE),
format!(
"[marketplaces.debug]\nsource_type = \"local\"\nsource = \"{}\"\n",
source_root.display()
),
)
.unwrap();
let source = MarketplaceSource::Local {
path: source_root.clone(),
};
let install_metadata = MarketplaceInstallMetadata::from_source(&source, &[]);
record_added_marketplace_entry(codex_home.path(), "debug", &install_metadata).unwrap();
let root = installed_marketplace_root_for_source(
codex_home.path(),
codex-rs/core/src/plugins/marketplace_add/source.rs
+2 -2
View File
@@ -124,9 +124,9 @@ fn normalize_git_url(url: &str) -> String {
}
fn looks_like_local_path(source: &str) -> bool {
source.starts_with("./")
Path::new(source).is_absolute()
|| source.starts_with("./")
|| source.starts_with("../")
|| source.starts_with('/')
|| source.starts_with("~/")
|| source == "."
|| source == ".."
codex-rs/core/src/plugins/marketplace_tests.rs
+245 -10
View File
@@ -1,8 +1,18 @@
use super::*;
use codex_protocol::protocol::Product;
use pretty_assertions::assert_eq;
use std::path::Path;
use tempfile::tempdir;
const ALTERNATE_MARKETPLACE_RELATIVE_PATH: &str = ".claude-plugin/marketplace.json";
fn write_alternate_marketplace(repo_root: &Path, contents: &str) -> AbsolutePathBuf {
let marketplace_path = repo_root.join(ALTERNATE_MARKETPLACE_RELATIVE_PATH);
fs::create_dir_all(marketplace_path.parent().unwrap()).unwrap();
fs::write(&marketplace_path, contents).unwrap();
AbsolutePathBuf::try_from(marketplace_path).unwrap()
}
#[test]
fn resolve_marketplace_plugin_finds_repo_marketplace_plugin() {
let tmp = tempdir().unwrap();
@@ -45,6 +55,46 @@ fn resolve_marketplace_plugin_finds_repo_marketplace_plugin() {
);
}
#[test]
fn resolve_marketplace_plugin_supports_alternate_layout_and_string_local_source() {
let tmp = tempdir().unwrap();
let repo_root = tmp.path().join("repo");
fs::create_dir_all(repo_root.join(".git")).unwrap();
let marketplace_path = write_alternate_marketplace(
&repo_root,
r#"{
"name": "alternate-marketplace",
"plugins": [
{
"name": "string-source-plugin",
"source": "./plugins/string-source-plugin"
}
]
}"#,
);
let resolved = resolve_marketplace_plugin(
&marketplace_path,
"string-source-plugin",
Some(Product::Codex),
)
.unwrap();
assert_eq!(
resolved,
ResolvedMarketplacePlugin {
plugin_id: PluginId::new(
"string-source-plugin".to_string(),
"alternate-marketplace".to_string()
)
.unwrap(),
source_path: AbsolutePathBuf::try_from(repo_root.join("plugins/string-source-plugin"))
.unwrap(),
auth_policy: MarketplacePluginAuthPolicy::OnInstall,
}
);
}
#[test]
fn resolve_marketplace_plugin_reports_missing_plugin() {
let tmp = tempdir().unwrap();
@@ -70,6 +120,106 @@ fn resolve_marketplace_plugin_reports_missing_plugin() {
);
}
#[test]
fn list_marketplaces_supports_alternate_manifest_layout() {
let tmp = tempdir().unwrap();
let repo_root = tmp.path().join("repo");
fs::create_dir_all(repo_root.join(".git")).unwrap();
let marketplace_path = write_alternate_marketplace(
&repo_root,
r#"{
"name": "alternate-marketplace",
"plugins": [
{
"name": "string-source-plugin",
"source": "./plugins/string-source-plugin"
}
]
}"#,
);
let marketplaces = list_marketplaces_with_home(
&[AbsolutePathBuf::try_from(repo_root.clone()).unwrap()],
/*home_dir*/ None,
)
.unwrap()
.marketplaces;
assert_eq!(
marketplaces,
vec![Marketplace {
name: "alternate-marketplace".to_string(),
path: marketplace_path,
interface: None,
plugins: vec![MarketplacePlugin {
name: "string-source-plugin".to_string(),
source: MarketplacePluginSource::Local {
path: AbsolutePathBuf::try_from(repo_root.join("plugins/string-source-plugin"))
.unwrap(),
},
policy: MarketplacePluginPolicy {
installation: MarketplacePluginInstallPolicy::Available,
authentication: MarketplacePluginAuthPolicy::OnInstall,
products: None,
},
interface: None,
}],
}]
);
}
#[test]
fn list_marketplaces_prefers_first_supported_manifest_layout() {
let tmp = tempdir().unwrap();
let repo_root = tmp.path().join("repo");
fs::create_dir_all(repo_root.join(".git")).unwrap();
fs::create_dir_all(repo_root.join(".agents/plugins")).unwrap();
fs::write(
repo_root.join(".agents/plugins/marketplace.json"),
r#"{
"name": "agents-marketplace",
"plugins": [
{
"name": "agents-plugin",
"source": {
"source": "local",
"path": "./plugins/agents-plugin"
}
}
]
}"#,
)
.unwrap();
write_alternate_marketplace(
&repo_root,
r#"{
"name": "alternate-marketplace",
"plugins": [
{
"name": "string-source-plugin",
"source": "./plugins/string-source-plugin"
}
]
}"#,
);
let marketplaces = list_marketplaces_with_home(
&[AbsolutePathBuf::try_from(repo_root.clone()).unwrap()],
/*home_dir*/ None,
)
.unwrap()
.marketplaces;
assert_eq!(marketplaces.len(), 1);
assert_eq!(marketplaces[0].name, "agents-marketplace");
assert_eq!(
marketplaces[0].path,
AbsolutePathBuf::try_from(repo_root.join(".agents/plugins/marketplace.json")).unwrap()
);
}
#[test]
fn list_marketplaces_returns_home_and_repo_marketplaces() {
let tmp = tempdir().unwrap();
@@ -413,7 +563,7 @@ fn list_marketplaces_reads_marketplace_display_name() {
}
#[test]
fn list_marketplaces_skips_marketplaces_that_fail_to_load() {
fn list_marketplaces_skips_invalid_plugins_but_keeps_marketplace() {
let tmp = tempdir().unwrap();
let valid_repo_root = tmp.path().join("valid-repo");
let invalid_repo_root = tmp.path().join("invalid-repo");
@@ -465,8 +615,10 @@ fn list_marketplaces_skips_marketplaces_that_fail_to_load() {
.unwrap()
.marketplaces;
assert_eq!(marketplaces.len(), 1);
assert_eq!(marketplaces.len(), 2);
assert_eq!(marketplaces[0].name, "valid-marketplace");
assert_eq!(marketplaces[1].name, "invalid-marketplace");
assert!(marketplaces[1].plugins.is_empty());
}
#[test]
@@ -522,6 +674,60 @@ fn list_marketplaces_reports_marketplace_load_errors() {
);
}
#[test]
fn list_marketplaces_skips_unsupported_plugin_sources_but_keeps_local_plugins() {
let tmp = tempdir().unwrap();
let repo_root = tmp.path().join("repo");
fs::create_dir_all(repo_root.join(".git")).unwrap();
write_alternate_marketplace(
&repo_root,
r#"{
"name": "mixed-source-marketplace",
"plugins": [
{
"name": "local-plugin",
"source": "./plugins/local-plugin"
},
{
"name": "url-plugin",
"source": {
"source": "url",
"url": "https://github.com/example/plugin.git"
}
},
{
"name": "git-subdir-plugin",
"source": {
"source": "git-subdir",
"url": "owner/repo",
"path": "plugins/example",
"ref": "main"
}
}
]
}"#,
);
let marketplaces = list_marketplaces_with_home(
&[AbsolutePathBuf::try_from(repo_root.clone()).unwrap()],
/*home_dir*/ None,
)
.unwrap()
.marketplaces;
assert_eq!(marketplaces.len(), 1);
assert_eq!(marketplaces[0].name, "mixed-source-marketplace");
assert_eq!(marketplaces[0].plugins.len(), 1);
assert_eq!(marketplaces[0].plugins[0].name, "local-plugin");
assert_eq!(
marketplaces[0].plugins[0].source,
MarketplacePluginSource::Local {
path: AbsolutePathBuf::try_from(repo_root.join("plugins/local-plugin")).unwrap(),
}
);
}
#[test]
fn list_marketplaces_resolves_plugin_interface_paths_to_absolute() {
let tmp = tempdir().unwrap();
@@ -734,7 +940,7 @@ fn list_marketplaces_ignores_plugin_interface_assets_without_dot_slash() {
}
#[test]
fn resolve_marketplace_plugin_rejects_non_relative_local_paths() {
fn resolve_marketplace_plugin_skips_invalid_local_paths() {
let tmp = tempdir().unwrap();
let repo_root = tmp.path().join("repo");
fs::create_dir_all(repo_root.join(".git")).unwrap();
@@ -756,17 +962,46 @@ fn resolve_marketplace_plugin_rejects_non_relative_local_paths() {
)
.unwrap();
let marketplace_path =
AbsolutePathBuf::try_from(repo_root.join(".agents/plugins/marketplace.json")).unwrap();
let err = resolve_marketplace_plugin(&marketplace_path, "local-plugin", Some(Product::Codex))
let err = resolve_marketplace_plugin(
&AbsolutePathBuf::try_from(repo_root.join(".agents/plugins/marketplace.json")).unwrap(),
"local-plugin",
Some(Product::Codex),
)
.unwrap_err();
assert_eq!(
err.to_string(),
"plugin `local-plugin` was not found in marketplace `codex-curated`"
);
}
#[test]
fn resolve_marketplace_plugin_skips_unsupported_sources() {
let tmp = tempdir().unwrap();
let repo_root = tmp.path().join("repo");
fs::create_dir_all(repo_root.join(".git")).unwrap();
let marketplace_path = write_alternate_marketplace(
&repo_root,
r#"{
"name": "alternate-marketplace",
"plugins": [
{
"name": "remote-plugin",
"source": {
"source": "url",
"url": "https://github.com/example/plugin.git"
}
}
]
}"#,
);
let err = resolve_marketplace_plugin(&marketplace_path, "remote-plugin", Some(Product::Codex))
.unwrap_err();
assert_eq!(
err.to_string(),
format!(
"invalid marketplace file `{}`: local plugin source path must start with `./`",
marketplace_path.display()
)
"plugin `remote-plugin` was not found in marketplace `alternate-marketplace`"
);
}
codex-rs/core/src/realtime_context.rs
+25 -24
View File
@@ -4,8 +4,10 @@ use crate::event_mapping::is_contextual_user_message_content;
use chrono::Utc;
use codex_git_utils::resolve_root_git_project_for_trust;
use codex_protocol::models::ResponseItem;
use codex_state::SortKey;
use codex_state::ThreadMetadata;
use codex_thread_store::ListThreadsParams;
use codex_thread_store::StoredThread;
use codex_thread_store::ThreadSortKey;
use codex_thread_store::ThreadStore;
use codex_utils_output_truncation::TruncationPolicy;
use codex_utils_output_truncation::truncate_text;
use dirs::home_dir;
@@ -98,7 +100,7 @@ pub(crate) async fn build_realtime_startup_context(
}
if let Some(section) = format_section(
"Notes",
Some("Built at realtime startup from the current thread history, persisted thread metadata in the state DB, and a bounded local workspace scan. This excludes repo memory instructions, AGENTS files, project-doc prompt blends, and memory summaries.".to_string()),
Some("Built at realtime startup from the current thread history, local thread metadata, and a bounded local workspace scan. This excludes repo memory instructions, AGENTS files, project-doc prompt blends, and memory summaries.".to_string()),
NOTES_SECTION_TOKEN_BUDGET,
) {
parts.push(section);
@@ -117,33 +119,31 @@ pub(crate) async fn build_realtime_startup_context(
Some(context)
}
async fn load_recent_threads(sess: &Session) -> Vec<ThreadMetadata> {
let Some(state_db) = sess.services.state_db.as_ref() else {
return Vec::new();
};
match state_db
.list_threads(
MAX_RECENT_THREADS,
/*anchor*/ None,
SortKey::UpdatedAt,
&[],
/*model_providers*/ None,
/*archived_only*/ false,
/*search_term*/ None,
)
async fn load_recent_threads(sess: &Session) -> Vec<StoredThread> {
match sess
.services
.thread_store
.list_threads(ListThreadsParams {
page_size: MAX_RECENT_THREADS,
cursor: None,
sort_key: ThreadSortKey::UpdatedAt,
allowed_sources: Vec::new(),
model_providers: None,
archived: false,
search_term: None,
})
.await
{
Ok(page) => page.items,
Err(err) => {
warn!("failed to load realtime startup threads from state db: {err}");
warn!("failed to load realtime startup threads from thread store: {err}");
Vec::new()
}
}
}
fn build_recent_work_section(cwd: &Path, recent_threads: &[ThreadMetadata]) -> Option<String> {
let mut groups: HashMap<PathBuf, Vec<&ThreadMetadata>> = HashMap::new();
fn build_recent_work_section(cwd: &Path, recent_threads: &[StoredThread]) -> Option<String> {
let mut groups: HashMap<PathBuf, Vec<&StoredThread>> = HashMap::new();
for entry in recent_threads {
let group =
resolve_root_git_project_for_trust(&entry.cwd).unwrap_or_else(|| entry.cwd.clone());
@@ -446,7 +446,7 @@ fn format_section(title: &str, body: Option<String>, budget_tokens: usize) -> Op
fn format_thread_group(
current_group: &Path,
group: &Path,
entries: Vec<&ThreadMetadata>,
entries: Vec<&StoredThread>,
) -> Option<String> {
let latest = entries.first()?;
let group_label = if resolve_root_git_project_for_trust(latest.cwd.as_path()).is_some() {
@@ -461,8 +461,9 @@ fn format_thread_group(
];
if let Some(git_branch) = latest
.git_branch
.as_deref()
.git_info
.as_ref()
.and_then(|git| git.branch.as_deref())
.filter(|git_branch| !git_branch.is_empty())
{
lines.push(format!("Latest branch: {git_branch}"));
codex-rs/core/src/realtime_context_tests.rs
+33 -23
View File
@@ -3,20 +3,31 @@ use super::build_recent_work_section;
use super::build_workspace_section_with_user_root;
use chrono::TimeZone;
use chrono::Utc;
use codex_git_utils::GitSha;
use codex_protocol::ThreadId;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ResponseItem;
use codex_state::ThreadMetadata;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::GitInfo;
use codex_protocol::protocol::SandboxPolicy;
use codex_protocol::protocol::SessionSource;
use codex_thread_store::StoredThread;
use pretty_assertions::assert_eq;
use std::fs;
use std::path::PathBuf;
use std::process::Command;
use tempfile::TempDir;
fn thread_metadata(cwd: &str, title: &str, first_user_message: &str) -> ThreadMetadata {
ThreadMetadata {
id: ThreadId::new(),
rollout_path: PathBuf::from("/tmp/rollout.jsonl"),
fn stored_thread(cwd: &str, title: &str, first_user_message: &str) -> StoredThread {
StoredThread {
thread_id: ThreadId::new(),
rollout_path: Some(PathBuf::from("/tmp/rollout.jsonl")),
forked_from_id: None,
preview: first_user_message.to_string(),
name: (!title.is_empty()).then(|| title.to_string()),
model_provider: "test-provider".to_string(),
model: Some("gpt-5".to_string()),
reasoning_effort: None,
created_at: Utc
.timestamp_opt(1_709_251_100, 0)
.single()
@@ -25,24 +36,23 @@ fn thread_metadata(cwd: &str, title: &str, first_user_message: &str) -> ThreadMe
.timestamp_opt(1_709_251_200, 0)
.single()
.expect("valid timestamp"),
source: "cli".to_string(),
agent_path: None,
agent_nickname: None,
agent_role: None,
model_provider: "test-provider".to_string(),
model: Some("gpt-5".to_string()),
reasoning_effort: None,
archived_at: None,
cwd: PathBuf::from(cwd),
cli_version: "test".to_string(),
title: title.to_string(),
sandbox_policy: "workspace-write".to_string(),
approval_mode: "never".to_string(),
tokens_used: 0,
source: SessionSource::Cli,
agent_nickname: None,
agent_role: None,
agent_path: None,
git_info: Some(GitInfo {
commit_hash: Some(GitSha::new("abcdef")),
branch: Some("main".to_string()),
repository_url: None,
}),
approval_mode: AskForApproval::Never,
sandbox_policy: SandboxPolicy::new_read_only_policy(),
token_usage: None,
first_user_message: Some(first_user_message.to_string()),
archived_at: None,
git_sha: None,
git_branch: Some("main".to_string()),
git_origin_url: None,
history: None,
}
}
@@ -224,17 +234,17 @@ fn recent_work_section_groups_threads_by_cwd() {
fs::create_dir_all(&outside).expect("create outside dir");
let recent_threads = vec![
thread_metadata(
stored_thread(
workspace_a.to_string_lossy().as_ref(),
"Investigate realtime startup context",
"Log the startup context before sending it",
),
thread_metadata(
stored_thread(
workspace_b.to_string_lossy().as_ref(),
"Trim websocket startup payload",
"Remove memories from the realtime startup context",
),
thread_metadata(outside.to_string_lossy().as_ref(), "", "Inspect flaky test"),
stored_thread(outside.to_string_lossy().as_ref(), "", "Inspect flaky test"),
];
let current_cwd = workspace_a;
let repo = fs::canonicalize(repo).expect("canonicalize repo");
codex-rs/core/src/realtime_conversation.rs
+17 -1
View File
@@ -805,7 +805,9 @@ async fn handle_start_inner(
if let Some(text) = maybe_routed_text {
debug!(text = %text, "[realtime-text] realtime conversation text output");
let sess_for_routed_text = Arc::clone(&sess_clone);
sess_for_routed_text.route_realtime_text_input(text).await;
sess_for_routed_text
.route_realtime_text_input(wrap_realtime_delegation_input(&text))
.await;
}
if !fanout_realtime_active.load(Ordering::Relaxed) {
break;
@@ -867,6 +869,20 @@ fn realtime_text_from_handoff_request(handoff: &RealtimeHandoffRequested) -> Opt
.or((!handoff.input_transcript.is_empty()).then_some(handoff.input_transcript.clone()))
}
fn wrap_realtime_delegation_input(input: &str) -> String {
format!(
"<realtime_delegation>\n <input>{}</input>\n</realtime_delegation>",
escape_xml_text(input)
)
}
fn escape_xml_text(input: &str) -> String {
input
.replace('&', "&amp;")
.replace('<', "&lt;")
.replace('>', "&gt;")
}
fn realtime_api_key(auth: Option<&CodexAuth>, provider: &ModelProviderInfo) -> CodexResult<String> {
if let Some(api_key) = provider.api_key()? {
return Ok(api_key);
codex-rs/core/src/realtime_conversation_tests.rs
+17
View File
@@ -1,6 +1,7 @@
use super::RealtimeHandoffState;
use super::RealtimeSessionKind;
use super::realtime_text_from_handoff_request;
use super::wrap_realtime_delegation_input;
use async_channel::bounded;
use codex_protocol::protocol::RealtimeHandoffRequested;
use codex_protocol::protocol::RealtimeTranscriptEntry;
@@ -54,6 +55,22 @@ fn ignores_empty_handoff_request_input_transcript() {
assert_eq!(realtime_text_from_handoff_request(&handoff), None);
}
#[test]
fn wraps_realtime_delegation_input() {
assert_eq!(
wrap_realtime_delegation_input("hello"),
"<realtime_delegation>\n <input>hello</input>\n</realtime_delegation>"
);
}
#[test]
fn wraps_realtime_delegation_input_with_xml_escaping() {
assert_eq!(
wrap_realtime_delegation_input("use a < b && c > d"),
"<realtime_delegation>\n <input>use a &lt; b &amp;&amp; c &gt; d</input>\n</realtime_delegation>"
);
}
#[tokio::test]
async fn clears_active_handoff_explicitly() {
let (tx, _rx) = bounded(1);
codex-rs/core/src/rollout.rs
-4
View File
@@ -46,11 +46,7 @@ impl codex_rollout::RolloutConfigView for Config {
}
pub(crate) mod list {
pub use codex_rollout::ThreadListConfig;
pub use codex_rollout::ThreadListLayout;
pub use codex_rollout::ThreadSortKey;
pub use codex_rollout::find_thread_path_by_id_str;
pub use codex_rollout::get_threads_in_root;
}
pub(crate) mod metadata {
codex-rs/core/src/seatbelt.rs
-48
View File
@@ -1,48 +0,0 @@
#![cfg(target_os = "macos")]
use crate::spawn::CODEX_SANDBOX_ENV_VAR;
use crate::spawn::SpawnChildRequest;
use crate::spawn::StdioPolicy;
use crate::spawn::spawn_child_async;
use codex_network_proxy::NetworkProxy;
use codex_protocol::permissions::FileSystemSandboxPolicy;
use codex_protocol::permissions::NetworkSandboxPolicy;
use codex_protocol::protocol::SandboxPolicy;
use codex_sandboxing::seatbelt::MACOS_PATH_TO_SEATBELT_EXECUTABLE;
use codex_sandboxing::seatbelt::create_seatbelt_command_args_for_policies;
use codex_utils_absolute_path::AbsolutePathBuf;
use std::collections::HashMap;
use std::path::PathBuf;
use tokio::process::Child;
pub async fn spawn_command_under_seatbelt(
command: Vec<String>,
command_cwd: AbsolutePathBuf,
sandbox_policy: &SandboxPolicy,
sandbox_policy_cwd: &AbsolutePathBuf,
stdio_policy: StdioPolicy,
network: Option<&NetworkProxy>,
mut env: HashMap<String, String>,
) -> std::io::Result<Child> {
let args = create_seatbelt_command_args_for_policies(
command,
&FileSystemSandboxPolicy::from_legacy_sandbox_policy(sandbox_policy, sandbox_policy_cwd),
NetworkSandboxPolicy::from(sandbox_policy),
sandbox_policy_cwd,
/*enforce_managed_network*/ false,
network,
);
let arg0 = None;
env.insert(CODEX_SANDBOX_ENV_VAR.to_string(), "seatbelt".to_string());
spawn_child_async(SpawnChildRequest {
program: PathBuf::from(MACOS_PATH_TO_SEATBELT_EXECUTABLE),
args,
arg0,
cwd: command_cwd,
network_sandbox_policy: NetworkSandboxPolicy::from(sandbox_policy),
network,
stdio_policy,
env,
})
.await
}
codex-rs/core/src/state/service.rs
+4
View File
@@ -4,6 +4,7 @@ use std::sync::Arc;
use crate::RolloutRecorder;
use crate::SkillsManager;
use crate::agent::AgentControl;
use crate::agent_identity::AgentIdentityManager;
use crate::client::ModelClient;
use crate::config::StartedNetworkProxy;
use crate::exec_policy::ExecPolicyManager;
@@ -23,6 +24,7 @@ use codex_mcp::McpConnectionManager;
use codex_models_manager::manager::ModelsManager;
use codex_otel::SessionTelemetry;
use codex_rollout::state_db::StateDbHandle;
use codex_thread_store::LocalThreadStore;
use std::path::PathBuf;
use tokio::sync::Mutex;
use tokio::sync::RwLock;
@@ -41,6 +43,7 @@ pub(crate) struct SessionServices {
pub(crate) hooks: Hooks,
pub(crate) rollout: Mutex<Option<RolloutRecorder>>,
pub(crate) user_shell: Arc<crate::shell::Shell>,
pub(crate) agent_identity_manager: Arc<AgentIdentityManager>,
pub(crate) shell_snapshot_tx: watch::Sender<Option<Arc<crate::shell_snapshot::ShellSnapshot>>>,
pub(crate) show_raw_agent_reasoning: bool,
pub(crate) exec_policy: Arc<ExecPolicyManager>,
@@ -57,6 +60,7 @@ pub(crate) struct SessionServices {
pub(crate) network_proxy: Option<StartedNetworkProxy>,
pub(crate) network_approval: Arc<NetworkApprovalService>,
pub(crate) state_db: Option<StateDbHandle>,
pub(crate) thread_store: LocalThreadStore,
/// Session-scoped model client shared across turns.
pub(crate) model_client: ModelClient,
pub(crate) code_mode_service: CodeModeService,
codex-rs/core/src/tools/code_mode/mod.rs
+29 -28
View File
@@ -116,7 +116,7 @@ struct CoreTurnHost {
impl CodeModeTurnHost for CoreTurnHost {
async fn invoke_tool(
&self,
tool_name: String,
tool_name: ToolName,
input: Option<JsonValue>,
cancellation_token: CancellationToken,
) -> Result<JsonValue, String> {
@@ -288,38 +288,39 @@ async fn build_nested_router(exec: &ExecContext) -> ToolRouter {
async fn call_nested_tool(
exec: ExecContext,
tool_runtime: ToolCallRuntime,
tool_name: String,
tool_name: ToolName,
input: Option<JsonValue>,
cancellation_token: CancellationToken,
) -> Result<JsonValue, FunctionCallError> {
if tool_name == PUBLIC_TOOL_NAME {
if tool_name.namespace.is_none() && tool_name.name == PUBLIC_TOOL_NAME {
return Err(FunctionCallError::RespondToModel(format!(
"{PUBLIC_TOOL_NAME} cannot invoke itself"
)));
}
let payload = if let Some(tool_info) = exec
.session
.resolve_mcp_tool_info(&tool_name, /*namespace*/ None)
.await
{
match serialize_function_tool_arguments(&tool_name, input) {
Ok(raw_arguments) => ToolPayload::Mcp {
server: tool_info.server_name,
tool: tool_info.tool.name.to_string(),
raw_arguments,
},
Err(error) => return Err(FunctionCallError::RespondToModel(error)),
}
} else {
match build_nested_tool_payload(tool_runtime.find_spec(&tool_name), &tool_name, input) {
Ok(payload) => payload,
Err(error) => return Err(FunctionCallError::RespondToModel(error)),
}
};
let (tool_call_name, payload) =
if let Some(tool_info) = exec.session.resolve_mcp_tool_info(&tool_name).await {
let raw_arguments = match serialize_function_tool_arguments(&tool_name, input) {
Ok(raw_arguments) => raw_arguments,
Err(error) => return Err(FunctionCallError::RespondToModel(error)),
};
(
tool_info.canonical_tool_name(),
ToolPayload::Mcp {
server: tool_info.server_name,
tool: tool_info.tool.name.to_string(),
raw_arguments,
},
)
} else {
match build_nested_tool_payload(tool_runtime.find_spec(&tool_name), &tool_name, input) {
Ok(payload) => (tool_name, payload),
Err(error) => return Err(FunctionCallError::RespondToModel(error)),
}
};
let call = ToolCall {
tool_name: ToolName::plain(tool_name.clone()),
tool_name: tool_call_name,
call_id: format!("{PUBLIC_TOOL_NAME}-{}", uuid::Uuid::new_v4()),
payload,
};
@@ -339,7 +340,7 @@ fn tool_kind_for_spec(spec: &ToolSpec) -> codex_code_mode::CodeModeToolKind {
fn tool_kind_for_name(
spec: Option<ToolSpec>,
tool_name: &str,
tool_name: &ToolName,
) -> Result<codex_code_mode::CodeModeToolKind, String> {
spec.as_ref()
.map(tool_kind_for_spec)
@@ -348,7 +349,7 @@ fn tool_kind_for_name(
fn build_nested_tool_payload(
spec: Option<ToolSpec>,
tool_name: &str,
tool_name: &ToolName,
input: Option<JsonValue>,
) -> Result<ToolPayload, String> {
let actual_kind = tool_kind_for_name(spec, tool_name)?;
@@ -363,7 +364,7 @@ fn build_nested_tool_payload(
}
fn build_function_tool_payload(
tool_name: &str,
tool_name: &ToolName,
input: Option<JsonValue>,
) -> Result<ToolPayload, String> {
let arguments = serialize_function_tool_arguments(tool_name, input)?;
@@ -371,7 +372,7 @@ fn build_function_tool_payload(
}
fn serialize_function_tool_arguments(
tool_name: &str,
tool_name: &ToolName,
input: Option<JsonValue>,
) -> Result<String, String> {
match input {
@@ -385,7 +386,7 @@ fn serialize_function_tool_arguments(
}
fn build_freeform_tool_payload(
tool_name: &str,
tool_name: &ToolName,
input: Option<JsonValue>,
) -> Result<ToolPayload, String> {
match input {

Some files were not shown because too many files have changed in this diff Show More