diff --git a/.codex/skills/codex-pr-body/SKILL.md b/.codex/skills/codex-pr-body/SKILL.md new file mode 100644 index 0000000000..76b37b8750 --- /dev/null +++ b/.codex/skills/codex-pr-body/SKILL.md @@ -0,0 +1,59 @@ +--- +name: codex-pr-body +description: Update the title and body of one or more pull requests. +--- + +## Determining the PR(s) + +When this skill is invoked, the PR(s) to update may be specified explicitly, but in the common case, the PR(s) to update will be inferred from the branch / commit that the user is currently working on. For ordinary Git usage (i.e., not Sapling as discussed below), you may have to use a combination of `git branch` and `gh pr view --repo openai/codex --json number --jq '.number'` to determine the PR associated with the current branch / commit. + +## PR Body Contents + +When invoked, use `gh` to edit the pull request body and title to reflect the contents of the specified PR. Make sure to check the existing pull request body to see if there is key information that should be preserved. For example, NEVER remove an image in the existing pull request body, as the author may have no way to recover it if you remove it. + +It is critically important to explain _why_ the change is being made. If the current conversation in which this skill is invoked has discussed the motivation, be sure to capture this in the pull request body. + +The body should also explain _what_ changed, but this should appear after the _why_. + +Limit discussion to the _net change_ of the commit. It is generally frowned upon to discuss changes that were attempted but later undone in the course of the development of the pull request. When rewriting the pull request body, you may need to eliminate details such as these when they are no longer appropriate / of interest to future readers. + +Avoid references to absolute paths on my local disk. When talking about a path that is within the repository, simply use the repo-relative path. + +It is generally helpful to discuss how the change was verified. That said, it is unnecessary to mention things that CI checks automatically, e.g., do not include "ran `just fmt`" as part of the test plan. Though identifying the new tests that were purposely introduced to verify the new behavior introduced by the pull request is often appropriate. + +Make use of Markdown to format the pull request professionally. Ensure "code things" appear in single backticks when referenced inline. Fenced code blocks are useful when referencing code or showing a shell transcript. Also, make use of GitHub permalinks when citing existing pieces of code that are relevant to the change. + +Make sure to reference any relevant pull requests or issues, though there should be no need to reference the pull request in its own PR body. + +If there is documentation that should be updated on https://developers.openai.com/codex as a result of this change, please note that in a separate section near the end of the pull request. Omit this section if there is no documentation that needs to be updated. + +## Working with Stacks + +Sometimes a pull request is composed of a stack of commits that build on one another. In these cases, the PR body should reflect the _net_ change introduced by the stack as a whole, rather than the individual commits that make up the stack. + +Similarly, sometimes a user may be using a tool like Sapling to leverage _stacked pull requests_, in which case the `base` of the PR may be the a branch that is the `head` of another PR in the stack rather than `main`. In this case, be sure to discuss only the net change between the `base` and `head` of the PR that is being opened against that stacked base, rather than the changes relative to `main`. + +## Sapling + +If `.git/sl/store` is present, then this Git repository is governed by Sapling SCM (https://sapling-scm.com). + +In Sapling, run the following to see if there is a GitHub pull request associated with the current revision: + +```shell +sl log --template '{github_pull_request_url}' -r . +``` + +Alternatively, you can run `sl sl` to see the current development branch and whether there is a GitHub pull request associated with the current commit. For example, if the output were: + +``` + @ cb032b31cf 72 minutes ago mbolin #11412 +╭─╯ tui: show non-file layer content in /debug-config +│ +o fdd0cd1de9 Today at 20:09 origin/main +│ +~ +``` + +- `@` indicates the current commit is `cb032b31cf` +- it is a development branch containing a single commit branched off of `origin/main` +- it is associated with GitHub pull request #11412 diff --git a/.github/scripts/run-bazel-ci.sh b/.github/scripts/run-bazel-ci.sh index fefca90b3a..c8c09f406a 100755 --- a/.github/scripts/run-bazel-ci.sh +++ b/.github/scripts/run-bazel-ci.sh @@ -94,6 +94,11 @@ print_bazel_test_log_tails() { local rel_path="${target#//}" rel_path="${rel_path/://}" local test_log="${testlogs_dir}/${rel_path}/test.log" + local reported_test_log + reported_test_log="$(grep -F "FAIL: ${target} " "$console_log" | sed -nE 's#.* \(see ([^)]+/test\.log)\).*#\1#p' | head -n 1 || true)" + if [[ -n "$reported_test_log" ]]; then + test_log="$reported_test_log" + fi echo "::group::Bazel test log tail for ${target}" if [[ -f "$test_log" ]]; then diff --git a/.github/workflows/rust-ci-full.yml b/.github/workflows/rust-ci-full.yml index a39c38857b..7cda682952 100644 --- a/.github/workflows/rust-ci-full.yml +++ b/.github/workflows/rust-ci-full.yml @@ -664,6 +664,7 @@ jobs: export CODEX_TEST_REMOTE_ENV_CONTAINER_NAME=codex-remote-test-env source "${GITHUB_WORKSPACE}/scripts/test-remote-env.sh" echo "CODEX_TEST_REMOTE_ENV=${CODEX_TEST_REMOTE_ENV}" >> "$GITHUB_ENV" + echo "CODEX_TEST_REMOTE_EXEC_SERVER_URL=${CODEX_TEST_REMOTE_EXEC_SERVER_URL}" >> "$GITHUB_ENV" - name: tests id: test diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index c270e6a9fa..ef80693c70 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -819,6 +819,8 @@ "dylint_linting_5.0.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"assert_cmd\",\"req\":\"^2.0\"},{\"name\":\"cargo_metadata\",\"req\":\"^0.23\"},{\"features\":[\"config\"],\"name\":\"dylint_internal\",\"req\":\"=5.0.0\"},{\"name\":\"paste\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rustc_version\",\"req\":\"^0.4\"},{\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.23\"},{\"name\":\"thiserror\",\"req\":\"^2.0\"},{\"name\":\"toml\",\"req\":\"^0.9\"},{\"kind\":\"build\",\"name\":\"toml\",\"req\":\"^0.9\"}],\"features\":{\"constituent\":[]}}", "dylint_testing_5.0.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0\"},{\"name\":\"cargo_metadata\",\"req\":\"^0.23\"},{\"name\":\"compiletest_rs\",\"req\":\"^0.11\"},{\"name\":\"dylint\",\"req\":\"=5.0.0\"},{\"name\":\"dylint_internal\",\"req\":\"=5.0.0\"},{\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"once_cell\",\"req\":\"^1.21\"},{\"name\":\"regex\",\"req\":\"^1.11\"},{\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"tempfile\",\"req\":\"^3.23\"}],\"features\":{\"default\":[],\"deny_warnings\":[]}}", "dyn-clone_1.0.20": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.66\"}],\"features\":{}}", + "ed25519-dalek_2.2.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"blake2\",\"req\":\"^0.10\"},{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"default_features\":false,\"features\":[\"digest\"],\"name\":\"curve25519-dalek\",\"req\":\"^4\"},{\"default_features\":false,\"features\":[\"digest\",\"rand_core\"],\"kind\":\"dev\",\"name\":\"curve25519-dalek\",\"req\":\"^4\"},{\"default_features\":false,\"name\":\"ed25519\",\"req\":\">=2.2, <2.3\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"hex-literal\",\"req\":\"^0.4\"},{\"default_features\":false,\"name\":\"merlin\",\"optional\":true,\"req\":\"^3\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"default_features\":false,\"name\":\"rand_core\",\"optional\":true,\"req\":\"^0.6.4\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"rand_core\",\"req\":\"^0.6.4\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"sha2\",\"req\":\"^0.10\"},{\"kind\":\"dev\",\"name\":\"sha3\",\"req\":\"^0.10\"},{\"default_features\":false,\"name\":\"signature\",\"optional\":true,\"req\":\">=2.0, <2.3\"},{\"default_features\":false,\"name\":\"subtle\",\"req\":\"^2.3.0\"},{\"kind\":\"dev\",\"name\":\"toml\",\"req\":\"^0.7\"},{\"default_features\":false,\"features\":[\"static_secrets\"],\"kind\":\"dev\",\"name\":\"x25519-dalek\",\"req\":\"^2\"},{\"default_features\":false,\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1.5\"}],\"features\":{\"alloc\":[\"curve25519-dalek/alloc\",\"ed25519/alloc\",\"serde?/alloc\",\"zeroize/alloc\"],\"asm\":[\"sha2/asm\"],\"batch\":[\"alloc\",\"merlin\",\"rand_core\"],\"default\":[\"fast\",\"std\",\"zeroize\"],\"digest\":[\"signature/digest\"],\"fast\":[\"curve25519-dalek/precomputed-tables\"],\"hazmat\":[],\"legacy_compatibility\":[\"curve25519-dalek/legacy_compatibility\"],\"pem\":[\"alloc\",\"ed25519/pem\",\"pkcs8\"],\"pkcs8\":[\"ed25519/pkcs8\"],\"rand_core\":[\"dep:rand_core\"],\"serde\":[\"dep:serde\",\"ed25519/serde\"],\"std\":[\"alloc\",\"ed25519/std\",\"serde?/std\",\"sha2/std\"],\"zeroize\":[\"dep:zeroize\",\"curve25519-dalek/zeroize\"]}}", + "ed25519_2.2.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1\"},{\"features\":[\"rand_core\"],\"kind\":\"dev\",\"name\":\"ed25519-dalek\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"hex-literal\",\"req\":\"^0.4\"},{\"name\":\"pkcs8\",\"optional\":true,\"req\":\"^0.10\"},{\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"rand_core\",\"req\":\"^0.6\"},{\"default_features\":false,\"features\":[\"signature\"],\"kind\":\"dev\",\"name\":\"ring-compat\",\"req\":\"^0.8\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"serde_bytes\",\"optional\":true,\"req\":\"^0.11\"},{\"default_features\":false,\"name\":\"signature\",\"req\":\"^2\"},{\"default_features\":false,\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"alloc\":[\"pkcs8?/alloc\"],\"default\":[\"std\"],\"pem\":[\"alloc\",\"pkcs8/pem\"],\"serde_bytes\":[\"serde\",\"dep:serde_bytes\"],\"std\":[\"pkcs8?/std\",\"signature/std\"]}}", "either_1.15.0": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"alloc\",\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.95\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[],\"use_std\":[\"std\"]}}", "ena_0.14.3": "{\"dependencies\":[{\"name\":\"dogged\",\"optional\":true,\"req\":\"^0.2.0\"},{\"name\":\"log\",\"req\":\"^0.4\"}],\"features\":{\"bench\":[],\"persistent\":[\"dogged\"]}}", "encode_unicode_1.0.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"ascii\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.0\",\"target\":\"cfg(unix)\"},{\"features\":[\"https-native\"],\"kind\":\"dev\",\"name\":\"minreq\",\"req\":\"^2.6\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", @@ -1201,9 +1203,11 @@ "process-wrap_9.0.1": "{\"dependencies\":[{\"name\":\"futures\",\"optional\":true,\"req\":\"^0.3.30\"},{\"name\":\"indexmap\",\"req\":\"^2.9.0\"},{\"default_features\":false,\"features\":[\"fs\",\"poll\",\"signal\"],\"name\":\"nix\",\"optional\":true,\"req\":\"^0.30.1\",\"target\":\"cfg(unix)\"},{\"kind\":\"dev\",\"name\":\"remoteprocess\",\"req\":\"^0.5.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.20.0\"},{\"features\":[\"io-util\",\"macros\",\"process\",\"rt\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.38.2\"},{\"features\":[\"io-util\",\"macros\",\"process\",\"rt\",\"rt-multi-thread\",\"time\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.38.2\"},{\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.40\"},{\"name\":\"windows\",\"optional\":true,\"req\":\"^0.62.2\",\"target\":\"cfg(windows)\"}],\"features\":{\"creation-flags\":[\"dep:windows\",\"windows/Win32_System_Threading\"],\"default\":[\"creation-flags\",\"job-object\",\"kill-on-drop\",\"process-group\",\"process-session\",\"tracing\"],\"job-object\":[\"dep:windows\",\"windows/Win32_Security\",\"windows/Win32_System_Diagnostics_ToolHelp\",\"windows/Win32_System_IO\",\"windows/Win32_System_JobObjects\",\"windows/Win32_System_Threading\"],\"kill-on-drop\":[],\"process-group\":[],\"process-session\":[\"process-group\"],\"reset-sigmask\":[],\"std\":[\"dep:nix\"],\"tokio1\":[\"dep:nix\",\"dep:futures\",\"dep:tokio\"],\"tracing\":[\"dep:tracing\"]}}", "proptest_1.9.0": "{\"dependencies\":[{\"name\":\"bit-set\",\"optional\":true,\"req\":\"^0.8.0\"},{\"name\":\"bit-vec\",\"optional\":true,\"req\":\"^0.8.0\"},{\"name\":\"bitflags\",\"req\":\"^2.9\"},{\"default_features\":false,\"name\":\"num-traits\",\"req\":\"^0.2.15\"},{\"name\":\"proptest-macro\",\"optional\":true,\"req\":\"^0.4.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"rand\",\"req\":\"^0.9\"},{\"default_features\":false,\"name\":\"rand_chacha\",\"req\":\"^0.9\"},{\"name\":\"rand_xorshift\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.0\"},{\"name\":\"regex-syntax\",\"optional\":true,\"req\":\"^0.8\"},{\"default_features\":false,\"name\":\"rusty-fork\",\"optional\":true,\"req\":\"^0.3.0\"},{\"name\":\"tempfile\",\"optional\":true,\"req\":\"^3.0\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.112\"},{\"name\":\"unarray\",\"req\":\"^0.1.4\"},{\"name\":\"x86\",\"optional\":true,\"req\":\"^0.52.0\"}],\"features\":{\"alloc\":[],\"atomic64bit\":[],\"attr-macro\":[\"proptest-macro\"],\"bit-set\":[\"dep:bit-set\",\"dep:bit-vec\"],\"default\":[\"std\",\"fork\",\"timeout\",\"bit-set\"],\"default-code-coverage\":[\"std\",\"fork\",\"timeout\",\"bit-set\"],\"fork\":[\"std\",\"rusty-fork\",\"tempfile\"],\"handle-panics\":[\"std\"],\"hardware-rng\":[\"x86\"],\"no_std\":[\"num-traits/libm\"],\"std\":[\"rand/std\",\"rand/os_rng\",\"regex-syntax\",\"num-traits/std\"],\"timeout\":[\"fork\",\"rusty-fork/timeout\"],\"unstable\":[]}}", "prost-build_0.12.6": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.10\"},{\"name\":\"heck\",\"req\":\">=0.4, <=0.5\"},{\"default_features\":false,\"features\":[\"use_alloc\"],\"name\":\"itertools\",\"req\":\">=0.10, <=0.12\"},{\"name\":\"log\",\"req\":\"^0.4.4\"},{\"default_features\":false,\"name\":\"multimap\",\"req\":\">=0.8, <=0.10\"},{\"name\":\"once_cell\",\"req\":\"^1.17.1\"},{\"default_features\":false,\"name\":\"petgraph\",\"req\":\"^0.6\"},{\"name\":\"prettyplease\",\"optional\":true,\"req\":\"^0.2\"},{\"default_features\":false,\"name\":\"prost\",\"req\":\"^0.12.6\"},{\"default_features\":false,\"name\":\"prost-types\",\"req\":\"^0.12.6\"},{\"default_features\":false,\"name\":\"pulldown-cmark\",\"optional\":true,\"req\":\"^0.9.1\"},{\"name\":\"pulldown-cmark-to-cmark\",\"optional\":true,\"req\":\"^10.0.1\"},{\"default_features\":false,\"features\":[\"std\",\"unicode-bool\"],\"name\":\"regex\",\"req\":\"^1.8.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"optional\":true,\"req\":\"^2\"},{\"name\":\"tempfile\",\"req\":\"^3\"}],\"features\":{\"cleanup-markdown\":[\"dep:pulldown-cmark\",\"dep:pulldown-cmark-to-cmark\"],\"default\":[\"format\"],\"format\":[\"dep:prettyplease\",\"dep:syn\"]}}", + "prost-build_0.14.3": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"heck\",\"req\":\">=0.4, <=0.5\"},{\"default_features\":false,\"features\":[\"use_alloc\"],\"name\":\"itertools\",\"req\":\">=0.10, <=0.14\"},{\"name\":\"log\",\"req\":\"^0.4.4\"},{\"default_features\":false,\"name\":\"multimap\",\"req\":\">=0.8, <=0.10\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"petgraph\",\"req\":\"^0.8\"},{\"name\":\"prettyplease\",\"optional\":true,\"req\":\"^0.2\"},{\"default_features\":false,\"name\":\"prost\",\"req\":\"^0.14.3\"},{\"default_features\":false,\"name\":\"prost-types\",\"req\":\"^0.14.3\"},{\"default_features\":false,\"name\":\"pulldown-cmark\",\"optional\":true,\"req\":\"^0.13\"},{\"name\":\"pulldown-cmark-to-cmark\",\"optional\":true,\"req\":\"^22\"},{\"default_features\":false,\"features\":[\"std\",\"unicode-bool\"],\"name\":\"regex\",\"req\":\"^1.8.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"optional\":true,\"req\":\"^2\"},{\"name\":\"tempfile\",\"req\":\"^3\"}],\"features\":{\"cleanup-markdown\":[\"dep:pulldown-cmark\",\"dep:pulldown-cmark-to-cmark\"],\"default\":[\"format\"],\"format\":[\"dep:prettyplease\",\"dep:syn\"]}}", "prost-derive_0.12.6": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.1\"},{\"default_features\":false,\"features\":[\"use_alloc\"],\"name\":\"itertools\",\"req\":\">=0.10, <=0.12\"},{\"name\":\"proc-macro2\",\"req\":\"^1\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2\"}],\"features\":{}}", "prost-derive_0.14.3": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.1\"},{\"name\":\"itertools\",\"req\":\">=0.10.1, <=0.14\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2\"}],\"features\":{}}", "prost-types_0.12.6": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"prost-derive\"],\"name\":\"prost\",\"req\":\"^0.12.6\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"prost/std\"]}}", + "prost-types_0.14.3": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.4\"},{\"default_features\":false,\"name\":\"chrono\",\"optional\":true,\"req\":\"^0.4.34\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"derive\"],\"name\":\"prost\",\"req\":\"^0.14.3\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"default\":[\"std\"],\"std\":[\"prost/std\"]}}", "prost_0.12.6": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"name\":\"prost-derive\",\"optional\":true,\"req\":\"^0.12.6\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"derive\",\"std\"],\"derive\":[\"dep:prost-derive\"],\"no-recursion-limit\":[],\"prost-derive\":[\"derive\"],\"std\":[]}}", "prost_0.14.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.7\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"name\":\"prost-derive\",\"optional\":true,\"req\":\"^0.14.3\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9\"}],\"features\":{\"default\":[\"derive\",\"std\"],\"derive\":[\"dep:prost-derive\"],\"no-recursion-limit\":[],\"std\":[]}}", "psl-types_2.0.11": "{\"dependencies\":[],\"features\":{}}", @@ -1239,7 +1243,7 @@ "rama-unix_0.3.0-alpha.4": "{\"dependencies\":[{\"name\":\"pin-project-lite\",\"req\":\"^0.2\"},{\"name\":\"rama-core\",\"req\":\"^0.3.0-alpha.4\"},{\"name\":\"rama-net\",\"req\":\"^0.3.0-alpha.4\"},{\"features\":[\"macros\",\"net\"],\"name\":\"tokio\",\"req\":\"^1.48\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.48\"}],\"features\":{\"default\":[]}}", "rama-utils_0.3.0-alpha.4": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"ahash\",\"req\":\"^0.8\"},{\"name\":\"const_format\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"loom\",\"req\":\"^0.7\",\"target\":\"cfg(loom)\"},{\"name\":\"parking_lot\",\"req\":\"^0.12\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"name\":\"rama-macros\",\"req\":\"^0.3.0-alpha.4\"},{\"name\":\"regex\",\"req\":\"^1.12\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1\"},{\"features\":[\"write\",\"serde\",\"const_generics\",\"const_new\"],\"name\":\"smallvec\",\"req\":\"^1.15\"},{\"name\":\"smol_str\",\"req\":\"^0.3\"},{\"features\":[\"time\",\"macros\"],\"name\":\"tokio\",\"req\":\"^1.48\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0\"},{\"name\":\"wildcard\",\"req\":\"^0.3\"}],\"features\":{}}", "rand_0.8.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.2.1\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.22\",\"target\":\"cfg(unix)\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.4\"},{\"features\":[\"into_bits\"],\"name\":\"packed_simd\",\"optional\":true,\"package\":\"packed_simd_2\",\"req\":\"^0.3.7\"},{\"default_features\":false,\"name\":\"rand_chacha\",\"optional\":true,\"req\":\"^0.3.0\"},{\"name\":\"rand_core\",\"req\":\"^0.6.0\"},{\"kind\":\"dev\",\"name\":\"rand_pcg\",\"req\":\"^0.3.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"}],\"features\":{\"alloc\":[\"rand_core/alloc\"],\"default\":[\"std\",\"std_rng\"],\"getrandom\":[\"rand_core/getrandom\"],\"min_const_gen\":[],\"nightly\":[],\"serde1\":[\"serde\",\"rand_core/serde1\"],\"simd_support\":[\"packed_simd\"],\"small_rng\":[],\"std\":[\"rand_core/std\",\"rand_chacha/std\",\"alloc\",\"getrandom\",\"libc\"],\"std_rng\":[\"rand_chacha\"]}}", - "rand_0.9.2": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.2.1\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.4\"},{\"default_features\":false,\"name\":\"rand_chacha\",\"optional\":true,\"req\":\"^0.9.0\"},{\"default_features\":false,\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rand_pcg\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.7\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.140\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"std_rng\",\"os_rng\",\"small_rng\",\"thread_rng\"],\"log\":[\"dep:log\"],\"nightly\":[],\"os_rng\":[\"rand_core/os_rng\"],\"serde\":[\"dep:serde\",\"rand_core/serde\"],\"simd_support\":[],\"small_rng\":[],\"std\":[\"rand_core/std\",\"rand_chacha?/std\",\"alloc\"],\"std_rng\":[\"dep:rand_chacha\"],\"thread_rng\":[\"std\",\"std_rng\",\"os_rng\"],\"unbiased\":[]}}", + "rand_0.9.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.2.1\"},{\"default_features\":false,\"name\":\"rand_chacha\",\"optional\":true,\"req\":\"^0.9.0\"},{\"default_features\":false,\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rand_pcg\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.7\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.140\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"std_rng\",\"os_rng\",\"small_rng\",\"thread_rng\"],\"log\":[],\"nightly\":[],\"os_rng\":[\"rand_core/os_rng\"],\"serde\":[\"dep:serde\",\"rand_core/serde\"],\"simd_support\":[],\"small_rng\":[],\"std\":[\"rand_core/std\",\"rand_chacha?/std\",\"alloc\"],\"std_rng\":[\"dep:rand_chacha\"],\"thread_rng\":[\"std\",\"std_rng\",\"os_rng\"],\"unbiased\":[]}}", "rand_chacha_0.3.1": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"ppv-lite86\",\"req\":\"^0.2.8\"},{\"name\":\"rand_core\",\"req\":\"^0.6.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"serde1\":[\"serde\"],\"simd\":[],\"std\":[\"ppv-lite86/std\"]}}", "rand_chacha_0.9.0": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"ppv-lite86\",\"req\":\"^0.2.14\"},{\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"features\":[\"os_rng\"],\"kind\":\"dev\",\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"os_rng\":[\"rand_core/os_rng\"],\"serde\":[\"dep:serde\"],\"std\":[\"ppv-lite86/std\",\"rand_core/std\"]}}", "rand_core_0.6.4": "{\"dependencies\":[{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.2\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"serde1\":[\"serde\"],\"std\":[\"alloc\",\"getrandom\",\"getrandom/std\"]}}", @@ -1287,7 +1291,7 @@ "rustix_1.1.4": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bitflags\",\"req\":\"^2.4.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4\",\"target\":\"cfg(all(criterion, not(any(target_os = \\\"emscripten\\\", target_os = \\\"wasi\\\"))))\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.182\",\"target\":\"cfg(all(not(windows), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.182\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.171\"},{\"default_features\":false,\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(all(not(windows), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(windows)\"},{\"default_features\":false,\"name\":\"libc_errno\",\"optional\":true,\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\"},{\"default_features\":false,\"features\":[\"general\",\"ioctl\",\"no_std\"],\"name\":\"linux-raw-sys\",\"req\":\"^0.12\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"features\":[\"auxvec\",\"general\",\"errno\",\"ioctl\",\"no_std\",\"elf\"],\"name\":\"linux-raw-sys\",\"req\":\"^0.12\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"kind\":\"dev\",\"name\":\"memoffset\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.20.3\",\"target\":\"cfg(windows)\"},{\"name\":\"rustc-std-workspace-alloc\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"serial_test\",\"req\":\"^2.0.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.5.0\"},{\"features\":[\"Win32_Foundation\",\"Win32_Networking_WinSock\"],\"name\":\"windows-sys\",\"req\":\">=0.52, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"all-apis\":[\"event\",\"fs\",\"io_uring\",\"mm\",\"mount\",\"net\",\"param\",\"pipe\",\"process\",\"pty\",\"rand\",\"runtime\",\"shm\",\"stdio\",\"system\",\"termios\",\"thread\",\"time\"],\"alloc\":[],\"default\":[\"std\"],\"event\":[],\"fs\":[],\"io_uring\":[\"event\",\"fs\",\"net\",\"thread\",\"linux-raw-sys/io_uring\"],\"linux_4_11\":[],\"linux_5_1\":[\"linux_4_11\"],\"linux_5_11\":[\"linux_5_1\"],\"linux_latest\":[\"linux_5_11\"],\"mm\":[],\"mount\":[],\"net\":[\"linux-raw-sys/net\",\"linux-raw-sys/netlink\",\"linux-raw-sys/if_ether\",\"linux-raw-sys/xdp\"],\"param\":[],\"pipe\":[],\"process\":[\"linux-raw-sys/prctl\"],\"pty\":[\"fs\"],\"rand\":[],\"runtime\":[\"linux-raw-sys/prctl\"],\"rustc-dep-of-std\":[\"core\",\"rustc-std-workspace-alloc\",\"linux-raw-sys/rustc-dep-of-std\",\"bitflags/rustc-dep-of-std\"],\"shm\":[\"fs\"],\"std\":[\"bitflags/std\",\"alloc\",\"libc?/std\",\"libc_errno?/std\"],\"stdio\":[],\"system\":[\"linux-raw-sys/system\"],\"termios\":[],\"thread\":[\"linux-raw-sys/prctl\"],\"time\":[],\"try_close\":[],\"use-explicitly-provided-auxv\":[],\"use-libc\":[\"libc_errno\",\"libc\"],\"use-libc-auxv\":[]}}", "rustls-native-certs_0.8.3": "{\"dependencies\":[{\"name\":\"openssl-probe\",\"req\":\"^0.2\",\"target\":\"cfg(all(unix, not(target_os = \\\"macos\\\")))\"},{\"features\":[\"std\"],\"name\":\"pki-types\",\"package\":\"rustls-pki-types\",\"req\":\"^1.10\"},{\"kind\":\"dev\",\"name\":\"ring\",\"req\":\"^0.17\"},{\"kind\":\"dev\",\"name\":\"rustls\",\"req\":\"^0.23\"},{\"kind\":\"dev\",\"name\":\"rustls-webpki\",\"req\":\"^0.103\"},{\"name\":\"schannel\",\"req\":\"^0.1\",\"target\":\"cfg(windows)\"},{\"name\":\"security-framework\",\"req\":\"^3\",\"target\":\"cfg(target_os = \\\"macos\\\")\"},{\"kind\":\"dev\",\"name\":\"serial_test\",\"req\":\"^3\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.5\"},{\"kind\":\"dev\",\"name\":\"untrusted\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"webpki-roots\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"x509-parser\",\"req\":\"^0.18\"}],\"features\":{}}", "rustls-pki-types_1.14.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"crabgrind\",\"req\":\"=0.1.9\",\"target\":\"cfg(all(target_os = \\\"linux\\\", target_arch = \\\"x86_64\\\"))\"},{\"name\":\"web-time\",\"optional\":true,\"req\":\"^1\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", target_os = \\\"unknown\\\"))\"},{\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"alloc\":[\"dep:zeroize\"],\"default\":[\"alloc\"],\"std\":[\"alloc\"],\"web\":[\"web-time\"]}}", - "rustls-webpki_0.103.10": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aws-lc-rs\",\"optional\":true,\"req\":\"^1.14\"},{\"kind\":\"dev\",\"name\":\"base64\",\"req\":\"^0.22\"},{\"kind\":\"dev\",\"name\":\"bencher\",\"req\":\"^0.1.5\"},{\"kind\":\"dev\",\"name\":\"bzip2\",\"req\":\"^0.6\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.17.2\"},{\"default_features\":false,\"name\":\"pki-types\",\"package\":\"rustls-pki-types\",\"req\":\"^1.12\"},{\"default_features\":false,\"features\":[\"aws_lc_rs\"],\"kind\":\"dev\",\"name\":\"rcgen\",\"req\":\"^0.14.2\"},{\"default_features\":false,\"name\":\"ring\",\"optional\":true,\"req\":\"^0.17\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"untrusted\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"x509-parser\",\"req\":\"^0.18.1\"}],\"features\":{\"alloc\":[\"ring?/alloc\",\"pki-types/alloc\"],\"aws-lc-rs\":[\"dep:aws-lc-rs\",\"aws-lc-rs/aws-lc-sys\",\"aws-lc-rs/prebuilt-nasm\"],\"aws-lc-rs-fips\":[\"dep:aws-lc-rs\",\"aws-lc-rs/fips\"],\"aws-lc-rs-unstable\":[\"aws-lc-rs\",\"aws-lc-rs/unstable\"],\"default\":[\"std\"],\"ring\":[\"dep:ring\"],\"std\":[\"alloc\",\"pki-types/std\"]}}", + "rustls-webpki_0.103.12": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aws-lc-rs\",\"optional\":true,\"req\":\"^1.14\"},{\"kind\":\"dev\",\"name\":\"base64\",\"req\":\"^0.22\"},{\"kind\":\"dev\",\"name\":\"bencher\",\"req\":\"^0.1.5\"},{\"kind\":\"dev\",\"name\":\"bzip2\",\"req\":\"^0.6\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.17.2\"},{\"default_features\":false,\"name\":\"pki-types\",\"package\":\"rustls-pki-types\",\"req\":\"^1.12\"},{\"default_features\":false,\"features\":[\"aws_lc_rs\"],\"kind\":\"dev\",\"name\":\"rcgen\",\"req\":\"^0.14.2\"},{\"default_features\":false,\"name\":\"ring\",\"optional\":true,\"req\":\"^0.17\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"untrusted\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"x509-parser\",\"req\":\"^0.18.1\"}],\"features\":{\"alloc\":[\"ring?/alloc\",\"pki-types/alloc\"],\"aws-lc-rs\":[\"dep:aws-lc-rs\",\"aws-lc-rs/aws-lc-sys\",\"aws-lc-rs/prebuilt-nasm\"],\"aws-lc-rs-fips\":[\"dep:aws-lc-rs\",\"aws-lc-rs/fips\"],\"aws-lc-rs-unstable\":[\"aws-lc-rs\",\"aws-lc-rs/unstable\"],\"default\":[\"std\"],\"ring\":[\"dep:ring\"],\"std\":[\"alloc\",\"pki-types/std\"]}}", "rustls_0.23.36": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aws-lc-rs\",\"optional\":true,\"req\":\"^1.14\"},{\"kind\":\"dev\",\"name\":\"base64\",\"req\":\"^0.22\"},{\"kind\":\"dev\",\"name\":\"bencher\",\"req\":\"^0.1.5\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"brotli\",\"optional\":true,\"req\":\"^8\"},{\"name\":\"brotli-decompressor\",\"optional\":true,\"req\":\"^5.0.0\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"default-hasher\",\"inline-more\"],\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.15\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.8\"},{\"kind\":\"dev\",\"name\":\"log\",\"req\":\"^0.4.8\"},{\"kind\":\"dev\",\"name\":\"macro_rules_attribute\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"num-bigint\",\"req\":\"^0.4.4\"},{\"default_features\":false,\"features\":[\"alloc\",\"race\"],\"name\":\"once_cell\",\"req\":\"^1.16\"},{\"features\":[\"alloc\"],\"name\":\"pki-types\",\"package\":\"rustls-pki-types\",\"req\":\"^1.12\"},{\"default_features\":false,\"features\":[\"pem\",\"aws_lc_rs\"],\"kind\":\"dev\",\"name\":\"rcgen\",\"req\":\"^0.14\"},{\"name\":\"ring\",\"optional\":true,\"req\":\"^0.17\"},{\"kind\":\"build\",\"name\":\"rustversion\",\"optional\":true,\"req\":\"^1.0.6\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"subtle\",\"req\":\"^2.5.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"time\",\"req\":\"^0.3.6\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"webpki\",\"package\":\"rustls-webpki\",\"req\":\"^0.103.5\"},{\"kind\":\"dev\",\"name\":\"webpki-roots\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"x509-parser\",\"req\":\"^0.17\"},{\"name\":\"zeroize\",\"req\":\"^1.8\"},{\"name\":\"zlib-rs\",\"optional\":true,\"req\":\"^0.5\"}],\"features\":{\"aws-lc-rs\":[\"aws_lc_rs\"],\"aws_lc_rs\":[\"dep:aws-lc-rs\",\"webpki/aws-lc-rs\",\"aws-lc-rs/aws-lc-sys\",\"aws-lc-rs/prebuilt-nasm\"],\"brotli\":[\"dep:brotli\",\"dep:brotli-decompressor\",\"std\"],\"custom-provider\":[],\"default\":[\"aws_lc_rs\",\"logging\",\"prefer-post-quantum\",\"std\",\"tls12\"],\"fips\":[\"aws_lc_rs\",\"aws-lc-rs?/fips\",\"webpki/aws-lc-rs-fips\"],\"logging\":[\"log\"],\"prefer-post-quantum\":[\"aws_lc_rs\"],\"read_buf\":[\"rustversion\",\"std\"],\"ring\":[\"dep:ring\",\"webpki/ring\"],\"std\":[\"webpki/std\",\"pki-types/std\",\"once_cell/std\"],\"tls12\":[],\"zlib\":[\"dep:zlib-rs\"]}}", "rustversion_1.0.22": "{\"dependencies\":[{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", "rustyline_14.0.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"assert_matches\",\"req\":\"^1.2\"},{\"name\":\"bitflags\",\"req\":\"^2.0\"},{\"default_features\":false,\"name\":\"buffer-redux\",\"optional\":true,\"req\":\"^1.0\",\"target\":\"cfg(unix)\"},{\"name\":\"cfg-if\",\"req\":\"^1.0\"},{\"name\":\"clipboard-win\",\"req\":\"^5.0\",\"target\":\"cfg(windows)\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"fd-lock\",\"optional\":true,\"req\":\"^4.0.0\"},{\"name\":\"home\",\"optional\":true,\"req\":\"^0.5.4\"},{\"name\":\"libc\",\"req\":\"^0.2\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"memchr\",\"req\":\"^2.0\"},{\"default_features\":false,\"features\":[\"fs\",\"ioctl\",\"poll\",\"signal\",\"term\"],\"name\":\"nix\",\"req\":\"^0.28\",\"target\":\"cfg(unix)\"},{\"name\":\"radix_trie\",\"optional\":true,\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"name\":\"regex\",\"optional\":true,\"req\":\"^1.5.5\"},{\"default_features\":false,\"features\":[\"bundled\",\"backup\"],\"name\":\"rusqlite\",\"optional\":true,\"req\":\"^0.31.0\"},{\"name\":\"rustyline-derive\",\"optional\":true,\"req\":\"^0.10.0\"},{\"default_features\":false,\"name\":\"signal-hook\",\"optional\":true,\"req\":\"^0.3\",\"target\":\"cfg(unix)\"},{\"default_features\":false,\"name\":\"skim\",\"optional\":true,\"req\":\"^0.10\",\"target\":\"cfg(unix)\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.1.0\"},{\"name\":\"termios\",\"optional\":true,\"req\":\"^0.3.3\",\"target\":\"cfg(unix)\"},{\"name\":\"unicode-segmentation\",\"req\":\"^1.0\"},{\"name\":\"unicode-width\",\"req\":\"^0.1\"},{\"name\":\"utf8parse\",\"req\":\"^0.2\",\"target\":\"cfg(unix)\"},{\"features\":[\"Win32_Foundation\",\"Win32_System_Console\",\"Win32_Security\",\"Win32_System_Threading\",\"Win32_UI_Input_KeyboardAndMouse\"],\"name\":\"windows-sys\",\"req\":\"^0.52.0\",\"target\":\"cfg(windows)\"}],\"features\":{\"case_insensitive_history_search\":[\"regex\"],\"custom-bindings\":[\"radix_trie\"],\"default\":[\"custom-bindings\",\"with-dirs\",\"with-file-history\"],\"derive\":[\"rustyline-derive\"],\"with-dirs\":[\"home\"],\"with-file-history\":[\"fd-lock\"],\"with-fuzzy\":[\"skim\"],\"with-sqlite-history\":[\"rusqlite\"]}}", @@ -1457,6 +1461,8 @@ "toml_parser_1.0.6+spec-1.1.0": "{\"dependencies\":[{\"name\":\"anstream\",\"optional\":true,\"req\":\"^0.6.20\"},{\"features\":[\"test\"],\"kind\":\"dev\",\"name\":\"anstream\",\"req\":\"^0.6.20\"},{\"name\":\"anstyle\",\"optional\":true,\"req\":\"^1.0.11\"},{\"kind\":\"dev\",\"name\":\"snapbox\",\"req\":\"^0.6.21\"},{\"default_features\":false,\"name\":\"winnow\",\"req\":\"^0.7.13\"}],\"features\":{\"alloc\":[],\"debug\":[\"std\",\"dep:anstream\",\"dep:anstyle\"],\"default\":[\"std\"],\"simd\":[\"winnow/simd\"],\"std\":[\"alloc\"],\"unsafe\":[]}}", "toml_parser_1.0.9+spec-1.1.0": "{\"dependencies\":[{\"name\":\"anstream\",\"optional\":true,\"req\":\"^0.6.20\"},{\"features\":[\"test\"],\"kind\":\"dev\",\"name\":\"anstream\",\"req\":\"^0.6.20\"},{\"name\":\"anstyle\",\"optional\":true,\"req\":\"^1.0.11\"},{\"kind\":\"dev\",\"name\":\"snapbox\",\"req\":\"^0.6.21\"},{\"default_features\":false,\"name\":\"winnow\",\"req\":\"^0.7.13\"}],\"features\":{\"alloc\":[],\"debug\":[\"std\",\"dep:anstream\",\"dep:anstyle\"],\"default\":[\"std\"],\"simd\":[\"winnow/simd\"],\"std\":[\"alloc\"],\"unsafe\":[]}}", "toml_writer_1.0.6+spec-1.1.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.7.0\"},{\"kind\":\"dev\",\"name\":\"snapbox\",\"req\":\"^0.6.21\"},{\"kind\":\"dev\",\"name\":\"toml_old\",\"package\":\"toml\",\"req\":\"^0.5.11\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", + "tonic-build_0.14.3": "{\"dependencies\":[{\"name\":\"prettyplease\",\"req\":\"^0.2\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"name\":\"quote\",\"req\":\"^1.0\"},{\"name\":\"syn\",\"req\":\"^2.0\"}],\"features\":{\"default\":[\"transport\"],\"transport\":[]}}", + "tonic-prost-build_0.14.3": "{\"dependencies\":[{\"name\":\"prettyplease\",\"req\":\"^0.2\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"name\":\"prost-build\",\"req\":\"^0.14\"},{\"name\":\"prost-types\",\"req\":\"^0.14\"},{\"name\":\"quote\",\"req\":\"^1.0\"},{\"name\":\"syn\",\"req\":\"^2.0\"},{\"name\":\"tempfile\",\"req\":\"^3.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"tonic\",\"req\":\"^0.14.0\"},{\"default_features\":false,\"name\":\"tonic-build\",\"req\":\"^0.14.0\"}],\"features\":{\"cleanup-markdown\":[\"prost-build/cleanup-markdown\"],\"default\":[\"transport\",\"cleanup-markdown\"],\"transport\":[\"tonic-build/transport\"]}}", "tonic-prost_0.14.3": "{\"dependencies\":[{\"name\":\"bytes\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"http-body\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"http-body-util\",\"req\":\"^0.1\"},{\"name\":\"prost\",\"req\":\"^0.14\"},{\"features\":[\"macros\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"default_features\":false,\"name\":\"tonic\",\"req\":\"^0.14.0\"}],\"features\":{}}", "tonic_0.14.3": "{\"dependencies\":[{\"name\":\"async-trait\",\"optional\":true,\"req\":\"^0.1.13\"},{\"default_features\":false,\"name\":\"axum\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"base64\",\"req\":\"^0.22\"},{\"kind\":\"dev\",\"name\":\"bencher\",\"req\":\"^0.1.5\"},{\"name\":\"bytes\",\"req\":\"^1.0\"},{\"name\":\"flate2\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"h2\",\"optional\":true,\"req\":\"^0.4\"},{\"name\":\"http\",\"req\":\"^1.1.0\"},{\"name\":\"http-body\",\"req\":\"^1\"},{\"name\":\"http-body-util\",\"req\":\"^0.1\"},{\"features\":[\"http1\",\"http2\"],\"name\":\"hyper\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"hyper-timeout\",\"optional\":true,\"req\":\"^0.5\"},{\"features\":[\"tokio\"],\"name\":\"hyper-util\",\"optional\":true,\"req\":\"^0.1.11\"},{\"name\":\"percent-encoding\",\"req\":\"^2.1\"},{\"name\":\"pin-project\",\"req\":\"^1.0.11\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1.0\"},{\"name\":\"rustls-native-certs\",\"optional\":true,\"req\":\"^0.8\"},{\"features\":[\"all\"],\"name\":\"socket2\",\"optional\":true,\"req\":\"^0.6\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.0\"},{\"name\":\"sync_wrapper\",\"req\":\"^1.0.2\"},{\"default_features\":false,\"name\":\"tokio\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"rt-multi-thread\",\"macros\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"logging\",\"tls12\"],\"name\":\"tokio-rustls\",\"optional\":true,\"req\":\"^0.26.1\"},{\"default_features\":false,\"name\":\"tokio-stream\",\"req\":\"^0.1.16\"},{\"default_features\":false,\"name\":\"tower\",\"optional\":true,\"req\":\"^0.5\"},{\"features\":[\"load-shed\",\"timeout\"],\"kind\":\"dev\",\"name\":\"tower\",\"req\":\"^0.5\"},{\"name\":\"tower-layer\",\"req\":\"^0.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"name\":\"tracing\",\"req\":\"^0.1\"},{\"name\":\"webpki-roots\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"zstd\",\"optional\":true,\"req\":\"^0.13.0\"}],\"features\":{\"_tls-any\":[\"dep:tokio\",\"tokio?/rt\",\"tokio?/macros\",\"tls-connect-info\"],\"channel\":[\"dep:hyper\",\"hyper?/client\",\"dep:hyper-util\",\"hyper-util?/client-legacy\",\"dep:tower\",\"tower?/balance\",\"tower?/buffer\",\"tower?/discover\",\"tower?/limit\",\"tower?/load-shed\",\"tower?/util\",\"dep:tokio\",\"tokio?/time\",\"dep:hyper-timeout\"],\"codegen\":[\"dep:async-trait\"],\"default\":[\"router\",\"transport\",\"codegen\"],\"deflate\":[\"dep:flate2\"],\"gzip\":[\"dep:flate2\"],\"router\":[\"dep:axum\",\"dep:tower\",\"tower?/util\"],\"server\":[\"dep:h2\",\"dep:hyper\",\"hyper?/server\",\"dep:hyper-util\",\"hyper-util?/service\",\"hyper-util?/server-auto\",\"dep:socket2\",\"dep:tokio\",\"tokio?/macros\",\"tokio?/net\",\"tokio?/time\",\"tokio-stream/net\",\"dep:tower\",\"tower?/util\",\"tower?/limit\",\"tower?/load-shed\"],\"tls-aws-lc\":[\"_tls-any\",\"tokio-rustls/aws-lc-rs\"],\"tls-connect-info\":[\"dep:tokio-rustls\"],\"tls-native-roots\":[\"_tls-any\",\"channel\",\"dep:rustls-native-certs\"],\"tls-ring\":[\"_tls-any\",\"tokio-rustls/ring\"],\"tls-webpki-roots\":[\"_tls-any\",\"channel\",\"dep:webpki-roots\"],\"transport\":[\"server\",\"channel\"],\"zstd\":[\"dep:zstd\"]}}", "tower-http_0.6.8": "{\"dependencies\":[{\"features\":[\"tokio\"],\"name\":\"async-compression\",\"optional\":true,\"req\":\"^0.4\"},{\"name\":\"base64\",\"optional\":true,\"req\":\"^0.22\"},{\"name\":\"bitflags\",\"req\":\"^2.0.2\"},{\"kind\":\"dev\",\"name\":\"brotli\",\"req\":\"^8\"},{\"name\":\"bytes\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"bytes\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.14\"},{\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3.14\"},{\"name\":\"http\",\"req\":\"^1.0\"},{\"name\":\"http-body\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"http-body\",\"req\":\"^1.0.0\"},{\"name\":\"http-body-util\",\"optional\":true,\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"http-body-util\",\"req\":\"^0.1.0\"},{\"name\":\"http-range-header\",\"optional\":true,\"req\":\"^0.4.0\"},{\"name\":\"httpdate\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"client-legacy\",\"http1\",\"tokio\"],\"kind\":\"dev\",\"name\":\"hyper-util\",\"req\":\"^0.1\"},{\"name\":\"iri-string\",\"optional\":true,\"req\":\"^0.7.0\"},{\"default_features\":false,\"name\":\"mime\",\"optional\":true,\"req\":\"^0.3.17\"},{\"default_features\":false,\"name\":\"mime_guess\",\"optional\":true,\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1\"},{\"name\":\"percent-encoding\",\"optional\":true,\"req\":\"^2.1.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"sync_wrapper\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.6\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"io\"],\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7\"},{\"name\":\"tower\",\"optional\":true,\"req\":\"^0.5\"},{\"features\":[\"buffer\",\"util\",\"retry\",\"make\",\"timeout\"],\"kind\":\"dev\",\"name\":\"tower\",\"req\":\"^0.5\"},{\"name\":\"tower-layer\",\"req\":\"^0.3.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"},{\"features\":[\"v4\"],\"name\":\"uuid\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"zstd\",\"req\":\"^0.13\"}],\"features\":{\"add-extension\":[],\"auth\":[\"base64\",\"validate-request\"],\"catch-panic\":[\"tracing\",\"futures-util/std\",\"dep:http-body\",\"dep:http-body-util\"],\"compression-br\":[\"async-compression/brotli\",\"futures-core\",\"dep:http-body\",\"tokio-util\",\"tokio\"],\"compression-deflate\":[\"async-compression/zlib\",\"futures-core\",\"dep:http-body\",\"tokio-util\",\"tokio\"],\"compression-full\":[\"compression-br\",\"compression-deflate\",\"compression-gzip\",\"compression-zstd\"],\"compression-gzip\":[\"async-compression/gzip\",\"futures-core\",\"dep:http-body\",\"tokio-util\",\"tokio\"],\"compression-zstd\":[\"async-compression/zstd\",\"futures-core\",\"dep:http-body\",\"tokio-util\",\"tokio\"],\"cors\":[],\"decompression-br\":[\"async-compression/brotli\",\"futures-core\",\"dep:http-body\",\"dep:http-body-util\",\"tokio-util\",\"tokio\"],\"decompression-deflate\":[\"async-compression/zlib\",\"futures-core\",\"dep:http-body\",\"dep:http-body-util\",\"tokio-util\",\"tokio\"],\"decompression-full\":[\"decompression-br\",\"decompression-deflate\",\"decompression-gzip\",\"decompression-zstd\"],\"decompression-gzip\":[\"async-compression/gzip\",\"futures-core\",\"dep:http-body\",\"dep:http-body-util\",\"tokio-util\",\"tokio\"],\"decompression-zstd\":[\"async-compression/zstd\",\"futures-core\",\"dep:http-body\",\"dep:http-body-util\",\"tokio-util\",\"tokio\"],\"default\":[],\"follow-redirect\":[\"futures-util\",\"dep:http-body\",\"iri-string\",\"tower/util\"],\"fs\":[\"futures-core\",\"futures-util\",\"dep:http-body\",\"dep:http-body-util\",\"tokio/fs\",\"tokio-util/io\",\"tokio/io-util\",\"dep:http-range-header\",\"mime_guess\",\"mime\",\"percent-encoding\",\"httpdate\",\"set-status\",\"futures-util/alloc\",\"tracing\"],\"full\":[\"add-extension\",\"auth\",\"catch-panic\",\"compression-full\",\"cors\",\"decompression-full\",\"follow-redirect\",\"fs\",\"limit\",\"map-request-body\",\"map-response-body\",\"metrics\",\"normalize-path\",\"propagate-header\",\"redirect\",\"request-id\",\"sensitive-headers\",\"set-header\",\"set-status\",\"timeout\",\"trace\",\"util\",\"validate-request\"],\"limit\":[\"dep:http-body\",\"dep:http-body-util\"],\"map-request-body\":[],\"map-response-body\":[],\"metrics\":[\"dep:http-body\",\"tokio/time\"],\"normalize-path\":[],\"propagate-header\":[],\"redirect\":[],\"request-id\":[\"uuid\"],\"sensitive-headers\":[],\"set-header\":[],\"set-status\":[],\"timeout\":[\"dep:http-body\",\"tokio/time\"],\"trace\":[\"dep:http-body\",\"tracing\"],\"util\":[\"tower\"],\"validate-request\":[\"mime\"]}}", diff --git a/SECURITY.md b/SECURITY.md index d6dd568910..29a9b903df 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -11,3 +11,7 @@ Our security program is managed through Bugcrowd, and we ask that any validated ## Vulnerability Disclosure Program Our Vulnerability Program Guidelines are defined on our [Bugcrowd program page](https://bugcrowd.com/engagements/openai). + +## How to operate CODEX safely + +For details on Codex security boundaries, including sandboxing, approvals, and network controls, see [Agent approvals & security](https://developers.openai.com/codex/agent-approvals-security). diff --git a/announcement_tip.toml b/announcement_tip.toml index 2473258368..79c0daea76 100644 --- a/announcement_tip.toml +++ b/announcement_tip.toml @@ -4,20 +4,14 @@ # version_regex matches against the CLI version (env!("CARGO_PKG_VERSION")); omit to apply to all versions. # target_app specify which app should display the announcement (cli, vsce, ...). -[[announcements]] -content = "Welcome to Codex! Check out the new onboarding flow." -from_date = "2024-10-01" -to_date = "2024-10-15" -target_app = "cli" - -# Test announcement only for local build version until 2026-01-10 excluded (past) +# Test announcement only for local build version until 2027-05-10 excluded [[announcements]] content = "This is a test announcement" version_regex = "^0\\.0\\.0$" -to_date = "2026-05-10" +to_date = "2027-05-10" [[announcements]] -content = "**BREAKING NEWS**: `gpt-5.3-codex` is out! Upgrade to `0.98.0` for a faster, smarter, more steerable agent." -from_date = "2026-02-01" -to_date = "2026-02-16" -version_regex = "^0\\.(?:[0-9]|[1-8][0-9]|9[0-7])\\." +content = "Update Required - This version will no longer be supported starting May 8th. Please upgrade to the latest version (https://github.com/openai/codex/releases/latest) using your preferred package manager." +# Matches 0.x.y versions from 0.0.y through 0.119.y; excludes 0.120.0 and newer. +version_regex = "^0\\.(?:[0-9]|[1-9][0-9]|1[01][0-9])\\." +to_date = "2026-05-08" diff --git a/codex-rs/.cargo/audit.toml b/codex-rs/.cargo/audit.toml index 425997f6ae..3760d28648 100644 --- a/codex-rs/.cargo/audit.toml +++ b/codex-rs/.cargo/audit.toml @@ -1,9 +1,10 @@ [advisories] -# Reviewed 2026-04-11. Keep this list in sync with ../deny.toml. +# Reviewed 2026-04-15. Keep this list in sync with ../deny.toml. ignore = [ "RUSTSEC-2024-0388", # derivative 2.2.0 via starlark; upstream crate is unmaintained "RUSTSEC-2025-0057", # fxhash 0.2.1 via starlark_map; upstream crate is unmaintained "RUSTSEC-2024-0436", # paste 1.0.15 via starlark/ratatui; upstream crate is unmaintained "RUSTSEC-2024-0320", # yaml-rust via syntect; remove when syntect drops or updates it "RUSTSEC-2025-0141", # bincode via syntect; remove when syntect drops or updates it + "RUSTSEC-2026-0097", # rand 0.8.5 via age/codex-secrets and zbus/keyring; remove when transitive deps move to rand >=0.9.3 ] diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index ea79776bdc..296e51725c 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1446,6 +1446,7 @@ dependencies = [ "codex-cloud-requirements", "codex-config", "codex-core", + "codex-core-plugins", "codex-exec-server", "codex-features", "codex-feedback", @@ -1461,6 +1462,7 @@ dependencies = [ "codex-sandboxing", "codex-shell-command", "codex-state", + "codex-thread-store", "codex-tools", "codex-utils-absolute-path", "codex-utils-cargo-bin", @@ -1647,6 +1649,7 @@ version = "0.0.0" dependencies = [ "anyhow", "clap", + "codex-app-server-protocol", "codex-config", "codex-connectors", "codex-core", @@ -1694,6 +1697,7 @@ dependencies = [ "codex-stdio-to-uds", "codex-terminal-detection", "codex-tui", + "codex-utils-absolute-path", "codex-utils-cargo-bin", "codex-utils-cli", "codex-utils-path", @@ -1728,7 +1732,7 @@ dependencies = [ "opentelemetry", "opentelemetry_sdk", "pretty_assertions", - "rand 0.9.2", + "rand 0.9.3", "reqwest", "rustls", "rustls-native-certs", @@ -1830,6 +1834,7 @@ name = "codex-code-mode" version = "0.0.0" dependencies = [ "async-trait", + "codex-protocol", "deno_core_icudata", "pretty_assertions", "serde", @@ -1905,10 +1910,12 @@ dependencies = [ "codex-api", "codex-app-server-protocol", "codex-apply-patch", + "codex-arg0", "codex-async-utils", "codex-code-mode", "codex-config", "codex-connectors", + "codex-core-plugins", "codex-core-skills", "codex-exec-server", "codex-execpolicy", @@ -1935,6 +1942,7 @@ dependencies = [ "codex-state", "codex-terminal-detection", "codex-test-binary-support", + "codex-thread-store", "codex-tools", "codex-utils-absolute-path", "codex-utils-cache", @@ -1956,6 +1964,7 @@ dependencies = [ "ctor 0.6.3", "dirs", "dunce", + "ed25519-dalek", "env-flags", "eventsource-stream", "futures", @@ -1973,7 +1982,7 @@ dependencies = [ "opentelemetry_sdk", "predicates", "pretty_assertions", - "rand 0.9.2", + "rand 0.9.3", "regex-lite", "reqwest", "rmcp", @@ -2007,6 +2016,34 @@ dependencies = [ "zstd 0.13.3", ] +[[package]] +name = "codex-core-plugins" +version = "0.0.0" +dependencies = [ + "chrono", + "codex-app-server-protocol", + "codex-config", + "codex-core-skills", + "codex-exec-server", + "codex-git-utils", + "codex-login", + "codex-plugin", + "codex-protocol", + "codex-utils-absolute-path", + "codex-utils-plugins", + "dirs", + "pretty_assertions", + "reqwest", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.18", + "tokio", + "toml 0.9.11+spec-1.1.0", + "tracing", + "url", +] + [[package]] name = "codex-core-skills" version = "0.0.0" @@ -2249,6 +2286,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "codex-install-context" +version = "0.0.0" +dependencies = [ + "codex-utils-home-dir", + "pretty_assertions", + "tempfile", +] + [[package]] name = "codex-instructions" version = "0.0.0" @@ -2326,7 +2372,7 @@ dependencies = [ "once_cell", "os_info", "pretty_assertions", - "rand 0.9.2", + "rand 0.9.3", "regex-lite", "reqwest", "serde", @@ -2570,6 +2616,7 @@ dependencies = [ "codex-utils-string", "codex-utils-template", "encoding_rs", + "globset", "http 1.4.0", "icu_decimal", "icu_locale_core", @@ -2694,6 +2741,8 @@ dependencies = [ name = "codex-sandboxing" version = "0.0.0" dependencies = [ + "anyhow", + "async-trait", "codex-network-proxy", "codex-protocol", "codex-utils-absolute-path", @@ -2702,6 +2751,7 @@ dependencies = [ "pretty_assertions", "serde_json", "tempfile", + "tokio", "tracing", "url", "which 8.0.0", @@ -2718,7 +2768,7 @@ dependencies = [ "codex-keyring-store", "keyring", "pretty_assertions", - "rand 0.9.2", + "rand 0.9.3", "regex", "schemars 0.8.22", "serde", @@ -2834,9 +2884,22 @@ version = "0.0.0" dependencies = [ "async-trait", "chrono", + "codex-git-utils", "codex-protocol", + "codex-rollout", + "codex-state", + "pretty_assertions", + "prost 0.14.3", "serde", + "serde_json", + "tempfile", "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tonic", + "tonic-prost", + "tonic-prost-build", + "uuid", ] [[package]] @@ -2874,11 +2937,13 @@ dependencies = [ "codex-cli", "codex-cloud-requirements", "codex-config", + "codex-connectors", "codex-exec-server", "codex-features", "codex-feedback", "codex-file-search", "codex-git-utils", + "codex-install-context", "codex-login", "codex-mcp", "codex-model-provider-info", @@ -2917,7 +2982,7 @@ dependencies = [ "pathdiff", "pretty_assertions", "pulldown-cmark", - "rand 0.9.2", + "rand 0.9.3", "ratatui", "ratatui-macros", "regex-lite", @@ -3650,6 +3715,7 @@ dependencies = [ "cfg-if", "cpufeatures", "curve25519-dalek-derive", + "digest", "fiat-crypto", "rustc_version", "subtle", @@ -4126,7 +4192,7 @@ dependencies = [ "libc", "option-ext", "redox_users 0.5.2", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4230,6 +4296,30 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "serde", + "sha2", + "subtle", + "zeroize", +] + [[package]] name = "either" version = "1.15.0" @@ -5159,7 +5249,7 @@ dependencies = [ "idna", "ipnet", "once_cell", - "rand 0.9.2", + "rand 0.9.3", "ring", "thiserror 2.0.18", "tinyvec", @@ -5181,7 +5271,7 @@ dependencies = [ "moka", "once_cell", "parking_lot", - "rand 0.9.2", + "rand 0.9.3", "resolv-conf", "smallvec", "thiserror 2.0.18", @@ -6728,7 +6818,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7327,7 +7417,7 @@ dependencies = [ "futures-util", "opentelemetry", "percent-encoding", - "rand 0.9.2", + "rand 0.9.3", "thiserror 2.0.18", "tokio", "tokio-stream", @@ -7480,7 +7570,7 @@ dependencies = [ "heck 0.4.1", "itertools 0.11.0", "prost 0.12.6", - "prost-types", + "prost-types 0.12.6", ] [[package]] @@ -7494,7 +7584,7 @@ dependencies = [ "pbjson", "pbjson-build", "prost 0.12.6", - "prost-build", + "prost-build 0.12.6", "serde", ] @@ -7876,7 +7966,7 @@ checksum = "bee689443a2bd0a16ab0348b52ee43e3b2d1b1f931c8aa5c9f8de4c86fbe8c40" dependencies = [ "bitflags 2.10.0", "num-traits", - "rand 0.9.2", + "rand 0.9.3", "rand_chacha 0.9.0", "rand_xorshift", "regex-syntax 0.8.8", @@ -7918,7 +8008,26 @@ dependencies = [ "petgraph 0.6.5", "prettyplease", "prost 0.12.6", - "prost-types", + "prost-types 0.12.6", + "regex", + "syn 2.0.114", + "tempfile", +] + +[[package]] +name = "prost-build" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" +dependencies = [ + "heck 0.5.0", + "itertools 0.14.0", + "log", + "multimap", + "petgraph 0.8.3", + "prettyplease", + "prost 0.14.3", + "prost-types 0.14.3", "regex", "syn 2.0.114", "tempfile", @@ -7959,6 +8068,15 @@ dependencies = [ "prost 0.12.6", ] +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost 0.14.3", +] + [[package]] name = "psl" version = "2.1.184" @@ -8047,7 +8165,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.2", + "rand 0.9.3", "ring", "rustc-hash 2.1.1", "rustls", @@ -8179,7 +8297,7 @@ dependencies = [ "rama-http-types", "rama-net", "rama-utils", - "rand 0.9.2", + "rand 0.9.3", "serde", "serde_html_form", "serde_json", @@ -8249,7 +8367,7 @@ dependencies = [ "rama-macros", "rama-net", "rama-utils", - "rand 0.9.2", + "rand 0.9.3", "serde", "sha1", ] @@ -8277,7 +8395,7 @@ dependencies = [ "rama-error", "rama-macros", "rama-utils", - "rand 0.9.2", + "rand 0.9.3", "serde", "serde_json", "sync_wrapper", @@ -8351,7 +8469,7 @@ dependencies = [ "rama-http-types", "rama-net", "rama-utils", - "rand 0.9.2", + "rand 0.9.3", "tokio", ] @@ -8431,9 +8549,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "7ec095654a25171c2124e9e3393a930bddbffdc939556c914957a4c3e0a87166" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -8748,7 +8866,7 @@ dependencies = [ "pastey", "pin-project-lite", "process-wrap", - "rand 0.9.2", + "rand 0.9.3", "reqwest", "rmcp-macros", "schemars 1.2.1", @@ -8955,9 +9073,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.10" +version = "0.103.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06" dependencies = [ "aws-lc-rs", "ring", @@ -9332,7 +9450,7 @@ version = "0.46.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26ab054c34b87f96c3e4701bea1888317cde30cc7e4a6136d2c48454ab96661c" dependencies = [ - "rand 0.9.2", + "rand 0.9.3", "sentry-types", "serde", "serde_json", @@ -9380,7 +9498,7 @@ checksum = "eecbd63e9d15a26a40675ed180d376fcb434635d2e33de1c24003f61e3e2230d" dependencies = [ "debugid", "hex", - "rand 0.9.2", + "rand 0.9.3", "serde", "serde_json", "thiserror 2.0.18", @@ -10867,8 +10985,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a286e33f82f8a1ee2df63f4fa35c0becf4a85a0cb03091a15fd7bf0b402dc94a" dependencies = [ "async-trait", + "axum", "base64 0.22.1", "bytes", + "h2", "http 1.4.0", "http-body", "http-body-util", @@ -10878,6 +10998,7 @@ dependencies = [ "percent-encoding", "pin-project", "rustls-native-certs", + "socket2 0.6.2", "sync_wrapper", "tokio", "tokio-rustls", @@ -10888,6 +11009,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "tonic-build" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27aac809edf60b741e2d7db6367214d078856b8a5bff0087e94ff330fb97b6fc" +dependencies = [ + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "tonic-prost" version = "0.14.3" @@ -10899,6 +11032,22 @@ dependencies = [ "tonic", ] +[[package]] +name = "tonic-prost-build" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4556786613791cfef4ed134aa670b61a85cfcacf71543ef33e8d801abae988f" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build 0.14.3", + "prost-types 0.14.3", + "quote", + "syn 2.0.114", + "tempfile", + "tonic-build", +] + [[package]] name = "tower" version = "0.5.3" @@ -11165,7 +11314,7 @@ dependencies = [ "http 1.4.0", "httparse", "log", - "rand 0.9.2", + "rand 0.9.3", "rustls", "rustls-pki-types", "sha1", @@ -11852,7 +12001,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index e4985e4582..deb3738812 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -13,6 +13,7 @@ members = [ "arg0", "feedback", "features", + "install-context", "codex-backend-openapi-models", "code-mode", "cloud-requirements", @@ -27,6 +28,7 @@ members = [ "shell-escalation", "skills", "core", + "core-plugins", "core-skills", "hooks", "instructions", @@ -127,6 +129,7 @@ codex-code-mode = { path = "code-mode" } codex-config = { path = "config" } codex-connectors = { path = "connectors" } codex-core = { path = "core" } +codex-core-plugins = { path = "core-plugins" } codex-core-skills = { path = "core-skills" } codex-exec = { path = "exec" } codex-exec-server = { path = "exec-server" } @@ -134,6 +137,7 @@ codex-execpolicy = { path = "execpolicy" } codex-experimental-api-macros = { path = "codex-experimental-api-macros" } codex-features = { path = "features" } codex-feedback = { path = "feedback" } +codex-install-context = { path = "install-context" } codex-file-search = { path = "file-search" } codex-git-utils = { path = "git-utils" } codex-hooks = { path = "hooks" } @@ -166,6 +170,7 @@ codex-state = { path = "state" } codex-stdio-to-uds = { path = "stdio-to-uds" } codex-terminal-detection = { path = "terminal-detection" } codex-test-binary-support = { path = "test-binary-support" } +codex-thread-store = { path = "thread-store" } codex-tools = { path = "tools" } codex-tui = { path = "tui" } codex-utils-absolute-path = { path = "utils/absolute-path" } @@ -227,6 +232,7 @@ diffy = "0.4.2" dirs = "6" dotenvy = "0.15.7" dunce = "1.0.4" +ed25519-dalek = { version = "2.2.0", features = ["pkcs8"] } encoding_rs = "0.8.35" env-flags = "0.1.1" env_logger = "0.11.9" @@ -341,6 +347,8 @@ tracing-appender = "0.2.3" tracing-opentelemetry = "0.32.0" tracing-subscriber = "0.3.22" tracing-test = "0.2.5" +tonic = { version = "0.14.3", default-features = false, features = ["channel", "codegen"] } +tonic-prost = "0.14.3" tree-sitter = "0.25.10" tree-sitter-bash = "0.25" ts-rs = "11" diff --git a/codex-rs/README.md b/codex-rs/README.md index 6307668f39..2ad7158f98 100644 --- a/codex-rs/README.md +++ b/codex-rs/README.md @@ -1,6 +1,6 @@ # Codex CLI (Rust Implementation) -We provide Codex CLI as a standalone, native executable to ensure a zero-dependency install. +We provide Codex CLI as a standalone executable to ensure a zero-dependency install. ## Installing Codex diff --git a/codex-rs/app-server-client/src/lib.rs b/codex-rs/app-server-client/src/lib.rs index eb3d984965..71a8784e0a 100644 --- a/codex-rs/app-server-client/src/lib.rs +++ b/codex-rs/app-server-client/src/lib.rs @@ -28,6 +28,7 @@ use std::time::Duration; pub use codex_app_server::in_process::DEFAULT_IN_PROCESS_CHANNEL_CAPACITY; pub use codex_app_server::in_process::InProcessServerEvent; use codex_app_server::in_process::InProcessStartArgs; +use codex_app_server::in_process::LogDbLayer; use codex_app_server_protocol::ClientInfo; use codex_app_server_protocol::ClientNotification; use codex_app_server_protocol::ClientRequest; @@ -64,9 +65,9 @@ pub use crate::remote::RemoteAppServerConnectArgs; /// while legacy startup/config paths are migrated to RPCs. pub mod legacy_core { pub use codex_core::Cursor; - pub use codex_core::DEFAULT_PROJECT_DOC_FILENAME; + pub use codex_core::DEFAULT_AGENTS_MD_FILENAME; pub use codex_core::INTERACTIVE_SESSION_SOURCES; - pub use codex_core::LOCAL_PROJECT_DOC_FILENAME; + pub use codex_core::LOCAL_AGENTS_MD_FILENAME; pub use codex_core::McpManager; pub use codex_core::PLUGIN_TEXT_MENTION_SIGIL; pub use codex_core::RolloutRecorder; @@ -76,7 +77,6 @@ pub mod legacy_core { pub use codex_core::ThreadsPage; pub use codex_core::append_message_history_entry; pub use codex_core::check_execpolicy_for_warnings; - pub use codex_core::discover_project_doc_paths; pub use codex_core::find_thread_meta_by_name_str; pub use codex_core::find_thread_name_by_id; pub use codex_core::find_thread_names_by_ids; @@ -354,6 +354,8 @@ pub struct InProcessClientStartArgs { pub cloud_requirements: CloudRequirementsLoader, /// Feedback sink used by app-server/core telemetry and logs. pub feedback: CodexFeedback, + /// SQLite tracing layer used to flush recently emitted logs before feedback upload. + pub log_db: Option, /// Environment manager used by core execution and filesystem operations. pub environment_manager: Arc, /// Startup warnings emitted after initialize succeeds. @@ -405,6 +407,7 @@ impl InProcessClientStartArgs { loader_overrides: self.loader_overrides, cloud_requirements: self.cloud_requirements, feedback: self.feedback, + log_db: self.log_db, environment_manager: self.environment_manager, config_warnings: self.config_warnings, session_source: self.session_source, @@ -967,6 +970,7 @@ mod tests { match ConfigBuilder::default().build().await { Ok(config) => config, Err(_) => Config::load_default_with_cli_overrides(Vec::new()) + .await .expect("default config should load"), } } @@ -982,6 +986,7 @@ mod tests { loader_overrides: LoaderOverrides::default(), cloud_requirements: CloudRequirementsLoader::default(), feedback: CodexFeedback::new(), + log_db: None, environment_manager: Arc::new(EnvironmentManager::new(/*exec_server_url*/ None)), config_warnings: Vec::new(), session_source, @@ -1994,6 +1999,7 @@ mod tests { loader_overrides: LoaderOverrides::default(), cloud_requirements: CloudRequirementsLoader::default(), feedback: CodexFeedback::new(), + log_db: None, environment_manager: environment_manager.clone(), config_warnings: Vec::new(), session_source: SessionSource::Exec, diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index 2649f506db..194eba34b3 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -2911,6 +2911,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 1adb5adfcb..5a0bfd0513 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -13453,6 +13453,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index fc1f5cd778..a572aa1a91 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -11297,6 +11297,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json index 4e8cfcb621..c5a84c4272 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json @@ -787,6 +787,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json index 5defb06618..cb9201f0a6 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json @@ -787,6 +787,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json index b7490d5e12..05dfaa4b72 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json @@ -930,6 +930,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json index 6db7d990f0..909c2af4d8 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json @@ -1444,6 +1444,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json index 7435fbeff7..8ebebd39f9 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json @@ -1206,6 +1206,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json index c6abbab166..40da3390ec 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json @@ -1206,6 +1206,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json index fb658030f4..400c0b661a 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json @@ -1206,6 +1206,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json index 9879db55f6..aef0cd737c 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json @@ -1444,6 +1444,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json index 84d2768855..6a50d6d978 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json @@ -1206,6 +1206,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json index 42d76d6818..a3d20555f1 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json @@ -1444,6 +1444,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json index 4b24a129bb..c4b1e54490 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json @@ -1206,6 +1206,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json index bebc8a6bbf..e55c9651fa 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json @@ -1206,6 +1206,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json index 381b78ab71..2848940085 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json @@ -930,6 +930,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json index 02727fbadd..c3f31e5322 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json @@ -930,6 +930,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json index 25c15d78dc..7e8ca749c0 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json @@ -930,6 +930,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts index e7fe940aa7..747a70c54d 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts @@ -53,7 +53,7 @@ exitCode: number | null, /** * The duration of the command execution in milliseconds. */ -durationMs: number | null, } | { "type": "fileChange", id: string, changes: Array, status: PatchApplyStatus, } | { "type": "mcpToolCall", id: string, server: string, tool: string, status: McpToolCallStatus, arguments: JsonValue, result: McpToolCallResult | null, error: McpToolCallError | null, +durationMs: number | null, } | { "type": "fileChange", id: string, changes: Array, status: PatchApplyStatus, } | { "type": "mcpToolCall", id: string, server: string, tool: string, status: McpToolCallStatus, arguments: JsonValue, mcpAppResourceUri?: string, result: McpToolCallResult | null, error: McpToolCallError | null, /** * The duration of the MCP tool call in milliseconds. */ diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index a000b7bb35..fd6fa61a8b 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -289,6 +289,11 @@ client_request_definitions! { params: v2::ThreadMemoryModeSetParams, response: v2::ThreadMemoryModeSetResponse, }, + #[experimental("memory/reset")] + MemoryReset => "memory/reset" { + params: #[ts(type = "undefined")] #[serde(skip_serializing_if = "Option::is_none")] Option<()>, + response: v2::MemoryResetResponse, + }, ThreadUnarchive => "thread/unarchive" { params: v2::ThreadUnarchiveParams, response: v2::ThreadUnarchiveResponse, @@ -764,6 +769,7 @@ macro_rules! server_notification_definitions { Display, ExperimentalApi, )] + #[allow(clippy::large_enum_variant)] #[serde(tag = "method", content = "params", rename_all = "camelCase")] #[strum(serialize_all = "camelCase")] pub enum ServerNotification { diff --git a/codex-rs/app-server-protocol/src/protocol/thread_history.rs b/codex-rs/app-server-protocol/src/protocol/thread_history.rs index 9e94515dd9..54f882cf29 100644 --- a/codex-rs/app-server-protocol/src/protocol/thread_history.rs +++ b/codex-rs/app-server-protocol/src/protocol/thread_history.rs @@ -502,6 +502,7 @@ impl ThreadHistoryBuilder { .arguments .clone() .unwrap_or(serde_json::Value::Null), + mcp_app_resource_uri: payload.mcp_app_resource_uri.clone(), result: None, error: None, duration_ms: None, @@ -518,11 +519,11 @@ impl ThreadHistoryBuilder { let duration_ms = i64::try_from(payload.duration.as_millis()).ok(); let (result, error) = match &payload.result { Ok(value) => ( - Some(McpToolCallResult { + Some(Box::new(McpToolCallResult { content: value.content.clone(), structured_content: value.structured_content.clone(), meta: value.meta.clone(), - }), + })), None, ), Err(message) => ( @@ -542,6 +543,7 @@ impl ThreadHistoryBuilder { .arguments .clone() .unwrap_or(serde_json::Value::Null), + mcp_app_resource_uri: payload.mcp_app_resource_uri.clone(), result, error, duration_ms, @@ -1809,6 +1811,7 @@ mod tests { tool: "lookup".into(), arguments: Some(serde_json::json!({"id":"123"})), }, + mcp_app_resource_uri: None, duration: Duration::from_millis(8), result: Err("boom".into()), }), @@ -1857,6 +1860,7 @@ mod tests { tool: "lookup".into(), status: McpToolCallStatus::Failed, arguments: serde_json::json!({"id":"123"}), + mcp_app_resource_uri: None, result: None, error: Some(McpToolCallError { message: "boom".into(), @@ -1882,6 +1886,7 @@ mod tests { tool: "lookup".into(), arguments: Some(serde_json::json!({"id":"123"})), }, + mcp_app_resource_uri: Some("ui://widget/lookup.html".into()), duration: Duration::from_millis(8), result: Ok(CallToolResult { content: vec![serde_json::json!({ @@ -1911,7 +1916,8 @@ mod tests { tool: "lookup".into(), status: McpToolCallStatus::Completed, arguments: serde_json::json!({"id":"123"}), - result: Some(McpToolCallResult { + mcp_app_resource_uri: Some("ui://widget/lookup.html".into()), + result: Some(Box::new(McpToolCallResult { content: vec![serde_json::json!({ "type": "text", "text": "result" @@ -1920,7 +1926,7 @@ mod tests { meta: Some(serde_json::json!({ "ui/resourceUri": "ui://widget/lookup.html" })), - }), + })), error: None, duration_ms: Some(8), } diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 13c49dc030..1e49340750 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -3145,6 +3145,11 @@ pub struct ThreadMemoryModeSetParams { #[ts(export_to = "v2/")] pub struct ThreadMemoryModeSetResponse {} +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct MemoryResetResponse {} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] @@ -4611,7 +4616,10 @@ pub enum ThreadItem { tool: String, status: McpToolCallStatus, arguments: JsonValue, - result: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + mcp_app_resource_uri: Option, + result: Option>, error: Option, /// The duration of the MCP tool call in milliseconds. #[ts(type = "number | null")] diff --git a/codex-rs/app-server/Cargo.toml b/codex-rs/app-server/Cargo.toml index 5dc3a31485..097abdb713 100644 --- a/codex-rs/app-server/Cargo.toml +++ b/codex-rs/app-server/Cargo.toml @@ -34,6 +34,7 @@ codex-arg0 = { workspace = true } codex-cloud-requirements = { workspace = true } codex-config = { workspace = true } codex-core = { workspace = true } +codex-core-plugins = { workspace = true } codex-exec-server = { workspace = true } codex-features = { workspace = true } codex-git-utils = { workspace = true } @@ -54,6 +55,7 @@ codex-rmcp-client = { workspace = true } codex-rollout = { workspace = true } codex-sandboxing = { workspace = true } codex-state = { workspace = true } +codex-thread-store = { workspace = true } codex-tools = { workspace = true } codex-utils-absolute-path = { workspace = true } codex-utils-json-to-toml = { workspace = true } diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 61730909a0..0d7e86dc39 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -41,8 +41,11 @@ Security note: - Non-loopback websocket listeners currently allow unauthenticated connections by default during rollout. If you expose one remotely, configure websocket auth explicitly now. - Supported auth modes are app-server flags: - `--ws-auth capability-token --ws-token-file /absolute/path` + - `--ws-auth capability-token --ws-token-sha256 HEX` - `--ws-auth signed-bearer-token --ws-shared-secret-file /absolute/path` for HMAC-signed JWT/JWS bearer tokens, with optional `--ws-issuer`, `--ws-audience`, `--ws-max-clock-skew-seconds` - Clients present the credential as `Authorization: Bearer ` during the websocket handshake. Auth is enforced before JSON-RPC `initialize`. +- When starting `codex app-server` manually, prefer `--ws-token-file` over passing raw bearer tokens on the command line. Store a high-entropy token in a file readable only by your user, then have your client present that token in the websocket `Authorization` header. +- `--ws-token-sha256` is intended for clients that keep the raw token in a separate local secret store and only need the server to know the SHA-256 verifier. The hash may appear in process listings, but it is not sufficient to authenticate; clients still need the original raw token. Only use this mode with randomly generated high-entropy tokens, not passwords or other guessable values. Tracing/log output: @@ -141,6 +144,7 @@ Example with notification opt-out: - `thread/read` — read a stored thread by id without resuming it; optionally include turns via `includeTurns`. The returned `thread` includes `status` (`ThreadStatus`), defaulting to `notLoaded` when the thread is not currently loaded. - `thread/metadata/update` — patch stored thread metadata in sqlite; currently supports updating persisted `gitInfo` fields and returns the refreshed `thread`. - `thread/memoryMode/set` — experimental; set a thread’s persisted memory eligibility to `"enabled"` or `"disabled"` for either a loaded thread or a stored rollout; returns `{}` on success. +- `memory/reset` — experimental; clear the current `CODEX_HOME/memories` directory and reset persisted memory stage data in sqlite while preserving existing thread memory modes; returns `{}` on success. - `thread/status/changed` — notification emitted when a loaded thread’s status changes (`threadId` + new `status`). - `thread/archive` — move a thread’s rollout file into the archived directory; returns `{}` on success and emits `thread/archived`. - `thread/unsubscribe` — unsubscribe this connection from thread turn/item events. If this was the last subscriber, the server keeps the thread loaded and unloads it only after it has had no subscribers and no thread activity for 30 minutes, then emits `thread/closed`. @@ -413,6 +417,13 @@ Experimental: use `thread/memoryMode/set` to change whether a thread remains eli { "id": 26, "result": {} } ``` +Experimental: use `memory/reset` to clear local memory artifacts and sqlite-backed memory stage data for the current Codex home. This preserves existing thread memory modes; use `thread/memoryMode/set` separately when a thread's future memory eligibility should change. + +```json +{ "method": "memory/reset", "id": 27 } +{ "id": 27, "result": {} } +``` + ### Example: Archive a thread Use `thread/archive` to move the persisted rollout (stored as a JSONL file on disk) into the archived sessions directory. diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index 5f5ff0633b..9b2cf57d74 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -2910,6 +2910,7 @@ async fn construct_mcp_tool_call_notification( tool: begin_event.invocation.tool, status: McpToolCallStatus::InProgress, arguments: begin_event.invocation.arguments.unwrap_or(JsonValue::Null), + mcp_app_resource_uri: begin_event.mcp_app_resource_uri, result: None, error: None, duration_ms: None, @@ -2936,11 +2937,11 @@ async fn construct_mcp_tool_call_end_notification( let (result, error) = match &end_event.result { Ok(value) => ( - Some(McpToolCallResult { + Some(Box::new(McpToolCallResult { content: value.content.clone(), structured_content: value.structured_content.clone(), meta: value.meta.clone(), - }), + })), None, ), Err(message) => ( @@ -2957,6 +2958,7 @@ async fn construct_mcp_tool_call_end_notification( tool: end_event.invocation.tool, status, arguments: end_event.invocation.arguments.unwrap_or(JsonValue::Null), + mcp_app_resource_uri: end_event.mcp_app_resource_uri, result, error, duration_ms, @@ -4220,6 +4222,7 @@ mod tests { tool: "list_mcp_resources".to_string(), arguments: Some(serde_json::json!({"server": ""})), }, + mcp_app_resource_uri: Some("ui://widget/list-resources.html".to_string()), }; let thread_id = ThreadId::new().to_string(); @@ -4240,6 +4243,7 @@ mod tests { tool: begin_event.invocation.tool, status: McpToolCallStatus::InProgress, arguments: serde_json::json!({"server": ""}), + mcp_app_resource_uri: Some("ui://widget/list-resources.html".to_string()), result: None, error: None, duration_ms: None, @@ -4380,6 +4384,7 @@ mod tests { tool: "list_mcp_resources".to_string(), arguments: None, }, + mcp_app_resource_uri: None, }; let thread_id = ThreadId::new().to_string(); @@ -4400,6 +4405,7 @@ mod tests { tool: begin_event.invocation.tool, status: McpToolCallStatus::InProgress, arguments: JsonValue::Null, + mcp_app_resource_uri: None, result: None, error: None, duration_ms: None, @@ -4431,6 +4437,7 @@ mod tests { tool: "list_mcp_resources".to_string(), arguments: Some(serde_json::json!({"server": ""})), }, + mcp_app_resource_uri: Some("ui://widget/list-resources.html".to_string()), duration: Duration::from_nanos(92708), result: Ok(result), }; @@ -4453,13 +4460,14 @@ mod tests { tool: end_event.invocation.tool, status: McpToolCallStatus::Completed, arguments: serde_json::json!({"server": ""}), - result: Some(McpToolCallResult { + mcp_app_resource_uri: Some("ui://widget/list-resources.html".to_string()), + result: Some(Box::new(McpToolCallResult { content, structured_content: None, meta: Some(serde_json::json!({ "ui/resourceUri": "ui://widget/list-resources.html" })), - }), + })), error: None, duration_ms: Some(0), }, @@ -4477,6 +4485,7 @@ mod tests { tool: "list_mcp_resources".to_string(), arguments: None, }, + mcp_app_resource_uri: None, duration: Duration::from_millis(1), result: Err("boom".to_string()), }; @@ -4499,6 +4508,7 @@ mod tests { tool: end_event.invocation.tool, status: McpToolCallStatus::Failed, arguments: JsonValue::Null, + mcp_app_resource_uri: None, result: None, error: Some(McpToolCallError { message: "boom".to_string(), diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index b89f38b958..ce05874604 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -94,6 +94,7 @@ use codex_app_server_protocol::McpServerStatus; use codex_app_server_protocol::McpServerStatusDetail; use codex_app_server_protocol::McpServerToolCallParams; use codex_app_server_protocol::McpServerToolCallResponse; +use codex_app_server_protocol::MemoryResetResponse; use codex_app_server_protocol::MockExperimentalMethodParams; use codex_app_server_protocol::MockExperimentalMethodResponse; use codex_app_server_protocol::ModelListParams; @@ -189,6 +190,7 @@ use codex_app_server_protocol::ThreadUnsubscribeStatus; use codex_app_server_protocol::Turn; use codex_app_server_protocol::TurnError; use codex_app_server_protocol::TurnInterruptParams; +use codex_app_server_protocol::TurnInterruptResponse; use codex_app_server_protocol::TurnStartParams; use codex_app_server_protocol::TurnStartResponse; use codex_app_server_protocol::TurnStatus; @@ -207,7 +209,6 @@ use codex_chatgpt::connectors; use codex_cloud_requirements::cloud_requirements_loader; use codex_config::types::McpServerTransportConfig; use codex_core::CodexThread; -use codex_core::Cursor as RolloutCursor; use codex_core::ForkSnapshot; use codex_core::NewThread; use codex_core::RolloutRecorder; @@ -215,8 +216,8 @@ use codex_core::SessionMeta; use codex_core::SteerInputError; use codex_core::ThreadConfigSnapshot; use codex_core::ThreadManager; -use codex_core::ThreadSortKey as CoreThreadSortKey; use codex_core::append_thread_name; +use codex_core::clear_memory_roots_contents; use codex_core::config::Config; use codex_core::config::ConfigOverrides; use codex_core::config::NetworkProxyAuditMetadata; @@ -236,26 +237,25 @@ use codex_core::find_archived_thread_path_by_id_str; use codex_core::find_thread_name_by_id; use codex_core::find_thread_names_by_ids; use codex_core::find_thread_path_by_id_str; -use codex_core::parse_cursor; use codex_core::path_utils; use codex_core::plugins::MarketplaceAddError; -use codex_core::plugins::MarketplaceError; -use codex_core::plugins::MarketplacePluginSource; use codex_core::plugins::OPENAI_CURATED_MARKETPLACE_NAME; use codex_core::plugins::PluginInstallError as CorePluginInstallError; use codex_core::plugins::PluginInstallRequest; use codex_core::plugins::PluginReadRequest; use codex_core::plugins::PluginUninstallError as CorePluginUninstallError; use codex_core::plugins::add_marketplace as add_marketplace_to_codex_home; -use codex_core::plugins::load_plugin_apps; -use codex_core::plugins::load_plugin_mcp_servers; use codex_core::read_head_for_summary; use codex_core::read_session_meta_line; -use codex_core::rollout_date_parts; use codex_core::sandboxing::SandboxPermissions; use codex_core::windows_sandbox::WindowsSandboxLevelExt; use codex_core::windows_sandbox::WindowsSandboxSetupMode as CoreWindowsSandboxSetupMode; use codex_core::windows_sandbox::WindowsSandboxSetupRequest; +use codex_core_plugins::loader::load_plugin_apps; +use codex_core_plugins::loader::load_plugin_mcp_servers; +use codex_core_plugins::manifest::PluginManifestInterface; +use codex_core_plugins::marketplace::MarketplaceError; +use codex_core_plugins::marketplace::MarketplacePluginSource; use codex_exec_server::LOCAL_FS; use codex_features::FEATURES; use codex_features::Feature; @@ -326,15 +326,20 @@ use codex_state::StateRuntime; use codex_state::ThreadMetadata; use codex_state::ThreadMetadataBuilder; use codex_state::log_db::LogDbLayer; +use codex_thread_store::ArchiveThreadParams as StoreArchiveThreadParams; +use codex_thread_store::ListThreadsParams as StoreListThreadsParams; +use codex_thread_store::LocalThreadStore; +use codex_thread_store::ReadThreadParams as StoreReadThreadParams; +use codex_thread_store::StoredThread; +use codex_thread_store::ThreadSortKey as StoreThreadSortKey; +use codex_thread_store::ThreadStore; +use codex_thread_store::ThreadStoreError; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_json_to_toml::json_to_toml; use codex_utils_pty::DEFAULT_OUTPUT_BYTES_CAP; use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::HashSet; -use std::ffi::OsStr; -use std::fs::FileTimes; -use std::fs::OpenOptions; use std::io::Error as IoError; use std::path::Path; use std::path::PathBuf; @@ -344,7 +349,6 @@ use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; use std::time::Duration; use std::time::Instant; -use std::time::SystemTime; use tokio::sync::Mutex; use tokio::sync::broadcast; use tokio::sync::oneshot; @@ -448,6 +452,7 @@ pub(crate) struct CodexMessageProcessor { analytics_events_client: AnalyticsEventsClient, arg0_paths: Arg0DispatchPaths, config: Arc, + thread_store: LocalThreadStore, cli_overrides: Arc>>, runtime_feature_enablement: Arc>>, cloud_requirements: Arc>, @@ -617,17 +622,9 @@ pub(crate) struct CodexMessageProcessorArgs { impl CodexMessageProcessor { async fn instruction_sources_from_config(config: &Config) -> Vec { - let mut paths: Vec = - config.user_instructions_path.iter().cloned().collect(); - match codex_core::discover_project_doc_paths(config, LOCAL_FS.as_ref()).await { - Ok(project_doc_paths) => { - paths.extend(project_doc_paths); - } - Err(err) => { - tracing::warn!(error = %err, "failed to discover project docs for thread response"); - } - } - paths + codex_core::AgentsMdManager::new(config) + .instruction_sources(LOCAL_FS.as_ref()) + .await } pub(crate) fn handle_config_mutation(&self) { @@ -706,6 +703,7 @@ impl CodexMessageProcessor { outgoing: outgoing.clone(), analytics_events_client, arg0_paths, + thread_store: LocalThreadStore::new(codex_rollout::RolloutConfig::from_view(&config)), config, cli_overrides, runtime_feature_enablement, @@ -913,6 +911,10 @@ impl CodexMessageProcessor { self.thread_memory_mode_set(to_connection_request_id(request_id), params) .await; } + ClientRequest::MemoryReset { request_id, params } => { + self.memory_reset(to_connection_request_id(request_id), params) + .await; + } ClientRequest::ThreadUnarchive { request_id, params } => { self.thread_unarchive(to_connection_request_id(request_id), params) .await; @@ -2512,6 +2514,7 @@ impl CodexMessageProcessor { )) { let trust_target = resolve_root_git_project_for_trust(config.cwd.as_path()) + .await .unwrap_or_else(|| config.cwd.to_path_buf()); let cli_overrides_with_trust; let cli_overrides_for_reload = if let Err(err) = @@ -2786,7 +2789,6 @@ impl CodexMessageProcessor { } async fn thread_archive(&self, request_id: ConnectionRequestId, params: ThreadArchiveParams) { - // TODO(jif) mostly rewrite this using sqlite after phase 1 let thread_id = match ThreadId::from_string(¶ms.thread_id) { Ok(id) => id, Err(err) => { @@ -2800,32 +2802,28 @@ impl CodexMessageProcessor { } }; - let rollout_path = - match find_thread_path_by_id_str(&self.config.codex_home, &thread_id.to_string()).await - { - Ok(Some(p)) => p, - Ok(None) => { - let error = JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!("no rollout found for thread id {thread_id}"), - data: None, - }; - self.outgoing.send_error(request_id, error).await; - return; - } - Err(err) => { - let error = JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!("failed to locate thread id {thread_id}: {err}"), - data: None, - }; - self.outgoing.send_error(request_id, error).await; - return; - } - }; - let thread_id_str = thread_id.to_string(); - match self.archive_thread_common(thread_id, &rollout_path).await { + if let Err(err) = self + .thread_store + .read_thread(StoreReadThreadParams { + thread_id, + include_archived: false, + include_history: false, + }) + .await + { + self.outgoing + .send_error(request_id, thread_store_archive_error("archive", err)) + .await; + return; + } + self.prepare_thread_for_archive(thread_id).await; + + match self + .thread_store + .archive_thread(StoreArchiveThreadParams { thread_id }) + .await + { Ok(()) => { let response = ThreadArchiveResponse {}; self.outgoing.send_response(request_id, response).await; @@ -2837,7 +2835,9 @@ impl CodexMessageProcessor { .await; } Err(err) => { - self.outgoing.send_error(request_id, err).await; + self.outgoing + .send_error(request_id, thread_store_archive_error("archive", err)) + .await; } } } @@ -3123,6 +3123,50 @@ impl CodexMessageProcessor { .await; } + async fn memory_reset(&self, request_id: ConnectionRequestId, _params: Option<()>) { + let state_db = match StateRuntime::init( + self.config.sqlite_home.clone(), + self.config.model_provider_id.clone(), + ) + .await + { + Ok(state_db) => state_db, + Err(err) => { + self.send_internal_error( + request_id, + format!("failed to open state db for memory reset: {err}"), + ) + .await; + return; + } + }; + + if let Err(err) = state_db.clear_memory_data().await { + self.send_internal_error( + request_id, + format!("failed to clear memory rows in state db: {err}"), + ) + .await; + return; + } + + if let Err(err) = clear_memory_roots_contents(&self.config.codex_home).await { + self.send_internal_error( + request_id, + format!( + "failed to clear memory directories under {}: {err}", + self.config.codex_home.display() + ), + ) + .await; + return; + } + + self.outgoing + .send_response(request_id, MemoryResetResponse {}) + .await; + } + async fn thread_metadata_update( &self, request_id: ConnectionRequestId, @@ -3426,7 +3470,6 @@ impl CodexMessageProcessor { request_id: ConnectionRequestId, params: ThreadUnarchiveParams, ) { - // TODO(jif) mostly rewrite this using sqlite after phase 1 let thread_id = match ThreadId::from_string(¶ms.thread_id) { Ok(id) => id, Err(err) => { @@ -3440,152 +3483,21 @@ impl CodexMessageProcessor { } }; - let archived_path = match find_archived_thread_path_by_id_str( - &self.config.codex_home, - &thread_id.to_string(), - ) - .await - { - Ok(Some(path)) => path, - Ok(None) => { - let error = JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!("no archived rollout found for thread id {thread_id}"), - data: None, - }; - self.outgoing.send_error(request_id, error).await; - return; - } - Err(err) => { - let error = JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!("failed to locate archived thread id {thread_id}: {err}"), - data: None, - }; - self.outgoing.send_error(request_id, error).await; - return; - } - }; - - let rollout_path_display = archived_path.display().to_string(); let fallback_provider = self.config.model_provider_id.clone(); - let state_db_ctx = get_state_db(&self.config).await; - let archived_folder = self - .config - .codex_home - .join(codex_core::ARCHIVED_SESSIONS_SUBDIR); - - let result: Result = async { - let canonical_archived_dir = tokio::fs::canonicalize(&archived_folder).await.map_err( - |err| JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!( - "failed to unarchive thread: unable to resolve archived directory: {err}" - ), - data: None, - }, - )?; - let canonical_rollout_path = tokio::fs::canonicalize(&archived_path).await; - let canonical_rollout_path = if let Ok(path) = canonical_rollout_path - && path.starts_with(&canonical_archived_dir) - { - path - } else { - return Err(JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!( - "rollout path `{rollout_path_display}` must be in archived directory" - ), - data: None, - }); - }; - - let required_suffix = format!("{thread_id}.jsonl"); - let Some(file_name) = canonical_rollout_path.file_name().map(OsStr::to_owned) else { - return Err(JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!("rollout path `{rollout_path_display}` missing file name"), - data: None, - }); - }; - if !file_name - .to_string_lossy() - .ends_with(required_suffix.as_str()) - { - return Err(JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!( - "rollout path `{rollout_path_display}` does not match thread id {thread_id}" - ), - data: None, - }); - } - - let Some((year, month, day)) = rollout_date_parts(&file_name) else { - return Err(JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!( - "rollout path `{rollout_path_display}` missing filename timestamp" - ), - data: None, - }); - }; - - let sessions_folder = self.config.codex_home.join(codex_core::SESSIONS_SUBDIR); - let dest_dir = sessions_folder.join(year).join(month).join(day); - let restored_path = dest_dir.join(&file_name); - tokio::fs::create_dir_all(&dest_dir) - .await - .map_err(|err| JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!("failed to unarchive thread: {err}"), - data: None, - })?; - tokio::fs::rename(&canonical_rollout_path, &restored_path) - .await - .map_err(|err| JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!("failed to unarchive thread: {err}"), - data: None, - })?; - tokio::task::spawn_blocking({ - let restored_path = restored_path.clone(); - move || -> std::io::Result<()> { - let times = FileTimes::new().set_modified(SystemTime::now()); - OpenOptions::new() - .append(true) - .open(&restored_path)? - .set_times(times)?; - Ok(()) - } - }) + let result = self + .thread_store + .unarchive_thread(StoreArchiveThreadParams { thread_id }) .await - .map_err(|err| JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!("failed to update unarchived thread timestamp: {err}"), - data: None, - })? - .map_err(|err| JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!("failed to update unarchived thread timestamp: {err}"), - data: None, - })?; - if let Some(ctx) = state_db_ctx { - let _ = ctx - .mark_unarchived(thread_id, restored_path.as_path()) - .await; - } - let summary = - read_summary_from_rollout(restored_path.as_path(), fallback_provider.as_str()) - .await - .map_err(|err| JSONRPCErrorError { + .map_err(|err| thread_store_archive_error("unarchive", err)) + .and_then(|stored_thread| { + summary_from_stored_thread(stored_thread, fallback_provider.as_str()) + .map(|summary| summary_to_thread(summary, &self.config.cwd)) + .ok_or_else(|| JSONRPCErrorError { code: INTERNAL_ERROR_CODE, - message: format!("failed to read unarchived thread: {err}"), + message: format!("failed to read unarchived thread {thread_id}"), data: None, - })?; - Ok(summary_to_thread(summary, &self.config.cwd)) - } - .await; + }) + }); match result { Ok(mut thread) => { @@ -3811,15 +3723,15 @@ impl CodexMessageProcessor { .map(|value| value as usize) .unwrap_or(THREAD_LIST_DEFAULT_LIMIT) .clamp(1, THREAD_LIST_MAX_LIMIT); - let core_sort_key = match sort_key.unwrap_or(ThreadSortKey::CreatedAt) { - ThreadSortKey::CreatedAt => CoreThreadSortKey::CreatedAt, - ThreadSortKey::UpdatedAt => CoreThreadSortKey::UpdatedAt, + let store_sort_key = match sort_key.unwrap_or(ThreadSortKey::CreatedAt) { + ThreadSortKey::CreatedAt => StoreThreadSortKey::CreatedAt, + ThreadSortKey::UpdatedAt => StoreThreadSortKey::UpdatedAt, }; let (summaries, next_cursor) = match self .list_threads_common( requested_page_size, cursor, - core_sort_key, + store_sort_key, ThreadListFilters { model_providers, source_kinds, @@ -5063,7 +4975,7 @@ impl CodexMessageProcessor { &self, requested_page_size: usize, cursor: Option, - sort_key: CoreThreadSortKey, + sort_key: StoreThreadSortKey, filters: ThreadListFilters, ) -> Result<(Vec, Option), JSONRPCErrorError> { let ThreadListFilters { @@ -5073,16 +4985,7 @@ impl CodexMessageProcessor { cwd, search_term, } = filters; - let mut cursor_obj: Option = match cursor.as_ref() { - Some(cursor_str) => { - Some(parse_cursor(cursor_str).ok_or_else(|| JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!("invalid cursor: {cursor_str}"), - data: None, - })?) - } - None => None, - }; + let mut cursor_obj = cursor; let mut last_cursor = cursor_obj.clone(); let mut remaining = requested_page_size; let mut items = Vec::with_capacity(requested_page_size); @@ -5101,54 +5004,26 @@ impl CodexMessageProcessor { let fallback_provider = self.config.model_provider_id.clone(); let (allowed_sources_vec, source_kind_filter) = compute_source_filters(source_kinds); let allowed_sources = allowed_sources_vec.as_slice(); - let state_db_ctx = get_state_db(&self.config).await; while remaining > 0 { let page_size = remaining.min(THREAD_LIST_MAX_LIMIT); - let page = if archived { - RolloutRecorder::list_archived_threads( - &self.config, + let page = self + .thread_store + .list_threads(StoreListThreadsParams { page_size, - cursor_obj.as_ref(), + cursor: cursor_obj.clone(), sort_key, - allowed_sources, - model_provider_filter.as_deref(), - fallback_provider.as_str(), - search_term.as_deref(), - ) + allowed_sources: allowed_sources.to_vec(), + model_providers: model_provider_filter.clone(), + archived, + search_term: search_term.clone(), + }) .await - .map_err(|err| JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!("failed to list threads: {err}"), - data: None, - })? - } else { - RolloutRecorder::list_threads( - &self.config, - page_size, - cursor_obj.as_ref(), - sort_key, - allowed_sources, - model_provider_filter.as_deref(), - fallback_provider.as_str(), - search_term.as_deref(), - ) - .await - .map_err(|err| JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!("failed to list threads: {err}"), - data: None, - })? - }; + .map_err(thread_store_list_error)?; let mut filtered = Vec::with_capacity(page.items.len()); for it in page.items { - let Some(summary) = summary_from_thread_list_item( - it, - fallback_provider.as_str(), - state_db_ctx.as_ref(), - ) - .await + let Some(summary) = summary_from_stored_thread(it, fallback_provider.as_str()) else { continue; }; @@ -5168,12 +5043,8 @@ impl CodexMessageProcessor { items.extend(filtered); remaining = requested_page_size.saturating_sub(items.len()); - // Encode RolloutCursor into the JSON-RPC string form returned to clients. let next_cursor_value = page.next_cursor.clone(); - next_cursor = next_cursor_value - .as_ref() - .and_then(|cursor| serde_json::to_value(cursor).ok()) - .and_then(|value| value.as_str().map(str::to_owned)); + next_cursor = next_cursor_value.clone(); if remaining == 0 { break; } @@ -5974,76 +5845,10 @@ impl CodexMessageProcessor { .await; } - async fn archive_thread_common( - &self, - thread_id: ThreadId, - rollout_path: &Path, - ) -> Result<(), JSONRPCErrorError> { - // Verify rollout_path is under sessions dir. - let rollout_folder = self.config.codex_home.join(codex_core::SESSIONS_SUBDIR); - - let canonical_sessions_dir = match tokio::fs::canonicalize(&rollout_folder).await { - Ok(path) => path, - Err(err) => { - return Err(JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!( - "failed to archive thread: unable to resolve sessions directory: {err}" - ), - data: None, - }); - } - }; - let canonical_rollout_path = tokio::fs::canonicalize(rollout_path).await; - let canonical_rollout_path = if let Ok(path) = canonical_rollout_path - && path.starts_with(&canonical_sessions_dir) - { - path - } else { - return Err(JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!( - "rollout path `{}` must be in sessions directory", - rollout_path.display() - ), - data: None, - }); - }; - - // Verify file name matches thread id. - let required_suffix = format!("{thread_id}.jsonl"); - let Some(file_name) = canonical_rollout_path.file_name().map(OsStr::to_owned) else { - return Err(JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!( - "rollout path `{}` missing file name", - rollout_path.display() - ), - data: None, - }); - }; - if !file_name - .to_string_lossy() - .ends_with(required_suffix.as_str()) - { - return Err(JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!( - "rollout path `{}` does not match thread id {thread_id}", - rollout_path.display() - ), - data: None, - }); - } - - let mut state_db_ctx = None; - + async fn prepare_thread_for_archive(&self, thread_id: ThreadId) { // If the thread is active, request shutdown and wait briefly. let removed_conversation = self.thread_manager.remove_thread(&thread_id).await; if let Some(conversation) = removed_conversation { - if let Some(ctx) = conversation.state_db() { - state_db_ctx = Some(ctx); - } info!("thread {thread_id} was active; shutting down"); match Self::wait_for_thread_shutdown(&conversation).await { ThreadShutdownResult::Complete => {} @@ -6058,34 +5863,6 @@ impl CodexMessageProcessor { } } self.finalize_thread_teardown(thread_id).await; - - if state_db_ctx.is_none() { - state_db_ctx = get_state_db(&self.config).await; - } - - // Move the rollout file to archived. - let result: std::io::Result<()> = async move { - let archive_folder = self - .config - .codex_home - .join(codex_core::ARCHIVED_SESSIONS_SUBDIR); - tokio::fs::create_dir_all(&archive_folder).await?; - let archived_path = archive_folder.join(&file_name); - tokio::fs::rename(&canonical_rollout_path, &archived_path).await?; - if let Some(ctx) = state_db_ctx { - let _ = ctx - .mark_archived(thread_id, archived_path.as_path(), Utc::now()) - .await; - } - Ok(()) - } - .await; - - result.map_err(|err| JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!("failed to archive thread: {err}"), - data: None, - }) } async fn apps_list(&self, request_id: ConnectionRequestId, params: AppsListParams) { @@ -6470,7 +6247,7 @@ impl CodexMessageProcessor { force_remote_sync, } = params; let roots = cwds.unwrap_or_default(); - plugins_manager.maybe_start_non_curated_plugin_cache_refresh_for_roots(&roots); + plugins_manager.maybe_start_non_curated_plugin_cache_refresh(&roots); let mut config = match self.load_latest_config(/*fallback_cwd*/ None).await { Ok(config) => config, @@ -7817,9 +7594,12 @@ impl CodexMessageProcessor { async fn turn_interrupt(&self, request_id: ConnectionRequestId, params: TurnInterruptParams) { let TurnInterruptParams { thread_id, turn_id } = params; - self.outgoing - .record_request_turn_id(&request_id, &turn_id) - .await; + let is_startup_interrupt = turn_id.is_empty(); + if !is_startup_interrupt { + self.outgoing + .record_request_turn_id(&request_id, &turn_id) + .await; + } let (thread_uuid, thread) = match self.load_thread(&thread_id).await { Ok(v) => v, @@ -7829,21 +7609,48 @@ impl CodexMessageProcessor { } }; - let request = request_id.clone(); - - // Record the pending interrupt so we can reply when TurnAborted arrives. - { + // Record turn interrupts so we can reply when TurnAborted arrives. Startup + // interrupts do not have a turn and are acknowledged after submission. + if !is_startup_interrupt { let thread_state = self.thread_state_manager.thread_state(thread_uuid).await; let mut thread_state = thread_state.lock().await; thread_state .pending_interrupts - .push((request, ApiVersion::V2)); + .push((request_id.clone(), ApiVersion::V2)); } - // Submit the interrupt; we'll respond upon TurnAborted. - let _ = self + // Submit the interrupt. Turn interrupts respond upon TurnAborted; startup + // interrupts respond here because startup cancellation has no turn event. + let submit_result = self .submit_core_op(&request_id, thread.as_ref(), Op::Interrupt) .await; + match submit_result { + Ok(_) if is_startup_interrupt => { + self.outgoing + .send_response(request_id, TurnInterruptResponse {}) + .await; + } + Ok(_) => {} + Err(err) => { + if !is_startup_interrupt { + let thread_state = self.thread_state_manager.thread_state(thread_uuid).await; + let mut thread_state = thread_state.lock().await; + thread_state + .pending_interrupts + .retain(|(pending_request_id, _)| pending_request_id != &request_id); + } + let interrupt_target = if is_startup_interrupt { + "startup" + } else { + "turn" + }; + self.send_internal_error( + request_id, + format!("failed to interrupt {interrupt_target}: {err}"), + ) + .await; + } + } } async fn ensure_conversation_listener( @@ -9093,9 +8900,7 @@ fn plugin_skills_to_info( .collect() } -fn plugin_interface_to_info( - interface: codex_core::plugins::PluginManifestInterface, -) -> PluginInterface { +fn plugin_interface_to_info(interface: PluginManifestInterface) -> PluginInterface { PluginInterface { display_name: interface.display_name, short_description: interface.short_description, @@ -9413,67 +9218,67 @@ fn set_thread_name_from_title(thread: &mut Thread, title: String) { thread.name = Some(title); } -async fn summary_from_thread_list_item( - it: codex_core::ThreadItem, - fallback_provider: &str, - state_db_ctx: Option<&StateDbHandle>, -) -> Option { - if let Some(thread_id) = it.thread_id { - let timestamp = it.created_at.clone(); - let updated_at = it.updated_at.clone().or_else(|| timestamp.clone()); - let model_provider = it - .model_provider - .clone() - .unwrap_or_else(|| fallback_provider.to_string()); - let cwd = it.cwd?; - let cli_version = it.cli_version.unwrap_or_default(); - let source = with_thread_spawn_agent_metadata( - it.source - .unwrap_or(codex_protocol::protocol::SessionSource::Unknown), - it.agent_nickname.clone(), - it.agent_role.clone(), - ); - return Some(ConversationSummary { - conversation_id: thread_id, - path: it.path, - preview: it.first_user_message.unwrap_or_default(), - timestamp, - updated_at, - model_provider, - cwd, - cli_version, - source, - git_info: if it.git_sha.is_none() - && it.git_branch.is_none() - && it.git_origin_url.is_none() - { - None - } else { - Some(ConversationGitInfo { - sha: it.git_sha, - branch: it.git_branch, - origin_url: it.git_origin_url, - }) - }, - }); +fn thread_store_list_error(err: ThreadStoreError) -> JSONRPCErrorError { + match err { + ThreadStoreError::InvalidRequest { message } => JSONRPCErrorError { + code: INVALID_REQUEST_ERROR_CODE, + message, + data: None, + }, + err => JSONRPCErrorError { + code: INTERNAL_ERROR_CODE, + message: format!("failed to list threads: {err}"), + data: None, + }, } - if let Some(thread_id) = thread_id_from_rollout_path(it.path.as_path()) { - return read_summary_from_state_db_context_by_thread_id(state_db_ctx, thread_id).await; - } - None } -fn thread_id_from_rollout_path(path: &Path) -> Option { - let file_name = path.file_name()?.to_str()?; - let stem = file_name.strip_suffix(".jsonl")?; - if stem.len() < 37 { - return None; +fn thread_store_archive_error(operation: &str, err: ThreadStoreError) -> JSONRPCErrorError { + match err { + ThreadStoreError::InvalidRequest { message } => JSONRPCErrorError { + code: INVALID_REQUEST_ERROR_CODE, + message, + data: None, + }, + err => JSONRPCErrorError { + code: INTERNAL_ERROR_CODE, + message: format!("failed to {operation} thread: {err}"), + data: None, + }, } - let uuid_start = stem.len().saturating_sub(36); - if !stem[..uuid_start].ends_with('-') { - return None; - } - ThreadId::from_string(&stem[uuid_start..]).ok() +} + +fn summary_from_stored_thread( + thread: StoredThread, + fallback_provider: &str, +) -> Option { + let path = thread.rollout_path?; + let source = with_thread_spawn_agent_metadata( + thread.source, + thread.agent_nickname.clone(), + thread.agent_role.clone(), + ); + let git_info = thread.git_info.map(|git| ConversationGitInfo { + sha: git.commit_hash.map(|sha| sha.0), + branch: git.branch, + origin_url: git.repository_url, + }); + Some(ConversationSummary { + conversation_id: thread.thread_id, + path, + preview: thread.first_user_message.unwrap_or(thread.preview), + timestamp: Some(thread.created_at.to_rfc3339_opts(SecondsFormat::Secs, true)), + updated_at: Some(thread.updated_at.to_rfc3339_opts(SecondsFormat::Secs, true)), + model_provider: if thread.model_provider.is_empty() { + fallback_provider.to_string() + } else { + thread.model_provider + }, + cwd: thread.cwd, + cli_version: thread.cli_version, + source, + git_info, + }) } #[allow(clippy::too_many_arguments)] diff --git a/codex-rs/app-server/src/config_api.rs b/codex-rs/app-server/src/config_api.rs index 4d0b450d73..2bc7447300 100644 --- a/codex-rs/app-server/src/config_api.rs +++ b/codex-rs/app-server/src/config_api.rs @@ -27,8 +27,8 @@ use codex_core::config_loader::LoaderOverrides; use codex_core::config_loader::ResidencyRequirement as CoreResidencyRequirement; use codex_core::config_loader::SandboxModeRequirement as CoreSandboxModeRequirement; use codex_core::plugins::PluginId; -use codex_core::plugins::collect_plugin_enabled_candidates; -use codex_core::plugins::installed_plugin_telemetry_metadata; +use codex_core_plugins::loader::installed_plugin_telemetry_metadata; +use codex_core_plugins::toggles::collect_plugin_enabled_candidates; use codex_features::canonical_feature_for_key; use codex_features::feature_for_key; use codex_protocol::config_types::WebSearchMode; diff --git a/codex-rs/app-server/src/external_agent_config_api.rs b/codex-rs/app-server/src/external_agent_config_api.rs index 3e20632db2..2f90a55a13 100644 --- a/codex-rs/app-server/src/external_agent_config_api.rs +++ b/codex-rs/app-server/src/external_agent_config_api.rs @@ -37,6 +37,7 @@ impl ExternalAgentConfigApi { include_home: params.include_home, cwds: params.cwds, }) + .await .map_err(map_io_error)?; Ok(ExternalAgentConfigDetectResponse { diff --git a/codex-rs/app-server/src/in_process.rs b/codex-rs/app-server/src/in_process.rs index 9ac0e6cf42..4458bce89d 100644 --- a/codex-rs/app-server/src/in_process.rs +++ b/codex-rs/app-server/src/in_process.rs @@ -82,6 +82,7 @@ use codex_exec_server::EnvironmentManager; use codex_feedback::CodexFeedback; use codex_login::AuthManager; use codex_protocol::protocol::SessionSource; +pub use codex_state::log_db::LogDbLayer; use tokio::sync::mpsc; use tokio::sync::oneshot; use tokio::time::timeout; @@ -117,6 +118,8 @@ pub struct InProcessStartArgs { pub cloud_requirements: CloudRequirementsLoader, /// Feedback sink used by app-server/core telemetry and logs. pub feedback: CodexFeedback, + /// SQLite tracing layer used to flush recently emitted logs before feedback upload. + pub log_db: Option, /// Environment manager used by core execution and filesystem operations. pub environment_manager: Arc, /// Startup warnings emitted after initialize succeeds. @@ -395,7 +398,7 @@ fn start_uninitialized(args: InProcessStartArgs) -> InProcessClientHandle { loader_overrides: args.loader_overrides, cloud_requirements: args.cloud_requirements, feedback: args.feedback, - log_db: None, + log_db: args.log_db, config_warnings: args.config_warnings, session_source: args.session_source, auth_manager, @@ -713,6 +716,7 @@ mod tests { match ConfigBuilder::default().build().await { Ok(config) => config, Err(_) => Config::load_default_with_cli_overrides(Vec::new()) + .await .expect("default config should load"), } } @@ -728,6 +732,7 @@ mod tests { loader_overrides: LoaderOverrides::default(), cloud_requirements: CloudRequirementsLoader::default(), feedback: CodexFeedback::new(), + log_db: None, environment_manager: Arc::new(EnvironmentManager::new(/*exec_server_url*/ None)), config_warnings: Vec::new(), session_source, diff --git a/codex-rs/app-server/src/lib.rs b/codex-rs/app-server/src/lib.rs index ca48ad0f41..ba946c67b6 100644 --- a/codex-rs/app-server/src/lib.rs +++ b/codex-rs/app-server/src/lib.rs @@ -432,12 +432,14 @@ pub async fn run_main_with_transport( Err(err) => { let message = config_warning_from_error("Invalid configuration; using defaults.", &err); config_warnings.push(message); - Config::load_default_with_cli_overrides(cli_kv_overrides.clone()).map_err(|e| { - std::io::Error::new( - ErrorKind::InvalidData, - format!("error loading default config after config error: {e}"), - ) - })? + Config::load_default_with_cli_overrides(cli_kv_overrides.clone()) + .await + .map_err(|e| { + std::io::Error::new( + ErrorKind::InvalidData, + format!("error loading default config after config error: {e}"), + ) + })? } }; diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index 023d744502..6221cebee5 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -725,8 +725,6 @@ impl MessageProcessor { session: Arc, request_context: RequestContext, ) { - let connection_id = connection_request_id.connection_id; - if !session.initialized() { let error = JSONRPCErrorError { code: INVALID_REQUEST_ERROR_CODE, @@ -748,6 +746,7 @@ impl MessageProcessor { self.outgoing.send_error(connection_request_id, error).await; return; } + let connection_id = connection_request_id.connection_id; if self.config.features.enabled(Feature::GeneralAnalytics) && let ClientRequest::TurnStart { request_id, .. } | ClientRequest::TurnSteer { request_id, .. } = &codex_request diff --git a/codex-rs/app-server/src/transport/auth.rs b/codex-rs/app-server/src/transport/auth.rs index a67c692b7d..45f44a36c9 100644 --- a/codex-rs/app-server/src/transport/auth.rs +++ b/codex-rs/app-server/src/transport/auth.rs @@ -34,6 +34,10 @@ pub struct AppServerWebsocketAuthArgs { #[arg(long = "ws-token-file", value_name = "PATH")] pub ws_token_file: Option, + /// Hex-encoded SHA-256 digest of the capability token. + #[arg(long = "ws-token-sha256", value_name = "HEX")] + pub ws_token_sha256: Option, + /// Absolute path to the shared secret file for signed JWT bearer tokens. #[arg(long = "ws-shared-secret-file", value_name = "PATH")] pub ws_shared_secret_file: Option, @@ -65,7 +69,7 @@ pub struct AppServerWebsocketAuthSettings { #[derive(Debug, Clone, PartialEq, Eq)] pub enum AppServerWebsocketAuthConfig { CapabilityToken { - token_file: AbsolutePathBuf, + source: AppServerWebsocketCapabilityTokenSource, }, SignedBearerToken { shared_secret_file: AbsolutePathBuf, @@ -75,6 +79,12 @@ pub enum AppServerWebsocketAuthConfig { }, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AppServerWebsocketCapabilityTokenSource { + TokenFile { token_file: AbsolutePathBuf }, + TokenSha256 { token_sha256: [u8; 32] }, +} + #[derive(Clone, Debug, Default)] pub(crate) struct WebsocketAuthPolicy { pub(crate) mode: Option, @@ -144,17 +154,34 @@ impl AppServerWebsocketAuthArgs { "`--ws-shared-secret-file`, `--ws-issuer`, `--ws-audience`, and `--ws-max-clock-skew-seconds` require `--ws-auth signed-bearer-token`" ); } - let token_file = self.ws_token_file.context( - "`--ws-token-file` is required when `--ws-auth capability-token` is set", - )?; - Some(AppServerWebsocketAuthConfig::CapabilityToken { - token_file: absolute_path_arg("--ws-token-file", token_file)?, - }) + let source = match (self.ws_token_file, self.ws_token_sha256) { + (Some(_), Some(_)) => { + anyhow::bail!( + "`--ws-token-file` and `--ws-token-sha256` are mutually exclusive" + ); + } + (Some(token_file), None) => { + AppServerWebsocketCapabilityTokenSource::TokenFile { + token_file: absolute_path_arg("--ws-token-file", token_file)?, + } + } + (None, Some(token_sha256)) => { + AppServerWebsocketCapabilityTokenSource::TokenSha256 { + token_sha256: sha256_digest_arg("--ws-token-sha256", &token_sha256)?, + } + } + (None, None) => { + anyhow::bail!( + "`--ws-token-file` or `--ws-token-sha256` is required when `--ws-auth capability-token` is set" + ); + } + }; + Some(AppServerWebsocketAuthConfig::CapabilityToken { source }) } Some(WebsocketAuthCliMode::SignedBearerToken) => { - if self.ws_token_file.is_some() { + if self.ws_token_file.is_some() || self.ws_token_sha256.is_some() { anyhow::bail!( - "`--ws-token-file` requires `--ws-auth capability-token`, not `signed-bearer-token`" + "`--ws-token-file` and `--ws-token-sha256` require `--ws-auth capability-token`, not `signed-bearer-token`" ); } let shared_secret_file = self.ws_shared_secret_file.context( @@ -174,6 +201,7 @@ impl AppServerWebsocketAuthArgs { } None => { if self.ws_token_file.is_some() + || self.ws_token_sha256.is_some() || self.ws_shared_secret_file.is_some() || self.ws_issuer.is_some() || self.ws_audience.is_some() @@ -195,12 +223,19 @@ pub(crate) fn policy_from_settings( settings: &AppServerWebsocketAuthSettings, ) -> io::Result { let mode = match settings.config.as_ref() { - Some(AppServerWebsocketAuthConfig::CapabilityToken { token_file }) => { - let token = read_trimmed_secret(token_file.as_ref())?; - Some(WebsocketAuthMode::CapabilityToken { - token_sha256: sha256_digest(token.as_bytes()), - }) - } + Some(AppServerWebsocketAuthConfig::CapabilityToken { source }) => match source { + AppServerWebsocketCapabilityTokenSource::TokenFile { token_file } => { + let token = read_trimmed_secret(token_file.as_ref())?; + Some(WebsocketAuthMode::CapabilityToken { + token_sha256: sha256_digest(token.as_bytes()), + }) + } + AppServerWebsocketCapabilityTokenSource::TokenSha256 { token_sha256 } => { + Some(WebsocketAuthMode::CapabilityToken { + token_sha256: *token_sha256, + }) + } + }, Some(AppServerWebsocketAuthConfig::SignedBearerToken { shared_secret_file, issuer, @@ -387,6 +422,30 @@ fn absolute_path_arg(flag_name: &str, path: PathBuf) -> anyhow::Result anyhow::Result<[u8; 32]> { + let trimmed = value.trim(); + if trimmed.len() != 64 { + anyhow::bail!("{flag_name} must be a 64-character hex SHA-256 digest"); + } + + let mut digest = [0u8; 32]; + for (index, pair) in trimmed.as_bytes().chunks_exact(2).enumerate() { + let high = hex_nibble(flag_name, pair[0])?; + let low = hex_nibble(flag_name, pair[1])?; + digest[index] = (high << 4) | low; + } + Ok(digest) +} + +fn hex_nibble(flag_name: &str, byte: u8) -> anyhow::Result { + match byte { + b'0'..=b'9' => Ok(byte - b'0'), + b'a'..=b'f' => Ok(byte - b'a' + 10), + b'A'..=b'F' => Ok(byte - b'A' + 10), + _ => anyhow::bail!("{flag_name} must be a 64-character hex SHA-256 digest"), + } +} + fn sha256_digest(input: &[u8]) -> [u8; 32] { let mut digest = [0u8; 32]; digest.copy_from_slice(&Sha256::digest(input)); @@ -403,6 +462,7 @@ fn unauthorized(message: &'static str) -> WebsocketAuthError { #[cfg(test)] mod tests { use super::*; + use axum::http::HeaderValue; use base64::Engine; use base64::engine::general_purpose::URL_SAFE_NO_PAD; use hmac::Hmac; @@ -443,19 +503,98 @@ mod tests { } #[test] - fn capability_token_args_require_token_file() { + fn capability_token_args_require_token_file_or_hash() { let err = AppServerWebsocketAuthArgs { ws_auth: Some(WebsocketAuthCliMode::CapabilityToken), ..Default::default() } .try_into_settings() - .expect_err("capability-token mode should require a token file"); + .expect_err("capability-token mode should require a token source"); assert!( - err.to_string().contains("--ws-token-file"), + err.to_string().contains("--ws-token-file") + && err.to_string().contains("--ws-token-sha256"), "unexpected error: {err}" ); } + #[test] + fn capability_token_args_accept_token_hash() { + let settings = AppServerWebsocketAuthArgs { + ws_auth: Some(WebsocketAuthCliMode::CapabilityToken), + ws_token_sha256: Some("ab".repeat(32)), + ..Default::default() + } + .try_into_settings() + .expect("capability-token hash args should parse"); + + assert_eq!( + settings, + AppServerWebsocketAuthSettings { + config: Some(AppServerWebsocketAuthConfig::CapabilityToken { + source: AppServerWebsocketCapabilityTokenSource::TokenSha256 { + token_sha256: [0xab; 32], + }, + }), + } + ); + } + + #[test] + fn capability_token_args_reject_multiple_token_sources() { + let err = AppServerWebsocketAuthArgs { + ws_auth: Some(WebsocketAuthCliMode::CapabilityToken), + ws_token_file: Some(PathBuf::from("/tmp/token")), + ws_token_sha256: Some("ab".repeat(32)), + ..Default::default() + } + .try_into_settings() + .expect_err("capability-token mode should reject multiple token sources"); + assert!( + err.to_string().contains("mutually exclusive"), + "unexpected error: {err}" + ); + } + + #[test] + fn capability_token_args_reject_malformed_token_hash() { + let err = AppServerWebsocketAuthArgs { + ws_auth: Some(WebsocketAuthCliMode::CapabilityToken), + ws_token_sha256: Some("not-a-sha256".to_string()), + ..Default::default() + } + .try_into_settings() + .expect_err("capability-token mode should reject malformed token hashes"); + assert!( + err.to_string().contains("64-character hex"), + "unexpected error: {err}" + ); + } + + #[test] + fn capability_token_hash_policy_authorizes_matching_bearer_token() { + let settings = AppServerWebsocketAuthSettings { + config: Some(AppServerWebsocketAuthConfig::CapabilityToken { + source: AppServerWebsocketCapabilityTokenSource::TokenSha256 { + token_sha256: sha256_digest(b"super-secret-token"), + }, + }), + }; + let policy = policy_from_settings(&settings).expect("hash policy should build"); + let mut headers = HeaderMap::new(); + headers.insert( + AUTHORIZATION, + HeaderValue::from_static("Bearer super-secret-token"), + ); + authorize_upgrade(&headers, &policy).expect("matching token should authorize"); + + headers.insert( + AUTHORIZATION, + HeaderValue::from_static("Bearer wrong-token"), + ); + let err = authorize_upgrade(&headers, &policy).expect_err("wrong token should fail"); + assert_eq!(err.status_code(), StatusCode::UNAUTHORIZED); + } + #[test] fn signed_bearer_args_require_mode_when_mode_specific_flags_are_set() { let err = AppServerWebsocketAuthArgs { diff --git a/codex-rs/app-server/src/transport/remote_control/mod.rs b/codex-rs/app-server/src/transport/remote_control/mod.rs index 1ea89bb643..c014c7a2c9 100644 --- a/codex-rs/app-server/src/transport/remote_control/mod.rs +++ b/codex-rs/app-server/src/transport/remote_control/mod.rs @@ -4,7 +4,6 @@ mod protocol; mod websocket; use crate::transport::remote_control::websocket::RemoteControlWebsocket; -use crate::transport::remote_control::websocket::load_remote_control_auth; pub use self::protocol::ClientId; use self::protocol::ServerEvent; @@ -59,9 +58,6 @@ pub(crate) async fn start_remote_control( } else { None }; - if initial_enabled { - validate_remote_control_auth(&auth_manager).await?; - } let (enabled_tx, enabled_rx) = watch::channel(initial_enabled); let join_handle = tokio::spawn(async move { @@ -86,15 +82,5 @@ pub(crate) async fn start_remote_control( )) } -pub(crate) async fn validate_remote_control_auth( - auth_manager: &Arc, -) -> io::Result<()> { - match load_remote_control_auth(auth_manager).await { - Ok(_) => Ok(()), - Err(err) if err.kind() == io::ErrorKind::WouldBlock => Ok(()), - Err(err) => Err(err), - } -} - #[cfg(test)] mod tests; diff --git a/codex-rs/app-server/src/transport/remote_control/protocol.rs b/codex-rs/app-server/src/transport/remote_control/protocol.rs index 857855f2a0..dcb79e5c35 100644 --- a/codex-rs/app-server/src/transport/remote_control/protocol.rs +++ b/codex-rs/app-server/src/transport/remote_control/protocol.rs @@ -48,9 +48,8 @@ pub enum ClientEvent { message: JSONRPCMessage, }, /// Backend-generated acknowledgement for all server envelopes addressed to - /// `client_id` whose envelope `seq_id` is less than or equal to this ack's - /// `seq_id`. This cursor is client-scoped, not stream-scoped, so receivers - /// must not use `stream_id` to partition acks. + /// `client_id` and `stream_id` whose envelope `seq_id` is less than or equal + /// to this ack's `seq_id`. This cursor is stream-scoped. Ack, Ping, ClientClosed, @@ -65,7 +64,7 @@ pub(crate) struct ClientEnvelope { pub(crate) client_id: ClientId, #[serde(rename = "stream_id", skip_serializing_if = "Option::is_none")] pub(crate) stream_id: Option, - /// For `Ack`, this is the backend-generated per-client cursor over + /// For `Ack`, this is the backend-generated per-stream cursor over /// `ServerEnvelope.seq_id`. #[serde(rename = "seq_id", skip_serializing_if = "Option::is_none")] pub(crate) seq_id: Option, diff --git a/codex-rs/app-server/src/transport/remote_control/tests.rs b/codex-rs/app-server/src/transport/remote_control/tests.rs index 21808a3a18..d702a99c2c 100644 --- a/codex-rs/app-server/src/transport/remote_control/tests.rs +++ b/codex-rs/app-server/src/transport/remote_control/tests.rs @@ -5,6 +5,7 @@ use super::enroll::update_persisted_remote_control_enrollment; use super::protocol::ClientEnvelope; use super::protocol::ClientEvent; use super::protocol::ClientId; +use super::protocol::StreamId; use super::protocol::normalize_remote_control_url; use super::websocket::REMOTE_CONTROL_PROTOCOL_VERSION; use super::*; @@ -96,6 +97,7 @@ fn remote_control_auth_dot_json(account_id: Option<&str>) -> AuthDotJson { account_id: account_id.map(str::to_string), }), last_refresh: Some(chrono::Utc::now()), + agent_identity: None, } } @@ -162,7 +164,7 @@ async fn remote_control_transport_manages_virtual_clients_and_routes_messages() json!({ "type": "pong", "client_id": "client-1", - "seq_id": 0, + "seq_id": 1, "status": "unknown", }) ); @@ -368,7 +370,7 @@ async fn remote_control_transport_manages_virtual_clients_and_routes_messages() json!({ "type": "pong", "client_id": "client-1", - "seq_id": 3, + "seq_id": 1, "status": "unknown", }) ); @@ -478,6 +480,44 @@ async fn remote_control_start_allows_remote_control_invalid_url_when_disabled() .expect("remote control task should join"); } +#[tokio::test] +async fn remote_control_start_allows_missing_auth_when_enabled() { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("listener should bind"); + let remote_control_url = remote_control_url_for_listener(&listener); + let codex_home = TempDir::new().expect("temp dir should create"); + let auth_manager = AuthManager::shared( + codex_home.path().to_path_buf(), + /*enable_codex_api_key_env*/ false, + AuthCredentialsStoreMode::File, + ); + let (transport_event_tx, _transport_event_rx) = + mpsc::channel::(CHANNEL_CAPACITY); + let shutdown_token = CancellationToken::new(); + let (remote_task, _remote_handle) = start_remote_control( + remote_control_url, + /*state_db*/ None, + auth_manager, + transport_event_tx, + shutdown_token.clone(), + /*app_server_client_name_rx*/ None, + /*initial_enabled*/ true, + ) + .await + .expect("remote control should start before ChatGPT auth is available"); + + timeout(Duration::from_millis(100), listener.accept()) + .await + .expect_err("remote control should wait for auth before connecting"); + + shutdown_token.cancel(); + timeout(Duration::from_secs(1), remote_task) + .await + .expect("remote control task should stop") + .expect("remote control task should join"); +} + #[tokio::test] async fn remote_control_handle_set_enabled_stops_and_restarts_connections() { let listener = TcpListener::bind("127.0.0.1:0") @@ -617,12 +657,13 @@ async fn remote_control_transport_clears_outgoing_buffer_when_backend_acks() { )) .await .expect("remote writer should accept outgoing message"); + let (server_event, stream_id) = read_server_event_with_stream_id(&mut first_websocket).await; assert_eq!( - read_server_event(&mut first_websocket).await, + server_event, json!({ "type": "server_message", "client_id": "client-1", - "seq_id": 0, + "seq_id": 1, "message": { "method": "configWarning", "params": { @@ -638,8 +679,8 @@ async fn remote_control_transport_clears_outgoing_buffer_when_backend_acks() { ClientEnvelope { event: ClientEvent::Ack, client_id: client_id.clone(), - stream_id: None, - seq_id: Some(0), + stream_id: Some(stream_id), + seq_id: Some(1), cursor: None, }, ) @@ -853,7 +894,7 @@ async fn remote_control_http_mode_enrolls_before_connecting() { json!({ "type": "server_message", "client_id": backend_client_id.0.clone(), - "seq_id": 0, + "seq_id": 1, "message": { "id": 11, "result": { @@ -881,7 +922,7 @@ async fn remote_control_http_mode_enrolls_before_connecting() { json!({ "type": "server_message", "client_id": backend_client_id.0.clone(), - "seq_id": 1, + "seq_id": 2, "message": { "method": "configWarning", "params": { @@ -1355,6 +1396,12 @@ async fn send_client_event( } async fn read_server_event(websocket: &mut WebSocketStream) -> serde_json::Value { + read_server_event_with_stream_id(websocket).await.0 +} + +async fn read_server_event_with_stream_id( + websocket: &mut WebSocketStream, +) -> (serde_json::Value, StreamId) { loop { let frame = timeout(Duration::from_secs(5), websocket.next()) .await @@ -1365,13 +1412,15 @@ async fn read_server_event(websocket: &mut WebSocketStream) -> serde_ tungstenite::Message::Text(text) => { let mut event: serde_json::Value = serde_json::from_str(text.as_ref()).expect("server event should deserialize"); - if let Some(stream_id) = event + let stream_id = event .as_object_mut() .and_then(|event| event.remove("stream_id")) - { - assert!(stream_id.is_string(), "stream_id should be a string"); - } - return event; + .expect("stream_id should be present"); + let stream_id = stream_id + .as_str() + .expect("stream_id should be a string") + .to_string(); + return (event, StreamId(stream_id)); } tungstenite::Message::Ping(payload) => { websocket diff --git a/codex-rs/app-server/src/transport/remote_control/websocket.rs b/codex-rs/app-server/src/transport/remote_control/websocket.rs index 42924c2d34..d813be1f6a 100644 --- a/codex-rs/app-server/src/transport/remote_control/websocket.rs +++ b/codex-rs/app-server/src/transport/remote_control/websocket.rs @@ -14,6 +14,7 @@ use super::protocol::ClientEvent; use super::protocol::ClientId; use super::protocol::RemoteControlTarget; use super::protocol::ServerEnvelope; +use super::protocol::StreamId; use axum::http::HeaderValue; use base64::Engine; use codex_core::util::backoff; @@ -25,8 +26,8 @@ use futures::SinkExt; use futures::StreamExt; use futures::stream::SplitSink; use futures::stream::SplitStream; -use std::collections::BTreeMap; use std::collections::HashMap; +use std::collections::VecDeque; use std::io; use std::io::ErrorKind; use std::sync::Arc; @@ -57,10 +58,7 @@ const REMOTE_CONTROL_ACCOUNT_ID_RETRY_INTERVAL: std::time::Duration = std::time::Duration::from_secs(1); struct BoundedOutboundBuffer { - // Remote-control acks are generated by the backend at client scope, so - // retransmit retention is keyed by client_id only. stream_id stays on each - // envelope for routing, but it is not part of the ack cursor. - buffer_by_client: HashMap>, + buffer_by_stream: HashMap<(ClientId, StreamId), VecDeque>, used_tx: watch::Sender, } @@ -68,46 +66,50 @@ impl BoundedOutboundBuffer { fn new() -> (Self, watch::Receiver) { let (used_tx, used_rx) = watch::channel(0); let buffer = Self { - buffer_by_client: HashMap::new(), + buffer_by_stream: HashMap::new(), used_tx, }; (buffer, used_rx) } fn insert(&mut self, server_envelope: &ServerEnvelope) { - self.buffer_by_client - .entry(server_envelope.client_id.clone()) + self.buffer_by_stream + .entry(( + server_envelope.client_id.clone(), + server_envelope.stream_id.clone(), + )) .or_default() - .insert(server_envelope.seq_id, server_envelope.clone()); + .push_back(server_envelope.clone()); self.used_tx.send_modify(|used| *used += 1); } - fn ack(&mut self, client_id: &ClientId, acked_seq_id: u64) { - let Some(buffer) = self.buffer_by_client.get_mut(client_id) else { + fn ack(&mut self, client_id: &ClientId, stream_id: &StreamId, acked_seq_id: u64) { + let key = (client_id.clone(), stream_id.clone()); + let Some(buffer) = self.buffer_by_stream.get_mut(&key) else { return; }; - while let Some(seq_id) = buffer.first_key_value().map(|(seq_id, _)| seq_id) - && *seq_id <= acked_seq_id + while let Some(server_envelope) = buffer.front() + && server_envelope.seq_id <= acked_seq_id { - buffer.pop_first(); + buffer.pop_front(); self.used_tx.send_modify(|used| *used -= 1); } if buffer.is_empty() { - self.buffer_by_client.remove(client_id); + self.buffer_by_stream.remove(&key); } } fn server_envelopes(&self) -> impl Iterator { - self.buffer_by_client + self.buffer_by_stream .values() - .flat_map(|buffer| buffer.values()) + .flat_map(|buffer| buffer.iter()) } } struct WebsocketState { outbound_buffer: BoundedOutboundBuffer, subscribe_cursor: Option, - next_seq_id: u64, + next_seq_id_by_stream: HashMap<(ClientId, StreamId), u64>, } pub(crate) struct RemoteControlWebsocket { @@ -162,7 +164,7 @@ impl RemoteControlWebsocket { state: Arc::new(Mutex::new(WebsocketState { outbound_buffer, subscribe_cursor: None, - next_seq_id: 0, + next_seq_id_by_stream: HashMap::new(), })), server_event_rx: Arc::new(Mutex::new(server_event_rx)), used_rx, @@ -445,10 +447,16 @@ impl RemoteControlWebsocket { } } }; - let (server_envelope, write_complete_tx) = { + let (payload, write_complete_tx) = { let mut state = state.lock().await; - let seq_id = state.next_seq_id; - state.next_seq_id = state.next_seq_id.saturating_add(1); + let seq_key = ( + queued_server_envelope.client_id.clone(), + queued_server_envelope.stream_id.clone(), + ); + let seq_id = *state + .next_seq_id_by_stream + .entry(seq_key.clone()) + .or_insert(1); let server_envelope = ServerEnvelope { event: queued_server_envelope.event, @@ -456,17 +464,19 @@ impl RemoteControlWebsocket { seq_id, stream_id: queued_server_envelope.stream_id, }; + let payload = match serde_json::to_string(&server_envelope) { + Ok(payload) => payload, + Err(err) => { + error!("failed to serialize remote-control server event: {err}"); + continue; + } + }; + state + .next_seq_id_by_stream + .insert(seq_key, seq_id.saturating_add(1)); state.outbound_buffer.insert(&server_envelope); - (server_envelope, queued_server_envelope.write_complete_tx) - }; - - let payload = match serde_json::to_string(&server_envelope) { - Ok(payload) => payload, - Err(err) => { - error!("failed to serialize remote-control server event: {err}"); - continue; - } + (payload, queued_server_envelope.write_complete_tx) }; tokio::select! { @@ -590,10 +600,13 @@ impl RemoteControlWebsocket { } if let ClientEvent::Ack = &client_envelope.event && let Some(acked_seq_id) = client_envelope.seq_id + && let Some(stream_id) = client_envelope.stream_id.as_ref() { - websocket_state - .outbound_buffer - .ack(&client_envelope.client_id, acked_seq_id); + websocket_state.outbound_buffer.ack( + &client_envelope.client_id, + stream_id, + acked_seq_id, + ); } drop(websocket_state); @@ -920,6 +933,13 @@ mod tests { use tokio::time::timeout; use tokio_tungstenite::accept_async; + // Windows Bazel CI can take longer than a few seconds for the websocket + // client connection attempt to reach the local test listener. + #[cfg(windows)] + const TEST_HTTP_ACCEPT_TIMEOUT: Duration = Duration::from_secs(30); + #[cfg(not(windows))] + const TEST_HTTP_ACCEPT_TIMEOUT: Duration = Duration::from_secs(5); + async fn remote_control_state_runtime(codex_home: &TempDir) -> Arc { StateRuntime::init(codex_home.path().to_path_buf(), "test-provider".to_string()) .await @@ -971,6 +991,7 @@ mod tests { account_id: Some("account_id".to_string()), }), last_refresh: Some(Utc::now()), + agent_identity: None, } } @@ -1219,7 +1240,7 @@ mod tests { let state = Arc::new(Mutex::new(WebsocketState { outbound_buffer, subscribe_cursor: None, - next_seq_id: 0, + next_seq_id_by_stream: HashMap::new(), })); let (_server_event_tx, server_event_rx) = mpsc::channel(super::super::CHANNEL_CAPACITY); let server_event_rx = Arc::new(Mutex::new(server_event_rx)); @@ -1247,6 +1268,83 @@ mod tests { .expect("writer should stop cleanly"); } + #[tokio::test] + async fn run_server_writer_inner_assigns_contiguous_seq_ids_per_stream() { + let (client_stream, mut server_stream) = connected_websocket_pair().await; + let (websocket_writer, _websocket_reader) = client_stream.split(); + let (outbound_buffer, used_rx) = BoundedOutboundBuffer::new(); + let state = Arc::new(Mutex::new(WebsocketState { + outbound_buffer, + subscribe_cursor: None, + next_seq_id_by_stream: HashMap::new(), + })); + let (server_event_tx, server_event_rx) = mpsc::channel(super::super::CHANNEL_CAPACITY); + let server_event_rx = Arc::new(Mutex::new(server_event_rx)); + let shutdown_token = CancellationToken::new(); + let writer_task = tokio::spawn(RemoteControlWebsocket::run_server_writer_inner( + state, + server_event_rx, + used_rx, + websocket_writer, + Duration::from_secs(60), + shutdown_token.clone(), + )); + + let client_id = ClientId("client-1".to_string()); + let first_stream = StreamId("stream-1".to_string()); + let second_stream = StreamId("stream-2".to_string()); + for stream_id in [&first_stream, &second_stream, &first_stream] { + server_event_tx + .send(super::super::QueuedServerEnvelope { + event: ServerEvent::Pong { + status: crate::transport::remote_control::protocol::PongStatus::Active, + }, + client_id: client_id.clone(), + stream_id: stream_id.clone(), + write_complete_tx: None, + }) + .await + .expect("server event should queue"); + } + + assert_eq!( + read_server_text_event(&mut server_stream).await, + serde_json::json!({ + "type": "pong", + "client_id": "client-1", + "stream_id": "stream-1", + "seq_id": 1, + "status": "active", + }) + ); + assert_eq!( + read_server_text_event(&mut server_stream).await, + serde_json::json!({ + "type": "pong", + "client_id": "client-1", + "stream_id": "stream-2", + "seq_id": 1, + "status": "active", + }) + ); + assert_eq!( + read_server_text_event(&mut server_stream).await, + serde_json::json!({ + "type": "pong", + "client_id": "client-1", + "stream_id": "stream-1", + "seq_id": 2, + "status": "active", + }) + ); + + shutdown_token.cancel(); + writer_task + .await + .expect("writer task should join") + .expect("writer should stop cleanly"); + } + #[tokio::test] async fn run_websocket_reader_inner_times_out_without_pong_frames() { let (client_stream, _server_stream) = connected_websocket_pair().await; @@ -1255,7 +1353,7 @@ mod tests { let state = Arc::new(Mutex::new(WebsocketState { outbound_buffer, subscribe_cursor: None, - next_seq_id: 0, + next_seq_id_by_stream: HashMap::new(), })); let (server_event_tx, _server_event_rx) = mpsc::channel(super::super::CHANNEL_CAPACITY); let (transport_event_tx, _transport_event_rx) = @@ -1286,70 +1384,32 @@ mod tests { } #[test] - fn outbound_buffer_acks_by_client_id_across_stream_ids() { + fn outbound_buffer_acks_by_stream_id() { let (mut outbound_buffer, used_rx) = BoundedOutboundBuffer::new(); let client_1 = ClientId("client-1".to_string()); let client_2 = ClientId("client-2".to_string()); + let stream_1 = StreamId("stream-1".to_string()); outbound_buffer.insert(&server_envelope( &client_1, "stream-1", - /*seq_id*/ 0, + /*seq_id*/ 1, "first-client-old-stream", )); outbound_buffer.insert(&server_envelope( &client_2, "stream-1", - /*seq_id*/ 1, + /*seq_id*/ 2, "second-client", )); outbound_buffer.insert(&server_envelope( &client_1, "stream-2", - /*seq_id*/ 2, + /*seq_id*/ 3, "first-client-new-stream", )); - outbound_buffer.ack(&client_1, /*acked_seq_id*/ 2); - - let mut retained = outbound_buffer - .server_envelopes() - .map(|server_envelope| { - ( - server_envelope.client_id.0.as_str(), - server_envelope.stream_id.0.as_str(), - server_envelope.seq_id, - ) - }) - .collect::>(); - retained.sort_unstable(); - assert_eq!(retained, vec![("client-2", "stream-1", 1)]); - assert_eq!(*used_rx.borrow(), 1); - } - - #[test] - fn outbound_buffer_retains_unacked_messages_until_ack_advances() { - let (mut outbound_buffer, used_rx) = BoundedOutboundBuffer::new(); - let client_1 = ClientId("client-1".to_string()); - let client_2 = ClientId("client-2".to_string()); - - outbound_buffer.insert(&server_envelope( - &client_1, - "stream-1", - /*seq_id*/ 0, - "first-old", - )); - outbound_buffer.insert(&server_envelope( - &client_1, - "stream-2", - /*seq_id*/ 1, - "first-new", - )); - outbound_buffer.insert(&server_envelope( - &client_2, "stream-1", /*seq_id*/ 2, "second", - )); - - outbound_buffer.ack(&client_1, /*acked_seq_id*/ 0); + outbound_buffer.ack(&client_1, &stream_1, /*acked_seq_id*/ 3); let mut retained = outbound_buffer .server_envelopes() @@ -1364,7 +1424,50 @@ mod tests { retained.sort_unstable(); assert_eq!( retained, - vec![("client-1", "stream-2", 1), ("client-2", "stream-1", 2)] + vec![("client-1", "stream-2", 3), ("client-2", "stream-1", 2)] + ); + assert_eq!(*used_rx.borrow(), 2); + } + + #[test] + fn outbound_buffer_retains_unacked_messages_until_ack_advances() { + let (mut outbound_buffer, used_rx) = BoundedOutboundBuffer::new(); + let client_1 = ClientId("client-1".to_string()); + let client_2 = ClientId("client-2".to_string()); + let stream_1 = StreamId("stream-1".to_string()); + + outbound_buffer.insert(&server_envelope( + &client_1, + "stream-1", + /*seq_id*/ 1, + "first-old", + )); + outbound_buffer.insert(&server_envelope( + &client_1, + "stream-2", + /*seq_id*/ 2, + "first-new", + )); + outbound_buffer.insert(&server_envelope( + &client_2, "stream-1", /*seq_id*/ 3, "second", + )); + + outbound_buffer.ack(&client_1, &stream_1, /*acked_seq_id*/ 1); + + let mut retained = outbound_buffer + .server_envelopes() + .map(|server_envelope| { + ( + server_envelope.client_id.0.as_str(), + server_envelope.stream_id.0.as_str(), + server_envelope.seq_id, + ) + }) + .collect::>(); + retained.sort_unstable(); + assert_eq!( + retained, + vec![("client-1", "stream-2", 2), ("client-2", "stream-1", 3)] ); assert_eq!(*used_rx.borrow(), 2); } @@ -1393,7 +1496,7 @@ mod tests { } async fn accept_http_request(listener: &TcpListener) -> (TcpStream, String) { - let (stream, _) = timeout(Duration::from_secs(5), listener.accept()) + let (stream, _) = timeout(TEST_HTTP_ACCEPT_TIMEOUT, listener.accept()) .await .expect("HTTP request should arrive in time") .expect("listener accept should succeed"); @@ -1449,6 +1552,20 @@ mod tests { (client_stream, server_stream) } + async fn read_server_text_event( + server_stream: &mut WebSocketStream, + ) -> serde_json::Value { + let message = timeout(Duration::from_secs(5), server_stream.next()) + .await + .expect("server event should arrive in time") + .expect("server websocket should stay open") + .expect("server event should read"); + let tungstenite::Message::Text(text) = message else { + panic!("expected text event, got {message:?}"); + }; + serde_json::from_str(text.as_ref()).expect("server event should deserialize") + } + async fn respond_with_status_and_headers( mut stream: TcpStream, status: &str, diff --git a/codex-rs/app-server/tests/common/auth_fixtures.rs b/codex-rs/app-server/tests/common/auth_fixtures.rs index 99334f0770..86f0fb456d 100644 --- a/codex-rs/app-server/tests/common/auth_fixtures.rs +++ b/codex-rs/app-server/tests/common/auth_fixtures.rs @@ -163,6 +163,7 @@ pub fn write_chatgpt_auth( openai_api_key: None, tokens: Some(tokens), last_refresh, + agent_identity: None, }; save_auth(codex_home, &auth, cli_auth_credentials_store_mode).context("write auth.json") diff --git a/codex-rs/app-server/tests/suite/auth.rs b/codex-rs/app-server/tests/suite/auth.rs index e6134e4802..1e60871012 100644 --- a/codex-rs/app-server/tests/suite/auth.rs +++ b/codex-rs/app-server/tests/suite/auth.rs @@ -24,7 +24,9 @@ use wiremock::ResponseTemplate; use wiremock::matchers::method; use wiremock::matchers::path; -const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); +// Bazel CI can spend tens of seconds starting app-server subprocesses or +// processing auth RPCs under load. +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); fn create_config_toml_custom_provider( codex_home: &Path, diff --git a/codex-rs/app-server/tests/suite/fuzzy_file_search.rs b/codex-rs/app-server/tests/suite/fuzzy_file_search.rs index 1520d99e3e..f508e0c987 100644 --- a/codex-rs/app-server/tests/suite/fuzzy_file_search.rs +++ b/codex-rs/app-server/tests/suite/fuzzy_file_search.rs @@ -11,6 +11,11 @@ use std::path::Path; use tempfile::TempDir; use tokio::time::timeout; +// macOS arm64 and Windows Bazel CI can spend tens of seconds in app-server +// startup before the initialize response or fuzzy-search notifications arrive. +#[cfg(any(target_os = "macos", windows))] +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); +#[cfg(not(any(target_os = "macos", windows)))] const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); const SHORT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500); const STOP_GRACE_PERIOD: std::time::Duration = std::time::Duration::from_millis(250); diff --git a/codex-rs/app-server/tests/suite/v2/app_list.rs b/codex-rs/app-server/tests/suite/v2/app_list.rs index 57a27961af..78a915d178 100644 --- a/codex-rs/app-server/tests/suite/v2/app_list.rs +++ b/codex-rs/app-server/tests/suite/v2/app_list.rs @@ -56,7 +56,9 @@ use tokio::net::TcpListener; use tokio::task::JoinHandle; use tokio::time::timeout; -const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10); +// Bazel CI can spend tens of seconds starting app-server subprocesses or +// processing app-list RPCs under load. +const DEFAULT_TIMEOUT: Duration = Duration::from_secs(60); #[tokio::test] async fn list_apps_returns_empty_when_connectors_disabled() -> Result<()> { @@ -117,6 +119,7 @@ async fn list_apps_returns_empty_with_api_key_auth() -> Result<()> { openai_api_key: Some("test-api-key".to_string()), tokens: None, last_refresh: None, + agent_identity: None, }, AuthCredentialsStoreMode::File, )?; diff --git a/codex-rs/app-server/tests/suite/v2/client_metadata.rs b/codex-rs/app-server/tests/suite/v2/client_metadata.rs index c85febd7d4..8d68888e7e 100644 --- a/codex-rs/app-server/tests/suite/v2/client_metadata.rs +++ b/codex-rs/app-server/tests/suite/v2/client_metadata.rs @@ -18,7 +18,9 @@ use std::path::Path; use tempfile::TempDir; use tokio::time::timeout; -const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); +// Bazel CI can spend tens of seconds starting app-server subprocesses or +// processing turn RPCs under load. +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); #[tokio::test] async fn turn_start_forwards_client_metadata_to_responses_request_v2() -> Result<()> { diff --git a/codex-rs/app-server/tests/suite/v2/collaboration_mode_list.rs b/codex-rs/app-server/tests/suite/v2/collaboration_mode_list.rs index 7c36827e6d..3c8a3e573e 100644 --- a/codex-rs/app-server/tests/suite/v2/collaboration_mode_list.rs +++ b/codex-rs/app-server/tests/suite/v2/collaboration_mode_list.rs @@ -21,7 +21,9 @@ use pretty_assertions::assert_eq; use tempfile::TempDir; use tokio::time::timeout; -const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10); +// Bazel CI can spend tens of seconds starting app-server subprocesses or +// processing list RPCs under load. +const DEFAULT_TIMEOUT: Duration = Duration::from_secs(60); /// Confirms the server returns the default collaboration mode presets in a stable order. #[tokio::test] diff --git a/codex-rs/app-server/tests/suite/v2/command_exec.rs b/codex-rs/app-server/tests/suite/v2/command_exec.rs index 50e8285665..d7645f13e3 100644 --- a/codex-rs/app-server/tests/suite/v2/command_exec.rs +++ b/codex-rs/app-server/tests/suite/v2/command_exec.rs @@ -411,11 +411,14 @@ async fn command_exec_streaming_does_not_buffer_output() -> Result<()> { }) .await?; - let delta = read_command_exec_delta(&mut mcp).await?; - assert_eq!(delta.process_id, process_id.as_str()); - assert_eq!(delta.stream, CommandExecOutputStream::Stdout); - assert_eq!(STANDARD.decode(&delta.delta_base64)?, b"abcde"); - assert!(delta.cap_reached); + let output = collect_command_exec_output_until( + CommandExecDeltaReader::Mcp(&mut mcp), + process_id.as_str(), + "capped stdout", + |_output, delta| delta.stream == CommandExecOutputStream::Stdout && delta.cap_reached, + ) + .await?; + assert_eq!(output.stdout, "abcde"); let terminate_request_id = mcp .send_command_exec_terminate_request(CommandExecTerminateParams { process_id: process_id.clone(), @@ -471,21 +474,13 @@ async fn command_exec_pipe_streams_output_and_accepts_write() -> Result<()> { }) .await?; - let first_stdout = read_command_exec_delta(&mut mcp).await?; - let first_stderr = read_command_exec_delta(&mut mcp).await?; - let seen = [first_stdout, first_stderr]; - assert!( - seen.iter() - .all(|delta| delta.process_id == process_id.as_str()) - ); - assert!(seen.iter().any(|delta| { - delta.stream == CommandExecOutputStream::Stdout - && delta.delta_base64 == STANDARD.encode("out-start\n") - })); - assert!(seen.iter().any(|delta| { - delta.stream == CommandExecOutputStream::Stderr - && delta.delta_base64 == STANDARD.encode("err-start\n") - })); + wait_for_command_exec_outputs_contains( + &mut mcp, + process_id.as_str(), + "out-start\n", + "err-start\n", + ) + .await?; let write_request_id = mcp .send_command_exec_write_request(CommandExecWriteParams { @@ -499,21 +494,13 @@ async fn command_exec_pipe_streams_output_and_accepts_write() -> Result<()> { .await?; assert_eq!(write_response.result, serde_json::json!({})); - let next_delta = read_command_exec_delta(&mut mcp).await?; - let final_delta = read_command_exec_delta(&mut mcp).await?; - let seen = [next_delta, final_delta]; - assert!( - seen.iter() - .all(|delta| delta.process_id == process_id.as_str()) - ); - assert!(seen.iter().any(|delta| { - delta.stream == CommandExecOutputStream::Stdout - && delta.delta_base64 == STANDARD.encode("out:hello\n") - })); - assert!(seen.iter().any(|delta| { - delta.stream == CommandExecOutputStream::Stderr - && delta.delta_base64 == STANDARD.encode("err:hello\n") - })); + wait_for_command_exec_outputs_contains( + &mut mcp, + process_id.as_str(), + "out:hello\n", + "err:hello\n", + ) + .await?; let response = mcp .read_stream_until_response_message(RequestId::Integer(command_request_id)) @@ -562,17 +549,13 @@ async fn command_exec_tty_implies_streaming_and_reports_pty_output() -> Result<( }) .await?; - let started_text = read_command_exec_output_until_contains( + wait_for_command_exec_output_contains( &mut mcp, process_id.as_str(), CommandExecOutputStream::Stdout, "tty\n", ) .await?; - assert!( - started_text.contains("tty\n"), - "expected TTY startup output, got {started_text:?}" - ); let write_request_id = mcp .send_command_exec_write_request(CommandExecWriteParams { @@ -586,17 +569,13 @@ async fn command_exec_tty_implies_streaming_and_reports_pty_output() -> Result<( .await?; assert_eq!(write_response.result, serde_json::json!({})); - let echoed_text = read_command_exec_output_until_contains( + wait_for_command_exec_output_contains( &mut mcp, process_id.as_str(), CommandExecOutputStream::Stdout, "echo:world\n", ) .await?; - assert!( - echoed_text.contains("echo:world\n"), - "expected TTY echo output, got {echoed_text:?}" - ); let response = mcp .read_stream_until_response_message(RequestId::Integer(command_request_id)) @@ -643,17 +622,13 @@ async fn command_exec_tty_supports_initial_size_and_resize() -> Result<()> { }) .await?; - let started_text = read_command_exec_output_until_contains( + wait_for_command_exec_output_contains( &mut mcp, process_id.as_str(), CommandExecOutputStream::Stdout, "start:31 101\n", ) .await?; - assert!( - started_text.contains("start:31 101\n"), - "unexpected initial size output: {started_text:?}" - ); let resize_request_id = mcp .send_command_exec_resize_request(CommandExecResizeParams { @@ -681,17 +656,13 @@ async fn command_exec_tty_supports_initial_size_and_resize() -> Result<()> { .await?; assert_eq!(write_response.result, serde_json::json!({})); - let resized_text = read_command_exec_output_until_contains( + wait_for_command_exec_output_contains( &mut mcp, process_id.as_str(), CommandExecOutputStream::Stdout, "after:45 132\n", ) .await?; - assert!( - resized_text.contains("after:45 132\n"), - "unexpected resized output: {resized_text:?}" - ); let response = mcp .read_stream_until_response_message(RequestId::Integer(command_request_id)) @@ -744,11 +715,13 @@ async fn command_exec_process_ids_are_connection_scoped_and_disconnect_terminate ) .await?; - let delta = read_command_exec_delta_ws(&mut ws1).await?; - assert_eq!(delta.process_id, "shared-process"); - assert_eq!(delta.stream, CommandExecOutputStream::Stdout); - let delta_text = String::from_utf8(STANDARD.decode(&delta.delta_base64)?)?; - assert!(delta_text.contains("ready")); + collect_command_exec_output_until( + CommandExecDeltaReader::Websocket(&mut ws1), + "shared-process", + "websocket ready output", + |output, _delta| output.stdout.contains("ready\n"), + ) + .await?; wait_for_process_marker(&marker, /*should_exist*/ true).await?; send_request( @@ -796,31 +769,98 @@ async fn read_command_exec_delta( decode_delta_notification(notification) } -async fn read_command_exec_output_until_contains( +async fn wait_for_command_exec_output_contains( mcp: &mut McpProcess, process_id: &str, stream: CommandExecOutputStream, expected: &str, -) -> Result { +) -> Result<()> { + let stream_name = match stream { + CommandExecOutputStream::Stdout => "stdout", + CommandExecOutputStream::Stderr => "stderr", + }; + collect_command_exec_output_until( + CommandExecDeltaReader::Mcp(mcp), + process_id, + format!("{stream_name} containing {expected:?}"), + |output, _delta| match stream { + CommandExecOutputStream::Stdout => output.stdout.contains(expected), + CommandExecOutputStream::Stderr => output.stderr.contains(expected), + }, + ) + .await?; + Ok(()) +} + +async fn wait_for_command_exec_outputs_contains( + mcp: &mut McpProcess, + process_id: &str, + stdout_expected: &str, + stderr_expected: &str, +) -> Result<()> { + collect_command_exec_output_until( + CommandExecDeltaReader::Mcp(mcp), + process_id, + format!("stdout containing {stdout_expected:?} and stderr containing {stderr_expected:?}"), + |output, _delta| { + output.stdout.contains(stdout_expected) && output.stderr.contains(stderr_expected) + }, + ) + .await?; + Ok(()) +} + +enum CommandExecDeltaReader<'a> { + Mcp(&'a mut McpProcess), + Websocket(&'a mut super::connection_handling_websocket::WsClient), +} + +#[derive(Default)] +struct CollectedCommandExecOutput { + stdout: String, + stderr: String, +} + +async fn collect_command_exec_output_until( + mut reader: CommandExecDeltaReader<'_>, + process_id: &str, + waiting_for: impl Into, + mut should_stop: impl FnMut( + &CollectedCommandExecOutput, + &CommandExecOutputDeltaNotification, + ) -> bool, +) -> Result { + let waiting_for = waiting_for.into(); let deadline = Instant::now() + DEFAULT_READ_TIMEOUT; - let mut collected = String::new(); + let mut output = CollectedCommandExecOutput::default(); loop { let remaining = deadline.saturating_duration_since(Instant::now()); - let delta = timeout(remaining, read_command_exec_delta(mcp)) - .await - .with_context(|| { - format!( - "timed out waiting for {expected:?} in command/exec output for {process_id}; collected {collected:?}" - ) - })??; + let delta = timeout(remaining, async { + match &mut reader { + CommandExecDeltaReader::Mcp(mcp) => read_command_exec_delta(mcp).await, + CommandExecDeltaReader::Websocket(stream) => { + read_command_exec_delta_ws(stream).await + } + } + }) + .await + .with_context(|| { + format!( + "timed out waiting for {waiting_for} in command/exec output for {process_id}; collected stdout={:?}, stderr={:?}", + output.stdout, output.stderr + ) + })??; assert_eq!(delta.process_id, process_id); - assert_eq!(delta.stream, stream); let delta_text = String::from_utf8(STANDARD.decode(&delta.delta_base64)?)?; - collected.push_str(&delta_text.replace('\r', "")); - if collected.contains(expected) { - return Ok(collected); + let delta_text = delta_text.replace('\r', ""); + match delta.stream { + CommandExecOutputStream::Stdout => output.stdout.push_str(&delta_text), + CommandExecOutputStream::Stderr => output.stderr.push_str(&delta_text), + } + if should_stop(&output, &delta) { + return Ok(output); } } } diff --git a/codex-rs/app-server/tests/suite/v2/compaction.rs b/codex-rs/app-server/tests/suite/v2/compaction.rs index e7661546ac..44b5dd6dc6 100644 --- a/codex-rs/app-server/tests/suite/v2/compaction.rs +++ b/codex-rs/app-server/tests/suite/v2/compaction.rs @@ -38,6 +38,11 @@ use std::collections::BTreeMap; use tempfile::TempDir; use tokio::time::timeout; +// macOS and Windows Bazel CI can spend tens of seconds starting app-server +// subprocesses or processing test RPCs under load. +#[cfg(any(target_os = "macos", windows))] +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); +#[cfg(not(any(target_os = "macos", windows)))] const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); const AUTO_COMPACT_LIMIT: i64 = 1_000; const COMPACT_PROMPT: &str = "Summarize the conversation."; diff --git a/codex-rs/app-server/tests/suite/v2/config_rpc.rs b/codex-rs/app-server/tests/suite/v2/config_rpc.rs index 5c04cc3c44..b5f795740c 100644 --- a/codex-rs/app-server/tests/suite/v2/config_rpc.rs +++ b/codex-rs/app-server/tests/suite/v2/config_rpc.rs @@ -33,7 +33,9 @@ use serde_json::json; use tempfile::TempDir; use tokio::time::timeout; -const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); +// Bazel CI can spend tens of seconds starting app-server subprocesses or +// processing config RPCs under load. +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); fn write_config(codex_home: &TempDir, contents: &str) -> Result<()> { Ok(std::fs::write( diff --git a/codex-rs/app-server/tests/suite/v2/connection_handling_websocket.rs b/codex-rs/app-server/tests/suite/v2/connection_handling_websocket.rs index 05d26d55e3..0cc8b51e56 100644 --- a/codex-rs/app-server/tests/suite/v2/connection_handling_websocket.rs +++ b/codex-rs/app-server/tests/suite/v2/connection_handling_websocket.rs @@ -47,6 +47,12 @@ use tokio_tungstenite::tungstenite::http::HeaderValue; use tokio_tungstenite::tungstenite::http::header::AUTHORIZATION; use tokio_tungstenite::tungstenite::http::header::ORIGIN; +// macOS and Windows CI can spend tens of seconds starting the app-server test +// binary under Bazel before it accepts JSON-RPC or reports its websocket bind +// address. +#[cfg(any(target_os = "macos", windows))] +pub(super) const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(60); +#[cfg(not(any(target_os = "macos", windows)))] pub(super) const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(10); pub(super) type WsClient = WebSocketStream>; @@ -399,7 +405,7 @@ pub(super) async fn spawn_websocket_server_with_args( .take() .context("failed to capture websocket app-server stderr")?; let mut stderr_reader = BufReader::new(stderr).lines(); - let deadline = Instant::now() + Duration::from_secs(10); + let deadline = Instant::now() + DEFAULT_READ_TIMEOUT; let bind_addr = loop { let line = timeout( deadline.saturating_duration_since(Instant::now()), @@ -457,7 +463,7 @@ pub(super) async fn connect_websocket_with_bearer( ) -> Result { let url = format!("ws://{}", connectable_bind_addr(bind_addr)); let request = websocket_request(url.as_str(), bearer_token, /*origin*/ None)?; - let deadline = Instant::now() + Duration::from_secs(10); + let deadline = Instant::now() + DEFAULT_READ_TIMEOUT; loop { match connect_async(request.clone()).await { Ok((stream, _response)) => return Ok(stream), @@ -524,7 +530,7 @@ async fn run_websocket_server_to_completion_with_args( .stderr(Stdio::piped()) .env("CODEX_HOME", codex_home) .env("RUST_LOG", "debug"); - timeout(Duration::from_secs(10), cmd.output()) + timeout(DEFAULT_READ_TIMEOUT, cmd.output()) .await .context("timed out waiting for websocket app-server to exit")? .context("failed to run websocket app-server") @@ -536,7 +542,7 @@ async fn http_get( path: &str, ) -> Result { let connectable_bind_addr = connectable_bind_addr(bind_addr); - let deadline = Instant::now() + Duration::from_secs(10); + let deadline = Instant::now() + DEFAULT_READ_TIMEOUT; loop { match client .get(format!("http://{connectable_bind_addr}{path}")) diff --git a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs index 0ab3f47235..0a3315a07a 100644 --- a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs +++ b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs @@ -34,6 +34,11 @@ use tempfile::TempDir; use tokio::time::timeout; use wiremock::MockServer; +// macOS and Windows Bazel CI can spend tens of seconds starting app-server +// subprocesses or processing test RPCs under load. +#[cfg(any(target_os = "macos", windows))] +const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(60); +#[cfg(not(any(target_os = "macos", windows)))] const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(10); /// Ensures dynamic tool specs are serialized into the model request payload. diff --git a/codex-rs/app-server/tests/suite/v2/fs.rs b/codex-rs/app-server/tests/suite/v2/fs.rs index c7f28f09f5..642844eb92 100644 --- a/codex-rs/app-server/tests/suite/v2/fs.rs +++ b/codex-rs/app-server/tests/suite/v2/fs.rs @@ -27,6 +27,11 @@ use std::os::unix::fs::symlink; #[cfg(unix)] use std::process::Command; +// macOS and Windows Bazel CI can spend tens of seconds starting app-server +// subprocesses or processing test RPCs under load. +#[cfg(any(target_os = "macos", windows))] +const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(60); +#[cfg(not(any(target_os = "macos", windows)))] const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(10); async fn initialized_mcp(codex_home: &TempDir) -> Result { diff --git a/codex-rs/app-server/tests/suite/v2/mcp_server_elicitation.rs b/codex-rs/app-server/tests/suite/v2/mcp_server_elicitation.rs index 862bdf8487..13ebe0b99c 100644 --- a/codex-rs/app-server/tests/suite/v2/mcp_server_elicitation.rs +++ b/codex-rs/app-server/tests/suite/v2/mcp_server_elicitation.rs @@ -66,8 +66,9 @@ use tokio::time::timeout; const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); const CONNECTOR_ID: &str = "calendar"; const CONNECTOR_NAME: &str = "Calendar"; +const TOOL_NAMESPACE: &str = "mcp__codex_apps__calendar"; +const CALLABLE_TOOL_NAME: &str = "_confirm_action"; const TOOL_NAME: &str = "calendar_confirm_action"; -const QUALIFIED_TOOL_NAME: &str = "mcp__codex_apps__calendar_confirm_action"; const TOOL_CALL_ID: &str = "call-calendar-confirm"; const ELICITATION_MESSAGE: &str = "Allow this request?"; @@ -85,9 +86,10 @@ async fn mcp_server_elicitation_round_trip() -> Result<()> { ]), responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call( + responses::ev_function_call_with_namespace( TOOL_CALL_ID, - QUALIFIED_TOOL_NAME, + TOOL_NAMESPACE, + CALLABLE_TOOL_NAME, &tool_call_arguments, ), responses::ev_completed("resp-1"), diff --git a/codex-rs/app-server/tests/suite/v2/memory_reset.rs b/codex-rs/app-server/tests/suite/v2/memory_reset.rs new file mode 100644 index 0000000000..3c7ae38671 --- /dev/null +++ b/codex-rs/app-server/tests/suite/v2/memory_reset.rs @@ -0,0 +1,145 @@ +use anyhow::Result; +use app_test_support::McpProcess; +use app_test_support::to_response; +use chrono::Utc; +use codex_app_server_protocol::JSONRPCResponse; +use codex_app_server_protocol::MemoryResetResponse; +use codex_app_server_protocol::RequestId; +use codex_protocol::ThreadId; +use codex_protocol::protocol::SessionSource; +use codex_state::Stage1JobClaimOutcome; +use codex_state::StateRuntime; +use codex_state::ThreadMetadataBuilder; +use pretty_assertions::assert_eq; +use std::path::Path; +use std::sync::Arc; +use tempfile::TempDir; +use tokio::time::timeout; +use uuid::Uuid; + +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + +#[tokio::test] +async fn memory_reset_clears_memory_files_and_rows_preserves_threads() -> Result<()> { + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path())?; + let state_db = init_state_db(codex_home.path()).await?; + + let memory_root = codex_home.path().join("memories"); + tokio::fs::create_dir_all(memory_root.join("rollout_summaries")).await?; + tokio::fs::write(memory_root.join("MEMORY.md"), "stale memory\n").await?; + tokio::fs::write( + memory_root.join("rollout_summaries").join("stale.md"), + "stale rollout summary\n", + ) + .await?; + + let thread_id = seed_stage1_output(&state_db, codex_home.path()).await?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let request_id = mcp + .send_raw_request("memory/reset", /*params*/ None) + .await?; + let response: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let _: MemoryResetResponse = to_response::(response)?; + + let stage1_outputs = state_db.list_stage1_outputs_for_global(/*n*/ 10).await?; + assert_eq!(stage1_outputs, Vec::new()); + assert_eq!( + state_db.get_thread_memory_mode(thread_id).await?.as_deref(), + Some("enabled") + ); + + let mut remaining_entries = tokio::fs::read_dir(&memory_root).await?; + assert!( + remaining_entries.next_entry().await?.is_none(), + "memory root should be empty after reset" + ); + + Ok(()) +} + +async fn seed_stage1_output(state_db: &Arc, codex_home: &Path) -> Result { + let now = Utc::now(); + let thread_id = ThreadId::from_string(&Uuid::new_v4().to_string())?; + let worker_id = ThreadId::from_string(&Uuid::new_v4().to_string())?; + let mut builder = ThreadMetadataBuilder::new( + thread_id, + codex_home.join("sessions").join("test.jsonl"), + now, + SessionSource::Cli, + ); + builder.updated_at = Some(now); + builder.cwd = codex_home.to_path_buf(); + let metadata = builder.build("mock_provider"); + state_db.upsert_thread(&metadata).await?; + + let claim = state_db + .try_claim_stage1_job( + thread_id, + worker_id, + now.timestamp(), + /*lease_seconds*/ 3600, + /*max_running_jobs*/ 64, + ) + .await?; + let Stage1JobClaimOutcome::Claimed { ownership_token } = claim else { + anyhow::bail!("unexpected stage1 claim outcome: {claim:?}"); + }; + assert!( + state_db + .mark_stage1_job_succeeded( + thread_id, + ownership_token.as_str(), + now.timestamp(), + "raw memory", + "rollout summary", + /*rollout_slug*/ None, + ) + .await?, + "stage1 success should be recorded" + ); + state_db + .enqueue_global_consolidation(now.timestamp()) + .await?; + + Ok(thread_id) +} + +async fn init_state_db(codex_home: &Path) -> Result> { + let state_db = StateRuntime::init(codex_home.to_path_buf(), "mock_provider".into()).await?; + state_db + .mark_backfill_complete(/*last_watermark*/ None) + .await?; + Ok(state_db) +} + +fn create_config_toml(codex_home: &Path) -> std::io::Result<()> { + let config_toml = codex_home.join("config.toml"); + std::fs::write( + config_toml, + r#" +model = "mock-model" +approval_policy = "never" +sandbox_mode = "read-only" +model_provider = "mock_provider" +suppress_unstable_features_warning = true + +[features] +sqlite = true + +[model_providers.mock_provider] +name = "Mock provider for test" +base_url = "http://127.0.0.1:9/v1" +wire_api = "responses" +request_max_retries = 0 +stream_max_retries = 0 +"#, + ) +} diff --git a/codex-rs/app-server/tests/suite/v2/mod.rs b/codex-rs/app-server/tests/suite/v2/mod.rs index 95e07c031e..bbf04006e6 100644 --- a/codex-rs/app-server/tests/suite/v2/mod.rs +++ b/codex-rs/app-server/tests/suite/v2/mod.rs @@ -20,6 +20,7 @@ mod mcp_resource; mod mcp_server_elicitation; mod mcp_server_status; mod mcp_tool; +mod memory_reset; mod model_list; mod output_schema; mod plan_item; diff --git a/codex-rs/app-server/tests/suite/v2/plugin_install.rs b/codex-rs/app-server/tests/suite/v2/plugin_install.rs index a3bea53172..e51fac725f 100644 --- a/codex-rs/app-server/tests/suite/v2/plugin_install.rs +++ b/codex-rs/app-server/tests/suite/v2/plugin_install.rs @@ -51,7 +51,9 @@ use wiremock::matchers::header; use wiremock::matchers::method; use wiremock::matchers::path; -const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10); +// Plugin install tests wait on connector discovery after the install response path +// starts, which is noticeably slower on Windows CI. +const DEFAULT_TIMEOUT: Duration = Duration::from_secs(60); #[tokio::test] async fn plugin_install_rejects_relative_marketplace_paths() -> Result<()> { diff --git a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs index 9870d410f1..1f7b8a3e4b 100644 --- a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs +++ b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs @@ -75,6 +75,8 @@ const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10); const STARTUP_CONTEXT_HEADER: &str = "Startup context from Codex."; const V2_STEERING_ACKNOWLEDGEMENT: &str = "This was sent to steer the previous background agent task."; +const V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT: &str = + "Background agent finished. Use the preceding [BACKEND] messages as the result."; #[derive(Debug, Clone, Copy)] enum StartupContextConfig<'a> { @@ -1359,7 +1361,8 @@ async fn webrtc_v2_forwards_audio_and_text_between_client_and_sideband() -> Resu request["type"] == "conversation.item.create" && request["item"]["type"] == "message" && request["item"]["role"] == "user" - && request["item"]["content"][0]["text"] == "hello" + && request["item"]["content"][0]["type"] == "input_text" + && request["item"]["content"][0]["text"] == "[USER] hello" }), "sideband requests should include user text item: {requests:?}" ); @@ -1558,7 +1561,7 @@ async fn webrtc_v2_background_agent_tool_call_delegates_and_returns_function_out assert_v2_progress_update(&progress, "delegated from v2"); let tool_output = harness.sideband_outbound_request(/*request_index*/ 2).await; - assert_v2_function_call_output(&tool_output, "call_v2", "delegated from v2"); + assert_v2_function_call_output(&tool_output, "call_v2", V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT); assert_eq!( function_call_output_sideband_requests(&harness.realtime_server).len(), 1 @@ -1693,7 +1696,11 @@ async fn webrtc_v2_background_agent_progress_is_sent_before_function_output() -> assert_v2_progress_update(&progress, "progress before final"); let tool_output = harness.sideband_outbound_request(/*request_index*/ 2).await; - assert_v2_function_call_output(&tool_output, "call_progress_order", "progress before final"); + assert_v2_function_call_output( + &tool_output, + "call_progress_order", + V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT, + ); harness.shutdown().await; Ok(()) @@ -1777,7 +1784,11 @@ async fn webrtc_v2_tool_call_delegated_turn_can_execute_shell_tool() -> Result<( assert_v2_progress_update(&progress, "shell tool finished"); let tool_output = harness.sideband_outbound_request(/*request_index*/ 2).await; - assert_v2_function_call_output(&tool_output, "call_shell", "shell tool finished"); + assert_v2_function_call_output( + &tool_output, + "call_shell", + V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT, + ); assert_eq!( function_call_output_sideband_requests(&harness.realtime_server).len(), 1 @@ -1857,7 +1868,11 @@ async fn webrtc_v2_tool_call_does_not_block_sideband_audio() -> Result<()> { assert_v2_progress_update(&progress, "late delegated result"); let tool_output = harness.sideband_outbound_request(/*request_index*/ 2).await; - assert_v2_function_call_output(&tool_output, "call_audio", "late delegated result"); + assert_v2_function_call_output( + &tool_output, + "call_audio", + V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT, + ); harness.shutdown().await; Ok(()) @@ -2090,7 +2105,7 @@ fn assert_v2_function_call_output(request: &Value, call_id: &str, expected_outpu "item": { "type": "function_call_output", "call_id": call_id, - "output": format!("\"Agent Final Message\":\n\n{expected_output}"), + "output": expected_output, } }) ); @@ -2106,7 +2121,7 @@ fn assert_v2_progress_update(request: &Value, expected_text: &str) { "role": "user", "content": [{ "type": "input_text", - "text": format!("{expected_text}\n\nUpdate from background agent (task hasn't finished yet):") + "text": format!("[BACKEND] {expected_text}") }] } }) @@ -2123,7 +2138,7 @@ fn assert_v2_user_text_item(request: &Value, expected_text: &str) { "role": "user", "content": [{ "type": "input_text", - "text": expected_text + "text": format!("[USER] {expected_text}") }] } }) diff --git a/codex-rs/app-server/tests/suite/v2/thread_fork.rs b/codex-rs/app-server/tests/suite/v2/thread_fork.rs index 576a46d643..649919db8e 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_fork.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_fork.rs @@ -43,6 +43,9 @@ use super::analytics::enable_analytics_capture; use super::analytics::thread_initialized_event; use super::analytics::wait_for_analytics_payload; +#[cfg(windows)] +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(25); +#[cfg(not(windows))] const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); #[tokio::test] diff --git a/codex-rs/app-server/tests/suite/v2/thread_list.rs b/codex-rs/app-server/tests/suite/v2/thread_list.rs index cf1ac4ff11..8fd4e47303 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_list.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_list.rs @@ -563,15 +563,34 @@ sqlite = true /*git_info*/ None, )?; - // `thread/list` only applies `search_term` on the sqlite path. In this test we - // create rollouts manually, so we must also create the sqlite DB and mark backfill - // complete; otherwise app-server will permanently use filesystem fallback. + // `thread/list` applies `search_term` on the sqlite fast path. This test creates + // rollouts manually, so mark the DB backfill complete and then run an unsearched + // list large enough to repair every rollout the searched list should find. let state_db = codex_state::StateRuntime::init(codex_home.path().to_path_buf(), "mock_provider".into()) .await?; state_db .mark_backfill_complete(/*last_watermark*/ None) .await?; + let rollout_config = codex_rollout::RolloutConfig { + codex_home: codex_home.path().to_path_buf(), + sqlite_home: codex_home.path().to_path_buf(), + cwd: codex_home.path().to_path_buf(), + model_provider_id: "mock_provider".to_string(), + generate_memories: false, + }; + let repaired_page = codex_core::RolloutRecorder::list_threads( + &rollout_config, + /*page_size*/ 10, + /*cursor*/ None, + codex_core::ThreadSortKey::CreatedAt, + &[], + /*model_providers*/ None, + "mock_provider", + /*search_term*/ None, + ) + .await?; + assert_eq!(repaired_page.items.len(), 3); let mut mcp = init_mcp(codex_home.path()).await?; let request_id = mcp diff --git a/codex-rs/app-server/tests/suite/v2/thread_read.rs b/codex-rs/app-server/tests/suite/v2/thread_read.rs index e4ff900150..4ab1181624 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_read.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_read.rs @@ -36,6 +36,9 @@ use std::path::Path; use tempfile::TempDir; use tokio::time::timeout; +#[cfg(windows)] +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(25); +#[cfg(not(windows))] const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); #[tokio::test] diff --git a/codex-rs/app-server/tests/suite/v2/thread_resume.rs b/codex-rs/app-server/tests/suite/v2/thread_resume.rs index db3a0897fc..46dee9b5d1 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_resume.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_resume.rs @@ -76,6 +76,9 @@ use super::analytics::enable_analytics_capture; use super::analytics::thread_initialized_event; use super::analytics::wait_for_analytics_payload; +#[cfg(windows)] +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(25); +#[cfg(not(windows))] const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); const CODEX_5_2_INSTRUCTIONS_TEMPLATE_DEFAULT: &str = "You are Codex, a coding agent based on GPT-5. You and the user share the same workspace and collaborate to achieve the user's goals."; diff --git a/codex-rs/app-server/tests/suite/v2/thread_start.rs b/codex-rs/app-server/tests/suite/v2/thread_start.rs index b8d0db9a01..ab514252e4 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_start.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_start.rs @@ -717,6 +717,7 @@ model_reasoning_effort = "high" let config_toml = std::fs::read_to_string(codex_home.path().join("config.toml"))?; let trusted_root = resolve_root_git_project_for_trust(workspace.path()) + .await .unwrap_or_else(|| workspace.path().to_path_buf()); assert!(config_toml.contains(&persisted_trust_path(&trusted_root))); assert!(config_toml.contains("trust_level = \"trusted\"")); @@ -753,8 +754,9 @@ async fn thread_start_with_nested_git_cwd_trusts_repo_root() -> Result<()> { .await??; let config_toml = std::fs::read_to_string(codex_home.path().join("config.toml"))?; - let trusted_root = - resolve_root_git_project_for_trust(&nested).expect("git root should resolve"); + let trusted_root = resolve_root_git_project_for_trust(&nested) + .await + .expect("git root should resolve"); assert!(config_toml.contains(&persisted_trust_path(&trusted_root))); assert!(!config_toml.contains(&persisted_trust_path(&nested))); diff --git a/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs b/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs index 6aab3d186f..7a3ae61b72 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs @@ -31,13 +31,17 @@ use tempfile::TempDir; use tokio::time::timeout; const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); +// Windows CI can spend most of the default read timeout launching PowerShell +// before the command finishes and the follow-up model request is sent. +const TURN_COMPLETION_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); async fn wait_for_responses_request_count_to_stabilize( server: &wiremock::MockServer, expected_count: usize, settle_duration: std::time::Duration, + timeout_duration: std::time::Duration, ) -> Result<()> { - timeout(DEFAULT_READ_TIMEOUT, async { + timeout(timeout_duration, async { let mut stable_since: Option = None; loop { let requests = server @@ -130,9 +134,9 @@ async fn thread_unsubscribe_keeps_thread_loaded_until_idle_timeout() -> Result<( async fn thread_unsubscribe_during_turn_keeps_turn_running() -> Result<()> { #[cfg(target_os = "windows")] let shell_command = vec![ - "powershell".to_string(), - "-Command".to_string(), - "Start-Sleep -Seconds 1".to_string(), + "Start-Sleep".to_string(), + "-Seconds".to_string(), + "1".to_string(), ]; #[cfg(not(target_os = "windows"))] let shell_command = vec!["sleep".to_string(), "1".to_string()]; @@ -210,6 +214,7 @@ async fn thread_unsubscribe_during_turn_keeps_turn_running() -> Result<()> { &server, /*expected_count*/ 2, std::time::Duration::from_millis(200), + TURN_COMPLETION_TIMEOUT, ) .await?; diff --git a/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs b/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs index b553137752..f8eaf799da 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs @@ -127,10 +127,17 @@ async fn turn_interrupt_aborts_running_turn() -> Result<()> { #[tokio::test] async fn turn_interrupt_resolves_pending_command_approval_request() -> Result<()> { + #[cfg(target_os = "windows")] + let shell_command = vec![ + "powershell".to_string(), + "-Command".to_string(), + "Start-Sleep -Seconds 10".to_string(), + ]; + #[cfg(not(target_os = "windows"))] let shell_command = vec![ "python3".to_string(), "-c".to_string(), - "print(42)".to_string(), + "import time; time.sleep(10)".to_string(), ]; let tmp = TempDir::new()?; @@ -143,7 +150,7 @@ async fn turn_interrupt_resolves_pending_command_approval_request() -> Result<() shell_command.clone(), Some(&working_directory), Some(10_000), - "call_python_approval", + "call_sleep_approval", )?]) .await; create_config_toml(&codex_home, &server.uri(), "untrusted", "read-only")?; @@ -172,6 +179,7 @@ async fn turn_interrupt_resolves_pending_command_approval_request() -> Result<() text_elements: Vec::new(), }], cwd: Some(working_directory), + approval_policy: Some(codex_app_server_protocol::AskForApproval::UnlessTrusted), ..Default::default() }) .await?; @@ -190,7 +198,7 @@ async fn turn_interrupt_resolves_pending_command_approval_request() -> Result<() let ServerRequest::CommandExecutionRequestApproval { request_id, params } = request else { panic!("expected CommandExecutionRequestApproval request"); }; - assert_eq!(params.item_id, "call_python_approval"); + assert_eq!(params.item_id, "call_sleep_approval"); assert_eq!(params.thread_id, thread.id); assert_eq!(params.turn_id, turn.id); @@ -251,6 +259,7 @@ fn create_config_toml( r#" model = "mock-model" approval_policy = "{approval_policy}" +approvals_reviewer = "user" sandbox_mode = "{sandbox_mode}" model_provider = "mock_provider" diff --git a/codex-rs/backend-client/src/client.rs b/codex-rs/backend-client/src/client.rs index e024974a0e..d5242fdf53 100644 --- a/codex-rs/backend-client/src/client.rs +++ b/codex-rs/backend-client/src/client.rs @@ -119,6 +119,7 @@ pub struct Client { bearer_token: Option, user_agent: Option, chatgpt_account_id: Option, + chatgpt_account_is_fedramp: bool, path_style: PathStyle, } @@ -144,6 +145,7 @@ impl Client { bearer_token: None, user_agent: None, chatgpt_account_id: None, + chatgpt_account_is_fedramp: false, path_style, }) } @@ -156,6 +158,9 @@ impl Client { if let Some(account_id) = auth.get_account_id() { client = client.with_chatgpt_account_id(account_id); } + if auth.is_fedramp_account() { + client = client.with_fedramp_routing_header(); + } Ok(client) } @@ -176,6 +181,11 @@ impl Client { self } + pub fn with_fedramp_routing_header(mut self) -> Self { + self.chatgpt_account_is_fedramp = true; + self + } + pub fn with_path_style(mut self, style: PathStyle) -> Self { self.path_style = style; self @@ -200,6 +210,11 @@ impl Client { { h.insert(name, hv); } + if self.chatgpt_account_is_fedramp + && let Ok(name) = HeaderName::from_bytes(b"X-OpenAI-Fedramp") + { + h.insert(name, HeaderValue::from_static("true")); + } h } diff --git a/codex-rs/chatgpt/Cargo.toml b/codex-rs/chatgpt/Cargo.toml index 381b4fc873..354449934a 100644 --- a/codex-rs/chatgpt/Cargo.toml +++ b/codex-rs/chatgpt/Cargo.toml @@ -10,6 +10,7 @@ workspace = true [dependencies] anyhow = { workspace = true } clap = { workspace = true, features = ["derive"] } +codex-app-server-protocol = { workspace = true } codex-connectors = { workspace = true } codex-config = { workspace = true } codex-core = { workspace = true } diff --git a/codex-rs/chatgpt/src/connectors.rs b/codex-rs/chatgpt/src/connectors.rs index f37609be31..5f6efbc124 100644 --- a/codex-rs/chatgpt/src/connectors.rs +++ b/codex-rs/chatgpt/src/connectors.rs @@ -1,7 +1,3 @@ -use codex_core::config::Config; -use codex_login::AuthManager; -use codex_login::CodexAuth; -use codex_login::token_data::TokenData; use std::collections::HashSet; use std::time::Duration; @@ -9,21 +5,24 @@ use crate::chatgpt_client::chatgpt_get_request_with_timeout; use crate::chatgpt_token::get_chatgpt_token_data; use crate::chatgpt_token::init_chatgpt_token_from_auth; +use codex_app_server_protocol::AppInfo; use codex_connectors::AllConnectorsCacheKey; use codex_connectors::DirectoryListResponse; - -pub use codex_core::connectors::AppInfo; -pub use codex_core::connectors::connector_display_label; -use codex_core::connectors::filter_disallowed_connectors; +use codex_connectors::filter::filter_disallowed_connectors; +use codex_connectors::merge::merge_connectors; +use codex_connectors::merge::merge_plugin_connectors; +use codex_core::config::Config; pub use codex_core::connectors::list_accessible_connectors_from_mcp_tools; pub use codex_core::connectors::list_accessible_connectors_from_mcp_tools_with_options; pub use codex_core::connectors::list_accessible_connectors_from_mcp_tools_with_options_and_status; pub use codex_core::connectors::list_cached_accessible_connectors_from_mcp_tools; -use codex_core::connectors::merge_connectors; -use codex_core::connectors::merge_plugin_apps; pub use codex_core::connectors::with_app_enabled_state; use codex_core::plugins::AppConnectorId; use codex_core::plugins::PluginsManager; +use codex_login::AuthManager; +use codex_login::CodexAuth; +use codex_login::default_client::originator; +use codex_login::token_data::TokenData; const DIRECTORY_CONNECTORS_TIMEOUT: Duration = Duration::from_secs(60); @@ -74,8 +73,17 @@ pub async fn list_cached_all_connectors(config: &Config) -> Option> let token_data = get_chatgpt_token_data()?; let cache_key = all_connectors_cache_key(config, &token_data); let connectors = codex_connectors::cached_all_connectors(&cache_key)?; - let connectors = merge_plugin_apps(connectors, plugin_apps_for_config(config).await); - Some(filter_disallowed_connectors(connectors)) + let connectors = merge_plugin_connectors( + connectors, + plugin_apps_for_config(config) + .await + .into_iter() + .map(|connector_id| connector_id.0), + ); + Some(filter_disallowed_connectors( + connectors, + originator().value.as_str(), + )) } pub async fn list_all_connectors_with_options( @@ -105,8 +113,17 @@ pub async fn list_all_connectors_with_options( }, ) .await?; - let connectors = merge_plugin_apps(connectors, plugin_apps_for_config(config).await); - Ok(filter_disallowed_connectors(connectors)) + let connectors = merge_plugin_connectors( + connectors, + plugin_apps_for_config(config) + .await + .into_iter() + .map(|connector_id| connector_id.0), + ); + Ok(filter_disallowed_connectors( + connectors, + originator().value.as_str(), + )) } fn all_connectors_cache_key(config: &Config, token_data: &TokenData) -> AllConnectorsCacheKey { @@ -134,7 +151,13 @@ pub fn connectors_for_plugin_apps( .map(|connector_id| connector_id.0.as_str()) .collect::>(); - filter_disallowed_connectors(merge_plugin_apps(connectors, plugin_apps.to_vec())) + let connectors = merge_plugin_connectors( + connectors, + plugin_apps + .iter() + .map(|connector_id| connector_id.0.clone()), + ); + filter_disallowed_connectors(connectors, originator().value.as_str()) .into_iter() .filter(|connector| plugin_app_ids.contains(connector.id.as_str())) .collect() @@ -158,13 +181,13 @@ pub fn merge_connectors_with_accessible( accessible_connectors }; let merged = merge_connectors(connectors, accessible_connectors); - filter_disallowed_connectors(merged) + filter_disallowed_connectors(merged, originator().value.as_str()) } #[cfg(test)] mod tests { use super::*; - use codex_core::connectors::connector_install_url; + use codex_connectors::metadata::connector_install_url; use codex_core::plugins::AppConnectorId; use pretty_assertions::assert_eq; @@ -186,46 +209,6 @@ mod tests { } } - #[test] - fn allows_asdk_connectors() { - let filtered = filter_disallowed_connectors(vec![app("asdk_app_hidden"), app("alpha")]); - assert_eq!(filtered, vec![app("asdk_app_hidden"), app("alpha")]); - } - - #[test] - fn allows_whitelisted_asdk_connectors() { - let filtered = filter_disallowed_connectors(vec![ - app("asdk_app_69781557cc1481919cf5e9824fa2e792"), - app("beta"), - ]); - assert_eq!( - filtered, - vec![ - app("asdk_app_69781557cc1481919cf5e9824fa2e792"), - app("beta") - ] - ); - } - - #[test] - fn filters_openai_prefixed_connectors() { - let filtered = filter_disallowed_connectors(vec![ - app("connector_openai_foo"), - app("connector_openai_bar"), - app("gamma"), - ]); - assert_eq!(filtered, vec![app("gamma")]); - } - - #[test] - fn filters_disallowed_connector_ids() { - let filtered = filter_disallowed_connectors(vec![ - app("asdk_app_6938a94a61d881918ef32cb999ff937c"), - app("delta"), - ]); - assert_eq!(filtered, vec![app("delta")]); - } - fn merged_app(id: &str, is_accessible: bool) -> AppInfo { AppInfo { id: id.to_string(), diff --git a/codex-rs/cli/Cargo.toml b/codex-rs/cli/Cargo.toml index 60caf888f8..11d3eb3bbd 100644 --- a/codex-rs/cli/Cargo.toml +++ b/codex-rs/cli/Cargo.toml @@ -45,6 +45,7 @@ codex-state = { workspace = true } codex-stdio-to-uds = { workspace = true } codex-terminal-detection = { workspace = true } codex-tui = { workspace = true } +codex-utils-absolute-path = { workspace = true } codex-utils-path = { workspace = true } libc = { workspace = true } owo-colors = { workspace = true } diff --git a/codex-rs/cli/src/debug_sandbox.rs b/codex-rs/cli/src/debug_sandbox.rs index 0ad2f36382..a2cbf61874 100644 --- a/codex-rs/cli/src/debug_sandbox.rs +++ b/codex-rs/cli/src/debug_sandbox.rs @@ -18,7 +18,10 @@ use codex_protocol::config_types::SandboxMode; use codex_protocol::permissions::NetworkSandboxPolicy; use codex_sandboxing::landlock::create_linux_sandbox_command_args_for_policies; #[cfg(target_os = "macos")] -use codex_sandboxing::seatbelt::create_seatbelt_command_args_for_policies; +use codex_sandboxing::seatbelt::CreateSeatbeltCommandArgsParams; +#[cfg(target_os = "macos")] +use codex_sandboxing::seatbelt::create_seatbelt_command_args; +use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_cli::CliConfigOverrides; use tokio::process::Child; use tokio::process::Command as TokioCommand; @@ -39,6 +42,7 @@ pub async fn run_command_under_seatbelt( ) -> anyhow::Result<()> { let SeatbeltCommand { full_auto, + allow_unix_sockets, log_denials, config_overrides, command, @@ -50,6 +54,7 @@ pub async fn run_command_under_seatbelt( codex_linux_sandbox_exe, SandboxType::Seatbelt, log_denials, + &allow_unix_sockets, ) .await } @@ -78,6 +83,7 @@ pub async fn run_command_under_landlock( codex_linux_sandbox_exe, SandboxType::Landlock, /*log_denials*/ false, + &[], ) .await } @@ -98,6 +104,7 @@ pub async fn run_command_under_windows( codex_linux_sandbox_exe, SandboxType::Windows, /*log_denials*/ false, + &[], ) .await } @@ -116,6 +123,8 @@ async fn run_command_under_sandbox( codex_linux_sandbox_exe: Option, sandbox_type: SandboxType, log_denials: bool, + #[cfg_attr(not(target_os = "macos"), allow(unused_variables))] + allow_unix_sockets: &[AbsolutePathBuf], ) -> anyhow::Result<()> { let config = load_debug_sandbox_config( config_overrides @@ -252,14 +261,15 @@ async fn run_command_under_sandbox( let mut child = match sandbox_type { #[cfg(target_os = "macos")] SandboxType::Seatbelt => { - let args = create_seatbelt_command_args_for_policies( + let args = create_seatbelt_command_args(CreateSeatbeltCommandArgsParams { command, - &config.permissions.file_system_sandbox_policy, - config.permissions.network_sandbox_policy, - sandbox_policy_cwd.as_path(), - /*enforce_managed_network*/ false, - network.as_ref(), - ); + file_system_sandbox_policy: &config.permissions.file_system_sandbox_policy, + network_sandbox_policy: config.permissions.network_sandbox_policy, + sandbox_policy_cwd: sandbox_policy_cwd.as_path(), + enforce_managed_network: false, + network: network.as_ref(), + extra_allow_unix_sockets: allow_unix_sockets, + }); let network_policy = config.permissions.network_sandbox_policy; spawn_debug_sandbox_child( PathBuf::from("/usr/bin/sandbox-exec"), diff --git a/codex-rs/cli/src/lib.rs b/codex-rs/cli/src/lib.rs index 848391293c..cac34b3b61 100644 --- a/codex-rs/cli/src/lib.rs +++ b/codex-rs/cli/src/lib.rs @@ -3,6 +3,7 @@ mod exit_status; pub(crate) mod login; use clap::Parser; +use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_cli::CliConfigOverrides; pub use debug_sandbox::run_command_under_landlock; @@ -22,6 +23,10 @@ pub struct SeatbeltCommand { #[arg(long = "full-auto", default_value_t = false)] pub full_auto: bool, + /// Allow the sandboxed command to bind/connect AF_UNIX sockets rooted at this path. Relative paths are resolved against the current directory. Repeat to allow multiple paths. + #[arg(long = "allow-unix-socket", value_parser = parse_allow_unix_socket_path)] + pub allow_unix_sockets: Vec, + /// While the command runs, capture macOS sandbox denials via `log stream` and print them after exit #[arg(long = "log-denials", default_value_t = false)] pub log_denials: bool, @@ -34,6 +39,11 @@ pub struct SeatbeltCommand { pub command: Vec, } +fn parse_allow_unix_socket_path(raw: &str) -> Result { + AbsolutePathBuf::relative_to_current_dir(raw) + .map_err(|err| format!("invalid path {raw}: {err}")) +} + #[derive(Debug, Parser)] pub struct LandlockCommand { /// Convenience alias for low-friction sandboxed automatic execution (network-disabled sandbox that can write to cwd and TMPDIR) diff --git a/codex-rs/cli/src/main.rs b/codex-rs/cli/src/main.rs index 5ad86a80fd..25bf80a6e3 100644 --- a/codex-rs/cli/src/main.rs +++ b/codex-rs/cli/src/main.rs @@ -49,6 +49,7 @@ use crate::mcp_cmd::McpCli; use crate::responses_cmd::ResponsesCommand; use crate::responses_cmd::run_responses_command; +use codex_core::clear_memory_roots_contents; use codex_core::config::Config; use codex_core::config::ConfigOverrides; use codex_core::config::edit::ConfigEditsBuilder; @@ -516,10 +517,19 @@ fn run_update_action(action: UpdateAction) -> anyhow::Result<()> { let status = { #[cfg(windows)] { - // On Windows, run via cmd.exe so .CMD/.BAT are correctly resolved (PATHEXT semantics). - std::process::Command::new("cmd") - .args(["/C", &cmd_str]) - .status()? + if action == UpdateAction::StandaloneWindows { + let (cmd, args) = action.command_args(); + // Run the standalone PowerShell installer with PowerShell + // itself. Routing this through `cmd.exe /C` would parse + // PowerShell metacharacters like `|` before PowerShell sees + // the installer command. + std::process::Command::new(cmd).args(args).status()? + } else { + // On Windows, run via cmd.exe so .CMD/.BAT are correctly resolved (PATHEXT semantics). + std::process::Command::new("cmd") + .args(["/C", &cmd_str]) + .status()? + } } #[cfg(not(windows))] { @@ -1269,31 +1279,21 @@ async fn run_debug_clear_memories_command( let state_db = StateRuntime::init(config.sqlite_home.clone(), config.model_provider_id.clone()) .await?; - state_db.reset_memory_data_for_fresh_start().await?; + state_db.clear_memory_data().await?; cleared_state_db = true; } - let memory_root = config.codex_home.join("memories"); - let removed_memory_root = match tokio::fs::remove_dir_all(&memory_root).await { - Ok(()) => true, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => false, - Err(err) => return Err(err.into()), - }; + clear_memory_roots_contents(&config.codex_home).await?; let mut message = if cleared_state_db { format!("Cleared memory state from {}.", state_path.display()) } else { format!("No state db found at {}.", state_path.display()) }; - - if removed_memory_root { - message.push_str(&format!(" Removed {}.", memory_root.display())); - } else { - message.push_str(&format!( - " No memory directory found at {}.", - memory_root.display() - )); - } + message.push_str(&format!( + " Cleared memory directories under {}.", + config.codex_home.display() + )); println!("{message}"); diff --git a/codex-rs/cli/src/marketplace_cmd.rs b/codex-rs/cli/src/marketplace_cmd.rs index cde50a2bf4..ce1f99390a 100644 --- a/codex-rs/cli/src/marketplace_cmd.rs +++ b/codex-rs/cli/src/marketplace_cmd.rs @@ -1,8 +1,12 @@ use anyhow::Context; use anyhow::Result; +use anyhow::bail; use clap::Parser; +use codex_core::config::Config; use codex_core::config::find_codex_home; use codex_core::plugins::MarketplaceAddRequest; +use codex_core::plugins::PluginMarketplaceUpgradeOutcome; +use codex_core::plugins::PluginsManager; use codex_core::plugins::add_marketplace; use codex_utils_cli::CliConfigOverrides; @@ -17,8 +21,8 @@ pub struct MarketplaceCli { #[derive(Debug, clap::Subcommand)] enum MarketplaceSubcommand { - /// Add a remote marketplace repository. Add(AddMarketplaceArgs), + Upgrade(UpgradeMarketplaceArgs), } #[derive(Debug, Parser)] @@ -27,11 +31,9 @@ struct AddMarketplaceArgs { /// or local marketplace root directories. source: String, - /// Git ref to check out. Overrides any @ref or #ref suffix in SOURCE. #[arg(long = "ref", value_name = "REF")] ref_name: Option, - /// Sparse-checkout path to use while cloning git sources. Repeat to include multiple paths. #[arg( long = "sparse", value_name = "PATH", @@ -40,6 +42,11 @@ struct AddMarketplaceArgs { sparse_paths: Vec, } +#[derive(Debug, Parser)] +struct UpgradeMarketplaceArgs { + marketplace_name: Option, +} + impl MarketplaceCli { pub async fn run(self) -> Result<()> { let MarketplaceCli { @@ -47,14 +54,13 @@ impl MarketplaceCli { subcommand, } = self; - // Validate overrides now. This command writes to CODEX_HOME only; marketplace discovery - // happens from that cache root after the next plugin/list or app-server start. - config_overrides + let overrides = config_overrides .parse_overrides() .map_err(anyhow::Error::msg)?; match subcommand { MarketplaceSubcommand::Add(args) => run_add(args).await?, + MarketplaceSubcommand::Upgrade(args) => run_upgrade(overrides, args).await?, } Ok(()) @@ -98,6 +104,60 @@ async fn run_add(args: AddMarketplaceArgs) -> Result<()> { Ok(()) } +async fn run_upgrade( + overrides: Vec<(String, toml::Value)>, + args: UpgradeMarketplaceArgs, +) -> Result<()> { + let UpgradeMarketplaceArgs { marketplace_name } = args; + let config = Config::load_with_cli_overrides(overrides) + .await + .context("failed to load configuration")?; + let codex_home = find_codex_home().context("failed to resolve CODEX_HOME")?; + let manager = PluginsManager::new(codex_home.to_path_buf()); + let outcome = manager + .upgrade_configured_marketplaces_for_config(&config, marketplace_name.as_deref()) + .map_err(anyhow::Error::msg)?; + print_upgrade_outcome(&outcome, marketplace_name.as_deref()) +} + +fn print_upgrade_outcome( + outcome: &PluginMarketplaceUpgradeOutcome, + marketplace_name: Option<&str>, +) -> Result<()> { + for error in &outcome.errors { + eprintln!( + "Failed to upgrade marketplace `{}`: {}", + error.marketplace_name, error.message + ); + } + if !outcome.all_succeeded() { + bail!("{} upgrade failure(s) occurred.", outcome.errors.len()); + } + + let selection_label = marketplace_name.unwrap_or("all configured Git marketplaces"); + if outcome.selected_marketplaces.is_empty() { + println!("No configured Git marketplaces to upgrade."); + } else if outcome.upgraded_roots.is_empty() { + if marketplace_name.is_some() { + println!("Marketplace `{selection_label}` is already up to date."); + } else { + println!("All configured Git marketplaces are already up to date."); + } + } else if marketplace_name.is_some() { + println!("Upgraded marketplace `{selection_label}` to the latest configured revision."); + for root in &outcome.upgraded_roots { + println!("Installed marketplace root: {}", root.display()); + } + } else { + println!("Upgraded {} marketplace(s).", outcome.upgraded_roots.len()); + for root in &outcome.upgraded_roots { + println!("Installed marketplace root: {}", root.display()); + } + } + + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -132,4 +192,13 @@ mod tests { vec!["plugins/foo", "skills/bar"] ); } + + #[test] + fn upgrade_subcommand_parses_optional_marketplace_name() { + let upgrade_all = UpgradeMarketplaceArgs::try_parse_from(["upgrade"]).unwrap(); + assert_eq!(upgrade_all.marketplace_name, None); + + let upgrade_one = UpgradeMarketplaceArgs::try_parse_from(["upgrade", "debug"]).unwrap(); + assert_eq!(upgrade_one.marketplace_name.as_deref(), Some("debug")); + } } diff --git a/codex-rs/cli/src/mcp_cmd.rs b/codex-rs/cli/src/mcp_cmd.rs index cac6ef2169..d413f72ddc 100644 --- a/codex-rs/cli/src/mcp_cmd.rs +++ b/codex-rs/cli/src/mcp_cmd.rs @@ -6,6 +6,7 @@ use anyhow::Result; use anyhow::anyhow; use anyhow::bail; use clap::ArgGroup; +use codex_config::types::AppToolApproval; use codex_config::types::McpServerConfig; use codex_config::types::McpServerTransportConfig; use codex_core::McpManager; @@ -297,12 +298,14 @@ async fn run_add(config_overrides: &CliConfigOverrides, add_args: AddArgs) -> Re let new_entry = McpServerConfig { transport: transport.clone(), + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -878,6 +881,14 @@ async fn run_get(config_overrides: &CliConfigOverrides, get_args: GetArgs) -> Re if let Some(timeout) = server.tool_timeout_sec { println!(" tool_timeout_sec: {}", timeout.as_secs_f64()); } + if let Some(approval_mode) = server.default_tools_approval_mode { + let approval_mode = match approval_mode { + AppToolApproval::Auto => "auto", + AppToolApproval::Prompt => "prompt", + AppToolApproval::Approve => "approve", + }; + println!(" default_tools_approval_mode: {approval_mode}"); + } println!(" remove: codex mcp remove {}", get_args.name); Ok(()) diff --git a/codex-rs/cli/tests/debug_clear_memories.rs b/codex-rs/cli/tests/debug_clear_memories.rs index c68172ba35..a24f7ebdde 100644 --- a/codex-rs/cli/tests/debug_clear_memories.rs +++ b/codex-rs/cli/tests/debug_clear_memories.rs @@ -125,13 +125,8 @@ INSERT INTO jobs ( .fetch_one(&pool) .await?; assert_eq!(memory_jobs_count, 0); - - let memory_mode: String = sqlx::query_scalar("SELECT memory_mode FROM threads WHERE id = ?") - .bind(thread_id) - .fetch_one(&pool) - .await?; - assert_eq!(memory_mode, "disabled"); - assert!(!memory_root.exists()); + assert!(memory_root.exists()); + assert_eq!(std::fs::read_dir(memory_root)?.count(), 0); Ok(()) } diff --git a/codex-rs/code-mode/Cargo.toml b/codex-rs/code-mode/Cargo.toml index fa7ce63a42..d2f42359d4 100644 --- a/codex-rs/code-mode/Cargo.toml +++ b/codex-rs/code-mode/Cargo.toml @@ -14,6 +14,7 @@ workspace = true [dependencies] async-trait = { workspace = true } +codex-protocol = { workspace = true } deno_core_icudata = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } diff --git a/codex-rs/code-mode/src/description.rs b/codex-rs/code-mode/src/description.rs index ac18fafd89..7f5e47fa25 100644 --- a/codex-rs/code-mode/src/description.rs +++ b/codex-rs/code-mode/src/description.rs @@ -1,3 +1,4 @@ +use codex_protocol::ToolName; use serde::Deserialize; use serde::Serialize; use serde_json::Value as JsonValue; @@ -8,6 +9,8 @@ use crate::PUBLIC_TOOL_NAME; const MAX_JS_SAFE_INTEGER: u64 = (1_u64 << 53) - 1; const CODE_MODE_ONLY_PREFACE: &str = "Use `exec/wait` tool to run all other tools, do not attempt to use any other tools directly"; +const DEFERRED_NESTED_TOOLS_GUIDANCE: &str = r#"Some nested MCP/app tools may be omitted from this description. They are still available on the global `tools` object and listed in `ALL_TOOLS`. +To find one, filter `ALL_TOOLS` by `name` and `description`; do not print the full `ALL_TOOLS` array. Print only a small set of relevant matches if you need to inspect them."#; const EXEC_DESCRIPTION_TEMPLATE: &str = r#"Run JavaScript code to orchestrate/compose tool calls - Evaluates the provided JavaScript code in a fresh V8 isolate as an async module. - All nested tools are available on the global `tools` object, for example `await tools.exec_command(...)`. Tool names are exposed as normalized JavaScript identifiers, for example `await tools.mcp__ologs__get_profile(...)`. @@ -23,7 +26,7 @@ const EXEC_DESCRIPTION_TEMPLATE: &str = r#"Run JavaScript code to orchestrate/co - Global helpers: - `exit()`: Immediately ends the current script successfully (like an early return from the top level). - `text(value: string | number | boolean | undefined | null)`: Appends a text item. Non-string values are stringified with `JSON.stringify(...)` when possible. -- `image(imageUrlOrItem: string | { image_url: string; detail?: "auto" | "low" | "high" | "original" | null })`: Appends an image item. `image_url` can be an HTTPS URL or a base64-encoded `data:` URL. +- `image(imageUrlOrItem: string | { image_url: string; detail?: "auto" | "low" | "high" | "original" | null } | ImageContent, detail?: "auto" | "low" | "high" | "original" | null)`: Appends an image item. `image_url` can be an HTTPS URL or a base64-encoded `data:` URL. To forward an MCP tool image, pass an individual `ImageContent` block from `result.content`, for example `image(result.content[0])`. MCP image blocks may request original detail with `_meta: { "codex/imageDetail": "original" }`. When provided, the second `detail` argument overrides any detail embedded in the first argument. - `store(key: string, value: any)`: stores a serializable value under a string key for later `exec` calls in the same session. - `load(key: string)`: returns the stored value for a string key, or `undefined` if it is missing. - `notify(value: string | number | boolean | undefined | null)`: immediately injects an extra `custom_tool_call_output` for the current `exec` call. Values are stringified like `text(...)`. @@ -129,6 +132,7 @@ pub enum CodeModeToolKind { #[derive(Clone, Debug, PartialEq)] pub struct ToolDefinition { pub name: String, + pub tool_name: ToolName, pub description: String, pub kind: CodeModeToolKind, pub input_schema: Option, @@ -249,15 +253,19 @@ pub fn build_exec_tool_description( enabled_tools: &[ToolDefinition], namespace_descriptions: &BTreeMap, code_mode_only: bool, + deferred_tools_available: bool, ) -> String { - if !code_mode_only { - return EXEC_DESCRIPTION_TEMPLATE.to_string(); + let mut sections = Vec::new(); + if code_mode_only { + sections.push(CODE_MODE_ONLY_PREFACE.to_string()); + } + sections.push(EXEC_DESCRIPTION_TEMPLATE.to_string()); + if deferred_tools_available { + sections.push(DEFERRED_NESTED_TOOLS_GUIDANCE.to_string()); + } + if !code_mode_only { + return sections.join("\n\n"); } - - let mut sections = vec![ - CODE_MODE_ONLY_PREFACE.to_string(), - EXEC_DESCRIPTION_TEMPLATE.to_string(), - ]; if !enabled_tools.is_empty() { let mut current_namespace: Option<&str> = None; @@ -269,11 +277,15 @@ pub fn build_exec_tool_description( for tool in enabled_tools { let name = tool.name.as_str(); let nested_description = render_code_mode_sample_for_definition(tool); - let next_namespace = namespace_descriptions - .get(name) + let namespace_description = tool + .tool_name + .namespace + .as_ref() + .and_then(|namespace| namespace_descriptions.get(namespace)); + let next_namespace = namespace_description .map(|namespace_description| namespace_description.name.as_str()); if next_namespace != current_namespace { - if let Some(namespace_description) = namespace_descriptions.get(name) { + if let Some(namespace_description) = namespace_description { let namespace_description_text = namespace_description.description.trim(); if !namespace_description_text.is_empty() { nested_tool_sections.push(format!( @@ -346,7 +358,7 @@ pub fn augment_tool_definition(mut definition: ToolDefinition) -> ToolDefinition pub fn enabled_tool_metadata(definition: &ToolDefinition) -> EnabledToolMetadata { EnabledToolMetadata { - tool_name: definition.name.clone(), + tool_name: definition.tool_name.clone(), global_name: normalize_code_mode_identifier(&definition.name), description: definition.description.clone(), kind: definition.kind, @@ -355,7 +367,7 @@ pub fn enabled_tool_metadata(definition: &ToolDefinition) -> EnabledToolMetadata #[derive(Clone, Debug, Eq, PartialEq, Serialize)] pub struct EnabledToolMetadata { - pub tool_name: String, + pub tool_name: ToolName, pub global_name: String, pub description: String, pub kind: CodeModeToolKind, @@ -706,6 +718,7 @@ mod tests { use super::build_exec_tool_description; use super::normalize_code_mode_identifier; use super::parse_exec_source; + use codex_protocol::ToolName; use pretty_assertions::assert_eq; use serde_json::Value as JsonValue; use serde_json::json; @@ -770,6 +783,7 @@ mod tests { fn augment_tool_definition_appends_typed_declaration() { let definition = ToolDefinition { name: "hidden_dynamic_tool".to_string(), + tool_name: ToolName::plain("hidden_dynamic_tool"), description: "Test tool".to_string(), kind: CodeModeToolKind::Function, input_schema: Some(json!({ @@ -798,6 +812,7 @@ mod tests { fn augment_tool_definition_includes_property_descriptions_as_comments() { let definition = ToolDefinition { name: "weather_tool".to_string(), + tool_name: ToolName::plain("weather_tool"), description: "Weather tool".to_string(), kind: CodeModeToolKind::Function, input_schema: Some(json!({ @@ -846,6 +861,7 @@ mod tests { let description = build_exec_tool_description( &[ToolDefinition { name: "foo".to_string(), + tool_name: ToolName::plain("foo"), description: "bar".to_string(), kind: CodeModeToolKind::Function, input_schema: None, @@ -853,6 +869,7 @@ mod tests { }], &BTreeMap::new(), /*code_mode_only*/ true, + /*deferred_tools_available*/ false, ); assert!(description.contains( "### `foo` @@ -862,34 +879,30 @@ bar" #[test] fn exec_description_mentions_timeout_helpers() { - let description = - build_exec_tool_description(&[], &BTreeMap::new(), /*code_mode_only*/ false); + let description = build_exec_tool_description( + &[], + &BTreeMap::new(), + /*code_mode_only*/ false, + /*deferred_tools_available*/ false, + ); assert!(description.contains("`setTimeout(callback: () => void, delayMs?: number)`")); assert!(description.contains("`clearTimeout(timeoutId?: number)`")); } #[test] fn code_mode_only_description_groups_namespace_instructions_once() { - let namespace_descriptions = BTreeMap::from([ - ( - "mcp__sample__alpha".to_string(), - ToolNamespaceDescription { - name: "mcp__sample".to_string(), - description: "Shared namespace guidance.".to_string(), - }, - ), - ( - "mcp__sample__beta".to_string(), - ToolNamespaceDescription { - name: "mcp__sample".to_string(), - description: "Shared namespace guidance.".to_string(), - }, - ), - ]); + let namespace_descriptions = BTreeMap::from([( + "mcp__sample__".to_string(), + ToolNamespaceDescription { + name: "mcp__sample".to_string(), + description: "Shared namespace guidance.".to_string(), + }, + )]); let description = build_exec_tool_description( &[ ToolDefinition { name: "mcp__sample__alpha".to_string(), + tool_name: ToolName::namespaced("mcp__sample__", "alpha"), description: "First tool".to_string(), kind: CodeModeToolKind::Function, input_schema: Some(json!({ @@ -905,6 +918,7 @@ bar" }, ToolDefinition { name: "mcp__sample__beta".to_string(), + tool_name: ToolName::namespaced("mcp__sample__", "beta"), description: "Second tool".to_string(), kind: CodeModeToolKind::Function, input_schema: Some(json!({ @@ -921,6 +935,7 @@ bar" ], &namespace_descriptions, /*code_mode_only*/ true, + /*deferred_tools_available*/ false, ); assert_eq!(description.matches("## mcp__sample").count(), 1); assert!(description.contains("## mcp__sample\nShared namespace guidance.")); @@ -935,7 +950,7 @@ bar" #[test] fn code_mode_only_description_omits_empty_namespace_sections() { let namespace_descriptions = BTreeMap::from([( - "mcp__sample__alpha".to_string(), + "mcp__sample__".to_string(), ToolNamespaceDescription { name: "mcp__sample".to_string(), description: String::new(), @@ -944,6 +959,7 @@ bar" let description = build_exec_tool_description( &[ToolDefinition { name: "mcp__sample__alpha".to_string(), + tool_name: ToolName::namespaced("mcp__sample__", "alpha"), description: "First tool".to_string(), kind: CodeModeToolKind::Function, input_schema: Some(json!({ @@ -959,6 +975,7 @@ bar" }], &namespace_descriptions, /*code_mode_only*/ true, + /*deferred_tools_available*/ false, ); assert!(!description.contains("## mcp__sample")); @@ -969,6 +986,7 @@ bar" fn code_mode_only_description_renders_shared_mcp_types_once() { let first_tool = augment_tool_definition(ToolDefinition { name: "mcp__sample__alpha".to_string(), + tool_name: ToolName::namespaced("mcp__sample__", "alpha"), description: "First tool".to_string(), kind: CodeModeToolKind::Function, input_schema: Some(json!({ @@ -1002,6 +1020,7 @@ bar" }); let second_tool = augment_tool_definition(ToolDefinition { name: "mcp__sample__beta".to_string(), + tool_name: ToolName::namespaced("mcp__sample__", "beta"), description: "Second tool".to_string(), kind: CodeModeToolKind::Function, input_schema: Some(json!({ @@ -1038,6 +1057,7 @@ bar" &[ ToolDefinition { name: first_tool.name, + tool_name: first_tool.tool_name, description: "First tool".to_string(), kind: first_tool.kind, input_schema: first_tool.input_schema, @@ -1045,6 +1065,7 @@ bar" }, ToolDefinition { name: second_tool.name, + tool_name: second_tool.tool_name, description: "Second tool".to_string(), kind: second_tool.kind, input_schema: second_tool.input_schema, @@ -1053,6 +1074,7 @@ bar" ], &BTreeMap::new(), /*code_mode_only*/ true, + /*deferred_tools_available*/ false, ); assert_eq!( @@ -1063,4 +1085,17 @@ bar" ); assert_eq!(description.matches("Shared MCP Types:").count(), 1); } + + #[test] + fn exec_description_mentions_deferred_nested_tools_when_available() { + let description = build_exec_tool_description( + &[], + &BTreeMap::new(), + /*code_mode_only*/ false, + /*deferred_tools_available*/ true, + ); + + assert!(description.contains("Some nested MCP/app tools may be omitted")); + assert!(description.contains("filter `ALL_TOOLS` by `name` and `description`")); + } } diff --git a/codex-rs/code-mode/src/runtime/callbacks.rs b/codex-rs/code-mode/src/runtime/callbacks.rs index 5511baca23..a9755f6eb0 100644 --- a/codex-rs/code-mode/src/runtime/callbacks.rs +++ b/codex-rs/code-mode/src/runtime/callbacks.rs @@ -15,7 +15,13 @@ pub(super) fn tool_callback( args: v8::FunctionCallbackArguments, mut retval: v8::ReturnValue, ) { - let tool_name = args.data().to_rust_string_lossy(scope); + let tool_index = match args.data().to_rust_string_lossy(scope).parse::() { + Ok(tool_index) => tool_index, + Err(_) => { + throw_type_error(scope, "invalid tool callback data"); + return; + } + }; let input = if args.length() == 0 { Ok(None) } else { @@ -36,6 +42,22 @@ pub(super) fn tool_callback( let promise = resolver.get_promise(scope); let resolver = v8::Global::new(scope, resolver); + let tool_name = { + let Some(state) = scope.get_slot::() else { + throw_type_error(scope, "runtime state unavailable"); + return; + }; + let Some(tool_name) = state + .enabled_tools + .get(tool_index) + .map(|tool| tool.tool_name.clone()) + else { + throw_type_error(scope, "tool callback data is out of range"); + return; + }; + tool_name + }; + let Some(state) = scope.get_slot_mut::() else { throw_type_error(scope, "runtime state unavailable"); return; @@ -87,7 +109,20 @@ pub(super) fn image_callback( } else { args.get(0) }; - let image_item = match normalize_output_image(scope, value) { + let detail_override = if args.length() < 2 { + None + } else { + let detail = args.get(1); + if detail.is_string() { + Some(detail.to_rust_string_lossy(scope)) + } else if detail.is_null() || detail.is_undefined() { + None + } else { + throw_type_error(scope, "image detail must be a string when provided"); + return; + } + }; + let image_item = match normalize_output_image(scope, value, detail_override) { Ok(image_item) => image_item, Err(()) => return, }; diff --git a/codex-rs/code-mode/src/runtime/globals.rs b/codex-rs/code-mode/src/runtime/globals.rs index 2d419db908..b40136c44c 100644 --- a/codex-rs/code-mode/src/runtime/globals.rs +++ b/codex-rs/code-mode/src/runtime/globals.rs @@ -53,10 +53,10 @@ fn build_tools_object<'s>( .map(|state| state.enabled_tools.clone()) .unwrap_or_default(); - for tool in enabled_tools { + for (tool_index, tool) in enabled_tools.iter().enumerate() { let name = v8::String::new(scope, &tool.global_name) .ok_or_else(|| "failed to allocate tool name".to_string())?; - let function = tool_function(scope, &tool.tool_name)?; + let function = tool_function(scope, tool_index)?; tools.set(scope, name.into(), function.into()); } Ok(tools) @@ -116,9 +116,9 @@ where fn tool_function<'s>( scope: &mut v8::PinScope<'s, '_>, - tool_name: &str, + tool_index: usize, ) -> Result, String> { - let data = v8::String::new(scope, tool_name) + let data = v8::String::new(scope, &tool_index.to_string()) .ok_or_else(|| "failed to allocate tool callback data".to_string())?; let template = v8::FunctionTemplate::builder(tool_callback) .data(data.into()) diff --git a/codex-rs/code-mode/src/runtime/mod.rs b/codex-rs/code-mode/src/runtime/mod.rs index febe54216a..0f50edd329 100644 --- a/codex-rs/code-mode/src/runtime/mod.rs +++ b/codex-rs/code-mode/src/runtime/mod.rs @@ -9,6 +9,7 @@ use std::sync::OnceLock; use std::sync::mpsc as std_mpsc; use std::thread; +use codex_protocol::ToolName; use serde_json::Value as JsonValue; use tokio::sync::mpsc; @@ -62,7 +63,7 @@ pub(crate) enum TurnMessage { ToolCall { cell_id: String, id: String, - name: String, + name: ToolName, input: Option, }, Notify { @@ -87,7 +88,7 @@ pub(crate) enum RuntimeEvent { YieldRequested, ToolCall { id: String, - name: String, + name: ToolName, input: Option, }, Notify { diff --git a/codex-rs/code-mode/src/runtime/value.rs b/codex-rs/code-mode/src/runtime/value.rs index eb0280142c..5c63434f4f 100644 --- a/codex-rs/code-mode/src/runtime/value.rs +++ b/codex-rs/code-mode/src/runtime/value.rs @@ -3,6 +3,9 @@ use serde_json::Value as JsonValue; use crate::response::FunctionCallOutputContentItem; use crate::response::ImageDetail; +const IMAGE_HELPER_EXPECTS_MESSAGE: &str = "image expects a non-empty image URL string, an object with image_url and optional detail, or a raw MCP image block"; +const CODEX_IMAGE_DETAIL_META_KEY: &str = "codex/imageDetail"; + pub(super) fn serialize_output_text( scope: &mut v8::PinScope<'_, '_>, value: v8::Local<'_, v8::Value>, @@ -34,45 +37,25 @@ pub(super) fn serialize_output_text( pub(super) fn normalize_output_image( scope: &mut v8::PinScope<'_, '_>, value: v8::Local<'_, v8::Value>, + detail_override: Option, ) -> Result { let result = (|| -> Result { let (image_url, detail) = if value.is_string() { (value.to_rust_string_lossy(scope), None) } else if value.is_object() && !value.is_array() { - let object = v8::Local::::try_from(value).map_err(|_| { - "image expects a non-empty image URL string or an object with image_url and optional detail".to_string() - })?; - let image_url_key = v8::String::new(scope, "image_url") - .ok_or_else(|| "failed to allocate image helper keys".to_string())?; - let detail_key = v8::String::new(scope, "detail") - .ok_or_else(|| "failed to allocate image helper keys".to_string())?; - let image_url = object - .get(scope, image_url_key.into()) - .filter(|value| value.is_string()) - .map(|value| value.to_rust_string_lossy(scope)) - .ok_or_else(|| { - "image expects a non-empty image URL string or an object with image_url and optional detail" - .to_string() - })?; - let detail = match object.get(scope, detail_key.into()) { - Some(value) if value.is_string() => Some(value.to_rust_string_lossy(scope)), - Some(value) if value.is_null() || value.is_undefined() => None, - Some(_) => return Err("image detail must be a string when provided".to_string()), - None => None, - }; - (image_url, detail) + let object = v8::Local::::try_from(value) + .map_err(|_| IMAGE_HELPER_EXPECTS_MESSAGE.to_string())?; + if let Some(image) = parse_non_mcp_output_image(scope, object)? { + image + } else { + parse_mcp_output_image(scope, value)? + } } else { - return Err( - "image expects a non-empty image URL string or an object with image_url and optional detail" - .to_string(), - ); + return Err(IMAGE_HELPER_EXPECTS_MESSAGE.to_string()); }; if image_url.is_empty() { - return Err( - "image expects a non-empty image URL string or an object with image_url and optional detail" - .to_string(), - ); + return Err(IMAGE_HELPER_EXPECTS_MESSAGE.to_string()); } let lower = image_url.to_ascii_lowercase(); if !(lower.starts_with("http://") @@ -82,6 +65,7 @@ pub(super) fn normalize_output_image( return Err("image expects an http(s) or data URL".to_string()); } + let detail = detail_override.or(detail); let detail = match detail { Some(detail) => { let normalized = detail.to_ascii_lowercase(); @@ -112,6 +96,86 @@ pub(super) fn normalize_output_image( } } +fn parse_non_mcp_output_image( + scope: &mut v8::PinScope<'_, '_>, + object: v8::Local<'_, v8::Object>, +) -> Result)>, String> { + let image_url_key = v8::String::new(scope, "image_url") + .ok_or_else(|| "failed to allocate image helper keys".to_string())?; + let Some(image_url) = object.get(scope, image_url_key.into()) else { + return Ok(None); + }; + if image_url.is_undefined() { + return Ok(None); + } + if !image_url.is_string() { + return Err(IMAGE_HELPER_EXPECTS_MESSAGE.to_string()); + } + let detail_key = v8::String::new(scope, "detail") + .ok_or_else(|| "failed to allocate image helper keys".to_string())?; + let detail = parse_image_detail_value(scope, object.get(scope, detail_key.into()))?; + Ok(Some((image_url.to_rust_string_lossy(scope), detail))) +} + +fn parse_mcp_output_image( + scope: &mut v8::PinScope<'_, '_>, + value: v8::Local<'_, v8::Value>, +) -> Result<(String, Option), String> { + let Some(result) = v8_value_to_json(scope, value)? else { + return Err(IMAGE_HELPER_EXPECTS_MESSAGE.to_string()); + }; + let JsonValue::Object(result) = result else { + return Err(IMAGE_HELPER_EXPECTS_MESSAGE.to_string()); + }; + let Some(item_type) = result.get("type").and_then(JsonValue::as_str) else { + return Err(IMAGE_HELPER_EXPECTS_MESSAGE.to_string()); + }; + if item_type != "image" { + return Err(format!( + "image only accepts MCP image blocks, got \"{item_type}\"" + )); + } + let data = result + .get("data") + .and_then(JsonValue::as_str) + .ok_or_else(|| "image expected MCP image data".to_string())?; + if data.is_empty() { + return Err("image expected MCP image data".to_string()); + } + + let image_url = if data.to_ascii_lowercase().starts_with("data:") { + data.to_string() + } else { + let mime_type = result + .get("mimeType") + .or_else(|| result.get("mime_type")) + .and_then(JsonValue::as_str) + .filter(|mime_type| !mime_type.is_empty()) + .unwrap_or("application/octet-stream"); + format!("data:{mime_type};base64,{data}") + }; + let detail = result + .get("_meta") + .and_then(JsonValue::as_object) + .and_then(|meta| meta.get(CODEX_IMAGE_DETAIL_META_KEY)) + .and_then(JsonValue::as_str) + .filter(|detail| *detail == "original") + .map(str::to_string); + Ok((image_url, detail)) +} + +fn parse_image_detail_value<'s>( + scope: &mut v8::PinScope<'s, '_>, + value: Option>, +) -> Result, String> { + match value { + Some(value) if value.is_string() => Ok(Some(value.to_rust_string_lossy(scope))), + Some(value) if value.is_null() || value.is_undefined() => Ok(None), + Some(_) => Err("image detail must be a string when provided".to_string()), + None => Ok(None), + } +} + pub(super) fn v8_value_to_json( scope: &mut v8::PinScope<'_, '_>, value: v8::Local<'_, v8::Value>, diff --git a/codex-rs/code-mode/src/service.rs b/codex-rs/code-mode/src/service.rs index c14fee1957..23ca7a7460 100644 --- a/codex-rs/code-mode/src/service.rs +++ b/codex-rs/code-mode/src/service.rs @@ -5,6 +5,7 @@ use std::sync::atomic::Ordering; use std::time::Duration; use async_trait::async_trait; +use codex_protocol::ToolName; use serde_json::Value as JsonValue; use tokio::sync::Mutex; use tokio::sync::mpsc; @@ -26,7 +27,7 @@ use crate::runtime::spawn_runtime; pub trait CodeModeTurnHost: Send + Sync { async fn invoke_tool( &self, - tool_name: String, + tool_name: ToolName, input: Option, cancellation_token: CancellationToken, ) -> Result; @@ -683,6 +684,154 @@ text(JSON.stringify(returnsUndefined)); ); } + #[tokio::test] + async fn image_helper_accepts_raw_mcp_image_block_with_original_detail() { + let service = CodeModeService::new(); + + let response = service + .execute(ExecuteRequest { + source: r#" +image({ + type: "image", + data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==", + mimeType: "image/png", + _meta: { "codex/imageDetail": "original" }, +}); +"# + .to_string(), + yield_time_ms: None, + ..execute_request("") + }) + .await + .unwrap(); + + assert_eq!( + response, + RuntimeResponse::Result { + cell_id: "1".to_string(), + content_items: vec![FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==".to_string(), + detail: Some(crate::ImageDetail::Original), + }], + stored_values: HashMap::new(), + error_text: None, + } + ); + } + + #[tokio::test] + async fn image_helper_second_arg_overrides_explicit_object_detail() { + let service = CodeModeService::new(); + + let response = service + .execute(ExecuteRequest { + source: r#" +image( + { + image_url: "https://example.com/image.jpg", + detail: "low", + }, + "original", +); +"# + .to_string(), + yield_time_ms: None, + ..execute_request("") + }) + .await + .unwrap(); + + assert_eq!( + response, + RuntimeResponse::Result { + cell_id: "1".to_string(), + content_items: vec![FunctionCallOutputContentItem::InputImage { + image_url: "https://example.com/image.jpg".to_string(), + detail: Some(crate::ImageDetail::Original), + }], + stored_values: HashMap::new(), + error_text: None, + } + ); + } + + #[tokio::test] + async fn image_helper_second_arg_overrides_raw_mcp_image_detail() { + let service = CodeModeService::new(); + + let response = service + .execute(ExecuteRequest { + source: r#" +image( + { + type: "image", + data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==", + mimeType: "image/png", + _meta: { "codex/imageDetail": "original" }, + }, + "low", +); +"# + .to_string(), + yield_time_ms: None, + ..execute_request("") + }) + .await + .unwrap(); + + assert_eq!( + response, + RuntimeResponse::Result { + cell_id: "1".to_string(), + content_items: vec![FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==".to_string(), + detail: Some(crate::ImageDetail::Low), + }], + stored_values: HashMap::new(), + error_text: None, + } + ); + } + + #[tokio::test] + async fn image_helper_rejects_raw_mcp_result_container() { + let service = CodeModeService::new(); + + let response = service + .execute(ExecuteRequest { + source: r#" +image({ + content: [ + { + type: "image", + data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==", + mimeType: "image/png", + _meta: { "codex/imageDetail": "original" }, + }, + ], + isError: false, +}); +"# + .to_string(), + yield_time_ms: None, + ..execute_request("") + }) + .await + .unwrap(); + + assert_eq!( + response, + RuntimeResponse::Result { + cell_id: "1".to_string(), + content_items: Vec::new(), + stored_values: HashMap::new(), + error_text: Some( + "image expects a non-empty image URL string, an object with image_url and optional detail, or a raw MCP image block".to_string(), + ), + } + ); + } + #[tokio::test] async fn terminate_waits_for_runtime_shutdown_before_responding() { let inner = test_inner(); diff --git a/codex-rs/codex-api/src/api_bridge.rs b/codex-rs/codex-api/src/api_bridge.rs index 0ad2b13979..2677ff3e55 100644 --- a/codex-rs/codex-api/src/api_bridge.rs +++ b/codex-rs/codex-api/src/api_bridge.rs @@ -12,6 +12,7 @@ use codex_protocol::error::RetryLimitReachedError; use codex_protocol::error::UnexpectedResponseError; use codex_protocol::error::UsageLimitReachedError; use http::HeaderMap; +use http::HeaderValue; use serde::Deserialize; use serde_json::Value; @@ -178,6 +179,7 @@ struct UsageErrorBody { pub struct CoreAuthProvider { pub token: Option, pub account_id: Option, + pub is_fedramp_account: bool, } impl CoreAuthProvider { @@ -195,16 +197,25 @@ impl CoreAuthProvider { Self { token: token.map(str::to_string), account_id: account_id.map(str::to_string), + is_fedramp_account: false, } } } impl ApiAuthProvider for CoreAuthProvider { - fn bearer_token(&self) -> Option { - self.token.clone() - } - - fn account_id(&self) -> Option { - self.account_id.clone() + fn add_auth_headers(&self, headers: &mut HeaderMap) { + if let Some(token) = self.token.as_ref() + && let Ok(header) = HeaderValue::from_str(&format!("Bearer {token}")) + { + let _ = headers.insert(http::header::AUTHORIZATION, header); + } + if let Some(account_id) = self.account_id.as_ref() + && let Ok(header) = HeaderValue::from_str(account_id) + { + let _ = headers.insert("ChatGPT-Account-ID", header); + } + if self.is_fedramp_account { + crate::auth::add_fedramp_routing_header(headers); + } } } diff --git a/codex-rs/codex-api/src/api_bridge_tests.rs b/codex-rs/codex-api/src/api_bridge_tests.rs index 71d3889915..c7d4bbbdab 100644 --- a/codex-rs/codex-api/src/api_bridge_tests.rs +++ b/codex-rs/codex-api/src/api_bridge_tests.rs @@ -136,8 +136,49 @@ fn core_auth_provider_reports_when_auth_header_will_attach() { let auth = CoreAuthProvider { token: Some("access-token".to_string()), account_id: None, + is_fedramp_account: false, }; assert!(auth.auth_header_attached()); assert_eq!(auth.auth_header_name(), Some("authorization")); } + +#[test] +fn core_auth_provider_adds_auth_headers() { + let auth = CoreAuthProvider::for_test(Some("access-token"), Some("workspace-123")); + let mut headers = HeaderMap::new(); + + crate::AuthProvider::add_auth_headers(&auth, &mut headers); + + assert_eq!( + headers + .get(http::header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()), + Some("Bearer access-token") + ); + assert_eq!( + headers + .get("ChatGPT-Account-ID") + .and_then(|value| value.to_str().ok()), + Some("workspace-123") + ); +} + +#[test] +fn core_auth_provider_adds_fedramp_routing_header_for_fedramp_accounts() { + let auth = CoreAuthProvider { + token: Some("access-token".to_string()), + account_id: Some("workspace-123".to_string()), + is_fedramp_account: true, + }; + let mut headers = HeaderMap::new(); + + crate::AuthProvider::add_auth_headers(&auth, &mut headers); + + assert_eq!( + headers + .get("X-OpenAI-Fedramp") + .and_then(|value| value.to_str().ok()), + Some("true") + ); +} diff --git a/codex-rs/codex-api/src/auth.rs b/codex-rs/codex-api/src/auth.rs index f649062db1..efa5fb3288 100644 --- a/codex-rs/codex-api/src/auth.rs +++ b/codex-rs/codex-api/src/auth.rs @@ -1,33 +1,34 @@ -use codex_client::Request; use http::HeaderMap; use http::HeaderValue; -/// Provides bearer and account identity information for API requests. +/// Adds authentication headers to API requests. /// /// Implementations should be cheap and non-blocking; any asynchronous /// refresh or I/O should be handled by higher layers before requests /// reach this interface. pub trait AuthProvider: Send + Sync { - fn bearer_token(&self) -> Option; - fn account_id(&self) -> Option { - None - } + fn add_auth_headers(&self, headers: &mut HeaderMap); } -pub(crate) fn add_auth_headers_to_header_map(auth: &A, headers: &mut HeaderMap) { - if let Some(token) = auth.bearer_token() - && let Ok(header) = HeaderValue::from_str(&format!("Bearer {token}")) - { - let _ = headers.insert(http::header::AUTHORIZATION, header); - } - if let Some(account_id) = auth.account_id() - && let Ok(header) = HeaderValue::from_str(&account_id) - { - let _ = headers.insert("ChatGPT-Account-ID", header); - } +pub(crate) fn add_fedramp_routing_header(headers: &mut HeaderMap) { + headers.insert("X-OpenAI-Fedramp", HeaderValue::from_static("true")); } -pub(crate) fn add_auth_headers(auth: &A, mut req: Request) -> Request { - add_auth_headers_to_header_map(auth, &mut req.headers); - req +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn add_fedramp_routing_header_sets_header() { + let mut headers = HeaderMap::new(); + + add_fedramp_routing_header(&mut headers); + + assert_eq!( + headers + .get("X-OpenAI-Fedramp") + .and_then(|v| v.to_str().ok()), + Some("true") + ); + } } diff --git a/codex-rs/codex-api/src/endpoint/compact.rs b/codex-rs/codex-api/src/endpoint/compact.rs index 44a56a11a7..748ac35558 100644 --- a/codex-rs/codex-api/src/endpoint/compact.rs +++ b/codex-rs/codex-api/src/endpoint/compact.rs @@ -90,9 +90,7 @@ mod tests { struct DummyAuth; impl AuthProvider for DummyAuth { - fn bearer_token(&self) -> Option { - None - } + fn add_auth_headers(&self, _headers: &mut HeaderMap) {} } #[test] diff --git a/codex-rs/codex-api/src/endpoint/memories.rs b/codex-rs/codex-api/src/endpoint/memories.rs index 5cb2a65b1f..3047c859db 100644 --- a/codex-rs/codex-api/src/endpoint/memories.rs +++ b/codex-rs/codex-api/src/endpoint/memories.rs @@ -103,9 +103,7 @@ mod tests { struct DummyAuth; impl AuthProvider for DummyAuth { - fn bearer_token(&self) -> Option { - None - } + fn add_auth_headers(&self, _headers: &mut HeaderMap) {} } #[derive(Clone)] diff --git a/codex-rs/codex-api/src/endpoint/models.rs b/codex-rs/codex-api/src/endpoint/models.rs index 97781ac419..17342d6f99 100644 --- a/codex-rs/codex-api/src/endpoint/models.rs +++ b/codex-rs/codex-api/src/endpoint/models.rs @@ -132,9 +132,7 @@ mod tests { struct DummyAuth; impl AuthProvider for DummyAuth { - fn bearer_token(&self) -> Option { - None - } + fn add_auth_headers(&self, _headers: &mut HeaderMap) {} } fn provider(base_url: &str) -> Provider { diff --git a/codex-rs/codex-api/src/endpoint/realtime_call.rs b/codex-rs/codex-api/src/endpoint/realtime_call.rs index 4d0bcfa7bf..a9a8b963cf 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_call.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_call.rs @@ -284,8 +284,11 @@ mod tests { struct DummyAuth; impl AuthProvider for DummyAuth { - fn bearer_token(&self) -> Option { - Some("test-token".to_string()) + fn add_auth_headers(&self, headers: &mut HeaderMap) { + headers.insert( + http::header::AUTHORIZATION, + HeaderValue::from_static("Bearer test-token"), + ); } } diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs index a6964e6614..6a5a032156 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs @@ -1670,7 +1670,7 @@ mod tests { ); assert_eq!( third_json["item"]["output"], - Value::String("\"Agent Final Message\":\n\ndelegated result".to_string()) + Value::String("delegated result".to_string()) ); }); diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_common.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_common.rs index 67345bdc7e..c0efcce165 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_common.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_common.rs @@ -45,9 +45,11 @@ pub(super) fn conversation_handoff_append_message( handoff_id: String, output_text: String, ) -> RealtimeOutboundMessage { - let output_text = format!("{AGENT_FINAL_MESSAGE_PREFIX}{output_text}"); match event_parser { - RealtimeEventParser::V1 => v1_conversation_handoff_append_message(handoff_id, output_text), + RealtimeEventParser::V1 => v1_conversation_handoff_append_message( + handoff_id, + format!("{AGENT_FINAL_MESSAGE_PREFIX}{output_text}"), + ), RealtimeEventParser::RealtimeV2 => { v2_conversation_handoff_append_message(handoff_id, output_text) } diff --git a/codex-rs/codex-api/src/endpoint/responses_websocket.rs b/codex-rs/codex-api/src/endpoint/responses_websocket.rs index a60d188bda..d2b775cdd5 100644 --- a/codex-rs/codex-api/src/endpoint/responses_websocket.rs +++ b/codex-rs/codex-api/src/endpoint/responses_websocket.rs @@ -1,5 +1,4 @@ use crate::auth::AuthProvider; -use crate::auth::add_auth_headers_to_header_map; use crate::common::ResponseEvent; use crate::common::ResponseStream; use crate::common::ResponsesWsRequest; @@ -310,7 +309,7 @@ impl ResponsesWebsocketClient { let mut headers = merge_request_headers(&self.provider.headers, extra_headers, default_headers); - add_auth_headers_to_header_map(&self.auth, &mut headers); + self.auth.add_auth_headers(&mut headers); let (stream, server_reasoning_included, models_etag, server_model) = connect_websocket(ws_url, headers, turn_state.clone()).await?; diff --git a/codex-rs/codex-api/src/endpoint/session.rs b/codex-rs/codex-api/src/endpoint/session.rs index 00919a0c54..e4a470ceee 100644 --- a/codex-rs/codex-api/src/endpoint/session.rs +++ b/codex-rs/codex-api/src/endpoint/session.rs @@ -1,5 +1,4 @@ use crate::auth::AuthProvider; -use crate::auth::add_auth_headers; use crate::error::ApiError; use crate::provider::Provider; use crate::telemetry::run_with_request_telemetry; @@ -56,7 +55,8 @@ impl EndpointSession { if let Some(body) = body { req.body = Some(RequestBody::Json(body.clone())); } - add_auth_headers(&self.auth, req) + self.auth.add_auth_headers(&mut req.headers); + req } pub(crate) async fn execute( diff --git a/codex-rs/codex-api/src/files.rs b/codex-rs/codex-api/src/files.rs index 6fad5b62f5..ebe35af5ba 100644 --- a/codex-rs/codex-api/src/files.rs +++ b/codex-rs/codex-api/src/files.rs @@ -256,17 +256,14 @@ fn authorized_request( method: reqwest::Method, url: &str, ) -> reqwest::RequestBuilder { + let mut headers = http::HeaderMap::new(); + auth.add_auth_headers(&mut headers); + let client = build_reqwest_client(); - let mut request = client + client .request(method, url) - .timeout(OPENAI_FILE_REQUEST_TIMEOUT); - if let Some(token) = auth.bearer_token() { - request = request.bearer_auth(token); - } - if let Some(account_id) = auth.account_id() { - request = request.header("chatgpt-account-id", account_id); - } - request + .timeout(OPENAI_FILE_REQUEST_TIMEOUT) + .headers(headers) } fn build_reqwest_client() -> reqwest::Client { diff --git a/codex-rs/codex-api/src/lib.rs b/codex-rs/codex-api/src/lib.rs index f4f90b289c..82a1004849 100644 --- a/codex-rs/codex-api/src/lib.rs +++ b/codex-rs/codex-api/src/lib.rs @@ -57,7 +57,7 @@ pub use crate::error::ApiError; pub use crate::files::upload_local_file; pub use crate::provider::Provider; pub use crate::provider::RetryConfig; -pub use crate::provider::is_azure_responses_wire_base_url; +pub use crate::provider::is_azure_responses_provider; pub use crate::requests::Compression; pub use crate::sse::stream_from_fixture; pub use crate::telemetry::SseTelemetry; diff --git a/codex-rs/codex-api/src/provider.rs b/codex-rs/codex-api/src/provider.rs index 81a168ffd7..45f2512dc3 100644 --- a/codex-rs/codex-api/src/provider.rs +++ b/codex-rs/codex-api/src/provider.rs @@ -86,7 +86,7 @@ impl Provider { } pub fn is_azure_responses_endpoint(&self) -> bool { - is_azure_responses_wire_base_url(&self.name, Some(&self.base_url)) + is_azure_responses_provider(&self.name, Some(&self.base_url)) } pub fn websocket_url_for_path(&self, path: &str) -> Result { @@ -103,21 +103,20 @@ impl Provider { } } -pub fn is_azure_responses_wire_base_url(name: &str, base_url: Option<&str>) -> bool { +pub fn is_azure_responses_provider(name: &str, base_url: Option<&str>) -> bool { if name.eq_ignore_ascii_case("azure") { - return true; + true + } else if let Some(base_url) = base_url { + matches_azure_responses_base_url(base_url) + } else { + false } - - let Some(base_url) = base_url else { - return false; - }; - - let base = base_url.to_ascii_lowercase(); - base.contains("openai.azure.") || matches_azure_responses_base_url(&base) } fn matches_azure_responses_base_url(base_url: &str) -> bool { - const AZURE_MARKERS: [&str; 5] = [ + let base_url = base_url.to_ascii_lowercase(); + const AZURE_MARKERS: [&str; 6] = [ + "openai.azure.", "cognitiveservices.azure.", "aoai.azure.", "azure-api.", @@ -144,12 +143,12 @@ mod tests { for base_url in positive_cases { assert!( - is_azure_responses_wire_base_url("test", Some(base_url)), + is_azure_responses_provider("test", Some(base_url)), "expected {base_url} to be detected as Azure" ); } - assert!(is_azure_responses_wire_base_url( + assert!(is_azure_responses_provider( "Azure", Some("https://example.com") )); @@ -162,7 +161,7 @@ mod tests { for base_url in negative_cases { assert!( - !is_azure_responses_wire_base_url("test", Some(base_url)), + !is_azure_responses_provider("test", Some(base_url)), "expected {base_url} not to be detected as Azure" ); } diff --git a/codex-rs/codex-api/tests/clients.rs b/codex-rs/codex-api/tests/clients.rs index b11c6f9d6b..d82fcc14ce 100644 --- a/codex-rs/codex-api/tests/clients.rs +++ b/codex-rs/codex-api/tests/clients.rs @@ -91,9 +91,7 @@ impl HttpTransport for RecordingTransport { struct NoAuth; impl AuthProvider for NoAuth { - fn bearer_token(&self) -> Option { - None - } + fn add_auth_headers(&self, _headers: &mut HeaderMap) {} } #[derive(Clone)] @@ -112,12 +110,14 @@ impl StaticAuth { } impl AuthProvider for StaticAuth { - fn bearer_token(&self) -> Option { - Some(self.token.clone()) - } - - fn account_id(&self) -> Option { - Some(self.account_id.clone()) + fn add_auth_headers(&self, headers: &mut HeaderMap) { + let token = &self.token; + if let Ok(header) = HeaderValue::from_str(&format!("Bearer {token}")) { + headers.insert(http::header::AUTHORIZATION, header); + } + if let Ok(header) = HeaderValue::from_str(&self.account_id) { + headers.insert("ChatGPT-Account-ID", header); + } } } diff --git a/codex-rs/codex-api/tests/models_integration.rs b/codex-rs/codex-api/tests/models_integration.rs index 91a8477a92..fab135e0ab 100644 --- a/codex-rs/codex-api/tests/models_integration.rs +++ b/codex-rs/codex-api/tests/models_integration.rs @@ -24,9 +24,7 @@ use wiremock::matchers::path; struct DummyAuth; impl AuthProvider for DummyAuth { - fn bearer_token(&self) -> Option { - None - } + fn add_auth_headers(&self, _headers: &mut HeaderMap) {} } fn provider(base_url: &str) -> Provider { diff --git a/codex-rs/codex-api/tests/sse_end_to_end.rs b/codex-rs/codex-api/tests/sse_end_to_end.rs index b15de296a8..4d32e82242 100644 --- a/codex-rs/codex-api/tests/sse_end_to_end.rs +++ b/codex-rs/codex-api/tests/sse_end_to_end.rs @@ -53,9 +53,7 @@ impl HttpTransport for FixtureSseTransport { struct NoAuth; impl AuthProvider for NoAuth { - fn bearer_token(&self) -> Option { - None - } + fn add_auth_headers(&self, _headers: &mut HeaderMap) {} } fn provider(name: &str) -> Provider { diff --git a/codex-rs/codex-mcp/src/lib.rs b/codex-rs/codex-mcp/src/lib.rs index bfbe1c60f8..ed0d9b4122 100644 --- a/codex-rs/codex-mcp/src/lib.rs +++ b/codex-rs/codex-mcp/src/lib.rs @@ -36,8 +36,7 @@ pub use mcp::tool_plugin_provenance; pub use mcp::with_codex_apps_mcp; pub use mcp_connection_manager::CodexAppsToolsCacheKey; pub use mcp_connection_manager::DEFAULT_STARTUP_TIMEOUT; -pub use mcp_connection_manager::MCP_SANDBOX_STATE_CAPABILITY; -pub use mcp_connection_manager::MCP_SANDBOX_STATE_METHOD; +pub use mcp_connection_manager::MCP_SANDBOX_STATE_META_CAPABILITY; pub use mcp_connection_manager::McpConnectionManager; pub use mcp_connection_manager::SandboxState; pub use mcp_connection_manager::ToolInfo; diff --git a/codex-rs/codex-mcp/src/mcp/mod.rs b/codex-rs/codex-mcp/src/mcp/mod.rs index d53532b951..1dc9db0789 100644 --- a/codex-rs/codex-mcp/src/mcp/mod.rs +++ b/codex-rs/codex-mcp/src/mcp/mod.rs @@ -35,7 +35,6 @@ use codex_protocol::protocol::SandboxPolicy; use serde_json::Value; use crate::mcp_connection_manager::McpConnectionManager; -use crate::mcp_connection_manager::SandboxState; use crate::mcp_connection_manager::codex_apps_tools_cache_key; pub type McpManager = McpConnectionManager; @@ -270,12 +269,14 @@ fn codex_apps_mcp_server_config(config: &McpConfig, auth: Option<&CodexAuth>) -> http_headers, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(30)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -347,14 +348,6 @@ pub async fn collect_mcp_snapshot_with_detail( let (tx_event, rx_event) = unbounded(); drop(rx_event); - // Use ReadOnly sandbox policy for MCP snapshot collection (safest default) - let sandbox_state = SandboxState { - sandbox_policy: SandboxPolicy::new_read_only_policy(), - codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(), - sandbox_cwd: env::current_dir().unwrap_or_else(|_| PathBuf::from("/")), - use_legacy_landlock: config.use_legacy_landlock, - }; - let (mcp_connection_manager, cancel_token) = McpConnectionManager::new( &mcp_servers, config.mcp_oauth_credentials_store_mode, @@ -362,7 +355,7 @@ pub async fn collect_mcp_snapshot_with_detail( &config.approval_policy, submit_id, tx_event, - sandbox_state, + SandboxPolicy::new_read_only_policy(), config.codex_home.clone(), codex_apps_tools_cache_key(auth), tool_plugin_provenance, @@ -421,13 +414,6 @@ pub async fn collect_mcp_server_status_snapshot_with_detail( let (tx_event, rx_event) = unbounded(); drop(rx_event); - let sandbox_state = SandboxState { - sandbox_policy: SandboxPolicy::new_read_only_policy(), - codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(), - sandbox_cwd: env::current_dir().unwrap_or_else(|_| PathBuf::from("/")), - use_legacy_landlock: config.use_legacy_landlock, - }; - let (mcp_connection_manager, cancel_token) = McpConnectionManager::new( &mcp_servers, config.mcp_oauth_credentials_store_mode, @@ -435,7 +421,7 @@ pub async fn collect_mcp_server_status_snapshot_with_detail( &config.approval_policy, submit_id, tx_event, - sandbox_state, + SandboxPolicy::new_read_only_policy(), config.codex_home.clone(), codex_apps_tools_cache_key(auth), tool_plugin_provenance, diff --git a/codex-rs/codex-mcp/src/mcp/mod_tests.rs b/codex-rs/codex-mcp/src/mcp/mod_tests.rs index eaffbb8a5d..8db52a9d83 100644 --- a/codex-rs/codex-mcp/src/mcp/mod_tests.rs +++ b/codex-rs/codex-mcp/src/mcp/mod_tests.rs @@ -193,12 +193,14 @@ async fn effective_mcp_servers_preserve_user_servers_and_add_codex_apps() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -215,12 +217,14 @@ async fn effective_mcp_servers_preserve_user_servers_and_add_codex_apps() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/codex-mcp/src/mcp/skill_dependencies.rs b/codex-rs/codex-mcp/src/mcp/skill_dependencies.rs index aa0c4d4e7a..f785fe4bde 100644 --- a/codex-rs/codex-mcp/src/mcp/skill_dependencies.rs +++ b/codex-rs/codex-mcp/src/mcp/skill_dependencies.rs @@ -119,12 +119,14 @@ fn mcp_dependency_to_server_config( http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -146,12 +148,14 @@ fn mcp_dependency_to_server_config( env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/codex-mcp/src/mcp/skill_dependencies_tests.rs b/codex-rs/codex-mcp/src/mcp/skill_dependencies_tests.rs index 0fe2856f01..2d8390d15e 100644 --- a/codex-rs/codex-mcp/src/mcp/skill_dependencies_tests.rs +++ b/codex-rs/codex-mcp/src/mcp/skill_dependencies_tests.rs @@ -39,12 +39,14 @@ fn collect_missing_respects_canonical_installed_key() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -90,12 +92,14 @@ fn collect_missing_dedupes_by_canonical_key_but_preserves_original_name() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/codex-mcp/src/mcp_connection_manager.rs b/codex-rs/codex-mcp/src/mcp_connection_manager.rs index 9021f7d7a9..d1696b5931 100644 --- a/codex-rs/codex-mcp/src/mcp_connection_manager.rs +++ b/codex-rs/codex-mcp/src/mcp_connection_manager.rs @@ -36,6 +36,7 @@ use codex_async_utils::CancelErr; use codex_async_utils::OrCancelExt; use codex_config::Constrained; use codex_config::types::OAuthCredentialsStoreMode; +use codex_protocol::ToolName; use codex_protocol::approvals::ElicitationRequest; use codex_protocol::approvals::ElicitationRequestEvent; use codex_protocol::mcp::CallToolResult; @@ -155,6 +156,12 @@ pub struct ToolInfo { pub connector_description: Option, } +impl ToolInfo { + pub fn canonical_tool_name(&self) -> ToolName { + ToolName::namespaced(self.callable_namespace.clone(), self.callable_name.clone()) + } +} + const META_OPENAI_FILE_PARAMS: &str = "openai/fileParams"; pub fn declared_openai_file_input_param_names( @@ -432,7 +439,7 @@ struct ManagedClient { tool_filter: ToolFilter, tool_timeout: Option, server_instructions: Option, - server_supports_sandbox_state_capability: bool, + server_supports_sandbox_state_meta_capability: bool, codex_apps_tools_cache_context: Option, } @@ -461,22 +468,6 @@ impl ManagedClient { self.tools.clone() } - - /// Returns once the server has ack'd the sandbox state update. - async fn notify_sandbox_state_change(&self, sandbox_state: &SandboxState) -> Result<()> { - if !self.server_supports_sandbox_state_capability { - return Ok(()); - } - - let _response = self - .client - .send_custom_request( - MCP_SANDBOX_STATE_METHOD, - Some(serde_json::to_value(sandbox_state)?), - ) - .await?; - Ok(()) - } } #[derive(Clone)] @@ -634,18 +625,11 @@ impl AsyncManagedClient { }; tools.map(annotate_tools) } - - async fn notify_sandbox_state_change(&self, sandbox_state: &SandboxState) -> Result<()> { - let managed = self.client().await?; - managed.notify_sandbox_state_change(sandbox_state).await - } } -pub const MCP_SANDBOX_STATE_CAPABILITY: &str = "codex/sandbox-state"; - -/// Custom MCP request to push sandbox state updates. -/// When used, the `params` field of the notification is [`SandboxState`]. -pub const MCP_SANDBOX_STATE_METHOD: &str = "codex/sandbox-state/update"; +/// MCP server capability indicating that Codex should include [`SandboxState`] +/// in tool-call request `_meta` under this key. +pub const MCP_SANDBOX_STATE_META_CAPABILITY: &str = "codex/sandbox-state-meta"; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -723,7 +707,7 @@ impl McpConnectionManager { approval_policy: &Constrained, submit_id: String, tx_event: Sender, - initial_sandbox_state: SandboxState, + initial_sandbox_policy: SandboxPolicy, codex_home: PathBuf, codex_apps_tools_cache_key: CodexAppsToolsCacheKey, tool_plugin_provenance: ToolPluginProvenance, @@ -732,10 +716,8 @@ impl McpConnectionManager { let mut clients = HashMap::new(); let mut server_origins = HashMap::new(); let mut join_set = JoinSet::new(); - let elicitation_requests = ElicitationRequestManager::new( - approval_policy.value(), - initial_sandbox_state.sandbox_policy.clone(), - ); + let elicitation_requests = + ElicitationRequestManager::new(approval_policy.value(), initial_sandbox_policy); let tool_plugin_provenance = Arc::new(tool_plugin_provenance); let startup_submit_id = submit_id.clone(); let mcp_servers = mcp_servers.clone(); @@ -775,25 +757,14 @@ impl McpConnectionManager { let tx_event = tx_event.clone(); let submit_id = startup_submit_id.clone(); let auth_entry = auth_entries.get(&server_name).cloned(); - let sandbox_state = initial_sandbox_state.clone(); join_set.spawn(async move { - let outcome = async_managed_client.client().await; + let mut outcome = async_managed_client.client().await; if cancel_token.is_cancelled() { - return (server_name, Err(StartupOutcomeError::Cancelled)); + outcome = Err(StartupOutcomeError::Cancelled); } let status = match &outcome { - Ok(_) => { - // Send sandbox state notification immediately after Ready - if let Err(e) = async_managed_client - .notify_sandbox_state_change(&sandbox_state) - .await - { - warn!( - "Failed to notify sandbox state to MCP server {server_name}: {e:#}", - ); - } - McpStartupStatus::Ready - } + Ok(_) => McpStartupStatus::Ready, + Err(StartupOutcomeError::Cancelled) => McpStartupStatus::Cancelled, Err(error) => { let error_str = mcp_init_error_display( server_name.as_str(), @@ -1142,6 +1113,16 @@ impl McpConnectionManager { }) } + pub async fn server_supports_sandbox_state_meta_capability( + &self, + server: &str, + ) -> Result { + Ok(self + .client_by_name(server) + .await? + .server_supports_sandbox_state_meta_capability) + } + /// List resources from the specified server. pub async fn list_resources( &self, @@ -1191,42 +1172,11 @@ impl McpConnectionManager { .with_context(|| format!("resources/read failed for `{server}` ({uri})")) } - pub async fn resolve_tool_info(&self, name: &str, namespace: Option<&str>) -> Option { - let qualified_name = match namespace { - Some(namespace) if name.starts_with(namespace) => name.to_string(), - Some(namespace) => format!("{namespace}{name}"), - None => name.to_string(), - }; - - self.list_all_tools().await.get(&qualified_name).cloned() - } - - pub async fn notify_sandbox_state_change(&self, sandbox_state: &SandboxState) -> Result<()> { - let mut join_set = JoinSet::new(); - - for async_managed_client in self.clients.values() { - let sandbox_state = sandbox_state.clone(); - let async_managed_client = async_managed_client.clone(); - join_set.spawn(async move { - async_managed_client - .notify_sandbox_state_change(&sandbox_state) - .await - }); - } - - while let Some(join_res) = join_set.join_next().await { - match join_res { - Ok(Ok(())) => {} - Ok(Err(err)) => { - warn!("Failed to notify sandbox state change to MCP server: {err:#}"); - } - Err(err) => { - warn!("Task panic when notifying sandbox state change to MCP server: {err:#}"); - } - } - } - - Ok(()) + pub async fn resolve_tool_info(&self, tool_name: &ToolName) -> Option { + let all_tools = self.list_all_tools().await; + all_tools + .into_values() + .find(|tool| tool.canonical_tool_name() == *tool_name) } } @@ -1473,11 +1423,11 @@ async fn start_server_task( .await .map_err(StartupOutcomeError::from)?; - let server_supports_sandbox_state_capability = initialize_result + let server_supports_sandbox_state_meta_capability = initialize_result .capabilities .experimental .as_ref() - .and_then(|exp| exp.get(MCP_SANDBOX_STATE_CAPABILITY)) + .and_then(|exp| exp.get(MCP_SANDBOX_STATE_META_CAPABILITY)) .is_some(); let list_start = Instant::now(); let fetch_start = Instant::now(); @@ -1514,7 +1464,7 @@ async fn start_server_task( tool_timeout: Some(tool_timeout), tool_filter, server_instructions: initialize_result.instructions, - server_supports_sandbox_state_capability, + server_supports_sandbox_state_meta_capability, codex_apps_tools_cache_context, }; diff --git a/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs b/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs index bfa50c4182..cf2889ccde 100644 --- a/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs +++ b/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs @@ -1,4 +1,5 @@ use super::*; +use codex_protocol::ToolName; use codex_protocol::protocol::GranularApprovalConfig; use codex_protocol::protocol::McpAuthStatus; use pretty_assertions::assert_eq; @@ -646,6 +647,42 @@ async fn list_all_tools_uses_startup_snapshot_while_client_is_pending() { assert_eq!(tool.callable_name, "calendar_create_event"); } +#[tokio::test] +async fn resolve_tool_info_accepts_canonical_namespaced_tool_names() { + let startup_tools = vec![create_test_tool("rmcp", "echo")]; + let pending_client = futures::future::pending::>() + .boxed() + .shared(); + let approval_policy = Constrained::allow_any(AskForApproval::OnFailure); + let sandbox_policy = Constrained::allow_any(SandboxPolicy::new_read_only_policy()); + let mut manager = McpConnectionManager::new_uninitialized(&approval_policy, &sandbox_policy); + manager.clients.insert( + "rmcp".to_string(), + AsyncManagedClient { + client: pending_client, + startup_snapshot: Some(startup_tools), + startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(false)), + tool_plugin_provenance: Arc::new(ToolPluginProvenance::default()), + }, + ); + + let tool = manager + .resolve_tool_info(&ToolName::namespaced("mcp__rmcp__", "echo")) + .await + .expect("split MCP tool namespace and name should resolve"); + + let expected = ("rmcp", "mcp__rmcp__", "echo", "echo"); + assert_eq!( + ( + tool.server_name.as_str(), + tool.callable_namespace.as_str(), + tool.callable_name.as_str(), + tool.tool.name.as_ref(), + ), + expected + ); +} + #[tokio::test] async fn list_all_tools_blocks_while_client_is_pending_without_startup_snapshot() { let pending_client = futures::future::pending::>() @@ -755,12 +792,14 @@ fn mcp_init_error_display_prompts_for_github_pat() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -805,12 +844,14 @@ fn mcp_init_error_display_reports_generic_errors() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/config/src/config_toml.rs b/codex-rs/config/src/config_toml.rs index 92e5304fe5..7de4e253e5 100644 --- a/codex-rs/config/src/config_toml.rs +++ b/codex-rs/config/src/config_toml.rs @@ -596,7 +596,7 @@ pub struct GhostSnapshotToml { impl ConfigToml { /// Derive the effective sandbox policy from the configuration. - pub fn derive_sandbox_policy( + pub async fn derive_sandbox_policy( &self, sandbox_mode_override: Option, profile_sandbox_mode: Option, @@ -610,11 +610,13 @@ impl ConfigToml { let resolved_sandbox_mode = sandbox_mode_override .or(profile_sandbox_mode) .or(self.sandbox_mode) - .or_else(|| { + .or(if sandbox_mode_was_explicit { + None + } else { // If no sandbox_mode is set but this directory has a trust decision, // default to workspace-write except on unsandboxed Windows where we // default to read-only. - self.get_active_project(resolved_cwd).and_then(|p| { + self.get_active_project(resolved_cwd).await.and_then(|p| { if p.is_trusted() || p.is_untrusted() { if cfg!(target_os = "windows") && windows_sandbox_level == WindowsSandboxLevel::Disabled @@ -676,7 +678,8 @@ impl ConfigToml { /// Resolves the cwd to an existing project, or returns None if ConfigToml /// does not contain a project corresponding to cwd or a git repo for cwd - pub fn get_active_project(&self, resolved_cwd: &Path) -> Option { + pub async fn get_active_project(&self, resolved_cwd: &Path) -> Option { + let repo_root = resolve_root_git_project_for_trust(resolved_cwd).await; let projects = self.projects.clone().unwrap_or_default(); let resolved_cwd_key = project_trust_key(resolved_cwd); @@ -691,8 +694,8 @@ impl ConfigToml { // If cwd lives inside a git repo/worktree, check whether the root git project // (the primary repository working directory) is trusted. This lets // worktrees inherit trust from the main project. - if let Some(repo_root) = resolve_root_git_project_for_trust(resolved_cwd) { - let repo_root_key = project_trust_key(repo_root.as_path()); + if let Some(repo_root) = repo_root.as_deref() { + let repo_root_key = project_trust_key(repo_root); let repo_root_raw_key = repo_root.to_string_lossy().to_string(); if let Some(project_config_for_root) = projects .get(&repo_root_key) diff --git a/codex-rs/config/src/marketplace_edit.rs b/codex-rs/config/src/marketplace_edit.rs index 33cdd8e163..e20a75a02e 100644 --- a/codex-rs/config/src/marketplace_edit.rs +++ b/codex-rs/config/src/marketplace_edit.rs @@ -12,6 +12,7 @@ use crate::CONFIG_TOML_FILE; pub struct MarketplaceConfigUpdate<'a> { pub last_updated: &'a str, + pub last_revision: Option<&'a str>, pub source_type: &'a str, pub source: &'a str, pub ref_name: Option<&'a str>, @@ -63,6 +64,9 @@ fn upsert_marketplace( let mut entry = TomlTable::new(); entry.set_implicit(false); entry["last_updated"] = value(update.last_updated.to_string()); + if let Some(last_revision) = update.last_revision { + entry["last_revision"] = value(last_revision.to_string()); + } entry["source_type"] = value(update.source_type.to_string()); entry["source"] = value(update.source.to_string()); if let Some(ref_name) = update.ref_name { diff --git a/codex-rs/config/src/mcp_edit.rs b/codex-rs/config/src/mcp_edit.rs index 965f869362..c4bb38c543 100644 --- a/codex-rs/config/src/mcp_edit.rs +++ b/codex-rs/config/src/mcp_edit.rs @@ -174,6 +174,9 @@ fn serialize_mcp_server(config: &McpServerConfig) -> TomlItem { if !config.enabled { entry["enabled"] = value(false); } + if let Some(environment) = &config.experimental_environment { + entry["experimental_environment"] = value(environment.clone()); + } if config.required { entry["required"] = value(true); } @@ -186,6 +189,13 @@ fn serialize_mcp_server(config: &McpServerConfig) -> TomlItem { if let Some(timeout) = config.tool_timeout_sec { entry["tool_timeout_sec"] = value(timeout.as_secs_f64()); } + if let Some(approval_mode) = config.default_tools_approval_mode { + entry["default_tools_approval_mode"] = value(match approval_mode { + AppToolApproval::Auto => "auto", + AppToolApproval::Prompt => "prompt", + AppToolApproval::Approve => "approve", + }); + } if let Some(enabled_tools) = &config.enabled_tools && !enabled_tools.is_empty() { diff --git a/codex-rs/config/src/mcp_edit_tests.rs b/codex-rs/config/src/mcp_edit_tests.rs index 38ab0852fe..cfcd73c3e5 100644 --- a/codex-rs/config/src/mcp_edit_tests.rs +++ b/codex-rs/config/src/mcp_edit_tests.rs @@ -22,12 +22,14 @@ async fn replace_mcp_servers_serializes_per_tool_approval_overrides() -> anyhow: env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: true, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: Some(AppToolApproval::Auto), enabled_tools: None, disabled_tools: None, scopes: None, @@ -61,6 +63,7 @@ async fn replace_mcp_servers_serializes_per_tool_approval_overrides() -> anyhow: r#"[mcp_servers.docs] command = "docs-server" supports_parallel_tool_calls = true +default_tools_approval_mode = "auto" [mcp_servers.docs.tools] diff --git a/codex-rs/config/src/mcp_types.rs b/codex-rs/config/src/mcp_types.rs index c00988c6ff..75b68c3f94 100644 --- a/codex-rs/config/src/mcp_types.rs +++ b/codex-rs/config/src/mcp_types.rs @@ -61,6 +61,10 @@ pub struct McpServerConfig { #[serde(flatten)] pub transport: McpServerTransportConfig, + /// Experimental environment selector for where Codex should start this MCP server. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub experimental_environment: Option, + /// When `false`, Codex skips initializing this MCP server. #[serde(default = "default_enabled")] pub enabled: bool, @@ -89,6 +93,10 @@ pub struct McpServerConfig { #[serde(default, with = "option_duration_secs")] pub tool_timeout_sec: Option, + /// Approval mode for tools in this server unless a tool override exists. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub default_tools_approval_mode: Option, + /// Explicit allow-list of tools exposed from this server. When set, only these tools will be registered. #[serde(default, skip_serializing_if = "Option::is_none")] pub enabled_tools: Option>, @@ -139,6 +147,8 @@ pub struct RawMcpServerConfig { // shared #[serde(default)] + pub experimental_environment: Option, + #[serde(default)] pub startup_timeout_sec: Option, #[serde(default)] pub startup_timeout_ms: Option, @@ -152,6 +162,8 @@ pub struct RawMcpServerConfig { #[serde(default)] pub supports_parallel_tool_calls: Option, #[serde(default)] + pub default_tools_approval_mode: Option, + #[serde(default)] pub enabled_tools: Option>, #[serde(default)] pub disabled_tools: Option>, @@ -181,12 +193,14 @@ impl TryFrom for McpServerConfig { url, bearer_token, bearer_token_env_var, + experimental_environment, startup_timeout_sec, startup_timeout_ms, tool_timeout_sec, enabled, required, supports_parallel_tool_calls, + default_tools_approval_mode, enabled_tools, disabled_tools, scopes, @@ -246,12 +260,14 @@ impl TryFrom for McpServerConfig { Ok(Self { transport, + experimental_environment, startup_timeout_sec, tool_timeout_sec, enabled: enabled.unwrap_or_else(default_enabled), required: required.unwrap_or_default(), supports_parallel_tool_calls: supports_parallel_tool_calls.unwrap_or_default(), disabled_reason: None, + default_tools_approval_mode, enabled_tools, disabled_tools, scopes, diff --git a/codex-rs/config/src/mcp_types_tests.rs b/codex-rs/config/src/mcp_types_tests.rs index 6b9fb16e90..dff4a6bbf5 100644 --- a/codex-rs/config/src/mcp_types_tests.rs +++ b/codex-rs/config/src/mcp_types_tests.rs @@ -258,6 +258,38 @@ fn deserialize_server_config_with_parallel_tool_calls() { assert!(cfg.supports_parallel_tool_calls); } +#[test] +fn deserialize_server_config_with_default_tool_approval_mode() { + let cfg: McpServerConfig = toml::from_str( + r#" + command = "echo" + default_tools_approval_mode = "approve" + + [tools.search] + approval_mode = "prompt" + "#, + ) + .expect("should deserialize default tool approval mode"); + + assert_eq!( + cfg.default_tools_approval_mode, + Some(AppToolApproval::Approve) + ); + assert_eq!( + cfg.tools.get("search"), + Some(&McpServerToolConfig { + approval_mode: Some(AppToolApproval::Prompt), + }) + ); + + let serialized = toml::to_string(&cfg).expect("should serialize MCP config"); + assert!(serialized.contains("default_tools_approval_mode = \"approve\"")); + + let round_tripped: McpServerConfig = + toml::from_str(&serialized).expect("should deserialize serialized MCP config"); + assert_eq!(round_tripped, cfg); +} + #[test] fn serialize_round_trips_server_config_with_parallel_tool_calls() { let cfg: McpServerConfig = toml::from_str( @@ -297,12 +329,14 @@ fn deserialize_ignores_unknown_server_fields() { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/config/src/types.rs b/codex-rs/config/src/types.rs index 73a5763fe1..b3576cb3c1 100644 --- a/codex-rs/config/src/types.rs +++ b/codex-rs/config/src/types.rs @@ -31,6 +31,10 @@ pub const DEFAULT_MEMORIES_MAX_ROLLOUT_AGE_DAYS: i64 = 30; pub const DEFAULT_MEMORIES_MIN_ROLLOUT_IDLE_HOURS: i64 = 6; pub const DEFAULT_MEMORIES_MAX_RAW_MEMORIES_FOR_CONSOLIDATION: usize = 256; pub const DEFAULT_MEMORIES_MAX_UNUSED_DAYS: i64 = 30; +const MIN_MEMORIES_MAX_RAW_MEMORIES_FOR_CONSOLIDATION: usize = 1; +const MAX_MEMORIES_MAX_RAW_MEMORIES_FOR_CONSOLIDATION: usize = 4096; +const MIN_MEMORIES_MAX_ROLLOUTS_PER_STARTUP: usize = 1; +const MAX_MEMORIES_MAX_ROLLOUTS_PER_STARTUP: usize = 128; const fn default_enabled() -> bool { true @@ -185,12 +189,14 @@ pub struct MemoriesToml { /// When `false`, skip injecting memory usage instructions into developer prompts. pub use_memories: Option, /// Maximum number of recent raw memories retained for global consolidation. + #[schemars(range(min = 1, max = 4096))] pub max_raw_memories_for_consolidation: Option, /// Maximum number of days since a memory was last used before it becomes ineligible for phase 2 selection. pub max_unused_days: Option, /// Maximum age of the threads used for memories. pub max_rollout_age_days: Option, /// Maximum number of rollout candidates processed per pass. + #[schemars(range(min = 1, max = 128))] pub max_rollouts_per_startup: Option, /// Minimum idle time between last thread activity and memory creation (hours). > 12h recommended. pub min_rollout_idle_hours: Option, @@ -244,7 +250,10 @@ impl From for MemoriesConfig { max_raw_memories_for_consolidation: toml .max_raw_memories_for_consolidation .unwrap_or(defaults.max_raw_memories_for_consolidation) - .min(4096), + .clamp( + MIN_MEMORIES_MAX_RAW_MEMORIES_FOR_CONSOLIDATION, + MAX_MEMORIES_MAX_RAW_MEMORIES_FOR_CONSOLIDATION, + ), max_unused_days: toml .max_unused_days .unwrap_or(defaults.max_unused_days) @@ -256,7 +265,10 @@ impl From for MemoriesConfig { max_rollouts_per_startup: toml .max_rollouts_per_startup .unwrap_or(defaults.max_rollouts_per_startup) - .min(128), + .clamp( + MIN_MEMORIES_MAX_ROLLOUTS_PER_STARTUP, + MAX_MEMORIES_MAX_ROLLOUTS_PER_STARTUP, + ), min_rollout_idle_hours: toml .min_rollout_idle_hours .unwrap_or(defaults.min_rollout_idle_hours) @@ -614,6 +626,9 @@ pub struct MarketplaceConfig { /// Last time Codex successfully added or refreshed this marketplace. #[serde(default)] pub last_updated: Option, + /// Git revision Codex last successfully activated for this marketplace. + #[serde(default)] + pub last_revision: Option, /// Source kind used to install this marketplace. #[serde(default)] pub source_type: Option, diff --git a/codex-rs/config/src/types_tests.rs b/codex-rs/config/src/types_tests.rs index fbcd2bdc64..b18c1cc645 100644 --- a/codex-rs/config/src/types_tests.rs +++ b/codex-rs/config/src/types_tests.rs @@ -41,3 +41,21 @@ fn deserialize_skill_config_with_path_selector() { } ); } + +#[test] +fn memories_config_clamps_count_limits_to_nonzero_values() { + let config = MemoriesConfig::from(MemoriesToml { + max_raw_memories_for_consolidation: Some(0), + max_rollouts_per_startup: Some(0), + ..Default::default() + }); + + assert_eq!( + config, + MemoriesConfig { + max_raw_memories_for_consolidation: 1, + max_rollouts_per_startup: 1, + ..MemoriesConfig::default() + } + ); +} diff --git a/codex-rs/connectors/src/accessible.rs b/codex-rs/connectors/src/accessible.rs new file mode 100644 index 0000000000..c44f8d8a38 --- /dev/null +++ b/codex-rs/connectors/src/accessible.rs @@ -0,0 +1,76 @@ +use std::collections::BTreeSet; +use std::collections::HashMap; + +use crate::metadata::connector_install_url; +use crate::normalize_connector_value; +use codex_app_server_protocol::AppInfo; + +pub struct AccessibleConnectorTool { + pub connector_id: String, + pub connector_name: Option, + pub connector_description: Option, + pub plugin_display_names: Vec, +} + +pub fn collect_accessible_connectors(tools: I) -> Vec +where + I: IntoIterator, +{ + let mut connectors: HashMap)> = HashMap::new(); + for tool in tools { + let connector_id = tool.connector_id; + let connector_name = normalize_connector_value(tool.connector_name.as_deref()) + .unwrap_or_else(|| connector_id.clone()); + let connector_description = + normalize_connector_value(tool.connector_description.as_deref()); + if let Some((existing, existing_plugin_display_names)) = connectors.get_mut(&connector_id) { + if existing.name == connector_id && connector_name != connector_id { + existing.name = connector_name; + } + if existing.description.is_none() && connector_description.is_some() { + existing.description = connector_description; + } + existing_plugin_display_names.extend(tool.plugin_display_names); + } else { + connectors.insert( + connector_id.clone(), + ( + AppInfo { + id: connector_id.clone(), + name: connector_name, + description: connector_description, + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + branding: None, + app_metadata: None, + labels: None, + install_url: None, + is_accessible: true, + is_enabled: true, + plugin_display_names: Vec::new(), + }, + tool.plugin_display_names + .into_iter() + .collect::>(), + ), + ); + } + } + let mut accessible: Vec = connectors + .into_values() + .map(|(mut connector, plugin_display_names)| { + connector.plugin_display_names = plugin_display_names.into_iter().collect(); + connector.install_url = Some(connector_install_url(&connector.name, &connector.id)); + connector + }) + .collect(); + accessible.sort_by(|left, right| { + right + .is_accessible + .cmp(&left.is_accessible) + .then_with(|| left.name.cmp(&right.name)) + .then_with(|| left.id.cmp(&right.id)) + }); + accessible +} diff --git a/codex-rs/connectors/src/filter.rs b/codex-rs/connectors/src/filter.rs new file mode 100644 index 0000000000..82c334f82d --- /dev/null +++ b/codex-rs/connectors/src/filter.rs @@ -0,0 +1,68 @@ +use std::collections::HashSet; + +use codex_app_server_protocol::AppInfo; + +pub fn filter_tool_suggest_discoverable_connectors( + directory_connectors: Vec, + accessible_connectors: &[AppInfo], + discoverable_connector_ids: &HashSet, + originator_value: &str, +) -> Vec { + let accessible_connector_ids: HashSet<&str> = accessible_connectors + .iter() + .filter(|connector| connector.is_accessible) + .map(|connector| connector.id.as_str()) + .collect(); + + let mut connectors = filter_disallowed_connectors(directory_connectors, originator_value) + .into_iter() + .filter(|connector| !accessible_connector_ids.contains(connector.id.as_str())) + .filter(|connector| discoverable_connector_ids.contains(connector.id.as_str())) + .collect::>(); + connectors.sort_by(|left, right| { + left.name + .cmp(&right.name) + .then_with(|| left.id.cmp(&right.id)) + }); + connectors +} + +const DISALLOWED_CONNECTOR_IDS: &[&str] = &[ + "asdk_app_6938a94a61d881918ef32cb999ff937c", + "connector_2b0a9009c9c64bf9933a3dae3f2b1254", + "connector_3f8d1a79f27c4c7ba1a897ab13bf37dc", + "connector_68de829bf7648191acd70a907364c67c", + "connector_68e004f14af881919eb50893d3d9f523", + "connector_69272cb413a081919685ec3c88d1744e", +]; +const FIRST_PARTY_CHAT_DISALLOWED_CONNECTOR_IDS: &[&str] = + &["connector_0f9c9d4592e54d0a9a12b3f44a1e2010"]; +const DISALLOWED_CONNECTOR_PREFIX: &str = "connector_openai_"; + +pub fn filter_disallowed_connectors( + connectors: Vec, + originator_value: &str, +) -> Vec { + let first_party_chat_originator = is_first_party_chat_originator(originator_value); + connectors + .into_iter() + .filter(|connector| { + is_connector_id_allowed(connector.id.as_str(), first_party_chat_originator) + }) + .collect() +} + +fn is_first_party_chat_originator(originator_value: &str) -> bool { + originator_value == "codex_atlas" || originator_value == "codex_chatgpt_desktop" +} + +fn is_connector_id_allowed(connector_id: &str, first_party_chat_originator: bool) -> bool { + let disallowed_connector_ids = if first_party_chat_originator { + FIRST_PARTY_CHAT_DISALLOWED_CONNECTOR_IDS + } else { + DISALLOWED_CONNECTOR_IDS + }; + + !connector_id.starts_with(DISALLOWED_CONNECTOR_PREFIX) + && !disallowed_connector_ids.contains(&connector_id) +} diff --git a/codex-rs/connectors/src/lib.rs b/codex-rs/connectors/src/lib.rs index a74ece5a35..d7c5fb405c 100644 --- a/codex-rs/connectors/src/lib.rs +++ b/codex-rs/connectors/src/lib.rs @@ -10,6 +10,11 @@ use codex_app_server_protocol::AppInfo; use codex_app_server_protocol::AppMetadata; use serde::Deserialize; +pub mod accessible; +pub mod filter; +pub mod merge; +pub mod metadata; + pub const CONNECTORS_CACHE_TTL: Duration = Duration::from_secs(3600); #[derive(Clone, Debug, PartialEq, Eq)] diff --git a/codex-rs/connectors/src/merge.rs b/codex-rs/connectors/src/merge.rs new file mode 100644 index 0000000000..b41ee63add --- /dev/null +++ b/codex-rs/connectors/src/merge.rs @@ -0,0 +1,119 @@ +use std::collections::HashMap; +use std::collections::HashSet; + +use crate::metadata::connector_install_url; +use crate::metadata::sort_connectors_by_accessibility_and_name; +use codex_app_server_protocol::AppInfo; + +pub fn merge_connectors( + connectors: Vec, + accessible_connectors: Vec, +) -> Vec { + let mut merged: HashMap = connectors + .into_iter() + .map(|mut connector| { + connector.is_accessible = false; + (connector.id.clone(), connector) + }) + .collect(); + + for mut connector in accessible_connectors { + connector.is_accessible = true; + let connector_id = connector.id.clone(); + if let Some(existing) = merged.get_mut(&connector_id) { + existing.is_accessible = true; + if existing.name == existing.id && connector.name != connector.id { + existing.name = connector.name; + } + if existing.description.is_none() && connector.description.is_some() { + existing.description = connector.description; + } + if existing.logo_url.is_none() && connector.logo_url.is_some() { + existing.logo_url = connector.logo_url; + } + if existing.logo_url_dark.is_none() && connector.logo_url_dark.is_some() { + existing.logo_url_dark = connector.logo_url_dark; + } + if existing.distribution_channel.is_none() && connector.distribution_channel.is_some() { + existing.distribution_channel = connector.distribution_channel; + } + existing + .plugin_display_names + .extend(connector.plugin_display_names); + } else { + merged.insert(connector_id, connector); + } + } + + let mut merged = merged.into_values().collect::>(); + for connector in &mut merged { + if connector.install_url.is_none() { + connector.install_url = Some(connector_install_url(&connector.name, &connector.id)); + } + connector.plugin_display_names.sort_unstable(); + connector.plugin_display_names.dedup(); + } + sort_connectors_by_accessibility_and_name(&mut merged); + merged +} + +pub fn merge_plugin_connectors(connectors: Vec, plugin_app_ids: I) -> Vec +where + I: IntoIterator, +{ + let mut merged = connectors; + let mut connector_ids = merged + .iter() + .map(|connector| connector.id.clone()) + .collect::>(); + + for connector_id in plugin_app_ids { + if connector_ids.insert(connector_id.clone()) { + merged.push(plugin_connector_to_app_info(connector_id)); + } + } + + sort_connectors_by_accessibility_and_name(&mut merged); + merged +} + +pub fn merge_plugin_connectors_with_accessible( + plugin_app_ids: I, + accessible_connectors: Vec, +) -> Vec +where + I: IntoIterator, +{ + let accessible_connector_ids: HashSet<&str> = accessible_connectors + .iter() + .map(|connector| connector.id.as_str()) + .collect(); + let plugin_connectors = plugin_app_ids + .into_iter() + .filter(|connector_id| accessible_connector_ids.contains(connector_id.as_str())) + .map(plugin_connector_to_app_info) + .collect::>(); + merge_connectors(plugin_connectors, accessible_connectors) +} + +pub fn plugin_connector_to_app_info(connector_id: String) -> AppInfo { + // Leave the placeholder name as the connector id so merge_connectors() can + // replace it with canonical app metadata from directory fetches or + // connector_name values from codex_apps tool discovery. + let name = connector_id.clone(); + AppInfo { + id: connector_id.clone(), + name: name.clone(), + description: None, + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + branding: None, + app_metadata: None, + labels: None, + install_url: Some(connector_install_url(&name, &connector_id)), + is_accessible: false, + is_enabled: true, + plugin_display_names: Vec::new(), + } +} diff --git a/codex-rs/connectors/src/metadata.rs b/codex-rs/connectors/src/metadata.rs new file mode 100644 index 0000000000..64becb766d --- /dev/null +++ b/codex-rs/connectors/src/metadata.rs @@ -0,0 +1,27 @@ +use codex_app_server_protocol::AppInfo; + +pub fn connector_display_label(connector: &AppInfo) -> String { + connector.name.clone() +} + +pub fn connector_mention_slug(connector: &AppInfo) -> String { + crate::connector_name_slug(&connector_display_label(connector)) +} + +pub fn connector_install_url(name: &str, connector_id: &str) -> String { + crate::connector_install_url(name, connector_id) +} + +pub fn sanitize_name(name: &str) -> String { + crate::connector_name_slug(name).replace("-", "_") +} + +pub(crate) fn sort_connectors_by_accessibility_and_name(connectors: &mut [AppInfo]) { + connectors.sort_by(|left, right| { + right + .is_accessible + .cmp(&left.is_accessible) + .then_with(|| left.name.cmp(&right.name)) + .then_with(|| left.id.cmp(&right.id)) + }); +} diff --git a/codex-rs/core-plugins/BUILD.bazel b/codex-rs/core-plugins/BUILD.bazel new file mode 100644 index 0000000000..aa19b9f368 --- /dev/null +++ b/codex-rs/core-plugins/BUILD.bazel @@ -0,0 +1,15 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "core-plugins", + crate_name = "codex_core_plugins", + compile_data = glob( + include = ["**"], + exclude = [ + "**/* *", + "BUILD.bazel", + "Cargo.toml", + ], + allow_empty = True, + ), +) diff --git a/codex-rs/core-plugins/Cargo.toml b/codex-rs/core-plugins/Cargo.toml new file mode 100644 index 0000000000..0372d9a14a --- /dev/null +++ b/codex-rs/core-plugins/Cargo.toml @@ -0,0 +1,40 @@ +[package] +edition.workspace = true +license.workspace = true +name = "codex-core-plugins" +version.workspace = true + +[lib] +doctest = false +name = "codex_core_plugins" +path = "src/lib.rs" + +[lints] +workspace = true + +[dependencies] +codex-app-server-protocol = { workspace = true } +codex-config = { workspace = true } +codex-core-skills = { workspace = true } +codex-exec-server = { workspace = true } +codex-git-utils = { workspace = true } +codex-login = { workspace = true } +codex-plugin = { workspace = true } +codex-protocol = { workspace = true } +codex-utils-absolute-path = { workspace = true } +codex-utils-plugins = { workspace = true } +chrono = { workspace = true } +dirs = { workspace = true } +reqwest = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +tempfile = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["fs"] } +toml = { workspace = true } +tracing = { workspace = true } +url = { workspace = true } + +[dev-dependencies] +pretty_assertions = { workspace = true } +tempfile = { workspace = true } diff --git a/codex-rs/core-plugins/src/lib.rs b/codex-rs/core-plugins/src/lib.rs new file mode 100644 index 0000000000..82ff4df3c7 --- /dev/null +++ b/codex-rs/core-plugins/src/lib.rs @@ -0,0 +1,7 @@ +pub mod loader; +pub mod manifest; +pub mod marketplace; +pub mod marketplace_upgrade; +pub mod remote; +pub mod store; +pub mod toggles; diff --git a/codex-rs/core-plugins/src/loader.rs b/codex-rs/core-plugins/src/loader.rs new file mode 100644 index 0000000000..ff3b8fc494 --- /dev/null +++ b/codex-rs/core-plugins/src/loader.rs @@ -0,0 +1,838 @@ +use crate::manifest::PluginManifestPaths; +use crate::manifest::load_plugin_manifest; +use crate::marketplace::MarketplacePluginSource; +use crate::marketplace::list_marketplaces; +use crate::marketplace::load_marketplace; +use crate::store::PluginStore; +use crate::store::plugin_version_for_source; +use codex_config::ConfigLayerStack; +use codex_config::types::McpServerConfig; +use codex_config::types::PluginConfig; +use codex_core_skills::SkillMetadata; +use codex_core_skills::config_rules::SkillConfigRules; +use codex_core_skills::config_rules::resolve_disabled_skill_paths; +use codex_core_skills::config_rules::skill_config_rules_from_stack; +use codex_core_skills::loader::SkillRoot; +use codex_core_skills::loader::load_skills_from_roots; +use codex_exec_server::LOCAL_FS; +use codex_plugin::AppConnectorId; +use codex_plugin::LoadedPlugin; +use codex_plugin::PluginCapabilitySummary; +use codex_plugin::PluginId; +use codex_plugin::PluginIdError; +use codex_plugin::PluginLoadOutcome; +use codex_plugin::PluginTelemetryMetadata; +use codex_protocol::protocol::Product; +use codex_protocol::protocol::SkillScope; +use codex_utils_absolute_path::AbsolutePathBuf; +use serde::Deserialize; +use serde_json::Map as JsonMap; +use serde_json::Value as JsonValue; +use std::collections::HashMap; +use std::collections::HashSet; +use std::fs; +use std::path::Path; +use std::sync::Arc; +use tracing::warn; + +const DEFAULT_SKILLS_DIR_NAME: &str = "skills"; +const DEFAULT_MCP_CONFIG_FILE: &str = ".mcp.json"; +const DEFAULT_APP_CONFIG_FILE: &str = ".app.json"; +const OPENAI_CURATED_MARKETPLACE_NAME: &str = "openai-curated"; +const CONFIG_TOML_FILE: &str = "config.toml"; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum NonCuratedCacheRefreshMode { + IfVersionChanged, + ForceReinstall, +} + +pub fn log_plugin_load_errors(outcome: &PluginLoadOutcome) { + for plugin in outcome + .plugins() + .iter() + .filter(|plugin| plugin.error.is_some()) + { + if let Some(error) = plugin.error.as_deref() { + warn!( + plugin = plugin.config_name, + path = %plugin.root.display(), + "failed to load plugin: {error}" + ); + } + } +} + +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +struct PluginMcpFile { + #[serde(default)] + mcp_servers: HashMap, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +struct PluginAppFile { + #[serde(default)] + apps: HashMap, +} + +#[derive(Debug, Default, Deserialize)] +struct PluginAppConfig { + id: String, +} + +pub async fn load_plugins_from_layer_stack( + config_layer_stack: &ConfigLayerStack, + store: &PluginStore, + restriction_product: Option, +) -> PluginLoadOutcome { + let skill_config_rules = skill_config_rules_from_stack(config_layer_stack); + let mut configured_plugins: Vec<_> = configured_plugins_from_stack(config_layer_stack) + .into_iter() + .collect(); + configured_plugins.sort_unstable_by(|(a, _), (b, _)| a.cmp(b)); + + let mut plugins = Vec::with_capacity(configured_plugins.len()); + let mut seen_mcp_server_names = HashMap::::new(); + for (configured_name, plugin) in configured_plugins { + let loaded_plugin = load_plugin( + configured_name.clone(), + &plugin, + store, + restriction_product, + &skill_config_rules, + ) + .await; + for name in loaded_plugin.mcp_servers.keys() { + if let Some(previous_plugin) = + seen_mcp_server_names.insert(name.clone(), configured_name.clone()) + { + warn!( + plugin = configured_name, + previous_plugin, + server = name, + "skipping duplicate plugin MCP server name" + ); + } + } + plugins.push(loaded_plugin); + } + + PluginLoadOutcome::from_plugins(plugins) +} + +pub fn refresh_curated_plugin_cache( + codex_home: &Path, + plugin_version: &str, + configured_curated_plugin_ids: &[PluginId], +) -> Result { + let store = PluginStore::new(codex_home.to_path_buf()); + let curated_marketplace_path = AbsolutePathBuf::try_from( + codex_home + .join(".tmp/plugins") + .join(".agents/plugins/marketplace.json"), + ) + .map_err(|_| "local curated marketplace is not available".to_string())?; + let curated_marketplace = load_marketplace(&curated_marketplace_path) + .map_err(|err| format!("failed to load curated marketplace for cache refresh: {err}"))?; + + let mut plugin_sources = HashMap::::new(); + for plugin in curated_marketplace.plugins { + let plugin_name = plugin.name; + if plugin_sources.contains_key(&plugin_name) { + warn!( + plugin = plugin_name, + marketplace = OPENAI_CURATED_MARKETPLACE_NAME, + "ignoring duplicate curated plugin entry during cache refresh" + ); + continue; + } + let source_path = match plugin.source { + MarketplacePluginSource::Local { path } => path, + }; + plugin_sources.insert(plugin_name, source_path); + } + + let mut cache_refreshed = false; + for plugin_id in configured_curated_plugin_ids { + if store.active_plugin_version(plugin_id).as_deref() == Some(plugin_version) { + continue; + } + + let Some(source_path) = plugin_sources.get(&plugin_id.plugin_name).cloned() else { + warn!( + plugin = plugin_id.plugin_name, + marketplace = OPENAI_CURATED_MARKETPLACE_NAME, + "configured curated plugin no longer exists in curated marketplace during cache refresh" + ); + continue; + }; + + store + .install_with_version(source_path, plugin_id.clone(), plugin_version.to_string()) + .map_err(|err| { + format!( + "failed to refresh curated plugin cache for {}: {err}", + plugin_id.as_key() + ) + })?; + cache_refreshed = true; + } + + Ok(cache_refreshed) +} + +pub fn refresh_non_curated_plugin_cache( + codex_home: &Path, + additional_roots: &[AbsolutePathBuf], +) -> Result { + refresh_non_curated_plugin_cache_with_mode( + codex_home, + additional_roots, + NonCuratedCacheRefreshMode::IfVersionChanged, + ) +} + +pub fn refresh_non_curated_plugin_cache_force_reinstall( + codex_home: &Path, + additional_roots: &[AbsolutePathBuf], +) -> Result { + refresh_non_curated_plugin_cache_with_mode( + codex_home, + additional_roots, + NonCuratedCacheRefreshMode::ForceReinstall, + ) +} + +fn refresh_non_curated_plugin_cache_with_mode( + codex_home: &Path, + additional_roots: &[AbsolutePathBuf], + mode: NonCuratedCacheRefreshMode, +) -> Result { + let configured_non_curated_plugin_ids = + non_curated_plugin_ids_from_config_keys(configured_plugins_from_codex_home( + codex_home, + "failed to read user config while refreshing non-curated plugin cache", + "failed to parse user config while refreshing non-curated plugin cache", + )); + if configured_non_curated_plugin_ids.is_empty() { + return Ok(false); + } + let configured_non_curated_plugin_keys = configured_non_curated_plugin_ids + .iter() + .map(PluginId::as_key) + .collect::>(); + + let store = PluginStore::new(codex_home.to_path_buf()); + let marketplace_outcome = list_marketplaces(additional_roots) + .map_err(|err| format!("failed to discover marketplaces for cache refresh: {err}"))?; + let mut plugin_sources = HashMap::::new(); + + for marketplace in marketplace_outcome.marketplaces { + if marketplace.name == OPENAI_CURATED_MARKETPLACE_NAME { + continue; + } + + for plugin in marketplace.plugins { + let plugin_id = + PluginId::new(plugin.name.clone(), marketplace.name.clone()).map_err(|err| { + match err { + PluginIdError::Invalid(message) => { + format!("failed to prepare non-curated plugin cache refresh: {message}") + } + } + })?; + let plugin_key = plugin_id.as_key(); + if !configured_non_curated_plugin_keys.contains(&plugin_key) { + continue; + } + if plugin_sources.contains_key(&plugin_key) { + warn!( + plugin = plugin.name, + marketplace = marketplace.name, + "ignoring duplicate non-curated plugin entry during cache refresh" + ); + continue; + } + + let source_path = match plugin.source { + MarketplacePluginSource::Local { path } => path, + }; + let plugin_version = plugin_version_for_source(source_path.as_path()) + .map_err(|err| format!("failed to read plugin version for {plugin_key}: {err}"))?; + plugin_sources.insert(plugin_key, (source_path, plugin_version)); + } + } + + let mut cache_refreshed = false; + for plugin_id in configured_non_curated_plugin_ids { + let plugin_key = plugin_id.as_key(); + let Some((source_path, plugin_version)) = plugin_sources.get(&plugin_key).cloned() else { + warn!( + plugin = plugin_id.plugin_name, + marketplace = plugin_id.marketplace_name, + "configured non-curated plugin no longer exists in discovered marketplaces during cache refresh" + ); + continue; + }; + + if mode == NonCuratedCacheRefreshMode::IfVersionChanged + && store.active_plugin_version(&plugin_id).as_deref() == Some(plugin_version.as_str()) + { + continue; + } + + store + .install_with_version(source_path, plugin_id.clone(), plugin_version) + .map_err(|err| format!("failed to refresh plugin cache for {plugin_key}: {err}"))?; + cache_refreshed = true; + } + + Ok(cache_refreshed) +} + +fn configured_plugins_from_stack( + config_layer_stack: &ConfigLayerStack, +) -> HashMap { + let Some(user_layer) = config_layer_stack.get_user_layer() else { + return HashMap::new(); + }; + configured_plugins_from_user_config_value(&user_layer.config) +} + +fn configured_plugins_from_user_config_value( + user_config: &toml::Value, +) -> HashMap { + let Some(plugins_value) = user_config.get("plugins") else { + return HashMap::new(); + }; + match plugins_value.clone().try_into() { + Ok(plugins) => plugins, + Err(err) => { + warn!("invalid plugins config: {err}"); + HashMap::new() + } + } +} + +fn configured_plugins_from_codex_home( + codex_home: &Path, + read_error_message: &str, + parse_error_message: &str, +) -> HashMap { + let config_path = codex_home.join(CONFIG_TOML_FILE); + let user_config = match fs::read_to_string(&config_path) { + Ok(user_config) => user_config, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return HashMap::new(), + Err(err) => { + warn!( + path = %config_path.display(), + error = %err, + "{read_error_message}" + ); + return HashMap::new(); + } + }; + + let user_config = match toml::from_str::(&user_config) { + Ok(user_config) => user_config, + Err(err) => { + warn!( + path = %config_path.display(), + error = %err, + "{parse_error_message}" + ); + return HashMap::new(); + } + }; + + configured_plugins_from_user_config_value(&user_config) +} + +fn configured_plugin_ids( + configured_plugins: HashMap, + invalid_plugin_key_message: &str, +) -> Vec { + configured_plugins + .into_keys() + .filter_map(|plugin_key| match PluginId::parse(&plugin_key) { + Ok(plugin_id) => Some(plugin_id), + Err(err) => { + warn!( + plugin_key, + error = %err, + "{invalid_plugin_key_message}" + ); + None + } + }) + .collect() +} + +fn curated_plugin_ids_from_config_keys( + configured_plugins: HashMap, +) -> Vec { + let mut configured_curated_plugin_ids = configured_plugin_ids( + configured_plugins, + "ignoring invalid configured plugin key during curated sync setup", + ) + .into_iter() + .filter(|plugin_id| plugin_id.marketplace_name == OPENAI_CURATED_MARKETPLACE_NAME) + .collect::>(); + configured_curated_plugin_ids.sort_unstable_by_key(PluginId::as_key); + configured_curated_plugin_ids +} + +fn non_curated_plugin_ids_from_config_keys( + configured_plugins: HashMap, +) -> Vec { + let mut configured_non_curated_plugin_ids = configured_plugin_ids( + configured_plugins, + "ignoring invalid plugin key during non-curated cache refresh setup", + ) + .into_iter() + .filter(|plugin_id| plugin_id.marketplace_name != OPENAI_CURATED_MARKETPLACE_NAME) + .collect::>(); + configured_non_curated_plugin_ids.sort_unstable_by_key(PluginId::as_key); + configured_non_curated_plugin_ids +} + +pub fn configured_curated_plugin_ids_from_codex_home(codex_home: &Path) -> Vec { + curated_plugin_ids_from_config_keys(configured_plugins_from_codex_home( + codex_home, + "failed to read user config while refreshing curated plugin cache", + "failed to parse user config while refreshing curated plugin cache", + )) +} + +async fn load_plugin( + config_name: String, + plugin: &PluginConfig, + store: &PluginStore, + restriction_product: Option, + skill_config_rules: &SkillConfigRules, +) -> LoadedPlugin { + let plugin_id = PluginId::parse(&config_name); + let active_plugin_root = plugin_id + .as_ref() + .ok() + .and_then(|plugin_id| store.active_plugin_root(plugin_id)); + let root = active_plugin_root + .clone() + .unwrap_or_else(|| match &plugin_id { + Ok(plugin_id) => store.plugin_base_root(plugin_id), + Err(_) => store.root().clone(), + }); + let mut loaded_plugin = LoadedPlugin { + config_name, + manifest_name: None, + manifest_description: None, + root, + enabled: plugin.enabled, + skill_roots: Vec::new(), + disabled_skill_paths: HashSet::new(), + has_enabled_skills: false, + mcp_servers: HashMap::new(), + apps: Vec::new(), + error: None, + }; + + if !plugin.enabled { + return loaded_plugin; + } + + let plugin_root = match plugin_id { + Ok(_) => match active_plugin_root { + Some(plugin_root) => plugin_root, + None => { + loaded_plugin.error = Some("plugin is not installed".to_string()); + return loaded_plugin; + } + }, + Err(err) => { + loaded_plugin.error = Some(err.to_string()); + return loaded_plugin; + } + }; + + if !plugin_root.as_path().is_dir() { + loaded_plugin.error = Some("path does not exist or is not a directory".to_string()); + return loaded_plugin; + } + + let Some(manifest) = load_plugin_manifest(plugin_root.as_path()) else { + loaded_plugin.error = Some("missing or invalid .codex-plugin/plugin.json".to_string()); + return loaded_plugin; + }; + + let manifest_paths = &manifest.paths; + loaded_plugin.manifest_name = manifest + .interface + .as_ref() + .and_then(|interface| interface.display_name.as_deref()) + .map(str::trim) + .filter(|display_name| !display_name.is_empty()) + .map(str::to_string) + .or_else(|| Some(manifest.name.clone())); + loaded_plugin.manifest_description = manifest.description.clone(); + loaded_plugin.skill_roots = plugin_skill_roots(&plugin_root, manifest_paths); + let resolved_skills = load_plugin_skills( + &plugin_root, + manifest_paths, + restriction_product, + skill_config_rules, + ) + .await; + let has_enabled_skills = resolved_skills.has_enabled_skills(); + loaded_plugin.disabled_skill_paths = resolved_skills.disabled_skill_paths; + loaded_plugin.has_enabled_skills = has_enabled_skills; + let mut mcp_servers = HashMap::new(); + for mcp_config_path in plugin_mcp_config_paths(plugin_root.as_path(), manifest_paths) { + let plugin_mcp = load_mcp_servers_from_file(plugin_root.as_path(), &mcp_config_path).await; + for (name, config) in plugin_mcp.mcp_servers { + if mcp_servers.insert(name.clone(), config).is_some() { + warn!( + plugin = %plugin_root.display(), + path = %mcp_config_path.display(), + server = name, + "plugin MCP file overwrote an earlier server definition" + ); + } + } + } + loaded_plugin.mcp_servers = mcp_servers; + loaded_plugin.apps = load_plugin_apps(plugin_root.as_path()).await; + loaded_plugin +} + +#[derive(Debug, Clone)] +pub struct ResolvedPluginSkills { + pub skills: Vec, + pub disabled_skill_paths: HashSet, + pub had_errors: bool, +} + +impl ResolvedPluginSkills { + pub fn has_enabled_skills(&self) -> bool { + self.had_errors + || self + .skills + .iter() + .any(|skill| !self.disabled_skill_paths.contains(&skill.path_to_skills_md)) + } +} + +pub async fn load_plugin_skills( + plugin_root: &AbsolutePathBuf, + manifest_paths: &PluginManifestPaths, + restriction_product: Option, + skill_config_rules: &SkillConfigRules, +) -> ResolvedPluginSkills { + let roots = plugin_skill_roots(plugin_root, manifest_paths) + .into_iter() + .map(|path| SkillRoot { + path, + scope: SkillScope::User, + file_system: Arc::clone(&LOCAL_FS), + }) + .collect::>(); + let outcome = load_skills_from_roots(roots).await; + let had_errors = !outcome.errors.is_empty(); + let skills = outcome + .skills + .into_iter() + .filter(|skill| skill.matches_product_restriction_for_product(restriction_product)) + .collect::>(); + let disabled_skill_paths = resolve_disabled_skill_paths(&skills, skill_config_rules); + + ResolvedPluginSkills { + skills, + disabled_skill_paths, + had_errors, + } +} + +fn plugin_skill_roots( + plugin_root: &AbsolutePathBuf, + manifest_paths: &PluginManifestPaths, +) -> Vec { + let mut paths = default_skill_roots(plugin_root); + if let Some(path) = &manifest_paths.skills { + paths.push(path.clone()); + } + paths.sort_unstable(); + paths.dedup(); + paths +} + +fn default_skill_roots(plugin_root: &AbsolutePathBuf) -> Vec { + let skills_dir = plugin_root.join(DEFAULT_SKILLS_DIR_NAME); + if skills_dir.is_dir() { + vec![skills_dir] + } else { + Vec::new() + } +} + +fn plugin_mcp_config_paths( + plugin_root: &Path, + manifest_paths: &PluginManifestPaths, +) -> Vec { + if let Some(path) = &manifest_paths.mcp_servers { + return vec![path.clone()]; + } + default_mcp_config_paths(plugin_root) +} + +fn default_mcp_config_paths(plugin_root: &Path) -> Vec { + let mut paths = Vec::new(); + let default_path = plugin_root.join(DEFAULT_MCP_CONFIG_FILE); + if default_path.is_file() + && let Ok(default_path) = AbsolutePathBuf::try_from(default_path) + { + paths.push(default_path); + } + paths.sort_unstable_by(|left, right| left.as_path().cmp(right.as_path())); + paths.dedup_by(|left, right| left.as_path() == right.as_path()); + paths +} + +pub async fn load_plugin_apps(plugin_root: &Path) -> Vec { + if let Some(manifest) = load_plugin_manifest(plugin_root) { + return load_apps_from_paths( + plugin_root, + plugin_app_config_paths(plugin_root, &manifest.paths), + ) + .await; + } + load_apps_from_paths(plugin_root, default_app_config_paths(plugin_root)).await +} + +fn plugin_app_config_paths( + plugin_root: &Path, + manifest_paths: &PluginManifestPaths, +) -> Vec { + if let Some(path) = &manifest_paths.apps { + return vec![path.clone()]; + } + default_app_config_paths(plugin_root) +} + +fn default_app_config_paths(plugin_root: &Path) -> Vec { + let mut paths = Vec::new(); + let default_path = plugin_root.join(DEFAULT_APP_CONFIG_FILE); + if default_path.is_file() + && let Ok(default_path) = AbsolutePathBuf::try_from(default_path) + { + paths.push(default_path); + } + paths.sort_unstable_by(|left, right| left.as_path().cmp(right.as_path())); + paths.dedup_by(|left, right| left.as_path() == right.as_path()); + paths +} + +async fn load_apps_from_paths( + plugin_root: &Path, + app_config_paths: Vec, +) -> Vec { + let mut connector_ids = Vec::new(); + for app_config_path in app_config_paths { + let Ok(contents) = tokio::fs::read_to_string(app_config_path.as_path()).await else { + continue; + }; + let parsed = match serde_json::from_str::(&contents) { + Ok(parsed) => parsed, + Err(err) => { + warn!( + path = %app_config_path.display(), + "failed to parse plugin app config: {err}" + ); + continue; + } + }; + + let mut apps: Vec = parsed.apps.into_values().collect(); + apps.sort_unstable_by(|left, right| left.id.cmp(&right.id)); + + connector_ids.extend(apps.into_iter().filter_map(|app| { + if app.id.trim().is_empty() { + warn!( + plugin = %plugin_root.display(), + "plugin app config is missing an app id" + ); + None + } else { + Some(AppConnectorId(app.id)) + } + })); + } + connector_ids.dedup(); + connector_ids +} + +pub async fn plugin_telemetry_metadata_from_root( + plugin_id: &PluginId, + plugin_root: &AbsolutePathBuf, +) -> PluginTelemetryMetadata { + let Some(manifest) = load_plugin_manifest(plugin_root.as_path()) else { + return PluginTelemetryMetadata::from_plugin_id(plugin_id); + }; + + let manifest_paths = &manifest.paths; + let has_skills = !plugin_skill_roots(plugin_root, manifest_paths).is_empty(); + let mut mcp_server_names = Vec::new(); + for path in plugin_mcp_config_paths(plugin_root.as_path(), manifest_paths) { + mcp_server_names.extend( + load_mcp_servers_from_file(plugin_root.as_path(), &path) + .await + .mcp_servers + .into_keys(), + ); + } + mcp_server_names.sort_unstable(); + mcp_server_names.dedup(); + + PluginTelemetryMetadata { + plugin_id: plugin_id.clone(), + capability_summary: Some(PluginCapabilitySummary { + config_name: plugin_id.as_key(), + display_name: plugin_id.plugin_name.clone(), + description: None, + has_skills, + mcp_server_names, + app_connector_ids: load_apps_from_paths( + plugin_root.as_path(), + plugin_app_config_paths(plugin_root.as_path(), manifest_paths), + ) + .await, + }), + } +} + +pub async fn load_plugin_mcp_servers(plugin_root: &Path) -> HashMap { + let Some(manifest) = load_plugin_manifest(plugin_root) else { + return HashMap::new(); + }; + + let mut mcp_servers = HashMap::new(); + for mcp_config_path in plugin_mcp_config_paths(plugin_root, &manifest.paths) { + let plugin_mcp = load_mcp_servers_from_file(plugin_root, &mcp_config_path).await; + for (name, config) in plugin_mcp.mcp_servers { + mcp_servers.entry(name).or_insert(config); + } + } + + mcp_servers +} + +pub async fn installed_plugin_telemetry_metadata( + codex_home: &Path, + plugin_id: &PluginId, +) -> PluginTelemetryMetadata { + let store = PluginStore::new(codex_home.to_path_buf()); + let Some(plugin_root) = store.active_plugin_root(plugin_id) else { + return PluginTelemetryMetadata::from_plugin_id(plugin_id); + }; + + plugin_telemetry_metadata_from_root(plugin_id, &plugin_root).await +} + +async fn load_mcp_servers_from_file( + plugin_root: &Path, + mcp_config_path: &AbsolutePathBuf, +) -> PluginMcpDiscovery { + let Ok(contents) = tokio::fs::read_to_string(mcp_config_path.as_path()).await else { + return PluginMcpDiscovery::default(); + }; + let parsed = match serde_json::from_str::(&contents) { + Ok(parsed) => parsed, + Err(err) => { + warn!( + path = %mcp_config_path.display(), + "failed to parse plugin MCP config: {err}" + ); + return PluginMcpDiscovery::default(); + } + }; + normalize_plugin_mcp_servers( + plugin_root, + parsed.mcp_servers, + mcp_config_path.to_string_lossy().as_ref(), + ) +} + +fn normalize_plugin_mcp_servers( + plugin_root: &Path, + plugin_mcp_servers: HashMap, + source: &str, +) -> PluginMcpDiscovery { + let mut mcp_servers = HashMap::new(); + + for (name, config_value) in plugin_mcp_servers { + let normalized = normalize_plugin_mcp_server_value(plugin_root, config_value); + match serde_json::from_value::(JsonValue::Object(normalized)) { + Ok(config) => { + mcp_servers.insert(name, config); + } + Err(err) => { + warn!( + plugin = %plugin_root.display(), + server = name, + "failed to parse plugin MCP server from {source}: {err}" + ); + } + } + } + + PluginMcpDiscovery { mcp_servers } +} + +fn normalize_plugin_mcp_server_value( + plugin_root: &Path, + value: JsonValue, +) -> JsonMap { + let mut object = match value { + JsonValue::Object(object) => object, + _ => return JsonMap::new(), + }; + + if let Some(JsonValue::String(transport_type)) = object.remove("type") { + match transport_type.as_str() { + "http" | "streamable_http" | "streamable-http" => {} + "stdio" => {} + other => { + warn!( + plugin = %plugin_root.display(), + transport = other, + "plugin MCP server uses an unknown transport type" + ); + } + } + } + + if let Some(JsonValue::Object(oauth)) = object.remove("oauth") + && oauth.contains_key("callbackPort") + { + warn!( + plugin = %plugin_root.display(), + "plugin MCP server OAuth callbackPort is ignored; Codex uses global MCP OAuth callback settings" + ); + } + + if let Some(JsonValue::String(cwd)) = object.get("cwd") + && !Path::new(cwd).is_absolute() + { + object.insert( + "cwd".to_string(), + JsonValue::String(plugin_root.join(cwd).display().to_string()), + ); + } + + object +} + +#[derive(Debug, Default)] +struct PluginMcpDiscovery { + mcp_servers: HashMap, +} diff --git a/codex-rs/core/src/plugins/manifest.rs b/codex-rs/core-plugins/src/manifest.rs similarity index 98% rename from codex-rs/core/src/plugins/manifest.rs rename to codex-rs/core-plugins/src/manifest.rs index f1253441bd..f58b882ca1 100644 --- a/codex-rs/core/src/plugins/manifest.rs +++ b/codex-rs/core-plugins/src/manifest.rs @@ -30,12 +30,12 @@ struct RawPluginManifest { } #[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct PluginManifest { - pub(crate) name: String, - pub(crate) version: Option, - pub(crate) description: Option, - pub(crate) paths: PluginManifestPaths, - pub(crate) interface: Option, +pub struct PluginManifest { + pub name: String, + pub version: Option, + pub description: Option, + pub paths: PluginManifestPaths, + pub interface: Option, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -114,7 +114,7 @@ enum RawPluginManifestDefaultPromptEntry { Invalid(JsonValue), } -pub(crate) fn load_plugin_manifest(plugin_root: &Path) -> Option { +pub fn load_plugin_manifest(plugin_root: &Path) -> Option { let manifest_path = plugin_root.join(PLUGIN_MANIFEST_PATH); if !manifest_path.is_file() { return None; diff --git a/codex-rs/core/src/plugins/marketplace.rs b/codex-rs/core-plugins/src/marketplace.rs similarity index 61% rename from codex-rs/core/src/plugins/marketplace.rs rename to codex-rs/core-plugins/src/marketplace.rs index 3cb1756f80..79e5237ad0 100644 --- a/codex-rs/core/src/plugins/marketplace.rs +++ b/codex-rs/core-plugins/src/marketplace.rs @@ -1,5 +1,5 @@ -use super::PluginManifestInterface; -use super::load_plugin_manifest; +use crate::manifest::PluginManifestInterface; +use crate::manifest::load_plugin_manifest; use codex_app_server_protocol::PluginAuthPolicy; use codex_app_server_protocol::PluginInstallPolicy; use codex_git_utils::get_git_repo_root; @@ -9,6 +9,8 @@ use codex_protocol::protocol::Product; use codex_utils_absolute_path::AbsolutePathBuf; use dirs::home_dir; use serde::Deserialize; +use serde::Deserializer; +use serde_json::Value as JsonValue; use std::fs; use std::io; use std::path::Component; @@ -16,7 +18,10 @@ use std::path::Path; use std::path::PathBuf; use tracing::warn; -const MARKETPLACE_RELATIVE_PATH: &str = ".agents/plugins/marketplace.json"; +const MARKETPLACE_MANIFEST_RELATIVE_PATHS: &[&str] = &[ + ".agents/plugins/marketplace.json", + ".claude-plugin/marketplace.json", +]; #[derive(Debug, Clone, PartialEq, Eq)] pub struct ResolvedMarketplacePlugin { @@ -162,46 +167,50 @@ pub fn resolve_marketplace_plugin( ) -> Result { let marketplace = load_raw_marketplace_manifest(marketplace_path)?; let marketplace_name = marketplace.name; - let plugin = marketplace - .plugins - .into_iter() - .find(|plugin| plugin.name == plugin_name); - - let Some(plugin) = plugin else { - return Err(MarketplaceError::PluginNotFound { - plugin_name: plugin_name.to_string(), - marketplace_name, - }); - }; - - let RawMarketplaceManifestPlugin { - name, - source, - policy, - .. - } = plugin; - let install_policy = policy.installation; - let product_allowed = match policy.products.as_deref() { - None => true, - Some([]) => false, - Some(products) => { - restriction_product.is_some_and(|product| product.matches_product_restriction(products)) + let marketplace_name_for_not_found = marketplace_name.clone(); + for plugin in marketplace.plugins { + if plugin.name != plugin_name { + continue; } - }; - if install_policy == MarketplacePluginInstallPolicy::NotAvailable || !product_allowed { - return Err(MarketplaceError::PluginNotAvailable { - plugin_name: name, - marketplace_name, + + let RawMarketplaceManifestPlugin { + name, + source, + policy, + .. + } = plugin; + let install_policy = policy.installation; + let product_allowed = match policy.products.as_deref() { + None => true, + Some([]) => false, + Some(products) => restriction_product + .is_some_and(|product| product.matches_product_restriction(products)), + }; + if install_policy == MarketplacePluginInstallPolicy::NotAvailable || !product_allowed { + return Err(MarketplaceError::PluginNotAvailable { + plugin_name: name, + marketplace_name, + }); + } + + let Some(source_path) = + resolve_supported_plugin_source_path(marketplace_path, &name, source) + else { + continue; + }; + + return Ok(ResolvedMarketplacePlugin { + plugin_id: PluginId::new(name, marketplace_name).map_err(|err| match err { + PluginIdError::Invalid(message) => MarketplaceError::InvalidPlugin(message), + })?, + source_path, + auth_policy: policy.authentication, }); } - let plugin_id = PluginId::new(name, marketplace_name).map_err(|err| match err { - PluginIdError::Invalid(message) => MarketplaceError::InvalidPlugin(message), - })?; - Ok(ResolvedMarketplacePlugin { - plugin_id, - source_path: resolve_plugin_source_path(marketplace_path, source)?, - auth_policy: policy.authentication, + Err(MarketplaceError::PluginNotFound { + plugin_name: plugin_name.to_string(), + marketplace_name: marketplace_name_for_not_found, }) } @@ -212,17 +221,51 @@ pub fn list_marketplaces( } pub fn validate_marketplace_root(root: &Path) -> Result { - let path = AbsolutePathBuf::try_from(root.join(MARKETPLACE_RELATIVE_PATH)).map_err(|err| { - MarketplaceError::InvalidMarketplaceFile { - path: root.join(MARKETPLACE_RELATIVE_PATH), - message: format!("marketplace path must resolve to an absolute path: {err}"), - } - })?; + let Some(path) = find_marketplace_manifest_path(root) else { + return Err(MarketplaceError::InvalidMarketplaceFile { + path: root.to_path_buf(), + message: "marketplace root does not contain a supported manifest".to_string(), + }); + }; let marketplace = load_marketplace(&path)?; Ok(marketplace.name) } -pub(crate) fn load_marketplace(path: &AbsolutePathBuf) -> Result { +pub fn find_marketplace_manifest_path(root: &Path) -> Option { + MARKETPLACE_MANIFEST_RELATIVE_PATHS + .iter() + .find_map(|relative_path| { + let path = root.join(relative_path); + if !path.is_file() { + return None; + } + AbsolutePathBuf::try_from(path).ok() + }) +} + +fn invalid_marketplace_layout_error(path: &AbsolutePathBuf) -> MarketplaceError { + MarketplaceError::InvalidMarketplaceFile { + path: path.to_path_buf(), + message: "marketplace file is not in a supported location".to_string(), + } +} + +fn marketplace_root_from_layout(marketplace_path: &Path, relative_path: &str) -> Option { + let mut current = marketplace_path; + for component in Path::new(relative_path).components().rev() { + let expected = match component { + Component::Normal(expected) => expected, + _ => return None, + }; + if current.file_name() != Some(expected) { + return None; + } + current = current.parent()?; + } + Some(current.to_path_buf()) +} + +pub fn load_marketplace(path: &AbsolutePathBuf) -> Result { let marketplace = load_raw_marketplace_manifest(path)?; let mut plugins = Vec::new(); @@ -233,7 +276,9 @@ pub(crate) fn load_marketplace(path: &AbsolutePathBuf) -> Result Result, ) -> Result { @@ -298,30 +344,27 @@ fn discover_marketplace_paths_from_roots( ) -> Vec { let mut paths = Vec::new(); - if let Some(home) = home_dir { - let path = home.join(MARKETPLACE_RELATIVE_PATH); - if path.is_file() - && let Ok(path) = AbsolutePathBuf::try_from(path) - { - paths.push(path); - } + if let Some(home) = home_dir + && let Some(path) = find_marketplace_manifest_path(home) + { + paths.push(path); } for root in additional_roots { // Curated marketplaces can now come from an HTTP-downloaded directory that is not a git // checkout, so check the root directly before falling back to repo-root discovery. - let path = root.join(MARKETPLACE_RELATIVE_PATH); - if path.as_path().is_file() && !paths.contains(&path) { + if let Some(path) = find_marketplace_manifest_path(root.as_path()) + && !paths.contains(&path) + { paths.push(path); continue; } if let Some(repo_root) = get_git_repo_root(root.as_path()) && let Ok(repo_root) = AbsolutePathBuf::try_from(repo_root) + && let Some(path) = find_marketplace_manifest_path(repo_root.as_path()) + && !paths.contains(&path) { - let path = repo_root.join(MARKETPLACE_RELATIVE_PATH); - if path.as_path().is_file() && !paths.contains(&path) { - paths.push(path); - } + paths.push(path); } } @@ -346,80 +389,83 @@ fn load_raw_marketplace_manifest( }) } -fn resolve_plugin_source_path( +fn resolve_supported_plugin_source_path( marketplace_path: &AbsolutePathBuf, + plugin_name: &str, source: RawMarketplaceManifestPluginSource, -) -> Result { +) -> Option { match source { RawMarketplaceManifestPluginSource::Local { path } => { - let Some(path) = path.strip_prefix("./") else { - return Err(MarketplaceError::InvalidMarketplaceFile { - path: marketplace_path.to_path_buf(), - message: "local plugin source path must start with `./`".to_string(), - }); - }; - if path.is_empty() { - return Err(MarketplaceError::InvalidMarketplaceFile { - path: marketplace_path.to_path_buf(), - message: "local plugin source path must not be empty".to_string(), - }); + match resolve_local_plugin_source_path(marketplace_path, &path) { + Ok(path) => Some(path), + Err(err) => { + warn!( + path = %marketplace_path.display(), + plugin = plugin_name, + error = %err, + "skipping marketplace plugin that failed to resolve" + ); + None + } } - - let relative_source_path = Path::new(path); - if relative_source_path - .components() - .any(|component| !matches!(component, Component::Normal(_))) - { - return Err(MarketplaceError::InvalidMarketplaceFile { - path: marketplace_path.to_path_buf(), - message: "local plugin source path must stay within the marketplace root" - .to_string(), - }); - } - - // `marketplace.json` lives under `/.agents/plugins/`, but local plugin paths - // are resolved relative to ``, not relative to the `plugins/` directory. - Ok(marketplace_root_dir(marketplace_path)?.join(relative_source_path)) + } + RawMarketplaceManifestPluginSource::Unsupported => { + warn!( + path = %marketplace_path.display(), + plugin = plugin_name, + "skipping marketplace plugin with unsupported source" + ); + None } } } +fn resolve_local_plugin_source_path( + marketplace_path: &AbsolutePathBuf, + source_path: &str, +) -> Result { + let Some(source_path) = source_path.strip_prefix("./") else { + return Err(MarketplaceError::InvalidMarketplaceFile { + path: marketplace_path.to_path_buf(), + message: "local plugin source path must start with `./`".to_string(), + }); + }; + if source_path.is_empty() { + return Err(MarketplaceError::InvalidMarketplaceFile { + path: marketplace_path.to_path_buf(), + message: "local plugin source path must not be empty".to_string(), + }); + } + + let relative_source_path = Path::new(source_path); + if relative_source_path + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(MarketplaceError::InvalidMarketplaceFile { + path: marketplace_path.to_path_buf(), + message: "local plugin source path must stay within the marketplace root".to_string(), + }); + } + + // `marketplace.json` lives under a supported marketplace layout beneath ``, + // but local plugin paths are resolved relative to ``. + Ok(marketplace_root_dir(marketplace_path)?.join(relative_source_path)) +} + fn marketplace_root_dir( marketplace_path: &AbsolutePathBuf, ) -> Result { - let Some(plugins_dir) = marketplace_path.parent() else { - return Err(MarketplaceError::InvalidMarketplaceFile { - path: marketplace_path.to_path_buf(), - message: "marketplace file must live under `/.agents/plugins/`".to_string(), - }); - }; - let Some(dot_agents_dir) = plugins_dir.parent() else { - return Err(MarketplaceError::InvalidMarketplaceFile { - path: marketplace_path.to_path_buf(), - message: "marketplace file must live under `/.agents/plugins/`".to_string(), - }); - }; - let Some(marketplace_root) = dot_agents_dir.parent() else { - return Err(MarketplaceError::InvalidMarketplaceFile { - path: marketplace_path.to_path_buf(), - message: "marketplace file must live under `/.agents/plugins/`".to_string(), - }); - }; - - if plugins_dir.as_path().file_name().and_then(|s| s.to_str()) != Some("plugins") - || dot_agents_dir - .as_path() - .file_name() - .and_then(|s| s.to_str()) - != Some(".agents") - { - return Err(MarketplaceError::InvalidMarketplaceFile { - path: marketplace_path.to_path_buf(), - message: "marketplace file must live under `/.agents/plugins/`".to_string(), - }); + for relative_path in MARKETPLACE_MANIFEST_RELATIVE_PATHS { + if let Some(marketplace_root) = + marketplace_root_from_layout(marketplace_path.as_path(), relative_path) + { + return AbsolutePathBuf::try_from(marketplace_root) + .map_err(|_| invalid_marketplace_layout_error(marketplace_path)); + } } - Ok(marketplace_root) + Err(invalid_marketplace_layout_error(marketplace_path)) } #[derive(Debug, Deserialize)] @@ -459,10 +505,33 @@ struct RawMarketplaceManifestPluginPolicy { products: Option>, } -#[derive(Debug, Deserialize)] -#[serde(tag = "source", rename_all = "lowercase")] +#[derive(Debug)] enum RawMarketplaceManifestPluginSource { Local { path: String }, + // Mixed-source marketplaces should still contribute the local plugins we can load. + Unsupported, +} + +impl<'de> Deserialize<'de> for RawMarketplaceManifestPluginSource { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let source = JsonValue::deserialize(deserializer)?; + Ok(match source { + JsonValue::String(path) => Self::Local { path }, + JsonValue::Object(object) => match object.get("source").and_then(JsonValue::as_str) { + Some("local") => match object.get("path").and_then(JsonValue::as_str) { + Some(path) => Self::Local { + path: path.to_string(), + }, + None => Self::Unsupported, + }, + _ => Self::Unsupported, + }, + _ => Self::Unsupported, + }) + } } fn resolve_marketplace_interface( diff --git a/codex-rs/core/src/plugins/marketplace_tests.rs b/codex-rs/core-plugins/src/marketplace_tests.rs similarity index 79% rename from codex-rs/core/src/plugins/marketplace_tests.rs rename to codex-rs/core-plugins/src/marketplace_tests.rs index 226ae3ff6e..3412eff72c 100644 --- a/codex-rs/core/src/plugins/marketplace_tests.rs +++ b/codex-rs/core-plugins/src/marketplace_tests.rs @@ -1,8 +1,18 @@ use super::*; use codex_protocol::protocol::Product; use pretty_assertions::assert_eq; +use std::path::Path; use tempfile::tempdir; +const ALTERNATE_MARKETPLACE_RELATIVE_PATH: &str = ".claude-plugin/marketplace.json"; + +fn write_alternate_marketplace(repo_root: &Path, contents: &str) -> AbsolutePathBuf { + let marketplace_path = repo_root.join(ALTERNATE_MARKETPLACE_RELATIVE_PATH); + fs::create_dir_all(marketplace_path.parent().unwrap()).unwrap(); + fs::write(&marketplace_path, contents).unwrap(); + AbsolutePathBuf::try_from(marketplace_path).unwrap() +} + #[test] fn resolve_marketplace_plugin_finds_repo_marketplace_plugin() { let tmp = tempdir().unwrap(); @@ -45,6 +55,46 @@ fn resolve_marketplace_plugin_finds_repo_marketplace_plugin() { ); } +#[test] +fn resolve_marketplace_plugin_supports_alternate_layout_and_string_local_source() { + let tmp = tempdir().unwrap(); + let repo_root = tmp.path().join("repo"); + fs::create_dir_all(repo_root.join(".git")).unwrap(); + let marketplace_path = write_alternate_marketplace( + &repo_root, + r#"{ + "name": "alternate-marketplace", + "plugins": [ + { + "name": "string-source-plugin", + "source": "./plugins/string-source-plugin" + } + ] +}"#, + ); + + let resolved = resolve_marketplace_plugin( + &marketplace_path, + "string-source-plugin", + Some(Product::Codex), + ) + .unwrap(); + + assert_eq!( + resolved, + ResolvedMarketplacePlugin { + plugin_id: PluginId::new( + "string-source-plugin".to_string(), + "alternate-marketplace".to_string() + ) + .unwrap(), + source_path: AbsolutePathBuf::try_from(repo_root.join("plugins/string-source-plugin")) + .unwrap(), + auth_policy: MarketplacePluginAuthPolicy::OnInstall, + } + ); +} + #[test] fn resolve_marketplace_plugin_reports_missing_plugin() { let tmp = tempdir().unwrap(); @@ -70,6 +120,106 @@ fn resolve_marketplace_plugin_reports_missing_plugin() { ); } +#[test] +fn list_marketplaces_supports_alternate_manifest_layout() { + let tmp = tempdir().unwrap(); + let repo_root = tmp.path().join("repo"); + + fs::create_dir_all(repo_root.join(".git")).unwrap(); + let marketplace_path = write_alternate_marketplace( + &repo_root, + r#"{ + "name": "alternate-marketplace", + "plugins": [ + { + "name": "string-source-plugin", + "source": "./plugins/string-source-plugin" + } + ] +}"#, + ); + + let marketplaces = list_marketplaces_with_home( + &[AbsolutePathBuf::try_from(repo_root.clone()).unwrap()], + /*home_dir*/ None, + ) + .unwrap() + .marketplaces; + + assert_eq!( + marketplaces, + vec![Marketplace { + name: "alternate-marketplace".to_string(), + path: marketplace_path, + interface: None, + plugins: vec![MarketplacePlugin { + name: "string-source-plugin".to_string(), + source: MarketplacePluginSource::Local { + path: AbsolutePathBuf::try_from(repo_root.join("plugins/string-source-plugin")) + .unwrap(), + }, + policy: MarketplacePluginPolicy { + installation: MarketplacePluginInstallPolicy::Available, + authentication: MarketplacePluginAuthPolicy::OnInstall, + products: None, + }, + interface: None, + }], + }] + ); +} + +#[test] +fn list_marketplaces_prefers_first_supported_manifest_layout() { + let tmp = tempdir().unwrap(); + let repo_root = tmp.path().join("repo"); + + fs::create_dir_all(repo_root.join(".git")).unwrap(); + fs::create_dir_all(repo_root.join(".agents/plugins")).unwrap(); + fs::write( + repo_root.join(".agents/plugins/marketplace.json"), + r#"{ + "name": "agents-marketplace", + "plugins": [ + { + "name": "agents-plugin", + "source": { + "source": "local", + "path": "./plugins/agents-plugin" + } + } + ] +}"#, + ) + .unwrap(); + write_alternate_marketplace( + &repo_root, + r#"{ + "name": "alternate-marketplace", + "plugins": [ + { + "name": "string-source-plugin", + "source": "./plugins/string-source-plugin" + } + ] +}"#, + ); + + let marketplaces = list_marketplaces_with_home( + &[AbsolutePathBuf::try_from(repo_root.clone()).unwrap()], + /*home_dir*/ None, + ) + .unwrap() + .marketplaces; + + assert_eq!(marketplaces.len(), 1); + assert_eq!(marketplaces[0].name, "agents-marketplace"); + assert_eq!( + marketplaces[0].path, + AbsolutePathBuf::try_from(repo_root.join(".agents/plugins/marketplace.json")).unwrap() + ); +} + #[test] fn list_marketplaces_returns_home_and_repo_marketplaces() { let tmp = tempdir().unwrap(); @@ -413,7 +563,7 @@ fn list_marketplaces_reads_marketplace_display_name() { } #[test] -fn list_marketplaces_skips_marketplaces_that_fail_to_load() { +fn list_marketplaces_skips_invalid_plugins_but_keeps_marketplace() { let tmp = tempdir().unwrap(); let valid_repo_root = tmp.path().join("valid-repo"); let invalid_repo_root = tmp.path().join("invalid-repo"); @@ -465,8 +615,10 @@ fn list_marketplaces_skips_marketplaces_that_fail_to_load() { .unwrap() .marketplaces; - assert_eq!(marketplaces.len(), 1); + assert_eq!(marketplaces.len(), 2); assert_eq!(marketplaces[0].name, "valid-marketplace"); + assert_eq!(marketplaces[1].name, "invalid-marketplace"); + assert!(marketplaces[1].plugins.is_empty()); } #[test] @@ -522,6 +674,60 @@ fn list_marketplaces_reports_marketplace_load_errors() { ); } +#[test] +fn list_marketplaces_skips_unsupported_plugin_sources_but_keeps_local_plugins() { + let tmp = tempdir().unwrap(); + let repo_root = tmp.path().join("repo"); + + fs::create_dir_all(repo_root.join(".git")).unwrap(); + write_alternate_marketplace( + &repo_root, + r#"{ + "name": "mixed-source-marketplace", + "plugins": [ + { + "name": "local-plugin", + "source": "./plugins/local-plugin" + }, + { + "name": "url-plugin", + "source": { + "source": "url", + "url": "https://github.com/example/plugin.git" + } + }, + { + "name": "git-subdir-plugin", + "source": { + "source": "git-subdir", + "url": "owner/repo", + "path": "plugins/example", + "ref": "main" + } + } + ] +}"#, + ); + + let marketplaces = list_marketplaces_with_home( + &[AbsolutePathBuf::try_from(repo_root.clone()).unwrap()], + /*home_dir*/ None, + ) + .unwrap() + .marketplaces; + + assert_eq!(marketplaces.len(), 1); + assert_eq!(marketplaces[0].name, "mixed-source-marketplace"); + assert_eq!(marketplaces[0].plugins.len(), 1); + assert_eq!(marketplaces[0].plugins[0].name, "local-plugin"); + assert_eq!( + marketplaces[0].plugins[0].source, + MarketplacePluginSource::Local { + path: AbsolutePathBuf::try_from(repo_root.join("plugins/local-plugin")).unwrap(), + } + ); +} + #[test] fn list_marketplaces_resolves_plugin_interface_paths_to_absolute() { let tmp = tempdir().unwrap(); @@ -734,7 +940,7 @@ fn list_marketplaces_ignores_plugin_interface_assets_without_dot_slash() { } #[test] -fn resolve_marketplace_plugin_rejects_non_relative_local_paths() { +fn resolve_marketplace_plugin_skips_invalid_local_paths() { let tmp = tempdir().unwrap(); let repo_root = tmp.path().join("repo"); fs::create_dir_all(repo_root.join(".git")).unwrap(); @@ -756,17 +962,46 @@ fn resolve_marketplace_plugin_rejects_non_relative_local_paths() { ) .unwrap(); - let marketplace_path = - AbsolutePathBuf::try_from(repo_root.join(".agents/plugins/marketplace.json")).unwrap(); - let err = resolve_marketplace_plugin(&marketplace_path, "local-plugin", Some(Product::Codex)) + let err = resolve_marketplace_plugin( + &AbsolutePathBuf::try_from(repo_root.join(".agents/plugins/marketplace.json")).unwrap(), + "local-plugin", + Some(Product::Codex), + ) + .unwrap_err(); + + assert_eq!( + err.to_string(), + "plugin `local-plugin` was not found in marketplace `codex-curated`" + ); +} + +#[test] +fn resolve_marketplace_plugin_skips_unsupported_sources() { + let tmp = tempdir().unwrap(); + let repo_root = tmp.path().join("repo"); + fs::create_dir_all(repo_root.join(".git")).unwrap(); + let marketplace_path = write_alternate_marketplace( + &repo_root, + r#"{ + "name": "alternate-marketplace", + "plugins": [ + { + "name": "remote-plugin", + "source": { + "source": "url", + "url": "https://github.com/example/plugin.git" + } + } + ] +}"#, + ); + + let err = resolve_marketplace_plugin(&marketplace_path, "remote-plugin", Some(Product::Codex)) .unwrap_err(); assert_eq!( err.to_string(), - format!( - "invalid marketplace file `{}`: local plugin source path must start with `./`", - marketplace_path.display() - ) + "plugin `remote-plugin` was not found in marketplace `alternate-marketplace`" ); } diff --git a/codex-rs/core-plugins/src/marketplace_upgrade.rs b/codex-rs/core-plugins/src/marketplace_upgrade.rs new file mode 100644 index 0000000000..81474cf66f --- /dev/null +++ b/codex-rs/core-plugins/src/marketplace_upgrade.rs @@ -0,0 +1,298 @@ +mod activation; +mod git; + +use self::activation::activate_marketplace_root; +use self::activation::installed_marketplace_metadata_matches; +use self::activation::write_installed_marketplace_metadata; +use self::git::clone_git_source; +use self::git::git_remote_revision; +use crate::marketplace::validate_marketplace_root; +use codex_config::CONFIG_TOML_FILE; +use codex_config::ConfigLayerStack; +use codex_config::MarketplaceConfigUpdate; +use codex_config::record_user_marketplace; +use codex_config::types::MarketplaceConfig; +use codex_config::types::MarketplaceSourceType; +use codex_plugin::validate_plugin_segment; +use codex_utils_absolute_path::AbsolutePathBuf; +use std::collections::HashMap; +use std::path::Path; +use std::path::PathBuf; +use std::time::Duration; +use tracing::warn; + +const INSTALLED_MARKETPLACES_DIR: &str = ".tmp/marketplaces"; +const MARKETPLACE_UPGRADE_GIT_TIMEOUT: Duration = Duration::from_secs(30); + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ConfiguredMarketplaceUpgradeError { + pub marketplace_name: String, + pub message: String, +} + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct ConfiguredMarketplaceUpgradeOutcome { + pub selected_marketplaces: Vec, + pub upgraded_roots: Vec, + pub errors: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ConfiguredGitMarketplace { + name: String, + source: String, + ref_name: Option, + sparse_paths: Vec, + last_revision: Option, +} + +impl ConfiguredMarketplaceUpgradeOutcome { + pub fn all_succeeded(&self) -> bool { + self.errors.is_empty() + } +} + +pub fn configured_git_marketplace_names(config_layer_stack: &ConfigLayerStack) -> Vec { + let mut names = configured_git_marketplaces(config_layer_stack) + .into_iter() + .map(|marketplace| marketplace.name) + .collect::>(); + names.sort_unstable(); + names +} + +pub fn upgrade_configured_git_marketplaces( + codex_home: &Path, + config_layer_stack: &ConfigLayerStack, + marketplace_name: Option<&str>, +) -> ConfiguredMarketplaceUpgradeOutcome { + let marketplaces = configured_git_marketplaces(config_layer_stack) + .into_iter() + .filter(|marketplace| marketplace_name.is_none_or(|name| marketplace.name.as_str() == name)) + .collect::>(); + if marketplaces.is_empty() { + return ConfiguredMarketplaceUpgradeOutcome::default(); + } + + let install_root = marketplace_install_root(codex_home); + let selected_marketplaces = marketplaces + .iter() + .map(|marketplace| marketplace.name.clone()) + .collect(); + let mut upgraded_roots = Vec::new(); + let mut errors = Vec::new(); + for marketplace in marketplaces { + match upgrade_configured_git_marketplace(codex_home, &install_root, &marketplace) { + Ok(Some(upgraded_root)) => upgraded_roots.push(upgraded_root), + Ok(None) => {} + Err(err) => { + errors.push(ConfiguredMarketplaceUpgradeError { + marketplace_name: marketplace.name, + message: err, + }); + } + } + } + + ConfiguredMarketplaceUpgradeOutcome { + selected_marketplaces, + upgraded_roots, + errors, + } +} + +fn marketplace_install_root(codex_home: &Path) -> PathBuf { + codex_home.join(INSTALLED_MARKETPLACES_DIR) +} + +fn configured_git_marketplaces( + config_layer_stack: &ConfigLayerStack, +) -> Vec { + let Some(user_layer) = config_layer_stack.get_user_layer() else { + return Vec::new(); + }; + let Some(marketplaces_value) = user_layer.config.get("marketplaces") else { + return Vec::new(); + }; + let marketplaces = match marketplaces_value + .clone() + .try_into::>() + { + Ok(marketplaces) => marketplaces, + Err(err) => { + warn!("invalid marketplaces config while preparing auto-upgrade: {err}"); + return Vec::new(); + } + }; + + let mut configured = marketplaces + .into_iter() + .filter_map(|(name, marketplace)| configured_git_marketplace_from_config(name, marketplace)) + .collect::>(); + configured.sort_unstable_by(|left, right| left.name.cmp(&right.name)); + configured +} + +fn configured_git_marketplace_from_config( + name: String, + marketplace: MarketplaceConfig, +) -> Option { + let MarketplaceConfig { + last_updated: _, + last_revision, + source_type, + source, + ref_name, + sparse_paths, + } = marketplace; + if source_type != Some(MarketplaceSourceType::Git) { + return None; + } + let Some(source) = source else { + warn!( + marketplace = name, + "ignoring configured Git marketplace without source" + ); + return None; + }; + Some(ConfiguredGitMarketplace { + name, + source, + ref_name, + sparse_paths: sparse_paths.unwrap_or_default(), + last_revision, + }) +} + +fn upgrade_configured_git_marketplace( + codex_home: &Path, + install_root: &Path, + marketplace: &ConfiguredGitMarketplace, +) -> Result, String> { + validate_plugin_segment(&marketplace.name, "marketplace name")?; + let remote_revision = git_remote_revision( + &marketplace.source, + marketplace.ref_name.as_deref(), + MARKETPLACE_UPGRADE_GIT_TIMEOUT, + )?; + let destination = install_root.join(&marketplace.name); + if destination + .join(".agents/plugins/marketplace.json") + .is_file() + && marketplace.last_revision.as_deref() == Some(remote_revision.as_str()) + && installed_marketplace_metadata_matches(&destination, marketplace, &remote_revision) + { + return Ok(None); + } + + let staging_parent = install_root.join(".staging"); + std::fs::create_dir_all(&staging_parent).map_err(|err| { + format!( + "failed to create marketplace upgrade staging directory {}: {err}", + staging_parent.display() + ) + })?; + let staged_dir = tempfile::Builder::new() + .prefix("marketplace-upgrade-") + .tempdir_in(&staging_parent) + .map_err(|err| { + format!( + "failed to create temporary marketplace upgrade directory in {}: {err}", + staging_parent.display() + ) + })?; + + let activated_revision = clone_git_source( + &marketplace.source, + marketplace.ref_name.as_deref(), + &marketplace.sparse_paths, + staged_dir.path(), + MARKETPLACE_UPGRADE_GIT_TIMEOUT, + )?; + let marketplace_name = validate_marketplace_root(staged_dir.path()) + .map_err(|err| format!("failed to validate upgraded marketplace root: {err}"))?; + if marketplace_name != marketplace.name { + return Err(format!( + "upgraded marketplace name `{marketplace_name}` does not match configured marketplace `{}`", + marketplace.name + )); + } + write_installed_marketplace_metadata(staged_dir.path(), marketplace, &activated_revision)?; + + let last_updated = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true); + let update = MarketplaceConfigUpdate { + last_updated: &last_updated, + last_revision: Some(&activated_revision), + source_type: "git", + source: &marketplace.source, + ref_name: marketplace.ref_name.as_deref(), + sparse_paths: &marketplace.sparse_paths, + }; + activate_marketplace_root(&destination, staged_dir, || { + ensure_configured_git_marketplace_unchanged(codex_home, marketplace)?; + record_user_marketplace(codex_home, &marketplace.name, &update).map_err(|err| { + format!( + "failed to record upgraded marketplace `{}` in user config.toml: {err}", + marketplace.name + ) + }) + })?; + + AbsolutePathBuf::try_from(destination) + .map(Some) + .map_err(|err| format!("upgraded marketplace path is not absolute: {err}")) +} +fn ensure_configured_git_marketplace_unchanged( + codex_home: &Path, + expected: &ConfiguredGitMarketplace, +) -> Result<(), String> { + let current = read_configured_git_marketplace(codex_home, &expected.name)?; + match current { + Some(current) if current == *expected => Ok(()), + Some(_) => Err(format!( + "configured marketplace `{}` changed while auto-upgrade was in flight", + expected.name + )), + None => Err(format!( + "configured marketplace `{}` was removed or is no longer a Git marketplace", + expected.name + )), + } +} + +fn read_configured_git_marketplace( + codex_home: &Path, + marketplace_name: &str, +) -> Result, String> { + let config_path = codex_home.join(CONFIG_TOML_FILE); + let raw_config = match std::fs::read_to_string(&config_path) { + Ok(raw_config) => raw_config, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(err) => { + return Err(format!( + "failed to read user config {} while checking marketplace auto-upgrade: {err}", + config_path.display() + )); + } + }; + let config: toml::Value = toml::from_str(&raw_config).map_err(|err| { + format!( + "failed to parse user config {} while checking marketplace auto-upgrade: {err}", + config_path.display() + ) + })?; + let Some(marketplaces_value) = config.get("marketplaces") else { + return Ok(None); + }; + let mut marketplaces = marketplaces_value + .clone() + .try_into::>() + .map_err(|err| format!("invalid marketplaces config while checking auto-upgrade: {err}"))?; + let Some(marketplace) = marketplaces.remove(marketplace_name) else { + return Ok(None); + }; + Ok(configured_git_marketplace_from_config( + marketplace_name.to_string(), + marketplace, + )) +} diff --git a/codex-rs/core-plugins/src/marketplace_upgrade/activation.rs b/codex-rs/core-plugins/src/marketplace_upgrade/activation.rs new file mode 100644 index 0000000000..366b35fb43 --- /dev/null +++ b/codex-rs/core-plugins/src/marketplace_upgrade/activation.rs @@ -0,0 +1,167 @@ +use super::ConfiguredGitMarketplace; +use codex_config::types::MarketplaceSourceType; +use serde::Deserialize; +use serde::Serialize; +use std::path::Path; +use std::path::PathBuf; +use tempfile::TempDir; +use tracing::warn; + +const MARKETPLACE_INSTALL_METADATA_FILE: &str = ".codex-marketplace-install.json"; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +struct InstalledMarketplaceMetadata { + source_type: MarketplaceSourceType, + source: String, + ref_name: Option, + sparse_paths: Vec, + revision: String, +} + +pub(super) fn installed_marketplace_metadata_matches( + root: &Path, + marketplace: &ConfiguredGitMarketplace, + revision: &str, +) -> bool { + let metadata = match std::fs::read_to_string(installed_marketplace_metadata_path(root)) { + Ok(metadata) => metadata, + Err(_) => return false, + }; + let metadata = match serde_json::from_str::(&metadata) { + Ok(metadata) => metadata, + Err(err) => { + warn!( + marketplace = marketplace.name, + error = %err, + "failed to parse activated marketplace metadata" + ); + return false; + } + }; + metadata == installed_marketplace_metadata(marketplace, revision) +} + +pub(super) fn write_installed_marketplace_metadata( + root: &Path, + marketplace: &ConfiguredGitMarketplace, + revision: &str, +) -> Result<(), String> { + let metadata = installed_marketplace_metadata(marketplace, revision); + let contents = serde_json::to_string_pretty(&metadata) + .map_err(|err| format!("failed to serialize activated marketplace metadata: {err}"))?; + std::fs::write(installed_marketplace_metadata_path(root), contents) + .map_err(|err| format!("failed to write activated marketplace metadata: {err}")) +} + +pub(super) fn activate_marketplace_root( + destination: &Path, + staged_dir: TempDir, + after_activate: impl FnOnce() -> Result<(), String>, +) -> Result<(), String> { + let staged_root = staged_dir.path(); + let Some(parent) = destination.parent() else { + return Err(format!( + "failed to determine marketplace install parent for {}", + destination.display() + )); + }; + std::fs::create_dir_all(parent).map_err(|err| { + format!( + "failed to create marketplace install parent {}: {err}", + parent.display() + ) + })?; + + if destination.exists() { + let backup_dir = tempfile::Builder::new() + .prefix("marketplace-backup-") + .tempdir_in(parent) + .map_err(|err| { + format!( + "failed to create marketplace backup directory in {}: {err}", + parent.display() + ) + })?; + let backup_root = backup_dir.path().join("root"); + std::fs::rename(destination, &backup_root).map_err(|err| { + format!( + "failed to move previous marketplace root out of the way at {}: {err}", + destination.display() + ) + })?; + + if let Err(err) = std::fs::rename(staged_root, destination) { + let rollback_result = std::fs::rename(&backup_root, destination); + return match rollback_result { + Ok(()) => Err(format!( + "failed to activate upgraded marketplace at {}: {err}", + destination.display() + )), + Err(rollback_err) => { + let backup_path = backup_dir.keep().join("root"); + Err(format!( + "failed to activate upgraded marketplace at {}: {err}; failed to restore previous marketplace root (left at {}): {rollback_err}", + destination.display(), + backup_path.display() + )) + } + }; + } + + if let Err(err) = after_activate() { + let remove_result = std::fs::remove_dir_all(destination); + let rollback_result = + remove_result.and_then(|()| std::fs::rename(&backup_root, destination)); + return match rollback_result { + Ok(()) => Err(err), + Err(rollback_err) => { + let backup_path = backup_dir.keep().join("root"); + Err(format!( + "{err}; failed to restore previous marketplace root at {} (left at {}): {rollback_err}", + destination.display(), + backup_path.display() + )) + } + }; + } + + return Ok(()); + } + + std::fs::rename(staged_root, destination).map_err(|err| { + format!( + "failed to activate upgraded marketplace at {}: {err}", + destination.display() + ) + })?; + if let Err(err) = after_activate() { + let remove_result = std::fs::remove_dir_all(destination); + return match remove_result { + Ok(()) => Err(err), + Err(remove_err) => Err(format!( + "{err}; failed to remove newly activated marketplace root at {}: {remove_err}", + destination.display() + )), + }; + } + + Ok(()) +} + +fn installed_marketplace_metadata( + marketplace: &ConfiguredGitMarketplace, + revision: &str, +) -> InstalledMarketplaceMetadata { + InstalledMarketplaceMetadata { + source_type: MarketplaceSourceType::Git, + source: marketplace.source.clone(), + ref_name: marketplace.ref_name.clone(), + sparse_paths: marketplace.sparse_paths.clone(), + revision: revision.to_string(), + } +} + +fn installed_marketplace_metadata_path(root: &Path) -> PathBuf { + root.join(MARKETPLACE_INSTALL_METADATA_FILE) +} diff --git a/codex-rs/core-plugins/src/marketplace_upgrade/git.rs b/codex-rs/core-plugins/src/marketplace_upgrade/git.rs new file mode 100644 index 0000000000..80a7c68f68 --- /dev/null +++ b/codex-rs/core-plugins/src/marketplace_upgrade/git.rs @@ -0,0 +1,238 @@ +use std::path::Path; +use std::process::Command; +use std::process::Output; +use std::process::Stdio; +use std::time::Duration; + +pub(super) fn git_remote_revision( + source: &str, + ref_name: Option<&str>, + timeout: Duration, +) -> Result { + if let Some(ref_name) = ref_name + && is_full_git_sha(ref_name) + { + return Ok(ref_name.to_string()); + } + + let ref_name = ref_name.unwrap_or("HEAD"); + let output = run_git_command_with_timeout( + git_command().arg("ls-remote").arg(source).arg(ref_name), + "git ls-remote marketplace source", + timeout, + )?; + ensure_git_success(&output, "git ls-remote marketplace source")?; + + let stdout = String::from_utf8_lossy(&output.stdout); + let Some(first_line) = stdout.lines().next() else { + return Err("git ls-remote returned empty output for marketplace source".to_string()); + }; + let Some((revision, _)) = first_line.split_once('\t') else { + return Err(format!( + "unexpected git ls-remote output for marketplace source: {first_line}" + )); + }; + let revision = revision.trim(); + if revision.is_empty() { + return Err("git ls-remote returned empty revision for marketplace source".to_string()); + } + Ok(revision.to_string()) +} + +pub(super) fn clone_git_source( + source: &str, + ref_name: Option<&str>, + sparse_paths: &[String], + destination: &Path, + timeout: Duration, +) -> Result { + if sparse_paths.is_empty() { + let output = run_git_command_with_timeout( + git_command().arg("clone").arg(source).arg(destination), + "git clone marketplace source", + timeout, + )?; + ensure_git_success(&output, "git clone marketplace source")?; + if let Some(ref_name) = ref_name { + let output = run_git_command_with_timeout( + git_command() + .arg("-C") + .arg(destination) + .arg("checkout") + .arg(ref_name), + "git checkout marketplace ref", + timeout, + )?; + ensure_git_success(&output, "git checkout marketplace ref")?; + } + return git_worktree_revision(destination, timeout); + } + + let output = run_git_command_with_timeout( + git_command() + .arg("clone") + .arg("--filter=blob:none") + .arg("--no-checkout") + .arg(source) + .arg(destination), + "git clone marketplace source", + timeout, + )?; + ensure_git_success(&output, "git clone marketplace source")?; + + let mut sparse_checkout = git_command(); + sparse_checkout + .arg("-C") + .arg(destination) + .arg("sparse-checkout") + .arg("set") + .args(sparse_paths); + let output = run_git_command_with_timeout( + &mut sparse_checkout, + "git sparse-checkout marketplace source", + timeout, + )?; + ensure_git_success(&output, "git sparse-checkout marketplace source")?; + + let output = run_git_command_with_timeout( + git_command() + .arg("-C") + .arg(destination) + .arg("checkout") + .arg(ref_name.unwrap_or("HEAD")), + "git checkout marketplace ref", + timeout, + )?; + ensure_git_success(&output, "git checkout marketplace ref")?; + git_worktree_revision(destination, timeout) +} + +fn git_worktree_revision(destination: &Path, timeout: Duration) -> Result { + let output = run_git_command_with_timeout( + git_command() + .arg("-C") + .arg(destination) + .arg("rev-parse") + .arg("HEAD"), + "git rev-parse marketplace revision", + timeout, + )?; + ensure_git_success(&output, "git rev-parse marketplace revision")?; + + let revision = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if revision.is_empty() { + Err("git rev-parse returned empty revision for marketplace source".to_string()) + } else { + Ok(revision) + } +} + +fn is_full_git_sha(value: &str) -> bool { + value.len() == 40 && value.chars().all(|ch| ch.is_ascii_hexdigit()) +} + +fn git_command() -> Command { + let mut command = Command::new("git"); + command + .env("GIT_OPTIONAL_LOCKS", "0") + .env("GIT_TERMINAL_PROMPT", "0"); + command +} + +fn run_git_command_with_timeout( + command: &mut Command, + context: &str, + timeout: Duration, +) -> Result { + let mut child = command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|err| format!("failed to run {context}: {err}"))?; + let start = std::time::Instant::now(); + loop { + match child.try_wait() { + Ok(Some(_)) => { + return child + .wait_with_output() + .map_err(|err| format!("failed to wait for {context}: {err}")); + } + Ok(None) => {} + Err(err) => return Err(format!("failed to poll {context}: {err}")), + } + + if start.elapsed() >= timeout { + let _ = child.kill(); + let output = child + .wait_with_output() + .map_err(|err| format!("failed to wait for {context} after timeout: {err}"))?; + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + return if stderr.is_empty() { + Err(format!("{context} timed out after {}s", timeout.as_secs())) + } else { + Err(format!( + "{context} timed out after {}s: {stderr}", + timeout.as_secs() + )) + }; + } + + std::thread::sleep(Duration::from_millis(100)); + } +} + +fn ensure_git_success(output: &Output, context: &str) -> Result<(), String> { + if output.status.success() { + return Ok(()); + } + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + if stderr.is_empty() { + Err(format!("{context} failed with status {}", output.status)) + } else { + Err(format!( + "{context} failed with status {}: {stderr}", + output.status + )) + } +} + +#[cfg(test)] +mod tests { + use super::git_command; + use super::is_full_git_sha; + use std::ffi::OsStr; + + #[test] + fn full_git_sha_ref_is_already_a_remote_revision() { + assert!(is_full_git_sha("0123456789abcdef0123456789abcdef01234567")); + assert!(!is_full_git_sha("main")); + assert!(!is_full_git_sha("0123456")); + } + + #[test] + fn git_command_uses_path_lookup_with_stable_noninteractive_env() { + let command = git_command(); + + assert_eq!(command.get_program(), OsStr::new("git")); + assert_eq!( + command_env(&command, "GIT_OPTIONAL_LOCKS"), + Some(Some(OsStr::new("0"))) + ); + assert_eq!( + command_env(&command, "GIT_TERMINAL_PROMPT"), + Some(Some(OsStr::new("0"))) + ); + assert_eq!(command_env(&command, "PATH"), None); + } + + fn command_env<'a>( + command: &'a std::process::Command, + name: &str, + ) -> Option> { + command + .get_envs() + .find(|(key, _)| key == &OsStr::new(name)) + .map(|(_, value)| value) + } +} diff --git a/codex-rs/core/src/plugins/remote.rs b/codex-rs/core-plugins/src/remote.rs similarity index 94% rename from codex-rs/core/src/plugins/remote.rs rename to codex-rs/core-plugins/src/remote.rs index f26bbe8880..34ece0880a 100644 --- a/codex-rs/core/src/plugins/remote.rs +++ b/codex-rs/core-plugins/src/remote.rs @@ -1,4 +1,3 @@ -use crate::config::Config; use codex_login::CodexAuth; use codex_login::default_client::build_reqwest_client; use codex_protocol::protocol::Product; @@ -11,12 +10,17 @@ const REMOTE_PLUGIN_FETCH_TIMEOUT: Duration = Duration::from_secs(30); const REMOTE_FEATURED_PLUGIN_FETCH_TIMEOUT: Duration = Duration::from_secs(10); const REMOTE_PLUGIN_MUTATION_TIMEOUT: Duration = Duration::from_secs(30); +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RemotePluginServiceConfig { + pub chatgpt_base_url: String, +} + #[derive(Debug, Clone, PartialEq, Eq, Deserialize)] -pub(crate) struct RemotePluginStatusSummary { - pub(crate) name: String, +pub struct RemotePluginStatusSummary { + pub name: String, #[serde(default = "default_remote_marketplace_name")] - pub(crate) marketplace_name: String, - pub(crate) enabled: bool, + pub marketplace_name: String, + pub enabled: bool, } #[derive(Debug, Clone, PartialEq, Eq, Deserialize)] @@ -116,8 +120,8 @@ pub enum RemotePluginFetchError { }, } -pub(crate) async fn fetch_remote_plugin_status( - config: &Config, +pub async fn fetch_remote_plugin_status( + config: &RemotePluginServiceConfig, auth: Option<&CodexAuth>, ) -> Result, RemotePluginFetchError> { let Some(auth) = auth else { @@ -161,7 +165,7 @@ pub(crate) async fn fetch_remote_plugin_status( } pub async fn fetch_remote_featured_plugin_ids( - config: &Config, + config: &RemotePluginServiceConfig, auth: Option<&CodexAuth>, product: Option, ) -> Result, RemotePluginFetchError> { @@ -205,8 +209,8 @@ pub async fn fetch_remote_featured_plugin_ids( }) } -pub(crate) async fn enable_remote_plugin( - config: &Config, +pub async fn enable_remote_plugin( + config: &RemotePluginServiceConfig, auth: Option<&CodexAuth>, plugin_id: &str, ) -> Result<(), RemotePluginMutationError> { @@ -214,8 +218,8 @@ pub(crate) async fn enable_remote_plugin( Ok(()) } -pub(crate) async fn uninstall_remote_plugin( - config: &Config, +pub async fn uninstall_remote_plugin( + config: &RemotePluginServiceConfig, auth: Option<&CodexAuth>, plugin_id: &str, ) -> Result<(), RemotePluginMutationError> { @@ -238,7 +242,7 @@ fn default_remote_marketplace_name() -> String { } async fn post_remote_plugin_mutation( - config: &Config, + config: &RemotePluginServiceConfig, auth: Option<&CodexAuth>, plugin_id: &str, action: &str, @@ -294,7 +298,7 @@ async fn post_remote_plugin_mutation( } fn remote_plugin_mutation_url( - config: &Config, + config: &RemotePluginServiceConfig, plugin_id: &str, action: &str, ) -> Result { diff --git a/codex-rs/core/src/plugins/store.rs b/codex-rs/core-plugins/src/store.rs similarity index 97% rename from codex-rs/core/src/plugins/store.rs rename to codex-rs/core-plugins/src/store.rs index 4316f18002..2f530b8648 100644 --- a/codex-rs/core/src/plugins/store.rs +++ b/codex-rs/core-plugins/src/store.rs @@ -1,5 +1,5 @@ -use super::load_plugin_manifest; -use super::manifest::PluginManifest; +use crate::manifest::PluginManifest; +use crate::manifest::load_plugin_manifest; use codex_plugin::PluginId; use codex_plugin::validate_plugin_segment; use codex_utils_absolute_path::AbsolutePathBuf; @@ -11,8 +11,8 @@ use std::io; use std::path::Path; use std::path::PathBuf; -pub(crate) const DEFAULT_PLUGIN_VERSION: &str = "local"; -pub(crate) const PLUGINS_CACHE_DIR: &str = "plugins/cache"; +pub const DEFAULT_PLUGIN_VERSION: &str = "local"; +pub const PLUGINS_CACHE_DIR: &str = "plugins/cache"; #[derive(Debug, Clone, PartialEq, Eq)] pub struct PluginInstallResult { @@ -157,7 +157,7 @@ impl PluginStoreError { } } -pub(crate) fn plugin_version_for_source(source_path: &Path) -> Result { +pub fn plugin_version_for_source(source_path: &Path) -> Result { let plugin_version = plugin_manifest_version_for_source(source_path)? .unwrap_or_else(|| DEFAULT_PLUGIN_VERSION.to_string()); validate_plugin_version_segment(&plugin_version).map_err(PluginStoreError::Invalid)?; diff --git a/codex-rs/core/src/plugins/store_tests.rs b/codex-rs/core-plugins/src/store_tests.rs similarity index 100% rename from codex-rs/core/src/plugins/store_tests.rs rename to codex-rs/core-plugins/src/store_tests.rs diff --git a/codex-rs/core/src/plugins/toggles.rs b/codex-rs/core-plugins/src/toggles.rs similarity index 100% rename from codex-rs/core/src/plugins/toggles.rs rename to codex-rs/core-plugins/src/toggles.rs diff --git a/codex-rs/core/BUILD.bazel b/codex-rs/core/BUILD.bazel index ed01996273..434dc1f6a4 100644 --- a/codex-rs/core/BUILD.bazel +++ b/codex-rs/core/BUILD.bazel @@ -53,6 +53,7 @@ codex_rust_crate( "//codex-rs/linux-sandbox:codex-linux-sandbox", "//codex-rs/rmcp-client:test_stdio_server", "//codex-rs/rmcp-client:test_streamable_http_server", + "//codex-rs/responses-api-proxy:codex-responses-api-proxy", "//codex-rs/cli:codex", ], ) diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index c91f07f705..4ce4192b81 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -33,6 +33,7 @@ codex-async-utils = { workspace = true } codex-code-mode = { workspace = true } codex-connectors = { workspace = true } codex-config = { workspace = true } +codex-core-plugins = { workspace = true } codex-core-skills = { workspace = true } codex-exec-server = { workspace = true } codex-features = { workspace = true } @@ -41,6 +42,7 @@ codex-login = { workspace = true } codex-mcp = { workspace = true } codex-model-provider-info = { workspace = true } codex-models-manager = { workspace = true } +ed25519-dalek = { workspace = true } codex-shell-command = { workspace = true } codex-execpolicy = { workspace = true } codex-git-utils = { workspace = true } @@ -56,6 +58,7 @@ codex-rmcp-client = { workspace = true } codex-sandboxing = { workspace = true } codex-state = { workspace = true } codex-terminal-detection = { workspace = true } +codex-thread-store = { workspace = true } codex-tools = { workspace = true } codex-utils-absolute-path = { workspace = true } codex-utils-cache = { workspace = true } @@ -143,6 +146,7 @@ codex-shell-escalation = { workspace = true } [dev-dependencies] assert_cmd = { workspace = true } assert_matches = { workspace = true } +codex-arg0 = { workspace = true } codex-otel = { workspace = true } codex-test-binary-support = { workspace = true } codex-utils-cargo-bin = { workspace = true } diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index e878649215..fd47d2e2cf 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -500,6 +500,9 @@ "tui_app_server": { "type": "boolean" }, + "unavailable_dummy_tools": { + "type": "boolean" + }, "undo": { "type": "boolean" }, @@ -524,6 +527,9 @@ "web_search_request": { "type": "boolean" }, + "workspace_dependencies": { + "type": "boolean" + }, "workspace_owner_usage_nudge": { "type": "boolean" } @@ -776,6 +782,11 @@ "MarketplaceConfig": { "additionalProperties": false, "properties": { + "last_revision": { + "default": null, + "description": "Git revision Codex last successfully activated for this marketplace.", + "type": "string" + }, "last_updated": { "default": null, "description": "Last time Codex successfully added or refreshed this marketplace.", @@ -852,7 +863,8 @@ "max_raw_memories_for_consolidation": { "description": "Maximum number of recent raw memories retained for global consolidation.", "format": "uint", - "minimum": 0.0, + "maximum": 4096.0, + "minimum": 1.0, "type": "integer" }, "max_rollout_age_days": { @@ -863,7 +875,8 @@ "max_rollouts_per_startup": { "description": "Maximum number of rollout candidates processed per pass.", "format": "uint", - "minimum": 0.0, + "maximum": 128.0, + "minimum": 1.0, "type": "integer" }, "max_unused_days": { @@ -1453,6 +1466,14 @@ "default": null, "type": "string" }, + "default_tools_approval_mode": { + "allOf": [ + { + "$ref": "#/definitions/AppToolApproval" + } + ], + "default": null + }, "disabled_tools": { "default": null, "items": { @@ -1492,6 +1513,10 @@ }, "type": "array" }, + "experimental_environment": { + "default": null, + "type": "string" + }, "http_headers": { "additionalProperties": { "type": "string" @@ -2360,6 +2385,9 @@ "tui_app_server": { "type": "boolean" }, + "unavailable_dummy_tools": { + "type": "boolean" + }, "undo": { "type": "boolean" }, @@ -2384,6 +2412,9 @@ "web_search_request": { "type": "boolean" }, + "workspace_dependencies": { + "type": "boolean" + }, "workspace_owner_usage_nudge": { "type": "boolean" } diff --git a/codex-rs/core/src/agent/control_tests.rs b/codex-rs/core/src/agent/control_tests.rs index 6fc74b30e8..b444d8c67c 100644 --- a/codex-rs/core/src/agent/control_tests.rs +++ b/codex-rs/core/src/agent/control_tests.rs @@ -7,7 +7,6 @@ use crate::config::Config; use crate::config::ConfigBuilder; use crate::contextual_user_message::SUBAGENT_NOTIFICATION_OPEN_TAG; use assert_matches::assert_matches; -use chrono::Utc; use codex_features::Feature; use codex_login::CodexAuth; use codex_protocol::AgentPath; @@ -24,6 +23,9 @@ use codex_protocol::protocol::TurnAbortReason; use codex_protocol::protocol::TurnAbortedEvent; use codex_protocol::protocol::TurnCompleteEvent; use codex_protocol::protocol::TurnStartedEvent; +use codex_thread_store::ArchiveThreadParams; +use codex_thread_store::LocalThreadStore; +use codex_thread_store::ThreadStore; use pretty_assertions::assert_eq; use tempfile::TempDir; use tokio::time::Duration; @@ -186,7 +188,9 @@ async fn wait_for_subagent_notification(parent_thread: &Arc) -> boo sleep(Duration::from_millis(25)).await; } }; - timeout(Duration::from_secs(2), wait).await.is_ok() + // CI can take several seconds to schedule the detached completion watcher, + // especially on slower Windows runners. + timeout(Duration::from_secs(10), wait).await.is_ok() } async fn persist_thread_for_tree_resume(thread: &Arc, message: &str) { @@ -1658,38 +1662,18 @@ async fn resume_agent_from_rollout_reads_archived_rollout_path() { .await .expect("child thread should exist"); persist_thread_for_tree_resume(&child_thread, "persist before archiving").await; - let rollout_path = child_thread - .rollout_path() - .expect("thread should have rollout path"); - let state_db = child_thread - .state_db() - .expect("thread should have state db handle"); - let _ = harness .control .shutdown_live_agent(child_thread_id) .await .expect("child shutdown should succeed"); - - let archived_root = harness - .config - .codex_home - .join(crate::ARCHIVED_SESSIONS_SUBDIR); - tokio::fs::create_dir_all(&archived_root) + let store = LocalThreadStore::new(codex_rollout::RolloutConfig::from_view(&harness.config)); + store + .archive_thread(ArchiveThreadParams { + thread_id: child_thread_id, + }) .await - .expect("archived root should exist"); - let archived_rollout_path = archived_root.join( - rollout_path - .file_name() - .expect("rollout file name should be present"), - ); - tokio::fs::rename(&rollout_path, &archived_rollout_path) - .await - .expect("rollout should move to archived path"); - state_db - .mark_archived(child_thread_id, archived_rollout_path.as_path(), Utc::now()) - .await - .expect("state db archive update should succeed"); + .expect("child thread should archive"); let resumed_thread_id = harness .control diff --git a/codex-rs/core/src/agent/role.rs b/codex-rs/core/src/agent/role.rs index 83c2b1843b..61847f49ba 100644 --- a/codex-rs/core/src/agent/role.rs +++ b/codex-rs/core/src/agent/role.rs @@ -78,7 +78,8 @@ async fn apply_role_to_config_inner( role_layer_toml, preserve_current_profile, preserve_current_provider, - )?; + ) + .await?; Ok(()) } @@ -150,7 +151,7 @@ fn preservation_policy(config: &Config, role_layer_toml: &TomlValue) -> (bool, b mod reload { use super::*; - pub(super) fn build_next_config( + pub(super) async fn build_next_config( config: &Config, role_layer_toml: TomlValue, preserve_current_profile: bool, @@ -171,7 +172,8 @@ mod reload { reload_overrides(config, preserve_current_provider), config.codex_home.clone(), config_layer_stack, - )?; + ) + .await?; if preserve_current_profile { next_config.active_profile = config.active_profile.clone(); } diff --git a/codex-rs/core/src/agent_identity.rs b/codex-rs/core/src/agent_identity.rs new file mode 100644 index 0000000000..2c897b5456 --- /dev/null +++ b/codex-rs/core/src/agent_identity.rs @@ -0,0 +1,767 @@ +use std::sync::Arc; +use std::time::Duration; + +use anyhow::Context; +use anyhow::Result; +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use chrono::SecondsFormat; +use chrono::Utc; +use codex_features::Feature; +use codex_login::AgentIdentityAuthRecord; +use codex_login::AuthManager; +use codex_login::CodexAuth; +use codex_login::default_client::create_client; +use codex_protocol::protocol::SessionSource; +use ed25519_dalek::SigningKey; +use ed25519_dalek::VerifyingKey; +use ed25519_dalek::pkcs8::DecodePrivateKey; +use ed25519_dalek::pkcs8::EncodePrivateKey; +use rand::TryRngCore; +use rand::rngs::OsRng; +use serde::Deserialize; +use serde::Serialize; +use tokio::sync::Mutex; +use tracing::debug; +use tracing::info; +use tracing::warn; + +use crate::config::Config; + +const AGENT_REGISTRATION_TIMEOUT: Duration = Duration::from_secs(15); +const AGENT_IDENTITY_BISCUIT_TIMEOUT: Duration = Duration::from_secs(15); + +#[derive(Clone)] +pub(crate) struct AgentIdentityManager { + auth_manager: Arc, + chatgpt_base_url: String, + feature_enabled: bool, + abom: AgentBillOfMaterials, + ensure_lock: Arc>, +} + +impl std::fmt::Debug for AgentIdentityManager { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AgentIdentityManager") + .field("chatgpt_base_url", &self.chatgpt_base_url) + .field("feature_enabled", &self.feature_enabled) + .field("abom", &self.abom) + .finish_non_exhaustive() + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub(crate) struct StoredAgentIdentity { + pub(crate) binding_id: String, + pub(crate) chatgpt_account_id: String, + pub(crate) chatgpt_user_id: Option, + pub(crate) agent_runtime_id: String, + pub(crate) private_key_pkcs8_base64: String, + pub(crate) public_key_ssh: String, + pub(crate) registered_at: String, + pub(crate) abom: AgentBillOfMaterials, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub(crate) struct AgentBillOfMaterials { + pub(crate) agent_version: String, + pub(crate) agent_harness_id: String, + pub(crate) running_location: String, +} + +#[derive(Debug, Serialize)] +struct RegisterAgentRequest { + abom: AgentBillOfMaterials, + agent_public_key: String, + capabilities: Vec, +} + +#[derive(Debug, Deserialize)] +struct RegisterAgentResponse { + agent_runtime_id: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct AgentIdentityBinding { + binding_id: String, + chatgpt_account_id: String, + chatgpt_user_id: Option, + access_token: String, +} + +struct GeneratedAgentKeyMaterial { + private_key_pkcs8_base64: String, + public_key_ssh: String, +} + +impl AgentIdentityManager { + pub(crate) fn new( + config: &Config, + auth_manager: Arc, + session_source: SessionSource, + ) -> Self { + Self { + auth_manager, + chatgpt_base_url: config.chatgpt_base_url.clone(), + feature_enabled: config.features.enabled(Feature::UseAgentIdentity), + abom: build_abom(session_source), + ensure_lock: Arc::new(Mutex::new(())), + } + } + + pub(crate) fn is_enabled(&self) -> bool { + self.feature_enabled + } + + pub(crate) async fn ensure_registered_identity(&self) -> Result> { + if !self.feature_enabled { + return Ok(None); + } + + let Some(auth) = self.auth_manager.auth().await else { + debug!("skipping agent identity registration because no auth is available"); + return Ok(None); + }; + + let Some(binding) = + AgentIdentityBinding::from_auth(&auth, self.auth_manager.forced_chatgpt_workspace_id()) + else { + debug!("skipping agent identity registration because ChatGPT auth is unavailable"); + return Ok(None); + }; + + let _guard = self.ensure_lock.lock().await; + + if let Some(stored_identity) = self.load_stored_identity(&auth, &binding)? { + info!( + agent_runtime_id = %stored_identity.agent_runtime_id, + binding_id = %binding.binding_id, + "reusing stored agent identity" + ); + return Ok(Some(stored_identity)); + } + + let stored_identity = self.register_agent_identity(&binding).await?; + self.store_identity(&auth, &stored_identity)?; + Ok(Some(stored_identity)) + } + + async fn register_agent_identity( + &self, + binding: &AgentIdentityBinding, + ) -> Result { + let key_material = generate_agent_key_material()?; + let request_body = RegisterAgentRequest { + abom: self.abom.clone(), + agent_public_key: key_material.public_key_ssh.clone(), + capabilities: Vec::new(), + }; + + let url = agent_registration_url(&self.chatgpt_base_url); + let human_biscuit = self.mint_human_biscuit(binding, "POST", &url).await?; + let client = create_client(); + let response = client + .post(&url) + .header("X-OpenAI-Authorization", human_biscuit) + .json(&request_body) + .timeout(AGENT_REGISTRATION_TIMEOUT) + .send() + .await + .with_context(|| { + format!("failed to send agent identity registration request to {url}") + })?; + + if response.status().is_success() { + let response_body = response + .json::() + .await + .with_context(|| format!("failed to parse agent identity response from {url}"))?; + let stored_identity = StoredAgentIdentity { + binding_id: binding.binding_id.clone(), + chatgpt_account_id: binding.chatgpt_account_id.clone(), + chatgpt_user_id: binding.chatgpt_user_id.clone(), + agent_runtime_id: response_body.agent_runtime_id, + private_key_pkcs8_base64: key_material.private_key_pkcs8_base64, + public_key_ssh: key_material.public_key_ssh, + registered_at: Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), + abom: self.abom.clone(), + }; + info!( + agent_runtime_id = %stored_identity.agent_runtime_id, + binding_id = %binding.binding_id, + "registered agent identity" + ); + return Ok(stored_identity); + } + + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("agent identity registration failed with status {status} from {url}: {body}") + } + + async fn mint_human_biscuit( + &self, + binding: &AgentIdentityBinding, + target_method: &str, + target_url: &str, + ) -> Result { + let url = agent_identity_biscuit_url(&self.chatgpt_base_url); + let request_id = agent_identity_request_id()?; + let client = create_client(); + let response = client + .get(&url) + .bearer_auth(&binding.access_token) + .header("X-Request-Id", request_id.clone()) + .header("X-Original-Method", target_method) + .header("X-Original-Url", target_url) + .timeout(AGENT_IDENTITY_BISCUIT_TIMEOUT) + .send() + .await + .with_context(|| format!("failed to send agent identity biscuit request to {url}"))?; + + if response.status().is_success() { + let human_biscuit = response + .headers() + .get("x-openai-authorization") + .context("agent identity biscuit response did not include x-openai-authorization")? + .to_str() + .context("agent identity biscuit response header was not valid UTF-8")? + .to_string(); + info!( + request_id = %request_id, + "minted human biscuit for agent identity registration" + ); + return Ok(human_biscuit); + } + + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + anyhow::bail!( + "agent identity biscuit minting failed with status {status} from {url}: {body}" + ) + } + + fn load_stored_identity( + &self, + auth: &CodexAuth, + binding: &AgentIdentityBinding, + ) -> Result> { + let Some(record) = auth.get_agent_identity(&binding.chatgpt_account_id) else { + return Ok(None); + }; + + let stored_identity = + match StoredAgentIdentity::from_auth_record(binding, record, self.abom.clone()) { + Ok(stored_identity) => stored_identity, + Err(error) => { + warn!( + binding_id = %binding.binding_id, + error = %error, + "stored agent identity is invalid; deleting cached value" + ); + auth.remove_agent_identity()?; + return Ok(None); + } + }; + + if !stored_identity.matches_binding(binding) { + warn!( + binding_id = %binding.binding_id, + "stored agent identity binding no longer matches current auth; deleting cached value" + ); + auth.remove_agent_identity()?; + return Ok(None); + } + + if let Err(error) = stored_identity.validate_key_material() { + warn!( + agent_runtime_id = %stored_identity.agent_runtime_id, + binding_id = %binding.binding_id, + error = %error, + "stored agent identity key material is invalid; deleting cached value" + ); + auth.remove_agent_identity()?; + return Ok(None); + } + + Ok(Some(stored_identity)) + } + + fn store_identity( + &self, + auth: &CodexAuth, + stored_identity: &StoredAgentIdentity, + ) -> Result<()> { + auth.set_agent_identity(stored_identity.to_auth_record())?; + Ok(()) + } + + #[cfg(test)] + fn new_for_tests( + auth_manager: Arc, + feature_enabled: bool, + chatgpt_base_url: String, + session_source: SessionSource, + ) -> Self { + Self { + auth_manager, + chatgpt_base_url, + feature_enabled, + abom: build_abom(session_source), + ensure_lock: Arc::new(Mutex::new(())), + } + } +} + +impl StoredAgentIdentity { + fn from_auth_record( + binding: &AgentIdentityBinding, + record: AgentIdentityAuthRecord, + abom: AgentBillOfMaterials, + ) -> Result { + if record.workspace_id != binding.chatgpt_account_id { + anyhow::bail!( + "stored agent identity workspace {:?} does not match current workspace {:?}", + record.workspace_id, + binding.chatgpt_account_id + ); + } + let signing_key = signing_key_from_private_key_pkcs8_base64(&record.agent_private_key)?; + Ok(Self { + binding_id: binding.binding_id.clone(), + chatgpt_account_id: binding.chatgpt_account_id.clone(), + chatgpt_user_id: record.chatgpt_user_id, + agent_runtime_id: record.agent_runtime_id, + private_key_pkcs8_base64: record.agent_private_key, + public_key_ssh: encode_ssh_ed25519_public_key(&signing_key.verifying_key()), + registered_at: record.registered_at, + abom, + }) + } + + fn to_auth_record(&self) -> AgentIdentityAuthRecord { + AgentIdentityAuthRecord { + workspace_id: self.chatgpt_account_id.clone(), + chatgpt_user_id: self.chatgpt_user_id.clone(), + agent_runtime_id: self.agent_runtime_id.clone(), + agent_private_key: self.private_key_pkcs8_base64.clone(), + registered_at: self.registered_at.clone(), + } + } + + fn matches_binding(&self, binding: &AgentIdentityBinding) -> bool { + self.binding_id == binding.binding_id + && self.chatgpt_account_id == binding.chatgpt_account_id + && match binding.chatgpt_user_id.as_deref() { + Some(chatgpt_user_id) => self.chatgpt_user_id.as_deref() == Some(chatgpt_user_id), + None => true, + } + } + + fn validate_key_material(&self) -> Result<()> { + let signing_key = self.signing_key()?; + let derived_public_key = encode_ssh_ed25519_public_key(&signing_key.verifying_key()); + anyhow::ensure!( + self.public_key_ssh == derived_public_key, + "stored public key does not match the private key" + ); + Ok(()) + } + + pub(crate) fn signing_key(&self) -> Result { + signing_key_from_private_key_pkcs8_base64(&self.private_key_pkcs8_base64) + } +} + +impl AgentIdentityBinding { + fn from_auth(auth: &CodexAuth, forced_workspace_id: Option) -> Option { + if !auth.is_chatgpt_auth() { + return None; + } + + let token_data = auth.get_token_data().ok()?; + let resolved_account_id = + forced_workspace_id + .filter(|value| !value.is_empty()) + .or(token_data + .account_id + .clone() + .filter(|value| !value.is_empty()))?; + + Some(Self { + binding_id: format!("chatgpt-account-{resolved_account_id}"), + chatgpt_account_id: resolved_account_id, + chatgpt_user_id: token_data + .id_token + .chatgpt_user_id + .filter(|value| !value.is_empty()), + access_token: token_data.access_token, + }) + } +} + +fn build_abom(session_source: SessionSource) -> AgentBillOfMaterials { + AgentBillOfMaterials { + agent_version: env!("CARGO_PKG_VERSION").to_string(), + agent_harness_id: match &session_source { + SessionSource::VSCode => "codex-app".to_string(), + SessionSource::Cli + | SessionSource::Exec + | SessionSource::Mcp + | SessionSource::Custom(_) + | SessionSource::SubAgent(_) + | SessionSource::Unknown => "codex-cli".to_string(), + }, + running_location: format!("{}-{}", session_source, std::env::consts::OS), + } +} + +fn generate_agent_key_material() -> Result { + let mut secret_key_bytes = [0u8; 32]; + OsRng + .try_fill_bytes(&mut secret_key_bytes) + .context("failed to generate agent identity private key bytes")?; + let signing_key = SigningKey::from_bytes(&secret_key_bytes); + let private_key_pkcs8 = signing_key + .to_pkcs8_der() + .context("failed to encode agent identity private key as PKCS#8")?; + + Ok(GeneratedAgentKeyMaterial { + private_key_pkcs8_base64: BASE64_STANDARD.encode(private_key_pkcs8.as_bytes()), + public_key_ssh: encode_ssh_ed25519_public_key(&signing_key.verifying_key()), + }) +} + +fn encode_ssh_ed25519_public_key(verifying_key: &VerifyingKey) -> String { + let mut blob = Vec::with_capacity(4 + 11 + 4 + 32); + append_ssh_string(&mut blob, b"ssh-ed25519"); + append_ssh_string(&mut blob, verifying_key.as_bytes()); + format!("ssh-ed25519 {}", BASE64_STANDARD.encode(blob)) +} + +fn append_ssh_string(buf: &mut Vec, value: &[u8]) { + buf.extend_from_slice(&(value.len() as u32).to_be_bytes()); + buf.extend_from_slice(value); +} + +fn agent_registration_url(chatgpt_base_url: &str) -> String { + let trimmed = chatgpt_base_url.trim_end_matches('/'); + format!("{trimmed}/v1/agent/register") +} + +fn signing_key_from_private_key_pkcs8_base64(private_key_pkcs8_base64: &str) -> Result { + let private_key = BASE64_STANDARD + .decode(private_key_pkcs8_base64) + .context("stored agent identity private key is not valid base64")?; + SigningKey::from_pkcs8_der(&private_key) + .context("stored agent identity private key is not valid PKCS#8") +} + +fn agent_identity_biscuit_url(chatgpt_base_url: &str) -> String { + let trimmed = chatgpt_base_url.trim_end_matches('/'); + format!("{trimmed}/authenticate_app_v2") +} + +fn agent_identity_request_id() -> Result { + let mut request_id_bytes = [0u8; 16]; + OsRng + .try_fill_bytes(&mut request_id_bytes) + .context("failed to generate agent identity request id")?; + Ok(format!( + "codex-agent-identity-{}", + URL_SAFE_NO_PAD.encode(request_id_bytes) + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + use base64::engine::general_purpose::URL_SAFE_NO_PAD; + use codex_app_server_protocol::AuthMode as ApiAuthMode; + use codex_login::AuthCredentialsStoreMode; + use codex_login::AuthDotJson; + use codex_login::save_auth; + use codex_login::token_data::IdTokenInfo; + use codex_login::token_data::TokenData; + use pretty_assertions::assert_eq; + use wiremock::Mock; + use wiremock::MockServer; + use wiremock::ResponseTemplate; + use wiremock::matchers::header; + use wiremock::matchers::method; + use wiremock::matchers::path; + + #[tokio::test] + async fn ensure_registered_identity_skips_when_feature_is_disabled() { + let auth_manager = + AuthManager::from_auth_for_testing(make_chatgpt_auth("account-123", Some("user-123"))); + let manager = AgentIdentityManager::new_for_tests( + auth_manager, + /*feature_enabled*/ false, + "https://chatgpt.com/backend-api/".to_string(), + SessionSource::Cli, + ); + + assert_eq!(manager.ensure_registered_identity().await.unwrap(), None); + } + + #[tokio::test] + async fn ensure_registered_identity_skips_for_api_key_auth() { + let auth_manager = AuthManager::from_auth_for_testing(CodexAuth::from_api_key("test-key")); + let manager = AgentIdentityManager::new_for_tests( + auth_manager, + /*feature_enabled*/ true, + "https://chatgpt.com/backend-api/".to_string(), + SessionSource::Cli, + ); + + assert_eq!(manager.ensure_registered_identity().await.unwrap(), None); + } + + #[tokio::test] + async fn ensure_registered_identity_registers_and_reuses_cached_identity() { + let server = MockServer::start().await; + let chatgpt_base_url = server.uri(); + mount_human_biscuit(&server, &chatgpt_base_url).await; + Mock::given(method("POST")) + .and(path("/v1/agent/register")) + .and(header("x-openai-authorization", "human-biscuit")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "agent_runtime_id": "agent_123", + }))) + .expect(1) + .mount(&server) + .await; + + let auth_manager = + AuthManager::from_auth_for_testing(make_chatgpt_auth("account-123", Some("user-123"))); + let manager = AgentIdentityManager::new_for_tests( + auth_manager, + /*feature_enabled*/ true, + chatgpt_base_url, + SessionSource::Cli, + ); + + let first = manager + .ensure_registered_identity() + .await + .unwrap() + .expect("identity should be registered"); + let second = manager + .ensure_registered_identity() + .await + .unwrap() + .expect("identity should be reused"); + + assert_eq!(first.agent_runtime_id, "agent_123"); + assert_eq!(first, second); + assert_eq!(first.abom.agent_harness_id, "codex-cli"); + assert_eq!(first.chatgpt_account_id, "account-123"); + assert_eq!(first.chatgpt_user_id.as_deref(), Some("user-123")); + } + + #[tokio::test] + async fn ensure_registered_identity_deletes_invalid_cached_identity_and_reregisters() { + let server = MockServer::start().await; + let chatgpt_base_url = server.uri(); + mount_human_biscuit(&server, &chatgpt_base_url).await; + Mock::given(method("POST")) + .and(path("/v1/agent/register")) + .and(header("x-openai-authorization", "human-biscuit")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "agent_runtime_id": "agent_456", + }))) + .expect(1) + .mount(&server) + .await; + + let auth = make_chatgpt_auth("account-123", Some("user-123")); + let auth_manager = AuthManager::from_auth_for_testing(auth.clone()); + let manager = AgentIdentityManager::new_for_tests( + auth_manager, + /*feature_enabled*/ true, + chatgpt_base_url, + SessionSource::Cli, + ); + + let binding = + AgentIdentityBinding::from_auth(&auth, /*forced_workspace_id*/ None).expect("binding"); + auth.set_agent_identity(AgentIdentityAuthRecord { + workspace_id: "account-123".to_string(), + chatgpt_user_id: Some("user-123".to_string()), + agent_runtime_id: "agent_invalid".to_string(), + agent_private_key: "not-valid-base64".to_string(), + registered_at: "2026-01-01T00:00:00Z".to_string(), + }) + .expect("seed invalid identity"); + + let stored = manager + .ensure_registered_identity() + .await + .unwrap() + .expect("identity should be registered"); + + assert_eq!(stored.agent_runtime_id, "agent_456"); + let persisted = auth + .get_agent_identity(&binding.chatgpt_account_id) + .expect("stored identity"); + assert_eq!(persisted.agent_runtime_id, "agent_456"); + } + + #[tokio::test] + async fn ensure_registered_identity_deletes_different_user_identity_and_reregisters() { + let server = MockServer::start().await; + let chatgpt_base_url = server.uri(); + mount_human_biscuit(&server, &chatgpt_base_url).await; + Mock::given(method("POST")) + .and(path("/v1/agent/register")) + .and(header("x-openai-authorization", "human-biscuit")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "agent_runtime_id": "agent_new", + }))) + .expect(1) + .mount(&server) + .await; + + let auth = make_chatgpt_auth("account-123", Some("user-new")); + let stale_key = generate_agent_key_material().expect("key material"); + auth.set_agent_identity(AgentIdentityAuthRecord { + workspace_id: "account-123".to_string(), + chatgpt_user_id: Some("user-old".to_string()), + agent_runtime_id: "agent_old".to_string(), + agent_private_key: stale_key.private_key_pkcs8_base64, + registered_at: "2026-01-01T00:00:00Z".to_string(), + }) + .expect("seed stale identity"); + + let auth_manager = AuthManager::from_auth_for_testing(auth.clone()); + let manager = AgentIdentityManager::new_for_tests( + auth_manager, + /*feature_enabled*/ true, + chatgpt_base_url, + SessionSource::Cli, + ); + + let stored = manager + .ensure_registered_identity() + .await + .unwrap() + .expect("identity should be registered"); + + assert_eq!(stored.agent_runtime_id, "agent_new"); + assert_eq!(stored.chatgpt_user_id.as_deref(), Some("user-new")); + let persisted = auth + .get_agent_identity("account-123") + .expect("stored identity"); + assert_eq!(persisted.agent_runtime_id, "agent_new"); + assert_eq!(persisted.chatgpt_user_id.as_deref(), Some("user-new")); + } + + #[tokio::test] + async fn ensure_registered_identity_uses_chatgpt_base_url() { + let server = MockServer::start().await; + let chatgpt_base_url = format!("{}/backend-api", server.uri()); + mount_human_biscuit(&server, &chatgpt_base_url).await; + Mock::given(method("POST")) + .and(path("/backend-api/v1/agent/register")) + .and(header("x-openai-authorization", "human-biscuit")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "agent_runtime_id": "agent_canonical", + }))) + .expect(1) + .mount(&server) + .await; + + let auth_manager = + AuthManager::from_auth_for_testing(make_chatgpt_auth("account-123", Some("user-123"))); + let manager = AgentIdentityManager::new_for_tests( + auth_manager, + /*feature_enabled*/ true, + chatgpt_base_url, + SessionSource::Cli, + ); + + let stored = manager + .ensure_registered_identity() + .await + .unwrap() + .expect("identity should be registered"); + assert_eq!(stored.agent_runtime_id, "agent_canonical"); + } + + async fn mount_human_biscuit(server: &MockServer, chatgpt_base_url: &str) { + let biscuit_url = agent_identity_biscuit_url(chatgpt_base_url); + let biscuit_path = reqwest::Url::parse(&biscuit_url) + .expect("biscuit URL parses") + .path() + .to_string(); + let target_url = agent_registration_url(chatgpt_base_url); + Mock::given(method("GET")) + .and(path(biscuit_path)) + .and(header("authorization", "Bearer access-token-account-123")) + .and(header("x-original-method", "POST")) + .and(header("x-original-url", target_url)) + .respond_with( + ResponseTemplate::new(200).insert_header("x-openai-authorization", "human-biscuit"), + ) + .expect(1) + .mount(server) + .await; + } + + #[test] + fn encode_ssh_ed25519_public_key_matches_expected_wire_shape() { + let key_material = generate_agent_key_material().expect("key material"); + let (_, encoded_blob) = key_material + .public_key_ssh + .split_once(' ') + .expect("public key contains scheme"); + let decoded = BASE64_STANDARD.decode(encoded_blob).expect("base64"); + + assert_eq!(&decoded[..4], 11u32.to_be_bytes().as_slice()); + assert_eq!(&decoded[4..15], b"ssh-ed25519"); + assert_eq!(&decoded[15..19], 32u32.to_be_bytes().as_slice()); + assert_eq!(decoded.len(), 51); + } + + fn make_chatgpt_auth(account_id: &str, user_id: Option<&str>) -> CodexAuth { + let tempdir = tempfile::tempdir().expect("tempdir"); + let auth_json = AuthDotJson { + auth_mode: Some(ApiAuthMode::Chatgpt), + openai_api_key: None, + tokens: Some(TokenData { + id_token: IdTokenInfo { + email: None, + chatgpt_plan_type: None, + chatgpt_user_id: user_id.map(ToOwned::to_owned), + chatgpt_account_id: Some(account_id.to_string()), + chatgpt_account_is_fedramp: false, + raw_jwt: fake_id_token(account_id, user_id), + }, + access_token: format!("access-token-{account_id}"), + refresh_token: "refresh-token".to_string(), + account_id: Some(account_id.to_string()), + }), + last_refresh: Some(Utc::now()), + agent_identity: None, + }; + save_auth(tempdir.path(), &auth_json, AuthCredentialsStoreMode::File).expect("save auth"); + CodexAuth::from_auth_storage(tempdir.path(), AuthCredentialsStoreMode::File) + .expect("load auth") + .expect("auth") + } + + fn fake_id_token(account_id: &str, user_id: Option<&str>) -> String { + let header = URL_SAFE_NO_PAD.encode(r#"{"alg":"none","typ":"JWT"}"#); + let payload = serde_json::json!({ + "https://api.openai.com/auth": { + "chatgpt_user_id": user_id, + "chatgpt_account_id": account_id, + } + }); + let payload = URL_SAFE_NO_PAD.encode(payload.to_string()); + format!("{header}.{payload}.signature") + } +} diff --git a/codex-rs/core/src/agents_md.rs b/codex-rs/core/src/agents_md.rs new file mode 100644 index 0000000000..a1a883e839 --- /dev/null +++ b/codex-rs/core/src/agents_md.rs @@ -0,0 +1,367 @@ +//! AGENTS.md discovery and user instruction assembly. +//! +//! Project-level documentation is primarily stored in files named `AGENTS.md`. +//! Additional fallback filenames can be configured via `project_doc_fallback_filenames`. +//! We include the concatenation of all files found along the path from the +//! project root to the current working directory as follows: +//! +//! 1. Determine the project root by walking upwards from the current working +//! directory until a configured `project_root_markers` entry is found. +//! When `project_root_markers` is unset, the default marker list is used +//! (`.git`). If no marker is found, only the current working directory is +//! considered. An empty marker list disables parent traversal. +//! 2. Collect every `AGENTS.md` found from the project root down to the +//! current working directory (inclusive) and concatenate their contents in +//! that order. +//! 3. We do **not** walk past the project root. + +use crate::config::Config; +use crate::config_loader::ConfigLayerStackOrdering; +use crate::config_loader::default_project_root_markers; +use crate::config_loader::merge_toml_values; +use crate::config_loader::project_root_markers_from_config; +use codex_app_server_protocol::ConfigLayerSource; +use codex_exec_server::Environment; +use codex_exec_server::ExecutorFileSystem; +use codex_features::Feature; +use codex_utils_absolute_path::AbsolutePathBuf; +use dunce::canonicalize as normalize_path; +use std::io; +use toml::Value as TomlValue; +use tracing::error; + +pub(crate) const HIERARCHICAL_AGENTS_MESSAGE: &str = + include_str!("../hierarchical_agents_message.md"); + +/// Default filename scanned for AGENTS.md instructions. +pub const DEFAULT_AGENTS_MD_FILENAME: &str = "AGENTS.md"; +/// Preferred local override for AGENTS.md instructions. +pub const LOCAL_AGENTS_MD_FILENAME: &str = "AGENTS.override.md"; + +/// When both `Config::instructions` and AGENTS.md docs are present, they will +/// be concatenated with the following separator. +const AGENTS_MD_SEPARATOR: &str = "\n\n--- project-doc ---\n\n"; + +fn render_js_repl_instructions(config: &Config) -> Option { + if !config.features.enabled(Feature::JsRepl) { + return None; + } + + let mut section = String::from("## JavaScript REPL (Node)\n"); + section.push_str( + "- Use `js_repl` for Node-backed JavaScript with top-level await in a persistent kernel.\n", + ); + section.push_str("- `js_repl` is a freeform/custom tool. Direct `js_repl` calls must send raw JavaScript tool input (optionally with first-line `// codex-js-repl: timeout_ms=15000`). Do not wrap code in JSON (for example `{\"code\":\"...\"}`), quotes, or markdown code fences.\n"); + section.push_str( + "- Helpers: `codex.cwd`, `codex.homeDir`, `codex.tmpDir`, `codex.tool(name, args?)`, and `codex.emitImage(imageLike)`.\n", + ); + section.push_str("- `codex.tool` executes a normal tool call and resolves to the raw tool output object. Use it for shell and non-shell tools alike. Nested tool outputs stay inside JavaScript unless you emit them explicitly.\n"); + section.push_str("- `codex.emitImage(...)` adds one image to the outer `js_repl` function output each time you call it, so you can call it multiple times to emit multiple images. It accepts a data URL, a single `input_image` item, an object like `{ bytes, mimeType }`, or a raw tool response object with exactly one image and no text. It rejects mixed text-and-image content.\n"); + section.push_str("- `codex.tool(...)` and `codex.emitImage(...)` keep stable helper identities across cells. Saved references and persisted objects can reuse them in later cells, but async callbacks that fire after a cell finishes still fail because no exec is active.\n"); + section.push_str("- Request full-resolution image processing with `detail: \"original\"` only when the `view_image` tool schema includes a `detail` argument. The same availability applies to `codex.emitImage(...)`: if `view_image.detail` is present, you may also pass `detail: \"original\"` there. Use this when high-fidelity image perception or precise localization is needed, especially for CUA agents.\n"); + section.push_str("- Raw MCP image blocks can request the same behavior by returning `_meta: { \"codex/imageDetail\": \"original\" }` on the image content item.\n"); + section.push_str("- Example of sharing an in-memory Playwright screenshot: `await codex.emitImage({ bytes: await page.screenshot({ type: \"jpeg\", quality: 85 }), mimeType: \"image/jpeg\", detail: \"original\" })`.\n"); + section.push_str("- Example of sharing a local image tool result: `await codex.emitImage(codex.tool(\"view_image\", { path: \"/absolute/path\", detail: \"original\" }))`.\n"); + section.push_str("- When encoding an image to send with `codex.emitImage(...)` or `view_image`, prefer JPEG at about 85 quality when lossy compression is acceptable; use PNG when transparency or lossless detail matters. Smaller uploads are faster and less likely to hit size limits.\n"); + section.push_str("- Top-level bindings persist across cells. If a cell throws, prior bindings remain available and bindings that finished initializing before the throw often remain usable in later cells. For code you plan to reuse across cells, prefer declaring or assigning it in direct top-level statements before operations that might throw. If you hit `SyntaxError: Identifier 'x' has already been declared`, first reuse the existing binding, reassign a previously declared `let`, or pick a new descriptive name. Use `{ ... }` only for a short temporary block when you specifically need local scratch names; do not wrap an entire cell in block scope if you want those names reusable later. Reset the kernel with `js_repl_reset` only when you need a clean state.\n"); + section.push_str("- Top-level static import declarations (for example `import x from \"./file.js\"`) are currently unsupported in `js_repl`; use dynamic imports with `await import(\"pkg\")`, `await import(\"./file.js\")`, or `await import(\"/abs/path/file.mjs\")` instead. Imported local files must be ESM `.js`/`.mjs` files and run in the same REPL VM context. Bare package imports always resolve from REPL-global search roots (`CODEX_JS_REPL_NODE_MODULE_DIRS`, then cwd), not relative to the imported file location. Local files may statically import only other local relative/absolute/`file://` `.js`/`.mjs` files; package and builtin imports from local files must stay dynamic. `import.meta.resolve()` returns importable strings such as `file://...`, bare package names, and `node:...` specifiers. Local file modules reload between execs, while top-level bindings persist until `js_repl_reset`.\n"); + + if config.features.enabled(Feature::JsReplToolsOnly) { + section.push_str("- Do not call tools directly; use `js_repl` + `codex.tool(...)` for all tool calls, including shell commands.\n"); + section + .push_str("- MCP tools (if any) can also be called by name via `codex.tool(...)`.\n"); + } + + section.push_str("- Avoid direct access to `process.stdout` / `process.stderr` / `process.stdin`; it can corrupt the JSON line protocol. Use `console.log`, `codex.tool(...)`, and `codex.emitImage(...)`."); + + Some(section) +} + +/// Resolves AGENTS.md files into model-visible user instructions and source +/// paths. +pub struct AgentsMdManager<'a> { + config: &'a Config, +} + +pub(crate) struct LoadedAgentsMd { + pub(crate) contents: String, + pub(crate) path: AbsolutePathBuf, +} + +impl<'a> AgentsMdManager<'a> { + pub fn new(config: &'a Config) -> Self { + Self { config } + } + + pub(crate) fn load_global_instructions( + codex_dir: Option<&AbsolutePathBuf>, + ) -> Option { + let base = codex_dir?; + for candidate in [LOCAL_AGENTS_MD_FILENAME, DEFAULT_AGENTS_MD_FILENAME] { + let path = base.join(candidate); + if let Ok(contents) = std::fs::read_to_string(&path) { + let trimmed = contents.trim(); + if !trimmed.is_empty() { + return Some(LoadedAgentsMd { + contents: trimmed.to_string(), + path, + }); + } + } + } + None + } + + /// Combines configured user instructions and AGENTS.md content into a + /// single model-visible instruction string. + pub(crate) async fn user_instructions( + &self, + environment: Option<&Environment>, + ) -> Option { + let fs = environment?.get_filesystem(); + self.user_instructions_with_fs(fs.as_ref()).await + } + + pub(crate) async fn user_instructions_with_fs( + &self, + fs: &dyn ExecutorFileSystem, + ) -> Option { + let agents_md_docs = self.read_agents_md(fs).await; + + let mut output = String::new(); + + if let Some(instructions) = self.config.user_instructions.clone() { + output.push_str(&instructions); + } + + match agents_md_docs { + Ok(Some(docs)) => { + if !output.is_empty() { + output.push_str(AGENTS_MD_SEPARATOR); + } + output.push_str(&docs); + } + Ok(None) => {} + Err(e) => { + error!("error trying to find AGENTS.md docs: {e:#}"); + } + }; + + if let Some(js_repl_section) = render_js_repl_instructions(self.config) { + if !output.is_empty() { + output.push_str("\n\n"); + } + output.push_str(&js_repl_section); + } + + if self.config.features.enabled(Feature::ChildAgentsMd) { + if !output.is_empty() { + output.push_str("\n\n"); + } + output.push_str(HIERARCHICAL_AGENTS_MESSAGE); + } + + if !output.is_empty() { + Some(output) + } else { + None + } + } + + /// Returns all instruction source files included in the current config. + pub async fn instruction_sources(&self, fs: &dyn ExecutorFileSystem) -> Vec { + let mut paths = Self::load_global_instructions(Some(&self.config.codex_home)) + .map(|loaded| vec![loaded.path]) + .unwrap_or_default(); + match self.agents_md_paths(fs).await { + Ok(agents_md_paths) => paths.extend(agents_md_paths), + Err(err) => { + tracing::warn!(error = %err, "failed to discover AGENTS.md docs for instruction sources"); + } + } + paths + } + + /// Attempt to locate and load AGENTS.md documentation. + /// + /// On success returns `Ok(Some(contents))` where `contents` is the + /// concatenation of all discovered docs. If no documentation file is found + /// the function returns `Ok(None)`. Unexpected I/O failures bubble up as + /// `Err` so callers can decide how to handle them. + async fn read_agents_md(&self, fs: &dyn ExecutorFileSystem) -> io::Result> { + let max_total = self.config.project_doc_max_bytes; + + if max_total == 0 { + return Ok(None); + } + + let paths = self.agents_md_paths(fs).await?; + if paths.is_empty() { + return Ok(None); + } + + let mut remaining: u64 = max_total as u64; + let mut parts: Vec = Vec::new(); + + for p in paths { + if remaining == 0 { + break; + } + + match fs.get_metadata(&p, /*sandbox*/ None).await { + Ok(metadata) if !metadata.is_file => continue, + Ok(_) => {} + Err(err) if err.kind() == io::ErrorKind::NotFound => continue, + Err(err) => return Err(err), + } + + let mut data = match fs.read_file(&p, /*sandbox*/ None).await { + Ok(data) => data, + Err(err) if err.kind() == io::ErrorKind::NotFound => continue, + Err(err) => return Err(err), + }; + let size = data.len() as u64; + if size > remaining { + data.truncate(remaining as usize); + } + + if size > remaining { + tracing::warn!( + "Project doc `{}` exceeds remaining budget ({} bytes) - truncating.", + p.display(), + remaining, + ); + } + + let text = String::from_utf8_lossy(&data).to_string(); + if !text.trim().is_empty() { + parts.push(text); + remaining = remaining.saturating_sub(data.len() as u64); + } + } + + if parts.is_empty() { + Ok(None) + } else { + Ok(Some(parts.join("\n\n"))) + } + } + + /// Discover the list of AGENTS.md files using the same search rules as + /// `read_agents_md`, but return the file paths instead of concatenated + /// contents. The list is ordered from project root to the current working + /// directory (inclusive). Symlinks are allowed. When `project_doc_max_bytes` + /// is zero, returns an empty list. + async fn agents_md_paths( + &self, + fs: &dyn ExecutorFileSystem, + ) -> io::Result> { + if self.config.project_doc_max_bytes == 0 { + return Ok(Vec::new()); + } + + let mut dir = self.config.cwd.clone(); + if let Ok(canon) = normalize_path(&dir) { + dir = AbsolutePathBuf::try_from(canon)?; + } + + let mut merged = TomlValue::Table(toml::map::Map::new()); + for layer in self.config.config_layer_stack.get_layers( + ConfigLayerStackOrdering::LowestPrecedenceFirst, + /*include_disabled*/ false, + ) { + if matches!(layer.name, ConfigLayerSource::Project { .. }) { + continue; + } + merge_toml_values(&mut merged, &layer.config); + } + let project_root_markers = match project_root_markers_from_config(&merged) { + Ok(Some(markers)) => markers, + Ok(None) => default_project_root_markers(), + Err(err) => { + tracing::warn!("invalid project_root_markers: {err}"); + default_project_root_markers() + } + }; + let mut project_root = None; + if !project_root_markers.is_empty() { + for ancestor in dir.ancestors() { + for marker in &project_root_markers { + let marker_path = ancestor.join(marker); + let marker_exists = match fs.get_metadata(&marker_path, /*sandbox*/ None).await + { + Ok(_) => true, + Err(err) if err.kind() == io::ErrorKind::NotFound => false, + Err(err) => return Err(err), + }; + if marker_exists { + project_root = Some(ancestor.clone()); + break; + } + } + if project_root.is_some() { + break; + } + } + } + + let search_dirs: Vec = if let Some(root) = project_root { + let mut dirs = Vec::new(); + let mut cursor = dir.clone(); + loop { + dirs.push(cursor.clone()); + if cursor == root { + break; + } + let Some(parent) = cursor.parent() else { + break; + }; + cursor = parent; + } + dirs.reverse(); + dirs + } else { + vec![dir] + }; + + let mut found: Vec = Vec::new(); + let candidate_filenames = self.candidate_filenames(); + for d in search_dirs { + for name in &candidate_filenames { + let candidate = d.join(name); + match fs.get_metadata(&candidate, /*sandbox*/ None).await { + Ok(md) if md.is_file => { + found.push(candidate); + break; + } + Ok(_) => {} + Err(err) if err.kind() == io::ErrorKind::NotFound => continue, + Err(err) => return Err(err), + } + } + } + + Ok(found) + } + + fn candidate_filenames(&self) -> Vec<&str> { + let mut names: Vec<&str> = + Vec::with_capacity(2 + self.config.project_doc_fallback_filenames.len()); + names.push(LOCAL_AGENTS_MD_FILENAME); + names.push(DEFAULT_AGENTS_MD_FILENAME); + for candidate in &self.config.project_doc_fallback_filenames { + let candidate = candidate.as_str(); + if candidate.is_empty() { + continue; + } + if !names.contains(&candidate) { + names.push(candidate); + } + } + names + } +} + +#[cfg(test)] +#[path = "agents_md_tests.rs"] +mod tests; diff --git a/codex-rs/core/src/project_doc_tests.rs b/codex-rs/core/src/agents_md_tests.rs similarity index 70% rename from codex-rs/core/src/project_doc_tests.rs rename to codex-rs/core/src/agents_md_tests.rs index 705801317e..012724b43e 100644 --- a/codex-rs/core/src/project_doc_tests.rs +++ b/codex-rs/core/src/agents_md_tests.rs @@ -11,11 +11,15 @@ use std::path::PathBuf; use tempfile::TempDir; async fn get_user_instructions(config: &Config) -> Option { - super::get_user_instructions_with_fs(config, LOCAL_FS.as_ref()).await + AgentsMdManager::new(config) + .user_instructions_with_fs(LOCAL_FS.as_ref()) + .await } -async fn discover_project_doc_paths(config: &Config) -> std::io::Result> { - super::discover_project_doc_paths(config, LOCAL_FS.as_ref()).await +async fn agents_md_paths(config: &Config) -> std::io::Result> { + AgentsMdManager::new(config) + .agents_md_paths(LOCAL_FS.as_ref()) + .await } /// Helper that returns a `Config` pointing at `root` and using `limit` as @@ -101,7 +105,9 @@ async fn no_environment_returns_none() { let tmp = tempfile::tempdir().expect("tempdir"); let config = make_config(&tmp, /*limit*/ 4096, Some("user instructions")).await; - let res = super::get_user_instructions(&config, /*environment*/ None).await; + let res = AgentsMdManager::new(&config) + .user_instructions(/*environment*/ None) + .await; assert_eq!(res, None); } @@ -187,10 +193,9 @@ async fn zero_byte_limit_disables_discovery() { let tmp = tempfile::tempdir().expect("tempdir"); fs::write(tmp.path().join("AGENTS.md"), "something").unwrap(); - let discovery = - discover_project_doc_paths(&make_config(&tmp, /*limit*/ 0, /*instructions*/ None).await) - .await - .expect("discover paths"); + let discovery = agents_md_paths(&make_config(&tmp, /*limit*/ 0, /*instructions*/ None).await) + .await + .expect("discover paths"); assert_eq!(discovery, Vec::::new()); } @@ -205,7 +210,7 @@ async fn js_repl_instructions_are_appended_when_enabled() { let res = get_user_instructions(&cfg) .await .expect("js_repl instructions expected"); - let expected = "## JavaScript REPL (Node)\n- Use `js_repl` for Node-backed JavaScript with top-level await in a persistent kernel.\n- `js_repl` is a freeform/custom tool. Direct `js_repl` calls must send raw JavaScript tool input (optionally with first-line `// codex-js-repl: timeout_ms=15000`). Do not wrap code in JSON (for example `{\"code\":\"...\"}`), quotes, or markdown code fences.\n- Helpers: `codex.cwd`, `codex.homeDir`, `codex.tmpDir`, `codex.tool(name, args?)`, and `codex.emitImage(imageLike)`.\n- `codex.tool` executes a normal tool call and resolves to the raw tool output object. Use it for shell and non-shell tools alike. Nested tool outputs stay inside JavaScript unless you emit them explicitly.\n- `codex.emitImage(...)` adds one image to the outer `js_repl` function output each time you call it, so you can call it multiple times to emit multiple images. It accepts a data URL, a single `input_image` item, an object like `{ bytes, mimeType }`, or a raw tool response object with exactly one image and no text. It rejects mixed text-and-image content.\n- `codex.tool(...)` and `codex.emitImage(...)` keep stable helper identities across cells. Saved references and persisted objects can reuse them in later cells, but async callbacks that fire after a cell finishes still fail because no exec is active.\n- Request full-resolution image processing with `detail: \"original\"` only when the `view_image` tool schema includes a `detail` argument. The same availability applies to `codex.emitImage(...)`: if `view_image.detail` is present, you may also pass `detail: \"original\"` there. Use this when high-fidelity image perception or precise localization is needed, especially for CUA agents.\n- Example of sharing an in-memory Playwright screenshot: `await codex.emitImage({ bytes: await page.screenshot({ type: \"jpeg\", quality: 85 }), mimeType: \"image/jpeg\", detail: \"original\" })`.\n- Example of sharing a local image tool result: `await codex.emitImage(codex.tool(\"view_image\", { path: \"/absolute/path\", detail: \"original\" }))`.\n- When encoding an image to send with `codex.emitImage(...)` or `view_image`, prefer JPEG at about 85 quality when lossy compression is acceptable; use PNG when transparency or lossless detail matters. Smaller uploads are faster and less likely to hit size limits.\n- Top-level bindings persist across cells. If a cell throws, prior bindings remain available and bindings that finished initializing before the throw often remain usable in later cells. For code you plan to reuse across cells, prefer declaring or assigning it in direct top-level statements before operations that might throw. If you hit `SyntaxError: Identifier 'x' has already been declared`, first reuse the existing binding, reassign a previously declared `let`, or pick a new descriptive name. Use `{ ... }` only for a short temporary block when you specifically need local scratch names; do not wrap an entire cell in block scope if you want those names reusable later. Reset the kernel with `js_repl_reset` only when you need a clean state.\n- Top-level static import declarations (for example `import x from \"./file.js\"`) are currently unsupported in `js_repl`; use dynamic imports with `await import(\"pkg\")`, `await import(\"./file.js\")`, or `await import(\"/abs/path/file.mjs\")` instead. Imported local files must be ESM `.js`/`.mjs` files and run in the same REPL VM context. Bare package imports always resolve from REPL-global search roots (`CODEX_JS_REPL_NODE_MODULE_DIRS`, then cwd), not relative to the imported file location. Local files may statically import only other local relative/absolute/`file://` `.js`/`.mjs` files; package and builtin imports from local files must stay dynamic. `import.meta.resolve()` returns importable strings such as `file://...`, bare package names, and `node:...` specifiers. Local file modules reload between execs, while top-level bindings persist until `js_repl_reset`.\n- Avoid direct access to `process.stdout` / `process.stderr` / `process.stdin`; it can corrupt the JSON line protocol. Use `console.log`, `codex.tool(...)`, and `codex.emitImage(...)`."; + let expected = "## JavaScript REPL (Node)\n- Use `js_repl` for Node-backed JavaScript with top-level await in a persistent kernel.\n- `js_repl` is a freeform/custom tool. Direct `js_repl` calls must send raw JavaScript tool input (optionally with first-line `// codex-js-repl: timeout_ms=15000`). Do not wrap code in JSON (for example `{\"code\":\"...\"}`), quotes, or markdown code fences.\n- Helpers: `codex.cwd`, `codex.homeDir`, `codex.tmpDir`, `codex.tool(name, args?)`, and `codex.emitImage(imageLike)`.\n- `codex.tool` executes a normal tool call and resolves to the raw tool output object. Use it for shell and non-shell tools alike. Nested tool outputs stay inside JavaScript unless you emit them explicitly.\n- `codex.emitImage(...)` adds one image to the outer `js_repl` function output each time you call it, so you can call it multiple times to emit multiple images. It accepts a data URL, a single `input_image` item, an object like `{ bytes, mimeType }`, or a raw tool response object with exactly one image and no text. It rejects mixed text-and-image content.\n- `codex.tool(...)` and `codex.emitImage(...)` keep stable helper identities across cells. Saved references and persisted objects can reuse them in later cells, but async callbacks that fire after a cell finishes still fail because no exec is active.\n- Request full-resolution image processing with `detail: \"original\"` only when the `view_image` tool schema includes a `detail` argument. The same availability applies to `codex.emitImage(...)`: if `view_image.detail` is present, you may also pass `detail: \"original\"` there. Use this when high-fidelity image perception or precise localization is needed, especially for CUA agents.\n- Raw MCP image blocks can request the same behavior by returning `_meta: { \"codex/imageDetail\": \"original\" }` on the image content item.\n- Example of sharing an in-memory Playwright screenshot: `await codex.emitImage({ bytes: await page.screenshot({ type: \"jpeg\", quality: 85 }), mimeType: \"image/jpeg\", detail: \"original\" })`.\n- Example of sharing a local image tool result: `await codex.emitImage(codex.tool(\"view_image\", { path: \"/absolute/path\", detail: \"original\" }))`.\n- When encoding an image to send with `codex.emitImage(...)` or `view_image`, prefer JPEG at about 85 quality when lossy compression is acceptable; use PNG when transparency or lossless detail matters. Smaller uploads are faster and less likely to hit size limits.\n- Top-level bindings persist across cells. If a cell throws, prior bindings remain available and bindings that finished initializing before the throw often remain usable in later cells. For code you plan to reuse across cells, prefer declaring or assigning it in direct top-level statements before operations that might throw. If you hit `SyntaxError: Identifier 'x' has already been declared`, first reuse the existing binding, reassign a previously declared `let`, or pick a new descriptive name. Use `{ ... }` only for a short temporary block when you specifically need local scratch names; do not wrap an entire cell in block scope if you want those names reusable later. Reset the kernel with `js_repl_reset` only when you need a clean state.\n- Top-level static import declarations (for example `import x from \"./file.js\"`) are currently unsupported in `js_repl`; use dynamic imports with `await import(\"pkg\")`, `await import(\"./file.js\")`, or `await import(\"/abs/path/file.mjs\")` instead. Imported local files must be ESM `.js`/`.mjs` files and run in the same REPL VM context. Bare package imports always resolve from REPL-global search roots (`CODEX_JS_REPL_NODE_MODULE_DIRS`, then cwd), not relative to the imported file location. Local files may statically import only other local relative/absolute/`file://` `.js`/`.mjs` files; package and builtin imports from local files must stay dynamic. `import.meta.resolve()` returns importable strings such as `file://...`, bare package names, and `node:...` specifiers. Local file modules reload between execs, while top-level bindings persist until `js_repl_reset`.\n- Avoid direct access to `process.stdout` / `process.stderr` / `process.stdin`; it can corrupt the JSON line protocol. Use `console.log`, `codex.tool(...)`, and `codex.emitImage(...)`."; assert_eq!(res, expected); } @@ -224,14 +229,14 @@ async fn js_repl_tools_only_instructions_are_feature_gated() { let res = get_user_instructions(&cfg) .await .expect("js_repl instructions expected"); - let expected = "## JavaScript REPL (Node)\n- Use `js_repl` for Node-backed JavaScript with top-level await in a persistent kernel.\n- `js_repl` is a freeform/custom tool. Direct `js_repl` calls must send raw JavaScript tool input (optionally with first-line `// codex-js-repl: timeout_ms=15000`). Do not wrap code in JSON (for example `{\"code\":\"...\"}`), quotes, or markdown code fences.\n- Helpers: `codex.cwd`, `codex.homeDir`, `codex.tmpDir`, `codex.tool(name, args?)`, and `codex.emitImage(imageLike)`.\n- `codex.tool` executes a normal tool call and resolves to the raw tool output object. Use it for shell and non-shell tools alike. Nested tool outputs stay inside JavaScript unless you emit them explicitly.\n- `codex.emitImage(...)` adds one image to the outer `js_repl` function output each time you call it, so you can call it multiple times to emit multiple images. It accepts a data URL, a single `input_image` item, an object like `{ bytes, mimeType }`, or a raw tool response object with exactly one image and no text. It rejects mixed text-and-image content.\n- `codex.tool(...)` and `codex.emitImage(...)` keep stable helper identities across cells. Saved references and persisted objects can reuse them in later cells, but async callbacks that fire after a cell finishes still fail because no exec is active.\n- Request full-resolution image processing with `detail: \"original\"` only when the `view_image` tool schema includes a `detail` argument. The same availability applies to `codex.emitImage(...)`: if `view_image.detail` is present, you may also pass `detail: \"original\"` there. Use this when high-fidelity image perception or precise localization is needed, especially for CUA agents.\n- Example of sharing an in-memory Playwright screenshot: `await codex.emitImage({ bytes: await page.screenshot({ type: \"jpeg\", quality: 85 }), mimeType: \"image/jpeg\", detail: \"original\" })`.\n- Example of sharing a local image tool result: `await codex.emitImage(codex.tool(\"view_image\", { path: \"/absolute/path\", detail: \"original\" }))`.\n- When encoding an image to send with `codex.emitImage(...)` or `view_image`, prefer JPEG at about 85 quality when lossy compression is acceptable; use PNG when transparency or lossless detail matters. Smaller uploads are faster and less likely to hit size limits.\n- Top-level bindings persist across cells. If a cell throws, prior bindings remain available and bindings that finished initializing before the throw often remain usable in later cells. For code you plan to reuse across cells, prefer declaring or assigning it in direct top-level statements before operations that might throw. If you hit `SyntaxError: Identifier 'x' has already been declared`, first reuse the existing binding, reassign a previously declared `let`, or pick a new descriptive name. Use `{ ... }` only for a short temporary block when you specifically need local scratch names; do not wrap an entire cell in block scope if you want those names reusable later. Reset the kernel with `js_repl_reset` only when you need a clean state.\n- Top-level static import declarations (for example `import x from \"./file.js\"`) are currently unsupported in `js_repl`; use dynamic imports with `await import(\"pkg\")`, `await import(\"./file.js\")`, or `await import(\"/abs/path/file.mjs\")` instead. Imported local files must be ESM `.js`/`.mjs` files and run in the same REPL VM context. Bare package imports always resolve from REPL-global search roots (`CODEX_JS_REPL_NODE_MODULE_DIRS`, then cwd), not relative to the imported file location. Local files may statically import only other local relative/absolute/`file://` `.js`/`.mjs` files; package and builtin imports from local files must stay dynamic. `import.meta.resolve()` returns importable strings such as `file://...`, bare package names, and `node:...` specifiers. Local file modules reload between execs, while top-level bindings persist until `js_repl_reset`.\n- Do not call tools directly; use `js_repl` + `codex.tool(...)` for all tool calls, including shell commands.\n- MCP tools (if any) can also be called by name via `codex.tool(...)`.\n- Avoid direct access to `process.stdout` / `process.stderr` / `process.stdin`; it can corrupt the JSON line protocol. Use `console.log`, `codex.tool(...)`, and `codex.emitImage(...)`."; + let expected = "## JavaScript REPL (Node)\n- Use `js_repl` for Node-backed JavaScript with top-level await in a persistent kernel.\n- `js_repl` is a freeform/custom tool. Direct `js_repl` calls must send raw JavaScript tool input (optionally with first-line `// codex-js-repl: timeout_ms=15000`). Do not wrap code in JSON (for example `{\"code\":\"...\"}`), quotes, or markdown code fences.\n- Helpers: `codex.cwd`, `codex.homeDir`, `codex.tmpDir`, `codex.tool(name, args?)`, and `codex.emitImage(imageLike)`.\n- `codex.tool` executes a normal tool call and resolves to the raw tool output object. Use it for shell and non-shell tools alike. Nested tool outputs stay inside JavaScript unless you emit them explicitly.\n- `codex.emitImage(...)` adds one image to the outer `js_repl` function output each time you call it, so you can call it multiple times to emit multiple images. It accepts a data URL, a single `input_image` item, an object like `{ bytes, mimeType }`, or a raw tool response object with exactly one image and no text. It rejects mixed text-and-image content.\n- `codex.tool(...)` and `codex.emitImage(...)` keep stable helper identities across cells. Saved references and persisted objects can reuse them in later cells, but async callbacks that fire after a cell finishes still fail because no exec is active.\n- Request full-resolution image processing with `detail: \"original\"` only when the `view_image` tool schema includes a `detail` argument. The same availability applies to `codex.emitImage(...)`: if `view_image.detail` is present, you may also pass `detail: \"original\"` there. Use this when high-fidelity image perception or precise localization is needed, especially for CUA agents.\n- Raw MCP image blocks can request the same behavior by returning `_meta: { \"codex/imageDetail\": \"original\" }` on the image content item.\n- Example of sharing an in-memory Playwright screenshot: `await codex.emitImage({ bytes: await page.screenshot({ type: \"jpeg\", quality: 85 }), mimeType: \"image/jpeg\", detail: \"original\" })`.\n- Example of sharing a local image tool result: `await codex.emitImage(codex.tool(\"view_image\", { path: \"/absolute/path\", detail: \"original\" }))`.\n- When encoding an image to send with `codex.emitImage(...)` or `view_image`, prefer JPEG at about 85 quality when lossy compression is acceptable; use PNG when transparency or lossless detail matters. Smaller uploads are faster and less likely to hit size limits.\n- Top-level bindings persist across cells. If a cell throws, prior bindings remain available and bindings that finished initializing before the throw often remain usable in later cells. For code you plan to reuse across cells, prefer declaring or assigning it in direct top-level statements before operations that might throw. If you hit `SyntaxError: Identifier 'x' has already been declared`, first reuse the existing binding, reassign a previously declared `let`, or pick a new descriptive name. Use `{ ... }` only for a short temporary block when you specifically need local scratch names; do not wrap an entire cell in block scope if you want those names reusable later. Reset the kernel with `js_repl_reset` only when you need a clean state.\n- Top-level static import declarations (for example `import x from \"./file.js\"`) are currently unsupported in `js_repl`; use dynamic imports with `await import(\"pkg\")`, `await import(\"./file.js\")`, or `await import(\"/abs/path/file.mjs\")` instead. Imported local files must be ESM `.js`/`.mjs` files and run in the same REPL VM context. Bare package imports always resolve from REPL-global search roots (`CODEX_JS_REPL_NODE_MODULE_DIRS`, then cwd), not relative to the imported file location. Local files may statically import only other local relative/absolute/`file://` `.js`/`.mjs` files; package and builtin imports from local files must stay dynamic. `import.meta.resolve()` returns importable strings such as `file://...`, bare package names, and `node:...` specifiers. Local file modules reload between execs, while top-level bindings persist until `js_repl_reset`.\n- Do not call tools directly; use `js_repl` + `codex.tool(...)` for all tool calls, including shell commands.\n- MCP tools (if any) can also be called by name via `codex.tool(...)`.\n- Avoid direct access to `process.stdout` / `process.stderr` / `process.stdin`; it can corrupt the JSON line protocol. Use `console.log`, `codex.tool(...)`, and `codex.emitImage(...)`."; assert_eq!(res, expected); } -/// When both system instructions *and* a project doc are present the two +/// When both system instructions and AGENTS.md docs are present the two /// should be concatenated with the separator. #[tokio::test] -async fn merges_existing_instructions_with_project_doc() { +async fn merges_existing_instructions_with_agents_md() { let tmp = tempfile::tempdir().expect("tempdir"); fs::write(tmp.path().join("AGENTS.md"), "proj doc").unwrap(); @@ -241,12 +246,12 @@ async fn merges_existing_instructions_with_project_doc() { .await .expect("should produce a combined instruction string"); - let expected = format!("{INSTRUCTIONS}{PROJECT_DOC_SEPARATOR}{}", "proj doc"); + let expected = format!("{INSTRUCTIONS}{AGENTS_MD_SEPARATOR}{}", "proj doc"); assert_eq!(res, expected); } -/// If there are existing system instructions but the project doc is +/// If there are existing system instructions but AGENTS.md docs are /// missing we expect the original instructions to be returned unchanged. #[tokio::test] async fn keeps_existing_instructions_when_doc_missing() { @@ -307,9 +312,7 @@ async fn project_root_markers_are_honored_for_agents_discovery() { .await; cfg.cwd = nested.abs(); - let discovery = discover_project_doc_paths(&cfg) - .await - .expect("discover paths"); + let discovery = agents_md_paths(&cfg).await.expect("discover paths"); let expected_parent = AbsolutePathBuf::try_from( dunce::canonicalize(root.path().join("AGENTS.md")).expect("canonical parent doc path"), ) @@ -326,12 +329,33 @@ async fn project_root_markers_are_honored_for_agents_discovery() { assert_eq!(res, "parent doc\n\nchild doc"); } +#[tokio::test] +async fn instruction_sources_include_global_before_agents_md_docs() { + let tmp = tempfile::tempdir().expect("tempdir"); + fs::write(tmp.path().join("AGENTS.md"), "project doc").unwrap(); + + let cfg = make_config(&tmp, /*limit*/ 4096, Some("global doc")).await; + let global_agents = cfg.codex_home.join(DEFAULT_AGENTS_MD_FILENAME); + fs::create_dir_all(&cfg.codex_home).unwrap(); + fs::write(&global_agents, "global doc").unwrap(); + + let sources = AgentsMdManager::new(&cfg) + .instruction_sources(LOCAL_FS.as_ref()) + .await; + let project_agents = AbsolutePathBuf::try_from( + dunce::canonicalize(cfg.cwd.join("AGENTS.md")).expect("canonical project doc path"), + ) + .expect("absolute project doc path"); + + assert_eq!(sources, vec![global_agents, project_agents]); +} + /// AGENTS.override.md is preferred over AGENTS.md when both are present. #[tokio::test] async fn agents_local_md_preferred() { let tmp = tempfile::tempdir().expect("tempdir"); - fs::write(tmp.path().join(DEFAULT_PROJECT_DOC_FILENAME), "versioned").unwrap(); - fs::write(tmp.path().join(LOCAL_PROJECT_DOC_FILENAME), "local").unwrap(); + fs::write(tmp.path().join(DEFAULT_AGENTS_MD_FILENAME), "versioned").unwrap(); + fs::write(tmp.path().join(LOCAL_AGENTS_MD_FILENAME), "local").unwrap(); let cfg = make_config(&tmp, /*limit*/ 4096, /*instructions*/ None).await; @@ -341,13 +365,11 @@ async fn agents_local_md_preferred() { assert_eq!(res, "local"); - let discovery = discover_project_doc_paths(&cfg) - .await - .expect("discover paths"); + let discovery = agents_md_paths(&cfg).await.expect("discover paths"); assert_eq!(discovery.len(), 1); assert_eq!( discovery[0].file_name().unwrap().to_string_lossy(), - LOCAL_PROJECT_DOC_FILENAME + LOCAL_AGENTS_MD_FILENAME ); } @@ -393,16 +415,14 @@ async fn agents_md_preferred_over_fallbacks() { assert_eq!(res, "primary"); - let discovery = discover_project_doc_paths(&cfg) - .await - .expect("discover paths"); + let discovery = agents_md_paths(&cfg).await.expect("discover paths"); assert_eq!(discovery.len(), 1); assert!( discovery[0] .file_name() .unwrap() .to_string_lossy() - .eq(DEFAULT_PROJECT_DOC_FILENAME) + .eq(DEFAULT_AGENTS_MD_FILENAME) ); } @@ -416,9 +436,7 @@ async fn agents_md_directory_is_ignored() { let res = get_user_instructions(&cfg).await; assert_eq!(res, None); - let discovery = discover_project_doc_paths(&cfg) - .await - .expect("discover paths"); + let discovery = agents_md_paths(&cfg).await.expect("discover paths"); assert_eq!(discovery, Vec::::new()); } @@ -441,17 +459,15 @@ async fn agents_md_special_file_is_ignored() { let res = get_user_instructions(&cfg).await; assert_eq!(res, None); - let discovery = discover_project_doc_paths(&cfg) - .await - .expect("discover paths"); + let discovery = agents_md_paths(&cfg).await.expect("discover paths"); assert_eq!(discovery, Vec::::new()); } #[tokio::test] async fn override_directory_falls_back_to_agents_md_file() { let tmp = tempfile::tempdir().expect("tempdir"); - fs::create_dir(tmp.path().join(LOCAL_PROJECT_DOC_FILENAME)).unwrap(); - fs::write(tmp.path().join(DEFAULT_PROJECT_DOC_FILENAME), "primary").unwrap(); + fs::create_dir(tmp.path().join(LOCAL_AGENTS_MD_FILENAME)).unwrap(); + fs::write(tmp.path().join(DEFAULT_AGENTS_MD_FILENAME), "primary").unwrap(); let cfg = make_config(&tmp, /*limit*/ 4096, /*instructions*/ None).await; @@ -460,21 +476,19 @@ async fn override_directory_falls_back_to_agents_md_file() { .expect("AGENTS.md should be used when override is a directory"); assert_eq!(res, "primary"); - let discovery = discover_project_doc_paths(&cfg) - .await - .expect("discover paths"); + let discovery = agents_md_paths(&cfg).await.expect("discover paths"); assert_eq!(discovery.len(), 1); assert_eq!( discovery[0] .file_name() .expect("file name") .to_string_lossy(), - DEFAULT_PROJECT_DOC_FILENAME + DEFAULT_AGENTS_MD_FILENAME ); } #[tokio::test] -async fn skills_are_not_appended_to_project_doc() { +async fn skills_are_not_appended_to_agents_md() { let tmp = tempfile::tempdir().expect("tempdir"); fs::write(tmp.path().join("AGENTS.md"), "base doc").unwrap(); @@ -504,7 +518,7 @@ async fn apps_feature_does_not_emit_user_instructions_by_itself() { } #[tokio::test] -async fn apps_feature_does_not_append_to_project_doc_user_instructions() { +async fn apps_feature_does_not_append_to_agents_md_user_instructions() { let tmp = tempfile::tempdir().expect("tempdir"); fs::write(tmp.path().join("AGENTS.md"), "base doc").unwrap(); diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 16f743943a..bd83c81f0d 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -32,6 +32,7 @@ use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; use codex_api::ApiError; +use codex_api::AuthProvider; use codex_api::CompactClient as ApiCompactClient; use codex_api::CompactionInput as ApiCompactionInput; use codex_api::Compression; @@ -83,7 +84,6 @@ use futures::StreamExt; use http::HeaderMap as ApiHeaderMap; use http::HeaderValue; use http::StatusCode as HttpStatusCode; -use http::header::AUTHORIZATION; use reqwest::StatusCode; use std::time::Duration; use std::time::Instant; @@ -277,16 +277,7 @@ pub(crate) struct RealtimeWebrtcCallStart { /// `api.openai.com` sideband path. fn sideband_websocket_auth_headers(api_auth: &CoreAuthProvider) -> ApiHeaderMap { let mut headers = ApiHeaderMap::new(); - if let Some(token) = api_auth.token.as_ref() - && let Ok(value) = HeaderValue::from_str(&format!("Bearer {token}")) - { - headers.insert(AUTHORIZATION, value); - } - if let Some(account_id) = api_auth.account_id.as_ref() - && let Ok(value) = HeaderValue::from_str(account_id) - { - headers.insert("ChatGPT-Account-ID", value); - } + api_auth.add_auth_headers(&mut headers); headers } diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 95e932b1dc..5274d88664 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -13,6 +13,7 @@ use crate::agent::Mailbox; use crate::agent::MailboxReceiver; use crate::agent::agent_status_from_event; use crate::agent::status::is_final; +use crate::agent_identity::AgentIdentityManager; use crate::apps::render_apps_section; use crate::commit_attribution::commit_message_trailer_instruction; use crate::compact; @@ -40,9 +41,11 @@ use crate::stream_events_utils::HandleOutputCtx; use crate::stream_events_utils::handle_non_tool_response_item; use crate::stream_events_utils::handle_output_item_done; use crate::stream_events_utils::last_assistant_message_from_item; +use crate::stream_events_utils::mark_thread_memory_mode_polluted_if_external_context; use crate::stream_events_utils::raw_assistant_output_text_from_item; use crate::stream_events_utils::record_completed_response_item; use crate::turn_metadata::TurnMetadataState; +use crate::unavailable_tool::collect_unavailable_called_tools; use crate::util::error_or_panic; use async_channel::Receiver; use async_channel::Sender; @@ -77,7 +80,6 @@ use codex_login::CodexAuth; use codex_login::auth_env_telemetry::collect_auth_env_telemetry; use codex_login::default_client::originator; use codex_mcp::McpConnectionManager; -use codex_mcp::SandboxState; use codex_mcp::ToolInfo; use codex_mcp::codex_apps_tools_cache_key; #[cfg(test)] @@ -91,6 +93,7 @@ use codex_otel::current_span_trace_id; use codex_otel::current_span_w3c_trace_context; use codex_otel::set_parent_from_w3c_trace_context; use codex_protocol::ThreadId; +use codex_protocol::ToolName; use codex_protocol::approvals::ElicitationRequestEvent; use codex_protocol::approvals::ExecPolicyAmendment; use codex_protocol::approvals::NetworkPolicyAmendment; @@ -134,9 +137,11 @@ use codex_protocol::request_permissions::RequestPermissionsResponse; use codex_protocol::request_user_input::RequestUserInputArgs; use codex_protocol::request_user_input::RequestUserInputResponse; use codex_rmcp_client::ElicitationResponse; +use codex_rollout::RolloutConfig; use codex_rollout::state_db; use codex_shell_command::parse_command::parse_command; use codex_terminal_detection::user_agent; +use codex_thread_store::LocalThreadStore; use codex_tools::filter_tool_suggest_discoverable_tools_for_client; use codex_utils_output_truncation::TruncationPolicy; use codex_utils_stream_parser::AssistantTextChunk; @@ -260,6 +265,7 @@ use crate::SkillInjections; use crate::SkillLoadOutcome; use crate::SkillMetadata; use crate::SkillsManager; +use crate::agents_md::AgentsMdManager; use crate::build_skill_injections; use crate::collect_env_var_dependencies; use crate::collect_explicit_skill_mentions; @@ -288,7 +294,6 @@ use crate::network_policy_decision::execpolicy_network_rule_amendment; use crate::plugins::PluginsManager; use crate::plugins::build_plugin_injections; use crate::plugins::render_plugins_section; -use crate::project_doc::get_user_instructions; use crate::resolve_skill_dependencies_for_turn; use crate::rollout::RolloutRecorder; use crate::rollout::RolloutRecorderParams; @@ -551,7 +556,9 @@ impl Codex { config.startup_warnings.push(message); } - let user_instructions = get_user_instructions(&config, environment.as_deref()).await; + let user_instructions = AgentsMdManager::new(&config) + .user_instructions(environment.as_deref()) + .await; let exec_policy = if crate::guardian::is_guardian_reviewer_source(&session_source) { // Guardian review should rely on the built-in shell safety checks, @@ -1078,6 +1085,18 @@ impl TurnContext { } pub(crate) fn to_turn_context_item(&self) -> TurnContextItem { + let legacy_file_system_sandbox_policy = FileSystemSandboxPolicy::from_legacy_sandbox_policy( + self.sandbox_policy.get(), + &self.cwd, + ); + // Omit the derived split filesystem policy when it is equivalent to + // the legacy sandbox policy. This keeps turn-context payloads stable + // while both fields exist; once callers consume only the split policy, + // this comparison and the legacy projection should go away. + let file_system_sandbox_policy = (self.file_system_sandbox_policy + != legacy_file_system_sandbox_policy) + .then(|| self.file_system_sandbox_policy.clone()); + TurnContextItem { turn_id: Some(self.sub_id.clone()), trace_id: self.trace_id.clone(), @@ -1087,6 +1106,7 @@ impl TurnContext { approval_policy: self.approval_policy.value(), sandbox_policy: self.sandbox_policy.get().clone(), network: self.turn_context_network_item(), + file_system_sandbox_policy, model: self.model_info.slug.clone(), personality: self.personality, collaboration_mode: Some(self.collaboration_mode.clone()), @@ -1282,7 +1302,14 @@ impl SessionConfiguration { let cwd_changed = absolute_cwd.as_path() != self.cwd.as_path(); next_configuration.cwd = absolute_cwd; - if sandbox_policy_changed || (cwd_changed && file_system_policy_matches_legacy) { + if sandbox_policy_changed { + next_configuration.file_system_sandbox_policy = + FileSystemSandboxPolicy::from_legacy_sandbox_policy_preserving_deny_entries( + next_configuration.sandbox_policy.get(), + &next_configuration.cwd, + &self.file_system_sandbox_policy, + ); + } else if cwd_changed && file_system_policy_matches_legacy { // Preserve richer split policies across cwd-only updates; only // rederive when the session is already using the legacy bridge. next_configuration.file_system_sandbox_policy = @@ -1334,6 +1361,10 @@ impl Session { } } + fn managed_network_proxy_active_for_sandbox_policy(sandbox_policy: &SandboxPolicy) -> bool { + !matches!(sandbox_policy, SandboxPolicy::DangerFullAccess) + } + /// Builds the `x-codex-beta-features` header value for this session. /// /// `ModelClient` is session-scoped and intentionally does not depend on the full `Config`, so @@ -1509,6 +1540,56 @@ impl Session { }); } + fn start_agent_identity_registration(self: &Arc) { + if !self.services.agent_identity_manager.is_enabled() { + return; + } + + let weak_sess = Arc::downgrade(self); + let mut auth_state_rx = self.services.auth_manager.subscribe_auth_state(); + tokio::spawn(async move { + loop { + let Some(sess) = weak_sess.upgrade() else { + return; + }; + match sess + .services + .agent_identity_manager + .ensure_registered_identity() + .await + { + Ok(Some(_)) => return, + Ok(None) => { + drop(sess); + if auth_state_rx.changed().await.is_err() { + return; + } + } + Err(error) => { + sess.fail_agent_identity_registration(error).await; + return; + } + } + } + }); + } + + async fn fail_agent_identity_registration(self: &Arc, error: anyhow::Error) { + warn!(error = %error, "agent identity registration failed"); + let message = format!( + "Agent identity registration failed. Codex cannot continue while `features.use_agent_identity` is enabled: {error}" + ); + self.send_event_raw(Event { + id: self.next_internal_sub_id(), + msg: EventMsg::Error(ErrorEvent { + message, + codex_error_info: Some(CodexErrorInfo::Other), + }), + }) + .await; + handlers::shutdown(self, self.next_internal_sub_id()).await; + } + #[allow(clippy::too_many_arguments)] fn make_turn_context( conversation_id: ThreadId, @@ -1950,10 +2031,15 @@ impl Session { .await; session_configuration.thread_name = thread_name.clone(); let state = SessionState::new(session_configuration.clone()); + let managed_network_requirements_configured = config + .config_layer_stack + .requirements_toml() + .network + .is_some(); let managed_network_requirements_enabled = config.managed_network_requirements_enabled(); let network_approval = Arc::new(NetworkApprovalService::default()); // The managed proxy can call back into core for allowlist-miss decisions. - let network_policy_decider_session = if managed_network_requirements_enabled { + let network_policy_decider_session = if managed_network_requirements_configured { config .permissions .network @@ -1962,7 +2048,7 @@ impl Session { } else { None }; - let blocked_request_observer = if managed_network_requirements_enabled { + let blocked_request_observer = if managed_network_requirements_configured { config .permissions .network @@ -1989,7 +2075,7 @@ impl Session { config.permissions.sandbox_policy.get(), network_policy_decider.as_ref().map(Arc::clone), blocked_request_observer.as_ref().map(Arc::clone), - managed_network_requirements_enabled, + managed_network_requirements_configured, network_proxy_audit_metadata, ) .instrument(info_span!( @@ -2054,6 +2140,11 @@ impl Session { hooks, rollout: Mutex::new(rollout_recorder), user_shell: Arc::new(default_shell), + agent_identity_manager: Arc::new(AgentIdentityManager::new( + config.as_ref(), + Arc::clone(&auth_manager), + session_configuration.session_source.clone(), + )), shell_snapshot_tx, show_raw_agent_reasoning: config.show_raw_agent_reasoning, exec_policy, @@ -2070,6 +2161,7 @@ impl Session { network_proxy, network_approval: Arc::clone(&network_approval), state_db: state_db_ctx.clone(), + thread_store: LocalThreadStore::new(RolloutConfig::from_view(config.as_ref())), model_client: ModelClient::new( Some(Arc::clone(&auth_manager)), conversation_id, @@ -2140,7 +2232,11 @@ impl Session { history_log_id, history_entry_count, initial_messages, - network_proxy: session_network_proxy, + network_proxy: session_network_proxy.filter(|_| { + Self::managed_network_proxy_active_for_sandbox_policy( + session_configuration.sandbox_policy.get(), + ) + }), rollout_path, }), }) @@ -2151,14 +2247,7 @@ impl Session { // Start the watcher after SessionConfigured so it cannot emit earlier events. sess.start_skills_watcher_listener(); - // Construct sandbox_state before MCP startup so it can be sent to each - // MCP server immediately after it becomes ready (avoiding blocking). - let sandbox_state = SandboxState { - sandbox_policy: session_configuration.sandbox_policy.get().clone(), - codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(), - sandbox_cwd: session_configuration.cwd.to_path_buf(), - use_legacy_landlock: config.features.use_legacy_landlock(), - }; + sess.start_agent_identity_registration(); let mut required_mcp_servers: Vec = mcp_servers .iter() .filter(|(_, server)| server.enabled && server.required) @@ -2180,7 +2269,7 @@ impl Session { &session_configuration.approval_policy, INITIAL_SUBMIT_ID.to_owned(), tx_event.clone(), - sandbox_state, + session_configuration.sandbox_policy.get().clone(), config.codex_home.to_path_buf(), codex_apps_tools_cache_key(auth), tool_plugin_provenance, @@ -2612,12 +2701,16 @@ impl Session { &session_source, ); + if sandbox_policy_changed { + self.refresh_managed_network_proxy_for_current_sandbox_policy() + .await; + } + Ok(self .new_turn_from_configuration( sub_id, session_configuration, updates.final_output_json_schema, - sandbox_policy_changed, ) .await) } @@ -2627,7 +2720,6 @@ impl Session { sub_id: String, session_configuration: SessionConfiguration, final_output_json_schema: Option>, - sandbox_policy_changed: bool, ) -> Arc { let per_turn_config = Self::build_per_turn_config(&session_configuration); { @@ -2637,27 +2729,6 @@ impl Session { .set_sandbox_policy(per_turn_config.permissions.sandbox_policy.get()); } - if sandbox_policy_changed { - self.refresh_managed_network_proxy_for_current_sandbox_policy() - .await; - let sandbox_state = SandboxState { - sandbox_policy: per_turn_config.permissions.sandbox_policy.get().clone(), - codex_linux_sandbox_exe: per_turn_config.codex_linux_sandbox_exe.clone(), - sandbox_cwd: per_turn_config.cwd.to_path_buf(), - use_legacy_landlock: per_turn_config.features.use_legacy_landlock(), - }; - if let Err(e) = self - .services - .mcp_connection_manager - .read() - .await - .notify_sandbox_state_change(&sandbox_state) - .await - { - warn!("Failed to notify sandbox state change to MCP servers: {e:#}"); - } - } - let model_info = self .services .models_manager @@ -2699,7 +2770,12 @@ impl Session { self.services .network_proxy .as_ref() - .map(StartedNetworkProxy::proxy), + .and_then(|started_proxy| { + Self::managed_network_proxy_active_for_sandbox_policy( + session_configuration.sandbox_policy.get(), + ) + .then(|| started_proxy.proxy()) + }), self.services.environment.clone(), sub_id, Arc::clone(&self.js_repl), @@ -2806,7 +2882,6 @@ impl Session { sub_id, session_configuration, /*final_output_json_schema*/ None, - /*sandbox_policy_changed*/ false, ) .await } @@ -4453,16 +4528,12 @@ impl Session { .await } - pub(crate) async fn resolve_mcp_tool_info( - &self, - name: &str, - namespace: Option<&str>, - ) -> Option { + pub(crate) async fn resolve_mcp_tool_info(&self, tool_name: &ToolName) -> Option { self.services .mcp_connection_manager .read() .await - .resolve_tool_info(name, namespace) + .resolve_tool_info(tool_name) .await } @@ -4522,12 +4593,6 @@ impl Session { .await; let mcp_servers = with_codex_apps_mcp(mcp_servers, auth.as_ref(), &mcp_config); let auth_statuses = compute_auth_statuses(mcp_servers.iter(), store_mode).await; - let sandbox_state = SandboxState { - sandbox_policy: turn_context.sandbox_policy.get().clone(), - codex_linux_sandbox_exe: turn_context.codex_linux_sandbox_exe.clone(), - sandbox_cwd: turn_context.cwd.to_path_buf(), - use_legacy_landlock: turn_context.features.use_legacy_landlock(), - }; { let mut guard = self.services.mcp_startup_cancellation_token.lock().await; guard.cancel(); @@ -4540,7 +4605,7 @@ impl Session { &turn_context.config.permissions.approval_policy, turn_context.sub_id.clone(), self.get_tx_event(), - sandbox_state, + turn_context.sandbox_policy.get().clone(), config.codex_home.to_path_buf(), codex_apps_tools_cache_key(auth.as_ref()), tool_plugin_provenance, @@ -4920,6 +4985,8 @@ mod handlers { use crate::config_loader::load_config_layers_state; use crate::realtime_context::REALTIME_TURN_TOKEN_BUDGET; use crate::realtime_context::truncate_realtime_text_to_token_budget; + use crate::realtime_conversation::REALTIME_USER_TEXT_PREFIX; + use crate::realtime_conversation::prefix_realtime_v2_text; use codex_features::Feature; use codex_utils_absolute_path::AbsolutePathBuf; @@ -5140,6 +5207,11 @@ mod handlers { if text.is_empty() { return; } + let text = if sess.conversation.is_running_v2().await { + prefix_realtime_v2_text(text, REALTIME_USER_TEXT_PREFIX) + } else { + text + }; let text = truncate_realtime_text_to_token_budget(&text, REALTIME_TURN_TOKEN_BUDGET); if text.is_empty() { return; @@ -5518,15 +5590,15 @@ mod handlers { errors.push("state db unavailable; memory rows were not cleared".to_string()); } - let memory_root = crate::memories::memory_root(&config.codex_home); - if let Err(err) = crate::memories::clear_memory_root_contents(&memory_root).await { + if let Err(err) = crate::memories::clear_memory_roots_contents(&config.codex_home).await { errors.push(format!( - "failed clearing memory directory {}: {err}", - memory_root.display() + "failed clearing memory directories under {}: {err}", + config.codex_home.display() )); } if errors.is_empty() { + let memory_root = crate::memories::memory_root(&config.codex_home); sess.send_event_raw(Event { id: sub_id, msg: EventMsg::Warning(WarningEvent { @@ -6195,8 +6267,11 @@ pub(crate) async fn run_turn( HashMap::new() }; let available_connectors = if turn_context.apps_enabled() { - let connectors = connectors::merge_plugin_apps_with_accessible( - loaded_plugins.effective_apps(), + let connectors = codex_connectors::merge::merge_plugin_connectors_with_accessible( + loaded_plugins + .effective_apps() + .into_iter() + .map(|connector_id| connector_id.0), connectors::accessible_connectors_from_mcp_tools(&mcp_tools), ); connectors::with_app_enabled_state(connectors, &turn_context.config) @@ -6842,7 +6917,7 @@ fn collect_explicit_app_ids_from_skill_items( let connector_slug_counts = build_connector_slug_counts(connectors); for connector in connectors { - let slug = connectors::connector_mention_slug(connector); + let slug = codex_connectors::metadata::connector_mention_slug(connector); let connector_count = connector_slug_counts.get(&slug).copied().unwrap_or(0); let skill_count = skill_name_counts_lower.get(&slug).copied().unwrap_or(0); if connector_count == 1 && skill_count == 0 && mention_names_lower.contains(&slug) { @@ -6917,7 +6992,7 @@ fn connector_inserted_in_messages( return true; } - let mention_slug = connectors::connector_mention_slug(connector); + let mention_slug = codex_connectors::metadata::connector_mention_slug(connector); let connector_count = connector_slug_counts .get(&mention_slug) .copied() @@ -7145,8 +7220,11 @@ pub(crate) async fn built_tools( connectors::with_app_enabled_state(connectors.clone(), &turn_context.config) }); let connectors = if apps_enabled { - let connectors = connectors::merge_plugin_apps_with_accessible( - loaded_plugins.effective_apps(), + let connectors = codex_connectors::merge::merge_plugin_connectors_with_accessible( + loaded_plugins + .effective_apps() + .into_iter() + .map(|connector_id| connector_id.0), accessible_connectors.clone().unwrap_or_default(), ); Some(connectors::with_app_enabled_state( @@ -7206,7 +7284,22 @@ pub(crate) async fn built_tools( &turn_context.config, &turn_context.tools_config, ); - let direct_mcp_tools = has_mcp_servers.then_some(mcp_tool_exposure.direct_tools); + let mcp_tools = has_mcp_servers.then_some(mcp_tool_exposure.direct_tools); + let deferred_mcp_tools = mcp_tool_exposure.deferred_tools; + let unavailable_called_tools = if turn_context + .config + .features + .enabled(Feature::UnavailableDummyTools) + { + let exposed_tool_names = mcp_tools + .iter() + .chain(deferred_mcp_tools.iter()) + .flat_map(|tools| tools.keys().map(String::as_str)) + .collect::>(); + collect_unavailable_called_tools(input, &exposed_tool_names) + } else { + Vec::new() + }; let parallel_mcp_server_names = turn_context .config @@ -7223,8 +7316,9 @@ pub(crate) async fn built_tools( Ok(Arc::new(ToolRouter::from_config( &turn_context.tools_config, ToolRouterParams { - mcp_tools: direct_mcp_tools, - deferred_mcp_tools: mcp_tool_exposure.deferred_tools, + mcp_tools, + deferred_mcp_tools, + unavailable_called_tools, parallel_mcp_server_names, discoverable_tools, dynamic_tools: turn_context.dynamic_tools.as_slice(), @@ -7765,8 +7859,15 @@ async fn drain_in_flight( while let Some(res) = in_flight.next().await { match res { Ok(response_input) => { - sess.record_conversation_items(&turn_context, &[response_input.into()]) + let response_item = response_input.into(); + sess.record_conversation_items(&turn_context, std::slice::from_ref(&response_item)) .await; + mark_thread_memory_mode_polluted_if_external_context( + sess.as_ref(), + turn_context.as_ref(), + &response_item, + ) + .await; } Err(err) => { error_or_panic(format!("in-flight tool future failed during drain: {err}")); diff --git a/codex-rs/core/src/codex/rollout_reconstruction_tests.rs b/codex-rs/core/src/codex/rollout_reconstruction_tests.rs index 753244ac2b..432cfa7a40 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction_tests.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction_tests.rs @@ -68,6 +68,7 @@ async fn record_initial_history_resumed_bare_turn_context_does_not_hydrate_previ approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: previous_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), @@ -107,6 +108,7 @@ async fn record_initial_history_resumed_hydrates_previous_turn_settings_from_lif approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: previous_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), @@ -901,6 +903,7 @@ async fn record_initial_history_resumed_turn_context_after_compaction_reestablis approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: previous_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), @@ -976,6 +979,7 @@ async fn record_initial_history_resumed_turn_context_after_compaction_reestablis approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: previous_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), @@ -1005,6 +1009,7 @@ async fn record_initial_history_resumed_aborted_turn_without_id_clears_active_tu approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: previous_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), @@ -1117,6 +1122,7 @@ async fn record_initial_history_resumed_unmatched_abort_preserves_active_turn_fo approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: current_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), @@ -1227,6 +1233,7 @@ async fn record_initial_history_resumed_trailing_incomplete_turn_compaction_clea approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: previous_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), @@ -1376,6 +1383,7 @@ async fn record_initial_history_resumed_replaced_incomplete_compacted_turn_clear approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: previous_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), diff --git a/codex-rs/core/src/codex_tests.rs b/codex-rs/core/src/codex_tests.rs index 8a0b7cb232..73f3669e60 100644 --- a/codex-rs/core/src/codex_tests.rs +++ b/codex-rs/core/src/codex_tests.rs @@ -45,6 +45,8 @@ use crate::rollout::recorder::RolloutRecorder; use crate::state::TaskKind; use crate::tasks::SessionTask; use crate::tasks::SessionTaskContext; +use crate::tasks::UserShellCommandMode; +use crate::tasks::execute_user_shell_command; use crate::tools::ToolRouter; use crate::tools::context::ToolInvocation; use crate::tools::context::ToolPayload; @@ -110,6 +112,7 @@ use opentelemetry::trace::TraceId; use std::path::Path; use std::time::Duration; use tokio::time::sleep; +use tokio::time::timeout; use tracing_opentelemetry::OpenTelemetrySpanExt; use codex_protocol::mcp::CallToolResult as McpCallToolResult; @@ -312,6 +315,7 @@ fn test_tool_runtime(session: Arc, turn_context: Arc) -> T crate::tools::router::ToolRouterParams { mcp_tools: None, deferred_mcp_tools: None, + unavailable_called_tools: Vec::new(), parallel_mcp_server_names: HashSet::new(), discoverable_tools: None, dynamic_tools: turn_context.dynamic_tools.as_slice(), @@ -409,7 +413,7 @@ fn make_mcp_tool( ) -> ToolInfo { let tool_namespace = if server_name == CODEX_APPS_MCP_SERVER_NAME { connector_name - .map(crate::connectors::sanitize_name) + .map(codex_connectors::metadata::sanitize_name) .map(|connector_name| format!("mcp__{server_name}__{connector_name}")) .unwrap_or_else(|| server_name.to_string()) } else { @@ -453,8 +457,8 @@ fn numbered_mcp_tools(count: usize) -> HashMap { .collect() } -fn tools_config_for_mcp_tool_exposure(search_tool: bool) -> ToolsConfig { - let config = test_config(); +async fn tools_config_for_mcp_tool_exposure(search_tool: bool) -> ToolsConfig { + let config = test_config().await; let model_info = ModelsManager::construct_model_info_offline_for_tests( "gpt-5-codex", &config.to_models_manager_config(), @@ -655,6 +659,293 @@ async fn managed_network_proxy_decider_survives_full_access_start() -> anyhow::R Ok(()) } +#[tokio::test] +async fn new_turn_refreshes_managed_network_proxy_for_sandbox_change() -> anyhow::Result<()> { + let (mut session, _turn_context) = make_session_and_context().await; + let initial_policy = SandboxPolicy::new_workspace_write_policy(); + + let mut network_config = NetworkProxyConfig::default(); + network_config + .network + .set_allowed_domains(vec!["evil.com".to_string()]); + let requirements = NetworkConstraints { + domains: Some(NetworkDomainPermissionsToml { + entries: std::collections::BTreeMap::from([( + "*.example.com".to_string(), + NetworkDomainPermissionToml::Allow, + )]), + }), + ..Default::default() + }; + let spec = crate::config::NetworkProxySpec::from_config_and_constraints( + network_config, + Some(requirements), + &initial_policy, + )?; + let (started_proxy, _) = Session::start_managed_network_proxy( + &spec, + &Policy::empty(), + &initial_policy, + /*network_policy_decider*/ None, + /*blocked_request_observer*/ None, + /*managed_network_requirements_enabled*/ false, + crate::config::NetworkProxyAuditMetadata::default(), + ) + .await?; + assert_eq!( + started_proxy + .proxy() + .current_cfg() + .await? + .network + .allowed_domains(), + Some(vec!["*.example.com".to_string(), "evil.com".to_string()]) + ); + + { + let mut state = session.state.lock().await; + let mut config = (*state.session_configuration.original_config_do_not_use).clone(); + config.permissions.network = Some(spec); + config.permissions.sandbox_policy = + codex_config::Constrained::allow_any(initial_policy.clone()); + state.session_configuration.original_config_do_not_use = Arc::new(config); + state.session_configuration.sandbox_policy = + codex_config::Constrained::allow_any(initial_policy); + } + session.services.network_proxy = Some(started_proxy); + + session + .new_turn_with_sub_id( + "sandbox-policy-change".to_string(), + SessionSettingsUpdate { + sandbox_policy: Some(SandboxPolicy::DangerFullAccess), + ..Default::default() + }, + ) + .await?; + + let started_proxy = session + .services + .network_proxy + .as_ref() + .expect("managed network proxy should be present"); + assert_eq!( + started_proxy + .proxy() + .current_cfg() + .await? + .network + .allowed_domains(), + Some(vec!["*.example.com".to_string()]) + ); + + Ok(()) +} + +#[tokio::test] +async fn danger_full_access_turns_do_not_expose_managed_network_proxy() -> anyhow::Result<()> { + let network_spec = crate::config::NetworkProxySpec::from_config_and_constraints( + NetworkProxyConfig::default(), + Some(NetworkConstraints { + enabled: Some(true), + ..Default::default() + }), + &SandboxPolicy::DangerFullAccess, + )?; + + let session = make_session_with_config(move |config| { + config.permissions.sandbox_policy = + codex_config::Constrained::allow_any(SandboxPolicy::DangerFullAccess); + config.permissions.network = Some(network_spec); + }) + .await?; + + let turn_context = session.new_default_turn().await; + assert!(turn_context.network.is_none()); + Ok(()) +} + +#[tokio::test] +async fn danger_full_access_tool_attempts_do_not_enforce_managed_network() -> anyhow::Result<()> { + #[derive(Default)] + struct ProbeToolRuntime { + enforce_managed_network: Vec, + } + + impl crate::tools::sandboxing::Approvable<()> for ProbeToolRuntime { + type ApprovalKey = String; + + fn approval_keys(&self, _req: &()) -> Vec { + vec!["probe".to_string()] + } + + fn start_approval_async<'a>( + &'a mut self, + _req: &'a (), + _ctx: crate::tools::sandboxing::ApprovalCtx<'a>, + ) -> futures::future::BoxFuture<'a, ReviewDecision> { + Box::pin(async { ReviewDecision::Approved }) + } + } + + impl crate::tools::sandboxing::Sandboxable for ProbeToolRuntime { + fn sandbox_preference(&self) -> codex_sandboxing::SandboxablePreference { + codex_sandboxing::SandboxablePreference::Auto + } + } + + impl crate::tools::sandboxing::ToolRuntime<(), ()> for ProbeToolRuntime { + async fn run( + &mut self, + _req: &(), + attempt: &crate::tools::sandboxing::SandboxAttempt<'_>, + _ctx: &crate::tools::sandboxing::ToolCtx, + ) -> Result<(), crate::tools::sandboxing::ToolError> { + self.enforce_managed_network + .push(attempt.enforce_managed_network); + Ok(()) + } + } + + let network_spec = crate::config::NetworkProxySpec::from_config_and_constraints( + NetworkProxyConfig::default(), + Some(NetworkConstraints { + enabled: Some(true), + ..Default::default() + }), + &SandboxPolicy::DangerFullAccess, + )?; + + let session = make_session_with_config(move |config| { + config.permissions.sandbox_policy = + codex_config::Constrained::allow_any(SandboxPolicy::DangerFullAccess); + config.permissions.network = Some(network_spec); + + let layers = config + .config_layer_stack + .get_layers( + ConfigLayerStackOrdering::LowestPrecedenceFirst, + /*include_disabled*/ true, + ) + .into_iter() + .cloned() + .collect(); + let mut requirements = config.config_layer_stack.requirements().clone(); + requirements.network = Some(Sourced::new( + NetworkConstraints { + enabled: Some(true), + ..Default::default() + }, + RequirementSource::CloudRequirements, + )); + let mut requirements_toml = config.config_layer_stack.requirements_toml().clone(); + requirements_toml.network = Some(crate::config_loader::NetworkRequirementsToml { + enabled: Some(true), + ..Default::default() + }); + config.config_layer_stack = ConfigLayerStack::new(layers, requirements, requirements_toml) + .expect("rebuild config layer stack with network requirements"); + }) + .await?; + + let turn = session.new_default_turn().await; + assert!(turn.network.is_none()); + + let mut orchestrator = crate::tools::orchestrator::ToolOrchestrator::new(); + let mut tool = ProbeToolRuntime::default(); + let tool_ctx = crate::tools::sandboxing::ToolCtx { + session: Arc::clone(&session), + turn: Arc::clone(&turn), + call_id: "probe-call".to_string(), + tool_name: "probe".to_string(), + }; + + orchestrator + .run( + &mut tool, + &(), + &tool_ctx, + turn.as_ref(), + AskForApproval::Never, + ) + .await + .expect("probe runtime should succeed"); + + assert_eq!(tool.enforce_managed_network, vec![false]); + + Ok(()) +} + +#[tokio::test] +async fn workspace_write_turns_continue_to_expose_managed_network_proxy() -> anyhow::Result<()> { + let sandbox_policy = SandboxPolicy::new_workspace_write_policy(); + let network_spec = crate::config::NetworkProxySpec::from_config_and_constraints( + NetworkProxyConfig::default(), + Some(NetworkConstraints { + enabled: Some(true), + ..Default::default() + }), + &sandbox_policy, + )?; + + let session = make_session_with_config(move |config| { + config.permissions.sandbox_policy = codex_config::Constrained::allow_any(sandbox_policy); + config.permissions.network = Some(network_spec); + }) + .await?; + + let turn_context = session.new_default_turn().await; + assert!(turn_context.network.is_some()); + Ok(()) +} + +#[tokio::test] +async fn user_shell_commands_do_not_inherit_managed_network_proxy() -> anyhow::Result<()> { + let sandbox_policy = SandboxPolicy::new_workspace_write_policy(); + let network_spec = crate::config::NetworkProxySpec::from_config_and_constraints( + NetworkProxyConfig::default(), + Some(NetworkConstraints { + enabled: Some(true), + ..Default::default() + }), + &sandbox_policy, + )?; + + let (session, rx) = make_session_with_config_and_rx(move |config| { + config.permissions.sandbox_policy = codex_config::Constrained::allow_any(sandbox_policy); + config.permissions.network = Some(network_spec); + }) + .await?; + + let turn_context = session.new_default_turn().await; + assert!(turn_context.network.is_some()); + + #[cfg(windows)] + let command = r#"$val = $env:HTTP_PROXY; if ([string]::IsNullOrEmpty($val)) { $val = 'not-set' } ; [System.Console]::Write($val)"#.to_string(); + #[cfg(not(windows))] + let command = r#"sh -c "printf '%s' \"${HTTP_PROXY:-not-set}\"""#.to_string(); + + execute_user_shell_command( + Arc::clone(&session), + turn_context, + command, + CancellationToken::new(), + UserShellCommandMode::StandaloneTurn, + ) + .await; + + loop { + let event = rx.recv().await.expect("channel open"); + if let EventMsg::ExecCommandEnd(event) = event.msg { + assert_eq!(event.exit_code, 0); + assert_eq!(event.stdout.trim(), "not-set"); + break; + } + } + + Ok(()) +} + #[tokio::test] async fn get_base_instructions_no_user_content() { let prompt_with_apply_patch_instructions = @@ -690,7 +981,7 @@ async fn get_base_instructions_no_user_content() { ]; let (session, _turn_context) = make_session_and_context().await; - let config = test_config(); + let config = test_config().await; for test_case in test_cases { let model_info = model_info_for_slug(test_case.slug, &config); @@ -855,10 +1146,10 @@ fn collect_explicit_app_ids_from_skill_items_skips_plain_mentions_with_skill_con assert_eq!(connector_ids, HashSet::::new()); } -#[test] -fn mcp_tool_exposure_directly_exposes_small_effective_tool_sets() { - let config = test_config(); - let tools_config = tools_config_for_mcp_tool_exposure(/*search_tool*/ true); +#[tokio::test] +async fn mcp_tool_exposure_directly_exposes_small_effective_tool_sets() { + let config = test_config().await; + let tools_config = tools_config_for_mcp_tool_exposure(/*search_tool*/ true).await; let mcp_tools = numbered_mcp_tools(DIRECT_MCP_TOOL_EXPOSURE_THRESHOLD - 1); let exposure = build_mcp_tool_exposure( @@ -877,10 +1168,10 @@ fn mcp_tool_exposure_directly_exposes_small_effective_tool_sets() { assert!(exposure.deferred_tools.is_none()); } -#[test] -fn mcp_tool_exposure_searches_large_effective_tool_sets() { - let config = test_config(); - let tools_config = tools_config_for_mcp_tool_exposure(/*search_tool*/ true); +#[tokio::test] +async fn mcp_tool_exposure_searches_large_effective_tool_sets() { + let config = test_config().await; + let tools_config = tools_config_for_mcp_tool_exposure(/*search_tool*/ true).await; let mcp_tools = numbered_mcp_tools(DIRECT_MCP_TOOL_EXPOSURE_THRESHOLD); let exposure = build_mcp_tool_exposure( @@ -903,10 +1194,10 @@ fn mcp_tool_exposure_searches_large_effective_tool_sets() { assert_eq!(deferred_tool_names, expected_tool_names); } -#[test] -fn mcp_tool_exposure_directly_exposes_explicit_apps_in_large_search_sets() { - let config = test_config(); - let tools_config = tools_config_for_mcp_tool_exposure(/*search_tool*/ true); +#[tokio::test] +async fn mcp_tool_exposure_directly_exposes_explicit_apps_without_deferred_overlap() { + let config = test_config().await; + let tools_config = tools_config_for_mcp_tool_exposure(/*search_tool*/ true).await; let mut mcp_tools = numbered_mcp_tools(DIRECT_MCP_TOOL_EXPOSURE_THRESHOLD - 1); mcp_tools.extend([( "mcp__codex_apps__calendar_create_event".to_string(), @@ -935,13 +1226,19 @@ fn mcp_tool_exposure_directly_exposes_explicit_apps_in_large_search_sets() { ); assert_eq!( exposure.deferred_tools.as_ref().map(HashMap::len), - Some(DIRECT_MCP_TOOL_EXPOSURE_THRESHOLD) + Some(DIRECT_MCP_TOOL_EXPOSURE_THRESHOLD - 1) ); let deferred_tools = exposure .deferred_tools .as_ref() .expect("large tool sets should be discoverable through tool_search"); - assert!(deferred_tools.contains_key("mcp__codex_apps__calendar_create_event")); + assert!( + tool_names + .iter() + .all(|direct_tool_name| !deferred_tools.contains_key(direct_tool_name)), + "direct tools should not also be deferred: {tool_names:?}" + ); + assert!(!deferred_tools.contains_key("mcp__codex_apps__calendar_create_event")); assert!(deferred_tools.contains_key("mcp__rmcp__tool_0")); } @@ -1342,6 +1639,7 @@ async fn record_initial_history_forked_hydrates_previous_turn_settings() { approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: previous_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), @@ -2814,6 +3112,11 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) { }), rollout: Mutex::new(None), user_shell: Arc::new(default_user_shell()), + agent_identity_manager: Arc::new(crate::agent_identity::AgentIdentityManager::new( + config.as_ref(), + Arc::clone(&auth_manager), + session_configuration.session_source.clone(), + )), shell_snapshot_tx: watch::channel(None).0, show_raw_agent_reasoning: config.show_raw_agent_reasoning, exec_policy, @@ -2830,6 +3133,9 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) { network_proxy: None, network_approval: Arc::clone(&network_approval), state_db: None, + thread_store: codex_thread_store::LocalThreadStore::new( + codex_rollout::RolloutConfig::from_view(config.as_ref()), + ), model_client: ModelClient::new( Some(auth_manager.clone()), conversation_id, @@ -2908,6 +3214,109 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) { (session, turn_context) } +async fn make_session_with_config( + mutator: impl FnOnce(&mut Config), +) -> anyhow::Result> { + let (session, _rx_event) = make_session_with_config_and_rx(mutator).await?; + Ok(session) +} + +async fn make_session_with_config_and_rx( + mutator: impl FnOnce(&mut Config), +) -> anyhow::Result<(Arc, async_channel::Receiver)> { + let codex_home = tempfile::tempdir().expect("create temp dir"); + let mut config = build_test_config(codex_home.path()).await; + mutator(&mut config); + let config = Arc::new(config); + let auth_manager = AuthManager::from_auth_for_testing(CodexAuth::from_api_key("Test API Key")); + let models_manager = Arc::new(ModelsManager::new( + config.codex_home.to_path_buf(), + auth_manager.clone(), + /*model_catalog*/ None, + CollaborationModesConfig::default(), + )); + let model = ModelsManager::get_model_offline_for_tests(config.model.as_deref()); + let model_info = ModelsManager::construct_model_info_offline_for_tests( + model.as_str(), + &config.to_models_manager_config(), + ); + let collaboration_mode = CollaborationMode { + mode: ModeKind::Default, + settings: Settings { + model, + reasoning_effort: config.model_reasoning_effort, + developer_instructions: None, + }, + }; + let session_configuration = SessionConfiguration { + provider: config.model_provider.clone(), + collaboration_mode, + model_reasoning_summary: config.model_reasoning_summary, + developer_instructions: config.developer_instructions.clone(), + user_instructions: config.user_instructions.clone(), + service_tier: None, + personality: config.personality, + base_instructions: config + .base_instructions + .clone() + .unwrap_or_else(|| model_info.get_model_instructions(config.personality)), + compact_prompt: config.compact_prompt.clone(), + approval_policy: config.permissions.approval_policy.clone(), + approvals_reviewer: config.approvals_reviewer, + sandbox_policy: config.permissions.sandbox_policy.clone(), + file_system_sandbox_policy: config.permissions.file_system_sandbox_policy.clone(), + network_sandbox_policy: config.permissions.network_sandbox_policy, + windows_sandbox_level: WindowsSandboxLevel::from_config(&config), + cwd: config.cwd.clone(), + codex_home: config.codex_home.clone(), + thread_name: None, + original_config_do_not_use: Arc::clone(&config), + metrics_service_name: None, + app_server_client_name: None, + app_server_client_version: None, + session_source: SessionSource::Exec, + dynamic_tools: Vec::new(), + persist_extended_history: false, + inherited_shell_snapshot: None, + user_shell_override: None, + }; + + let (tx_event, rx_event) = async_channel::unbounded(); + let (agent_status_tx, _agent_status_rx) = watch::channel(AgentStatus::PendingInit); + let plugins_manager = Arc::new(PluginsManager::new(config.codex_home.to_path_buf())); + let mcp_manager = Arc::new(McpManager::new(Arc::clone(&plugins_manager))); + let skills_manager = Arc::new(SkillsManager::new( + config.codex_home.clone(), + /*bundled_skills_enabled*/ true, + )); + + let session = Session::new( + session_configuration, + Arc::clone(&config), + auth_manager, + models_manager, + Arc::new(ExecPolicyManager::default()), + tx_event, + agent_status_tx, + InitialHistory::New, + SessionSource::Exec, + skills_manager, + plugins_manager, + mcp_manager, + Arc::new(SkillsWatcher::noop()), + AgentControl::default(), + Some(Arc::new( + codex_exec_server::Environment::create(/*exec_server_url*/ None) + .await + .expect("create environment"), + )), + /*analytics_events_client*/ None, + ) + .await?; + + Ok((session, rx_event)) +} + #[tokio::test] async fn notify_request_permissions_response_ignores_unmatched_call_id() { let (session, _turn_context) = make_session_and_context().await; @@ -3666,6 +4075,11 @@ pub(crate) async fn make_session_and_context_with_dynamic_tools_and_rx( }), rollout: Mutex::new(None), user_shell: Arc::new(default_user_shell()), + agent_identity_manager: Arc::new(crate::agent_identity::AgentIdentityManager::new( + config.as_ref(), + Arc::clone(&auth_manager), + session_configuration.session_source.clone(), + )), shell_snapshot_tx: watch::channel(None).0, show_raw_agent_reasoning: config.show_raw_agent_reasoning, exec_policy, @@ -3682,6 +4096,9 @@ pub(crate) async fn make_session_and_context_with_dynamic_tools_and_rx( network_proxy: None, network_approval: Arc::clone(&network_approval), state_db: None, + thread_store: codex_thread_store::LocalThreadStore::new( + codex_rollout::RolloutConfig::from_view(config.as_ref()), + ), model_client: ModelClient::new( Some(Arc::clone(&auth_manager)), conversation_id, @@ -3770,6 +4187,42 @@ pub(crate) async fn make_session_and_context_with_rx() -> ( make_session_and_context_with_dynamic_tools_and_rx(Vec::new()).await } +#[tokio::test] +async fn fail_agent_identity_registration_emits_error_and_shutdown() { + let (session, _turn_context, rx_event) = make_session_and_context_with_rx().await; + + session + .fail_agent_identity_registration(anyhow::anyhow!("registration exploded")) + .await; + + let error_event = timeout(Duration::from_secs(1), rx_event.recv()) + .await + .expect("error event should arrive") + .expect("error event should be readable"); + match error_event.msg { + EventMsg::Error(ErrorEvent { + message, + codex_error_info, + }) => { + assert_eq!( + message, + "Agent identity registration failed. Codex cannot continue while `features.use_agent_identity` is enabled: registration exploded".to_string() + ); + assert_eq!(codex_error_info, Some(CodexErrorInfo::Other)); + } + other => panic!("expected error event, got {other:?}"), + } + + let shutdown_event = timeout(Duration::from_secs(1), rx_event.recv()) + .await + .expect("shutdown event should arrive") + .expect("shutdown event should be readable"); + match shutdown_event.msg { + EventMsg::ShutdownComplete => {} + other => panic!("expected shutdown event, got {other:?}"), + } +} + #[tokio::test] async fn refresh_mcp_servers_is_deferred_until_next_turn() { let (session, turn_context) = make_session_and_context().await; @@ -4268,6 +4721,47 @@ async fn build_initial_context_restates_realtime_start_when_reference_context_is ); } +fn file_system_policy_with_unreadable_glob(turn_context: &TurnContext) -> FileSystemSandboxPolicy { + let mut policy = FileSystemSandboxPolicy::from_legacy_sandbox_policy( + turn_context.sandbox_policy.get(), + &turn_context.cwd, + ); + policy.entries.push(FileSystemSandboxEntry { + path: FileSystemPath::GlobPattern { + pattern: format!("{}/**/*.env", turn_context.cwd.as_path().display()), + }, + access: FileSystemAccessMode::None, + }); + policy +} + +#[tokio::test] +async fn turn_context_item_omits_legacy_equivalent_file_system_sandbox_policy() { + let (_session, mut turn_context) = make_session_and_context().await; + turn_context.file_system_sandbox_policy = FileSystemSandboxPolicy::from_legacy_sandbox_policy( + turn_context.sandbox_policy.get(), + &turn_context.cwd, + ); + + let item = turn_context.to_turn_context_item(); + + assert_eq!(item.file_system_sandbox_policy, None); +} + +#[tokio::test] +async fn turn_context_item_stores_split_file_system_sandbox_policy_when_different() { + let (_session, mut turn_context) = make_session_and_context().await; + let file_system_sandbox_policy = file_system_policy_with_unreadable_glob(&turn_context); + turn_context.file_system_sandbox_policy = file_system_sandbox_policy.clone(); + + let item = turn_context.to_turn_context_item(); + + assert_eq!( + item.file_system_sandbox_policy, + Some(file_system_sandbox_policy) + ); +} + #[tokio::test] async fn record_context_updates_and_set_reference_context_item_injects_full_context_when_baseline_missing() { @@ -4406,6 +4900,56 @@ async fn record_context_updates_and_set_reference_context_item_persists_baseline ); } +#[tokio::test] +async fn record_context_updates_and_set_reference_context_item_persists_split_file_system_policy_to_rollout() + { + let (session, mut turn_context) = make_session_and_context().await; + let file_system_sandbox_policy = file_system_policy_with_unreadable_glob(&turn_context); + turn_context.file_system_sandbox_policy = file_system_sandbox_policy.clone(); + let config = session.get_config().await; + let recorder = RolloutRecorder::new( + config.as_ref(), + RolloutRecorderParams::new( + ThreadId::default(), + /*forked_from_id*/ None, + SessionSource::Exec, + BaseInstructions::default(), + Vec::new(), + EventPersistenceMode::Limited, + ), + /*state_db_ctx*/ None, + /*state_builder*/ None, + ) + .await + .expect("create rollout recorder"); + let rollout_path = recorder.rollout_path().to_path_buf(); + { + let mut rollout = session.services.rollout.lock().await; + *rollout = Some(recorder); + } + + session + .record_context_updates_and_set_reference_context_item(&turn_context) + .await; + session.ensure_rollout_materialized().await; + session.flush_rollout().await.expect("rollout should flush"); + + let InitialHistory::Resumed(resumed) = RolloutRecorder::get_rollout_history(&rollout_path) + .await + .expect("read rollout history") + else { + panic!("expected resumed rollout history"); + }; + let persisted_file_system_sandbox_policy = resumed.history.iter().find_map(|item| match item { + RolloutItem::TurnContext(ctx) => ctx.file_system_sandbox_policy.clone(), + _ => None, + }); + assert_eq!( + persisted_file_system_sandbox_policy, + Some(file_system_sandbox_policy) + ); +} + #[tokio::test] async fn build_initial_context_prepends_model_switch_message() { let (session, turn_context) = make_session_and_context().await; @@ -5315,6 +5859,7 @@ async fn fatal_tool_error_stops_turn_and_reports_error() { crate::tools::router::ToolRouterParams { deferred_mcp_tools, mcp_tools: Some(tools), + unavailable_called_tools: Vec::new(), parallel_mcp_server_names: HashSet::new(), discoverable_tools: None, dynamic_tools: turn_context.dynamic_tools.as_slice(), diff --git a/codex-rs/core/src/compact.rs b/codex-rs/core/src/compact.rs index 59fb45454d..eaf1d92109 100644 --- a/codex-rs/core/src/compact.rs +++ b/codex-rs/core/src/compact.rs @@ -58,7 +58,7 @@ pub(crate) enum InitialContextInjection { } pub(crate) fn should_use_remote_compact_task(provider: &ModelProviderInfo) -> bool { - provider.is_openai() + provider.supports_remote_compaction() } pub(crate) async fn run_inline_auto_compact_task( diff --git a/codex-rs/core/src/compact_tests.rs b/codex-rs/core/src/compact_tests.rs index cecf9ce9d7..f7a8d1dfff 100644 --- a/codex-rs/core/src/compact_tests.rs +++ b/codex-rs/core/src/compact_tests.rs @@ -1,4 +1,6 @@ use super::*; +use codex_model_provider_info::ModelProviderInfo; +use codex_model_provider_info::WireApi; use pretty_assertions::assert_eq; async fn process_compacted_history_with_test_session( @@ -185,6 +187,30 @@ fn build_token_limited_compacted_history_appends_summary_message() { assert_eq!(summary, summary_text); } +#[test] +fn should_use_remote_compact_task_for_azure_provider() { + let provider = ModelProviderInfo { + name: "Azure".into(), + base_url: Some("https://example.com/openai".into()), + env_key: Some("AZURE_OPENAI_API_KEY".into()), + env_key_instructions: None, + experimental_bearer_token: None, + auth: None, + wire_api: WireApi::Responses, + query_params: None, + http_headers: None, + env_http_headers: None, + request_max_retries: None, + stream_max_retries: None, + stream_idle_timeout_ms: None, + websocket_connect_timeout_ms: None, + requires_openai_auth: false, + supports_websockets: false, + }; + + assert!(should_use_remote_compact_task(&provider)); +} + #[tokio::test] async fn process_compacted_history_replaces_developer_messages() { let compacted_history = vec![ diff --git a/codex-rs/core/src/config/config_tests.rs b/codex-rs/core/src/config/config_tests.rs index b2931e1055..34002b129a 100644 --- a/codex-rs/core/src/config/config_tests.rs +++ b/codex-rs/core/src/config/config_tests.rs @@ -1,3 +1,5 @@ +use crate::agents_md::DEFAULT_AGENTS_MD_FILENAME; +use crate::agents_md::LOCAL_AGENTS_MD_FILENAME; use crate::config::edit::ConfigEdit; use crate::config::edit::ConfigEditsBuilder; use crate::config::edit::apply_blocking; @@ -81,12 +83,14 @@ fn stdio_mcp(command: &str) -> McpServerConfig { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -103,12 +107,14 @@ fn http_mcp(url: &str) -> McpServerConfig { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -117,8 +123,8 @@ fn http_mcp(url: &str) -> McpServerConfig { } } -#[test] -fn load_config_normalizes_relative_cwd_override() -> std::io::Result<()> { +#[tokio::test] +async fn load_config_normalizes_relative_cwd_override() -> std::io::Result<()> { let expected_cwd = AbsolutePathBuf::relative_to_current_dir("nested")?; let codex_home = tempdir()?; let config = Config::load_from_base_config_with_overrides( @@ -128,64 +134,61 @@ fn load_config_normalizes_relative_cwd_override() -> std::io::Result<()> { ..Default::default() }, codex_home.abs(), - )?; + ) + .await?; assert_eq!(config.cwd, expected_cwd); Ok(()) } -#[test] -fn load_config_records_global_agents_path() -> std::io::Result<()> { +#[tokio::test] +async fn load_config_loads_global_agents_instructions() -> std::io::Result<()> { let codex_home = tempdir()?; - let global_agents_path = codex_home.path().join(DEFAULT_PROJECT_DOC_FILENAME); - std::fs::write(&global_agents_path, "\n global instructions \n")?; + std::fs::write( + codex_home.path().join(DEFAULT_AGENTS_MD_FILENAME), + "\n global instructions \n", + )?; let config = Config::load_from_base_config_with_overrides( ConfigToml::default(), ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.user_instructions.as_deref(), Some("global instructions") ); - assert_eq!( - config.user_instructions_path.as_deref(), - Some(global_agents_path.as_path()) - ); Ok(()) } -#[test] -fn load_config_records_preferred_global_agents_override_path() -> std::io::Result<()> { +#[tokio::test] +async fn load_config_prefers_global_agents_override_instructions() -> std::io::Result<()> { let codex_home = tempdir()?; std::fs::write( - codex_home.path().join(DEFAULT_PROJECT_DOC_FILENAME), + codex_home.path().join(DEFAULT_AGENTS_MD_FILENAME), "global instructions", )?; - let global_agents_override_path = codex_home.path().join(LOCAL_PROJECT_DOC_FILENAME); + let global_agents_override_path = codex_home.path().join(LOCAL_AGENTS_MD_FILENAME); std::fs::write(&global_agents_override_path, "local override instructions")?; let config = Config::load_from_base_config_with_overrides( ConfigToml::default(), ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.user_instructions.as_deref(), Some("local override instructions") ); - assert_eq!( - config.user_instructions_path.as_deref(), - Some(global_agents_override_path.as_path()) - ); Ok(()) } -#[test] -fn test_toml_parsing() { +#[tokio::test] +async fn test_toml_parsing() { let history_with_persistence = r#" [history] persistence = "save-all" @@ -251,6 +254,7 @@ consolidation_model = "gpt-5" ConfigOverrides::default(), tempdir().expect("tempdir").abs(), ) + .await .expect("load config from memories settings"); assert_eq!( config.memories, @@ -376,13 +380,14 @@ fn config_toml_deserializes_model_availability_nux() { ); } -#[test] -fn runtime_config_defaults_model_availability_nux() { +#[tokio::test] +async fn runtime_config_defaults_model_availability_nux() { let cfg = Config::load_from_base_config_with_overrides( ConfigToml::default(), ConfigOverrides::default(), tempdir().expect("tempdir").abs(), ) + .await .expect("load config"); assert_eq!( @@ -462,8 +467,9 @@ allow_upstream_proxy = false ); } -#[test] -fn permissions_profiles_network_populates_runtime_network_proxy_spec() -> std::io::Result<()> { +#[tokio::test] +async fn permissions_profiles_network_populates_runtime_network_proxy_spec() -> std::io::Result<()> +{ let codex_home = TempDir::new()?; let cwd = TempDir::new()?; std::fs::write(cwd.path().join(".git"), "gitdir: nowhere")?; @@ -497,7 +503,8 @@ fn permissions_profiles_network_populates_runtime_network_proxy_spec() -> std::i ..Default::default() }, codex_home.abs(), - )?; + ) + .await?; let network = config .permissions .network @@ -509,8 +516,9 @@ fn permissions_profiles_network_populates_runtime_network_proxy_spec() -> std::i Ok(()) } -#[test] -fn permissions_profiles_network_disabled_by_default_does_not_start_proxy() -> std::io::Result<()> { +#[tokio::test] +async fn permissions_profiles_network_disabled_by_default_does_not_start_proxy() +-> std::io::Result<()> { let codex_home = TempDir::new()?; let cwd = TempDir::new()?; std::fs::write(cwd.path().join(".git"), "gitdir: nowhere")?; @@ -547,14 +555,15 @@ fn permissions_profiles_network_disabled_by_default_does_not_start_proxy() -> st ..Default::default() }, codex_home.abs(), - )?; + ) + .await?; assert!(config.permissions.network.is_none()); Ok(()) } -#[test] -fn default_permissions_profile_populates_runtime_sandbox_policy() -> std::io::Result<()> { +#[tokio::test] +async fn default_permissions_profile_populates_runtime_sandbox_policy() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cwd = TempDir::new()?; std::fs::create_dir_all(cwd.path().join("docs"))?; @@ -595,7 +604,8 @@ fn default_permissions_profile_populates_runtime_sandbox_policy() -> std::io::Re ..Default::default() }, codex_home.abs(), - )?; + ) + .await?; let memories_root = codex_home.path().join("memories").abs(); assert_eq!( @@ -647,8 +657,76 @@ fn default_permissions_profile_populates_runtime_sandbox_policy() -> std::io::Re Ok(()) } -#[test] -fn permissions_profiles_require_default_permissions() -> std::io::Result<()> { +#[tokio::test] +async fn project_root_glob_none_compiles_to_filesystem_pattern_entry() -> std::io::Result<()> { + let codex_home = TempDir::new()?; + let cwd = TempDir::new()?; + tokio::fs::write(cwd.path().join(".git"), "gitdir: nowhere").await?; + + let config = Config::load_from_base_config_with_overrides( + ConfigToml { + default_permissions: Some("workspace".to_string()), + permissions: Some(PermissionsToml { + entries: BTreeMap::from([( + "workspace".to_string(), + PermissionProfileToml { + filesystem: Some(FilesystemPermissionsToml { + entries: BTreeMap::from([( + ":project_roots".to_string(), + FilesystemPermissionToml::Scoped(BTreeMap::from([ + (".".to_string(), FileSystemAccessMode::Write), + ("**/*.env".to_string(), FileSystemAccessMode::None), + ])), + )]), + }), + network: None, + }, + )]), + }), + ..Default::default() + }, + ConfigOverrides { + cwd: Some(cwd.path().to_path_buf()), + ..Default::default() + }, + codex_home.abs(), + ) + .await?; + + let expected_pattern = AbsolutePathBuf::resolve_path_against_base("**/*.env", cwd.path()) + .to_string_lossy() + .into_owned(); + assert!( + config + .permissions + .file_system_sandbox_policy + .entries + .contains(&FileSystemSandboxEntry { + path: FileSystemPath::GlobPattern { + pattern: expected_pattern, + }, + access: FileSystemAccessMode::None, + }) + ); + assert!( + !config + .permissions + .file_system_sandbox_policy + .entries + .iter() + .any(|entry| matches!( + &entry.path, + FileSystemPath::Special { + value: FileSystemSpecialPath::ProjectRoots { subpath: Some(subpath) }, + } if subpath == std::path::Path::new("**/*.env") + )), + "glob should compile to a filesystem pattern entry, not a literal filesystem entry" + ); + Ok(()) +} + +#[tokio::test] +async fn permissions_profiles_require_default_permissions() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cwd = TempDir::new()?; std::fs::write(cwd.path().join(".git"), "gitdir: nowhere")?; @@ -677,6 +755,7 @@ fn permissions_profiles_require_default_permissions() -> std::io::Result<()> { }, codex_home.abs(), ) + .await .expect_err("missing default_permissions should be rejected"); assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); @@ -687,8 +766,8 @@ fn permissions_profiles_require_default_permissions() -> std::io::Result<()> { Ok(()) } -#[test] -fn permissions_profiles_reject_writes_outside_workspace_root() -> std::io::Result<()> { +#[tokio::test] +async fn permissions_profiles_reject_writes_outside_workspace_root() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cwd = TempDir::new()?; std::fs::write(cwd.path().join(".git"), "gitdir: nowhere")?; @@ -719,6 +798,7 @@ fn permissions_profiles_reject_writes_outside_workspace_root() -> std::io::Resul }, codex_home.abs(), ) + .await .expect_err("writes outside the workspace root should be rejected"); assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); @@ -730,8 +810,8 @@ fn permissions_profiles_reject_writes_outside_workspace_root() -> std::io::Resul Ok(()) } -#[test] -fn permissions_profiles_reject_nested_entries_for_non_project_roots() -> std::io::Result<()> { +#[tokio::test] +async fn permissions_profiles_reject_nested_entries_for_non_project_roots() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cwd = TempDir::new()?; std::fs::write(cwd.path().join(".git"), "gitdir: nowhere")?; @@ -764,6 +844,7 @@ fn permissions_profiles_reject_nested_entries_for_non_project_roots() -> std::io }, codex_home.abs(), ) + .await .expect_err("nested entries outside :project_roots should be rejected"); assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); @@ -774,7 +855,9 @@ fn permissions_profiles_reject_nested_entries_for_non_project_roots() -> std::io Ok(()) } -fn load_workspace_permission_profile(profile: PermissionProfileToml) -> std::io::Result { +async fn load_workspace_permission_profile( + profile: PermissionProfileToml, +) -> std::io::Result { let codex_home = TempDir::new()?; let cwd = TempDir::new()?; std::fs::write(cwd.path().join(".git"), "gitdir: nowhere")?; @@ -793,10 +876,11 @@ fn load_workspace_permission_profile(profile: PermissionProfileToml) -> std::io: }, codex_home.abs(), ) + .await } -#[test] -fn permissions_profiles_allow_unknown_special_paths() -> std::io::Result<()> { +#[tokio::test] +async fn permissions_profiles_allow_unknown_special_paths() -> std::io::Result<()> { let config = load_workspace_permission_profile(PermissionProfileToml { filesystem: Some(FilesystemPermissionsToml { entries: BTreeMap::from([( @@ -805,7 +889,8 @@ fn permissions_profiles_allow_unknown_special_paths() -> std::io::Result<()> { )]), }), network: None, - })?; + }) + .await?; assert_eq!( config.permissions.file_system_sandbox_policy, @@ -839,8 +924,9 @@ fn permissions_profiles_allow_unknown_special_paths() -> std::io::Result<()> { Ok(()) } -#[test] -fn permissions_profiles_allow_unknown_special_paths_with_nested_entries() -> std::io::Result<()> { +#[tokio::test] +async fn permissions_profiles_allow_unknown_special_paths_with_nested_entries() +-> std::io::Result<()> { let config = load_workspace_permission_profile(PermissionProfileToml { filesystem: Some(FilesystemPermissionsToml { entries: BTreeMap::from([( @@ -852,7 +938,8 @@ fn permissions_profiles_allow_unknown_special_paths_with_nested_entries() -> std )]), }), network: None, - })?; + }) + .await?; assert_eq!( config.permissions.file_system_sandbox_policy, @@ -873,12 +960,13 @@ fn permissions_profiles_allow_unknown_special_paths_with_nested_entries() -> std Ok(()) } -#[test] -fn permissions_profiles_allow_missing_filesystem_with_warning() -> std::io::Result<()> { +#[tokio::test] +async fn permissions_profiles_allow_missing_filesystem_with_warning() -> std::io::Result<()> { let config = load_workspace_permission_profile(PermissionProfileToml { filesystem: None, network: None, - })?; + }) + .await?; assert_eq!( config.permissions.file_system_sandbox_policy, @@ -904,14 +992,15 @@ fn permissions_profiles_allow_missing_filesystem_with_warning() -> std::io::Resu Ok(()) } -#[test] -fn permissions_profiles_allow_empty_filesystem_with_warning() -> std::io::Result<()> { +#[tokio::test] +async fn permissions_profiles_allow_empty_filesystem_with_warning() -> std::io::Result<()> { let config = load_workspace_permission_profile(PermissionProfileToml { filesystem: Some(FilesystemPermissionsToml { entries: BTreeMap::new(), }), network: None, - })?; + }) + .await?; assert_eq!( config.permissions.file_system_sandbox_policy, @@ -927,8 +1016,8 @@ fn permissions_profiles_allow_empty_filesystem_with_warning() -> std::io::Result Ok(()) } -#[test] -fn permissions_profiles_reject_project_root_parent_traversal() -> std::io::Result<()> { +#[tokio::test] +async fn permissions_profiles_reject_project_root_parent_traversal() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cwd = TempDir::new()?; std::fs::write(cwd.path().join(".git"), "gitdir: nowhere")?; @@ -961,6 +1050,7 @@ fn permissions_profiles_reject_project_root_parent_traversal() -> std::io::Resul }, codex_home.abs(), ) + .await .expect_err("parent traversal should be rejected for project root subpaths"); assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); @@ -971,8 +1061,8 @@ fn permissions_profiles_reject_project_root_parent_traversal() -> std::io::Resul Ok(()) } -#[test] -fn permissions_profiles_allow_network_enablement() -> std::io::Result<()> { +#[tokio::test] +async fn permissions_profiles_allow_network_enablement() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cwd = TempDir::new()?; std::fs::write(cwd.path().join(".git"), "gitdir: nowhere")?; @@ -1004,7 +1094,8 @@ fn permissions_profiles_allow_network_enablement() -> std::io::Result<()> { ..Default::default() }, codex_home.abs(), - )?; + ) + .await?; assert!( config.permissions.network_sandbox_policy.is_enabled(), @@ -1067,8 +1158,8 @@ fn tui_config_missing_notifications_field_defaults_to_enabled() { ); } -#[test] -fn test_sandbox_config_parsing() { +#[tokio::test] +async fn test_sandbox_config_parsing() { let sandbox_full_access = r#" sandbox_mode = "danger-full-access" @@ -1078,13 +1169,15 @@ network_access = false # This should be ignored. let sandbox_full_access_cfg = toml::from_str::(sandbox_full_access) .expect("TOML deserialization should succeed"); let sandbox_mode_override = None; - let resolution = sandbox_full_access_cfg.derive_sandbox_policy( - sandbox_mode_override, - /*profile_sandbox_mode*/ None, - WindowsSandboxLevel::Disabled, - &PathBuf::from("/tmp/test"), - /*sandbox_policy_constraint*/ None, - ); + let resolution = sandbox_full_access_cfg + .derive_sandbox_policy( + sandbox_mode_override, + /*profile_sandbox_mode*/ None, + WindowsSandboxLevel::Disabled, + &PathBuf::from("/tmp/test"), + /*sandbox_policy_constraint*/ None, + ) + .await; assert_eq!(resolution, SandboxPolicy::DangerFullAccess); let sandbox_read_only = r#" @@ -1097,13 +1190,15 @@ network_access = true # This should be ignored. let sandbox_read_only_cfg = toml::from_str::(sandbox_read_only) .expect("TOML deserialization should succeed"); let sandbox_mode_override = None; - let resolution = sandbox_read_only_cfg.derive_sandbox_policy( - sandbox_mode_override, - /*profile_sandbox_mode*/ None, - WindowsSandboxLevel::Disabled, - &PathBuf::from("/tmp/test"), - /*sandbox_policy_constraint*/ None, - ); + let resolution = sandbox_read_only_cfg + .derive_sandbox_policy( + sandbox_mode_override, + /*profile_sandbox_mode*/ None, + WindowsSandboxLevel::Disabled, + &PathBuf::from("/tmp/test"), + /*sandbox_policy_constraint*/ None, + ) + .await; assert_eq!(resolution, SandboxPolicy::new_read_only_policy()); let writable_root = test_absolute_path("/my/workspace"); @@ -1124,13 +1219,15 @@ exclude_slash_tmp = true let sandbox_workspace_write_cfg = toml::from_str::(&sandbox_workspace_write) .expect("TOML deserialization should succeed"); let sandbox_mode_override = None; - let resolution = sandbox_workspace_write_cfg.derive_sandbox_policy( - sandbox_mode_override, - /*profile_sandbox_mode*/ None, - WindowsSandboxLevel::Disabled, - &PathBuf::from("/tmp/test"), - /*sandbox_policy_constraint*/ None, - ); + let resolution = sandbox_workspace_write_cfg + .derive_sandbox_policy( + sandbox_mode_override, + /*profile_sandbox_mode*/ None, + WindowsSandboxLevel::Disabled, + &PathBuf::from("/tmp/test"), + /*sandbox_policy_constraint*/ None, + ) + .await; if cfg!(target_os = "windows") { assert_eq!(resolution, SandboxPolicy::new_read_only_policy()); } else { @@ -1166,13 +1263,15 @@ trust_level = "trusted" let sandbox_workspace_write_cfg = toml::from_str::(&sandbox_workspace_write) .expect("TOML deserialization should succeed"); let sandbox_mode_override = None; - let resolution = sandbox_workspace_write_cfg.derive_sandbox_policy( - sandbox_mode_override, - /*profile_sandbox_mode*/ None, - WindowsSandboxLevel::Disabled, - &PathBuf::from("/tmp/test"), - /*sandbox_policy_constraint*/ None, - ); + let resolution = sandbox_workspace_write_cfg + .derive_sandbox_policy( + sandbox_mode_override, + /*profile_sandbox_mode*/ None, + WindowsSandboxLevel::Disabled, + &PathBuf::from("/tmp/test"), + /*sandbox_policy_constraint*/ None, + ) + .await; if cfg!(target_os = "windows") { assert_eq!(resolution, SandboxPolicy::new_read_only_policy()); } else { @@ -1189,8 +1288,8 @@ trust_level = "trusted" } } -#[test] -fn legacy_sandbox_mode_config_builds_split_policies_without_drift() -> std::io::Result<()> { +#[tokio::test] +async fn legacy_sandbox_mode_config_builds_split_policies_without_drift() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cwd = TempDir::new()?; let extra_root = test_absolute_path("/tmp/legacy-extra-root"); @@ -1232,7 +1331,8 @@ exclude_slash_tmp = true ..Default::default() }, codex_home.abs(), - )?; + ) + .await?; let sandbox_policy = config.permissions.sandbox_policy.get(); assert_eq!( @@ -1393,8 +1493,8 @@ fn filter_mcp_servers_by_allowlist_blocks_all_when_empty() { ); } -#[test] -fn add_dir_override_extends_workspace_writable_roots() -> std::io::Result<()> { +#[tokio::test] +async fn add_dir_override_extends_workspace_writable_roots() -> std::io::Result<()> { let temp_dir = TempDir::new()?; let frontend = temp_dir.path().join("frontend"); let backend = temp_dir.path().join("backend"); @@ -1412,7 +1512,8 @@ fn add_dir_override_extends_workspace_writable_roots() -> std::io::Result<()> { ConfigToml::default(), overrides, temp_dir.path().abs(), - )?; + ) + .await?; let expected_backend = backend.abs(); if cfg!(target_os = "windows") { @@ -1440,8 +1541,8 @@ fn add_dir_override_extends_workspace_writable_roots() -> std::io::Result<()> { Ok(()) } -#[test] -fn sqlite_home_defaults_to_codex_home_for_workspace_write() -> std::io::Result<()> { +#[tokio::test] +async fn sqlite_home_defaults_to_codex_home_for_workspace_write() -> std::io::Result<()> { let codex_home = TempDir::new()?; let config = Config::load_from_base_config_with_overrides( ConfigToml::default(), @@ -1450,15 +1551,16 @@ fn sqlite_home_defaults_to_codex_home_for_workspace_write() -> std::io::Result<( ..Default::default() }, codex_home.abs(), - )?; + ) + .await?; assert_eq!(config.sqlite_home, codex_home.path().to_path_buf()); Ok(()) } -#[test] -fn workspace_write_always_includes_memories_root_once() -> std::io::Result<()> { +#[tokio::test] +async fn workspace_write_always_includes_memories_root_once() -> std::io::Result<()> { let codex_home = TempDir::new()?; let memories_root = codex_home.path().join("memories"); let config = Config::load_from_base_config_with_overrides( @@ -1474,7 +1576,8 @@ fn workspace_write_always_includes_memories_root_once() -> std::io::Result<()> { ..Default::default() }, codex_home.abs(), - )?; + ) + .await?; if cfg!(target_os = "windows") { match config.permissions.sandbox_policy.get() { @@ -1507,8 +1610,8 @@ fn workspace_write_always_includes_memories_root_once() -> std::io::Result<()> { Ok(()) } -#[test] -fn config_defaults_to_file_cli_auth_store_mode() -> std::io::Result<()> { +#[tokio::test] +async fn config_defaults_to_file_cli_auth_store_mode() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cfg = ConfigToml::default(); @@ -1516,7 +1619,8 @@ fn config_defaults_to_file_cli_auth_store_mode() -> std::io::Result<()> { cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.cli_auth_credentials_store_mode, @@ -1526,8 +1630,8 @@ fn config_defaults_to_file_cli_auth_store_mode() -> std::io::Result<()> { Ok(()) } -#[test] -fn config_resolves_explicit_keyring_auth_store_mode() -> std::io::Result<()> { +#[tokio::test] +async fn config_resolves_explicit_keyring_auth_store_mode() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cfg = ConfigToml { cli_auth_credentials_store: Some(AuthCredentialsStoreMode::Keyring), @@ -1538,7 +1642,8 @@ fn config_resolves_explicit_keyring_auth_store_mode() -> std::io::Result<()> { cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.cli_auth_credentials_store_mode, @@ -1551,8 +1656,8 @@ fn config_resolves_explicit_keyring_auth_store_mode() -> std::io::Result<()> { Ok(()) } -#[test] -fn config_resolves_default_oauth_store_mode() -> std::io::Result<()> { +#[tokio::test] +async fn config_resolves_default_oauth_store_mode() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cfg = ConfigToml::default(); @@ -1560,7 +1665,8 @@ fn config_resolves_default_oauth_store_mode() -> std::io::Result<()> { cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.mcp_oauth_credentials_store_mode, @@ -1624,8 +1730,8 @@ fn local_dev_builds_force_file_mcp_oauth_store_modes() { ); } -#[test] -fn feedback_enabled_defaults_to_true() -> std::io::Result<()> { +#[tokio::test] +async fn feedback_enabled_defaults_to_true() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cfg = ConfigToml { feedback: Some(FeedbackConfigToml::default()), @@ -1636,7 +1742,8 @@ fn feedback_enabled_defaults_to_true() -> std::io::Result<()> { cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!(config.feedback_enabled, true); @@ -1776,8 +1883,8 @@ profile = "project" Ok(()) } -#[test] -fn profile_sandbox_mode_overrides_base() -> std::io::Result<()> { +#[tokio::test] +async fn profile_sandbox_mode_overrides_base() -> std::io::Result<()> { let codex_home = TempDir::new()?; let mut profiles = HashMap::new(); profiles.insert( @@ -1798,7 +1905,8 @@ fn profile_sandbox_mode_overrides_base() -> std::io::Result<()> { cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert!(matches!( config.permissions.sandbox_policy.get(), @@ -1808,8 +1916,8 @@ fn profile_sandbox_mode_overrides_base() -> std::io::Result<()> { Ok(()) } -#[test] -fn cli_override_takes_precedence_over_profile_sandbox_mode() -> std::io::Result<()> { +#[tokio::test] +async fn cli_override_takes_precedence_over_profile_sandbox_mode() -> std::io::Result<()> { let codex_home = TempDir::new()?; let mut profiles = HashMap::new(); profiles.insert( @@ -1830,7 +1938,8 @@ fn cli_override_takes_precedence_over_profile_sandbox_mode() -> std::io::Result< ..Default::default() }; - let config = Config::load_from_base_config_with_overrides(cfg, overrides, codex_home.abs())?; + let config = + Config::load_from_base_config_with_overrides(cfg, overrides, codex_home.abs()).await?; if cfg!(target_os = "windows") { assert!(matches!( @@ -1847,8 +1956,8 @@ fn cli_override_takes_precedence_over_profile_sandbox_mode() -> std::io::Result< Ok(()) } -#[test] -fn feature_table_overrides_legacy_flags() -> std::io::Result<()> { +#[tokio::test] +async fn feature_table_overrides_legacy_flags() -> std::io::Result<()> { let codex_home = TempDir::new()?; let mut entries = BTreeMap::new(); entries.insert("apply_patch_freeform".to_string(), false); @@ -1861,7 +1970,8 @@ fn feature_table_overrides_legacy_flags() -> std::io::Result<()> { cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert!(!config.features.enabled(Feature::ApplyPatchFreeform)); assert!(!config.include_apply_patch_tool); @@ -1869,8 +1979,8 @@ fn feature_table_overrides_legacy_flags() -> std::io::Result<()> { Ok(()) } -#[test] -fn legacy_toggles_map_to_features() -> std::io::Result<()> { +#[tokio::test] +async fn legacy_toggles_map_to_features() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cfg = ConfigToml { experimental_use_unified_exec_tool: Some(true), @@ -1882,7 +1992,8 @@ fn legacy_toggles_map_to_features() -> std::io::Result<()> { cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert!(config.features.enabled(Feature::ApplyPatchFreeform)); assert!(config.features.enabled(Feature::UnifiedExec)); @@ -1894,8 +2005,8 @@ fn legacy_toggles_map_to_features() -> std::io::Result<()> { Ok(()) } -#[test] -fn responses_websocket_features_do_not_change_wire_api() -> std::io::Result<()> { +#[tokio::test] +async fn responses_websocket_features_do_not_change_wire_api() -> std::io::Result<()> { for feature_key in ["responses_websockets", "responses_websockets_v2"] { let codex_home = TempDir::new()?; let mut entries = BTreeMap::new(); @@ -1909,7 +2020,8 @@ fn responses_websocket_features_do_not_change_wire_api() -> std::io::Result<()> cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!(config.model_provider.wire_api, WireApi::Responses); } @@ -1917,8 +2029,8 @@ fn responses_websocket_features_do_not_change_wire_api() -> std::io::Result<()> Ok(()) } -#[test] -fn config_honors_explicit_file_oauth_store_mode() -> std::io::Result<()> { +#[tokio::test] +async fn config_honors_explicit_file_oauth_store_mode() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cfg = ConfigToml { mcp_oauth_credentials_store: Some(OAuthCredentialsStoreMode::File), @@ -1929,7 +2041,8 @@ fn config_honors_explicit_file_oauth_store_mode() -> std::io::Result<()> { cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.mcp_oauth_credentials_store_mode, @@ -1974,7 +2087,8 @@ async fn managed_config_overrides_oauth_store_mode() -> anyhow::Result<()> { cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( final_config.mcp_oauth_credentials_store_mode, resolve_mcp_oauth_credentials_store_mode( @@ -2011,12 +2125,14 @@ async fn replace_mcp_servers_round_trips_entries() -> anyhow::Result<()> { env_vars: Vec::new(), cwd: None, }, + experimental_environment: Some("remote".to_string()), enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(3)), tool_timeout_sec: Some(Duration::from_secs(5)), + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2052,6 +2168,7 @@ async fn replace_mcp_servers_round_trips_entries() -> anyhow::Result<()> { } assert_eq!(docs.startup_timeout_sec, Some(Duration::from_secs(3))); assert_eq!(docs.tool_timeout_sec, Some(Duration::from_secs(5))); + assert_eq!(docs.experimental_environment.as_deref(), Some("remote")); assert!(docs.enabled); let empty = BTreeMap::new(); @@ -2129,23 +2246,28 @@ fn mcp_servers_toml_parses_per_tool_approval_overrides() { [mcp_servers.docs] command = "docs-server" name = "Docs" +default_tools_approval_mode = "prompt" [mcp_servers.docs.tools.search] approval_mode = "approve" "#, ) .expect("TOML deserialization should succeed"); - let tool = config + let server = config .mcp_servers .get("docs") - .and_then(|server| server.tools.get("search")) - .expect("docs/search tool config exists"); + .expect("docs server config exists"); assert_eq!( - tool, - &McpServerToolConfig { + server.default_tools_approval_mode, + Some(AppToolApproval::Prompt) + ); + + assert_eq!( + server.tools.get("search"), + Some(&McpServerToolConfig { approval_mode: Some(AppToolApproval::Approve), - } + }) ); } @@ -2199,7 +2321,8 @@ async fn to_mcp_config_preserves_apps_feature_from_config() -> std::io::Result<( ConfigToml::default(), ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; let plugins_manager = PluginsManager::new(codex_home.path().to_path_buf()); let mcp_config = config.to_mcp_config(&plugins_manager).await; @@ -2258,12 +2381,14 @@ async fn replace_mcp_servers_serializes_env_sorted() -> anyhow::Result<()> { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2332,12 +2457,14 @@ async fn replace_mcp_servers_serializes_env_vars() -> anyhow::Result<()> { env_vars: vec!["ALPHA".to_string(), "BETA".to_string()], cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2386,12 +2513,14 @@ async fn replace_mcp_servers_serializes_cwd() -> anyhow::Result<()> { env_vars: Vec::new(), cwd: Some(cwd_path.clone()), }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2438,12 +2567,14 @@ async fn replace_mcp_servers_streamable_http_serializes_bearer_token() -> anyhow http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(2)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2506,12 +2637,14 @@ async fn replace_mcp_servers_streamable_http_serializes_custom_headers() -> anyh "DOCS_AUTH".to_string(), )])), }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(2)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2586,12 +2719,14 @@ async fn replace_mcp_servers_streamable_http_removes_optional_sections() -> anyh "DOCS_AUTH".to_string(), )])), }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(2)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2619,12 +2754,14 @@ async fn replace_mcp_servers_streamable_http_removes_optional_sections() -> anyh http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2687,12 +2824,14 @@ async fn replace_mcp_servers_streamable_http_isolates_headers_between_servers() "DOCS_AUTH".to_string(), )])), }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(2)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2710,12 +2849,14 @@ async fn replace_mcp_servers_streamable_http_isolates_headers_between_servers() env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2796,12 +2937,14 @@ async fn replace_mcp_servers_serializes_disabled_flag() -> anyhow::Result<()> { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: false, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2844,12 +2987,14 @@ async fn replace_mcp_servers_serializes_required_flag() -> anyhow::Result<()> { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: true, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2892,12 +3037,14 @@ async fn replace_mcp_servers_serializes_tool_filters() -> anyhow::Result<()> { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: Some(vec!["allowed".to_string()]), disabled_tools: Some(vec!["blocked".to_string()]), scopes: None, @@ -2944,12 +3091,14 @@ async fn replace_mcp_servers_streamable_http_serializes_oauth_resource() -> anyh http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -3248,8 +3397,8 @@ impl PrecedenceTestFixture { } } -#[test] -fn cli_override_sets_compact_prompt() -> std::io::Result<()> { +#[tokio::test] +async fn cli_override_sets_compact_prompt() -> std::io::Result<()> { let codex_home = TempDir::new()?; let overrides = ConfigOverrides { compact_prompt: Some("Use the compact override".to_string()), @@ -3260,7 +3409,8 @@ fn cli_override_sets_compact_prompt() -> std::io::Result<()> { ConfigToml::default(), overrides, codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.compact_prompt.as_deref(), @@ -3270,8 +3420,8 @@ fn cli_override_sets_compact_prompt() -> std::io::Result<()> { Ok(()) } -#[test] -fn loads_compact_prompt_from_file() -> std::io::Result<()> { +#[tokio::test] +async fn loads_compact_prompt_from_file() -> std::io::Result<()> { let codex_home = TempDir::new()?; let workspace = codex_home.path().join("workspace"); std::fs::create_dir_all(&workspace)?; @@ -3289,7 +3439,8 @@ fn loads_compact_prompt_from_file() -> std::io::Result<()> { ..Default::default() }; - let config = Config::load_from_base_config_with_overrides(cfg, overrides, codex_home.abs())?; + let config = + Config::load_from_base_config_with_overrides(cfg, overrides, codex_home.abs()).await?; assert_eq!( config.compact_prompt.as_deref(), @@ -3299,8 +3450,8 @@ fn loads_compact_prompt_from_file() -> std::io::Result<()> { Ok(()) } -#[test] -fn load_config_uses_requirements_guardian_policy_config() -> std::io::Result<()> { +#[tokio::test] +async fn load_config_uses_requirements_guardian_policy_config() -> std::io::Result<()> { let codex_home = TempDir::new()?; let config_layer_stack = ConfigLayerStack::new( Vec::new(), @@ -3322,7 +3473,8 @@ fn load_config_uses_requirements_guardian_policy_config() -> std::io::Result<()> }, codex_home.abs(), config_layer_stack, - )?; + ) + .await?; assert_eq!( config.guardian_policy_config.as_deref(), @@ -3332,8 +3484,8 @@ fn load_config_uses_requirements_guardian_policy_config() -> std::io::Result<()> Ok(()) } -#[test] -fn load_config_ignores_empty_requirements_guardian_policy_config() -> std::io::Result<()> { +#[tokio::test] +async fn load_config_ignores_empty_requirements_guardian_policy_config() -> std::io::Result<()> { let codex_home = TempDir::new()?; let config_layer_stack = ConfigLayerStack::new( Vec::new(), @@ -3353,15 +3505,16 @@ fn load_config_ignores_empty_requirements_guardian_policy_config() -> std::io::R }, codex_home.abs(), config_layer_stack, - )?; + ) + .await?; assert_eq!(config.guardian_policy_config, None); Ok(()) } -#[test] -fn load_config_rejects_missing_agent_role_config_file() -> std::io::Result<()> { +#[tokio::test] +async fn load_config_rejects_missing_agent_role_config_file() -> std::io::Result<()> { let codex_home = TempDir::new()?; let missing_path = codex_home.path().join("agents").join("researcher.toml"); let cfg = ConfigToml { @@ -3385,7 +3538,8 @@ fn load_config_rejects_missing_agent_role_config_file() -> std::io::Result<()> { cfg, ConfigOverrides::default(), codex_home.abs(), - ); + ) + .await; let err = result.expect_err("missing role config file should be rejected"); assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); let message = err.to_string(); @@ -4226,8 +4380,8 @@ model = "gpt-5-mini" Ok(()) } -#[test] -fn load_config_normalizes_agent_role_nickname_candidates() -> std::io::Result<()> { +#[tokio::test] +async fn load_config_normalizes_agent_role_nickname_candidates() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cfg = ConfigToml { agents: Some(AgentsToml { @@ -4253,7 +4407,8 @@ fn load_config_normalizes_agent_role_nickname_candidates() -> std::io::Result<() cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config @@ -4267,8 +4422,8 @@ fn load_config_normalizes_agent_role_nickname_candidates() -> std::io::Result<() Ok(()) } -#[test] -fn load_config_rejects_empty_agent_role_nickname_candidates() -> std::io::Result<()> { +#[tokio::test] +async fn load_config_rejects_empty_agent_role_nickname_candidates() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cfg = ConfigToml { agents: Some(AgentsToml { @@ -4291,7 +4446,8 @@ fn load_config_rejects_empty_agent_role_nickname_candidates() -> std::io::Result cfg, ConfigOverrides::default(), codex_home.abs(), - ); + ) + .await; let err = result.expect_err("empty nickname candidates should be rejected"); assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); assert!( @@ -4302,8 +4458,8 @@ fn load_config_rejects_empty_agent_role_nickname_candidates() -> std::io::Result Ok(()) } -#[test] -fn load_config_rejects_duplicate_agent_role_nickname_candidates() -> std::io::Result<()> { +#[tokio::test] +async fn load_config_rejects_duplicate_agent_role_nickname_candidates() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cfg = ConfigToml { agents: Some(AgentsToml { @@ -4326,7 +4482,8 @@ fn load_config_rejects_duplicate_agent_role_nickname_candidates() -> std::io::Re cfg, ConfigOverrides::default(), codex_home.abs(), - ); + ) + .await; let err = result.expect_err("duplicate nickname candidates should be rejected"); assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); assert!( @@ -4337,8 +4494,8 @@ fn load_config_rejects_duplicate_agent_role_nickname_candidates() -> std::io::Re Ok(()) } -#[test] -fn load_config_rejects_unsafe_agent_role_nickname_candidates() -> std::io::Result<()> { +#[tokio::test] +async fn load_config_rejects_unsafe_agent_role_nickname_candidates() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cfg = ConfigToml { agents: Some(AgentsToml { @@ -4361,7 +4518,8 @@ fn load_config_rejects_unsafe_agent_role_nickname_candidates() -> std::io::Resul cfg, ConfigOverrides::default(), codex_home.abs(), - ); + ) + .await; let err = result.expect_err("unsafe nickname candidates should be rejected"); assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); assert!(err.to_string().contains( @@ -4371,8 +4529,8 @@ fn load_config_rejects_unsafe_agent_role_nickname_candidates() -> std::io::Resul Ok(()) } -#[test] -fn model_catalog_json_loads_from_path() -> std::io::Result<()> { +#[tokio::test] +async fn model_catalog_json_loads_from_path() -> std::io::Result<()> { let codex_home = TempDir::new()?; let catalog_path = codex_home.path().join("catalog.json"); let mut catalog = bundled_models_response() @@ -4392,14 +4550,15 @@ fn model_catalog_json_loads_from_path() -> std::io::Result<()> { cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!(config.model_catalog, Some(catalog)); Ok(()) } -#[test] -fn model_catalog_json_rejects_empty_catalog() -> std::io::Result<()> { +#[tokio::test] +async fn model_catalog_json_rejects_empty_catalog() -> std::io::Result<()> { let codex_home = TempDir::new()?; let catalog_path = codex_home.path().join("catalog.json"); std::fs::write(&catalog_path, r#"{"models":[]}"#)?; @@ -4414,6 +4573,7 @@ fn model_catalog_json_rejects_empty_catalog() -> std::io::Result<()> { ConfigOverrides::default(), codex_home.abs(), ) + .await .expect_err("empty custom catalog should fail config load"); assert_eq!(err.kind(), ErrorKind::InvalidData); @@ -4538,8 +4698,8 @@ model_verbosity = "high" /// /// Note that profiles are the recommended way to specify a group of /// configuration options together. -#[test] -fn test_precedence_fixture_with_o3_profile() -> std::io::Result<()> { +#[tokio::test] +async fn test_precedence_fixture_with_o3_profile() -> std::io::Result<()> { let fixture = create_test_fixture()?; let o3_profile_overrides = ConfigOverrides { @@ -4551,7 +4711,8 @@ fn test_precedence_fixture_with_o3_profile() -> std::io::Result<()> { fixture.cfg.clone(), o3_profile_overrides, fixture.codex_home(), - )?; + ) + .await?; assert_eq!( Config { model: Some("o3".to_string()), @@ -4577,7 +4738,6 @@ fn test_precedence_fixture_with_o3_profile() -> std::io::Result<()> { approvals_reviewer: ApprovalsReviewer::User, enforce_residency: Constrained::allow_any(/*initial_value*/ None), user_instructions: None, - user_instructions_path: None, notify: None, cwd: fixture.cwd(), cli_auth_credentials_store_mode: Default::default(), @@ -4589,7 +4749,7 @@ fn test_precedence_fixture_with_o3_profile() -> std::io::Result<()> { mcp_oauth_callback_port: None, mcp_oauth_callback_url: None, model_providers: fixture.model_provider_map.clone(), - project_doc_max_bytes: PROJECT_DOC_MAX_BYTES, + project_doc_max_bytes: AGENTS_MD_MAX_BYTES, project_doc_fallback_filenames: Vec::new(), tool_output_token_limit: None, agent_max_threads: DEFAULT_AGENT_MAX_THREADS, @@ -4671,8 +4831,8 @@ fn test_precedence_fixture_with_o3_profile() -> std::io::Result<()> { Ok(()) } -#[test] -fn metrics_exporter_defaults_to_statsig_when_missing() -> std::io::Result<()> { +#[tokio::test] +async fn metrics_exporter_defaults_to_statsig_when_missing() -> std::io::Result<()> { let fixture = create_test_fixture()?; let config = Config::load_from_base_config_with_overrides( @@ -4682,14 +4842,15 @@ fn metrics_exporter_defaults_to_statsig_when_missing() -> std::io::Result<()> { ..Default::default() }, fixture.codex_home(), - )?; + ) + .await?; assert_eq!(config.otel.metrics_exporter, OtelExporterKind::Statsig); Ok(()) } -#[test] -fn test_precedence_fixture_with_gpt3_profile() -> std::io::Result<()> { +#[tokio::test] +async fn test_precedence_fixture_with_gpt3_profile() -> std::io::Result<()> { let fixture = create_test_fixture()?; let gpt3_profile_overrides = ConfigOverrides { @@ -4701,7 +4862,8 @@ fn test_precedence_fixture_with_gpt3_profile() -> std::io::Result<()> { fixture.cfg.clone(), gpt3_profile_overrides, fixture.codex_home(), - )?; + ) + .await?; let expected_gpt3_profile_config = Config { model: Some("gpt-3.5-turbo".to_string()), review_model: None, @@ -4726,7 +4888,6 @@ fn test_precedence_fixture_with_gpt3_profile() -> std::io::Result<()> { approvals_reviewer: ApprovalsReviewer::User, enforce_residency: Constrained::allow_any(/*initial_value*/ None), user_instructions: None, - user_instructions_path: None, notify: None, cwd: fixture.cwd(), cli_auth_credentials_store_mode: Default::default(), @@ -4738,7 +4899,7 @@ fn test_precedence_fixture_with_gpt3_profile() -> std::io::Result<()> { mcp_oauth_callback_port: None, mcp_oauth_callback_url: None, model_providers: fixture.model_provider_map.clone(), - project_doc_max_bytes: PROJECT_DOC_MAX_BYTES, + project_doc_max_bytes: AGENTS_MD_MAX_BYTES, project_doc_fallback_filenames: Vec::new(), tool_output_token_limit: None, agent_max_threads: DEFAULT_AGENT_MAX_THREADS, @@ -4829,14 +4990,15 @@ fn test_precedence_fixture_with_gpt3_profile() -> std::io::Result<()> { fixture.cfg.clone(), default_profile_overrides, fixture.codex_home(), - )?; + ) + .await?; assert_eq!(expected_gpt3_profile_config, default_profile_config); Ok(()) } -#[test] -fn test_precedence_fixture_with_zdr_profile() -> std::io::Result<()> { +#[tokio::test] +async fn test_precedence_fixture_with_zdr_profile() -> std::io::Result<()> { let fixture = create_test_fixture()?; let zdr_profile_overrides = ConfigOverrides { @@ -4848,7 +5010,8 @@ fn test_precedence_fixture_with_zdr_profile() -> std::io::Result<()> { fixture.cfg.clone(), zdr_profile_overrides, fixture.codex_home(), - )?; + ) + .await?; let expected_zdr_profile_config = Config { model: Some("o3".to_string()), review_model: None, @@ -4873,7 +5036,6 @@ fn test_precedence_fixture_with_zdr_profile() -> std::io::Result<()> { approvals_reviewer: ApprovalsReviewer::User, enforce_residency: Constrained::allow_any(/*initial_value*/ None), user_instructions: None, - user_instructions_path: None, notify: None, cwd: fixture.cwd(), cli_auth_credentials_store_mode: Default::default(), @@ -4885,7 +5047,7 @@ fn test_precedence_fixture_with_zdr_profile() -> std::io::Result<()> { mcp_oauth_callback_port: None, mcp_oauth_callback_url: None, model_providers: fixture.model_provider_map.clone(), - project_doc_max_bytes: PROJECT_DOC_MAX_BYTES, + project_doc_max_bytes: AGENTS_MD_MAX_BYTES, project_doc_fallback_filenames: Vec::new(), tool_output_token_limit: None, agent_max_threads: DEFAULT_AGENT_MAX_THREADS, @@ -4968,8 +5130,8 @@ fn test_precedence_fixture_with_zdr_profile() -> std::io::Result<()> { Ok(()) } -#[test] -fn test_precedence_fixture_with_gpt5_profile() -> std::io::Result<()> { +#[tokio::test] +async fn test_precedence_fixture_with_gpt5_profile() -> std::io::Result<()> { let fixture = create_test_fixture()?; let gpt5_profile_overrides = ConfigOverrides { @@ -4981,7 +5143,8 @@ fn test_precedence_fixture_with_gpt5_profile() -> std::io::Result<()> { fixture.cfg.clone(), gpt5_profile_overrides, fixture.codex_home(), - )?; + ) + .await?; let expected_gpt5_profile_config = Config { model: Some("gpt-5.1".to_string()), review_model: None, @@ -5006,7 +5169,6 @@ fn test_precedence_fixture_with_gpt5_profile() -> std::io::Result<()> { approvals_reviewer: ApprovalsReviewer::User, enforce_residency: Constrained::allow_any(/*initial_value*/ None), user_instructions: None, - user_instructions_path: None, notify: None, cwd: fixture.cwd(), cli_auth_credentials_store_mode: Default::default(), @@ -5018,7 +5180,7 @@ fn test_precedence_fixture_with_gpt5_profile() -> std::io::Result<()> { mcp_oauth_callback_port: None, mcp_oauth_callback_url: None, model_providers: fixture.model_provider_map.clone(), - project_doc_max_bytes: PROJECT_DOC_MAX_BYTES, + project_doc_max_bytes: AGENTS_MD_MAX_BYTES, project_doc_fallback_filenames: Vec::new(), tool_output_token_limit: None, agent_max_threads: DEFAULT_AGENT_MAX_THREADS, @@ -5101,8 +5263,9 @@ fn test_precedence_fixture_with_gpt5_profile() -> std::io::Result<()> { Ok(()) } -#[test] -fn test_requirements_web_search_mode_allowlist_does_not_warn_when_unset() -> anyhow::Result<()> { +#[tokio::test] +async fn test_requirements_web_search_mode_allowlist_does_not_warn_when_unset() -> anyhow::Result<()> +{ let fixture = create_test_fixture()?; let requirements_toml = crate::config_loader::ConfigRequirementsToml { @@ -5154,7 +5317,8 @@ fn test_requirements_web_search_mode_allowlist_does_not_warn_when_unset() -> any }, fixture.codex_home(), config_layer_stack, - )?; + ) + .await?; assert!( !config @@ -5317,9 +5481,9 @@ fn test_set_default_oss_provider_rejects_legacy_ollama_chat_provider() -> std::i Ok(()) } -#[test] -fn test_load_config_rejects_legacy_ollama_chat_provider_with_helpful_error() -> std::io::Result<()> -{ +#[tokio::test] +async fn test_load_config_rejects_legacy_ollama_chat_provider_with_helpful_error() +-> std::io::Result<()> { let codex_home = TempDir::new()?; let cfg = ConfigToml { model_provider: Some(LEGACY_OLLAMA_CHAT_PROVIDER_ID.to_string()), @@ -5330,7 +5494,8 @@ fn test_load_config_rejects_legacy_ollama_chat_provider_with_helpful_error() -> cfg, ConfigOverrides::default(), codex_home.abs(), - ); + ) + .await; assert!(result.is_err()); let error = result.unwrap_err(); assert_eq!(error.kind(), std::io::ErrorKind::NotFound); @@ -5343,8 +5508,8 @@ fn test_load_config_rejects_legacy_ollama_chat_provider_with_helpful_error() -> Ok(()) } -#[test] -fn test_untrusted_project_gets_workspace_write_sandbox() -> anyhow::Result<()> { +#[tokio::test] +async fn test_untrusted_project_gets_workspace_write_sandbox() -> anyhow::Result<()> { let config_with_untrusted = r#" [projects."/tmp/test"] trust_level = "untrusted" @@ -5353,13 +5518,15 @@ trust_level = "untrusted" let cfg = toml::from_str::(config_with_untrusted) .expect("TOML deserialization should succeed"); - let resolution = cfg.derive_sandbox_policy( - /*sandbox_mode_override*/ None, - /*profile_sandbox_mode*/ None, - WindowsSandboxLevel::Disabled, - &PathBuf::from("/tmp/test"), - /*sandbox_policy_constraint*/ None, - ); + let resolution = cfg + .derive_sandbox_policy( + /*sandbox_mode_override*/ None, + /*profile_sandbox_mode*/ None, + WindowsSandboxLevel::Disabled, + &PathBuf::from("/tmp/test"), + /*sandbox_policy_constraint*/ None, + ) + .await; // Verify that untrusted projects get WorkspaceWrite (or ReadOnly on Windows due to downgrade) if cfg!(target_os = "windows") { @@ -5377,9 +5544,9 @@ trust_level = "untrusted" Ok(()) } -#[test] -fn derive_sandbox_policy_falls_back_to_constraint_value_for_implicit_defaults() -> anyhow::Result<()> -{ +#[tokio::test] +async fn derive_sandbox_policy_falls_back_to_constraint_value_for_implicit_defaults() +-> anyhow::Result<()> { let project_dir = TempDir::new()?; let project_path = project_dir.path().to_path_buf(); let project_key = project_path.to_string_lossy().to_string(); @@ -5405,21 +5572,23 @@ fn derive_sandbox_policy_falls_back_to_constraint_value_for_implicit_defaults() } })?; - let resolution = cfg.derive_sandbox_policy( - /*sandbox_mode_override*/ None, - /*profile_sandbox_mode*/ None, - WindowsSandboxLevel::Disabled, - &project_path, - Some(&constrained), - ); + let resolution = cfg + .derive_sandbox_policy( + /*sandbox_mode_override*/ None, + /*profile_sandbox_mode*/ None, + WindowsSandboxLevel::Disabled, + &project_path, + Some(&constrained), + ) + .await; assert_eq!(resolution, SandboxPolicy::DangerFullAccess); Ok(()) } -#[test] -fn derive_sandbox_policy_preserves_windows_downgrade_for_unsupported_fallback() -> anyhow::Result<()> -{ +#[tokio::test] +async fn derive_sandbox_policy_preserves_windows_downgrade_for_unsupported_fallback() +-> anyhow::Result<()> { let project_dir = TempDir::new()?; let project_path = project_dir.path().to_path_buf(); let project_key = project_path.to_string_lossy().to_string(); @@ -5445,13 +5614,15 @@ fn derive_sandbox_policy_preserves_windows_downgrade_for_unsupported_fallback() } })?; - let resolution = cfg.derive_sandbox_policy( - /*sandbox_mode_override*/ None, - /*profile_sandbox_mode*/ None, - WindowsSandboxLevel::Disabled, - &project_path, - Some(&constrained), - ); + let resolution = cfg + .derive_sandbox_policy( + /*sandbox_mode_override*/ None, + /*profile_sandbox_mode*/ None, + WindowsSandboxLevel::Disabled, + &project_path, + Some(&constrained), + ) + .await; if cfg!(target_os = "windows") { assert_eq!(resolution, SandboxPolicy::new_read_only_policy()); @@ -5579,8 +5750,8 @@ fn config_toml_deserializes_mcp_oauth_callback_url() { ); } -#[test] -fn config_loads_mcp_oauth_callback_port_from_toml() -> std::io::Result<()> { +#[tokio::test] +async fn config_loads_mcp_oauth_callback_port_from_toml() -> std::io::Result<()> { let codex_home = TempDir::new()?; let toml = r#" model = "gpt-5.1" @@ -5593,14 +5764,15 @@ mcp_oauth_callback_port = 5678 cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!(config.mcp_oauth_callback_port, Some(5678)); Ok(()) } -#[test] -fn config_loads_allow_login_shell_from_toml() -> std::io::Result<()> { +#[tokio::test] +async fn config_loads_allow_login_shell_from_toml() -> std::io::Result<()> { let codex_home = TempDir::new()?; let cfg: ConfigToml = toml::from_str( r#" @@ -5614,14 +5786,15 @@ allow_login_shell = false cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert!(!config.permissions.allow_login_shell); Ok(()) } -#[test] -fn config_loads_mcp_oauth_callback_url_from_toml() -> std::io::Result<()> { +#[tokio::test] +async fn config_loads_mcp_oauth_callback_url_from_toml() -> std::io::Result<()> { let codex_home = TempDir::new()?; let toml = r#" model = "gpt-5.1" @@ -5634,7 +5807,8 @@ mcp_oauth_callback_url = "https://example.com/callback" cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.mcp_oauth_callback_url.as_deref(), @@ -5643,8 +5817,8 @@ mcp_oauth_callback_url = "https://example.com/callback" Ok(()) } -#[test] -fn test_untrusted_project_gets_unless_trusted_approval_policy() -> anyhow::Result<()> { +#[tokio::test] +async fn test_untrusted_project_gets_unless_trusted_approval_policy() -> anyhow::Result<()> { let codex_home = TempDir::new()?; let test_project_dir = TempDir::new()?; let test_path = test_project_dir.path(); @@ -5664,7 +5838,8 @@ fn test_untrusted_project_gets_unless_trusted_approval_policy() -> anyhow::Resul ..Default::default() }, codex_home.abs(), - )?; + ) + .await?; // Verify that untrusted projects get UnlessTrusted approval policy assert_eq!( @@ -6359,8 +6534,8 @@ async fn feature_requirements_reject_collab_legacy_alias() { ); } -#[test] -fn tool_suggest_discoverables_load_from_config_toml() -> std::io::Result<()> { +#[tokio::test] +async fn tool_suggest_discoverables_load_from_config_toml() -> std::io::Result<()> { let cfg: ConfigToml = toml::from_str( r#" [tool_suggest] @@ -6398,7 +6573,8 @@ discoverables = [ cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.tool_suggest, @@ -6418,8 +6594,8 @@ discoverables = [ Ok(()) } -#[test] -fn experimental_realtime_start_instructions_load_from_config_toml() -> std::io::Result<()> { +#[tokio::test] +async fn experimental_realtime_start_instructions_load_from_config_toml() -> std::io::Result<()> { let cfg: ConfigToml = toml::from_str( r#" experimental_realtime_start_instructions = "start instructions from config" @@ -6437,7 +6613,8 @@ experimental_realtime_start_instructions = "start instructions from config" cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.experimental_realtime_start_instructions.as_deref(), @@ -6446,8 +6623,8 @@ experimental_realtime_start_instructions = "start instructions from config" Ok(()) } -#[test] -fn experimental_realtime_ws_base_url_loads_from_config_toml() -> std::io::Result<()> { +#[tokio::test] +async fn experimental_realtime_ws_base_url_loads_from_config_toml() -> std::io::Result<()> { let cfg: ConfigToml = toml::from_str( r#" experimental_realtime_ws_base_url = "http://127.0.0.1:8011" @@ -6465,7 +6642,8 @@ experimental_realtime_ws_base_url = "http://127.0.0.1:8011" cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.experimental_realtime_ws_base_url.as_deref(), @@ -6474,8 +6652,8 @@ experimental_realtime_ws_base_url = "http://127.0.0.1:8011" Ok(()) } -#[test] -fn experimental_realtime_ws_backend_prompt_loads_from_config_toml() -> std::io::Result<()> { +#[tokio::test] +async fn experimental_realtime_ws_backend_prompt_loads_from_config_toml() -> std::io::Result<()> { let cfg: ConfigToml = toml::from_str( r#" experimental_realtime_ws_backend_prompt = "prompt from config" @@ -6493,7 +6671,8 @@ experimental_realtime_ws_backend_prompt = "prompt from config" cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.experimental_realtime_ws_backend_prompt.as_deref(), @@ -6502,8 +6681,8 @@ experimental_realtime_ws_backend_prompt = "prompt from config" Ok(()) } -#[test] -fn experimental_realtime_ws_startup_context_loads_from_config_toml() -> std::io::Result<()> { +#[tokio::test] +async fn experimental_realtime_ws_startup_context_loads_from_config_toml() -> std::io::Result<()> { let cfg: ConfigToml = toml::from_str( r#" experimental_realtime_ws_startup_context = "startup context from config" @@ -6521,7 +6700,8 @@ experimental_realtime_ws_startup_context = "startup context from config" cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.experimental_realtime_ws_startup_context.as_deref(), @@ -6530,8 +6710,8 @@ experimental_realtime_ws_startup_context = "startup context from config" Ok(()) } -#[test] -fn experimental_realtime_ws_model_loads_from_config_toml() -> std::io::Result<()> { +#[tokio::test] +async fn experimental_realtime_ws_model_loads_from_config_toml() -> std::io::Result<()> { let cfg: ConfigToml = toml::from_str( r#" experimental_realtime_ws_model = "realtime-test-model" @@ -6549,7 +6729,8 @@ experimental_realtime_ws_model = "realtime-test-model" cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.experimental_realtime_ws_model.as_deref(), @@ -6558,8 +6739,8 @@ experimental_realtime_ws_model = "realtime-test-model" Ok(()) } -#[test] -fn realtime_config_partial_table_uses_realtime_defaults() -> std::io::Result<()> { +#[tokio::test] +async fn realtime_config_partial_table_uses_realtime_defaults() -> std::io::Result<()> { let cfg: ConfigToml = toml::from_str( r#" [realtime] @@ -6573,7 +6754,8 @@ voice = "marin" cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.realtime, @@ -6585,8 +6767,8 @@ voice = "marin" Ok(()) } -#[test] -fn realtime_loads_from_config_toml() -> std::io::Result<()> { +#[tokio::test] +async fn realtime_loads_from_config_toml() -> std::io::Result<()> { let cfg: ConfigToml = toml::from_str( r#" [realtime] @@ -6613,7 +6795,8 @@ voice = "cedar" cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!( config.realtime, @@ -6627,8 +6810,8 @@ voice = "cedar" Ok(()) } -#[test] -fn realtime_audio_loads_from_config_toml() -> std::io::Result<()> { +#[tokio::test] +async fn realtime_audio_loads_from_config_toml() -> std::io::Result<()> { let cfg: ConfigToml = toml::from_str( r#" [audio] @@ -6650,7 +6833,8 @@ speaker = "Desk Speakers" cfg, ConfigOverrides::default(), codex_home.abs(), - )?; + ) + .await?; assert_eq!(config.realtime_audio.microphone.as_deref(), Some("USB Mic")); assert_eq!( diff --git a/codex-rs/core/src/config/edit.rs b/codex-rs/core/src/config/edit.rs index 443d00c93d..5adc0f998c 100644 --- a/codex-rs/core/src/config/edit.rs +++ b/codex-rs/core/src/config/edit.rs @@ -222,6 +222,9 @@ mod document_helpers { if !config.enabled { entry["enabled"] = value(false); } + if let Some(environment) = &config.experimental_environment { + entry["experimental_environment"] = value(environment.clone()); + } if config.required { entry["required"] = value(true); } @@ -234,6 +237,13 @@ mod document_helpers { if let Some(timeout) = config.tool_timeout_sec { entry["tool_timeout_sec"] = value(timeout.as_secs_f64()); } + if let Some(approval_mode) = config.default_tools_approval_mode { + entry["default_tools_approval_mode"] = value(match approval_mode { + AppToolApproval::Auto => "auto", + AppToolApproval::Prompt => "prompt", + AppToolApproval::Approve => "approve", + }); + } if let Some(enabled_tools) = &config.enabled_tools && !enabled_tools.is_empty() { diff --git a/codex-rs/core/src/config/edit_tests.rs b/codex-rs/core/src/config/edit_tests.rs index c9288be30b..4f340d89b3 100644 --- a/codex-rs/core/src/config/edit_tests.rs +++ b/codex-rs/core/src/config/edit_tests.rs @@ -575,12 +575,14 @@ fn blocking_replace_mcp_servers_round_trips() { env_vars: vec!["FOO".to_string()], cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: true, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: Some(vec!["one".to_string(), "two".to_string()]), disabled_tools: None, scopes: None, @@ -602,12 +604,14 @@ fn blocking_replace_mcp_servers_round_trips() { ), env_http_headers: None, }, + experimental_environment: None, enabled: false, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: Some(std::time::Duration::from_secs(5)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: Some(vec!["forbidden".to_string()]), scopes: None, @@ -666,12 +670,14 @@ fn blocking_replace_mcp_servers_serializes_tool_approval_overrides() { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: Some(AppToolApproval::Prompt), enabled_tools: None, disabled_tools: None, scopes: None, @@ -696,6 +702,7 @@ fn blocking_replace_mcp_servers_serializes_tool_approval_overrides() { let expected = "\ [mcp_servers.docs] command = \"docs-server\" +default_tools_approval_mode = \"prompt\" [mcp_servers.docs.tools.search] approval_mode = \"approve\" @@ -727,12 +734,14 @@ foo = { command = "cmd" } env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -779,12 +788,14 @@ foo = { command = "cmd" } # keep me env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: false, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -830,12 +841,14 @@ foo = { command = "cmd", args = ["--flag"] } # keep me env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -882,12 +895,14 @@ foo = { command = "cmd" } env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: false, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index b58acbf651..c4217bdf2a 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -1,3 +1,4 @@ +use crate::agents_md::AgentsMdManager; use crate::config::edit::ConfigEdit; use crate::config::edit::ConfigEditsBuilder; use crate::config_loader::CloudRequirementsLoader; @@ -15,8 +16,6 @@ use crate::config_loader::load_config_layers_state; use crate::config_loader::project_trust_key; use crate::memories::memory_root; use crate::path_utils::normalize_for_native_workdir; -use crate::project_doc::DEFAULT_PROJECT_DOC_FILENAME; -use crate::project_doc::LOCAL_PROJECT_DOC_FILENAME; use crate::unified_exec::DEFAULT_MAX_BACKGROUND_TERMINAL_TIMEOUT_MS; use crate::unified_exec::MIN_EMPTY_YIELD_TIME_MS; use crate::windows_sandbox::WindowsSandboxLevelExt; @@ -120,7 +119,7 @@ pub use codex_git_utils::GhostSnapshotConfig; /// Maximum number of bytes of the documentation that will be embedded. Larger /// files are *silently truncated* to this size so we do not take up too much of /// the context window. -pub(crate) const PROJECT_DOC_MAX_BYTES: usize = 32 * 1024; // 32 KiB +pub(crate) const AGENTS_MD_MAX_BYTES: usize = 32 * 1024; // 32 KiB pub(crate) const DEFAULT_AGENT_MAX_THREADS: Option = Some(6); pub(crate) const DEFAULT_AGENT_MAX_DEPTH: i32 = 1; pub(crate) const DEFAULT_AGENT_JOB_MAX_RUNTIME_SECONDS: Option = None; @@ -169,13 +168,14 @@ fn resolve_mcp_oauth_credentials_store_mode( } #[cfg(test)] -pub(crate) fn test_config() -> Config { +pub(crate) async fn test_config() -> Config { let codex_home = tempfile::tempdir().expect("create temp dir"); Config::load_from_base_config_with_overrides( ConfigToml::default(), ConfigOverrides::default(), AbsolutePathBuf::from_absolute_path(codex_home.path()).expect("temp dir should resolve"), ) + .await .expect("load default test config") } @@ -271,9 +271,6 @@ pub struct Config { /// User-provided instructions from AGENTS.md. pub user_instructions: Option, - /// Path to the global AGENTS file loaded into `user_instructions`. - pub user_instructions_path: Option, - /// Base instructions override. pub base_instructions: Option, @@ -726,6 +723,7 @@ impl ConfigBuilder { codex_home, config_layer_stack, ) + .await } #[cfg(test)] @@ -786,7 +784,7 @@ impl Config { } /// Load a default configuration when user config files are invalid. - pub fn load_default_with_cli_overrides( + pub async fn load_default_with_cli_overrides( cli_overrides: Vec<(String, TomlValue)>, ) -> std::io::Result { let codex_home = find_codex_home()?; @@ -794,11 +792,12 @@ impl Config { codex_home.to_path_buf(), cli_overrides, ) + .await } /// Load a default configuration for a specific Codex home without reading /// user, project, or system config layers. - pub fn load_default_with_cli_overrides_for_codex_home( + pub async fn load_default_with_cli_overrides_for_codex_home( codex_home: PathBuf, cli_overrides: Vec<(String, TomlValue)>, ) -> std::io::Result { @@ -818,6 +817,7 @@ impl Config { codex_home, ConfigLayerStack::default(), ) + .await } /// This is a secondary way of creating [Config], which is appropriate when @@ -1413,22 +1413,24 @@ pub(crate) fn resolve_web_search_mode_for_turn( impl Config { #[cfg(test)] - fn load_from_base_config_with_overrides( + async fn load_from_base_config_with_overrides( cfg: ConfigToml, overrides: ConfigOverrides, codex_home: AbsolutePathBuf, ) -> std::io::Result { // Note this ignores requirements.toml enforcement for tests. let config_layer_stack = ConfigLayerStack::default(); - Self::load_config_with_layer_stack(cfg, overrides, codex_home, config_layer_stack) + Self::load_config_with_layer_stack(cfg, overrides, codex_home, config_layer_stack).await } - pub(crate) fn load_config_with_layer_stack( + pub(crate) async fn load_config_with_layer_stack( cfg: ConfigToml, overrides: ConfigOverrides, codex_home: AbsolutePathBuf, config_layer_stack: ConfigLayerStack, ) -> std::io::Result { + // Keep the large config-construction future off small test thread stacks. + Box::pin(async move { validate_model_providers(&cfg.model_providers) .map_err(|message| std::io::Error::new(std::io::ErrorKind::InvalidInput, message))?; // Ensure that every field of ConfigRequirements is applied to the final @@ -1445,10 +1447,8 @@ impl Config { network: network_requirements, } = config_layer_stack.requirements().clone(); - let (user_instructions, user_instructions_path) = - Self::load_instructions(Some(&codex_home)) - .map(|loaded| (Some(loaded.contents), Some(loaded.path))) - .unwrap_or((None, None)); + let user_instructions = AgentsMdManager::load_global_instructions(Some(&codex_home)) + .map(|loaded| loaded.contents); let mut startup_warnings = Vec::new(); // Destructure ConfigOverrides fully to ensure all overrides are applied. @@ -1547,6 +1547,7 @@ impl Config { .collect(); let active_project = cfg .get_active_project(resolved_cwd.as_path()) + .await .unwrap_or(ProjectConfig { trust_level: None }); let permission_config_syntax = resolve_permission_config_syntax( &config_layer_stack, @@ -1615,6 +1616,7 @@ impl Config { compile_permission_profile( permissions, default_permissions, + resolved_cwd.as_path(), &mut startup_warnings, )?; let mut sandbox_policy = file_system_sandbox_policy @@ -1636,13 +1638,15 @@ impl Config { ) } else { let configured_network_proxy_config = NetworkProxyConfig::default(); - let mut sandbox_policy = cfg.derive_sandbox_policy( - sandbox_mode, - config_profile.sandbox_mode, - windows_sandbox_level, - resolved_cwd.as_path(), - Some(&constrained_sandbox_policy), - ); + let mut sandbox_policy = cfg + .derive_sandbox_policy( + sandbox_mode, + config_profile.sandbox_mode, + windows_sandbox_level, + resolved_cwd.as_path(), + Some(&constrained_sandbox_policy), + ) + .await; if let SandboxPolicy::WorkspaceWrite { writable_roots, .. } = &mut sandbox_policy { for path in &additional_writable_roots { if !writable_roots.iter().any(|existing| existing == path) { @@ -1997,9 +2001,10 @@ impl Config { if effective_sandbox_policy == original_sandbox_policy { file_system_sandbox_policy } else { - FileSystemSandboxPolicy::from_legacy_sandbox_policy( + FileSystemSandboxPolicy::from_legacy_sandbox_policy_preserving_deny_entries( &effective_sandbox_policy, resolved_cwd.as_path(), + &file_system_sandbox_policy, ) }; let effective_file_system_sandbox_policy = effective_file_system_sandbox_policy @@ -2035,7 +2040,6 @@ impl Config { enforce_residency: enforce_residency.value, notify: cfg.notify, user_instructions, - user_instructions_path, base_instructions, personality, developer_instructions, @@ -2060,7 +2064,7 @@ impl Config { mcp_oauth_callback_port: cfg.mcp_oauth_callback_port, mcp_oauth_callback_url: cfg.mcp_oauth_callback_url.clone(), model_providers, - project_doc_max_bytes: cfg.project_doc_max_bytes.unwrap_or(PROJECT_DOC_MAX_BYTES), + project_doc_max_bytes: cfg.project_doc_max_bytes.unwrap_or(AGENTS_MD_MAX_BYTES), project_doc_fallback_filenames: cfg .project_doc_fallback_filenames .unwrap_or_default() @@ -2207,23 +2211,8 @@ impl Config { }, }; Ok(config) - } - - fn load_instructions(codex_dir: Option<&AbsolutePathBuf>) -> Option { - let base = codex_dir?; - for candidate in [LOCAL_PROJECT_DOC_FILENAME, DEFAULT_PROJECT_DOC_FILENAME] { - let path = base.join(candidate); - if let Ok(contents) = std::fs::read_to_string(&path) { - let trimmed = contents.trim(); - if !trimmed.is_empty() { - return Some(LoadedUserInstructions { - contents: trimmed.to_string(), - path, - }); - } - } - } - None + }) + .await } /// If `path` is `Some`, attempts to read the file at the given path and @@ -2282,7 +2271,11 @@ impl Config { } pub fn managed_network_requirements_enabled(&self) -> bool { - self.config_layer_stack + !matches!( + self.permissions.sandbox_policy.get(), + SandboxPolicy::DangerFullAccess + ) && self + .config_layer_stack .requirements_toml() .network .is_some() @@ -2293,11 +2286,6 @@ impl Config { } } -struct LoadedUserInstructions { - contents: String, - path: AbsolutePathBuf, -} - pub(crate) fn uses_deprecated_instructions_file(config_layer_stack: &ConfigLayerStack) -> bool { config_layer_stack .layers_high_to_low() diff --git a/codex-rs/core/src/config/permissions.rs b/codex-rs/core/src/config/permissions.rs index 826d0139bf..1609b696d8 100644 --- a/codex-rs/core/src/config/permissions.rs +++ b/codex-rs/core/src/config/permissions.rs @@ -5,12 +5,14 @@ use std::path::Path; use std::path::PathBuf; use codex_config::permissions_toml::FilesystemPermissionToml; +use codex_config::permissions_toml::FilesystemPermissionsToml; use codex_config::permissions_toml::NetworkToml; use codex_config::permissions_toml::PermissionProfileToml; use codex_config::permissions_toml::PermissionsToml; use codex_network_proxy::NetworkProxyConfig; #[cfg(test)] use codex_network_proxy::NetworkUnixSocketPermission as ProxyNetworkUnixSocketPermission; +use codex_protocol::permissions::FileSystemAccessMode; use codex_protocol::permissions::FileSystemPath; use codex_protocol::permissions::FileSystemSandboxEntry; use codex_protocol::permissions::FileSystemSandboxPolicy; @@ -42,6 +44,7 @@ pub(crate) fn resolve_permission_profile<'a>( pub(crate) fn compile_permission_profile( permissions: &PermissionsToml, profile_name: &str, + policy_cwd: &Path, startup_warnings: &mut Vec, ) -> io::Result<(FileSystemSandboxPolicy, NetworkSandboxPolicy)> { let profile = resolve_permission_profile(permissions, profile_name)?; @@ -54,8 +57,23 @@ pub(crate) fn compile_permission_profile( missing_filesystem_entries_warning(profile_name), ); } else { + if cfg!(not(target_os = "macos")) { + for pattern in unsupported_read_write_glob_paths(filesystem) { + push_warning( + startup_warnings, + format!( + "Filesystem glob `{pattern}` uses `read` or `write` access, which is not fully supported by this platform's sandboxing. Use an exact path or trailing `/**` subtree rule instead. `none` deny-read globs are supported." + ), + ); + } + } for (path, permission) in &filesystem.entries { - compile_filesystem_permission(path, permission, &mut entries, startup_warnings)?; + entries.extend(compile_filesystem_permission( + path, + permission, + policy_cwd, + startup_warnings, + )?); } } } else { @@ -66,7 +84,6 @@ pub(crate) fn compile_permission_profile( } let network_sandbox_policy = compile_network_sandbox_policy(profile.network.as_ref()); - Ok(( FileSystemSandboxPolicy::restricted(entries), network_sandbox_policy, @@ -118,24 +135,71 @@ fn compile_network_sandbox_policy(network: Option<&NetworkToml>) -> NetworkSandb fn compile_filesystem_permission( path: &str, permission: &FilesystemPermissionToml, - entries: &mut Vec, + policy_cwd: &Path, startup_warnings: &mut Vec, -) -> io::Result<()> { +) -> io::Result> { + let mut entries = Vec::new(); match permission { - FilesystemPermissionToml::Access(access) => entries.push(FileSystemSandboxEntry { - path: compile_filesystem_path(path, startup_warnings)?, - access: *access, - }), + FilesystemPermissionToml::Access(access) => { + entries.push(FileSystemSandboxEntry { + path: compile_filesystem_access_path(path, *access, startup_warnings)?, + access: *access, + }); + } FilesystemPermissionToml::Scoped(scoped_entries) => { for (subpath, access) in scoped_entries { - entries.push(FileSystemSandboxEntry { - path: compile_scoped_filesystem_path(path, subpath, startup_warnings)?, - access: *access, - }); + let has_glob = contains_glob_chars(subpath); + let can_compile_as_pattern = match parse_special_path(path) { + Some(FileSystemSpecialPath::ProjectRoots { .. }) | None => true, + Some(_) => false, + }; + if has_glob && *access == FileSystemAccessMode::None && can_compile_as_pattern { + // Scoped glob syntax is a first-class filesystem policy + // pattern entry. Literal scoped paths continue through the + // exact-path parser so existing path semantics stay intact. + let entry = FileSystemSandboxEntry { + path: FileSystemPath::GlobPattern { + pattern: compile_scoped_filesystem_pattern( + path, subpath, *access, policy_cwd, + )?, + }, + access: *access, + }; + entries.push(entry); + } else { + let subpath = compile_read_write_glob_path(subpath, *access)?; + entries.push(FileSystemSandboxEntry { + path: compile_scoped_filesystem_path(path, subpath, startup_warnings)?, + access: *access, + }); + } } } } - Ok(()) + Ok(entries) +} + +fn compile_filesystem_access_path( + path: &str, + access: FileSystemAccessMode, + startup_warnings: &mut Vec, +) -> io::Result { + if !contains_glob_chars(path) { + return compile_filesystem_path(path, startup_warnings); + } + + if access == FileSystemAccessMode::None { + // At this point `path` is an unscoped filesystem table key. Top-level + // glob deny entries still go through the absolute-path parser before + // becoming policy patterns; relative project-root glob syntax is + // handled by `compile_scoped_filesystem_pattern`. + return Ok(FileSystemPath::GlobPattern { + pattern: parse_absolute_path(path)?.to_string_lossy().into_owned(), + }); + } + + let path = compile_read_write_glob_path(path, access)?; + compile_filesystem_path(path, startup_warnings) } fn compile_filesystem_path( @@ -186,6 +250,97 @@ fn compile_scoped_filesystem_path( Ok(FileSystemPath::Path { path }) } +fn compile_scoped_filesystem_pattern( + path: &str, + subpath: &str, + access: FileSystemAccessMode, + policy_cwd: &Path, +) -> io::Result { + // Pattern entries currently mean deny-read only. Supporting broader access + // modes here would imply glob-based read/write allow semantics that the + // sandbox policy does not express yet. + if access != FileSystemAccessMode::None { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("filesystem glob subpath `{subpath}` only supports `none` access"), + )); + } + let subpath = parse_relative_subpath(subpath)?; + + match parse_special_path(path) { + Some(FileSystemSpecialPath::ProjectRoots { .. }) => { + // `:project_roots` is represented as a special path, but current + // filesystem-policy resolution defines it relative to the session + // cwd. Use the same policy cwd here so glob entries and exact + // scoped entries resolve consistently. + Ok( + AbsolutePathBuf::resolve_path_against_base(&subpath, policy_cwd) + .to_string_lossy() + .to_string(), + ) + } + Some(_) => Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("filesystem path `{path}` does not support nested entries"), + )), + None => { + let base = parse_absolute_path(path)?; + Ok(base.join(&subpath).to_string_lossy().to_string()) + } + } +} + +fn compile_read_write_glob_path(path: &str, access: FileSystemAccessMode) -> io::Result<&str> { + if !contains_glob_chars(path) { + return Ok(path); + } + + let path_without_trailing_glob = remove_trailing_glob_suffix(path); + if !contains_glob_chars(path_without_trailing_glob) { + return Ok(path_without_trailing_glob); + } + + Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "filesystem glob path `{path}` only supports `none` access; use an exact path or trailing `/**` for `{access}` subtree access" + ), + )) +} + +fn unsupported_read_write_glob_paths(filesystem: &FilesystemPermissionsToml) -> Vec { + let mut patterns = Vec::new(); + for (path, permission) in &filesystem.entries { + match permission { + FilesystemPermissionToml::Access(access) => { + if *access != FileSystemAccessMode::None + && contains_glob_chars(remove_trailing_glob_suffix(path)) + { + patterns.push(path.clone()); + } + } + FilesystemPermissionToml::Scoped(scoped_entries) => { + for (subpath, access) in scoped_entries { + if *access != FileSystemAccessMode::None + && contains_glob_chars(remove_trailing_glob_suffix(subpath)) + { + patterns.push(format!("{path}/{subpath}")); + } + } + } + } + } + patterns +} + +fn contains_glob_chars(path: &str) -> bool { + path.chars().any(|ch| matches!(ch, '*' | '?' | '[' | ']')) +} + +fn remove_trailing_glob_suffix(path: &str) -> &str { + path.strip_suffix("/**").unwrap_or(path) +} + // WARNING: keep this parser forward-compatible. // Adding a new `:special_path` must not make older Codex versions reject the // config. Unknown values intentionally round-trip through diff --git a/codex-rs/core/src/config/permissions_tests.rs b/codex-rs/core/src/config/permissions_tests.rs index 268346794c..7116e32e70 100644 --- a/codex-rs/core/src/config/permissions_tests.rs +++ b/codex-rs/core/src/config/permissions_tests.rs @@ -2,6 +2,7 @@ use super::*; use crate::config::Config; use crate::config::ConfigOverrides; use codex_config::config_toml::ConfigToml; +use codex_config::permissions_toml::FilesystemPermissionToml; use codex_config::permissions_toml::FilesystemPermissionsToml; use codex_config::permissions_toml::NetworkDomainPermissionToml; use codex_config::permissions_toml::NetworkDomainPermissionsToml; @@ -10,6 +11,11 @@ use codex_config::permissions_toml::NetworkUnixSocketPermissionToml; use codex_config::permissions_toml::NetworkUnixSocketPermissionsToml; use codex_config::permissions_toml::PermissionProfileToml; use codex_config::permissions_toml::PermissionsToml; +use codex_protocol::permissions::FileSystemAccessMode; +use codex_protocol::permissions::FileSystemPath; +use codex_protocol::permissions::FileSystemSandboxEntry; +use codex_protocol::permissions::FileSystemSandboxPolicy; +use codex_protocol::permissions::FileSystemSpecialPath; use codex_utils_absolute_path::AbsolutePathBuf; use pretty_assertions::assert_eq; use std::collections::BTreeMap; @@ -24,8 +30,8 @@ fn normalize_absolute_path_for_platform_simplifies_windows_verbatim_paths() { assert_eq!(parsed, PathBuf::from(r"D:\c\x\worktrees\2508\swift-base")); } -#[test] -fn restricted_read_implicitly_allows_helper_executables() -> std::io::Result<()> { +#[tokio::test] +async fn restricted_read_implicitly_allows_helper_executables() -> std::io::Result<()> { let temp_dir = TempDir::new()?; let cwd = temp_dir.path().join("workspace"); let codex_home = temp_dir.path().join(".codex"); @@ -64,7 +70,8 @@ fn restricted_read_implicitly_allows_helper_executables() -> std::io::Result<()> ..Default::default() }, AbsolutePathBuf::from_absolute_path(&codex_home)?, - )?; + ) + .await?; let expected_zsh = AbsolutePathBuf::try_from(zsh_path)?; let expected_allowed_arg0_dir = AbsolutePathBuf::try_from(allowed_arg0_dir)?; @@ -215,3 +222,89 @@ fn network_toml_overlays_unix_socket_permissions_by_path() { }) ); } + +#[test] +fn read_write_glob_warnings_skip_supported_deny_read_globs_and_trailing_subpaths() { + let filesystem = FilesystemPermissionsToml { + entries: BTreeMap::from([ + ( + "/tmp/**/*.log".to_string(), + FilesystemPermissionToml::Access(FileSystemAccessMode::Read), + ), + ( + "/tmp/cache/**".to_string(), + FilesystemPermissionToml::Access(FileSystemAccessMode::Write), + ), + ( + ":project_roots".to_string(), + FilesystemPermissionToml::Scoped(BTreeMap::from([ + ("**/*.env".to_string(), FileSystemAccessMode::None), + ("docs/**".to_string(), FileSystemAccessMode::Read), + ("src/**/*.rs".to_string(), FileSystemAccessMode::Write), + ])), + ), + ]), + }; + + assert_eq!( + unsupported_read_write_glob_paths(&filesystem), + vec![ + "/tmp/**/*.log".to_string(), + ":project_roots/src/**/*.rs".to_string() + ], + "`none` glob patterns are supported as deny-read rules; only `read`/`write` globs should warn" + ); +} + +#[test] +fn read_write_trailing_glob_suffix_compiles_as_subpath() -> std::io::Result<()> { + let cwd = TempDir::new()?; + let mut startup_warnings = Vec::new(); + let (file_system_policy, _) = compile_permission_profile( + &PermissionsToml { + entries: BTreeMap::from([( + "workspace".to_string(), + PermissionProfileToml { + filesystem: Some(FilesystemPermissionsToml { + entries: BTreeMap::from([( + ":project_roots".to_string(), + FilesystemPermissionToml::Scoped(BTreeMap::from([( + "docs/**".to_string(), + FileSystemAccessMode::Read, + )])), + )]), + }), + network: None, + }, + )]), + }, + "workspace", + cwd.path(), + &mut startup_warnings, + )?; + + assert_eq!( + file_system_policy, + FileSystemSandboxPolicy::restricted(vec![FileSystemSandboxEntry { + path: FileSystemPath::Special { + value: FileSystemSpecialPath::project_roots(Some("docs".into())), + }, + access: FileSystemAccessMode::Read, + }]), + "trailing /** should compile as a subtree path instead of a glob pattern" + ); + Ok(()) +} + +#[test] +fn read_write_glob_patterns_still_reject_non_subpath_globs() { + let err = compile_read_write_glob_path("src/**/*.rs", FileSystemAccessMode::Read) + .expect_err("non-subpath read/write glob should be rejected"); + + assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); + assert!( + err.to_string() + .contains("filesystem glob path `src/**/*.rs` only supports `none` access"), + "{err}" + ); +} diff --git a/codex-rs/core/src/config/service_tests.rs b/codex-rs/core/src/config/service_tests.rs index ae62326605..faaffb34df 100644 --- a/codex-rs/core/src/config/service_tests.rs +++ b/codex-rs/core/src/config/service_tests.rs @@ -164,6 +164,49 @@ async fn write_value_supports_nested_app_paths() -> Result<()> { Ok(()) } +#[tokio::test] +async fn write_value_supports_custom_mcp_server_default_tool_approval_mode() -> Result<()> { + let tmp = tempdir().expect("tempdir"); + std::fs::write( + tmp.path().join(CONFIG_TOML_FILE), + "[mcp_servers.docs]\ncommand = \"docs-server\"\n", + )?; + + let service = ConfigService::without_managed_config_for_tests(tmp.path().to_path_buf()); + service + .write_value(ConfigValueWriteParams { + file_path: Some(tmp.path().join(CONFIG_TOML_FILE).display().to_string()), + key_path: "mcp_servers.docs.default_tools_approval_mode".to_string(), + value: serde_json::json!("approve"), + merge_strategy: MergeStrategy::Replace, + expected_version: None, + }) + .await + .expect("write mcp server default_tools_approval_mode succeeds"); + + let contents = std::fs::read_to_string(tmp.path().join(CONFIG_TOML_FILE))?; + assert!(contents.contains("default_tools_approval_mode = \"approve\"")); + + let read = service + .read(ConfigReadParams { + include_layers: false, + cwd: None, + }) + .await + .expect("config read succeeds"); + + assert_eq!( + read.config + .additional + .get("mcp_servers") + .and_then(|servers| servers.get("docs")) + .and_then(|docs| docs.get("default_tools_approval_mode")), + Some(&serde_json::json!("approve")) + ); + + Ok(()) +} + #[tokio::test] async fn read_includes_origins_and_layers() { let tmp = tempdir().expect("tempdir"); diff --git a/codex-rs/core/src/config_loader/mod.rs b/codex-rs/core/src/config_loader/mod.rs index 36fc956bf4..f7830b7089 100644 --- a/codex-rs/core/src/config_loader/mod.rs +++ b/codex-rs/core/src/config_loader/mod.rs @@ -650,7 +650,7 @@ async fn project_trust_context( let projects = project_trust_config.projects.unwrap_or_default(); let project_root_key = project_trust_key(project_root.as_path()); - let repo_root = resolve_root_git_project_for_trust(cwd.as_path()); + let repo_root = resolve_root_git_project_for_trust(cwd.as_path()).await; let repo_root_key = repo_root.as_ref().map(|root| project_trust_key(root)); let projects_trust = projects diff --git a/codex-rs/core/src/connectors.rs b/codex-rs/core/src/connectors.rs index eef6d5747b..103ce251fc 100644 --- a/codex-rs/core/src/connectors.rs +++ b/codex-rs/core/src/connectors.rs @@ -1,8 +1,5 @@ -use std::collections::BTreeSet; use std::collections::HashMap; use std::collections::HashSet; -use std::env; -use std::path::PathBuf; use std::sync::Arc; use std::sync::LazyLock; use std::sync::Mutex as StdMutex; @@ -28,7 +25,6 @@ use crate::codex::INITIAL_SUBMIT_ID; use crate::config::Config; use crate::config_loader::AppsRequirementsToml; use crate::mcp::McpManager; -use crate::plugins::AppConnectorId; use crate::plugins::PluginsManager; use crate::plugins::list_tool_suggest_discoverable_plugins; use codex_config::types::AppToolApproval; @@ -38,18 +34,15 @@ use codex_features::Feature; use codex_login::AuthManager; use codex_login::CodexAuth; use codex_login::default_client::create_client; -use codex_login::default_client::is_first_party_chat_originator; use codex_login::default_client::originator; use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; use codex_mcp::McpConnectionManager; -use codex_mcp::SandboxState; use codex_mcp::ToolInfo; use codex_mcp::ToolPluginProvenance; use codex_mcp::codex_apps_tools_cache_key; use codex_mcp::compute_auth_statuses; use codex_mcp::with_codex_apps_mcp; -pub use codex_connectors::CONNECTORS_CACHE_TTL; const CONNECTORS_READY_TIMEOUT_ON_EMPTY_TOOLS: Duration = Duration::from_secs(30); const DIRECTORY_CONNECTORS_TIMEOUT: Duration = Duration::from_secs(60); @@ -125,13 +118,15 @@ pub(crate) async fn list_tool_suggest_discoverable_tools_with_auth( let directory_connectors = list_directory_connectors_for_tool_suggest_with_auth(config, auth).await?; let connector_ids = tool_suggest_connector_ids(config).await; - let discoverable_connectors = filter_tool_suggest_discoverable_connectors( - directory_connectors, - accessible_connectors, - &connector_ids, - ) - .into_iter() - .map(DiscoverableTool::from); + let discoverable_connectors = + codex_connectors::filter::filter_tool_suggest_discoverable_connectors( + directory_connectors, + accessible_connectors, + &connector_ids, + originator().value.as_str(), + ) + .into_iter() + .map(DiscoverableTool::from); let discoverable_plugins = list_tool_suggest_discoverable_plugins(config) .await? .into_iter() @@ -154,7 +149,12 @@ pub async fn list_cached_accessible_connectors_from_mcp_tools( return Some(Vec::new()); } let cache_key = accessible_connectors_cache_key(config, auth.as_ref()); - read_cached_accessible_connectors(&cache_key).map(filter_disallowed_connectors) + read_cached_accessible_connectors(&cache_key).map(|connectors| { + codex_connectors::filter::filter_disallowed_connectors( + connectors, + originator().value.as_str(), + ) + }) } pub(crate) fn refresh_accessible_connectors_cache_from_mcp_tools( @@ -167,8 +167,10 @@ pub(crate) fn refresh_accessible_connectors_cache_from_mcp_tools( } let cache_key = accessible_connectors_cache_key(config, auth); - let accessible_connectors = - filter_disallowed_connectors(accessible_connectors_from_mcp_tools(mcp_tools)); + let accessible_connectors = codex_connectors::filter::filter_disallowed_connectors( + accessible_connectors_from_mcp_tools(mcp_tools), + originator().value.as_str(), + ); write_cached_accessible_connectors(cache_key, &accessible_connectors); } @@ -205,7 +207,10 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_options_and_status( let tool_plugin_provenance = mcp_manager.tool_plugin_provenance(config).await; if !force_refetch && let Some(cached_connectors) = read_cached_accessible_connectors(&cache_key) { - let cached_connectors = filter_disallowed_connectors(cached_connectors); + let cached_connectors = codex_connectors::filter::filter_disallowed_connectors( + cached_connectors, + originator().value.as_str(), + ); let cached_connectors = with_app_plugin_sources(cached_connectors, &tool_plugin_provenance); return Ok(AccessibleConnectorsStatus { connectors: cached_connectors, @@ -228,13 +233,6 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_options_and_status( let (tx_event, rx_event) = unbounded(); drop(rx_event); - let sandbox_state = SandboxState { - sandbox_policy: SandboxPolicy::new_read_only_policy(), - codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(), - sandbox_cwd: env::current_dir().unwrap_or_else(|_| PathBuf::from("/")), - use_legacy_landlock: config.features.use_legacy_landlock(), - }; - let (mcp_connection_manager, cancel_token) = McpConnectionManager::new( &mcp_servers, config.mcp_oauth_credentials_store_mode, @@ -242,7 +240,7 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_options_and_status( &config.permissions.approval_policy, INITIAL_SUBMIT_ID.to_owned(), tx_event, - sandbox_state, + SandboxPolicy::new_read_only_policy(), config.codex_home.to_path_buf(), codex_apps_tools_cache_key(auth.as_ref()), ToolPluginProvenance::default(), @@ -303,8 +301,10 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_options_and_status( cancel_token.cancel(); } - let accessible_connectors = - filter_disallowed_connectors(accessible_connectors_from_mcp_tools(&tools)); + let accessible_connectors = codex_connectors::filter::filter_disallowed_connectors( + accessible_connectors_from_mcp_tools(&tools), + originator().value.as_str(), + ); if codex_apps_ready || !accessible_connectors.is_empty() { write_cached_accessible_connectors(cache_key, &accessible_connectors); } @@ -367,35 +367,11 @@ fn write_cached_accessible_connectors( .unwrap_or_else(std::sync::PoisonError::into_inner); *cache_guard = Some(CachedAccessibleConnectors { key: cache_key, - expires_at: Instant::now() + CONNECTORS_CACHE_TTL, + expires_at: Instant::now() + codex_connectors::CONNECTORS_CACHE_TTL, connectors: connectors.to_vec(), }); } -fn filter_tool_suggest_discoverable_connectors( - directory_connectors: Vec, - accessible_connectors: &[AppInfo], - discoverable_connector_ids: &HashSet, -) -> Vec { - let accessible_connector_ids: HashSet<&str> = accessible_connectors - .iter() - .filter(|connector| connector.is_accessible) - .map(|connector| connector.id.as_str()) - .collect(); - - let mut connectors = filter_disallowed_connectors(directory_connectors) - .into_iter() - .filter(|connector| !accessible_connector_ids.contains(connector.id.as_str())) - .filter(|connector| discoverable_connector_ids.contains(connector.id.as_str())) - .collect::>(); - connectors.sort_by(|left, right| { - left.name - .cmp(&right.name) - .then_with(|| left.id.cmp(&right.id)) - }); - connectors -} - async fn tool_suggest_connector_ids(config: &Config) -> HashSet { let mut connector_ids = PluginsManager::new(config.codex_home.to_path_buf()) .plugins_for_config(config) @@ -503,14 +479,6 @@ async fn chatgpt_get_request_with_token( } } -pub fn connector_display_label(connector: &AppInfo) -> String { - format_connector_label(&connector.name, &connector.id) -} - -pub fn connector_mention_slug(connector: &AppInfo) -> String { - sanitize_slug(&connector_display_label(connector)) -} - pub(crate) fn accessible_connectors_from_mcp_tools( mcp_tools: &HashMap, ) -> Vec { @@ -521,114 +489,14 @@ pub(crate) fn accessible_connectors_from_mcp_tools( return None; } let connector_id = tool.connector_id.as_deref()?; - Some(( - connector_id.to_string(), - normalize_connector_value(tool.connector_name.as_deref()), - normalize_connector_value(tool.connector_description.as_deref()), - tool.plugin_display_names.clone(), - )) - }); - collect_accessible_connectors(tools) -} - -pub fn merge_connectors( - connectors: Vec, - accessible_connectors: Vec, -) -> Vec { - let mut merged: HashMap = connectors - .into_iter() - .map(|mut connector| { - connector.is_accessible = false; - (connector.id.clone(), connector) + Some(codex_connectors::accessible::AccessibleConnectorTool { + connector_id: connector_id.to_string(), + connector_name: tool.connector_name.clone(), + connector_description: tool.connector_description.clone(), + plugin_display_names: tool.plugin_display_names.clone(), }) - .collect(); - - for mut connector in accessible_connectors { - connector.is_accessible = true; - let connector_id = connector.id.clone(); - if let Some(existing) = merged.get_mut(&connector_id) { - existing.is_accessible = true; - if existing.name == existing.id && connector.name != connector.id { - existing.name = connector.name; - } - if existing.description.is_none() && connector.description.is_some() { - existing.description = connector.description; - } - if existing.logo_url.is_none() && connector.logo_url.is_some() { - existing.logo_url = connector.logo_url; - } - if existing.logo_url_dark.is_none() && connector.logo_url_dark.is_some() { - existing.logo_url_dark = connector.logo_url_dark; - } - if existing.distribution_channel.is_none() && connector.distribution_channel.is_some() { - existing.distribution_channel = connector.distribution_channel; - } - existing - .plugin_display_names - .extend(connector.plugin_display_names); - } else { - merged.insert(connector_id, connector); - } - } - - let mut merged = merged.into_values().collect::>(); - for connector in &mut merged { - if connector.install_url.is_none() { - connector.install_url = Some(connector_install_url(&connector.name, &connector.id)); - } - connector.plugin_display_names.sort_unstable(); - connector.plugin_display_names.dedup(); - } - merged.sort_by(|left, right| { - right - .is_accessible - .cmp(&left.is_accessible) - .then_with(|| left.name.cmp(&right.name)) - .then_with(|| left.id.cmp(&right.id)) }); - merged -} - -pub fn merge_plugin_apps( - connectors: Vec, - plugin_apps: Vec, -) -> Vec { - let mut merged = connectors; - let mut connector_ids = merged - .iter() - .map(|connector| connector.id.clone()) - .collect::>(); - - for connector_id in plugin_apps { - if connector_ids.insert(connector_id.0.clone()) { - merged.push(plugin_app_to_app_info(connector_id)); - } - } - - merged.sort_by(|left, right| { - right - .is_accessible - .cmp(&left.is_accessible) - .then_with(|| left.name.cmp(&right.name)) - .then_with(|| left.id.cmp(&right.id)) - }); - merged -} - -pub fn merge_plugin_apps_with_accessible( - plugin_apps: Vec, - accessible_connectors: Vec, -) -> Vec { - let accessible_connector_ids: HashSet<&str> = accessible_connectors - .iter() - .map(|connector| connector.id.as_str()) - .collect(); - let plugin_connectors = plugin_apps - .into_iter() - .filter(|connector_id| accessible_connector_ids.contains(connector_id.0.as_str())) - .map(plugin_app_to_app_info) - .collect::>(); - merge_connectors(plugin_connectors, accessible_connectors) + codex_connectors::accessible::collect_accessible_connectors(tools) } pub fn with_app_enabled_state(mut connectors: Vec, config: &Config) -> Vec { @@ -701,45 +569,6 @@ pub(crate) fn codex_app_tool_is_enabled(config: &Config, tool_info: &ToolInfo) - .enabled } -const DISALLOWED_CONNECTOR_IDS: &[&str] = &[ - "asdk_app_6938a94a61d881918ef32cb999ff937c", - "connector_2b0a9009c9c64bf9933a3dae3f2b1254", - "connector_3f8d1a79f27c4c7ba1a897ab13bf37dc", - "connector_68de829bf7648191acd70a907364c67c", - "connector_68e004f14af881919eb50893d3d9f523", - "connector_69272cb413a081919685ec3c88d1744e", -]; -const FIRST_PARTY_CHAT_DISALLOWED_CONNECTOR_IDS: &[&str] = - &["connector_0f9c9d4592e54d0a9a12b3f44a1e2010"]; -const DISALLOWED_CONNECTOR_PREFIX: &str = "connector_openai_"; - -pub fn filter_disallowed_connectors(connectors: Vec) -> Vec { - filter_disallowed_connectors_for_originator(connectors, originator().value.as_str()) -} - -fn filter_disallowed_connectors_for_originator( - connectors: Vec, - originator_value: &str, -) -> Vec { - connectors - .into_iter() - .filter(|connector| { - is_connector_id_allowed_for_originator(connector.id.as_str(), originator_value) - }) - .collect() -} - -fn is_connector_id_allowed_for_originator(connector_id: &str, originator_value: &str) -> bool { - let disallowed_connector_ids = if is_first_party_chat_originator(originator_value) { - FIRST_PARTY_CHAT_DISALLOWED_CONNECTOR_IDS - } else { - DISALLOWED_CONNECTOR_IDS - }; - - !connector_id.starts_with(DISALLOWED_CONNECTOR_PREFIX) - && !disallowed_connector_ids.contains(&connector_id) -} - fn read_apps_config(config: &Config) -> Option { let apps_config = read_user_apps_config(config); let had_apps_config = apps_config.is_some(); @@ -861,125 +690,6 @@ fn app_tool_policy_from_apps_config( AppToolPolicy { enabled, approval } } -fn collect_accessible_connectors(tools: I) -> Vec -where - I: IntoIterator, Option, Vec)>, -{ - let mut connectors: HashMap)> = HashMap::new(); - for (connector_id, connector_name, connector_description, plugin_display_names) in tools { - let connector_name = connector_name.unwrap_or_else(|| connector_id.clone()); - if let Some((existing, existing_plugin_display_names)) = connectors.get_mut(&connector_id) { - if existing.name == connector_id && connector_name != connector_id { - existing.name = connector_name; - } - if existing.description.is_none() && connector_description.is_some() { - existing.description = connector_description; - } - existing_plugin_display_names.extend(plugin_display_names); - } else { - connectors.insert( - connector_id.clone(), - ( - AppInfo { - id: connector_id.clone(), - name: connector_name, - description: connector_description, - logo_url: None, - logo_url_dark: None, - distribution_channel: None, - branding: None, - app_metadata: None, - labels: None, - install_url: None, - is_accessible: true, - is_enabled: true, - plugin_display_names: Vec::new(), - }, - plugin_display_names - .into_iter() - .collect::>(), - ), - ); - } - } - let mut accessible: Vec = connectors - .into_values() - .map(|(mut connector, plugin_display_names)| { - connector.plugin_display_names = plugin_display_names.into_iter().collect(); - connector.install_url = Some(connector_install_url(&connector.name, &connector.id)); - connector - }) - .collect(); - accessible.sort_by(|left, right| { - right - .is_accessible - .cmp(&left.is_accessible) - .then_with(|| left.name.cmp(&right.name)) - .then_with(|| left.id.cmp(&right.id)) - }); - accessible -} - -fn plugin_app_to_app_info(connector_id: AppConnectorId) -> AppInfo { - // Leave the placeholder name as the connector id so merge_connectors() can - // replace it with canonical app metadata from directory fetches or - // connector_name values from codex_apps tool discovery. - let connector_id = connector_id.0; - let name = connector_id.clone(); - AppInfo { - id: connector_id.clone(), - name: name.clone(), - description: None, - logo_url: None, - logo_url_dark: None, - distribution_channel: None, - branding: None, - app_metadata: None, - labels: None, - install_url: Some(connector_install_url(&name, &connector_id)), - is_accessible: false, - is_enabled: true, - plugin_display_names: Vec::new(), - } -} - -fn normalize_connector_value(value: Option<&str>) -> Option { - value - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_string) -} - -pub fn connector_install_url(name: &str, connector_id: &str) -> String { - let slug = sanitize_slug(name); - format!("https://chatgpt.com/apps/{slug}/{connector_id}") -} - -pub fn sanitize_name(name: &str) -> String { - sanitize_slug(name).replace("-", "_") -} - -fn sanitize_slug(name: &str) -> String { - let mut normalized = String::with_capacity(name.len()); - for character in name.chars() { - if character.is_ascii_alphanumeric() { - normalized.push(character.to_ascii_lowercase()); - } else { - normalized.push('-'); - } - } - let normalized = normalized.trim_matches('-'); - if normalized.is_empty() { - "app".to_string() - } else { - normalized.to_string() - } -} - -fn format_connector_label(name: &str, _id: &str) -> String { - name.to_string() -} - #[cfg(test)] #[path = "connectors_tests.rs"] mod tests; diff --git a/codex-rs/core/src/connectors_tests.rs b/codex-rs/core/src/connectors_tests.rs index c9f77d0469..e462c7939e 100644 --- a/codex-rs/core/src/connectors_tests.rs +++ b/codex-rs/core/src/connectors_tests.rs @@ -11,6 +11,13 @@ use codex_config::types::AppConfig; use codex_config::types::AppToolConfig; use codex_config::types::AppToolsConfig; use codex_config::types::AppsDefaultConfig; +use codex_connectors::filter::filter_disallowed_connectors; +use codex_connectors::filter::filter_tool_suggest_discoverable_connectors; +use codex_connectors::merge::merge_connectors; +use codex_connectors::merge::plugin_connector_to_app_info; +use codex_connectors::metadata::connector_install_url; +use codex_connectors::metadata::connector_mention_slug; +use codex_connectors::metadata::sanitize_name; use codex_features::Feature; use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; use codex_mcp::ToolInfo; @@ -138,7 +145,7 @@ fn with_accessible_connectors_cache_cleared(f: impl FnOnce() -> R) -> R { #[test] fn merge_connectors_replaces_plugin_placeholder_name_with_accessible_name() { - let plugin = plugin_app_to_app_info(AppConnectorId("calendar".to_string())); + let plugin = plugin_connector_to_app_info("calendar".to_string()); let accessible = google_calendar_accessible_connector(&[]); let merged = merge_connectors(vec![plugin], vec![accessible]); @@ -281,7 +288,7 @@ async fn refresh_accessible_connectors_cache_from_mcp_tools_writes_latest_instal #[test] fn merge_connectors_unions_and_dedupes_plugin_display_names() { - let mut plugin = plugin_app_to_app_info(AppConnectorId("calendar".to_string())); + let mut plugin = plugin_connector_to_app_info("calendar".to_string()); plugin.plugin_display_names = plugin_names(&["sample", "alpha", "sample"]); let accessible = google_calendar_accessible_connector(&["beta", "alpha"]); @@ -975,33 +982,40 @@ fn app_tool_policy_matches_prefix_stripped_tool_name_for_tool_config() { #[test] fn filter_disallowed_connectors_allows_non_disallowed_connectors() { - let filtered = filter_disallowed_connectors(vec![app("asdk_app_hidden"), app("alpha")]); + let filtered = + filter_disallowed_connectors(vec![app("asdk_app_hidden"), app("alpha")], "codex_cli"); assert_eq!(filtered, vec![app("asdk_app_hidden"), app("alpha")]); } #[test] fn filter_disallowed_connectors_filters_openai_prefix() { - let filtered = filter_disallowed_connectors(vec![ - app("connector_openai_foo"), - app("connector_openai_bar"), - app("gamma"), - ]); + let filtered = filter_disallowed_connectors( + vec![ + app("connector_openai_foo"), + app("connector_openai_bar"), + app("gamma"), + ], + "codex_cli", + ); assert_eq!(filtered, vec![app("gamma")]); } #[test] fn filter_disallowed_connectors_filters_disallowed_connector_ids() { - let filtered = filter_disallowed_connectors(vec![ - app("asdk_app_6938a94a61d881918ef32cb999ff937c"), - app("connector_3f8d1a79f27c4c7ba1a897ab13bf37dc"), - app("delta"), - ]); + let filtered = filter_disallowed_connectors( + vec![ + app("asdk_app_6938a94a61d881918ef32cb999ff937c"), + app("connector_3f8d1a79f27c4c7ba1a897ab13bf37dc"), + app("delta"), + ], + "codex_cli", + ); assert_eq!(filtered, vec![app("delta")]); } #[test] fn first_party_chat_originator_filters_target_and_openai_prefixed_connectors() { - let filtered = filter_disallowed_connectors_for_originator( + let filtered = filter_disallowed_connectors( vec![ app("connector_openai_foo"), app("asdk_app_6938a94a61d881918ef32cb999ff937c"), @@ -1064,6 +1078,7 @@ fn filter_tool_suggest_discoverable_connectors_keeps_only_plugin_backed_uninstal "connector_2128aebfecb84f64a069897515042a44".to_string(), "connector_68df038e0ba48191908c8434991bbac2".to_string(), ]), + "codex_cli", ); assert_eq!( @@ -1103,6 +1118,7 @@ fn filter_tool_suggest_discoverable_connectors_excludes_accessible_apps_even_whe "connector_2128aebfecb84f64a069897515042a44".to_string(), "connector_68df038e0ba48191908c8434991bbac2".to_string(), ]), + "codex_cli", ); assert_eq!(filtered, Vec::::new()); diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rs index 9720492c1f..ff71b797be 100644 --- a/codex-rs/core/src/context_manager/history_tests.rs +++ b/codex-rs/core/src/context_manager/history_tests.rs @@ -133,6 +133,7 @@ fn reference_context_item() -> TurnContextItem { approval_policy: AskForApproval::OnRequest, sandbox_policy: SandboxPolicy::new_read_only_policy(), network: None, + file_system_sandbox_policy: None, model: "gpt-test".to_string(), personality: None, collaboration_mode: None, diff --git a/codex-rs/core/src/exec.rs b/codex-rs/core/src/exec.rs index d5b96902dc..396b7701be 100644 --- a/codex-rs/core/src/exec.rs +++ b/codex-rs/core/src/exec.rs @@ -251,8 +251,24 @@ pub fn build_exec_request( codex_linux_sandbox_exe: &Option, use_legacy_landlock: bool, ) -> Result { - let windows_sandbox_level = params.windows_sandbox_level; - let enforce_managed_network = params.network.is_some(); + let ExecParams { + command, + cwd, + mut env, + expiration, + capture_policy, + network, + windows_sandbox_level, + windows_sandbox_private_desktop, + + // TODO: Should arg0 be set on the ExecRequest that is returned? + arg0: _, + // These fields are related to approvals, so can be ignored here. + justification: _, + sandbox_permissions: _, + } = params; + + let enforce_managed_network = network.is_some(); let sandbox_type = select_process_exec_tool_sandbox_type( file_system_sandbox_policy, network_sandbox_policy, @@ -261,19 +277,6 @@ pub fn build_exec_request( ); tracing::debug!("Sandbox type: {sandbox_type:?}"); - let ExecParams { - command, - cwd, - mut env, - expiration, - capture_policy, - network, - sandbox_permissions: _, - windows_sandbox_level, - windows_sandbox_private_desktop, - justification: _, - arg0: _, - } = params; if let Some(network) = network.as_ref() { network.apply_to_env(&mut env); } @@ -357,7 +360,8 @@ pub(crate) async fn execute_exec_request( windows_sandbox_level, windows_sandbox_private_desktop, sandbox_policy, - file_system_sandbox_policy, + // TODO(mbolin): Use file_system_sandbox_policy instead of sandbox_policy. + file_system_sandbox_policy: _, network_sandbox_policy, windows_sandbox_filesystem_overrides, arg0, @@ -378,21 +382,40 @@ pub(crate) async fn execute_exec_request( }; let start = Instant::now(); - let raw_output_result = exec( + let raw_output_result = get_raw_output_result( params, - sandbox, - &sandbox_policy, - &file_system_sandbox_policy, - windows_sandbox_filesystem_overrides.as_ref(), network_sandbox_policy, stdout_stream, after_spawn, + sandbox, + &sandbox_policy, + windows_sandbox_filesystem_overrides.as_ref(), ) .await; let duration = start.elapsed(); finalize_exec_result(raw_output_result, sandbox, duration) } +async fn get_raw_output_result( + params: ExecParams, + network_sandbox_policy: NetworkSandboxPolicy, + stdout_stream: Option, + after_spawn: Option>, + #[cfg_attr(not(windows), allow(unused_variables))] sandbox: SandboxType, + #[cfg_attr(not(windows), allow(unused_variables))] sandbox_policy: &SandboxPolicy, + #[cfg_attr(not(windows), allow(unused_variables))] windows_sandbox_filesystem_overrides: Option< + &WindowsSandboxFilesystemOverrides, + >, +) -> Result { + #[cfg(target_os = "windows")] + if sandbox == SandboxType::WindowsRestrictedToken { + return exec_windows_sandbox(params, sandbox_policy, windows_sandbox_filesystem_overrides) + .await; + } + + exec(params, network_sandbox_policy, stdout_stream, after_spawn).await +} + #[cfg(target_os = "windows")] fn extract_create_process_as_user_error_code(err: &str) -> Option { let marker = "CreateProcessAsUserW failed: "; @@ -799,26 +822,24 @@ fn aggregate_output( } } -#[allow(clippy::too_many_arguments)] +/// This is a general-purpose function for executing a command specified by +/// [ExecParams]. Events are reported via `stdout_stream`, if specified, and +/// `after_spawn` is invoked once the child process has been spawned, before +/// output consumption begins. +/// +/// `network_sandbox_policy` is used to determine whether +/// CODEX_SANDBOX_NETWORK_DISABLED=1 is added to the environment of the spawned +/// process. +/// +/// Note this command does not apply any sandboxing logic. The caller is +/// responsible for constructing [ExecParams::command] to include any sandboxing +/// wrapper args, as appropriate. async fn exec( params: ExecParams, - _sandbox: SandboxType, - _sandbox_policy: &SandboxPolicy, - _file_system_sandbox_policy: &FileSystemSandboxPolicy, - _windows_sandbox_filesystem_overrides: Option<&WindowsSandboxFilesystemOverrides>, network_sandbox_policy: NetworkSandboxPolicy, stdout_stream: Option, after_spawn: Option>, ) -> Result { - #[cfg(target_os = "windows")] - if _sandbox == SandboxType::WindowsRestrictedToken { - return exec_windows_sandbox( - params, - _sandbox_policy, - _windows_sandbox_filesystem_overrides, - ) - .await; - } let ExecParams { command, cwd, @@ -827,8 +848,14 @@ async fn exec( arg0, expiration, capture_policy, + + // If applicable, these fields should have been honored upstream of + // this exec call. windows_sandbox_level: _, - .. + windows_sandbox_private_desktop: _, + // These fields are related to approvals, so can be ignored here. + sandbox_permissions: _, + justification: _, } = params; if let Some(network) = network.as_ref() { network.apply_to_env(&mut env); @@ -954,6 +981,9 @@ pub(crate) fn resolve_windows_restricted_token_filesystem_overrides( if !file_system_sandbox_policy .get_unreadable_roots_with_cwd(sandbox_policy_cwd) .is_empty() + || !file_system_sandbox_policy + .get_unreadable_globs_with_cwd(sandbox_policy_cwd) + .is_empty() { return Err( "windows unelevated restricted-token sandbox cannot enforce unreadable split filesystem carveouts directly; refusing to run unsandboxed" @@ -1069,6 +1099,9 @@ pub(crate) fn resolve_windows_elevated_filesystem_overrides( if !file_system_sandbox_policy .get_unreadable_roots_with_cwd(sandbox_policy_cwd) .is_empty() + || !file_system_sandbox_policy + .get_unreadable_globs_with_cwd(sandbox_policy_cwd) + .is_empty() { return Err( "windows elevated sandbox cannot enforce unreadable split filesystem carveouts directly; refusing to run unsandboxed" diff --git a/codex-rs/core/src/exec_tests.rs b/codex-rs/core/src/exec_tests.rs index 796062b3fc..1cfa87ff3f 100644 --- a/codex-rs/core/src/exec_tests.rs +++ b/codex-rs/core/src/exec_tests.rs @@ -277,10 +277,6 @@ async fn exec_full_buffer_capture_ignores_expiration() -> Result<()> { justification: None, arg0: None, }, - SandboxType::None, - &SandboxPolicy::DangerFullAccess, - &FileSystemSandboxPolicy::unrestricted(), - /*windows_sandbox_filesystem_overrides*/ None, NetworkSandboxPolicy::Enabled, /*stdout_stream*/ None, /*after_spawn*/ None, @@ -317,10 +313,6 @@ async fn exec_full_buffer_capture_keeps_io_drain_timeout_when_descendant_holds_p justification: None, arg0: None, }, - SandboxType::None, - &SandboxPolicy::DangerFullAccess, - &FileSystemSandboxPolicy::unrestricted(), - /*windows_sandbox_filesystem_overrides*/ None, NetworkSandboxPolicy::Enabled, /*stdout_stream*/ None, /*after_spawn*/ None, @@ -814,6 +806,53 @@ fn windows_elevated_rejects_unreadable_split_carveouts() { ); } +#[test] +fn windows_elevated_rejects_unreadable_globs() { + let temp_dir = tempfile::TempDir::new().expect("tempdir"); + let policy = SandboxPolicy::WorkspaceWrite { + writable_roots: vec![], + read_only_access: codex_protocol::protocol::ReadOnlyAccess::FullAccess, + network_access: false, + exclude_tmpdir_env_var: true, + exclude_slash_tmp: true, + }; + let file_system_policy = FileSystemSandboxPolicy::restricted(vec![ + codex_protocol::permissions::FileSystemSandboxEntry { + path: codex_protocol::permissions::FileSystemPath::Special { + value: codex_protocol::permissions::FileSystemSpecialPath::Root, + }, + access: codex_protocol::permissions::FileSystemAccessMode::Read, + }, + codex_protocol::permissions::FileSystemSandboxEntry { + path: codex_protocol::permissions::FileSystemPath::Special { + value: codex_protocol::permissions::FileSystemSpecialPath::CurrentWorkingDirectory, + }, + access: codex_protocol::permissions::FileSystemAccessMode::Write, + }, + codex_protocol::permissions::FileSystemSandboxEntry { + path: codex_protocol::permissions::FileSystemPath::GlobPattern { + pattern: "**/*.env".to_string(), + }, + access: codex_protocol::permissions::FileSystemAccessMode::None, + }, + ]); + + assert_eq!( + unsupported_windows_restricted_token_sandbox_reason( + SandboxType::WindowsRestrictedToken, + &policy, + &file_system_policy, + NetworkSandboxPolicy::Restricted, + &temp_dir.path().abs(), + WindowsSandboxLevel::Elevated, + ), + Some( + "windows elevated sandbox cannot enforce unreadable split filesystem carveouts directly; refusing to run unsandboxed" + .to_string() + ) + ); +} + #[test] fn windows_elevated_rejects_reopened_writable_descendants() { let temp_dir = tempfile::TempDir::new().expect("tempdir"); @@ -931,10 +970,6 @@ async fn kill_child_process_group_kills_grandchildren_on_timeout() -> Result<()> let output = exec( params, - SandboxType::None, - &SandboxPolicy::new_read_only_policy(), - &FileSystemSandboxPolicy::from(&SandboxPolicy::new_read_only_policy()), - /*windows_sandbox_filesystem_overrides*/ None, NetworkSandboxPolicy::Restricted, /*stdout_stream*/ None, /*after_spawn*/ None, diff --git a/codex-rs/core/src/external_agent_config.rs b/codex-rs/core/src/external_agent_config.rs index 1699239ab4..f0f4443a75 100644 --- a/codex-rs/core/src/external_agent_config.rs +++ b/codex-rs/core/src/external_agent_config.rs @@ -1,9 +1,15 @@ +use crate::config::Config; +use crate::config::ConfigBuilder; use crate::plugins::MarketplaceAddRequest; use crate::plugins::PluginId; use crate::plugins::PluginInstallRequest; use crate::plugins::PluginsManager; use crate::plugins::add_marketplace; -use codex_utils_absolute_path::AbsolutePathBuf; +use crate::plugins::configured_plugins_from_stack; +use crate::plugins::find_marketplace_manifest_path; +use crate::plugins::parse_marketplace_source; +use codex_core_plugins::marketplace::MarketplacePluginInstallPolicy; +use codex_protocol::protocol::Product; use serde_json::Value as JsonValue; use std::collections::BTreeMap; use std::collections::HashSet; @@ -16,8 +22,8 @@ use toml::Value as TomlValue; const EXTERNAL_AGENT_CONFIG_DETECT_METRIC: &str = "codex.external_agent_config.detect"; const EXTERNAL_AGENT_CONFIG_IMPORT_METRIC: &str = "codex.external_agent_config.import"; -// Installed marketplace roots always expose their manifest at this relative path. -const INSTALLED_MARKETPLACE_MANIFEST_RELATIVE_PATH: &str = ".agents/plugins/marketplace.json"; +const EXTERNAL_AGENT_DIR: &str = ".claude"; +const EXTERNAL_AGENT_CONFIG_MD: &str = "CLAUDE.md"; #[derive(Debug, Clone, PartialEq, Eq)] pub struct ExternalAgentConfigDetectOptions { @@ -84,20 +90,21 @@ impl ExternalAgentConfigService { } } - pub fn detect( + pub async fn detect( &self, params: ExternalAgentConfigDetectOptions, ) -> io::Result> { let mut items = Vec::new(); if params.include_home { - self.detect_migrations(/*repo_root*/ None, &mut items)?; + self.detect_migrations(/*repo_root*/ None, &mut items) + .await?; } for cwd in params.cwds.as_deref().unwrap_or(&[]) { let Some(repo_root) = find_repo_root(Some(cwd))? else { continue; }; - self.detect_migrations(Some(&repo_root), &mut items)?; + self.detect_migrations(Some(&repo_root), &mut items).await?; } Ok(items) @@ -138,22 +145,11 @@ impl ExternalAgentConfigService { let cwd = migration_item.cwd; let details = migration_item.details; tokio::spawn(async move { - match service.import_plugins(cwd.as_deref(), details).await { - Ok(outcome) => { - tracing::info!( - succeeded_marketplaces = outcome.succeeded_marketplaces.len(), - succeeded_plugin_ids = outcome.succeeded_plugin_ids.len(), - failed_marketplaces = outcome.failed_marketplaces.len(), - failed_plugin_ids = outcome.failed_plugin_ids.len(), - "external agent config plugin import completed" - ); - } - Err(err) => { - tracing::warn!( - error = %err, - "external agent config plugin import failed" - ); - } + if let Err(err) = service.import_plugins(cwd.as_deref(), details).await { + tracing::warn!( + error = %err, + "external agent config plugin import failed" + ); } }); emit_migration_metric( @@ -169,7 +165,7 @@ impl ExternalAgentConfigService { Ok(()) } - fn detect_migrations( + async fn detect_migrations( &self, repo_root: Option<&Path>, items: &mut Vec, @@ -177,7 +173,7 @@ impl ExternalAgentConfigService { let cwd = repo_root.map(Path::to_path_buf); let source_settings = repo_root.map_or_else( || self.external_agent_home.join("settings.json"), - |repo_root| repo_root.join(".claude").join("settings.json"), + |repo_root| repo_root.join(EXTERNAL_AGENT_DIR).join("settings.json"), ); let settings = read_external_settings(&source_settings)?; let target_config = repo_root.map_or_else( @@ -185,7 +181,7 @@ impl ExternalAgentConfigService { |repo_root| repo_root.join(".codex").join("config.toml"), ); if let Some(settings) = settings.as_ref() { - let migrated = build_config_from_external(&settings)?; + let migrated = build_config_from_external(settings)?; if !is_empty_toml_table(&migrated) { let mut should_include = true; if target_config.exists() { @@ -220,16 +216,9 @@ impl ExternalAgentConfigService { } } - self.detect_plugin_migration( - source_settings.as_path(), - cwd.clone(), - settings.as_ref(), - items, - ); - let source_skills = repo_root.map_or_else( || self.external_agent_home.join("skills"), - |repo_root| repo_root.join(".claude").join("skills"), + |repo_root| repo_root.join(EXTERNAL_AGENT_DIR).join("skills"), ); let target_skills = repo_root.map_or_else( || self.home_target_skills_dir(), @@ -240,7 +229,7 @@ impl ExternalAgentConfigService { items.push(ExternalAgentConfigMigrationItem { item_type: ExternalAgentConfigMigrationItemType::Skills, description: format!( - "Copy skill folders from {} to {}", + "Migrate skills from {} to {}", source_skills.display(), target_skills.display() ), @@ -257,7 +246,7 @@ impl ExternalAgentConfigService { let source_agents_md = if let Some(repo_root) = repo_root { find_repo_agents_md_source(repo_root)? } else { - let path = self.external_agent_home.join("CLAUDE.md"); + let path = self.external_agent_home.join(EXTERNAL_AGENT_CONFIG_MD); is_non_empty_text_file(&path)?.then_some(path) }; let target_agents_md = repo_root.map_or_else( @@ -274,7 +263,7 @@ impl ExternalAgentConfigService { source_agents_md.display(), target_agents_md.display() ), - cwd, + cwd: cwd.clone(), details: None, }); emit_migration_metric( @@ -284,6 +273,43 @@ impl ExternalAgentConfigService { ); } + if let Some(settings) = settings.as_ref() { + match ConfigBuilder::default() + .codex_home(self.codex_home.clone()) + .fallback_cwd(Some(self.codex_home.clone())) + .build() + .await + { + Ok(config) => { + let configured_plugin_ids = + configured_plugins_from_stack(&config.config_layer_stack) + .into_keys() + .collect::>(); + let configured_marketplace_plugins = configured_marketplace_plugins( + &config, + &PluginsManager::new(self.codex_home.clone()), + )?; + if let Some(item) = self.detect_plugin_migration( + source_settings.as_path(), + repo_root.unwrap_or(self.external_agent_home.as_path()), + cwd.clone(), + settings, + &configured_plugin_ids, + &configured_marketplace_plugins, + ) { + items.push(item); + } + } + Err(err) => { + tracing::warn!( + error = %err, + settings_path = %source_settings.display(), + "skipping external agent plugin migration detection because config load failed" + ); + } + } + } + Ok(()) } @@ -297,25 +323,30 @@ impl ExternalAgentConfigService { fn detect_plugin_migration( &self, source_settings: &Path, + source_root: &Path, cwd: Option, - settings: Option<&JsonValue>, - items: &mut Vec, - ) { - let Some(plugin_details) = settings.and_then(extract_plugin_migration_details) else { - return; - }; - - items.push(ExternalAgentConfigMigrationItem { - item_type: ExternalAgentConfigMigrationItemType::Plugins, - description: format!("Import enabled plugins from {}", source_settings.display()), - cwd, - details: Some(plugin_details), - }); + settings: &JsonValue, + configured_plugin_ids: &HashSet, + configured_marketplace_plugins: &BTreeMap>, + ) -> Option { + let plugin_details = extract_plugin_migration_details( + settings, + source_root, + configured_plugin_ids, + configured_marketplace_plugins, + )?; emit_migration_metric( EXTERNAL_AGENT_CONFIG_DETECT_METRIC, ExternalAgentConfigMigrationItemType::Plugins, /*skills_count*/ None, ); + + Some(ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::Plugins, + description: format!("Import enabled plugins from {}", source_settings.display()), + cwd, + details: Some(plugin_details), + }) } async fn import_plugins( @@ -337,8 +368,14 @@ impl ExternalAgentConfigService { .iter() .map(|plugin_name| format!("{plugin_name}@{marketplace_name}")) .collect::>(); - let import_source = - read_marketplace_import_source(cwd, &self.external_agent_home, &marketplace_name)?; + let source_settings = cwd.map_or_else( + || self.external_agent_home.join("settings.json"), + |cwd| cwd.join(EXTERNAL_AGENT_DIR).join("settings.json"), + ); + let source_root = cwd.unwrap_or(self.external_agent_home.as_path()); + let import_source = read_external_settings(&source_settings)?.and_then(|settings| { + collect_marketplace_import_sources(&settings, source_root).remove(&marketplace_name) + }); let Some(import_source) = import_source else { outcome.failed_marketplaces.push(marketplace_name); outcome.failed_plugin_ids.extend(plugin_ids); @@ -352,23 +389,17 @@ impl ExternalAgentConfigService { let add_marketplace_outcome = add_marketplace(self.codex_home.clone(), request).await; let marketplace_path = match add_marketplace_outcome { Ok(add_marketplace_outcome) => { - match AbsolutePathBuf::try_from( - add_marketplace_outcome - .installed_root - .join(INSTALLED_MARKETPLACE_MANIFEST_RELATIVE_PATH), - ) { - Ok(path) => { - outcome - .succeeded_marketplaces - .push(marketplace_name.clone()); - path - } - Err(_) => { - outcome.failed_marketplaces.push(marketplace_name); - outcome.failed_plugin_ids.extend(plugin_ids); - continue; - } - } + let Some(marketplace_path) = find_marketplace_manifest_path( + add_marketplace_outcome.installed_root.as_path(), + ) else { + outcome.failed_marketplaces.push(marketplace_name); + outcome.failed_plugin_ids.extend(plugin_ids); + continue; + }; + outcome + .succeeded_marketplaces + .push(marketplace_name.clone()); + marketplace_path } Err(_) => { outcome.failed_marketplaces.push(marketplace_name); @@ -400,7 +431,7 @@ impl ExternalAgentConfigService { fn import_config(&self, cwd: Option<&Path>) -> io::Result<()> { let (source_settings, target_config) = if let Some(repo_root) = find_repo_root(cwd)? { ( - repo_root.join(".claude").join("settings.json"), + repo_root.join(EXTERNAL_AGENT_DIR).join("settings.json"), repo_root.join(".codex").join("config.toml"), ) } else if cwd.is_some_and(|cwd| !cwd.as_os_str().is_empty()) { @@ -452,7 +483,7 @@ impl ExternalAgentConfigService { fn import_skills(&self, cwd: Option<&Path>) -> io::Result { let (source_skills, target_skills) = if let Some(repo_root) = find_repo_root(cwd)? { ( - repo_root.join(".claude").join("skills"), + repo_root.join(EXTERNAL_AGENT_DIR).join("skills"), repo_root.join(".agents").join("skills"), ) } else if cwd.is_some_and(|cwd| !cwd.as_os_str().is_empty()) { @@ -499,7 +530,7 @@ impl ExternalAgentConfigService { return Ok(()); } else { ( - self.external_agent_home.join("CLAUDE.md"), + self.external_agent_home.join(EXTERNAL_AGENT_CONFIG_MD), self.codex_home.join("AGENTS.md"), ) }; @@ -520,10 +551,10 @@ impl ExternalAgentConfigService { fn default_external_agent_home() -> PathBuf { if let Some(home) = std::env::var_os("HOME").or_else(|| std::env::var_os("USERPROFILE")) { - return PathBuf::from(home).join(".claude"); + return PathBuf::from(home).join(EXTERNAL_AGENT_DIR); } - PathBuf::from(".claude") + PathBuf::from(EXTERNAL_AGENT_DIR) } fn read_external_settings(path: &Path) -> io::Result> { @@ -537,12 +568,37 @@ fn read_external_settings(path: &Path) -> io::Result> { Ok(Some(settings)) } -fn extract_plugin_migration_details(settings: &JsonValue) -> Option { +fn extract_plugin_migration_details( + settings: &JsonValue, + source_root: &Path, + configured_plugin_ids: &HashSet, + configured_marketplace_plugins: &BTreeMap>, +) -> Option { + let loadable_marketplaces = collect_marketplace_import_sources(settings, source_root) + .into_iter() + .filter_map(|(marketplace_name, source)| { + parse_marketplace_source(&source.source, source.ref_name) + .ok() + .map(|_| marketplace_name) + }) + .collect::>(); let mut plugins = BTreeMap::new(); - for plugin_id in collect_enabled_plugins(settings) { + for plugin_id in collect_enabled_plugins(settings) + .into_iter() + .filter(|plugin_id| !configured_plugin_ids.contains(plugin_id)) + { let Ok(plugin_id) = PluginId::parse(&plugin_id) else { continue; }; + if let Some(installable_plugins) = + configured_marketplace_plugins.get(&plugin_id.marketplace_name) + { + if !installable_plugins.contains(&plugin_id.plugin_name) { + continue; + } + } else if !loadable_marketplaces.contains(&plugin_id.marketplace_name) { + continue; + } let plugin_group = plugins .entry(plugin_id.marketplace_name.clone()) .or_insert_with(|| PluginsMigration { @@ -591,8 +647,40 @@ fn collect_enabled_plugins(settings: &JsonValue) -> Vec { .collect() } +fn configured_marketplace_plugins( + config: &Config, + plugins_manager: &PluginsManager, +) -> io::Result>> { + let marketplaces = plugins_manager + .list_marketplaces_for_config(config, &[]) + .map_err(|err| { + invalid_data_error(format!("failed to list configured marketplaces: {err}")) + })?; + let mut marketplace_plugins = BTreeMap::new(); + for marketplace in marketplaces.marketplaces { + let plugins = marketplace + .plugins + .into_iter() + .filter(|plugin| { + plugin.policy.installation != MarketplacePluginInstallPolicy::NotAvailable + }) + .filter(|plugin| { + plugin + .policy + .products + .as_deref() + .is_none_or(|products| Product::Codex.matches_product_restriction(products)) + }) + .map(|plugin| plugin.name) + .collect::>(); + marketplace_plugins.insert(marketplace.name, plugins); + } + Ok(marketplace_plugins) +} + fn collect_marketplace_import_sources( settings: &JsonValue, + source_root: &Path, ) -> BTreeMap { let Some(extra_known_marketplaces) = settings .as_object() @@ -623,6 +711,7 @@ fn collect_marketplace_import_sources( if source.is_empty() { return None; } + let source = resolve_external_marketplace_source(&source, source_root); let ref_name = source_fields .get("ref") @@ -643,20 +732,16 @@ struct MarketplaceImportSource { ref_name: Option, } -fn read_marketplace_import_source( - cwd: Option<&Path>, - external_agent_home: &Path, - marketplace_name: &str, -) -> io::Result> { - let source_settings = cwd.map_or_else( - || external_agent_home.join("settings.json"), - |cwd| cwd.join(".claude").join("settings.json"), - ); - let Some(settings) = read_external_settings(&source_settings)? else { - return Ok(None); - }; +fn resolve_external_marketplace_source(source: &str, source_root: &Path) -> String { + if !looks_like_relative_local_path(source) { + return source.to_string(); + } - Ok(collect_marketplace_import_sources(&settings).remove(marketplace_name)) + source_root.join(source).display().to_string() +} + +fn looks_like_relative_local_path(source: &str) -> bool { + source.starts_with("./") || source.starts_with("../") || source == "." || source == ".." } fn find_repo_root(cwd: Option<&Path>) -> io::Result> { @@ -741,8 +826,10 @@ fn is_non_empty_text_file(path: &Path) -> io::Result { fn find_repo_agents_md_source(repo_root: &Path) -> io::Result> { for candidate in [ - repo_root.join("CLAUDE.md"), - repo_root.join(".claude").join("CLAUDE.md"), + repo_root.join(EXTERNAL_AGENT_CONFIG_MD), + repo_root + .join(EXTERNAL_AGENT_DIR) + .join(EXTERNAL_AGENT_CONFIG_MD), ] { if is_non_empty_text_file(&candidate)? { return Ok(Some(candidate)); @@ -791,7 +878,11 @@ fn rewrite_and_copy_text_file(source: &Path, target: &Path) -> io::Result<()> { } fn rewrite_external_agent_terms(content: &str) -> String { - let mut rewritten = replace_case_insensitive_with_boundaries(content, "claude.md", "AGENTS.md"); + let mut rewritten = replace_case_insensitive_with_boundaries( + content, + &EXTERNAL_AGENT_CONFIG_MD.to_ascii_lowercase(), + "AGENTS.md", + ); for from in [ "claude code", "claude-code", diff --git a/codex-rs/core/src/external_agent_config_tests.rs b/codex-rs/core/src/external_agent_config_tests.rs index f2afab0170..f01a40e3d2 100644 --- a/codex-rs/core/src/external_agent_config_tests.rs +++ b/codex-rs/core/src/external_agent_config_tests.rs @@ -5,13 +5,16 @@ use tempfile::TempDir; fn fixture_paths() -> (TempDir, PathBuf, PathBuf) { let root = TempDir::new().expect("create tempdir"); - let claude_home = root.path().join(".claude"); + let external_agent_home = root.path().join(".claude"); let codex_home = root.path().join(".codex"); - (root, claude_home, codex_home) + (root, external_agent_home, codex_home) } -fn service_for_paths(claude_home: PathBuf, codex_home: PathBuf) -> ExternalAgentConfigService { - ExternalAgentConfigService::new_for_test(codex_home, claude_home) +fn service_for_paths( + external_agent_home: PathBuf, + codex_home: PathBuf, +) -> ExternalAgentConfigService { + ExternalAgentConfigService::new_for_test(codex_home, external_agent_home) } fn github_plugin_details() -> MigrationDetails { @@ -23,26 +26,28 @@ fn github_plugin_details() -> MigrationDetails { } } -#[test] -fn detect_home_lists_config_skills_and_agents_md() { - let (_root, claude_home, codex_home) = fixture_paths(); +#[tokio::test] +async fn detect_home_lists_config_skills_and_agents_md() { + let (_root, external_agent_home, codex_home) = fixture_paths(); let agents_skills = codex_home .parent() .map(|parent| parent.join(".agents").join("skills")) .unwrap_or_else(|| PathBuf::from(".agents").join("skills")); - fs::create_dir_all(claude_home.join("skills").join("skill-a")).expect("create skills"); - fs::write(claude_home.join("CLAUDE.md"), "claude rules").expect("write claude md"); + fs::create_dir_all(external_agent_home.join("skills").join("skill-a")).expect("create skills"); + fs::write(external_agent_home.join("CLAUDE.md"), "claude rules") + .expect("write external agent md"); fs::write( - claude_home.join("settings.json"), + external_agent_home.join("settings.json"), r#"{"model":"claude","env":{"FOO":"bar"}}"#, ) .expect("write settings"); - let items = service_for_paths(claude_home.clone(), codex_home.clone()) + let items = service_for_paths(external_agent_home.clone(), codex_home.clone()) .detect(ExternalAgentConfigDetectOptions { include_home: true, cwds: None, }) + .await .expect("detect"); let expected = vec![ @@ -50,7 +55,7 @@ fn detect_home_lists_config_skills_and_agents_md() { item_type: ExternalAgentConfigMigrationItemType::Config, description: format!( "Migrate {} into {}", - claude_home.join("settings.json").display(), + external_agent_home.join("settings.json").display(), codex_home.join("config.toml").display() ), cwd: None, @@ -59,8 +64,8 @@ fn detect_home_lists_config_skills_and_agents_md() { ExternalAgentConfigMigrationItem { item_type: ExternalAgentConfigMigrationItemType::Skills, description: format!( - "Copy skill folders from {} to {}", - claude_home.join("skills").display(), + "Migrate skills from {} to {}", + external_agent_home.join("skills").display(), agents_skills.display() ), cwd: None, @@ -70,7 +75,7 @@ fn detect_home_lists_config_skills_and_agents_md() { item_type: ExternalAgentConfigMigrationItemType::AgentsMd, description: format!( "Import {} to {}", - claude_home.join("CLAUDE.md").display(), + external_agent_home.join("CLAUDE.md").display(), codex_home.join("AGENTS.md").display() ), cwd: None, @@ -81,8 +86,8 @@ fn detect_home_lists_config_skills_and_agents_md() { assert_eq!(items, expected); } -#[test] -fn detect_repo_lists_agents_md_for_each_cwd() { +#[tokio::test] +async fn detect_repo_lists_agents_md_for_each_cwd() { let root = TempDir::new().expect("create tempdir"); let repo_root = root.path().join("repo"); let nested = repo_root.join("nested").join("child"); @@ -95,6 +100,7 @@ fn detect_repo_lists_agents_md_for_each_cwd() { include_home: false, cwds: Some(vec![nested, repo_root.clone()]), }) + .await .expect("detect"); let expected = vec![ @@ -123,27 +129,110 @@ fn detect_repo_lists_agents_md_for_each_cwd() { assert_eq!(items, expected); } +#[tokio::test] +async fn detect_repo_still_reports_non_plugin_items_when_home_config_is_invalid() { + let root = TempDir::new().expect("create tempdir"); + let repo_root = root.path().join("repo"); + let codex_home = root.path().join(".codex"); + fs::create_dir_all(repo_root.join(".git")).expect("create git dir"); + fs::create_dir_all(repo_root.join(".claude").join("skills").join("skill-a")) + .expect("create repo skills"); + fs::create_dir_all(&codex_home).expect("create codex home"); + fs::write(codex_home.join("config.toml"), "this is not valid = [toml") + .expect("write invalid codex config"); + fs::write( + repo_root.join(".claude").join("settings.json"), + r#"{"env":{"FOO":"bar"}}"#, + ) + .expect("write settings"); + fs::write( + repo_root + .join(".claude") + .join("skills") + .join("skill-a") + .join("SKILL.md"), + "Use Claude Code and CLAUDE utilities.", + ) + .expect("write skill"); + fs::write( + repo_root.join(".claude").join("CLAUDE.md"), + "Claude code guidance", + ) + .expect("write agents"); + + let items = service_for_paths(root.path().join(".claude"), codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: false, + cwds: Some(vec![repo_root.clone()]), + }) + .await + .expect("detect"); + + assert_eq!( + items, + vec![ + ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::Config, + description: format!( + "Migrate {} into {}", + repo_root.join(".claude").join("settings.json").display(), + repo_root.join(".codex").join("config.toml").display() + ), + cwd: Some(repo_root.clone()), + details: None, + }, + ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::Skills, + description: format!( + "Migrate skills from {} to {}", + repo_root.join(".claude").join("skills").display(), + repo_root.join(".agents").join("skills").display() + ), + cwd: Some(repo_root.clone()), + details: None, + }, + ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::AgentsMd, + description: format!( + "Import {} to {}", + repo_root.join(".claude").join("CLAUDE.md").display(), + repo_root.join("AGENTS.md").display(), + ), + cwd: Some(repo_root), + details: None, + }, + ] + ); +} + #[tokio::test] async fn import_home_migrates_supported_config_fields_skills_and_agents_md() { - let (_root, claude_home, codex_home) = fixture_paths(); + let (_root, external_agent_home, codex_home) = fixture_paths(); let agents_skills = codex_home .parent() .map(|parent| parent.join(".agents").join("skills")) .unwrap_or_else(|| PathBuf::from(".agents").join("skills")); - fs::create_dir_all(claude_home.join("skills").join("skill-a")).expect("create skills"); + fs::create_dir_all(external_agent_home.join("skills").join("skill-a")).expect("create skills"); fs::write( - claude_home.join("settings.json"), + external_agent_home.join("settings.json"), r#"{"model":"claude","permissions":{"ask":["git push"]},"env":{"FOO":"bar","CI":false,"MAX_RETRIES":3,"MY_TEAM":"codex","IGNORED":null,"LIST":["a","b"],"MAP":{"x":1}},"sandbox":{"enabled":true,"network":{"allowLocalBinding":true}}}"#, ) .expect("write settings"); fs::write( - claude_home.join("skills").join("skill-a").join("SKILL.md"), + external_agent_home + .join("skills") + .join("skill-a") + .join("SKILL.md"), "Use Claude Code and CLAUDE utilities.", ) .expect("write skill"); - fs::write(claude_home.join("CLAUDE.md"), "Claude code guidance").expect("write agents"); + fs::write( + external_agent_home.join("CLAUDE.md"), + "Claude code guidance", + ) + .expect("write agents"); - service_for_paths(claude_home, codex_home.clone()) + service_for_paths(external_agent_home, codex_home.clone()) .import(vec![ ExternalAgentConfigMigrationItem { item_type: ExternalAgentConfigMigrationItemType::AgentsMd, @@ -185,15 +274,15 @@ async fn import_home_migrates_supported_config_fields_skills_and_agents_md() { #[tokio::test] async fn import_home_skips_empty_config_migration() { - let (_root, claude_home, codex_home) = fixture_paths(); - fs::create_dir_all(&claude_home).expect("create claude home"); + let (_root, external_agent_home, codex_home) = fixture_paths(); + fs::create_dir_all(&external_agent_home).expect("create external agent home"); fs::write( - claude_home.join("settings.json"), + external_agent_home.join("settings.json"), r#"{"model":"claude","sandbox":{"enabled":false}}"#, ) .expect("write settings"); - service_for_paths(claude_home, codex_home.clone()) + service_for_paths(external_agent_home, codex_home.clone()) .import(vec![ExternalAgentConfigMigrationItem { item_type: ExternalAgentConfigMigrationItemType::Config, description: String::new(), @@ -206,13 +295,13 @@ async fn import_home_skips_empty_config_migration() { assert!(!codex_home.join("config.toml").exists()); } -#[test] -fn detect_home_skips_config_when_target_already_has_supported_fields() { - let (_root, claude_home, codex_home) = fixture_paths(); - fs::create_dir_all(&claude_home).expect("create claude home"); +#[tokio::test] +async fn detect_home_skips_config_when_target_already_has_supported_fields() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + fs::create_dir_all(&external_agent_home).expect("create external agent home"); fs::create_dir_all(&codex_home).expect("create codex home"); fs::write( - claude_home.join("settings.json"), + external_agent_home.join("settings.json"), r#"{"env":{"FOO":"bar"},"sandbox":{"enabled":true}}"#, ) .expect("write settings"); @@ -230,31 +319,33 @@ fn detect_home_skips_config_when_target_already_has_supported_fields() { ) .expect("write config"); - let items = service_for_paths(claude_home, codex_home) + let items = service_for_paths(external_agent_home, codex_home) .detect(ExternalAgentConfigDetectOptions { include_home: true, cwds: None, }) + .await .expect("detect"); assert_eq!(items, Vec::::new()); } -#[test] -fn detect_home_skips_skills_when_all_skill_directories_exist() { - let (_root, claude_home, codex_home) = fixture_paths(); +#[tokio::test] +async fn detect_home_skips_skills_when_all_skill_directories_exist() { + let (_root, external_agent_home, codex_home) = fixture_paths(); let agents_skills = codex_home .parent() .map(|parent| parent.join(".agents").join("skills")) .unwrap_or_else(|| PathBuf::from(".agents").join("skills")); - fs::create_dir_all(claude_home.join("skills").join("skill-a")).expect("create source"); + fs::create_dir_all(external_agent_home.join("skills").join("skill-a")).expect("create source"); fs::create_dir_all(agents_skills.join("skill-a")).expect("create target"); - let items = service_for_paths(claude_home, codex_home) + let items = service_for_paths(external_agent_home, codex_home) .detect(ExternalAgentConfigDetectOptions { include_home: true, cwds: None, }) + .await .expect("detect"); assert_eq!(items, Vec::::new()); @@ -332,8 +423,8 @@ async fn import_repo_agents_md_overwrites_empty_targets() { ); } -#[test] -fn detect_repo_prefers_non_empty_dot_claude_agents_source() { +#[tokio::test] +async fn detect_repo_prefers_non_empty_external_agent_agents_source() { let root = TempDir::new().expect("create tempdir"); let repo_root = root.path().join("repo"); fs::create_dir_all(repo_root.join(".git")).expect("create git"); @@ -350,6 +441,7 @@ fn detect_repo_prefers_non_empty_dot_claude_agents_source() { include_home: false, cwds: Some(vec![repo_root.clone()]), }) + .await .expect("detect"); assert_eq!( @@ -368,7 +460,7 @@ fn detect_repo_prefers_non_empty_dot_claude_agents_source() { } #[tokio::test] -async fn import_repo_uses_non_empty_dot_claude_agents_source() { +async fn import_repo_uses_non_empty_external_agent_agents_source() { let root = TempDir::new().expect("create tempdir"); let repo_root = root.path().join("repo"); fs::create_dir_all(repo_root.join(".git")).expect("create git"); @@ -407,27 +499,33 @@ fn migration_metric_tags_for_skills_include_skills_count() { ); } -#[test] -fn detect_home_lists_enabled_plugins_from_settings() { - let (_root, claude_home, codex_home) = fixture_paths(); - fs::create_dir_all(&claude_home).expect("create claude home"); +#[tokio::test] +async fn detect_home_lists_enabled_plugins_from_settings() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + fs::create_dir_all(&external_agent_home).expect("create external agent home"); fs::write( - claude_home.join("settings.json"), + external_agent_home.join("settings.json"), r#"{ "enabledPlugins": { "formatter@acme-tools": true, "deployer@acme-tools": true, "analyzer@security-plugins": false + }, + "extraKnownMarketplaces": { + "acme-tools": { + "source": "acme-corp/claude-plugins" + } } }"#, ) .expect("write settings"); - let items = service_for_paths(claude_home.clone(), codex_home) + let items = service_for_paths(external_agent_home.clone(), codex_home) .detect(ExternalAgentConfigDetectOptions { include_home: true, cwds: None, }) + .await .expect("detect"); assert_eq!( @@ -436,7 +534,7 @@ fn detect_home_lists_enabled_plugins_from_settings() { item_type: ExternalAgentConfigMigrationItemType::Plugins, description: format!( "Import enabled plugins from {}", - claude_home.join("settings.json").display() + external_agent_home.join("settings.json").display() ), cwd: None, details: Some(MigrationDetails { @@ -450,10 +548,155 @@ fn detect_home_lists_enabled_plugins_from_settings() { } #[tokio::test] -async fn import_plugins_requires_details() { - let (_root, claude_home, codex_home) = fixture_paths(); +async fn detect_repo_skips_plugins_that_are_already_configured_in_codex() { + let root = TempDir::new().expect("create tempdir"); + let external_agent_home = root.path().join(".claude"); + let codex_home = root.path().join(".codex"); + let repo_root = root.path().join("repo"); + fs::create_dir_all(repo_root.join(".git")).expect("create git dir"); + fs::create_dir_all(repo_root.join(".claude")).expect("create repo external agent dir"); + fs::create_dir_all(&codex_home).expect("create codex home"); + fs::write( + repo_root.join(".claude").join("settings.json"), + r#"{ + "enabledPlugins": { + "formatter@acme-tools": true, + "deployer@acme-tools": true + }, + "extraKnownMarketplaces": { + "acme-tools": { + "source": "acme-corp/claude-plugins" + } + } + }"#, + ) + .expect("write repo settings"); + fs::write( + codex_home.join("config.toml"), + r#" +[plugins."formatter@acme-tools"] +enabled = true +"#, + ) + .expect("write codex config"); - let err = service_for_paths(claude_home, codex_home) + let items = service_for_paths(external_agent_home, codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: false, + cwds: Some(vec![repo_root.clone()]), + }) + .await + .expect("detect"); + + assert_eq!( + items, + vec![ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::Plugins, + description: format!( + "Import enabled plugins from {}", + repo_root.join(".claude").join("settings.json").display() + ), + cwd: Some(repo_root), + details: Some(MigrationDetails { + plugins: vec![PluginsMigration { + marketplace_name: "acme-tools".to_string(), + plugin_names: vec!["deployer".to_string()], + }], + }), + }] + ); +} + +#[tokio::test] +async fn detect_repo_skips_plugins_that_are_disabled_in_codex() { + let root = TempDir::new().expect("create tempdir"); + let external_agent_home = root.path().join(".claude"); + let codex_home = root.path().join(".codex"); + let repo_root = root.path().join("repo"); + fs::create_dir_all(repo_root.join(".git")).expect("create git dir"); + fs::create_dir_all(repo_root.join(".claude")).expect("create repo external agent dir"); + fs::create_dir_all(&codex_home).expect("create codex home"); + fs::write( + repo_root.join(".claude").join("settings.json"), + r#"{ + "enabledPlugins": { + "formatter@acme-tools": true + }, + "extraKnownMarketplaces": { + "acme-tools": { + "source": "acme-corp/claude-plugins" + } + } + }"#, + ) + .expect("write repo settings"); + fs::write( + codex_home.join("config.toml"), + r#" +[plugins."formatter@acme-tools"] +enabled = false +"#, + ) + .expect("write codex config"); + + let items = service_for_paths(external_agent_home, codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: false, + cwds: Some(vec![repo_root]), + }) + .await + .expect("detect"); + + assert_eq!(items, Vec::::new()); +} + +#[tokio::test] +async fn detect_repo_skips_plugins_without_explicit_enabled_in_codex() { + let root = TempDir::new().expect("create tempdir"); + let external_agent_home = root.path().join(".claude"); + let codex_home = root.path().join(".codex"); + let repo_root = root.path().join("repo"); + fs::create_dir_all(repo_root.join(".git")).expect("create git dir"); + fs::create_dir_all(repo_root.join(".claude")).expect("create repo external agent dir"); + fs::create_dir_all(&codex_home).expect("create codex home"); + fs::write( + repo_root.join(".claude").join("settings.json"), + r#"{ + "enabledPlugins": { + "formatter@acme-tools": true + }, + "extraKnownMarketplaces": { + "acme-tools": { + "source": "acme-corp/claude-plugins" + } + } + }"#, + ) + .expect("write repo settings"); + fs::write( + codex_home.join("config.toml"), + r#" +[plugins."formatter@acme-tools"] +"#, + ) + .expect("write codex config"); + + let items = service_for_paths(external_agent_home, codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: false, + cwds: Some(vec![repo_root]), + }) + .await + .expect("detect"); + + assert_eq!(items, Vec::::new()); +} + +#[tokio::test] +async fn import_plugins_requires_details() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + + let err = service_for_paths(external_agent_home, codex_home) .import_plugins(/*cwd*/ None, /*details*/ None) .await .expect_err("expected missing details error"); @@ -463,11 +706,251 @@ async fn import_plugins_requires_details() { } #[tokio::test] -async fn import_plugins_requires_source_marketplace_details() { - let (_root, claude_home, codex_home) = fixture_paths(); - fs::create_dir_all(&claude_home).expect("create claude home"); +async fn detect_repo_does_not_skip_plugins_only_configured_in_project_codex() { + let root = TempDir::new().expect("create tempdir"); + let external_agent_home = root.path().join(".claude"); + let codex_home = root.path().join(".codex"); + let repo_root = root.path().join("repo"); + fs::create_dir_all(repo_root.join(".git")).expect("create git dir"); + fs::create_dir_all(repo_root.join(".claude")).expect("create repo external agent dir"); + fs::create_dir_all(repo_root.join(".codex")).expect("create repo codex dir"); + fs::create_dir_all(&codex_home).expect("create codex home"); fs::write( - claude_home.join("settings.json"), + repo_root.join(".claude").join("settings.json"), + r#"{ + "enabledPlugins": { + "formatter@acme-tools": true + }, + "extraKnownMarketplaces": { + "acme-tools": { + "source": "acme-corp/claude-plugins" + } + } + }"#, + ) + .expect("write repo settings"); + fs::write( + repo_root.join(".codex").join("config.toml"), + r#" +[plugins."formatter@acme-tools"] +enabled = true +"#, + ) + .expect("write project codex config"); + + let items = service_for_paths(external_agent_home, codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: false, + cwds: Some(vec![repo_root.clone()]), + }) + .await + .expect("detect"); + + assert_eq!( + items, + vec![ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::Plugins, + description: format!( + "Import enabled plugins from {}", + repo_root.join(".claude").join("settings.json").display() + ), + cwd: Some(repo_root), + details: Some(MigrationDetails { + plugins: vec![PluginsMigration { + marketplace_name: "acme-tools".to_string(), + plugin_names: vec!["formatter".to_string()], + }], + }), + }] + ); +} + +#[tokio::test] +async fn detect_home_skips_plugins_without_marketplace_source() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + fs::create_dir_all(&external_agent_home).expect("create external agent home"); + fs::write( + external_agent_home.join("settings.json"), + r#"{ + "enabledPlugins": { + "formatter@acme-tools": true + } + }"#, + ) + .expect("write settings"); + + let items = service_for_paths(external_agent_home, codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: true, + cwds: None, + }) + .await + .expect("detect"); + + assert_eq!(items, Vec::::new()); +} + +#[tokio::test] +async fn detect_home_skips_plugins_with_invalid_marketplace_source() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + fs::create_dir_all(&external_agent_home).expect("create external agent home"); + fs::write( + external_agent_home.join("settings.json"), + r#"{ + "enabledPlugins": { + "formatter@acme-tools": true + }, + "extraKnownMarketplaces": { + "acme-tools": { + "source": "github" + } + } + }"#, + ) + .expect("write settings"); + + let items = service_for_paths(external_agent_home, codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: true, + cwds: None, + }) + .await + .expect("detect"); + + assert_eq!(items, Vec::::new()); +} + +#[tokio::test] +async fn detect_repo_filters_plugins_against_installed_marketplace() { + let root = TempDir::new().expect("create tempdir"); + let external_agent_home = root.path().join(".claude"); + let codex_home = root.path().join(".codex"); + let repo_root = root.path().join("repo"); + let marketplace_root = codex_home.join(".tmp").join("marketplaces").join("debug"); + fs::create_dir_all(repo_root.join(".git")).expect("create git dir"); + fs::create_dir_all(repo_root.join(".claude")).expect("create repo external agent dir"); + fs::create_dir_all(marketplace_root.join(".agents").join("plugins")) + .expect("create marketplace manifest dir"); + fs::create_dir_all( + marketplace_root + .join("plugins") + .join("sample") + .join(".codex-plugin"), + ) + .expect("create sample plugin"); + fs::create_dir_all( + marketplace_root + .join("plugins") + .join("available") + .join(".codex-plugin"), + ) + .expect("create available plugin"); + fs::write( + repo_root.join(".claude").join("settings.json"), + r#"{ + "enabledPlugins": { + "sample@debug": true, + "available@debug": true, + "missing@debug": true + }, + "extraKnownMarketplaces": { + "debug": { + "source": "owner/debug-marketplace" + } + } + }"#, + ) + .expect("write repo settings"); + fs::write( + codex_home.join("config.toml"), + r#" +[marketplaces.debug] +source_type = "git" +source = "owner/debug-marketplace" +"#, + ) + .expect("write codex config"); + fs::write( + marketplace_root + .join(".agents") + .join("plugins") + .join("marketplace.json"), + r#"{ + "name": "debug", + "plugins": [ + { + "name": "sample", + "source": { + "source": "local", + "path": "./plugins/sample" + }, + "policy": { + "installation": "NOT_AVAILABLE" + } + }, + { + "name": "available", + "source": { + "source": "local", + "path": "./plugins/available" + } + } + ] +}"#, + ) + .expect("write marketplace manifest"); + fs::write( + marketplace_root + .join("plugins") + .join("sample") + .join(".codex-plugin") + .join("plugin.json"), + r#"{"name":"sample"}"#, + ) + .expect("write sample plugin manifest"); + fs::write( + marketplace_root + .join("plugins") + .join("available") + .join(".codex-plugin") + .join("plugin.json"), + r#"{"name":"available"}"#, + ) + .expect("write available plugin manifest"); + + let items = service_for_paths(external_agent_home, codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: false, + cwds: Some(vec![repo_root.clone()]), + }) + .await + .expect("detect"); + + assert_eq!( + items, + vec![ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::Plugins, + description: format!( + "Import enabled plugins from {}", + repo_root.join(".claude").join("settings.json").display() + ), + cwd: Some(repo_root), + details: Some(MigrationDetails { + plugins: vec![PluginsMigration { + marketplace_name: "debug".to_string(), + plugin_names: vec!["available".to_string()], + }], + }), + }] + ); +} + +#[tokio::test] +async fn import_plugins_requires_source_marketplace_details() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + fs::create_dir_all(&external_agent_home).expect("create external agent home"); + fs::write( + external_agent_home.join("settings.json"), r#"{ "enabledPlugins": { "formatter@acme-tools": true @@ -482,7 +965,7 @@ async fn import_plugins_requires_source_marketplace_details() { ) .expect("write settings"); - let outcome = service_for_paths(claude_home, codex_home) + let outcome = service_for_paths(external_agent_home, codex_home) .import_plugins( /*cwd*/ None, Some(MigrationDetails { @@ -508,10 +991,10 @@ async fn import_plugins_requires_source_marketplace_details() { #[tokio::test] async fn import_plugins_defers_marketplace_source_validation_to_add_marketplace() { - let (_root, claude_home, codex_home) = fixture_paths(); - fs::create_dir_all(&claude_home).expect("create claude home"); + let (_root, external_agent_home, codex_home) = fixture_paths(); + fs::create_dir_all(&external_agent_home).expect("create external agent home"); fs::write( - claude_home.join("settings.json"), + external_agent_home.join("settings.json"), r#"{ "enabledPlugins": { "formatter@acme-tools": true @@ -526,7 +1009,7 @@ async fn import_plugins_defers_marketplace_source_validation_to_add_marketplace( ) .expect("write settings"); - let outcome = service_for_paths(claude_home, codex_home) + let outcome = service_for_paths(external_agent_home, codex_home) .import_plugins(/*cwd*/ None, Some(github_plugin_details())) .await .expect("import plugins"); @@ -542,18 +1025,396 @@ async fn import_plugins_defers_marketplace_source_validation_to_add_marketplace( ); } +#[tokio::test] +async fn import_plugins_supports_external_agent_plugin_marketplace_layout() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + let marketplace_root = external_agent_home.join("my-marketplace"); + let plugin_root = marketplace_root.join("plugins").join("cloudflare"); + fs::create_dir_all(marketplace_root.join(".claude-plugin")) + .expect("create marketplace manifest dir"); + fs::create_dir_all(plugin_root.join(".codex-plugin")).expect("create plugin manifest dir"); + fs::create_dir_all(&codex_home).expect("create codex home"); + + fs::write( + external_agent_home.join("settings.json"), + serde_json::to_string_pretty(&serde_json::json!({ + "enabledPlugins": { + "cloudflare@my-plugins": true + }, + "extraKnownMarketplaces": { + "my-plugins": { + "source": "local", + "path": marketplace_root + } + } + })) + .expect("serialize settings"), + ) + .expect("write settings"); + fs::write( + marketplace_root + .join(".claude-plugin") + .join("marketplace.json"), + r#"{ + "name": "my-plugins", + "plugins": [ + { + "name": "cloudflare", + "source": "./plugins/cloudflare" + } + ] + }"#, + ) + .expect("write marketplace manifest"); + fs::write( + plugin_root.join(".codex-plugin").join("plugin.json"), + r#"{"name":"cloudflare","version":"0.1.0"}"#, + ) + .expect("write plugin manifest"); + + let outcome = service_for_paths(external_agent_home, codex_home.clone()) + .import_plugins( + /*cwd*/ None, + Some(MigrationDetails { + plugins: vec![PluginsMigration { + marketplace_name: "my-plugins".to_string(), + plugin_names: vec!["cloudflare".to_string()], + }], + }), + ) + .await + .expect("import plugins"); + + assert_eq!( + outcome, + PluginImportOutcome { + succeeded_marketplaces: vec!["my-plugins".to_string()], + succeeded_plugin_ids: vec!["cloudflare@my-plugins".to_string()], + failed_marketplaces: Vec::new(), + failed_plugin_ids: Vec::new(), + } + ); + let config = fs::read_to_string(codex_home.join("config.toml")).expect("read config"); + assert!(config.contains(r#"[plugins."cloudflare@my-plugins"]"#)); + assert!(config.contains("enabled = true")); +} + +#[tokio::test] +async fn detect_home_supports_relative_external_agent_plugin_marketplace_path() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + let marketplace_root = external_agent_home.join("my-marketplace"); + let plugin_root = marketplace_root.join("plugins").join("cloudflare"); + fs::create_dir_all(marketplace_root.join(".claude-plugin")) + .expect("create marketplace manifest dir"); + fs::create_dir_all(plugin_root.join(".codex-plugin")).expect("create plugin manifest dir"); + fs::create_dir_all(&codex_home).expect("create codex home"); + + fs::write( + external_agent_home.join("settings.json"), + r#"{ + "enabledPlugins": { + "cloudflare@my-plugins": true + }, + "extraKnownMarketplaces": { + "my-plugins": { + "source": "directory", + "path": "./my-marketplace" + } + } + }"#, + ) + .expect("write settings"); + fs::write( + marketplace_root + .join(".claude-plugin") + .join("marketplace.json"), + r#"{ + "name": "my-plugins", + "plugins": [ + { + "name": "cloudflare", + "source": "./plugins/cloudflare" + } + ] + }"#, + ) + .expect("write marketplace manifest"); + fs::write( + plugin_root.join(".codex-plugin").join("plugin.json"), + r#"{"name":"cloudflare","version":"0.1.0"}"#, + ) + .expect("write plugin manifest"); + + let items = service_for_paths(external_agent_home.clone(), codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: true, + cwds: None, + }) + .await + .expect("detect"); + + assert_eq!( + items, + vec![ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::Plugins, + description: format!( + "Import enabled plugins from {}", + external_agent_home.join("settings.json").display() + ), + cwd: None, + details: Some(MigrationDetails { + plugins: vec![PluginsMigration { + marketplace_name: "my-plugins".to_string(), + plugin_names: vec!["cloudflare".to_string()], + }], + }), + }] + ); +} + +#[tokio::test] +async fn import_plugins_supports_relative_external_agent_plugin_marketplace_path() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + let marketplace_root = external_agent_home.join("my-marketplace"); + let plugin_root = marketplace_root.join("plugins").join("cloudflare"); + fs::create_dir_all(marketplace_root.join(".claude-plugin")) + .expect("create marketplace manifest dir"); + fs::create_dir_all(plugin_root.join(".codex-plugin")).expect("create plugin manifest dir"); + fs::create_dir_all(&codex_home).expect("create codex home"); + + fs::write( + external_agent_home.join("settings.json"), + r#"{ + "enabledPlugins": { + "cloudflare@my-plugins": true + }, + "extraKnownMarketplaces": { + "my-plugins": { + "source": "directory", + "path": "./my-marketplace" + } + } + }"#, + ) + .expect("write settings"); + fs::write( + marketplace_root + .join(".claude-plugin") + .join("marketplace.json"), + r#"{ + "name": "my-plugins", + "plugins": [ + { + "name": "cloudflare", + "source": "./plugins/cloudflare" + } + ] + }"#, + ) + .expect("write marketplace manifest"); + fs::write( + plugin_root.join(".codex-plugin").join("plugin.json"), + r#"{"name":"cloudflare","version":"0.1.0"}"#, + ) + .expect("write plugin manifest"); + + let outcome = service_for_paths(external_agent_home, codex_home.clone()) + .import_plugins( + /*cwd*/ None, + Some(MigrationDetails { + plugins: vec![PluginsMigration { + marketplace_name: "my-plugins".to_string(), + plugin_names: vec!["cloudflare".to_string()], + }], + }), + ) + .await + .expect("import plugins"); + + assert_eq!( + outcome, + PluginImportOutcome { + succeeded_marketplaces: vec!["my-plugins".to_string()], + succeeded_plugin_ids: vec!["cloudflare@my-plugins".to_string()], + failed_marketplaces: Vec::new(), + failed_plugin_ids: Vec::new(), + } + ); + let config = fs::read_to_string(codex_home.join("config.toml")).expect("read config"); + assert!(config.contains(r#"[plugins."cloudflare@my-plugins"]"#)); + assert!(config.contains("enabled = true")); +} + +#[tokio::test] +async fn detect_repo_supports_project_relative_external_agent_plugin_marketplace_path() { + let root = TempDir::new().expect("create tempdir"); + let external_agent_home = root.path().join(".claude"); + let codex_home = root.path().join(".codex"); + let repo_root = root.path().join("repo"); + let marketplace_root = repo_root.join("my-marketplace"); + let plugin_root = marketplace_root.join("plugins").join("cloudflare"); + fs::create_dir_all(repo_root.join(".git")).expect("create git dir"); + fs::create_dir_all(repo_root.join(".claude")).expect("create repo external agent dir"); + fs::create_dir_all(marketplace_root.join(".claude-plugin")) + .expect("create marketplace manifest dir"); + fs::create_dir_all(plugin_root.join(".codex-plugin")).expect("create plugin manifest dir"); + fs::create_dir_all(&codex_home).expect("create codex home"); + + fs::write( + repo_root.join(".claude").join("settings.json"), + r#"{ + "enabledPlugins": { + "cloudflare@my-plugins": true + }, + "extraKnownMarketplaces": { + "my-plugins": { + "source": "directory", + "path": "./my-marketplace" + } + } + }"#, + ) + .expect("write settings"); + fs::write( + marketplace_root + .join(".claude-plugin") + .join("marketplace.json"), + r#"{ + "name": "my-plugins", + "plugins": [ + { + "name": "cloudflare", + "source": "./plugins/cloudflare" + } + ] + }"#, + ) + .expect("write marketplace manifest"); + fs::write( + plugin_root.join(".codex-plugin").join("plugin.json"), + r#"{"name":"cloudflare","version":"0.1.0"}"#, + ) + .expect("write plugin manifest"); + + let items = service_for_paths(external_agent_home, codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: false, + cwds: Some(vec![repo_root.clone()]), + }) + .await + .expect("detect"); + + assert_eq!( + items, + vec![ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::Plugins, + description: format!( + "Import enabled plugins from {}", + repo_root.join(".claude").join("settings.json").display() + ), + cwd: Some(repo_root), + details: Some(MigrationDetails { + plugins: vec![PluginsMigration { + marketplace_name: "my-plugins".to_string(), + plugin_names: vec!["cloudflare".to_string()], + }], + }), + }] + ); +} + +#[tokio::test] +async fn import_plugins_supports_project_relative_external_agent_plugin_marketplace_path() { + let root = TempDir::new().expect("create tempdir"); + let external_agent_home = root.path().join(".claude"); + let codex_home = root.path().join(".codex"); + let repo_root = root.path().join("repo"); + let marketplace_root = repo_root.join("my-marketplace"); + let plugin_root = marketplace_root.join("plugins").join("cloudflare"); + fs::create_dir_all(repo_root.join(".git")).expect("create git dir"); + fs::create_dir_all(repo_root.join(".claude")).expect("create repo external agent dir"); + fs::create_dir_all(marketplace_root.join(".claude-plugin")) + .expect("create marketplace manifest dir"); + fs::create_dir_all(plugin_root.join(".codex-plugin")).expect("create plugin manifest dir"); + fs::create_dir_all(&codex_home).expect("create codex home"); + + fs::write( + repo_root.join(".claude").join("settings.json"), + r#"{ + "enabledPlugins": { + "cloudflare@my-plugins": true + }, + "extraKnownMarketplaces": { + "my-plugins": { + "source": "directory", + "path": "./my-marketplace" + } + } + }"#, + ) + .expect("write settings"); + fs::write( + marketplace_root + .join(".claude-plugin") + .join("marketplace.json"), + r#"{ + "name": "my-plugins", + "plugins": [ + { + "name": "cloudflare", + "source": "./plugins/cloudflare" + } + ] + }"#, + ) + .expect("write marketplace manifest"); + fs::write( + plugin_root.join(".codex-plugin").join("plugin.json"), + r#"{"name":"cloudflare","version":"0.1.0"}"#, + ) + .expect("write plugin manifest"); + + let outcome = service_for_paths(external_agent_home, codex_home.clone()) + .import_plugins( + Some(repo_root.as_path()), + Some(MigrationDetails { + plugins: vec![PluginsMigration { + marketplace_name: "my-plugins".to_string(), + plugin_names: vec!["cloudflare".to_string()], + }], + }), + ) + .await + .expect("import plugins"); + + assert_eq!( + outcome, + PluginImportOutcome { + succeeded_marketplaces: vec!["my-plugins".to_string()], + succeeded_plugin_ids: vec!["cloudflare@my-plugins".to_string()], + failed_marketplaces: Vec::new(), + failed_plugin_ids: Vec::new(), + } + ); + let config = fs::read_to_string(codex_home.join("config.toml")).expect("read config"); + assert!(config.contains(r#"[plugins."cloudflare@my-plugins"]"#)); + assert!(config.contains("enabled = true")); +} + #[test] fn import_skills_returns_only_new_skill_directory_count() { - let (_root, claude_home, codex_home) = fixture_paths(); + let (_root, external_agent_home, codex_home) = fixture_paths(); let agents_skills = codex_home .parent() .map(|parent| parent.join(".agents").join("skills")) .unwrap_or_else(|| PathBuf::from(".agents").join("skills")); - fs::create_dir_all(claude_home.join("skills").join("skill-a")).expect("create source a"); - fs::create_dir_all(claude_home.join("skills").join("skill-b")).expect("create source b"); + fs::create_dir_all(external_agent_home.join("skills").join("skill-a")) + .expect("create source a"); + fs::create_dir_all(external_agent_home.join("skills").join("skill-b")) + .expect("create source b"); fs::create_dir_all(agents_skills.join("skill-a")).expect("create existing target"); - let copied_count = service_for_paths(claude_home, codex_home) + let copied_count = service_for_paths(external_agent_home, codex_home) .import_skills(/*cwd*/ None) .expect("import skills"); diff --git a/codex-rs/core/src/git_info_tests.rs b/codex-rs/core/src/git_info_tests.rs index 4cb50f02fd..e3ef1fa62f 100644 --- a/codex-rs/core/src/git_info_tests.rs +++ b/codex-rs/core/src/git_info_tests.rs @@ -426,10 +426,14 @@ async fn test_get_git_working_tree_state_branch_fallback() { assert_eq!(state.sha, GitSha::new(&remote_sha)); } -#[test] -fn resolve_root_git_project_for_trust_returns_none_outside_repo() { +#[tokio::test] +async fn resolve_root_git_project_for_trust_returns_none_outside_repo() { let tmp = TempDir::new().expect("tempdir"); - assert!(resolve_root_git_project_for_trust(tmp.path()).is_none()); + assert!( + resolve_root_git_project_for_trust(tmp.path()) + .await + .is_none() + ); } #[tokio::test] @@ -439,12 +443,15 @@ async fn resolve_root_git_project_for_trust_regular_repo_returns_repo_root() { let expected = std::fs::canonicalize(&repo_path).unwrap(); assert_eq!( - resolve_root_git_project_for_trust(&repo_path), + resolve_root_git_project_for_trust(&repo_path).await, Some(expected.clone()) ); let nested = repo_path.join("sub/dir"); std::fs::create_dir_all(&nested).unwrap(); - assert_eq!(resolve_root_git_project_for_trust(&nested), Some(expected)); + assert_eq!( + resolve_root_git_project_for_trust(&nested).await, + Some(expected) + ); } #[tokio::test] @@ -467,18 +474,20 @@ async fn resolve_root_git_project_for_trust_detects_worktree_and_returns_main_ro .expect("git worktree add"); let expected = std::fs::canonicalize(&repo_path).ok(); - let got = - resolve_root_git_project_for_trust(&wt_root).and_then(|p| std::fs::canonicalize(p).ok()); + let got = resolve_root_git_project_for_trust(&wt_root) + .await + .and_then(|p| std::fs::canonicalize(p).ok()); assert_eq!(got, expected); let nested = wt_root.join("nested/sub"); std::fs::create_dir_all(&nested).unwrap(); - let got_nested = - resolve_root_git_project_for_trust(&nested).and_then(|p| std::fs::canonicalize(p).ok()); + let got_nested = resolve_root_git_project_for_trust(&nested) + .await + .and_then(|p| std::fs::canonicalize(p).ok()); assert_eq!(got_nested, expected); } -#[test] -fn resolve_root_git_project_for_trust_detects_worktree_pointer_without_git_command() { +#[tokio::test] +async fn resolve_root_git_project_for_trust_detects_worktree_pointer_without_git_command() { let tmp = TempDir::new().expect("tempdir"); let repo_root = tmp.path().join("repo"); let common_dir = repo_root.join(".git"); @@ -495,17 +504,17 @@ fn resolve_root_git_project_for_trust_detects_worktree_pointer_without_git_comma let expected = std::fs::canonicalize(&repo_root).unwrap(); assert_eq!( - resolve_root_git_project_for_trust(&worktree_root), + resolve_root_git_project_for_trust(&worktree_root).await, Some(expected.clone()) ); assert_eq!( - resolve_root_git_project_for_trust(&worktree_root.join("nested")), + resolve_root_git_project_for_trust(&worktree_root.join("nested")).await, Some(expected) ); } -#[test] -fn resolve_root_git_project_for_trust_non_worktrees_gitdir_returns_none() { +#[tokio::test] +async fn resolve_root_git_project_for_trust_non_worktrees_gitdir_returns_none() { let tmp = TempDir::new().expect("tempdir"); let proj = tmp.path().join("proj"); std::fs::create_dir_all(proj.join("nested")).unwrap(); @@ -520,8 +529,12 @@ fn resolve_root_git_project_for_trust_non_worktrees_gitdir_returns_none() { ) .unwrap(); - assert!(resolve_root_git_project_for_trust(&proj).is_none()); - assert!(resolve_root_git_project_for_trust(&proj.join("nested")).is_none()); + assert!(resolve_root_git_project_for_trust(&proj).await.is_none()); + assert!( + resolve_root_git_project_for_trust(&proj.join("nested")) + .await + .is_none() + ); } #[tokio::test] diff --git a/codex-rs/core/src/guardian/review_session.rs b/codex-rs/core/src/guardian/review_session.rs index 6cb9aefa14..f1c6e1e054 100644 --- a/codex-rs/core/src/guardian/review_session.rs +++ b/codex-rs/core/src/guardian/review_session.rs @@ -823,9 +823,9 @@ async fn interrupt_and_drain_turn(codex: &Codex) -> anyhow::Result<()> { mod tests { use super::*; - #[test] - fn guardian_review_session_config_change_invalidates_cached_session() { - let parent_config = crate::config::test_config(); + #[tokio::test] + async fn guardian_review_session_config_change_invalidates_cached_session() { + let parent_config = crate::config::test_config().await; let cached_spawn_config = build_guardian_review_session_config( &parent_config, /*live_network_config*/ None, @@ -855,9 +855,9 @@ mod tests { ); } - #[test] - fn guardian_review_session_config_disables_hooks() { - let mut parent_config = crate::config::test_config(); + #[tokio::test] + async fn guardian_review_session_config_disables_hooks() { + let mut parent_config = crate::config::test_config().await; parent_config .features .enable(Feature::CodexHooks) diff --git a/codex-rs/core/src/guardian/tests.rs b/codex-rs/core/src/guardian/tests.rs index 8940753d27..485f222551 100644 --- a/codex-rs/core/src/guardian/tests.rs +++ b/codex-rs/core/src/guardian/tests.rs @@ -1332,8 +1332,8 @@ async fn guardian_review_surfaces_responses_api_errors_in_rejection_reason() -> Ok(()) } -#[test] -fn guardian_parallel_reviews_fork_from_last_committed_trunk_history() -> anyhow::Result<()> { +#[tokio::test] +async fn guardian_parallel_reviews_fork_from_last_committed_trunk_history() -> anyhow::Result<()> { const TEST_STACK_SIZE_BYTES: usize = 2 * 1024 * 1024; let handle = @@ -1569,9 +1569,9 @@ fn guardian_parallel_reviews_fork_from_last_committed_trunk_history() -> anyhow: )), } } -#[test] -fn guardian_review_session_config_preserves_parent_network_proxy() { - let mut parent_config = test_config(); +#[tokio::test] +async fn guardian_review_session_config_preserves_parent_network_proxy() { + let mut parent_config = test_config().await; let network = NetworkProxySpec::from_config_and_constraints( NetworkProxyConfig::default(), Some(NetworkConstraints { @@ -1616,9 +1616,9 @@ fn guardian_review_session_config_preserves_parent_network_proxy() { ); } -#[test] -fn guardian_review_session_config_overrides_parent_developer_instructions() { - let mut parent_config = test_config(); +#[tokio::test] +async fn guardian_review_session_config_overrides_parent_developer_instructions() { + let mut parent_config = test_config().await; parent_config.developer_instructions = Some("parent or managed config should not replace guardian policy".to_string()); @@ -1636,9 +1636,9 @@ fn guardian_review_session_config_overrides_parent_developer_instructions() { ); } -#[test] -fn guardian_review_session_config_uses_live_network_proxy_state() { - let mut parent_config = test_config(); +#[tokio::test] +async fn guardian_review_session_config_uses_live_network_proxy_state() { + let mut parent_config = test_config().await; let mut parent_network = NetworkProxyConfig::default(); parent_network.network.enabled = true; parent_network @@ -1680,9 +1680,9 @@ fn guardian_review_session_config_uses_live_network_proxy_state() { ); } -#[test] -fn guardian_review_session_config_rejects_pinned_collab_feature() { - let mut parent_config = test_config(); +#[tokio::test] +async fn guardian_review_session_config_rejects_pinned_collab_feature() { + let mut parent_config = test_config().await; parent_config.features = ManagedFeatures::from_configured( parent_config.features.get().clone(), Some(Sourced { @@ -1708,9 +1708,9 @@ fn guardian_review_session_config_rejects_pinned_collab_feature() { ); } -#[test] -fn guardian_review_session_config_uses_parent_active_model_instead_of_hardcoded_slug() { - let mut parent_config = test_config(); +#[tokio::test] +async fn guardian_review_session_config_uses_parent_active_model_instead_of_hardcoded_slug() { + let mut parent_config = test_config().await; parent_config.model = Some("configured-model".to_string()); let guardian_config = build_guardian_review_session_config_for_test( @@ -1724,8 +1724,8 @@ fn guardian_review_session_config_uses_parent_active_model_instead_of_hardcoded_ assert_eq!(guardian_config.model, Some("active-model".to_string())); } -#[test] -fn guardian_review_session_config_uses_requirements_guardian_policy_config() { +#[tokio::test] +async fn guardian_review_session_config_uses_requirements_guardian_policy_config() { let codex_home = tempfile::tempdir().expect("create temp dir"); let workspace = tempfile::tempdir().expect("create temp dir"); let config_layer_stack = ConfigLayerStack::new( @@ -1748,6 +1748,7 @@ fn guardian_review_session_config_uses_requirements_guardian_policy_config() { codex_home.abs(), config_layer_stack, ) + .await .expect("load config"); let guardian_config = build_guardian_review_session_config_for_test( @@ -1766,8 +1767,9 @@ fn guardian_review_session_config_uses_requirements_guardian_policy_config() { ); } -#[test] -fn guardian_review_session_config_uses_default_guardian_policy_without_requirements_override() { +#[tokio::test] +async fn guardian_review_session_config_uses_default_guardian_policy_without_requirements_override() +{ let codex_home = tempfile::tempdir().expect("create temp dir"); let workspace = tempfile::tempdir().expect("create temp dir"); let config_layer_stack = @@ -1782,6 +1784,7 @@ fn guardian_review_session_config_uses_default_guardian_policy_without_requireme codex_home.abs(), config_layer_stack, ) + .await .expect("load config"); let guardian_config = build_guardian_review_session_config_for_test( diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 5e62199d2d..50777ad157 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -5,6 +5,7 @@ // the TUI or the tracing stack). #![deny(clippy::print_stdout, clippy::print_stderr)] +mod agent_identity; mod apply_patch; mod apps; mod arc_monitor; @@ -54,12 +55,11 @@ pub use network_proxy_loader::MtimeConfigReloader; pub use network_proxy_loader::build_network_proxy_state; pub use network_proxy_loader::build_network_proxy_state_and_reloader; mod original_image_detail; -pub use codex_mcp::MCP_SANDBOX_STATE_CAPABILITY; -pub use codex_mcp::MCP_SANDBOX_STATE_METHOD; pub use codex_mcp::SandboxState; mod mcp_openai_file; mod mcp_tool_call; mod memories; +pub use memories::clear_memory_roots_contents; pub(crate) mod mention_syntax; pub(crate) mod message_history; pub(crate) mod utils; @@ -99,9 +99,7 @@ pub(crate) use skills::build_skill_injections; pub(crate) use skills::build_skill_name_counts; pub(crate) use skills::collect_env_var_dependencies; pub(crate) use skills::collect_explicit_skill_mentions; -pub(crate) use skills::config_rules; pub(crate) use skills::injection; -pub(crate) use skills::loader; pub(crate) use skills::manager; pub(crate) use skills::maybe_emit_implicit_skill_invocation; pub(crate) use skills::render_skills_section; @@ -132,14 +130,12 @@ pub type ConversationManager = ThreadManager; pub type NewConversation = NewThread; #[deprecated(note = "use CodexThread")] pub type CodexConversation = CodexThread; -pub(crate) mod project_doc; -pub use project_doc::DEFAULT_PROJECT_DOC_FILENAME; -pub use project_doc::LOCAL_PROJECT_DOC_FILENAME; -pub use project_doc::discover_project_doc_paths; -pub use project_doc::read_project_docs; +pub(crate) mod agents_md; +pub use agents_md::AgentsMdManager; +pub use agents_md::DEFAULT_AGENTS_MD_FILENAME; +pub use agents_md::LOCAL_AGENTS_MD_FILENAME; mod rollout; pub(crate) mod safety; -pub mod seatbelt; mod session_rollout_init_error; pub mod shell; pub(crate) mod shell_snapshot; @@ -152,6 +148,7 @@ mod tools; pub(crate) mod turn_diff_tracker; mod turn_metadata; mod turn_timing; +mod unavailable_tool; pub use rollout::ARCHIVED_SESSIONS_SUBDIR; pub use rollout::Cursor; pub use rollout::EventPersistenceMode; diff --git a/codex-rs/core/src/mcp_openai_file.rs b/codex-rs/core/src/mcp_openai_file.rs index 587dd4b770..33d0a3f1f0 100644 --- a/codex-rs/core/src/mcp_openai_file.rs +++ b/codex-rs/core/src/mcp_openai_file.rs @@ -115,6 +115,7 @@ async fn build_uploaded_local_argument_value( let upload_auth = CoreAuthProvider { token: Some(token_data.access_token), account_id: token_data.account_id, + is_fedramp_account: auth.is_fedramp_account(), }; let uploaded = upload_local_file( turn_context.config.chatgpt_base_url.trim_end_matches('/'), diff --git a/codex-rs/core/src/mcp_skill_dependencies.rs b/codex-rs/core/src/mcp_skill_dependencies.rs index 6cdd3cf084..c12fe28cb2 100644 --- a/codex-rs/core/src/mcp_skill_dependencies.rs +++ b/codex-rs/core/src/mcp_skill_dependencies.rs @@ -364,12 +364,14 @@ fn mcp_dependency_to_server_config( http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -391,12 +393,14 @@ fn mcp_dependency_to_server_config( env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/core/src/mcp_tool_call.rs b/codex-rs/core/src/mcp_tool_call.rs index 9aea9b87c8..b707d51890 100644 --- a/codex-rs/core/src/mcp_tool_call.rs +++ b/codex-rs/core/src/mcp_tool_call.rs @@ -36,6 +36,7 @@ use codex_analytics::build_track_events_context; use codex_config::types::AppToolApproval; use codex_features::Feature; use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; +use codex_mcp::SandboxState; use codex_mcp::declared_openai_file_input_param_names; use codex_mcp::mcp_permission_prompt_is_auto_approved; use codex_otel::sanitize_metric_tag_value; @@ -100,6 +101,9 @@ pub(crate) async fn handle_mcp_tool_call( let metadata = lookup_mcp_tool_metadata(sess.as_ref(), turn_context.as_ref(), &server, &tool_name).await; + let mcp_app_resource_uri = metadata + .as_ref() + .and_then(|metadata| metadata.mcp_app_resource_uri.clone()); let app_tool_policy = if server == CODEX_APPS_MCP_SERVER_NAME { connectors::app_tool_policy( &turn_context.config, @@ -129,6 +133,7 @@ pub(crate) async fn handle_mcp_tool_call( turn_context.as_ref(), &call_id, invocation, + mcp_app_resource_uri.clone(), "MCP tool call blocked by app configuration".to_string(), /*already_started*/ false, ) @@ -160,6 +165,7 @@ pub(crate) async fn handle_mcp_tool_call( let tool_call_begin_event = EventMsg::McpToolCallBegin(McpToolCallBeginEvent { call_id: call_id.clone(), invocation: invocation.clone(), + mcp_app_resource_uri: mcp_app_resource_uri.clone(), }); notify_mcp_tool_call_event(sess.as_ref(), turn_context.as_ref(), tool_call_begin_event).await; @@ -212,6 +218,7 @@ pub(crate) async fn handle_mcp_tool_call( let tool_call_end_event = EventMsg::McpToolCallEnd(McpToolCallEndEvent { call_id: call_id.clone(), invocation, + mcp_app_resource_uri: mcp_app_resource_uri.clone(), duration, result: result.clone(), }); @@ -238,6 +245,7 @@ pub(crate) async fn handle_mcp_tool_call( turn_context.as_ref(), &call_id, invocation, + mcp_app_resource_uri.clone(), message, /*already_started*/ true, ) @@ -253,6 +261,7 @@ pub(crate) async fn handle_mcp_tool_call( turn_context.as_ref(), &call_id, invocation, + mcp_app_resource_uri.clone(), message, /*already_started*/ true, ) @@ -267,6 +276,7 @@ pub(crate) async fn handle_mcp_tool_call( turn_context.as_ref(), &call_id, invocation, + mcp_app_resource_uri.clone(), message, /*already_started*/ true, ) @@ -324,6 +334,7 @@ pub(crate) async fn handle_mcp_tool_call( let tool_call_end_event = EventMsg::McpToolCallEnd(McpToolCallEndEvent { call_id: call_id.clone(), invocation, + mcp_app_resource_uri, duration, result: result.clone(), }); @@ -466,6 +477,10 @@ async fn execute_mcp_tool_call( metadata.and_then(|metadata| metadata.openai_file_input_params.as_deref()), ) .await?; + let request_meta = + augment_mcp_tool_request_meta_with_sandbox_state(sess, turn_context, server, request_meta) + .await + .map_err(|e| format!("failed to build MCP tool request metadata: {e:#}"))?; let result = sess .call_tool(server, tool_name, rewritten_arguments, request_meta) .await @@ -479,6 +494,52 @@ async fn execute_mcp_tool_call( ) } +async fn augment_mcp_tool_request_meta_with_sandbox_state( + sess: &Session, + turn_context: &TurnContext, + server: &str, + mut meta: Option, +) -> anyhow::Result> { + let supports_sandbox_state_meta = sess + .services + .mcp_connection_manager + .read() + .await + .server_supports_sandbox_state_meta_capability(server) + .await + .unwrap_or(false); + if !supports_sandbox_state_meta { + return Ok(meta); + } + + let sandbox_state = serde_json::to_value(SandboxState { + sandbox_policy: turn_context.sandbox_policy.get().clone(), + codex_linux_sandbox_exe: turn_context.codex_linux_sandbox_exe.clone(), + sandbox_cwd: turn_context.cwd.to_path_buf(), + use_legacy_landlock: turn_context.features.use_legacy_landlock(), + })?; + + match meta.as_mut() { + Some(serde_json::Value::Object(map)) => { + map.insert( + codex_mcp::MCP_SANDBOX_STATE_META_CAPABILITY.to_string(), + sandbox_state, + ); + } + Some(_) => {} + None => { + let mut map = serde_json::Map::new(); + map.insert( + codex_mcp::MCP_SANDBOX_STATE_META_CAPABILITY.to_string(), + sandbox_state, + ); + meta = Some(serde_json::Value::Object(map)); + } + } + + Ok(meta) +} + async fn maybe_mark_thread_memory_mode_polluted(sess: &Session, turn_context: &TurnContext) { if !turn_context .config @@ -591,11 +652,14 @@ pub(crate) struct McpToolApprovalMetadata { connector_description: Option, tool_title: Option, tool_description: Option, + mcp_app_resource_uri: Option, codex_apps_meta: Option>, openai_file_input_params: Option>, } const MCP_TOOL_CODEX_APPS_META_KEY: &str = "_codex_apps"; +const MCP_TOOL_OPENAI_OUTPUT_TEMPLATE_META_KEY: &str = "openai/outputTemplate"; +const MCP_TOOL_UI_RESOURCE_URI_META_KEY: &str = "ui/resourceUri"; fn custom_mcp_tool_approval_mode( turn_context: &TurnContext, @@ -612,9 +676,14 @@ fn custom_mcp_tool_approval_mode( .and_then(|value| { HashMap::::deserialize(value).ok() }) - .and_then(|servers| servers.get(server).cloned()) - .and_then(|server| server.tools.get(tool_name).cloned()) - .and_then(|tool| tool.approval_mode) + .and_then(|servers| { + let server_config = servers.get(server)?; + server_config + .tools + .get(tool_name) + .and_then(|tool| tool.approval_mode) + .or(server_config.default_tools_approval_mode) + }) .unwrap_or_default() } @@ -1049,6 +1118,7 @@ pub(crate) async fn lookup_mcp_tool_metadata( connector_description, tool_title: tool_info.tool.title, tool_description: tool_info.tool.description.map(std::borrow::Cow::into_owned), + mcp_app_resource_uri: get_mcp_app_resource_uri(tool_info.tool.meta.as_deref()), codex_apps_meta: tool_info .tool .meta @@ -1063,6 +1133,26 @@ pub(crate) async fn lookup_mcp_tool_metadata( }) } +fn get_mcp_app_resource_uri( + meta: Option<&serde_json::Map>, +) -> Option { + meta.and_then(|meta| { + meta.get("ui") + .and_then(serde_json::Value::as_object) + .and_then(|ui| ui.get("resourceUri")) + .and_then(serde_json::Value::as_str) + .or_else(|| { + meta.get(MCP_TOOL_UI_RESOURCE_URI_META_KEY) + .and_then(serde_json::Value::as_str) + }) + .or_else(|| { + meta.get(MCP_TOOL_OPENAI_OUTPUT_TEMPLATE_META_KEY) + .and_then(serde_json::Value::as_str) + }) + .map(str::to_string) + }) +} + async fn lookup_mcp_app_usage_metadata( sess: &Session, server: &str, @@ -1615,6 +1705,7 @@ async fn notify_mcp_tool_call_skip( turn_context: &TurnContext, call_id: &str, invocation: McpInvocation, + mcp_app_resource_uri: Option, message: String, already_started: bool, ) -> Result { @@ -1622,6 +1713,7 @@ async fn notify_mcp_tool_call_skip( let tool_call_begin_event = EventMsg::McpToolCallBegin(McpToolCallBeginEvent { call_id: call_id.to_string(), invocation: invocation.clone(), + mcp_app_resource_uri: mcp_app_resource_uri.clone(), }); notify_mcp_tool_call_event(sess, turn_context, tool_call_begin_event).await; } @@ -1629,6 +1721,7 @@ async fn notify_mcp_tool_call_skip( let tool_call_end_event = EventMsg::McpToolCallEnd(McpToolCallEndEvent { call_id: call_id.to_string(), invocation, + mcp_app_resource_uri, duration: Duration::ZERO, result: Err(message.clone()), }); diff --git a/codex-rs/core/src/mcp_tool_call_tests.rs b/codex-rs/core/src/mcp_tool_call_tests.rs index 201a162ef7..a85233c773 100644 --- a/codex-rs/core/src/mcp_tool_call_tests.rs +++ b/codex-rs/core/src/mcp_tool_call_tests.rs @@ -59,6 +59,7 @@ fn approval_metadata( connector_description: connector_description.map(str::to_string), tool_title: tool_title.map(str::to_string), tool_description: tool_description.map(str::to_string), + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, } @@ -74,6 +75,35 @@ fn prompt_options( } } +#[test] +fn mcp_app_resource_uri_reads_known_tool_meta_keys() { + let nested = serde_json::json!({ + "ui": { + "resourceUri": "ui://widget/nested.html", + }, + }); + assert_eq!( + get_mcp_app_resource_uri(nested.as_object()), + Some("ui://widget/nested.html".to_string()) + ); + + let flat = serde_json::json!({ + "ui/resourceUri": "ui://widget/flat.html", + }); + assert_eq!( + get_mcp_app_resource_uri(flat.as_object()), + Some("ui://widget/flat.html".to_string()) + ); + + let output_template = serde_json::json!({ + "openai/outputTemplate": "ui://widget/output-template.html", + }); + assert_eq!( + get_mcp_app_resource_uri(output_template.as_object()), + Some("ui://widget/output-template.html".to_string()) + ); +} + #[test] fn approval_required_when_read_only_false_and_destructive() { let annotations = annotations(Some(false), Some(true), /*open_world*/ None); @@ -589,6 +619,7 @@ async fn codex_apps_tool_call_request_meta_includes_turn_metadata_and_codex_apps connector_description: Some("Manage events".to_string()), tool_title: Some("Create Event".to_string()), tool_description: Some("Create a calendar event.".to_string()), + mcp_app_resource_uri: None, codex_apps_meta: Some( serde_json::json!({ "resource_uri": "connector://calendar/tools/calendar_create_event", @@ -746,6 +777,7 @@ fn guardian_mcp_review_request_includes_annotations_when_present() { connector_description: None, tool_title: None, tool_description: None, + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -1114,6 +1146,43 @@ async fn persist_custom_mcp_tool_approval_writes_tool_override() { assert!(contents.contains("[mcp_servers.docs.tools.search]")); } +#[tokio::test] +async fn custom_mcp_tool_approval_mode_uses_server_default_with_tool_override() { + let tmp = tempdir().expect("tempdir"); + std::fs::write( + tmp.path().join(CONFIG_TOML_FILE), + r#" +[mcp_servers.docs] +command = "docs-server" +default_tools_approval_mode = "approve" + +[mcp_servers.docs.tools.search] +approval_mode = "prompt" +"#, + ) + .expect("seed config"); + let config = ConfigBuilder::default() + .codex_home(tmp.path().to_path_buf()) + .build() + .await + .expect("load config"); + let (_session, mut turn_context) = make_session_and_context().await; + turn_context.config = Arc::new(config); + + assert_eq!( + custom_mcp_tool_approval_mode(&turn_context, "docs", "read"), + AppToolApproval::Approve + ); + assert_eq!( + custom_mcp_tool_approval_mode(&turn_context, "docs", "search"), + AppToolApproval::Prompt + ); + assert_eq!( + custom_mcp_tool_approval_mode(&turn_context, "unknown", "search"), + AppToolApproval::Auto + ); +} + #[tokio::test] async fn maybe_persist_mcp_tool_approval_reloads_session_config() { let (session, turn_context) = make_session_and_context().await; @@ -1272,6 +1341,7 @@ async fn approve_mode_skips_when_annotations_do_not_require_approval() { connector_description: None, tool_title: Some("Read Only Tool".to_string()), tool_description: None, + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -1340,6 +1410,7 @@ async fn guardian_mode_skips_auto_when_annotations_do_not_require_approval() { connector_description: None, tool_title: Some("Read Only Tool".to_string()), tool_description: None, + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -1411,6 +1482,7 @@ async fn guardian_mode_mcp_denial_returns_rationale_message() { connector_description: None, tool_title: Some("Dangerous Tool".to_string()), tool_description: Some("Reads calendar data.".to_string()), + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -1462,6 +1534,7 @@ async fn prompt_mode_waits_for_approval_when_annotations_do_not_require_approval connector_description: None, tool_title: Some("Read Only Tool".to_string()), tool_description: None, + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -1539,6 +1612,7 @@ async fn approve_mode_blocks_when_arc_returns_interrupt_for_model() { connector_description: Some("Manage events".to_string()), tool_title: Some("Dangerous Tool".to_string()), tool_description: Some("Performs a risky action.".to_string()), + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -1609,6 +1683,7 @@ async fn custom_approve_mode_blocks_when_arc_returns_interrupt_for_model() { connector_description: None, tool_title: Some("Dangerous Tool".to_string()), tool_description: Some("Performs a risky action.".to_string()), + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -1679,6 +1754,7 @@ async fn approve_mode_blocks_when_arc_returns_interrupt_without_annotations() { connector_description: Some("Manage events".to_string()), tool_title: Some("Dangerous Tool".to_string()), tool_description: Some("Performs a risky action.".to_string()), + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -1757,6 +1833,7 @@ async fn full_access_mode_skips_arc_monitor_for_all_approval_modes() { connector_description: Some("Manage events".to_string()), tool_title: Some("Dangerous Tool".to_string()), tool_description: Some("Performs a risky action.".to_string()), + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -1859,6 +1936,7 @@ async fn approve_mode_routes_arc_ask_user_to_guardian_when_guardian_reviewer_is_ connector_description: Some("Manage events".to_string()), tool_title: Some("Dangerous Tool".to_string()), tool_description: Some("Performs a risky action.".to_string()), + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; diff --git a/codex-rs/core/src/mcp_tool_exposure.rs b/codex-rs/core/src/mcp_tool_exposure.rs index d7858c17f0..16aa50cfd4 100644 --- a/codex-rs/core/src/mcp_tool_exposure.rs +++ b/codex-rs/core/src/mcp_tool_exposure.rs @@ -41,9 +41,13 @@ pub(crate) fn build_mcp_tool_exposure( let direct_tools = filter_codex_apps_mcp_tools(all_mcp_tools, explicitly_enabled_connectors, config); + for direct_tool_name in direct_tools.keys() { + deferred_tools.remove(direct_tool_name); + } + McpToolExposure { direct_tools, - deferred_tools: Some(deferred_tools), + deferred_tools: (!deferred_tools.is_empty()).then_some(deferred_tools), } } diff --git a/codex-rs/core/src/memories/README.md b/codex-rs/core/src/memories/README.md index eecaeaea28..a1d365435b 100644 --- a/codex-rs/core/src/memories/README.md +++ b/codex-rs/core/src/memories/README.md @@ -84,7 +84,11 @@ What it does: - `raw_memories.md` (merged raw memories, latest first) - `rollout_summaries/` (one summary file per retained rollout) - prunes stale rollout summaries that are no longer retained -- if there are no inputs, marks the job successful and exits +- finds old resource files from memory extensions under + `memories_extensions//resources/` for extension directories that + have an `instructions.md`, using the memory module retention window +- if there are no Phase 1 inputs or old extension resources, marks the job + successful and exits If there is input, it then: @@ -92,10 +96,13 @@ If there is input, it then: - builds the Phase 2 prompt with a diff of the current Phase 1 input selection versus the last successful Phase 2 selection (`added`, `retained`, `removed`) +- includes old extension resource paths in the prompt diff - runs it with no approvals, no network, and local write access only - disables collab for that agent (to prevent recursive delegation) - watches the agent status and heartbeats the global job lease while it runs - marks the phase-2 job success/failure in the state DB when the agent finishes +- prunes old extension resource files after the consolidation agent completes + and the successful Phase 2 job is recorded Selection diff behavior: diff --git a/codex-rs/core/src/memories/control.rs b/codex-rs/core/src/memories/control.rs index 10bd19246f..4f09d3e74c 100644 --- a/codex-rs/core/src/memories/control.rs +++ b/codex-rs/core/src/memories/control.rs @@ -1,5 +1,16 @@ use std::path::Path; +pub async fn clear_memory_roots_contents(codex_home: &Path) -> std::io::Result<()> { + for memory_root in [ + codex_home.join("memories"), + codex_home.join("memories_extensions"), + ] { + clear_memory_root_contents(memory_root.as_path()).await?; + } + + Ok(()) +} + pub(crate) async fn clear_memory_root_contents(memory_root: &Path) -> std::io::Result<()> { match tokio::fs::symlink_metadata(memory_root).await { Ok(metadata) if metadata.file_type().is_symlink() => { diff --git a/codex-rs/core/src/memories/extensions.rs b/codex-rs/core/src/memories/extensions.rs new file mode 100644 index 0000000000..91c96ebff0 --- /dev/null +++ b/codex-rs/core/src/memories/extensions.rs @@ -0,0 +1,251 @@ +use crate::memories::memory_extensions_root; +use chrono::DateTime; +use chrono::Duration; +use chrono::NaiveDateTime; +use chrono::Utc; +use std::path::Path; +use std::path::PathBuf; +use tracing::warn; + +const FILENAME_TS_FORMAT: &str = "%Y-%m-%dT%H-%M-%S"; +pub(super) const EXTENSION_RESOURCE_RETENTION_DAYS: i64 = 7; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct RemovedExtensionResource { + pub(super) extension: String, + pub(super) resource_path: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct PendingExtensionResourceRemoval { + pub(super) removed: RemovedExtensionResource, + path: PathBuf, +} + +pub(super) async fn find_old_extension_resources( + memory_root: &Path, +) -> Vec { + find_old_extension_resources_with_now(memory_root, Utc::now()).await +} + +async fn find_old_extension_resources_with_now( + memory_root: &Path, + now: DateTime, +) -> Vec { + let mut pending = Vec::new(); + let cutoff = now - Duration::days(EXTENSION_RESOURCE_RETENTION_DAYS); + let extensions_root = memory_extensions_root(memory_root); + let mut extensions = match tokio::fs::read_dir(&extensions_root).await { + Ok(extensions) => extensions, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return pending, + Err(err) => { + warn!( + "failed reading memory extensions root {}: {err}", + extensions_root.display() + ); + return pending; + } + }; + + while let Ok(Some(extension_entry)) = extensions.next_entry().await { + let extension_path = extension_entry.path(); + let Ok(file_type) = extension_entry.file_type().await else { + continue; + }; + if !file_type.is_dir() { + continue; + } + let Some(extension) = extension_path + .file_name() + .and_then(|name| name.to_str()) + .map(ToOwned::to_owned) + else { + continue; + }; + if !tokio::fs::try_exists(extension_path.join("instructions.md")) + .await + .unwrap_or(false) + { + continue; + } + + let resources_path = extension_path.join("resources"); + let mut resources = match tokio::fs::read_dir(&resources_path).await { + Ok(resources) => resources, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue, + Err(err) => { + warn!( + "failed reading memory extension resources {}: {err}", + resources_path.display() + ); + continue; + } + }; + + while let Ok(Some(resource_entry)) = resources.next_entry().await { + let resource_file_path = resource_entry.path(); + let Ok(file_type) = resource_entry.file_type().await else { + continue; + }; + if !file_type.is_file() { + continue; + } + let Some(file_name) = resource_file_path + .file_name() + .and_then(|name| name.to_str()) + else { + continue; + }; + if !file_name.ends_with(".md") { + continue; + } + let Some(resource_timestamp) = resource_timestamp(file_name) else { + continue; + }; + if resource_timestamp > cutoff { + continue; + } + + pending.push(PendingExtensionResourceRemoval { + removed: RemovedExtensionResource { + extension: extension.clone(), + resource_path: format!("resources/{file_name}"), + }, + path: resource_file_path, + }); + } + } + + pending.sort_by(|left, right| { + left.removed + .extension + .cmp(&right.removed.extension) + .then_with(|| left.removed.resource_path.cmp(&right.removed.resource_path)) + }); + pending +} + +pub(super) async fn remove_extension_resources(resources: &[PendingExtensionResourceRemoval]) { + for resource in resources { + if let Err(err) = tokio::fs::remove_file(&resource.path).await + && err.kind() != std::io::ErrorKind::NotFound + { + warn!( + "failed pruning old memory extension resource {}: {err}", + resource.path.display() + ); + } + } +} + +fn resource_timestamp(file_name: &str) -> Option> { + let timestamp = file_name.get(..19)?; + let naive = NaiveDateTime::parse_from_str(timestamp, FILENAME_TS_FORMAT).ok()?; + Some(DateTime::from_naive_utc_and_offset(naive, Utc)) +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use tempfile::TempDir; + + #[tokio::test] + async fn finds_only_old_resources_from_extensions_with_instructions() { + let codex_home = TempDir::new().expect("create temp codex home"); + let memory_root = codex_home.path().join("memories"); + let extensions_root = memory_extensions_root(&memory_root); + let telepathy_resources = extensions_root.join("telepathy/resources"); + tokio::fs::create_dir_all(&telepathy_resources) + .await + .expect("create telepathy resources"); + tokio::fs::write( + extensions_root.join("telepathy/instructions.md"), + "instructions", + ) + .await + .expect("write telepathy instructions"); + + let now = DateTime::from_naive_utc_and_offset( + NaiveDateTime::parse_from_str("2026-04-14T12-00-00", FILENAME_TS_FORMAT) + .expect("parse now"), + Utc, + ); + let old_file = telepathy_resources.join("2026-04-06T11-59-59-abcd-10min-old.md"); + let exact_cutoff_file = + telepathy_resources.join("2026-04-07T12-00-00-abcd-10min-cutoff.md"); + let recent_file = telepathy_resources.join("2026-04-08T12-00-00-abcd-10min-recent.md"); + let invalid_file = telepathy_resources.join("not-a-timestamp.md"); + for file in [&old_file, &exact_cutoff_file, &recent_file, &invalid_file] { + tokio::fs::write(file, "resource") + .await + .expect("write telepathy resource"); + } + + let ignored_resources = extensions_root.join("ignored/resources"); + tokio::fs::create_dir_all(&ignored_resources) + .await + .expect("create ignored resources"); + let ignored_old_file = ignored_resources.join("2026-04-06T11-59-59-abcd-10min-old.md"); + tokio::fs::write(&ignored_old_file, "ignored") + .await + .expect("write ignored resource"); + + let pending = find_old_extension_resources_with_now(&memory_root, now).await; + + assert_eq!( + pending + .iter() + .map(|resource| resource.removed.clone()) + .collect::>(), + vec![ + RemovedExtensionResource { + extension: "telepathy".to_string(), + resource_path: "resources/2026-04-06T11-59-59-abcd-10min-old.md".to_string(), + }, + RemovedExtensionResource { + extension: "telepathy".to_string(), + resource_path: "resources/2026-04-07T12-00-00-abcd-10min-cutoff.md".to_string(), + }, + ] + ); + assert!( + tokio::fs::try_exists(&old_file) + .await + .expect("check old file before remove") + ); + assert!( + tokio::fs::try_exists(&exact_cutoff_file) + .await + .expect("check cutoff file before remove") + ); + + remove_extension_resources(&pending).await; + + assert!( + !tokio::fs::try_exists(&old_file) + .await + .expect("check old file") + ); + assert!( + !tokio::fs::try_exists(&exact_cutoff_file) + .await + .expect("check cutoff file") + ); + assert!( + tokio::fs::try_exists(&recent_file) + .await + .expect("check recent file") + ); + assert!( + tokio::fs::try_exists(&invalid_file) + .await + .expect("check invalid file") + ); + assert!( + tokio::fs::try_exists(&ignored_old_file) + .await + .expect("check ignored old file") + ); + } +} diff --git a/codex-rs/core/src/memories/mod.rs b/codex-rs/core/src/memories/mod.rs index fb033a32c1..d796063d2d 100644 --- a/codex-rs/core/src/memories/mod.rs +++ b/codex-rs/core/src/memories/mod.rs @@ -17,7 +17,7 @@ pub(crate) mod usage; use codex_protocol::openai_models::ReasoningEffort; -pub(crate) use control::clear_memory_root_contents; +pub use control::clear_memory_roots_contents; /// Starts the memory startup pipeline for eligible root sessions. /// This is the single entrypoint that `codex` uses to trigger memory startup. /// @@ -30,6 +30,8 @@ mod artifacts { pub(super) const RAW_MEMORIES_FILENAME: &str = "raw_memories.md"; } +mod extensions; + /// Phase 1 (startup extraction). mod phase_one { /// Default model used for phase 1. diff --git a/codex-rs/core/src/memories/phase1.rs b/codex-rs/core/src/memories/phase1.rs index 37cb0e9f48..c005f4da4e 100644 --- a/codex-rs/core/src/memories/phase1.rs +++ b/codex-rs/core/src/memories/phase1.rs @@ -477,9 +477,10 @@ mod job { } }) .collect::>(); - serde_json::to_string(&filtered).map_err(|err| { + let serialized = serde_json::to_string(&filtered).map_err(|err| { CodexErr::InvalidRequest(format!("failed to serialize rollout memory: {err}")) - }) + })?; + Ok(redact_secrets(serialized)) } fn sanitize_response_item_for_memories(item: &ResponseItem) -> Option { diff --git a/codex-rs/core/src/memories/phase1_tests.rs b/codex-rs/core/src/memories/phase1_tests.rs index 9d824ab2b1..89bde1a877 100644 --- a/codex-rs/core/src/memories/phase1_tests.rs +++ b/codex-rs/core/src/memories/phase1_tests.rs @@ -3,6 +3,8 @@ use super::JobResult; use super::aggregate_stats; use super::job::serialize_filtered_rollout_response_items; use codex_protocol::models::ContentItem; +use codex_protocol::models::FunctionCallOutputBody; +use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::TokenUsage; @@ -72,6 +74,25 @@ fn serializes_memory_rollout_with_agents_removed_but_environment_kept() { ); } +#[test] +fn serializes_memory_rollout_redacts_secrets_before_prompt_upload() { + let serialized = serialize_filtered_rollout_response_items(&[RolloutItem::ResponseItem( + ResponseItem::FunctionCallOutput { + call_id: "call_123".to_string(), + output: FunctionCallOutputPayload { + body: FunctionCallOutputBody::Text( + r#"{"token":"sk-abcdefghijklmnopqrstuvwxyz123456"}"#.to_string(), + ), + success: Some(true), + }, + }, + )]) + .expect("serialize"); + + assert!(!serialized.contains("sk-abcdefghijklmnopqrstuvwxyz123456")); + assert!(serialized.contains("[REDACTED_SECRET]")); +} + #[test] fn count_outcomes_sums_token_usage_across_all_jobs() { let counts = aggregate_stats(vec![ diff --git a/codex-rs/core/src/memories/phase2.rs b/codex-rs/core/src/memories/phase2.rs index 2abc435ca7..dd194edfbf 100644 --- a/codex-rs/core/src/memories/phase2.rs +++ b/codex-rs/core/src/memories/phase2.rs @@ -3,6 +3,9 @@ use crate::agent::status::is_final as is_final_agent_status; use crate::codex::Session; use crate::codex::emit_subagent_session_started; use crate::config::Config; +use crate::memories::extensions::PendingExtensionResourceRemoval; +use crate::memories::extensions::find_old_extension_resources; +use crate::memories::extensions::remove_extension_resources; use crate::memories::memory_root; use crate::memories::metrics; use crate::memories::phase_two; @@ -112,7 +115,12 @@ pub(super) async fn run(session: &Arc, config: Arc) { job::failed(session, db, &claim, "failed_rebuild_raw_memories").await; return; } - if raw_memories.is_empty() { + let pending_extension_resource_removals = find_old_extension_resources(&root).await; + let removed_extension_resources = pending_extension_resource_removals + .iter() + .map(|resource| resource.removed.clone()) + .collect::>(); + if raw_memories.is_empty() && pending_extension_resource_removals.is_empty() { // We check only after sync of the file system. job::succeed( session, @@ -127,7 +135,7 @@ pub(super) async fn run(session: &Arc, config: Arc) { } // 5. Spawn the agent - let prompt = agent::get_prompt(config, &selection); + let prompt = agent::get_prompt(config, &selection, &removed_extension_resources); let source = SessionSource::SubAgent(SubAgentSource::MemoryConsolidation); let thread_id = match session .services @@ -170,6 +178,7 @@ pub(super) async fn run(session: &Arc, config: Arc) { claim, new_watermark, raw_memories.clone(), + pending_extension_resource_removals, thread_id, phase_two_e2e_timer, ); @@ -268,15 +277,15 @@ mod job { completion_watermark: i64, selected_outputs: &[codex_state::Stage1Output], reason: &'static str, - ) { + ) -> bool { session.services.session_telemetry.counter( metrics::MEMORY_PHASE_TWO_JOBS, /*inc*/ 1, &[("status", reason)], ); - let _ = db - .mark_global_phase2_job_succeeded(&claim.token, completion_watermark, selected_outputs) - .await; + db.mark_global_phase2_job_succeeded(&claim.token, completion_watermark, selected_outputs) + .await + .unwrap_or(false) } } @@ -328,9 +337,10 @@ mod agent { pub(super) fn get_prompt( config: Arc, selection: &codex_state::Phase2InputSelection, + removed_extension_resources: &[crate::memories::extensions::RemovedExtensionResource], ) -> Vec { let root = memory_root(&config.codex_home); - let prompt = build_consolidation_prompt(&root, selection); + let prompt = build_consolidation_prompt(&root, selection, removed_extension_resources); vec![UserInput::Text { text: prompt, text_elements: vec![], @@ -343,6 +353,7 @@ mod agent { claim: Claim, new_watermark: i64, selected_outputs: Vec, + pending_extension_resource_removals: Vec, thread_id: ThreadId, phase_two_e2e_timer: Option, ) { @@ -379,7 +390,7 @@ mod agent { if let Some(token_usage) = agent_control.get_total_token_usage(thread_id).await { emit_token_usage_metrics(&session, &token_usage); } - job::succeed( + if job::succeed( &session, &db, &claim, @@ -387,7 +398,10 @@ mod agent { &selected_outputs, "succeeded", ) - .await; + .await + { + remove_extension_resources(&pending_extension_resource_removals).await; + } } else { job::failed(&session, &db, &claim, "failed_agent").await; } diff --git a/codex-rs/core/src/memories/prompts.rs b/codex-rs/core/src/memories/prompts.rs index b6e18b4057..fc85917c9b 100644 --- a/codex-rs/core/src/memories/prompts.rs +++ b/codex-rs/core/src/memories/prompts.rs @@ -1,3 +1,5 @@ +use crate::memories::extensions::EXTENSION_RESOURCE_RETENTION_DAYS; +use crate::memories::extensions::RemovedExtensionResource; use crate::memories::memory_extensions_root; use crate::memories::memory_root; use crate::memories::phase_one; @@ -10,6 +12,7 @@ use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_output_truncation::TruncationPolicy; use codex_utils_output_truncation::truncate_text; use codex_utils_template::Template; +use std::fmt::Write as _; use std::path::Path; use std::sync::LazyLock; use tokio::fs; @@ -62,8 +65,9 @@ Memory extensions (under {{ memory_extensions_root }}/): source. If the user has any memory extensions, you MUST read the instructions for each extension to -determine how to use the memory source. If it has no extension folders, continue with the standard -memory inputs only. +determine how to use the memory source. If the Phase 2 diff lists removed memory extension +resources, use that extension-specific deletion diff to remove stale memories derived only from +those resources. If it has no extension folders, continue with the standard memory inputs only. "#; const MEMORY_EXTENSIONS_PRIMARY_INPUTS: &str = r#" @@ -73,12 +77,16 @@ Under `{{ memory_extensions_root }}/`: - `/instructions.md` - If extension folders exist, read each instructions.md first and follow it when interpreting that extension's memory source. + +If the Phase 2 diff lists removed memory extension resources, use that extension-specific deletion +diff to remove stale memories derived only from those resources. "#; /// Builds the consolidation subagent prompt for a specific memory root. pub(super) fn build_consolidation_prompt( memory_root: &Path, selection: &Phase2InputSelection, + removed_extension_resources: &[RemovedExtensionResource], ) -> String { let memory_extensions_root = memory_extensions_root(memory_root); let memory_extensions_exist = memory_extensions_root.is_dir(); @@ -100,7 +108,8 @@ pub(super) fn build_consolidation_prompt( } else { String::new() }; - let phase2_input_selection = render_phase2_input_selection(selection); + let phase2_input_selection = + render_phase2_input_selection(selection, removed_extension_resources); CONSOLIDATION_PROMPT_TEMPLATE .render([ ("memory_root", memory_root.as_str()), @@ -131,7 +140,10 @@ fn render_memory_extensions_block(template: &Template, memory_extensions_root: & }) } -fn render_phase2_input_selection(selection: &Phase2InputSelection) -> String { +fn render_phase2_input_selection( + selection: &Phase2InputSelection, + removed_extension_resources: &[RemovedExtensionResource], +) -> String { let retained = selection.retained_thread_ids.len(); let added = selection.selected.len().saturating_sub(retained); let selected = if selection.selected.is_empty() { @@ -160,11 +172,29 @@ fn render_phase2_input_selection(selection: &Phase2InputSelection) -> String { .join("\n") }; - format!( + let mut rendered = format!( "- selected inputs this run: {}\n- newly added since the last successful Phase 2 run: {added}\n- retained from the last successful Phase 2 run: {retained}\n- removed from the last successful Phase 2 run: {}\n\nCurrent selected Phase 1 inputs:\n{selected}\n\nRemoved from the last successful Phase 2 selection:\n{removed}\n", selection.selected.len(), selection.removed.len(), - ) + ); + + if !removed_extension_resources.is_empty() { + rendered.push_str("\nMemory extension resources removed by retention pruning:\n"); + let _ = writeln!( + rendered, + "- retention window: {EXTENSION_RESOURCE_RETENTION_DAYS} days" + ); + let mut current_extension = ""; + for removed_resource in removed_extension_resources { + if removed_resource.extension != current_extension { + current_extension = &removed_resource.extension; + let _ = writeln!(rendered, "- extension: {current_extension}"); + } + let _ = writeln!(rendered, " - {}", removed_resource.resource_path); + } + } + + rendered } fn render_selected_input_line(item: &Stage1Output, retained: bool) -> String { diff --git a/codex-rs/core/src/memories/prompts_tests.rs b/codex-rs/core/src/memories/prompts_tests.rs index 325d5e9234..959bb6911d 100644 --- a/codex-rs/core/src/memories/prompts_tests.rs +++ b/codex-rs/core/src/memories/prompts_tests.rs @@ -1,5 +1,7 @@ use super::*; +use crate::memories::extensions::RemovedExtensionResource; use codex_models_manager::model_info::model_info_from_slug; +use codex_state::Phase2InputSelection; use core_test_support::PathExt; use pretty_assertions::assert_eq; use tempfile::tempdir; @@ -54,6 +56,36 @@ fn build_stage_one_input_message_uses_default_limit_when_model_context_window_mi assert!(message.contains(&expected_truncated)); } +#[test] +fn build_consolidation_prompt_includes_removed_extension_resources() { + let temp = tempdir().unwrap(); + let memory_root = temp.path().join("memories"); + std::fs::create_dir_all(temp.path().join("memories_extensions")).unwrap(); + let removed_extension_resources = vec![ + RemovedExtensionResource { + extension: "telepathy".to_string(), + resource_path: "resources/2026-04-06T11-59-59-abcd-10min-old.md".to_string(), + }, + RemovedExtensionResource { + extension: "telepathy".to_string(), + resource_path: "resources/2026-04-07T12-00-00-abcd-10min-cutoff.md".to_string(), + }, + ]; + + let prompt = build_consolidation_prompt( + &memory_root, + &Phase2InputSelection::default(), + &removed_extension_resources, + ); + + assert!(prompt.contains("Memory extension resources removed by retention pruning:")); + assert!(prompt.contains("- retention window: 7 days")); + assert!(prompt.contains("- extension: telepathy")); + assert!(prompt.contains(" - resources/2026-04-06T11-59-59-abcd-10min-old.md")); + assert!(prompt.contains(" - resources/2026-04-07T12-00-00-abcd-10min-cutoff.md")); + assert!(prompt.contains("extension-specific deletion diff")); +} + #[tokio::test] async fn build_memory_tool_developer_instructions_renders_embedded_template() { let temp = tempdir().unwrap(); diff --git a/codex-rs/core/src/memories/tests.rs b/codex-rs/core/src/memories/tests.rs index a0ed29dbec..ab3538af67 100644 --- a/codex-rs/core/src/memories/tests.rs +++ b/codex-rs/core/src/memories/tests.rs @@ -1,6 +1,6 @@ +use super::control::clear_memory_root_contents; use super::storage::rebuild_raw_memories_file_from_memories; use super::storage::sync_rollout_summaries_from_memories; -use crate::memories::clear_memory_root_contents; use crate::memories::ensure_layout; use crate::memories::memory_root; use crate::memories::raw_memories_file; @@ -424,6 +424,7 @@ mod phase2 { use crate::memories::phase2; use crate::memories::raw_memories_file; use crate::memories::rollout_summaries_dir; + use chrono::Duration as ChronoDuration; use chrono::Utc; use codex_config::Constrained; use codex_login::CodexAuth; @@ -467,7 +468,7 @@ mod phase2 { impl DispatchHarness { async fn new() -> Self { let codex_home = tempfile::tempdir().expect("create temp codex home"); - let mut config = test_config(); + let mut config = test_config().await; config.codex_home = codex_utils_absolute_path::AbsolutePathBuf::from_absolute_path(codex_home.path()) .expect("codex home is absolute"); @@ -894,7 +895,7 @@ mod phase2 { #[tokio::test] async fn dispatch_marks_job_for_retry_when_spawn_agent_fails() { let codex_home = tempfile::tempdir().expect("create temp codex home"); - let mut config = test_config(); + let mut config = test_config().await; config.codex_home = codex_utils_absolute_path::AbsolutePathBuf::from_absolute_path(codex_home.path()) .expect("codex home is absolute"); @@ -960,6 +961,28 @@ mod phase2 { "stage-1 success should enqueue global consolidation" ); + let telepathy_resources = config + .codex_home + .join("memories_extensions/telepathy/resources"); + tokio::fs::create_dir_all(&telepathy_resources) + .await + .expect("create telepathy resources"); + tokio::fs::write( + config + .codex_home + .join("memories_extensions/telepathy/instructions.md"), + "instructions", + ) + .await + .expect("write telepathy instructions"); + let old_file = telepathy_resources.join(format!( + "{}-abcd-10min-old.md", + (Utc::now() - ChronoDuration::days(8)).format("%Y-%m-%dT%H-%M-%S") + )); + tokio::fs::write(&old_file, "old resource") + .await + .expect("write old extension resource"); + phase2::run(&session, Arc::clone(&config)).await; let retry_claim = state_db @@ -971,5 +994,11 @@ mod phase2 { Phase2JobClaimOutcome::SkippedNotDirty, "spawn failures should leave the job in retry backoff instead of running" ); + assert!( + tokio::fs::try_exists(&old_file) + .await + .expect("check old extension resource"), + "spawn failures should not prune extension resources before retry" + ); } } diff --git a/codex-rs/core/src/original_image_detail.rs b/codex-rs/core/src/original_image_detail.rs index c3e5c3d9a5..adfed321b8 100644 --- a/codex-rs/core/src/original_image_detail.rs +++ b/codex-rs/core/src/original_image_detail.rs @@ -1,2 +1,3 @@ pub(crate) use codex_tools::can_request_original_image_detail; pub(crate) use codex_tools::normalize_output_image_detail; +pub(crate) use codex_tools::sanitize_original_image_detail; diff --git a/codex-rs/core/src/personality_migration.rs b/codex-rs/core/src/personality_migration.rs index 52cabf55de..4b53a2629e 100644 --- a/codex-rs/core/src/personality_migration.rs +++ b/codex-rs/core/src/personality_migration.rs @@ -1,14 +1,10 @@ use crate::config::edit::ConfigEditsBuilder; -use crate::rollout::ARCHIVED_SESSIONS_SUBDIR; -use crate::rollout::SESSIONS_SUBDIR; -use crate::rollout::list::ThreadListConfig; -use crate::rollout::list::ThreadListLayout; -use crate::rollout::list::ThreadSortKey; -use crate::rollout::list::get_threads_in_root; use codex_config::config_toml::ConfigToml; use codex_protocol::config_types::Personality; -use codex_protocol::protocol::SessionSource; -use codex_rollout::state_db; +use codex_thread_store::ListThreadsParams; +use codex_thread_store::LocalThreadStore; +use codex_thread_store::ThreadSortKey; +use codex_thread_store::ThreadStore; use std::io; use std::path::Path; use tokio::fs::OpenOptions; @@ -64,57 +60,33 @@ pub async fn maybe_migrate_personality( } async fn has_recorded_sessions(codex_home: &Path, default_provider: &str) -> io::Result { - let allowed_sources: &[SessionSource] = &[]; + let store = LocalThreadStore::new(codex_rollout::RolloutConfig { + codex_home: codex_home.to_path_buf(), + sqlite_home: codex_home.to_path_buf(), + cwd: codex_home.to_path_buf(), + model_provider_id: default_provider.to_string(), + generate_memories: false, + }); + if has_threads(&store, /*archived*/ false).await? { + return Ok(true); + } + has_threads(&store, /*archived*/ true).await +} - if let Some(state_db_ctx) = state_db::open_if_present(codex_home, default_provider).await - && let Some(ids) = state_db::list_thread_ids_db( - Some(state_db_ctx.as_ref()), - codex_home, - /*page_size*/ 1, - /*cursor*/ None, - ThreadSortKey::CreatedAt, - allowed_sources, - /*model_providers*/ None, - /*archived_only*/ false, - "personality_migration", - ) +async fn has_threads(store: &LocalThreadStore, archived: bool) -> io::Result { + store + .list_threads(ListThreadsParams { + page_size: 1, + cursor: None, + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived, + search_term: None, + }) .await - && !ids.is_empty() - { - return Ok(true); - } - - let sessions = get_threads_in_root( - codex_home.join(SESSIONS_SUBDIR), - /*page_size*/ 1, - /*cursor*/ None, - ThreadSortKey::CreatedAt, - ThreadListConfig { - allowed_sources, - model_providers: None, - default_provider, - layout: ThreadListLayout::NestedByDate, - }, - ) - .await?; - if !sessions.items.is_empty() { - return Ok(true); - } - - let archived_sessions = get_threads_in_root( - codex_home.join(ARCHIVED_SESSIONS_SUBDIR), - /*page_size*/ 1, - /*cursor*/ None, - ThreadSortKey::CreatedAt, - ThreadListConfig { - allowed_sources, - model_providers: None, - default_provider, - layout: ThreadListLayout::Flat, - }, - ) - .await?; - Ok(!archived_sessions.items.is_empty()) + .map(|page| !page.items.is_empty()) + .map_err(io::Error::other) } async fn create_marker(marker_path: &Path) -> io::Result<()> { diff --git a/codex-rs/core/src/personality_migration_tests.rs b/codex-rs/core/src/personality_migration_tests.rs index de1070ad34..4aef53a5c4 100644 --- a/codex-rs/core/src/personality_migration_tests.rs +++ b/codex-rs/core/src/personality_migration_tests.rs @@ -7,6 +7,8 @@ use codex_protocol::protocol::SessionMeta; use codex_protocol::protocol::SessionMetaLine; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::UserMessageEvent; +use codex_rollout::ARCHIVED_SESSIONS_SUBDIR; +use codex_rollout::SESSIONS_SUBDIR; use pretty_assertions::assert_eq; use tempfile::TempDir; use tokio::io::AsyncWriteExt; @@ -25,6 +27,16 @@ async fn write_session_with_user_event(codex_home: &Path) -> io::Result<()> { .join("2025") .join("01") .join("01"); + write_rollout_with_user_event(&dir, thread_id).await +} + +async fn write_archived_session_with_user_event(codex_home: &Path) -> io::Result<()> { + let thread_id = ThreadId::new(); + let dir = codex_home.join(ARCHIVED_SESSIONS_SUBDIR); + write_rollout_with_user_event(&dir, thread_id).await +} + +async fn write_rollout_with_user_event(dir: &Path, thread_id: ThreadId) -> io::Result<()> { tokio::fs::create_dir_all(&dir).await?; let file_path = dir.join(format!("rollout-{TEST_TIMESTAMP}-{thread_id}.jsonl")); let mut file = tokio::fs::File::create(&file_path).await?; @@ -85,6 +97,22 @@ async fn applies_when_sessions_exist_and_no_personality() -> io::Result<()> { Ok(()) } +#[tokio::test] +async fn applies_when_only_archived_sessions_exist_and_no_personality() -> io::Result<()> { + let temp = TempDir::new()?; + write_archived_session_with_user_event(temp.path()).await?; + + let config_toml = ConfigToml::default(); + let status = maybe_migrate_personality(temp.path(), &config_toml).await?; + + assert_eq!(status, PersonalityMigrationStatus::Applied); + assert!(temp.path().join(PERSONALITY_MIGRATION_FILENAME).exists()); + + let persisted = read_config_toml(temp.path()).await?; + assert_eq!(persisted.personality, Some(Personality::Pragmatic)); + Ok(()) +} + #[tokio::test] async fn skips_when_marker_exists() -> io::Result<()> { let temp = TempDir::new()?; diff --git a/codex-rs/core/src/plugins/injection.rs b/codex-rs/core/src/plugins/injection.rs index 3e5bb6ecb5..2f6ec26260 100644 --- a/codex-rs/core/src/plugins/injection.rs +++ b/codex-rs/core/src/plugins/injection.rs @@ -1,6 +1,7 @@ use std::collections::BTreeSet; use std::collections::HashMap; +use codex_connectors::metadata::connector_display_label; use codex_protocol::models::DeveloperInstructions; use codex_protocol::models::ResponseItem; @@ -46,7 +47,7 @@ pub(crate) fn build_plugin_injections( .iter() .any(|plugin_name| plugin_name == &plugin.display_name) }) - .map(connectors::connector_display_label) + .map(connector_display_label) .collect::>() .into_iter() .collect::>(); diff --git a/codex-rs/core/src/plugins/installed_marketplaces.rs b/codex-rs/core/src/plugins/installed_marketplaces.rs index df69111a27..a6f80139a9 100644 --- a/codex-rs/core/src/plugins/installed_marketplaces.rs +++ b/codex-rs/core/src/plugins/installed_marketplaces.rs @@ -1,4 +1,5 @@ use crate::config::Config; +use codex_core_plugins::marketplace::find_marketplace_manifest_path; use codex_utils_absolute_path::AbsolutePathBuf; use std::path::Path; use std::path::PathBuf; @@ -50,9 +51,7 @@ pub(crate) fn installed_marketplace_roots_from_config( marketplace, &default_install_root, )?; - path.join(".agents/plugins/marketplace.json") - .is_file() - .then_some(path) + find_marketplace_manifest_path(&path).map(|_| path) }) .filter_map(|path| AbsolutePathBuf::try_from(path).ok()) .collect::>(); diff --git a/codex-rs/core/src/plugins/manager.rs b/codex-rs/core/src/plugins/manager.rs index a735180ab6..7a5991b03a 100644 --- a/codex-rs/core/src/plugins/manager.rs +++ b/codex-rs/core/src/plugins/manager.rs @@ -1,52 +1,53 @@ -use super::LoadedPlugin; use super::PluginLoadOutcome; -use super::PluginManifestPaths; use super::curated_plugins_repo_path; use super::installed_marketplaces::installed_marketplace_roots_from_config; -use super::load_plugin_manifest; -use super::manifest::PluginManifestInterface; -use super::marketplace::MarketplaceError; -use super::marketplace::MarketplaceInterface; -use super::marketplace::MarketplaceListError; -use super::marketplace::MarketplacePluginAuthPolicy; -use super::marketplace::MarketplacePluginPolicy; -use super::marketplace::MarketplacePluginSource; -use super::marketplace::ResolvedMarketplacePlugin; -use super::marketplace::list_marketplaces; -use super::marketplace::load_marketplace; -use super::marketplace::resolve_marketplace_plugin; use super::read_curated_plugins_sha; -use super::remote::RemotePluginFetchError; -use super::remote::RemotePluginMutationError; -use super::remote::enable_remote_plugin; -use super::remote::fetch_remote_featured_plugin_ids; -use super::remote::fetch_remote_plugin_status; -use super::remote::uninstall_remote_plugin; use super::startup_sync::start_startup_remote_plugin_sync_once; -use super::store::PluginInstallResult as StorePluginInstallResult; -use super::store::PluginStore; -use super::store::PluginStoreError; -use super::store::plugin_version_for_source; use super::sync_openai_plugins_repo; use crate::SkillMetadata; -use crate::config::CONFIG_TOML_FILE; use crate::config::Config; use crate::config::ConfigService; use crate::config::ConfigServiceError; use crate::config::edit::ConfigEdit; use crate::config::edit::ConfigEditsBuilder; use crate::config_loader::ConfigLayerStack; -use crate::config_rules::SkillConfigRules; -use crate::config_rules::resolve_disabled_skill_paths; -use crate::config_rules::skill_config_rules_from_stack; -use crate::loader::SkillRoot; -use crate::loader::load_skills_from_roots; use codex_analytics::AnalyticsEventsClient; use codex_app_server_protocol::ConfigValueWriteParams; use codex_app_server_protocol::MergeStrategy; -use codex_config::types::McpServerConfig; use codex_config::types::PluginConfig; -use codex_exec_server::LOCAL_FS; +use codex_core_plugins::loader::configured_curated_plugin_ids_from_codex_home; +use codex_core_plugins::loader::installed_plugin_telemetry_metadata; +use codex_core_plugins::loader::load_plugin_apps; +use codex_core_plugins::loader::load_plugin_mcp_servers; +use codex_core_plugins::loader::load_plugin_skills; +use codex_core_plugins::loader::load_plugins_from_layer_stack; +use codex_core_plugins::loader::log_plugin_load_errors; +use codex_core_plugins::loader::plugin_telemetry_metadata_from_root; +use codex_core_plugins::loader::refresh_curated_plugin_cache; +use codex_core_plugins::loader::refresh_non_curated_plugin_cache; +use codex_core_plugins::loader::refresh_non_curated_plugin_cache_force_reinstall; +use codex_core_plugins::manifest::PluginManifestInterface; +use codex_core_plugins::manifest::load_plugin_manifest; +use codex_core_plugins::marketplace::MarketplaceError; +use codex_core_plugins::marketplace::MarketplaceInterface; +use codex_core_plugins::marketplace::MarketplaceListError; +use codex_core_plugins::marketplace::MarketplacePluginAuthPolicy; +use codex_core_plugins::marketplace::MarketplacePluginPolicy; +use codex_core_plugins::marketplace::MarketplacePluginSource; +use codex_core_plugins::marketplace::ResolvedMarketplacePlugin; +use codex_core_plugins::marketplace::list_marketplaces; +use codex_core_plugins::marketplace::load_marketplace; +use codex_core_plugins::marketplace::resolve_marketplace_plugin; +use codex_core_plugins::marketplace_upgrade::ConfiguredMarketplaceUpgradeError; +use codex_core_plugins::marketplace_upgrade::ConfiguredMarketplaceUpgradeOutcome; +use codex_core_plugins::marketplace_upgrade::configured_git_marketplace_names; +use codex_core_plugins::marketplace_upgrade::upgrade_configured_git_marketplaces; +use codex_core_plugins::remote::RemotePluginFetchError; +use codex_core_plugins::remote::RemotePluginMutationError; +use codex_core_plugins::remote::RemotePluginServiceConfig; +use codex_core_plugins::store::PluginInstallResult as StorePluginInstallResult; +use codex_core_plugins::store::PluginStore; +use codex_core_plugins::store::PluginStoreError; use codex_features::Feature; use codex_login::AuthManager; use codex_login::CodexAuth; @@ -54,19 +55,12 @@ use codex_plugin::AppConnectorId; use codex_plugin::PluginCapabilitySummary; use codex_plugin::PluginId; use codex_plugin::PluginIdError; -use codex_plugin::PluginTelemetryMetadata; use codex_plugin::prompt_safe_plugin_description; use codex_protocol::protocol::Product; -use codex_protocol::protocol::SkillScope; use codex_utils_absolute_path::AbsolutePathBuf; -use serde::Deserialize; -use serde_json::Map as JsonMap; -use serde_json::Value as JsonValue; use serde_json::json; use std::collections::HashMap; use std::collections::HashSet; -use std::fs; -use std::path::Path; use std::path::PathBuf; use std::sync::Arc; use std::sync::RwLock; @@ -78,9 +72,6 @@ use toml_edit::value; use tracing::info; use tracing::warn; -const DEFAULT_SKILLS_DIR_NAME: &str = "skills"; -const DEFAULT_MCP_CONFIG_FILE: &str = ".mcp.json"; -const DEFAULT_APP_CONFIG_FILE: &str = ".app.json"; pub const OPENAI_CURATED_MARKETPLACE_NAME: &str = "openai-curated"; pub const OPENAI_CURATED_MARKETPLACE_DISPLAY_NAME: &str = "OpenAI Curated"; static CURATED_REPO_SYNC_STARTED: AtomicBool = AtomicBool::new(false); @@ -102,13 +93,36 @@ struct CachedFeaturedPluginIds { featured_plugin_ids: Vec, } +#[derive(Clone, PartialEq, Eq)] +struct NonCuratedCacheRefreshRequest { + roots: Vec, + mode: NonCuratedCacheRefreshMode, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum NonCuratedCacheRefreshMode { + IfVersionChanged, + ForceReinstall, +} + #[derive(Default)] struct NonCuratedCacheRefreshState { - requested_roots: Option>, - last_refreshed_roots: Option>, + requested: Option, + last_refreshed: Option, in_flight: bool, } +#[derive(Default)] +struct ConfiguredMarketplaceUpgradeState { + in_flight: bool, +} + +fn remote_plugin_service_config(config: &Config) -> RemotePluginServiceConfig { + RemotePluginServiceConfig { + chatgpt_base_url: config.chatgpt_base_url.clone(), + } +} + fn featured_plugin_ids_cache_key( config: &Config, auth: Option<&CodexAuth>, @@ -322,6 +336,7 @@ pub struct PluginsManager { codex_home: PathBuf, store: PluginStore, featured_plugin_ids_cache: RwLock>, + configured_marketplace_upgrade_state: RwLock, non_curated_cache_refresh_state: RwLock, cached_enabled_outcome: RwLock>, remote_sync_lock: Mutex<()>, @@ -349,6 +364,9 @@ impl PluginsManager { codex_home: codex_home.clone(), store: PluginStore::new(codex_home), featured_plugin_ids_cache: RwLock::new(None), + configured_marketplace_upgrade_state: RwLock::new( + ConfiguredMarketplaceUpgradeState::default(), + ), non_curated_cache_refresh_state: RwLock::new(NonCuratedCacheRefreshState::default()), cached_enabled_outcome: RwLock::new(None), remote_sync_lock: Mutex::new(()), @@ -503,8 +521,12 @@ impl PluginsManager { if let Some(featured_plugin_ids) = self.cached_featured_plugin_ids(&cache_key) { return Ok(featured_plugin_ids); } - let featured_plugin_ids = - fetch_remote_featured_plugin_ids(config, auth, self.restriction_product).await?; + let featured_plugin_ids = codex_core_plugins::remote::fetch_remote_featured_plugin_ids( + &remote_plugin_service_config(config), + auth, + self.restriction_product, + ) + .await?; self.write_featured_plugin_ids_cache(cache_key, &featured_plugin_ids); Ok(featured_plugin_ids) } @@ -536,9 +558,13 @@ impl PluginsManager { // This only forwards the backend mutation before the local install flow. We rely on // `plugin/list(forceRemoteSync=true)` to sync local state rather than doing an extra // reconcile pass here. - enable_remote_plugin(config, auth, &plugin_id) - .await - .map_err(PluginInstallError::from)?; + codex_core_plugins::remote::enable_remote_plugin( + &remote_plugin_service_config(config), + auth, + &plugin_id, + ) + .await + .map_err(PluginInstallError::from)?; self.install_resolved_plugin(resolved).await } @@ -619,9 +645,13 @@ impl PluginsManager { // This only forwards the backend mutation before the local uninstall flow. We rely on // `plugin/list(forceRemoteSync=true)` to sync local state rather than doing an extra // reconcile pass here. - uninstall_remote_plugin(config, auth, &plugin_key) - .await - .map_err(PluginUninstallError::from)?; + codex_core_plugins::remote::uninstall_remote_plugin( + &remote_plugin_service_config(config), + auth, + &plugin_key, + ) + .await + .map_err(PluginUninstallError::from)?; self.uninstall_plugin_id(plugin_id).await } @@ -670,9 +700,12 @@ impl PluginsManager { } info!("starting remote plugin sync"); - let remote_plugins = fetch_remote_plugin_status(config, auth) - .await - .map_err(PluginRemoteSyncError::from)?; + let remote_plugins = codex_core_plugins::remote::fetch_remote_plugin_status( + &remote_plugin_service_config(config), + auth, + ) + .await + .map_err(PluginRemoteSyncError::from)?; let configured_plugins = configured_plugins_from_stack(&config.config_layer_stack); let curated_marketplace_root = curated_plugins_repo_path(self.codex_home.as_path()); let curated_marketplace_path = AbsolutePathBuf::try_from( @@ -1026,7 +1059,9 @@ impl PluginsManager { })?; let description = manifest.description.clone(); let manifest_paths = &manifest.paths; - let skill_config_rules = skill_config_rules_from_stack(&config.config_layer_stack); + let skill_config_rules = codex_core_skills::config_rules::skill_config_rules_from_stack( + &config.config_layer_stack, + ); let resolved_skills = load_plugin_skills( &source_path, manifest_paths, @@ -1034,21 +1069,11 @@ impl PluginsManager { &skill_config_rules, ) .await; - let apps = load_apps_from_paths( - source_path.as_path(), - plugin_app_config_paths(source_path.as_path(), manifest_paths), - ) - .await; - let mcp_config_paths = plugin_mcp_config_paths(source_path.as_path(), manifest_paths); - let mut mcp_server_names = Vec::new(); - for mcp_config_path in mcp_config_paths { - mcp_server_names.extend( - load_mcp_servers_from_file(source_path.as_path(), &mcp_config_path) - .await - .mcp_servers - .into_keys(), - ); - } + let apps = load_plugin_apps(source_path.as_path()).await; + let mut mcp_server_names = load_plugin_mcp_servers(source_path.as_path()) + .await + .into_keys() + .collect::>(); mcp_server_names.sort_unstable(); mcp_server_names.dedup(); @@ -1075,6 +1100,57 @@ impl PluginsManager { ) { if config.features.enabled(Feature::Plugins) { self.start_curated_repo_sync(); + let should_spawn_marketplace_auto_upgrade = { + let mut state = match self.configured_marketplace_upgrade_state.write() { + Ok(state) => state, + Err(err) => err.into_inner(), + }; + if state.in_flight { + false + } else { + state.in_flight = true; + true + } + }; + if should_spawn_marketplace_auto_upgrade { + let manager = Arc::clone(self); + let config = config.clone(); + if let Err(err) = std::thread::Builder::new() + .name("plugins-marketplace-auto-upgrade".to_string()) + .spawn(move || { + let outcome = manager.upgrade_configured_marketplaces_for_config( + &config, /*marketplace_name*/ None, + ); + match outcome { + Ok(outcome) => { + for error in outcome.errors { + warn!( + marketplace = error.marketplace_name, + error = %error.message, + "failed to auto-upgrade configured marketplace" + ); + } + } + Err(err) => { + warn!("failed to auto-upgrade configured marketplaces: {err}"); + } + } + + let mut state = match manager.configured_marketplace_upgrade_state.write() { + Ok(state) => state, + Err(err) => err.into_inner(), + }; + state.in_flight = false; + }) + { + let mut state = match self.configured_marketplace_upgrade_state.write() { + Ok(state) => state, + Err(err) => err.into_inner(), + }; + state.in_flight = false; + warn!("failed to start configured marketplace auto-upgrade task: {err}"); + } + } start_startup_remote_plugin_sync_once( Arc::clone(self), self.codex_home.clone(), @@ -1099,9 +1175,66 @@ impl PluginsManager { } } - pub fn maybe_start_non_curated_plugin_cache_refresh_for_roots( + pub fn upgrade_configured_marketplaces_for_config( + &self, + config: &Config, + marketplace_name: Option<&str>, + ) -> Result { + if let Some(marketplace_name) = marketplace_name + && !configured_git_marketplace_names(&config.config_layer_stack) + .iter() + .any(|name| name == marketplace_name) + { + return Err(format!( + "marketplace `{marketplace_name}` is not configured as a Git marketplace" + )); + } + + let mut outcome = upgrade_configured_git_marketplaces( + self.codex_home.as_path(), + &config.config_layer_stack, + marketplace_name, + ); + if !outcome.upgraded_roots.is_empty() { + match refresh_non_curated_plugin_cache_force_reinstall( + self.codex_home.as_path(), + &outcome.upgraded_roots, + ) { + Ok(cache_refreshed) => { + if cache_refreshed { + self.clear_cache(); + } + } + Err(err) => { + self.clear_cache(); + outcome.errors.push(ConfiguredMarketplaceUpgradeError { + marketplace_name: marketplace_name + .unwrap_or("all configured marketplaces") + .to_string(), + message: format!( + "failed to refresh installed plugin cache after marketplace upgrade: {err}" + ), + }); + } + } + } + Ok(outcome) + } + + pub fn maybe_start_non_curated_plugin_cache_refresh( self: &Arc, roots: &[AbsolutePathBuf], + ) { + self.schedule_non_curated_plugin_cache_refresh( + roots, + NonCuratedCacheRefreshMode::IfVersionChanged, + ); + } + + fn schedule_non_curated_plugin_cache_refresh( + self: &Arc, + roots: &[AbsolutePathBuf], + mode: NonCuratedCacheRefreshMode, ) { let mut roots = roots.to_vec(); roots.sort_unstable(); @@ -1109,6 +1242,7 @@ impl PluginsManager { if roots.is_empty() { return; } + let request = NonCuratedCacheRefreshRequest { roots, mode }; let should_spawn = { let mut state = match self.non_curated_cache_refresh_state.write() { @@ -1116,13 +1250,25 @@ impl PluginsManager { Err(err) => err.into_inner(), }; // Collapse repeated plugin/list requests onto one worker and only queue another pass - // when the requested roots set actually changes. - if state.requested_roots.as_ref() == Some(&roots) - || (!state.in_flight && state.last_refreshed_roots.as_ref() == Some(&roots)) + // when the requested roots set actually changes. Forced reinstall requests are not + // deduped against the last completed pass because the same marketplace root path can + // point at newly activated files after an auto-upgrade. + if state.requested.as_ref() == Some(&request) + || (mode == NonCuratedCacheRefreshMode::IfVersionChanged + && !state.in_flight + && state.last_refreshed.as_ref() == Some(&request)) { return; } - state.requested_roots = Some(roots); + if mode == NonCuratedCacheRefreshMode::IfVersionChanged + && state.requested.as_ref().is_some_and(|requested| { + requested.mode == NonCuratedCacheRefreshMode::ForceReinstall + && requested.roots == request.roots + }) + { + return; + } + state.requested = Some(request); if state.in_flight { false } else { @@ -1144,7 +1290,7 @@ impl PluginsManager { Err(err) => err.into_inner(), }; state.in_flight = false; - state.requested_roots = None; + state.requested = None; warn!("failed to start non-curated plugin cache refresh task: {err}"); } } @@ -1160,13 +1306,8 @@ impl PluginsManager { .spawn( move || match sync_openai_plugins_repo(codex_home.as_path()) { Ok(curated_plugin_version) => { - let configured_curated_plugin_ids = curated_plugin_ids_from_config_keys( - configured_plugins_from_codex_home( - codex_home.as_path(), - "failed to read user config while refreshing curated plugin cache", - "failed to parse user config while refreshing curated plugin cache", - ), - ); + let configured_curated_plugin_ids = + configured_curated_plugin_ids_from_codex_home(codex_home.as_path()); match refresh_curated_plugin_cache( codex_home.as_path(), &curated_plugin_version, @@ -1198,15 +1339,15 @@ impl PluginsManager { fn run_non_curated_plugin_cache_refresh_loop(self: Arc) { loop { - let roots = { + let request = { let state = match self.non_curated_cache_refresh_state.read() { Ok(state) => state, Err(err) => err.into_inner(), }; - state.requested_roots.clone() + state.requested.clone() }; - let Some(roots) = roots else { + let Some(request) = request else { let mut state = match self.non_curated_cache_refresh_state.write() { Ok(state) => state, Err(err) => err.into_inner(), @@ -1215,30 +1356,40 @@ impl PluginsManager { return; }; - let refreshed = - match refresh_non_curated_plugin_cache(self.codex_home.as_path(), &roots) { - Ok(cache_refreshed) => { - if cache_refreshed { - self.clear_cache(); - } - true - } - Err(err) => { + let refresh_result = match request.mode { + NonCuratedCacheRefreshMode::IfVersionChanged => { + refresh_non_curated_plugin_cache(self.codex_home.as_path(), &request.roots) + } + NonCuratedCacheRefreshMode::ForceReinstall => { + refresh_non_curated_plugin_cache_force_reinstall( + self.codex_home.as_path(), + &request.roots, + ) + } + }; + let refreshed = match refresh_result { + Ok(cache_refreshed) => { + if cache_refreshed { self.clear_cache(); - warn!("failed to refresh non-curated plugin cache: {err}"); - false } - }; + true + } + Err(err) => { + self.clear_cache(); + warn!("failed to refresh non-curated plugin cache: {err}"); + false + } + }; let mut state = match self.non_curated_cache_refresh_state.write() { Ok(state) => state, Err(err) => err.into_inner(), }; if refreshed { - state.last_refreshed_roots = Some(roots.clone()); + state.last_refreshed = Some(request.clone()); } - if state.requested_roots.as_ref() == Some(&roots) { - state.requested_roots = None; + if state.requested.as_ref() == Some(&request) { + state.requested = None; state.in_flight = false; return; } @@ -1352,225 +1503,7 @@ impl PluginUninstallError { } } -fn log_plugin_load_errors(outcome: &PluginLoadOutcome) { - for plugin in outcome - .plugins() - .iter() - .filter(|plugin| plugin.error.is_some()) - { - if let Some(error) = plugin.error.as_deref() { - warn!( - plugin = plugin.config_name, - path = %plugin.root.display(), - "failed to load plugin: {error}" - ); - } - } -} - -#[derive(Debug, Default, Deserialize)] -#[serde(rename_all = "camelCase")] -struct PluginMcpFile { - #[serde(default)] - mcp_servers: HashMap, -} - -#[derive(Debug, Default, Deserialize)] -#[serde(rename_all = "camelCase")] -struct PluginAppFile { - #[serde(default)] - apps: HashMap, -} - -#[derive(Debug, Default, Deserialize)] -struct PluginAppConfig { - id: String, -} - -pub(crate) async fn load_plugins_from_layer_stack( - config_layer_stack: &ConfigLayerStack, - store: &PluginStore, - restriction_product: Option, -) -> PluginLoadOutcome { - let skill_config_rules = skill_config_rules_from_stack(config_layer_stack); - let mut configured_plugins: Vec<_> = configured_plugins_from_stack(config_layer_stack) - .into_iter() - .collect(); - configured_plugins.sort_unstable_by(|(a, _), (b, _)| a.cmp(b)); - - let mut plugins = Vec::with_capacity(configured_plugins.len()); - let mut seen_mcp_server_names = HashMap::::new(); - for (configured_name, plugin) in configured_plugins { - let loaded_plugin = load_plugin( - configured_name.clone(), - &plugin, - store, - restriction_product, - &skill_config_rules, - ) - .await; - for name in loaded_plugin.mcp_servers.keys() { - if let Some(previous_plugin) = - seen_mcp_server_names.insert(name.clone(), configured_name.clone()) - { - warn!( - plugin = configured_name, - previous_plugin, - server = name, - "skipping duplicate plugin MCP server name" - ); - } - } - plugins.push(loaded_plugin); - } - - PluginLoadOutcome::from_plugins(plugins) -} - -fn refresh_curated_plugin_cache( - codex_home: &Path, - plugin_version: &str, - configured_curated_plugin_ids: &[PluginId], -) -> Result { - let store = PluginStore::new(codex_home.to_path_buf()); - let curated_marketplace_path = AbsolutePathBuf::try_from( - curated_plugins_repo_path(codex_home).join(".agents/plugins/marketplace.json"), - ) - .map_err(|_| "local curated marketplace is not available".to_string())?; - let curated_marketplace = load_marketplace(&curated_marketplace_path) - .map_err(|err| format!("failed to load curated marketplace for cache refresh: {err}"))?; - - let mut plugin_sources = HashMap::::new(); - for plugin in curated_marketplace.plugins { - let plugin_name = plugin.name; - if plugin_sources.contains_key(&plugin_name) { - warn!( - plugin = plugin_name, - marketplace = OPENAI_CURATED_MARKETPLACE_NAME, - "ignoring duplicate curated plugin entry during cache refresh" - ); - continue; - } - let source_path = match plugin.source { - MarketplacePluginSource::Local { path } => path, - }; - plugin_sources.insert(plugin_name, source_path); - } - - let mut cache_refreshed = false; - for plugin_id in configured_curated_plugin_ids { - if store.active_plugin_version(plugin_id).as_deref() == Some(plugin_version) { - continue; - } - - let Some(source_path) = plugin_sources.get(&plugin_id.plugin_name).cloned() else { - warn!( - plugin = plugin_id.plugin_name, - marketplace = OPENAI_CURATED_MARKETPLACE_NAME, - "configured curated plugin no longer exists in curated marketplace during cache refresh" - ); - continue; - }; - - store - .install_with_version(source_path, plugin_id.clone(), plugin_version.to_string()) - .map_err(|err| { - format!( - "failed to refresh curated plugin cache for {}: {err}", - plugin_id.as_key() - ) - })?; - cache_refreshed = true; - } - - Ok(cache_refreshed) -} - -fn refresh_non_curated_plugin_cache( - codex_home: &Path, - additional_roots: &[AbsolutePathBuf], -) -> Result { - let configured_non_curated_plugin_ids = - non_curated_plugin_ids_from_config_keys(configured_plugins_from_codex_home( - codex_home, - "failed to read user config while refreshing non-curated plugin cache", - "failed to parse user config while refreshing non-curated plugin cache", - )); - if configured_non_curated_plugin_ids.is_empty() { - return Ok(false); - } - let configured_non_curated_plugin_keys = configured_non_curated_plugin_ids - .iter() - .map(PluginId::as_key) - .collect::>(); - - let store = PluginStore::new(codex_home.to_path_buf()); - let marketplace_outcome = list_marketplaces(additional_roots) - .map_err(|err| format!("failed to discover marketplaces for cache refresh: {err}"))?; - let mut plugin_sources = HashMap::::new(); - - for marketplace in marketplace_outcome.marketplaces { - if marketplace.name == OPENAI_CURATED_MARKETPLACE_NAME { - continue; - } - - for plugin in marketplace.plugins { - let plugin_id = - PluginId::new(plugin.name.clone(), marketplace.name.clone()).map_err(|err| { - match err { - PluginIdError::Invalid(message) => { - format!("failed to prepare non-curated plugin cache refresh: {message}") - } - } - })?; - let plugin_key = plugin_id.as_key(); - if !configured_non_curated_plugin_keys.contains(&plugin_key) { - continue; - } - if plugin_sources.contains_key(&plugin_key) { - warn!( - plugin = plugin.name, - marketplace = marketplace.name, - "ignoring duplicate non-curated plugin entry during cache refresh" - ); - continue; - } - - let source_path = match plugin.source { - MarketplacePluginSource::Local { path } => path, - }; - let plugin_version = plugin_version_for_source(source_path.as_path()) - .map_err(|err| format!("failed to read plugin version for {plugin_key}: {err}"))?; - plugin_sources.insert(plugin_key, (source_path, plugin_version)); - } - } - - let mut cache_refreshed = false; - for plugin_id in configured_non_curated_plugin_ids { - let plugin_key = plugin_id.as_key(); - let Some((source_path, plugin_version)) = plugin_sources.get(&plugin_key).cloned() else { - warn!( - plugin = plugin_id.plugin_name, - marketplace = plugin_id.marketplace_name, - "configured non-curated plugin no longer exists in discovered marketplaces during cache refresh" - ); - continue; - }; - - if store.active_plugin_version(&plugin_id).as_deref() == Some(plugin_version.as_str()) { - continue; - } - - store - .install_with_version(source_path, plugin_id.clone(), plugin_version) - .map_err(|err| format!("failed to refresh plugin cache for {plugin_key}: {err}"))?; - cache_refreshed = true; - } - - Ok(cache_refreshed) -} - -fn configured_plugins_from_stack( +pub(crate) fn configured_plugins_from_stack( config_layer_stack: &ConfigLayerStack, ) -> HashMap { // Plugin entries remain persisted user config only. @@ -1595,523 +1528,6 @@ fn configured_plugins_from_user_config_value( } } -fn configured_plugins_from_codex_home( - codex_home: &Path, - read_error_message: &str, - parse_error_message: &str, -) -> HashMap { - let config_path = codex_home.join(CONFIG_TOML_FILE); - let user_config = match fs::read_to_string(&config_path) { - Ok(user_config) => user_config, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => return HashMap::new(), - Err(err) => { - warn!( - path = %config_path.display(), - error = %err, - "{read_error_message}" - ); - return HashMap::new(); - } - }; - - let user_config = match toml::from_str::(&user_config) { - Ok(user_config) => user_config, - Err(err) => { - warn!( - path = %config_path.display(), - error = %err, - "{parse_error_message}" - ); - return HashMap::new(); - } - }; - - configured_plugins_from_user_config_value(&user_config) -} - -fn configured_plugin_ids( - configured_plugins: HashMap, - invalid_plugin_key_message: &str, -) -> Vec { - configured_plugins - .into_keys() - .filter_map(|plugin_key| match PluginId::parse(&plugin_key) { - Ok(plugin_id) => Some(plugin_id), - Err(err) => { - warn!( - plugin_key, - error = %err, - "{invalid_plugin_key_message}" - ); - None - } - }) - .collect() -} - -fn curated_plugin_ids_from_config_keys( - configured_plugins: HashMap, -) -> Vec { - let mut configured_curated_plugin_ids = configured_plugin_ids( - configured_plugins, - "ignoring invalid configured plugin key during curated sync setup", - ) - .into_iter() - .filter(|plugin_id| plugin_id.marketplace_name == OPENAI_CURATED_MARKETPLACE_NAME) - .collect::>(); - configured_curated_plugin_ids.sort_unstable_by_key(PluginId::as_key); - configured_curated_plugin_ids -} - -fn non_curated_plugin_ids_from_config_keys( - configured_plugins: HashMap, -) -> Vec { - let mut configured_non_curated_plugin_ids = configured_plugin_ids( - configured_plugins, - "ignoring invalid plugin key during non-curated cache refresh setup", - ) - .into_iter() - .filter(|plugin_id| plugin_id.marketplace_name != OPENAI_CURATED_MARKETPLACE_NAME) - .collect::>(); - configured_non_curated_plugin_ids.sort_unstable_by_key(PluginId::as_key); - configured_non_curated_plugin_ids -} - -async fn load_plugin( - config_name: String, - plugin: &PluginConfig, - store: &PluginStore, - restriction_product: Option, - skill_config_rules: &SkillConfigRules, -) -> LoadedPlugin { - let plugin_id = PluginId::parse(&config_name); - let active_plugin_root = plugin_id - .as_ref() - .ok() - .and_then(|plugin_id| store.active_plugin_root(plugin_id)); - let root = active_plugin_root - .clone() - .unwrap_or_else(|| match &plugin_id { - Ok(plugin_id) => store.plugin_base_root(plugin_id), - Err(_) => store.root().clone(), - }); - let mut loaded_plugin = LoadedPlugin { - config_name, - manifest_name: None, - manifest_description: None, - root, - enabled: plugin.enabled, - skill_roots: Vec::new(), - disabled_skill_paths: HashSet::new(), - has_enabled_skills: false, - mcp_servers: HashMap::new(), - apps: Vec::new(), - error: None, - }; - - if !plugin.enabled { - return loaded_plugin; - } - - let plugin_root = match plugin_id { - Ok(_) => match active_plugin_root { - Some(plugin_root) => plugin_root, - None => { - loaded_plugin.error = Some("plugin is not installed".to_string()); - return loaded_plugin; - } - }, - Err(err) => { - loaded_plugin.error = Some(err.to_string()); - return loaded_plugin; - } - }; - - if !plugin_root.as_path().is_dir() { - loaded_plugin.error = Some("path does not exist or is not a directory".to_string()); - return loaded_plugin; - } - - let Some(manifest) = load_plugin_manifest(plugin_root.as_path()) else { - loaded_plugin.error = Some("missing or invalid .codex-plugin/plugin.json".to_string()); - return loaded_plugin; - }; - - let manifest_paths = &manifest.paths; - loaded_plugin.manifest_name = manifest - .interface - .as_ref() - .and_then(|interface| interface.display_name.as_deref()) - .map(str::trim) - .filter(|display_name| !display_name.is_empty()) - .map(str::to_string) - .or_else(|| Some(manifest.name.clone())); - loaded_plugin.manifest_description = manifest.description.clone(); - loaded_plugin.skill_roots = plugin_skill_roots(&plugin_root, manifest_paths); - let resolved_skills = load_plugin_skills( - &plugin_root, - manifest_paths, - restriction_product, - skill_config_rules, - ) - .await; - let has_enabled_skills = resolved_skills.has_enabled_skills(); - loaded_plugin.disabled_skill_paths = resolved_skills.disabled_skill_paths; - loaded_plugin.has_enabled_skills = has_enabled_skills; - let mut mcp_servers = HashMap::new(); - for mcp_config_path in plugin_mcp_config_paths(plugin_root.as_path(), manifest_paths) { - let plugin_mcp = load_mcp_servers_from_file(plugin_root.as_path(), &mcp_config_path).await; - for (name, config) in plugin_mcp.mcp_servers { - if mcp_servers.insert(name.clone(), config).is_some() { - warn!( - plugin = %plugin_root.display(), - path = %mcp_config_path.display(), - server = name, - "plugin MCP file overwrote an earlier server definition" - ); - } - } - } - loaded_plugin.mcp_servers = mcp_servers; - loaded_plugin.apps = load_apps_from_paths( - plugin_root.as_path(), - plugin_app_config_paths(plugin_root.as_path(), manifest_paths), - ) - .await; - loaded_plugin -} - -struct ResolvedPluginSkills { - skills: Vec, - disabled_skill_paths: HashSet, - had_errors: bool, -} - -impl ResolvedPluginSkills { - fn has_enabled_skills(&self) -> bool { - // Keep the plugin visible in capability summaries if skill loading was partial. - self.had_errors - || self - .skills - .iter() - .any(|skill| !self.disabled_skill_paths.contains(&skill.path_to_skills_md)) - } -} - -async fn load_plugin_skills( - plugin_root: &AbsolutePathBuf, - manifest_paths: &PluginManifestPaths, - restriction_product: Option, - skill_config_rules: &SkillConfigRules, -) -> ResolvedPluginSkills { - let roots = plugin_skill_roots(plugin_root, manifest_paths) - .into_iter() - .map(|path| SkillRoot { - path, - scope: SkillScope::User, - file_system: Arc::clone(&LOCAL_FS), - }) - .collect::>(); - let outcome = load_skills_from_roots(roots).await; - let had_errors = !outcome.errors.is_empty(); - let skills = outcome - .skills - .into_iter() - .filter(|skill| skill.matches_product_restriction_for_product(restriction_product)) - .collect::>(); - let disabled_skill_paths = resolve_disabled_skill_paths(&skills, skill_config_rules); - - ResolvedPluginSkills { - skills, - disabled_skill_paths, - had_errors, - } -} - -fn plugin_skill_roots( - plugin_root: &AbsolutePathBuf, - manifest_paths: &PluginManifestPaths, -) -> Vec { - let mut paths = default_skill_roots(plugin_root); - if let Some(path) = &manifest_paths.skills { - paths.push(path.clone()); - } - paths.sort_unstable(); - paths.dedup(); - paths -} - -fn default_skill_roots(plugin_root: &AbsolutePathBuf) -> Vec { - let skills_dir = plugin_root.join(DEFAULT_SKILLS_DIR_NAME); - if skills_dir.is_dir() { - vec![skills_dir] - } else { - Vec::new() - } -} - -fn plugin_mcp_config_paths( - plugin_root: &Path, - manifest_paths: &PluginManifestPaths, -) -> Vec { - if let Some(path) = &manifest_paths.mcp_servers { - return vec![path.clone()]; - } - default_mcp_config_paths(plugin_root) -} - -fn default_mcp_config_paths(plugin_root: &Path) -> Vec { - let mut paths = Vec::new(); - let default_path = plugin_root.join(DEFAULT_MCP_CONFIG_FILE); - if default_path.is_file() - && let Ok(default_path) = AbsolutePathBuf::try_from(default_path) - { - paths.push(default_path); - } - paths.sort_unstable(); - paths.dedup(); - paths -} - -pub async fn load_plugin_apps(plugin_root: &Path) -> Vec { - if let Some(manifest) = load_plugin_manifest(plugin_root) { - return load_apps_from_paths( - plugin_root, - plugin_app_config_paths(plugin_root, &manifest.paths), - ) - .await; - } - load_apps_from_paths(plugin_root, default_app_config_paths(plugin_root)).await -} - -fn plugin_app_config_paths( - plugin_root: &Path, - manifest_paths: &PluginManifestPaths, -) -> Vec { - if let Some(path) = &manifest_paths.apps { - return vec![path.clone()]; - } - default_app_config_paths(plugin_root) -} - -fn default_app_config_paths(plugin_root: &Path) -> Vec { - let mut paths = Vec::new(); - let default_path = plugin_root.join(DEFAULT_APP_CONFIG_FILE); - if default_path.is_file() - && let Ok(default_path) = AbsolutePathBuf::try_from(default_path) - { - paths.push(default_path); - } - paths.sort_unstable(); - paths.dedup(); - paths -} - -async fn load_apps_from_paths( - plugin_root: &Path, - app_config_paths: Vec, -) -> Vec { - let mut connector_ids = Vec::new(); - for app_config_path in app_config_paths { - let Ok(contents) = tokio::fs::read_to_string(app_config_path.as_path()).await else { - continue; - }; - let parsed = match serde_json::from_str::(&contents) { - Ok(parsed) => parsed, - Err(err) => { - warn!( - path = %app_config_path.display(), - "failed to parse plugin app config: {err}" - ); - continue; - } - }; - - let mut apps: Vec = parsed.apps.into_values().collect(); - apps.sort_unstable_by(|left, right| left.id.cmp(&right.id)); - - connector_ids.extend(apps.into_iter().filter_map(|app| { - if app.id.trim().is_empty() { - warn!( - plugin = %plugin_root.display(), - "plugin app config is missing an app id" - ); - None - } else { - Some(AppConnectorId(app.id)) - } - })); - } - connector_ids.dedup(); - connector_ids -} - -pub async fn plugin_telemetry_metadata_from_root( - plugin_id: &PluginId, - plugin_root: &AbsolutePathBuf, -) -> PluginTelemetryMetadata { - let Some(manifest) = load_plugin_manifest(plugin_root.as_path()) else { - return PluginTelemetryMetadata::from_plugin_id(plugin_id); - }; - - let manifest_paths = &manifest.paths; - let has_skills = !plugin_skill_roots(plugin_root, manifest_paths).is_empty(); - let mut mcp_server_names = Vec::new(); - for path in plugin_mcp_config_paths(plugin_root.as_path(), manifest_paths) { - mcp_server_names.extend( - load_mcp_servers_from_file(plugin_root.as_path(), &path) - .await - .mcp_servers - .into_keys(), - ); - } - mcp_server_names.sort_unstable(); - mcp_server_names.dedup(); - - PluginTelemetryMetadata { - plugin_id: plugin_id.clone(), - capability_summary: Some(PluginCapabilitySummary { - config_name: plugin_id.as_key(), - display_name: plugin_id.plugin_name.clone(), - description: None, - has_skills, - mcp_server_names, - app_connector_ids: load_apps_from_paths( - plugin_root.as_path(), - plugin_app_config_paths(plugin_root.as_path(), manifest_paths), - ) - .await, - }), - } -} - -pub async fn load_plugin_mcp_servers(plugin_root: &Path) -> HashMap { - let Some(manifest) = load_plugin_manifest(plugin_root) else { - return HashMap::new(); - }; - - let mut mcp_servers = HashMap::new(); - for mcp_config_path in plugin_mcp_config_paths(plugin_root, &manifest.paths) { - let plugin_mcp = load_mcp_servers_from_file(plugin_root, &mcp_config_path).await; - for (name, config) in plugin_mcp.mcp_servers { - mcp_servers.entry(name).or_insert(config); - } - } - - mcp_servers -} - -pub async fn installed_plugin_telemetry_metadata( - codex_home: &Path, - plugin_id: &PluginId, -) -> PluginTelemetryMetadata { - let store = PluginStore::new(codex_home.to_path_buf()); - let Some(plugin_root) = store.active_plugin_root(plugin_id) else { - return PluginTelemetryMetadata::from_plugin_id(plugin_id); - }; - - plugin_telemetry_metadata_from_root(plugin_id, &plugin_root).await -} - -async fn load_mcp_servers_from_file( - plugin_root: &Path, - mcp_config_path: &AbsolutePathBuf, -) -> PluginMcpDiscovery { - let Ok(contents) = tokio::fs::read_to_string(mcp_config_path.as_path()).await else { - return PluginMcpDiscovery::default(); - }; - let parsed = match serde_json::from_str::(&contents) { - Ok(parsed) => parsed, - Err(err) => { - warn!( - path = %mcp_config_path.display(), - "failed to parse plugin MCP config: {err}" - ); - return PluginMcpDiscovery::default(); - } - }; - normalize_plugin_mcp_servers( - plugin_root, - parsed.mcp_servers, - mcp_config_path.to_string_lossy().as_ref(), - ) -} - -fn normalize_plugin_mcp_servers( - plugin_root: &Path, - plugin_mcp_servers: HashMap, - source: &str, -) -> PluginMcpDiscovery { - let mut mcp_servers = HashMap::new(); - - for (name, config_value) in plugin_mcp_servers { - let normalized = normalize_plugin_mcp_server_value(plugin_root, config_value); - match serde_json::from_value::(JsonValue::Object(normalized)) { - Ok(config) => { - mcp_servers.insert(name, config); - } - Err(err) => { - warn!( - plugin = %plugin_root.display(), - server = name, - "failed to parse plugin MCP server from {source}: {err}" - ); - } - } - } - - PluginMcpDiscovery { mcp_servers } -} - -fn normalize_plugin_mcp_server_value( - plugin_root: &Path, - value: JsonValue, -) -> JsonMap { - let mut object = match value { - JsonValue::Object(object) => object, - _ => return JsonMap::new(), - }; - - if let Some(JsonValue::String(transport_type)) = object.remove("type") { - match transport_type.as_str() { - "http" | "streamable_http" | "streamable-http" => {} - "stdio" => {} - other => { - warn!( - plugin = %plugin_root.display(), - transport = other, - "plugin MCP server uses an unknown transport type" - ); - } - } - } - - if let Some(JsonValue::Object(oauth)) = object.remove("oauth") - && oauth.contains_key("callbackPort") - { - warn!( - plugin = %plugin_root.display(), - "plugin MCP server OAuth callbackPort is ignored; Codex uses global MCP OAuth callback settings" - ); - } - - if let Some(JsonValue::String(cwd)) = object.get("cwd") - && !Path::new(cwd).is_absolute() - { - object.insert( - "cwd".to_string(), - JsonValue::String(plugin_root.join(cwd).display().to_string()), - ); - } - - object -} - -#[derive(Debug, Default)] -struct PluginMcpDiscovery { - mcp_servers: HashMap, -} - #[cfg(test)] #[path = "manager_tests.rs"] mod tests; diff --git a/codex-rs/core/src/plugins/manager_tests.rs b/codex-rs/core/src/plugins/manager_tests.rs index 23b202f1a8..2b2f486586 100644 --- a/codex-rs/core/src/plugins/manager_tests.rs +++ b/codex-rs/core/src/plugins/manager_tests.rs @@ -6,7 +6,6 @@ use crate::config_loader::ConfigLayerStack; use crate::config_loader::ConfigRequirements; use crate::config_loader::ConfigRequirementsToml; use crate::plugins::LoadedPlugin; -use crate::plugins::MarketplacePluginInstallPolicy; use crate::plugins::PluginLoadOutcome; use crate::plugins::marketplace_install_root; use crate::plugins::test_support::TEST_CURATED_PLUGIN_SHA; @@ -14,12 +13,17 @@ use crate::plugins::test_support::write_curated_plugin_sha_with as write_curated use crate::plugins::test_support::write_file; use crate::plugins::test_support::write_openai_curated_marketplace; use codex_app_server_protocol::ConfigLayerSource; +use codex_config::McpServerConfig; use codex_config::types::McpServerTransportConfig; +use codex_core_plugins::loader::refresh_non_curated_plugin_cache; +use codex_core_plugins::loader::refresh_non_curated_plugin_cache_force_reinstall; +use codex_core_plugins::marketplace::MarketplacePluginInstallPolicy; use codex_login::CodexAuth; use codex_protocol::protocol::Product; use codex_utils_absolute_path::test_support::PathBufExt; use pretty_assertions::assert_eq; use std::fs; +use std::path::Path; use tempfile::TempDir; use toml::Value; use wiremock::Mock; @@ -172,12 +176,14 @@ async fn load_plugins_loads_default_skills_and_mcp_servers() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -508,12 +514,14 @@ async fn load_plugins_uses_manifest_configured_component_paths() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -617,12 +625,14 @@ async fn load_plugins_ignores_manifest_component_paths_without_dot_slash() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -774,12 +784,14 @@ fn capability_index_filters_inactive_and_zero_capability_plugins() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2477,14 +2489,10 @@ enabled = true ); assert_eq!( - curated_plugin_ids_from_config_keys(configured_plugins_from_codex_home( - tmp.path(), - "failed to read user config while refreshing curated plugin cache", - "failed to parse user config while refreshing curated plugin cache", - )) - .into_iter() - .map(|plugin_id| plugin_id.as_key()) - .collect::>(), + configured_curated_plugin_ids_from_codex_home(tmp.path()) + .into_iter() + .map(|plugin_id| plugin_id.as_key()) + .collect::>(), vec!["slack@openai-curated".to_string()] ); @@ -2496,11 +2504,7 @@ plugins = true ); assert_eq!( - curated_plugin_ids_from_config_keys(configured_plugins_from_codex_home( - tmp.path(), - "failed to read user config while refreshing curated plugin cache", - "failed to parse user config while refreshing curated plugin cache", - )), + configured_curated_plugin_ids_from_codex_home(tmp.path()), Vec::::new() ); } @@ -2678,6 +2682,68 @@ enabled = true ); } +#[test] +fn refresh_non_curated_plugin_cache_force_reinstalls_current_local_version() { + let tmp = tempfile::tempdir().unwrap(); + let repo_root = tmp.path().join("repo"); + fs::create_dir_all(repo_root.join(".git")).unwrap(); + fs::create_dir_all(repo_root.join(".agents/plugins")).unwrap(); + write_plugin(&repo_root, "sample-plugin", "sample-plugin"); + fs::write(repo_root.join("sample-plugin/skills/SKILL.md"), "new skill").unwrap(); + write_file( + &repo_root.join(".agents/plugins/marketplace.json"), + r#"{ + "name": "debug", + "plugins": [ + { + "name": "sample-plugin", + "source": { + "source": "local", + "path": "./sample-plugin" + } + } + ] +}"#, + ); + write_plugin( + &tmp.path().join("plugins/cache/debug"), + "sample-plugin/local", + "sample-plugin", + ); + fs::write( + tmp.path() + .join("plugins/cache/debug/sample-plugin/local/skills/SKILL.md"), + "old skill", + ) + .unwrap(); + write_file( + &tmp.path().join(CONFIG_TOML_FILE), + r#"[features] +plugins = true + +[plugins."sample-plugin@debug"] +enabled = true +"#, + ); + + assert!( + refresh_non_curated_plugin_cache_force_reinstall( + tmp.path(), + &[AbsolutePathBuf::try_from(repo_root).unwrap()], + ) + .expect("cache refresh should reinstall unchanged local version") + ); + + assert_eq!( + fs::read_to_string( + tmp.path() + .join("plugins/cache/debug/sample-plugin/local/skills/SKILL.md") + ) + .unwrap(), + "new skill" + ); +} + #[test] fn refresh_non_curated_plugin_cache_ignores_invalid_unconfigured_plugin_versions() { let tmp = tempfile::tempdir().unwrap(); diff --git a/codex-rs/core/src/plugins/marketplace_add.rs b/codex-rs/core/src/plugins/marketplace_add.rs index 49c1c2e565..38fc04c199 100644 --- a/codex-rs/core/src/plugins/marketplace_add.rs +++ b/codex-rs/core/src/plugins/marketplace_add.rs @@ -20,7 +20,7 @@ use metadata::find_marketplace_root_by_name; use metadata::installed_marketplace_root_for_source; use metadata::record_added_marketplace_entry; use source::MarketplaceSource; -use source::parse_marketplace_source; +pub(crate) use source::parse_marketplace_source; use source::stage_marketplace_source; use source::validate_marketplace_source_root; @@ -275,9 +275,15 @@ mod tests { ); let config = fs::read_to_string(codex_home.path().join(codex_config::CONFIG_TOML_FILE))?; - assert!(config.contains("[marketplaces.debug]")); - assert!(config.contains("source_type = \"local\"")); - assert!(config.contains(&format!("source = \"{expected_source}\""))); + let config: toml::Value = toml::from_str(&config)?; + assert_eq!( + config["marketplaces"]["debug"]["source_type"].as_str(), + Some("local") + ); + assert_eq!( + config["marketplaces"]["debug"]["source"].as_str(), + Some(expected_source.as_str()) + ); Ok(()) } diff --git a/codex-rs/core/src/plugins/marketplace_add/metadata.rs b/codex-rs/core/src/plugins/marketplace_add/metadata.rs index ce271b116d..ccded11c87 100644 --- a/codex-rs/core/src/plugins/marketplace_add/metadata.rs +++ b/codex-rs/core/src/plugins/marketplace_add/metadata.rs @@ -1,10 +1,10 @@ use super::MarketplaceAddError; use super::MarketplaceSource; use crate::plugins::installed_marketplaces::resolve_configured_marketplace_root; -use crate::plugins::validate_marketplace_root; use codex_config::CONFIG_TOML_FILE; use codex_config::MarketplaceConfigUpdate; use codex_config::record_user_marketplace; +use codex_core_plugins::marketplace::validate_marketplace_root; use std::fs; use std::io::ErrorKind; use std::path::Path; @@ -38,6 +38,7 @@ pub(super) fn record_added_marketplace_entry( let timestamp = utc_timestamp_now()?; let update = MarketplaceConfigUpdate { last_updated: ×tamp, + last_revision: None, source_type: install_metadata.config_source_type(), source: &source, ref_name: install_metadata.ref_name(), @@ -296,19 +297,11 @@ mod tests { r#"{"name":"debug","plugins":[]}"#, ) .unwrap(); - fs::write( - codex_home.path().join(CONFIG_TOML_FILE), - format!( - "[marketplaces.debug]\nsource_type = \"local\"\nsource = \"{}\"\n", - source_root.display() - ), - ) - .unwrap(); - let source = MarketplaceSource::Local { path: source_root.clone(), }; let install_metadata = MarketplaceInstallMetadata::from_source(&source, &[]); + record_added_marketplace_entry(codex_home.path(), "debug", &install_metadata).unwrap(); let root = installed_marketplace_root_for_source( codex_home.path(), diff --git a/codex-rs/core/src/plugins/marketplace_add/source.rs b/codex-rs/core/src/plugins/marketplace_add/source.rs index 19fec57ade..e723c8865a 100644 --- a/codex-rs/core/src/plugins/marketplace_add/source.rs +++ b/codex-rs/core/src/plugins/marketplace_add/source.rs @@ -1,11 +1,11 @@ use super::MarketplaceAddError; -use crate::plugins::validate_marketplace_root; use crate::plugins::validate_plugin_segment; +use codex_core_plugins::marketplace::validate_marketplace_root; use std::path::Path; use std::path::PathBuf; #[derive(Debug, Clone, PartialEq, Eq)] -pub(super) enum MarketplaceSource { +pub(crate) enum MarketplaceSource { Git { url: String, ref_name: Option, @@ -15,7 +15,7 @@ pub(super) enum MarketplaceSource { }, } -pub(super) fn parse_marketplace_source( +pub(crate) fn parse_marketplace_source( source: &str, explicit_ref: Option, ) -> Result { @@ -124,14 +124,26 @@ fn normalize_git_url(url: &str) -> String { } fn looks_like_local_path(source: &str) -> bool { - source.starts_with("./") + Path::new(source).is_absolute() + || looks_like_windows_absolute_path(source) + || source.starts_with("./") + || source.starts_with(".\\") || source.starts_with("../") - || source.starts_with('/') + || source.starts_with("..\\") || source.starts_with("~/") || source == "." || source == ".." } +fn looks_like_windows_absolute_path(source: &str) -> bool { + let bytes = source.as_bytes(); + bytes.len() >= 3 + && bytes[0].is_ascii_alphabetic() + && bytes[1] == b':' + && matches!(bytes[2], b'\\' | b'/') + || source.starts_with(r"\\") +} + fn resolve_local_source_path(source: &str) -> Result { let path = expand_tilde_path(source); let path = if path.is_absolute() { @@ -310,6 +322,14 @@ mod tests { assert!(path.is_absolute()); } + #[test] + fn windows_absolute_paths_look_like_local_paths_on_every_host() { + assert!(looks_like_local_path(r"C:\Users\alice\marketplace")); + assert!(looks_like_local_path("C:/Users/alice/marketplace")); + assert!(looks_like_local_path(r"\\server\share\marketplace")); + assert!(!looks_like_local_path(r"C:relative\path")); + } + #[test] fn local_file_source_is_rejected() { let tempdir = TempDir::new().unwrap(); diff --git a/codex-rs/core/src/plugins/mentions.rs b/codex-rs/core/src/plugins/mentions.rs index 5e11b9b91e..a5e94345cb 100644 --- a/codex-rs/core/src/plugins/mentions.rs +++ b/codex-rs/core/src/plugins/mentions.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use std::collections::HashSet; +use codex_connectors::metadata::connector_mention_slug; use codex_protocol::user_input::UserInput; use crate::connectors; @@ -108,7 +109,7 @@ pub(crate) fn build_connector_slug_counts( ) -> HashMap { let mut counts: HashMap = HashMap::new(); for connector in connectors { - let slug = connectors::connector_mention_slug(connector); + let slug = connector_mention_slug(connector); *counts.entry(slug).or_insert(0) += 1; } counts diff --git a/codex-rs/core/src/plugins/mod.rs b/codex-rs/core/src/plugins/mod.rs index 7ffb80b449..16cc8d08c4 100644 --- a/codex-rs/core/src/plugins/mod.rs +++ b/codex-rs/core/src/plugins/mod.rs @@ -4,18 +4,15 @@ mod discoverable; mod injection; mod installed_marketplaces; mod manager; -mod manifest; -mod marketplace; mod marketplace_add; mod mentions; -mod remote; mod render; mod startup_sync; -mod store; #[cfg(test)] pub(crate) mod test_support; -mod toggles; +pub use codex_core_plugins::marketplace_upgrade::ConfiguredMarketplaceUpgradeError as PluginMarketplaceUpgradeError; +pub use codex_core_plugins::marketplace_upgrade::ConfiguredMarketplaceUpgradeOutcome as PluginMarketplaceUpgradeOutcome; pub use codex_plugin::AppConnectorId; pub use codex_plugin::EffectiveSkillRoots; pub use codex_plugin::PluginCapabilitySummary; @@ -27,6 +24,7 @@ pub use codex_plugin::validate_plugin_segment; pub type LoadedPlugin = codex_plugin::LoadedPlugin; pub type PluginLoadOutcome = codex_plugin::PluginLoadOutcome; +pub(crate) use codex_core_plugins::marketplace::find_marketplace_manifest_path; pub(crate) use discoverable::list_tool_suggest_discoverable_plugins; pub(crate) use injection::build_plugin_injections; pub use installed_marketplaces::INSTALLED_MARKETPLACES_DIR; @@ -45,32 +43,17 @@ pub use manager::PluginRemoteSyncError; pub use manager::PluginUninstallError; pub use manager::PluginsManager; pub use manager::RemotePluginSyncResult; -pub use manager::installed_plugin_telemetry_metadata; -pub use manager::load_plugin_apps; -pub use manager::load_plugin_mcp_servers; -pub use manager::plugin_telemetry_metadata_from_root; -pub use manifest::PluginManifestInterface; -pub(crate) use manifest::PluginManifestPaths; -pub(crate) use manifest::load_plugin_manifest; -pub use marketplace::MarketplaceError; -pub use marketplace::MarketplaceListError; -pub use marketplace::MarketplacePluginAuthPolicy; -pub use marketplace::MarketplacePluginInstallPolicy; -pub use marketplace::MarketplacePluginPolicy; -pub use marketplace::MarketplacePluginSource; -pub use marketplace::validate_marketplace_root; +pub(crate) use manager::configured_plugins_from_stack; pub use marketplace_add::MarketplaceAddError; pub use marketplace_add::MarketplaceAddOutcome; pub use marketplace_add::MarketplaceAddRequest; pub use marketplace_add::add_marketplace; -pub use remote::RemotePluginFetchError; -pub use remote::fetch_remote_featured_plugin_ids; +pub(crate) use marketplace_add::parse_marketplace_source; pub(crate) use render::render_explicit_plugin_instructions; pub(crate) use render::render_plugins_section; pub(crate) use startup_sync::curated_plugins_repo_path; pub(crate) use startup_sync::read_curated_plugins_sha; pub(crate) use startup_sync::sync_openai_plugins_repo; -pub use toggles::collect_plugin_enabled_candidates; pub(crate) use mentions::build_connector_slug_counts; pub(crate) use mentions::build_skill_name_counts; diff --git a/codex-rs/core/src/project_doc.rs b/codex-rs/core/src/project_doc.rs deleted file mode 100644 index 6659dfea7f..0000000000 --- a/codex-rs/core/src/project_doc.rs +++ /dev/null @@ -1,325 +0,0 @@ -//! Project-level documentation discovery. -//! -//! Project-level documentation is primarily stored in files named `AGENTS.md`. -//! Additional fallback filenames can be configured via `project_doc_fallback_filenames`. -//! We include the concatenation of all files found along the path from the -//! project root to the current working directory as follows: -//! -//! 1. Determine the project root by walking upwards from the current working -//! directory until a configured `project_root_markers` entry is found. -//! When `project_root_markers` is unset, the default marker list is used -//! (`.git`). If no marker is found, only the current working directory is -//! considered. An empty marker list disables parent traversal. -//! 2. Collect every `AGENTS.md` found from the project root down to the -//! current working directory (inclusive) and concatenate their contents in -//! that order. -//! 3. We do **not** walk past the project root. - -use crate::config::Config; -use crate::config_loader::ConfigLayerStackOrdering; -use crate::config_loader::default_project_root_markers; -use crate::config_loader::merge_toml_values; -use crate::config_loader::project_root_markers_from_config; -use codex_app_server_protocol::ConfigLayerSource; -use codex_exec_server::Environment; -use codex_exec_server::ExecutorFileSystem; -use codex_features::Feature; -use codex_utils_absolute_path::AbsolutePathBuf; -use dunce::canonicalize as normalize_path; -use std::io; -use toml::Value as TomlValue; -use tracing::error; - -pub(crate) const HIERARCHICAL_AGENTS_MESSAGE: &str = - include_str!("../hierarchical_agents_message.md"); - -/// Default filename scanned for project-level docs. -pub const DEFAULT_PROJECT_DOC_FILENAME: &str = "AGENTS.md"; -/// Preferred local override for project-level docs. -pub const LOCAL_PROJECT_DOC_FILENAME: &str = "AGENTS.override.md"; - -/// When both `Config::instructions` and the project doc are present, they will -/// be concatenated with the following separator. -const PROJECT_DOC_SEPARATOR: &str = "\n\n--- project-doc ---\n\n"; - -fn render_js_repl_instructions(config: &Config) -> Option { - if !config.features.enabled(Feature::JsRepl) { - return None; - } - - let mut section = String::from("## JavaScript REPL (Node)\n"); - section.push_str( - "- Use `js_repl` for Node-backed JavaScript with top-level await in a persistent kernel.\n", - ); - section.push_str("- `js_repl` is a freeform/custom tool. Direct `js_repl` calls must send raw JavaScript tool input (optionally with first-line `// codex-js-repl: timeout_ms=15000`). Do not wrap code in JSON (for example `{\"code\":\"...\"}`), quotes, or markdown code fences.\n"); - section.push_str( - "- Helpers: `codex.cwd`, `codex.homeDir`, `codex.tmpDir`, `codex.tool(name, args?)`, and `codex.emitImage(imageLike)`.\n", - ); - section.push_str("- `codex.tool` executes a normal tool call and resolves to the raw tool output object. Use it for shell and non-shell tools alike. Nested tool outputs stay inside JavaScript unless you emit them explicitly.\n"); - section.push_str("- `codex.emitImage(...)` adds one image to the outer `js_repl` function output each time you call it, so you can call it multiple times to emit multiple images. It accepts a data URL, a single `input_image` item, an object like `{ bytes, mimeType }`, or a raw tool response object with exactly one image and no text. It rejects mixed text-and-image content.\n"); - section.push_str("- `codex.tool(...)` and `codex.emitImage(...)` keep stable helper identities across cells. Saved references and persisted objects can reuse them in later cells, but async callbacks that fire after a cell finishes still fail because no exec is active.\n"); - section.push_str("- Request full-resolution image processing with `detail: \"original\"` only when the `view_image` tool schema includes a `detail` argument. The same availability applies to `codex.emitImage(...)`: if `view_image.detail` is present, you may also pass `detail: \"original\"` there. Use this when high-fidelity image perception or precise localization is needed, especially for CUA agents.\n"); - section.push_str("- Example of sharing an in-memory Playwright screenshot: `await codex.emitImage({ bytes: await page.screenshot({ type: \"jpeg\", quality: 85 }), mimeType: \"image/jpeg\", detail: \"original\" })`.\n"); - section.push_str("- Example of sharing a local image tool result: `await codex.emitImage(codex.tool(\"view_image\", { path: \"/absolute/path\", detail: \"original\" }))`.\n"); - section.push_str("- When encoding an image to send with `codex.emitImage(...)` or `view_image`, prefer JPEG at about 85 quality when lossy compression is acceptable; use PNG when transparency or lossless detail matters. Smaller uploads are faster and less likely to hit size limits.\n"); - section.push_str("- Top-level bindings persist across cells. If a cell throws, prior bindings remain available and bindings that finished initializing before the throw often remain usable in later cells. For code you plan to reuse across cells, prefer declaring or assigning it in direct top-level statements before operations that might throw. If you hit `SyntaxError: Identifier 'x' has already been declared`, first reuse the existing binding, reassign a previously declared `let`, or pick a new descriptive name. Use `{ ... }` only for a short temporary block when you specifically need local scratch names; do not wrap an entire cell in block scope if you want those names reusable later. Reset the kernel with `js_repl_reset` only when you need a clean state.\n"); - section.push_str("- Top-level static import declarations (for example `import x from \"./file.js\"`) are currently unsupported in `js_repl`; use dynamic imports with `await import(\"pkg\")`, `await import(\"./file.js\")`, or `await import(\"/abs/path/file.mjs\")` instead. Imported local files must be ESM `.js`/`.mjs` files and run in the same REPL VM context. Bare package imports always resolve from REPL-global search roots (`CODEX_JS_REPL_NODE_MODULE_DIRS`, then cwd), not relative to the imported file location. Local files may statically import only other local relative/absolute/`file://` `.js`/`.mjs` files; package and builtin imports from local files must stay dynamic. `import.meta.resolve()` returns importable strings such as `file://...`, bare package names, and `node:...` specifiers. Local file modules reload between execs, while top-level bindings persist until `js_repl_reset`.\n"); - - if config.features.enabled(Feature::JsReplToolsOnly) { - section.push_str("- Do not call tools directly; use `js_repl` + `codex.tool(...)` for all tool calls, including shell commands.\n"); - section - .push_str("- MCP tools (if any) can also be called by name via `codex.tool(...)`.\n"); - } - - section.push_str("- Avoid direct access to `process.stdout` / `process.stderr` / `process.stdin`; it can corrupt the JSON line protocol. Use `console.log`, `codex.tool(...)`, and `codex.emitImage(...)`."); - - Some(section) -} - -/// Combines `Config::instructions` and `AGENTS.md` (if present) into a single -/// string of instructions. -pub(crate) async fn get_user_instructions( - config: &Config, - environment: Option<&Environment>, -) -> Option { - let fs = environment?.get_filesystem(); - get_user_instructions_with_fs(config, fs.as_ref()).await -} - -pub(crate) async fn get_user_instructions_with_fs( - config: &Config, - fs: &dyn ExecutorFileSystem, -) -> Option { - let project_docs = read_project_docs_with_fs(config, fs).await; - - let mut output = String::new(); - - if let Some(instructions) = config.user_instructions.clone() { - output.push_str(&instructions); - } - - match project_docs { - Ok(Some(docs)) => { - if !output.is_empty() { - output.push_str(PROJECT_DOC_SEPARATOR); - } - output.push_str(&docs); - } - Ok(None) => {} - Err(e) => { - error!("error trying to find project doc: {e:#}"); - } - }; - - if let Some(js_repl_section) = render_js_repl_instructions(config) { - if !output.is_empty() { - output.push_str("\n\n"); - } - output.push_str(&js_repl_section); - } - - if config.features.enabled(Feature::ChildAgentsMd) { - if !output.is_empty() { - output.push_str("\n\n"); - } - output.push_str(HIERARCHICAL_AGENTS_MESSAGE); - } - - if !output.is_empty() { - Some(output) - } else { - None - } -} - -/// Attempt to locate and load the project documentation. -/// -/// On success returns `Ok(Some(contents))` where `contents` is the -/// concatenation of all discovered docs. If no documentation file is found the -/// function returns `Ok(None)`. Unexpected I/O failures bubble up as `Err` so -/// callers can decide how to handle them. -pub async fn read_project_docs( - config: &Config, - environment: &Environment, -) -> io::Result> { - let fs = environment.get_filesystem(); - read_project_docs_with_fs(config, fs.as_ref()).await -} - -async fn read_project_docs_with_fs( - config: &Config, - fs: &dyn ExecutorFileSystem, -) -> io::Result> { - let max_total = config.project_doc_max_bytes; - - if max_total == 0 { - return Ok(None); - } - - let paths = discover_project_doc_paths(config, fs).await?; - if paths.is_empty() { - return Ok(None); - } - - let mut remaining: u64 = max_total as u64; - let mut parts: Vec = Vec::new(); - - for p in paths { - if remaining == 0 { - break; - } - - match fs.get_metadata(&p, /*sandbox*/ None).await { - Ok(metadata) if !metadata.is_file => continue, - Ok(_) => {} - Err(err) if err.kind() == io::ErrorKind::NotFound => continue, - Err(err) => return Err(err), - } - - let mut data = match fs.read_file(&p, /*sandbox*/ None).await { - Ok(data) => data, - Err(err) if err.kind() == io::ErrorKind::NotFound => continue, - Err(err) => return Err(err), - }; - let size = data.len() as u64; - if size > remaining { - data.truncate(remaining as usize); - } - - if size > remaining { - tracing::warn!( - "Project doc `{}` exceeds remaining budget ({} bytes) - truncating.", - p.display(), - remaining, - ); - } - - let text = String::from_utf8_lossy(&data).to_string(); - if !text.trim().is_empty() { - parts.push(text); - remaining = remaining.saturating_sub(data.len() as u64); - } - } - - if parts.is_empty() { - Ok(None) - } else { - Ok(Some(parts.join("\n\n"))) - } -} - -/// Discover the list of AGENTS.md files using the same search rules as -/// `read_project_docs`, but return the file paths instead of concatenated -/// contents. The list is ordered from project root to the current working -/// directory (inclusive). Symlinks are allowed. When `project_doc_max_bytes` -/// is zero, returns an empty list. -pub async fn discover_project_doc_paths( - config: &Config, - fs: &dyn ExecutorFileSystem, -) -> io::Result> { - if config.project_doc_max_bytes == 0 { - return Ok(Vec::new()); - } - - let mut dir = config.cwd.clone(); - if let Ok(canon) = normalize_path(&dir) { - dir = AbsolutePathBuf::try_from(canon)?; - } - - let mut merged = TomlValue::Table(toml::map::Map::new()); - for layer in config.config_layer_stack.get_layers( - ConfigLayerStackOrdering::LowestPrecedenceFirst, - /*include_disabled*/ false, - ) { - if matches!(layer.name, ConfigLayerSource::Project { .. }) { - continue; - } - merge_toml_values(&mut merged, &layer.config); - } - let project_root_markers = match project_root_markers_from_config(&merged) { - Ok(Some(markers)) => markers, - Ok(None) => default_project_root_markers(), - Err(err) => { - tracing::warn!("invalid project_root_markers: {err}"); - default_project_root_markers() - } - }; - let mut project_root = None; - if !project_root_markers.is_empty() { - for ancestor in dir.ancestors() { - for marker in &project_root_markers { - let marker_path = ancestor.join(marker); - let marker_exists = match fs.get_metadata(&marker_path, /*sandbox*/ None).await { - Ok(_) => true, - Err(err) if err.kind() == io::ErrorKind::NotFound => false, - Err(err) => return Err(err), - }; - if marker_exists { - project_root = Some(ancestor.clone()); - break; - } - } - if project_root.is_some() { - break; - } - } - } - - let search_dirs: Vec = if let Some(root) = project_root { - let mut dirs = Vec::new(); - let mut cursor = dir.clone(); - loop { - dirs.push(cursor.clone()); - if cursor == root { - break; - } - let Some(parent) = cursor.parent() else { - break; - }; - cursor = parent; - } - dirs.reverse(); - dirs - } else { - vec![dir] - }; - - let mut found: Vec = Vec::new(); - let candidate_filenames = candidate_filenames(config); - for d in search_dirs { - for name in &candidate_filenames { - let candidate = d.join(name); - match fs.get_metadata(&candidate, /*sandbox*/ None).await { - Ok(md) if md.is_file => { - found.push(candidate); - break; - } - Ok(_) => {} - Err(err) if err.kind() == io::ErrorKind::NotFound => continue, - Err(err) => return Err(err), - } - } - } - - Ok(found) -} -fn candidate_filenames<'a>(config: &'a Config) -> Vec<&'a str> { - let mut names: Vec<&'a str> = - Vec::with_capacity(2 + config.project_doc_fallback_filenames.len()); - names.push(LOCAL_PROJECT_DOC_FILENAME); - names.push(DEFAULT_PROJECT_DOC_FILENAME); - for candidate in &config.project_doc_fallback_filenames { - let candidate = candidate.as_str(); - if candidate.is_empty() { - continue; - } - if !names.contains(&candidate) { - names.push(candidate); - } - } - names -} - -#[cfg(test)] -#[path = "project_doc_tests.rs"] -mod tests; diff --git a/codex-rs/core/src/prompt_debug.rs b/codex-rs/core/src/prompt_debug.rs index d5fa1863f2..fb64481523 100644 --- a/codex-rs/core/src/prompt_debug.rs +++ b/codex-rs/core/src/prompt_debug.rs @@ -106,7 +106,7 @@ mod tests { async fn build_prompt_input_includes_context_and_user_message() { let codex_home = tempfile::tempdir().expect("create codex home"); let cwd = tempfile::tempdir().expect("create cwd"); - let mut config = test_config(); + let mut config = test_config().await; config.codex_home = AbsolutePathBuf::from_absolute_path(codex_home.path()).expect("codex home is absolute"); config.cwd = AbsolutePathBuf::try_from(cwd.path().to_path_buf()).expect("absolute cwd"); diff --git a/codex-rs/core/src/realtime_context.rs b/codex-rs/core/src/realtime_context.rs index 6e9815f562..fc17dab99b 100644 --- a/codex-rs/core/src/realtime_context.rs +++ b/codex-rs/core/src/realtime_context.rs @@ -4,8 +4,10 @@ use crate::event_mapping::is_contextual_user_message_content; use chrono::Utc; use codex_git_utils::resolve_root_git_project_for_trust; use codex_protocol::models::ResponseItem; -use codex_state::SortKey; -use codex_state::ThreadMetadata; +use codex_thread_store::ListThreadsParams; +use codex_thread_store::StoredThread; +use codex_thread_store::ThreadSortKey; +use codex_thread_store::ThreadStore; use codex_utils_output_truncation::TruncationPolicy; use codex_utils_output_truncation::truncate_text; use dirs::home_dir; @@ -23,6 +25,8 @@ use tracing::info; use tracing::warn; const STARTUP_CONTEXT_HEADER: &str = "Startup context from Codex.\nThis is background context about recent work and machine/workspace layout. It may be incomplete or stale. Use it to inform responses, and do not repeat it back unless relevant."; +const STARTUP_CONTEXT_OPEN_TAG: &str = ""; +const STARTUP_CONTEXT_CLOSE_TAG: &str = ""; const CURRENT_THREAD_SECTION_TOKEN_BUDGET: usize = 1_200; const RECENT_WORK_SECTION_TOKEN_BUDGET: usize = 2_200; const WORKSPACE_SECTION_TOKEN_BUDGET: usize = 1_600; @@ -58,8 +62,8 @@ pub(crate) async fn build_realtime_startup_context( let history = sess.clone_history().await; let current_thread_section = build_current_thread_section(history.raw_items()); let recent_threads = load_recent_threads(sess).await; - let recent_work_section = build_recent_work_section(&cwd, &recent_threads); - let workspace_section = build_workspace_section_with_user_root(&cwd, home_dir()); + let recent_work_section = build_recent_work_section(&cwd, &recent_threads).await; + let workspace_section = build_workspace_section_with_user_root(&cwd, home_dir()).await; if current_thread_section.is_none() && recent_work_section.is_none() @@ -98,13 +102,13 @@ pub(crate) async fn build_realtime_startup_context( } if let Some(section) = format_section( "Notes", - Some("Built at realtime startup from the current thread history, persisted thread metadata in the state DB, and a bounded local workspace scan. This excludes repo memory instructions, AGENTS files, project-doc prompt blends, and memory summaries.".to_string()), + Some("Built at realtime startup from the current thread history, local thread metadata, and a bounded local workspace scan. This excludes repo memory instructions, AGENTS files, project-doc prompt blends, and memory summaries.".to_string()), NOTES_SECTION_TOKEN_BUDGET, ) { parts.push(section); } - let context = truncate_text(&parts.join("\n\n"), TruncationPolicy::Tokens(budget_tokens)); + let context = format_startup_context_blob(&parts.join("\n\n"), budget_tokens); debug!( approx_tokens = approx_token_count(&context), bytes = context.len(), @@ -117,41 +121,41 @@ pub(crate) async fn build_realtime_startup_context( Some(context) } -async fn load_recent_threads(sess: &Session) -> Vec { - let Some(state_db) = sess.services.state_db.as_ref() else { - return Vec::new(); - }; - - match state_db - .list_threads( - MAX_RECENT_THREADS, - /*anchor*/ None, - SortKey::UpdatedAt, - &[], - /*model_providers*/ None, - /*archived_only*/ false, - /*search_term*/ None, - ) +async fn load_recent_threads(sess: &Session) -> Vec { + match sess + .services + .thread_store + .list_threads(ListThreadsParams { + page_size: MAX_RECENT_THREADS, + cursor: None, + sort_key: ThreadSortKey::UpdatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived: false, + search_term: None, + }) .await { Ok(page) => page.items, Err(err) => { - warn!("failed to load realtime startup threads from state db: {err}"); + warn!("failed to load realtime startup threads from thread store: {err}"); Vec::new() } } } -fn build_recent_work_section(cwd: &Path, recent_threads: &[ThreadMetadata]) -> Option { - let mut groups: HashMap> = HashMap::new(); +async fn build_recent_work_section(cwd: &Path, recent_threads: &[StoredThread]) -> Option { + let mut groups: HashMap> = HashMap::new(); for entry in recent_threads { - let group = - resolve_root_git_project_for_trust(&entry.cwd).unwrap_or_else(|| entry.cwd.clone()); + let group = resolve_root_git_project_for_trust(&entry.cwd) + .await + .unwrap_or_else(|| entry.cwd.clone()); groups.entry(group).or_default().push(entry); } - let current_group = - resolve_root_git_project_for_trust(cwd).unwrap_or_else(|| cwd.to_path_buf()); + let current_group = resolve_root_git_project_for_trust(cwd) + .await + .unwrap_or_else(|| cwd.to_path_buf()); let mut groups = groups.into_iter().collect::>(); groups.sort_by(|(left_group, left_entries), (right_group, right_entries)| { let left_latest = left_entries @@ -176,14 +180,13 @@ fn build_recent_work_section(cwd: &Path, recent_threads: &[ThreadMetadata]) -> O )) }); - let sections = groups - .into_iter() - .take(MAX_RECENT_WORK_GROUPS) - .filter_map(|(group, mut entries)| { - entries.sort_by_key(|entry| Reverse(entry.updated_at)); - format_thread_group(¤t_group, &group, entries) - }) - .collect::>(); + let mut sections = Vec::new(); + for (group, mut entries) in groups.into_iter().take(MAX_RECENT_WORK_GROUPS) { + entries.sort_by_key(|entry| Reverse(entry.updated_at)); + if let Some(section) = format_thread_group(¤t_group, &group, entries).await { + sections.push(section); + } + } (!sections.is_empty()).then(|| sections.join("\n\n")) } @@ -304,11 +307,11 @@ pub(crate) fn truncate_realtime_text_to_token_budget(text: &str, budget_tokens: } } -fn build_workspace_section_with_user_root( +async fn build_workspace_section_with_user_root( cwd: &Path, user_root: Option, ) -> Option { - let git_root = resolve_root_git_project_for_trust(cwd); + let git_root = resolve_root_git_project_for_trust(cwd).await; let cwd_tree = render_tree(cwd); let git_root_tree = git_root .as_ref() @@ -443,13 +446,37 @@ fn format_section(title: &str, body: Option, budget_tokens: usize) -> Op )) } -fn format_thread_group( +fn format_startup_context_blob(body: &str, budget_tokens: usize) -> String { + let wrapper = format!("{STARTUP_CONTEXT_OPEN_TAG}\n\n{STARTUP_CONTEXT_CLOSE_TAG}"); + let mut body_budget = budget_tokens.saturating_sub(approx_token_count(&wrapper)); + + loop { + let body = truncate_text(body, TruncationPolicy::Tokens(body_budget)); + let wrapped = format!("{STARTUP_CONTEXT_OPEN_TAG}\n{body}\n{STARTUP_CONTEXT_CLOSE_TAG}"); + let wrapped_tokens = approx_token_count(&wrapped); + if wrapped_tokens <= budget_tokens || body_budget == 0 { + return wrapped; + } + + let excess_tokens = wrapped_tokens.saturating_sub(budget_tokens); + let next_budget = body_budget.saturating_sub(excess_tokens.max(1)); + if next_budget == body_budget { + return wrapped; + } + body_budget = next_budget; + } +} + +async fn format_thread_group( current_group: &Path, group: &Path, - entries: Vec<&ThreadMetadata>, + entries: Vec<&StoredThread>, ) -> Option { let latest = entries.first()?; - let group_label = if resolve_root_git_project_for_trust(latest.cwd.as_path()).is_some() { + let group_label = if resolve_root_git_project_for_trust(latest.cwd.as_path()) + .await + .is_some() + { format!("### Git repo: {}", group.display()) } else { format!("### Directory: {}", group.display()) @@ -461,8 +488,9 @@ fn format_thread_group( ]; if let Some(git_branch) = latest - .git_branch - .as_deref() + .git_info + .as_ref() + .and_then(|git| git.branch.as_deref()) .filter(|git_branch| !git_branch.is_empty()) { lines.push(format!("Latest branch: {git_branch}")); diff --git a/codex-rs/core/src/realtime_context_tests.rs b/codex-rs/core/src/realtime_context_tests.rs index 7495161d46..4f62affcae 100644 --- a/codex-rs/core/src/realtime_context_tests.rs +++ b/codex-rs/core/src/realtime_context_tests.rs @@ -1,22 +1,34 @@ use super::build_current_thread_section; use super::build_recent_work_section; use super::build_workspace_section_with_user_root; +use super::format_startup_context_blob; use chrono::TimeZone; use chrono::Utc; +use codex_git_utils::GitSha; use codex_protocol::ThreadId; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; -use codex_state::ThreadMetadata; +use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::GitInfo; +use codex_protocol::protocol::SandboxPolicy; +use codex_protocol::protocol::SessionSource; +use codex_thread_store::StoredThread; use pretty_assertions::assert_eq; use std::fs; use std::path::PathBuf; use std::process::Command; use tempfile::TempDir; -fn thread_metadata(cwd: &str, title: &str, first_user_message: &str) -> ThreadMetadata { - ThreadMetadata { - id: ThreadId::new(), - rollout_path: PathBuf::from("/tmp/rollout.jsonl"), +fn stored_thread(cwd: &str, title: &str, first_user_message: &str) -> StoredThread { + StoredThread { + thread_id: ThreadId::new(), + rollout_path: Some(PathBuf::from("/tmp/rollout.jsonl")), + forked_from_id: None, + preview: first_user_message.to_string(), + name: (!title.is_empty()).then(|| title.to_string()), + model_provider: "test-provider".to_string(), + model: Some("gpt-5".to_string()), + reasoning_effort: None, created_at: Utc .timestamp_opt(1_709_251_100, 0) .single() @@ -25,24 +37,23 @@ fn thread_metadata(cwd: &str, title: &str, first_user_message: &str) -> ThreadMe .timestamp_opt(1_709_251_200, 0) .single() .expect("valid timestamp"), - source: "cli".to_string(), - agent_path: None, - agent_nickname: None, - agent_role: None, - model_provider: "test-provider".to_string(), - model: Some("gpt-5".to_string()), - reasoning_effort: None, + archived_at: None, cwd: PathBuf::from(cwd), cli_version: "test".to_string(), - title: title.to_string(), - sandbox_policy: "workspace-write".to_string(), - approval_mode: "never".to_string(), - tokens_used: 0, + source: SessionSource::Cli, + agent_nickname: None, + agent_role: None, + agent_path: None, + git_info: Some(GitInfo { + commit_hash: Some(GitSha::new("abcdef")), + branch: Some("main".to_string()), + repository_url: None, + }), + approval_mode: AskForApproval::Never, + sandbox_policy: SandboxPolicy::new_read_only_policy(), + token_usage: None, first_user_message: Some(first_user_message.to_string()), - archived_at: None, - git_sha: None, - git_branch: Some("main".to_string()), - git_origin_url: None, + history: None, } } @@ -161,29 +172,47 @@ fn current_thread_section_keeps_latest_turns_when_history_exceeds_budget() { } #[test] -fn workspace_section_requires_meaningful_structure() { +fn startup_context_blob_is_wrapped_in_tags_and_fits_budget() { + let body = format!( + "Startup context from Codex.\n{}\n{}", + "recent work ".repeat(1_200), + "workspace tree ".repeat(800), + ); + + let wrapped = format_startup_context_blob(&body, /*budget_tokens*/ 200); + + assert!(wrapped.starts_with("\n")); + assert!(wrapped.ends_with("\n")); + assert!(wrapped.contains("Startup context from Codex.")); + assert!(wrapped.contains("tokens truncated")); + assert!(wrapped.len().div_ceil(4) <= 200); +} + +#[tokio::test] +async fn workspace_section_requires_meaningful_structure() { let cwd = TempDir::new().expect("tempdir"); assert_eq!( - build_workspace_section_with_user_root(cwd.path(), /*user_root*/ None), + build_workspace_section_with_user_root(cwd.path(), /*user_root*/ None).await, None ); } -#[test] -fn workspace_section_includes_tree_when_entries_exist() { +#[tokio::test] +async fn workspace_section_includes_tree_when_entries_exist() { let cwd = TempDir::new().expect("tempdir"); fs::create_dir(cwd.path().join("docs")).expect("create docs dir"); fs::write(cwd.path().join("README.md"), "hello").expect("write readme"); let section = build_workspace_section_with_user_root(cwd.path(), /*user_root*/ None) + .await .expect("workspace section"); assert!(section.contains("Working directory tree:")); assert!(section.contains("- docs/")); assert!(section.contains("- README.md")); } -#[test] -fn workspace_section_includes_user_root_tree_when_distinct() { +#[tokio::test] +async fn workspace_section_includes_user_root_tree_when_distinct() { let root = TempDir::new().expect("tempdir"); let cwd = root.path().join("cwd"); let git_root = root.path().join("git"); @@ -197,14 +226,15 @@ fn workspace_section_includes_user_root_tree_when_distinct() { fs::write(user_root.join(".zshrc"), "export TEST=1").expect("write home file"); let section = build_workspace_section_with_user_root(cwd.as_path(), Some(user_root)) + .await .expect("workspace section"); assert!(section.contains("User root tree:")); assert!(section.contains("- code/")); assert!(!section.contains("- .zshrc")); } -#[test] -fn recent_work_section_groups_threads_by_cwd() { +#[tokio::test] +async fn recent_work_section_groups_threads_by_cwd() { let root = TempDir::new().expect("tempdir"); let repo = root.path().join("repo"); let workspace_a = repo.join("workspace-a"); @@ -224,22 +254,23 @@ fn recent_work_section_groups_threads_by_cwd() { fs::create_dir_all(&outside).expect("create outside dir"); let recent_threads = vec![ - thread_metadata( + stored_thread( workspace_a.to_string_lossy().as_ref(), "Investigate realtime startup context", "Log the startup context before sending it", ), - thread_metadata( + stored_thread( workspace_b.to_string_lossy().as_ref(), "Trim websocket startup payload", "Remove memories from the realtime startup context", ), - thread_metadata(outside.to_string_lossy().as_ref(), "", "Inspect flaky test"), + stored_thread(outside.to_string_lossy().as_ref(), "", "Inspect flaky test"), ]; let current_cwd = workspace_a; let repo = fs::canonicalize(repo).expect("canonicalize repo"); let section = build_recent_work_section(current_cwd.as_path(), &recent_threads) + .await .expect("recent work section"); assert!(section.contains(&format!("### Git repo: {}", repo.display()))); assert!(section.contains("Recent sessions: 2")); diff --git a/codex-rs/core/src/realtime_conversation.rs b/codex-rs/core/src/realtime_conversation.rs index f81d6c0948..b79060259c 100644 --- a/codex-rs/core/src/realtime_conversation.rs +++ b/codex-rs/core/src/realtime_conversation.rs @@ -65,8 +65,10 @@ const HANDOFF_OUT_QUEUE_CAPACITY: usize = 64; const OUTPUT_EVENTS_QUEUE_CAPACITY: usize = 256; const REALTIME_STARTUP_CONTEXT_TOKEN_BUDGET: usize = 5_000; const DEFAULT_REALTIME_MODEL: &str = "gpt-realtime-1.5"; -const REALTIME_V2_PROGRESS_UPDATE_SUFFIX: &str = - "\n\nUpdate from background agent (task hasn't finished yet):"; +pub(crate) const REALTIME_USER_TEXT_PREFIX: &str = "[USER] "; +pub(crate) const REALTIME_BACKEND_TEXT_PREFIX: &str = "[BACKEND] "; +const REALTIME_V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT: &str = + "Background agent finished. Use the preceding [BACKEND] messages as the result."; const REALTIME_V2_STEER_ACKNOWLEDGEMENT: &str = "This was sent to steer the previous background agent task."; const REALTIME_ACTIVE_RESPONSE_ERROR_PREFIX: &str = @@ -209,6 +211,7 @@ impl RealtimeHandoffState { struct ConversationState { audio_tx: Sender, user_text_tx: Sender, + session_kind: RealtimeSessionKind, writer: RealtimeWebsocketWriter, handoff: RealtimeHandoffState, input_task: JoinHandle<()>, @@ -245,6 +248,16 @@ impl RealtimeConversationManager { .and_then(|state| state.realtime_active.load(Ordering::Relaxed).then_some(())) } + pub(crate) async fn is_running_v2(&self) -> bool { + let state = self.state.lock().await; + matches!( + state.as_ref(), + Some(state) + if state.realtime_active.load(Ordering::Relaxed) + && state.session_kind == RealtimeSessionKind::V2 + ) + } + async fn start(&self, start: RealtimeStart) -> CodexResult { let previous_state = { let mut guard = self.state.lock().await; @@ -331,6 +344,7 @@ impl RealtimeConversationManager { *guard = Some(ConversationState { audio_tx, user_text_tx, + session_kind, writer, handoff, input_task: task, @@ -406,15 +420,18 @@ impl RealtimeConversationManager { pub(crate) async fn text_in(&self, text: String) -> CodexResult<()> { let sender = { let guard = self.state.lock().await; - guard.as_ref().map(|state| state.user_text_tx.clone()) + guard + .as_ref() + .map(|state| (state.user_text_tx.clone(), state.session_kind)) }; - let Some(sender) = sender else { + let Some((sender, session_kind)) = sender else { return Err(CodexErr::InvalidRequest( "conversation is not running".to_string(), )); }; + let text = prefix_realtime_text(text, REALTIME_USER_TEXT_PREFIX, session_kind); sender .send(text) .await @@ -437,6 +454,11 @@ impl RealtimeConversationManager { return Ok(()); }; + let output_text = prefix_realtime_text( + output_text, + REALTIME_BACKEND_TEXT_PREFIX, + handoff.session_kind, + ); *handoff.last_output_text.lock().await = Some(output_text.clone()); handoff .output_tx @@ -695,6 +717,17 @@ fn default_realtime_voice(version: RealtimeWsVersion) -> RealtimeVoice { } } +fn prefix_realtime_text(text: String, prefix: &str, session_kind: RealtimeSessionKind) -> String { + if session_kind != RealtimeSessionKind::V2 || text.is_empty() || text.starts_with(prefix) { + return text; + } + format!("{prefix}{text}") +} + +pub(crate) fn prefix_realtime_v2_text(text: String, prefix: &str) -> String { + prefix_realtime_text(text, prefix, RealtimeSessionKind::V2) +} + fn validate_realtime_voice(version: RealtimeWsVersion, voice: RealtimeVoice) -> CodexResult<()> { let voices = RealtimeVoicesList::builtin(); let allowed = match version { @@ -1074,18 +1107,17 @@ async fn handle_handoff_output( return Ok(()); } } - writer - .send_conversation_item_create(format!( - "{output_text}{REALTIME_V2_PROGRESS_UPDATE_SUFFIX}" - )) - .await + writer.send_conversation_item_create(output_text).await } HandoffOutput::FinalUpdate { handoff_id, - output_text, + output_text: _, } => { if let Err(err) = writer - .send_conversation_handoff_append(handoff_id, output_text) + .send_conversation_handoff_append( + handoff_id, + REALTIME_V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT.to_string(), + ) .await { Err(err) diff --git a/codex-rs/core/src/rollout.rs b/codex-rs/core/src/rollout.rs index 26e3a84d5c..4913f957ea 100644 --- a/codex-rs/core/src/rollout.rs +++ b/codex-rs/core/src/rollout.rs @@ -46,11 +46,7 @@ impl codex_rollout::RolloutConfigView for Config { } pub(crate) mod list { - pub use codex_rollout::ThreadListConfig; - pub use codex_rollout::ThreadListLayout; - pub use codex_rollout::ThreadSortKey; pub use codex_rollout::find_thread_path_by_id_str; - pub use codex_rollout::get_threads_in_root; } pub(crate) mod metadata { diff --git a/codex-rs/core/src/seatbelt.rs b/codex-rs/core/src/seatbelt.rs deleted file mode 100644 index c54ff3fd5b..0000000000 --- a/codex-rs/core/src/seatbelt.rs +++ /dev/null @@ -1,48 +0,0 @@ -#![cfg(target_os = "macos")] - -use crate::spawn::CODEX_SANDBOX_ENV_VAR; -use crate::spawn::SpawnChildRequest; -use crate::spawn::StdioPolicy; -use crate::spawn::spawn_child_async; -use codex_network_proxy::NetworkProxy; -use codex_protocol::permissions::FileSystemSandboxPolicy; -use codex_protocol::permissions::NetworkSandboxPolicy; -use codex_protocol::protocol::SandboxPolicy; -use codex_sandboxing::seatbelt::MACOS_PATH_TO_SEATBELT_EXECUTABLE; -use codex_sandboxing::seatbelt::create_seatbelt_command_args_for_policies; -use codex_utils_absolute_path::AbsolutePathBuf; -use std::collections::HashMap; -use std::path::PathBuf; -use tokio::process::Child; - -pub async fn spawn_command_under_seatbelt( - command: Vec, - command_cwd: AbsolutePathBuf, - sandbox_policy: &SandboxPolicy, - sandbox_policy_cwd: &AbsolutePathBuf, - stdio_policy: StdioPolicy, - network: Option<&NetworkProxy>, - mut env: HashMap, -) -> std::io::Result { - let args = create_seatbelt_command_args_for_policies( - command, - &FileSystemSandboxPolicy::from_legacy_sandbox_policy(sandbox_policy, sandbox_policy_cwd), - NetworkSandboxPolicy::from(sandbox_policy), - sandbox_policy_cwd, - /*enforce_managed_network*/ false, - network, - ); - let arg0 = None; - env.insert(CODEX_SANDBOX_ENV_VAR.to_string(), "seatbelt".to_string()); - spawn_child_async(SpawnChildRequest { - program: PathBuf::from(MACOS_PATH_TO_SEATBELT_EXECUTABLE), - args, - arg0, - cwd: command_cwd, - network_sandbox_policy: NetworkSandboxPolicy::from(sandbox_policy), - network, - stdio_policy, - env, - }) - .await -} diff --git a/codex-rs/core/src/state/service.rs b/codex-rs/core/src/state/service.rs index 2e89585f99..5db38f7b72 100644 --- a/codex-rs/core/src/state/service.rs +++ b/codex-rs/core/src/state/service.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use crate::RolloutRecorder; use crate::SkillsManager; use crate::agent::AgentControl; +use crate::agent_identity::AgentIdentityManager; use crate::client::ModelClient; use crate::config::StartedNetworkProxy; use crate::exec_policy::ExecPolicyManager; @@ -23,6 +24,7 @@ use codex_mcp::McpConnectionManager; use codex_models_manager::manager::ModelsManager; use codex_otel::SessionTelemetry; use codex_rollout::state_db::StateDbHandle; +use codex_thread_store::LocalThreadStore; use std::path::PathBuf; use tokio::sync::Mutex; use tokio::sync::RwLock; @@ -41,6 +43,7 @@ pub(crate) struct SessionServices { pub(crate) hooks: Hooks, pub(crate) rollout: Mutex>, pub(crate) user_shell: Arc, + pub(crate) agent_identity_manager: Arc, pub(crate) shell_snapshot_tx: watch::Sender>>, pub(crate) show_raw_agent_reasoning: bool, pub(crate) exec_policy: Arc, @@ -57,6 +60,7 @@ pub(crate) struct SessionServices { pub(crate) network_proxy: Option, pub(crate) network_approval: Arc, pub(crate) state_db: Option, + pub(crate) thread_store: LocalThreadStore, /// Session-scoped model client shared across turns. pub(crate) model_client: ModelClient, pub(crate) code_mode_service: CodeModeService, diff --git a/codex-rs/core/src/stream_events_utils.rs b/codex-rs/core/src/stream_events_utils.rs index d491bb6adb..3838d6ed01 100644 --- a/codex-rs/core/src/stream_events_utils.rs +++ b/codex-rs/core/src/stream_events_utils.rs @@ -136,11 +136,20 @@ pub(crate) async fn record_completed_response_item( sess.defer_mailbox_delivery_to_next_turn(&turn_context.sub_id) .await; } - maybe_mark_thread_memory_mode_polluted_from_web_search(sess, turn_context, item).await; + mark_thread_memory_mode_polluted_if_external_context(sess, turn_context, item).await; record_stage1_output_usage_for_completed_item(turn_context, item).await; } -async fn maybe_mark_thread_memory_mode_polluted_from_web_search( +fn response_item_may_include_external_context(item: &ResponseItem) -> bool { + matches!( + item, + ResponseItem::ToolSearchCall { .. } + | ResponseItem::ToolSearchOutput { .. } + | ResponseItem::WebSearchCall { .. } + ) +} + +pub(crate) async fn mark_thread_memory_mode_polluted_if_external_context( sess: &Session, turn_context: &TurnContext, item: &ResponseItem, @@ -149,7 +158,7 @@ async fn maybe_mark_thread_memory_mode_polluted_from_web_search( .config .memories .no_memories_if_mcp_or_web_search - || !matches!(item, ResponseItem::WebSearchCall { .. }) + || !response_item_may_include_external_context(item) { return; } diff --git a/codex-rs/core/src/stream_events_utils_tests.rs b/codex-rs/core/src/stream_events_utils_tests.rs index e3947a06ed..2634a88742 100644 --- a/codex-rs/core/src/stream_events_utils_tests.rs +++ b/codex-rs/core/src/stream_events_utils_tests.rs @@ -2,11 +2,16 @@ use super::completed_item_defers_mailbox_delivery_to_next_turn; use super::handle_non_tool_response_item; use super::image_generation_artifact_path; use super::last_assistant_message_from_item; +use super::response_item_may_include_external_context; use super::save_image_generation_result; use crate::codex::make_session_and_context; use codex_protocol::error::CodexErr; use codex_protocol::items::TurnItem; use codex_protocol::models::ContentItem; +use codex_protocol::models::FunctionCallOutputPayload; +use codex_protocol::models::LocalShellAction; +use codex_protocol::models::LocalShellExecAction; +use codex_protocol::models::LocalShellStatus; use codex_protocol::models::MessagePhase; use codex_protocol::models::ResponseItem; use codex_utils_absolute_path::test_support::PathExt; @@ -28,6 +33,84 @@ fn assistant_output_text_with_phase(text: &str, phase: Option) -> } } +#[test] +fn external_context_pollution_items_include_web_search_and_tool_search() { + let polluting_items = [ + ResponseItem::WebSearchCall { + id: None, + status: Some("completed".to_string()), + action: None, + }, + ResponseItem::ToolSearchCall { + id: None, + call_id: Some("search-1".to_string()), + status: None, + execution: "client".to_string(), + arguments: serde_json::json!({"query": "calendar"}), + }, + ResponseItem::ToolSearchOutput { + call_id: Some("search-1".to_string()), + status: "completed".to_string(), + execution: "client".to_string(), + tools: Vec::new(), + }, + ]; + + assert!( + polluting_items + .iter() + .all(response_item_may_include_external_context) + ); +} + +#[test] +fn external_context_pollution_items_exclude_local_tool_calls() { + let non_polluting_items = [ + ResponseItem::LocalShellCall { + id: None, + call_id: Some("shell-1".to_string()), + status: LocalShellStatus::Completed, + action: LocalShellAction::Exec(LocalShellExecAction { + command: vec!["cat".to_string(), "README.md".to_string()], + timeout_ms: None, + working_directory: None, + env: None, + user: None, + }), + }, + ResponseItem::FunctionCall { + id: None, + name: "shell".to_string(), + namespace: None, + arguments: "{}".to_string(), + call_id: "call-1".to_string(), + }, + ResponseItem::FunctionCallOutput { + call_id: "call-1".to_string(), + output: FunctionCallOutputPayload::from_text("ok".to_string()), + }, + ResponseItem::CustomToolCall { + id: None, + status: None, + call_id: "custom-1".to_string(), + name: "apply_patch".to_string(), + input: "*** Begin Patch\n*** End Patch\n".to_string(), + }, + ResponseItem::CustomToolCallOutput { + call_id: "custom-1".to_string(), + name: Some("apply_patch".to_string()), + output: FunctionCallOutputPayload::from_text("ok".to_string()), + }, + assistant_output_text("plain assistant text"), + ]; + + assert!( + !non_polluting_items + .iter() + .any(response_item_may_include_external_context) + ); +} + #[tokio::test] async fn handle_non_tool_response_item_strips_citations_from_assistant_message() { let (session, turn_context) = make_session_and_context().await; diff --git a/codex-rs/core/src/tasks/user_shell.rs b/codex-rs/core/src/tasks/user_shell.rs index 76e531744b..8b108eee06 100644 --- a/codex-rs/core/src/tasks/user_shell.rs +++ b/codex-rs/core/src/tasks/user_shell.rs @@ -163,7 +163,9 @@ pub(crate) async fn execute_user_shell_command( cwd: cwd.clone(), env: exec_env_map, exec_server_env_config: None, - network: turn_context.network.clone(), + // `/shell` is the explicit full-access escape hatch, so it must not + // inherit a managed proxy from the surrounding session or turn. + network: None, // TODO(zhao-oai): Now that we have ExecExpiration::Cancellation, we // should use that instead of an "arbitrarily large" timeout here. expiration: USER_SHELL_TIMEOUT_MS.into(), diff --git a/codex-rs/core/src/thread_manager_tests.rs b/codex-rs/core/src/thread_manager_tests.rs index 57d6ba8d6d..06f89fc728 100644 --- a/codex-rs/core/src/thread_manager_tests.rs +++ b/codex-rs/core/src/thread_manager_tests.rs @@ -237,7 +237,7 @@ async fn ignores_session_prefix_messages_when_truncating() { #[tokio::test] async fn shutdown_all_threads_bounded_submits_shutdown_to_every_thread() { let temp_dir = tempdir().expect("tempdir"); - let mut config = test_config(); + let mut config = test_config().await; config.codex_home = temp_dir.path().join("codex-home").abs(); config.cwd = config.codex_home.abs(); std::fs::create_dir_all(&config.codex_home).expect("create codex home"); @@ -279,7 +279,7 @@ async fn new_uses_configured_openai_provider_for_model_refresh() { let models_mock = mount_models_once(&server, ModelsResponse { models: vec![] }).await; let temp_dir = tempdir().expect("tempdir"); - let mut config = test_config(); + let mut config = test_config().await; config.codex_home = temp_dir.path().join("codex-home").abs(); config.cwd = config.codex_home.abs(); std::fs::create_dir_all(&config.codex_home).expect("create codex home"); @@ -422,7 +422,7 @@ fn mixed_response_and_legacy_user_event_history_is_mid_turn() { #[tokio::test] async fn interrupted_fork_snapshot_does_not_synthesize_turn_id_for_legacy_history() { let temp_dir = tempdir().expect("tempdir"); - let mut config = test_config(); + let mut config = test_config().await; config.codex_home = temp_dir.path().join("codex-home").abs(); config.cwd = config.codex_home.abs(); std::fs::create_dir_all(&config.codex_home).expect("create codex home"); @@ -525,7 +525,7 @@ async fn interrupted_fork_snapshot_does_not_synthesize_turn_id_for_legacy_histor #[tokio::test] async fn interrupted_fork_snapshot_preserves_explicit_turn_id() { let temp_dir = tempdir().expect("tempdir"); - let mut config = test_config(); + let mut config = test_config().await; config.codex_home = temp_dir.path().join("codex-home").abs(); config.cwd = config.codex_home.abs(); std::fs::create_dir_all(&config.codex_home).expect("create codex home"); @@ -618,7 +618,7 @@ async fn interrupted_fork_snapshot_preserves_explicit_turn_id() { #[tokio::test] async fn interrupted_fork_snapshot_uses_persisted_mid_turn_history_without_live_source() { let temp_dir = tempdir().expect("tempdir"); - let mut config = test_config(); + let mut config = test_config().await; config.codex_home = temp_dir.path().join("codex-home").abs(); config.cwd = config.codex_home.abs(); std::fs::create_dir_all(&config.codex_home).expect("create codex home"); diff --git a/codex-rs/core/src/tools/code_mode/mod.rs b/codex-rs/core/src/tools/code_mode/mod.rs index de3a1fa330..7d66d56c39 100644 --- a/codex-rs/core/src/tools/code_mode/mod.rs +++ b/codex-rs/core/src/tools/code_mode/mod.rs @@ -18,6 +18,8 @@ use tokio_util::sync::CancellationToken; use crate::codex::Session; use crate::codex::TurnContext; use crate::function_tool::FunctionCallError; +use crate::original_image_detail::can_request_original_image_detail; +use crate::original_image_detail::sanitize_original_image_detail as sanitize_image_detail_items; use crate::tools::ToolRouter; use crate::tools::context::FunctionToolOutput; use crate::tools::context::SharedTurnDiffTracker; @@ -116,7 +118,7 @@ struct CoreTurnHost { impl CodeModeTurnHost for CoreTurnHost { async fn invoke_tool( &self, - tool_name: String, + tool_name: ToolName, input: Option, cancellation_token: CancellationToken, ) -> Result { @@ -160,12 +162,14 @@ pub(super) async fn handle_runtime_response( match response { RuntimeResponse::Yielded { content_items, .. } => { let mut content_items = into_function_call_output_content_items(content_items); + sanitize_runtime_image_detail(exec.turn.as_ref(), &mut content_items); content_items = truncate_code_mode_result(content_items, max_output_tokens); prepend_script_status(&mut content_items, &script_status, started_at.elapsed()); Ok(FunctionToolOutput::from_content(content_items, Some(true))) } RuntimeResponse::Terminated { content_items, .. } => { let mut content_items = into_function_call_output_content_items(content_items); + sanitize_runtime_image_detail(exec.turn.as_ref(), &mut content_items); content_items = truncate_code_mode_result(content_items, max_output_tokens); prepend_script_status(&mut content_items, &script_status, started_at.elapsed()); Ok(FunctionToolOutput::from_content(content_items, Some(true))) @@ -177,6 +181,7 @@ pub(super) async fn handle_runtime_response( .. } => { let mut content_items = into_function_call_output_content_items(content_items); + sanitize_runtime_image_detail(exec.turn.as_ref(), &mut content_items); exec.session .services .code_mode_service @@ -198,6 +203,10 @@ pub(super) async fn handle_runtime_response( } } +fn sanitize_runtime_image_detail(turn: &TurnContext, items: &mut [FunctionCallOutputContentItem]) { + sanitize_image_detail_items(can_request_original_image_detail(&turn.model_info), items); +} + fn format_script_status(response: &RuntimeResponse) -> String { match response { RuntimeResponse::Yielded { cell_id, .. } => { @@ -278,6 +287,7 @@ async fn build_nested_router(exec: &ExecContext) -> ToolRouter { ToolRouterParams { deferred_mcp_tools: None, mcp_tools: Some(listed_mcp_tools), + unavailable_called_tools: Vec::new(), parallel_mcp_server_names, discoverable_tools: None, dynamic_tools: exec.turn.dynamic_tools.as_slice(), @@ -288,38 +298,39 @@ async fn build_nested_router(exec: &ExecContext) -> ToolRouter { async fn call_nested_tool( exec: ExecContext, tool_runtime: ToolCallRuntime, - tool_name: String, + tool_name: ToolName, input: Option, cancellation_token: CancellationToken, ) -> Result { - if tool_name == PUBLIC_TOOL_NAME { + if tool_name.namespace.is_none() && tool_name.name == PUBLIC_TOOL_NAME { return Err(FunctionCallError::RespondToModel(format!( "{PUBLIC_TOOL_NAME} cannot invoke itself" ))); } - let payload = if let Some(tool_info) = exec - .session - .resolve_mcp_tool_info(&tool_name, /*namespace*/ None) - .await - { - match serialize_function_tool_arguments(&tool_name, input) { - Ok(raw_arguments) => ToolPayload::Mcp { - server: tool_info.server_name, - tool: tool_info.tool.name.to_string(), - raw_arguments, - }, - Err(error) => return Err(FunctionCallError::RespondToModel(error)), - } - } else { - match build_nested_tool_payload(tool_runtime.find_spec(&tool_name), &tool_name, input) { - Ok(payload) => payload, - Err(error) => return Err(FunctionCallError::RespondToModel(error)), - } - }; + let (tool_call_name, payload) = + if let Some(tool_info) = exec.session.resolve_mcp_tool_info(&tool_name).await { + let raw_arguments = match serialize_function_tool_arguments(&tool_name, input) { + Ok(raw_arguments) => raw_arguments, + Err(error) => return Err(FunctionCallError::RespondToModel(error)), + }; + ( + tool_info.canonical_tool_name(), + ToolPayload::Mcp { + server: tool_info.server_name, + tool: tool_info.tool.name.to_string(), + raw_arguments, + }, + ) + } else { + match build_nested_tool_payload(tool_runtime.find_spec(&tool_name), &tool_name, input) { + Ok(payload) => (tool_name, payload), + Err(error) => return Err(FunctionCallError::RespondToModel(error)), + } + }; let call = ToolCall { - tool_name: ToolName::plain(tool_name.clone()), + tool_name: tool_call_name, call_id: format!("{PUBLIC_TOOL_NAME}-{}", uuid::Uuid::new_v4()), payload, }; @@ -339,7 +350,7 @@ fn tool_kind_for_spec(spec: &ToolSpec) -> codex_code_mode::CodeModeToolKind { fn tool_kind_for_name( spec: Option, - tool_name: &str, + tool_name: &ToolName, ) -> Result { spec.as_ref() .map(tool_kind_for_spec) @@ -348,7 +359,7 @@ fn tool_kind_for_name( fn build_nested_tool_payload( spec: Option, - tool_name: &str, + tool_name: &ToolName, input: Option, ) -> Result { let actual_kind = tool_kind_for_name(spec, tool_name)?; @@ -363,7 +374,7 @@ fn build_nested_tool_payload( } fn build_function_tool_payload( - tool_name: &str, + tool_name: &ToolName, input: Option, ) -> Result { let arguments = serialize_function_tool_arguments(tool_name, input)?; @@ -371,7 +382,7 @@ fn build_function_tool_payload( } fn serialize_function_tool_arguments( - tool_name: &str, + tool_name: &ToolName, input: Option, ) -> Result { match input { @@ -385,7 +396,7 @@ fn serialize_function_tool_arguments( } fn build_freeform_tool_payload( - tool_name: &str, + tool_name: &ToolName, input: Option, ) -> Result { match input { diff --git a/codex-rs/core/src/tools/context.rs b/codex-rs/core/src/tools/context.rs index dff1e444da..60bcbcda23 100644 --- a/codex-rs/core/src/tools/context.rs +++ b/codex-rs/core/src/tools/context.rs @@ -1,5 +1,6 @@ use crate::codex::Session; use crate::codex::TurnContext; +use crate::original_image_detail::sanitize_original_image_detail; use crate::tools::TELEMETRY_PREVIEW_MAX_BYTES; use crate::tools::TELEMETRY_PREVIEW_MAX_LINES; use crate::tools::TELEMETRY_PREVIEW_TRUNCATION_NOTICE; @@ -122,6 +123,7 @@ impl ToolOutput for CallToolResult { pub struct McpToolOutput { pub result: CallToolResult, pub wall_time: Duration, + pub original_image_detail_supported: bool, } impl ToolOutput for McpToolOutput { @@ -155,6 +157,10 @@ impl ToolOutput for McpToolOutput { impl McpToolOutput { fn response_payload(&self) -> FunctionCallOutputPayload { let mut payload = self.result.as_function_call_output_payload(); + if let Some(items) = payload.content_items_mut() { + sanitize_original_image_detail(self.original_image_detail_supported, items); + } + let wall_time_seconds = self.wall_time.as_secs_f64(); let header = format!("Wall time: {wall_time_seconds:.4} seconds\nOutput:"); diff --git a/codex-rs/core/src/tools/context_tests.rs b/codex-rs/core/src/tools/context_tests.rs index d45f351833..8df9159edd 100644 --- a/codex-rs/core/src/tools/context_tests.rs +++ b/codex-rs/core/src/tools/context_tests.rs @@ -98,6 +98,7 @@ fn mcp_tool_output_response_item_includes_wall_time() { meta: None, }, wall_time: std::time::Duration::from_millis(1250), + original_image_detail_supported: false, }; let response = output.to_response_item( @@ -149,6 +150,7 @@ fn mcp_tool_output_response_item_preserves_content_items() { meta: None, }, wall_time: std::time::Duration::from_millis(500), + original_image_detail_supported: false, }; let response = output.to_response_item( @@ -201,6 +203,7 @@ fn mcp_tool_output_code_mode_result_stays_raw_call_tool_result() { meta: None, }, wall_time: std::time::Duration::from_millis(1250), + original_image_detail_supported: false, }; let result = output.code_mode_result(&ToolPayload::Mcp { diff --git a/codex-rs/core/src/tools/handlers/list_dir.rs b/codex-rs/core/src/tools/handlers/list_dir.rs index be4e356c5c..22909c1d18 100644 --- a/codex-rs/core/src/tools/handlers/list_dir.rs +++ b/codex-rs/core/src/tools/handlers/list_dir.rs @@ -4,6 +4,7 @@ use std::fs::FileType; use std::path::Path; use std::path::PathBuf; +use codex_protocol::permissions::ReadDenyMatcher; use codex_utils_string::take_bytes_at_char_boundary; use serde::Deserialize; use tokio::fs; @@ -18,6 +19,8 @@ use crate::tools::registry::ToolKind; pub struct ListDirHandler; +const DENY_READ_POLICY_MESSAGE: &str = + "access denied: reading this path is blocked by filesystem deny_read policy"; const MAX_ENTRY_LENGTH: usize = 500; const INDENTATION_SPACES: usize = 2; @@ -52,7 +55,7 @@ impl ToolHandler for ListDirHandler { } async fn handle(&self, invocation: ToolInvocation) -> Result { - let ToolInvocation { payload, .. } = invocation; + let ToolInvocation { payload, turn, .. } = invocation; let arguments = match payload { ToolPayload::Function { arguments } => arguments, @@ -96,8 +99,20 @@ impl ToolHandler for ListDirHandler { "dir_path must be an absolute path".to_string(), )); } + let read_deny_matcher = ReadDenyMatcher::new(&turn.file_system_sandbox_policy, &turn.cwd); + if read_deny_matcher + .as_ref() + .is_some_and(|matcher| matcher.is_read_denied(&path)) + { + return Err(FunctionCallError::RespondToModel(format!( + "{DENY_READ_POLICY_MESSAGE}: `{}`", + path.display() + ))); + } - let entries = list_dir_slice(&path, offset, limit, depth).await?; + let entries = + list_dir_slice_with_policy(&path, offset, limit, depth, read_deny_matcher.as_ref()) + .await?; let mut output = Vec::with_capacity(entries.len() + 1); output.push(format!("Absolute path: {}", path.display())); output.extend(entries); @@ -105,14 +120,15 @@ impl ToolHandler for ListDirHandler { } } -async fn list_dir_slice( +async fn list_dir_slice_with_policy( path: &Path, offset: usize, limit: usize, depth: usize, + read_deny_matcher: Option<&ReadDenyMatcher>, ) -> Result, FunctionCallError> { let mut entries = Vec::new(); - collect_entries(path, Path::new(""), depth, &mut entries).await?; + collect_entries(path, Path::new(""), depth, read_deny_matcher, &mut entries).await?; if entries.is_empty() { return Ok(Vec::new()); @@ -148,6 +164,7 @@ async fn collect_entries( dir_path: &Path, relative_prefix: &Path, depth: usize, + read_deny_matcher: Option<&ReadDenyMatcher>, entries: &mut Vec, ) -> Result<(), FunctionCallError> { let mut queue = VecDeque::new(); @@ -163,6 +180,13 @@ async fn collect_entries( while let Some(entry) = read_dir.next_entry().await.map_err(|err| { FunctionCallError::RespondToModel(format!("failed to read directory: {err}")) })? { + let entry_path = entry.path(); + if let Some(read_deny_matcher) = read_deny_matcher + && read_deny_matcher.is_read_denied(&entry_path) + { + continue; + } + let file_type = entry.file_type().await.map_err(|err| { FunctionCallError::RespondToModel(format!("failed to inspect entry: {err}")) })?; @@ -179,7 +203,7 @@ async fn collect_entries( let sort_key = format_entry_name(&relative_path); let kind = DirEntryKind::from(&file_type); dir_entries.push(( - entry.path(), + entry_path, relative_path, kind, DirEntry { diff --git a/codex-rs/core/src/tools/handlers/list_dir_tests.rs b/codex-rs/core/src/tools/handlers/list_dir_tests.rs index ccc30b5326..7de107a18d 100644 --- a/codex-rs/core/src/tools/handlers/list_dir_tests.rs +++ b/codex-rs/core/src/tools/handlers/list_dir_tests.rs @@ -1,7 +1,21 @@ use super::*; +use codex_protocol::permissions::FileSystemAccessMode; +use codex_protocol::permissions::FileSystemPath; +use codex_protocol::permissions::FileSystemSandboxEntry; +use codex_protocol::permissions::FileSystemSandboxPolicy; +use codex_protocol::permissions::ReadDenyMatcher; use pretty_assertions::assert_eq; use tempfile::tempdir; +async fn list_dir_slice( + path: &Path, + offset: usize, + limit: usize, + depth: usize, +) -> Result, FunctionCallError> { + list_dir_slice_with_policy(path, offset, limit, depth, /*read_deny_matcher*/ None).await +} + #[tokio::test] async fn lists_directory_entries() { let temp = tempdir().expect("create tempdir"); @@ -258,3 +272,60 @@ async fn truncation_respects_sorted_order() -> anyhow::Result<()> { Ok(()) } + +#[tokio::test] +async fn hides_denied_entries_and_prunes_denied_subtrees() { + let temp = tempdir().expect("create tempdir"); + let dir_path = temp.path(); + let visible_dir = dir_path.join("visible"); + let denied_dir = dir_path.join("private"); + tokio::fs::create_dir(&visible_dir) + .await + .expect("create visible dir"); + tokio::fs::create_dir(&denied_dir) + .await + .expect("create denied dir"); + tokio::fs::write(visible_dir.join("ok.txt"), b"ok") + .await + .expect("write visible file"); + tokio::fs::write(denied_dir.join("secret.txt"), b"secret") + .await + .expect("write denied file"); + tokio::fs::write(dir_path.join("top_secret.txt"), b"secret") + .await + .expect("write denied top-level file"); + + let policy = FileSystemSandboxPolicy::restricted(vec![ + FileSystemSandboxEntry { + path: FileSystemPath::Path { + path: denied_dir.try_into().expect("absolute denied dir"), + }, + access: FileSystemAccessMode::None, + }, + FileSystemSandboxEntry { + path: FileSystemPath::Path { + path: dir_path + .join("top_secret.txt") + .try_into() + .expect("absolute denied file"), + }, + access: FileSystemAccessMode::None, + }, + ]); + + let read_deny_matcher = ReadDenyMatcher::new(&policy, dir_path); + let entries = list_dir_slice_with_policy( + dir_path, + /*offset*/ 1, + /*limit*/ 20, + /*depth*/ 3, + read_deny_matcher.as_ref(), + ) + .await + .expect("list directory"); + + assert_eq!( + entries, + vec!["visible/".to_string(), " ok.txt".to_string(),] + ); +} diff --git a/codex-rs/core/src/tools/handlers/mcp.rs b/codex-rs/core/src/tools/handlers/mcp.rs index cfe219f443..b24db734b9 100644 --- a/codex-rs/core/src/tools/handlers/mcp.rs +++ b/codex-rs/core/src/tools/handlers/mcp.rs @@ -3,6 +3,7 @@ use std::time::Instant; use crate::function_tool::FunctionCallError; use crate::mcp_tool_call::handle_mcp_tool_call; +use crate::original_image_detail::can_request_original_image_detail; use crate::tools::context::McpToolOutput; use crate::tools::context::ToolInvocation; use crate::tools::context::ToolPayload; @@ -56,6 +57,7 @@ impl ToolHandler for McpHandler { Ok(McpToolOutput { result, wall_time: started.elapsed(), + original_image_detail_supported: can_request_original_image_detail(&turn.model_info), }) } } diff --git a/codex-rs/core/src/tools/handlers/mcp_resource.rs b/codex-rs/core/src/tools/handlers/mcp_resource.rs index 69058a6226..d7d0256dda 100644 --- a/codex-rs/core/src/tools/handlers/mcp_resource.rs +++ b/codex-rs/core/src/tools/handlers/mcp_resource.rs @@ -562,6 +562,7 @@ async fn emit_tool_call_begin( EventMsg::McpToolCallBegin(McpToolCallBeginEvent { call_id: call_id.to_string(), invocation, + mcp_app_resource_uri: None, }), ) .await; @@ -581,6 +582,7 @@ async fn emit_tool_call_end( EventMsg::McpToolCallEnd(McpToolCallEndEvent { call_id: call_id.to_string(), invocation, + mcp_app_resource_uri: None, duration, result, }), diff --git a/codex-rs/core/src/tools/handlers/mod.rs b/codex-rs/core/src/tools/handlers/mod.rs index 022e02c39e..6b48680b8a 100644 --- a/codex-rs/core/src/tools/handlers/mod.rs +++ b/codex-rs/core/src/tools/handlers/mod.rs @@ -15,6 +15,7 @@ mod shell; mod test_sync; mod tool_search; mod tool_suggest; +mod unavailable_tool; pub(crate) mod unified_exec; mod view_image; @@ -49,6 +50,8 @@ pub use shell::ShellHandler; pub use test_sync::TestSyncHandler; pub use tool_search::ToolSearchHandler; pub use tool_suggest::ToolSuggestHandler; +pub use unavailable_tool::UnavailableToolHandler; +pub(crate) use unavailable_tool::unavailable_tool_message; pub use unified_exec::UnifiedExecHandler; pub use view_image::ViewImageHandler; diff --git a/codex-rs/core/src/tools/handlers/multi_agents_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_tests.rs index e35d6a06fb..10f41d97eb 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_tests.rs @@ -1492,24 +1492,25 @@ async fn multi_agent_v2_followup_task_interrupts_busy_child_without_losing_messa let session = Arc::new(session); let turn = Arc::new(turn); - SpawnAgentHandlerV2 - .handle(invocation( - session.clone(), - turn.clone(), - "spawn_agent", - function_payload(json!({ - "message": "boot worker", - "task_name": "worker" - })), - )) - .await - .expect("spawn worker"); + let worker_path = AgentPath::try_from("/root/worker").expect("worker path"); let agent_id = session .services .agent_control - .resolve_agent_reference(session.conversation_id, &turn.session_source, "worker") + .spawn_agent_with_metadata( + (*turn.config).clone(), + Op::CleanBackgroundTerminals, + Some(SessionSource::SubAgent(SubAgentSource::ThreadSpawn { + parent_thread_id: root.thread_id, + depth: 1, + agent_path: Some(worker_path.clone()), + agent_nickname: None, + agent_role: None, + })), + crate::agent::control::SpawnAgentOptions::default(), + ) .await - .expect("worker should resolve"); + .expect("worker spawn should succeed") + .thread_id; let thread = manager .get_thread(agent_id) .await @@ -1566,7 +1567,7 @@ async fn multi_agent_v2_followup_task_interrupts_busy_child_without_losing_messa &thread, &InterAgentCommunication::new( AgentPath::root(), - AgentPath::try_from("/root/worker").expect("agent path"), + worker_path, Vec::new(), "continue".to_string(), /*trigger_turn*/ true, diff --git a/codex-rs/core/src/tools/handlers/unavailable_tool.rs b/codex-rs/core/src/tools/handlers/unavailable_tool.rs new file mode 100644 index 0000000000..eb00cf8ff1 --- /dev/null +++ b/codex-rs/core/src/tools/handlers/unavailable_tool.rs @@ -0,0 +1,44 @@ +use crate::function_tool::FunctionCallError; +use crate::tools::context::FunctionToolOutput; +use crate::tools::context::ToolInvocation; +use crate::tools::context::ToolPayload; +use crate::tools::registry::ToolHandler; +use crate::tools::registry::ToolKind; + +pub struct UnavailableToolHandler; + +pub(crate) fn unavailable_tool_message( + tool_name: impl std::fmt::Display, + next_step: &str, +) -> String { + format!( + "Tool `{tool_name}` is not currently available. It appeared in earlier tool calls in this conversation, but its implementation is not available in the current request. {next_step}" + ) +} + +impl ToolHandler for UnavailableToolHandler { + type Output = FunctionToolOutput; + + fn kind(&self) -> ToolKind { + ToolKind::Function + } + + async fn handle(&self, invocation: ToolInvocation) -> Result { + let ToolInvocation { + tool_name, payload, .. + } = invocation; + + match payload { + ToolPayload::Function { .. } => Ok(FunctionToolOutput::from_text( + unavailable_tool_message( + tool_name.display(), + "Retry after the tool becomes available or ask the user to re-enable it.", + ), + Some(false), + )), + _ => Err(FunctionCallError::RespondToModel( + "unavailable tool handler received unsupported payload".to_string(), + )), + } + } +} diff --git a/codex-rs/core/src/tools/js_repl/kernel.js b/codex-rs/core/src/tools/js_repl/kernel.js index 5d31818521..3e5cf855f7 100644 --- a/codex-rs/core/src/tools/js_repl/kernel.js +++ b/codex-rs/core/src/tools/js_repl/kernel.js @@ -1330,6 +1330,13 @@ function normalizeMcpImageData(data, mimeType) { return `data:${normalizedMimeType};base64,${data}`; } +function parseMcpImageDetail(meta) { + if (!isPlainObject(meta) || meta["codex/imageDetail"] !== "original") { + return undefined; + } + return "original"; +} + function parseMcpToolResult(result) { if (typeof result === "string") { return { images: [], textCount: result.length > 0 ? 1 : 0 }; @@ -1362,6 +1369,7 @@ function parseMcpToolResult(result) { if (item.type === "image") { images.push({ image_url: normalizeMcpImageData(item.data, item.mimeType ?? item.mime_type), + detail: parseMcpImageDetail(item._meta), }); continue; } diff --git a/codex-rs/core/src/tools/js_repl/mod.rs b/codex-rs/core/src/tools/js_repl/mod.rs index b728009499..48bfb8ac41 100644 --- a/codex-rs/core/src/tools/js_repl/mod.rs +++ b/codex-rs/core/src/tools/js_repl/mod.rs @@ -45,6 +45,8 @@ use codex_sandboxing::SandboxCommand; use codex_sandboxing::SandboxManager; use codex_sandboxing::SandboxTransformRequest; use codex_sandboxing::SandboxablePreference; +use codex_tools::ResponsesApiNamespaceTool; +use codex_tools::ToolName; use codex_tools::ToolSpec; use codex_utils_output_truncation::TruncationPolicy; use codex_utils_output_truncation::truncate_text; @@ -1031,18 +1033,13 @@ impl JsReplManager { } let sandbox = SandboxManager::new(); - let has_managed_network_requirements = turn - .config - .config_layer_stack - .requirements_toml() - .network - .is_some(); + let managed_network_active = turn.network.is_some(); let sandbox_type = sandbox.select_initial( &turn.file_system_sandbox_policy, turn.network_sandbox_policy, SandboxablePreference::Auto, turn.windows_sandbox_level, - has_managed_network_requirements, + managed_network_active, ); let command = SandboxCommand { program: node_path.into_os_string(), @@ -1065,7 +1062,7 @@ impl JsReplManager { file_system_policy: &turn.file_system_sandbox_policy, network_policy: turn.network_sandbox_policy, sandbox: sandbox_type, - enforce_managed_network: has_managed_network_requirements, + enforce_managed_network: managed_network_active, network: None, sandbox_policy_cwd: &turn.cwd, codex_linux_sandbox_exe: turn.codex_linux_sandbox_exe.as_deref(), @@ -1566,6 +1563,7 @@ impl JsReplManager { crate::tools::router::ToolRouterParams { deferred_mcp_tools: None, mcp_tools: Some(mcp_tools), + unavailable_called_tools: Vec::new(), // JS REPL dispatches nested tool calls directly, not through // `ToolCallRuntime`'s parallel scheduling lock. parallel_mcp_server_names: std::collections::HashSet::new(), @@ -1574,29 +1572,67 @@ impl JsReplManager { }, ); - let payload = if let Some(tool_info) = exec + let specs = router.specs(); + let requested_tool_name = specs + .iter() + .find_map(|spec| match spec { + ToolSpec::Function(tool) if tool.name == req.tool_name => { + Some(ToolName::plain(req.tool_name.clone())) + } + ToolSpec::Freeform(tool) if tool.name == req.tool_name => { + Some(ToolName::plain(req.tool_name.clone())) + } + ToolSpec::Namespace(namespace) => { + namespace.tools.iter().find_map(|tool| match tool { + ResponsesApiNamespaceTool::Function(tool) => { + let tool_name = + ToolName::namespaced(namespace.name.clone(), tool.name.clone()); + (tool_name.display() == req.tool_name).then_some(tool_name) + } + }) + } + ToolSpec::LocalShell {} + | ToolSpec::ImageGeneration { .. } + | ToolSpec::ToolSearch { .. } + | ToolSpec::WebSearch { .. } + | ToolSpec::Function(_) + | ToolSpec::Freeform(_) => None, + }) + .unwrap_or_else(|| ToolName::plain(req.tool_name.clone())); + let (tool_call_name, payload) = if let Some(tool_info) = exec .session - .resolve_mcp_tool_info(&req.tool_name, /*namespace*/ None) + .resolve_mcp_tool_info(&requested_tool_name) .await { - crate::tools::context::ToolPayload::Mcp { - server: tool_info.server_name, - tool: tool_info.tool.name.to_string(), - raw_arguments: req.arguments.clone(), - } - } else if is_freeform_tool(&router.specs(), &req.tool_name) { - crate::tools::context::ToolPayload::Custom { - input: req.arguments.clone(), - } + ( + tool_info.canonical_tool_name(), + crate::tools::context::ToolPayload::Mcp { + server: tool_info.server_name, + tool: tool_info.tool.name.to_string(), + raw_arguments: req.arguments.clone(), + }, + ) + } else if matches!( + router.find_spec(&requested_tool_name), + Some(ToolSpec::Freeform(_)) + ) { + ( + requested_tool_name, + crate::tools::context::ToolPayload::Custom { + input: req.arguments.clone(), + }, + ) } else { - crate::tools::context::ToolPayload::Function { - arguments: req.arguments.clone(), - } + ( + requested_tool_name, + crate::tools::context::ToolPayload::Function { + arguments: req.arguments.clone(), + }, + ) }; - let tool_name = req.tool_name.clone(); let call = crate::tools::router::ToolCall { - tool_name: codex_tools::ToolName::plain(tool_name.clone()), + tool_name: tool_call_name, call_id: req.id.clone(), payload, }; @@ -1755,12 +1791,6 @@ fn split_exec_result_content_items( } } -fn is_freeform_tool(specs: &[ToolSpec], name: &str) -> bool { - specs - .iter() - .any(|spec| spec.name() == name && matches!(spec, ToolSpec::Freeform(_))) -} - fn is_js_repl_internal_tool(name: &str) -> bool { matches!(name, "js_repl" | "js_repl_reset") } diff --git a/codex-rs/core/src/tools/network_approval.rs b/codex-rs/core/src/tools/network_approval.rs index c91b6df7ca..de689ea25d 100644 --- a/codex-rs/core/src/tools/network_approval.rs +++ b/codex-rs/core/src/tools/network_approval.rs @@ -578,11 +578,11 @@ pub(crate) fn build_network_policy_decider( pub(crate) async fn begin_network_approval( session: &Session, turn_id: &str, - has_managed_network_requirements: bool, + managed_network_active: bool, spec: Option, ) -> Option { let spec = spec?; - if !has_managed_network_requirements || spec.network.is_none() { + if !managed_network_active || spec.network.is_none() { return None; } diff --git a/codex-rs/core/src/tools/orchestrator.rs b/codex-rs/core/src/tools/orchestrator.rs index 36450da3f7..f9b2a64966 100644 --- a/codex-rs/core/src/tools/orchestrator.rs +++ b/codex-rs/core/src/tools/orchestrator.rs @@ -55,7 +55,7 @@ impl ToolOrchestrator { req: &Rq, tool_ctx: &ToolCtx, attempt: &SandboxAttempt<'_>, - has_managed_network_requirements: bool, + managed_network_active: bool, ) -> (Result, Option) where T: ToolRuntime, @@ -63,7 +63,7 @@ impl ToolOrchestrator { let network_approval = begin_network_approval( &tool_ctx.session, &tool_ctx.turn.sub_id, - has_managed_network_requirements, + managed_network_active, tool.network_approval_spec(req, tool_ctx), ) .await; @@ -180,12 +180,7 @@ impl ToolOrchestrator { } // 2) First attempt under the selected sandbox. - let has_managed_network_requirements = turn_ctx - .config - .config_layer_stack - .requirements_toml() - .network - .is_some(); + let managed_network_active = turn_ctx.network.is_some(); let initial_sandbox = match tool.sandbox_mode_for_first_attempt(req) { SandboxOverride::BypassSandboxFirstAttempt => SandboxType::None, SandboxOverride::NoOverride => self.sandbox.select_initial( @@ -193,7 +188,7 @@ impl ToolOrchestrator { turn_ctx.network_sandbox_policy, tool.sandbox_preference(), turn_ctx.windows_sandbox_level, - has_managed_network_requirements, + managed_network_active, ), }; @@ -204,7 +199,7 @@ impl ToolOrchestrator { policy: &turn_ctx.sandbox_policy, file_system_policy: &turn_ctx.file_system_sandbox_policy, network_policy: turn_ctx.network_sandbox_policy, - enforce_managed_network: has_managed_network_requirements, + enforce_managed_network: managed_network_active, manager: &self.sandbox, sandbox_cwd: &turn_ctx.cwd, codex_linux_sandbox_exe: turn_ctx.codex_linux_sandbox_exe.as_ref(), @@ -221,7 +216,7 @@ impl ToolOrchestrator { req, tool_ctx, &initial_attempt, - has_managed_network_requirements, + managed_network_active, ) .await; match first_result { @@ -236,7 +231,7 @@ impl ToolOrchestrator { output, network_policy_decision, }))) => { - let network_approval_context = if has_managed_network_requirements { + let network_approval_context = if managed_network_active { network_policy_decision .as_ref() .and_then(network_approval_context_from_payload) @@ -341,7 +336,7 @@ impl ToolOrchestrator { policy: &turn_ctx.sandbox_policy, file_system_policy: &turn_ctx.file_system_sandbox_policy, network_policy: turn_ctx.network_sandbox_policy, - enforce_managed_network: has_managed_network_requirements, + enforce_managed_network: managed_network_active, manager: &self.sandbox, sandbox_cwd: &turn_ctx.cwd, codex_linux_sandbox_exe: None, @@ -359,7 +354,7 @@ impl ToolOrchestrator { req, tool_ctx, &escalated_attempt, - has_managed_network_requirements, + managed_network_active, ) .await; retry_result.map(|output| OrchestratorRunResult { diff --git a/codex-rs/core/src/tools/parallel.rs b/codex-rs/core/src/tools/parallel.rs index a1965db5cc..b4dbfaa6cf 100644 --- a/codex-rs/core/src/tools/parallel.rs +++ b/codex-rs/core/src/tools/parallel.rs @@ -48,7 +48,7 @@ impl ToolCallRuntime { } } - pub(crate) fn find_spec(&self, tool_name: &str) -> Option { + pub(crate) fn find_spec(&self, tool_name: &codex_tools::ToolName) -> Option { self.router.find_spec(tool_name) } diff --git a/codex-rs/core/src/tools/router.rs b/codex-rs/core/src/tools/router.rs index 645650525c..ef7ae32d4b 100644 --- a/codex-rs/core/src/tools/router.rs +++ b/codex-rs/core/src/tools/router.rs @@ -16,6 +16,7 @@ use codex_protocol::models::SearchToolCallParams; use codex_protocol::models::ShellToolCallParams; use codex_tools::ConfiguredToolSpec; use codex_tools::DiscoverableTool; +use codex_tools::ResponsesApiNamespaceTool; use codex_tools::ToolName; use codex_tools::ToolSpec; use codex_tools::ToolsConfig; @@ -43,6 +44,7 @@ pub struct ToolRouter { pub(crate) struct ToolRouterParams<'a> { pub(crate) mcp_tools: Option>, pub(crate) deferred_mcp_tools: Option>, + pub(crate) unavailable_called_tools: Vec, pub(crate) parallel_mcp_server_names: HashSet, pub(crate) discoverable_tools: Option>, pub(crate) dynamic_tools: &'a [DynamicToolSpec], @@ -53,6 +55,7 @@ impl ToolRouter { let ToolRouterParams { mcp_tools, deferred_mcp_tools, + unavailable_called_tools, parallel_mcp_server_names, discoverable_tools, dynamic_tools, @@ -61,6 +64,7 @@ impl ToolRouter { config, mcp_tools, deferred_mcp_tools, + unavailable_called_tools, discoverable_tools, dynamic_tools, ); @@ -102,20 +106,48 @@ impl ToolRouter { self.model_visible_specs.clone() } - pub fn find_spec(&self, tool_name: &str) -> Option { - self.specs - .iter() - .find(|config| config.name() == tool_name) - .map(|config| config.spec.clone()) + pub fn find_spec(&self, tool_name: &ToolName) -> Option { + self.specs.iter().find_map(|config| match &config.spec { + ToolSpec::Function(tool) + if tool_name.namespace.is_none() && tool.name == tool_name.name => + { + Some(config.spec.clone()) + } + ToolSpec::Freeform(tool) + if tool_name.namespace.is_none() && tool.name == tool_name.name => + { + Some(config.spec.clone()) + } + ToolSpec::Namespace(namespace) => namespace.tools.iter().find_map(|tool| match tool { + ResponsesApiNamespaceTool::Function(tool) + if tool_name.namespace.as_deref() == Some(namespace.name.as_str()) + && tool.name == tool_name.name => + { + Some(ToolSpec::Function(tool.clone())) + } + _ => None, + }), + _ => None, + }) } fn configured_tool_supports_parallel(&self, tool_name: &ToolName) -> bool { - tool_name.namespace.is_none() - && self - .specs - .iter() - .filter(|config| config.supports_parallel_tool_calls) - .any(|config| config.name() == tool_name.name.as_str()) + if tool_name.namespace.is_some() { + return false; + } + + self.specs + .iter() + .filter(|config| config.supports_parallel_tool_calls) + .any(|config| match &config.spec { + ToolSpec::Function(tool) => tool.name == tool_name.name.as_str(), + ToolSpec::Freeform(tool) => tool.name == tool_name.name.as_str(), + ToolSpec::Namespace(_) + | ToolSpec::ToolSearch { .. } + | ToolSpec::LocalShell {} + | ToolSpec::ImageGeneration { .. } + | ToolSpec::WebSearch { .. } => false, + }) } pub fn tool_supports_parallel(&self, call: &ToolCall) -> bool { @@ -141,16 +173,10 @@ impl ToolRouter { call_id, .. } => { - let mcp_tool = session - .resolve_mcp_tool_info(&name, namespace.as_deref()) - .await; - let tool_name = match namespace { - Some(namespace) => ToolName::namespaced(namespace, name), - None => ToolName::plain(name), - }; - if let Some(tool_info) = mcp_tool { + let tool_name = ToolName::new(namespace, name); + if let Some(tool_info) = session.resolve_mcp_tool_info(&tool_name).await { Ok(Some(ToolCall { - tool_name, + tool_name: tool_info.canonical_tool_name(), call_id, payload: ToolPayload::Mcp { server: tool_info.server_name, diff --git a/codex-rs/core/src/tools/router_tests.rs b/codex-rs/core/src/tools/router_tests.rs index f4c0fb7c5a..d93796ab64 100644 --- a/codex-rs/core/src/tools/router_tests.rs +++ b/codex-rs/core/src/tools/router_tests.rs @@ -33,6 +33,7 @@ async fn js_repl_tools_only_blocks_direct_tool_calls() -> anyhow::Result<()> { ToolRouterParams { deferred_mcp_tools, mcp_tools: Some(mcp_tools), + unavailable_called_tools: Vec::new(), parallel_mcp_server_names: HashSet::new(), discoverable_tools: None, dynamic_tools: turn.dynamic_tools.as_slice(), @@ -86,6 +87,7 @@ async fn js_repl_tools_only_allows_js_repl_source_calls() -> anyhow::Result<()> ToolRouterParams { deferred_mcp_tools, mcp_tools: Some(mcp_tools), + unavailable_called_tools: Vec::new(), parallel_mcp_server_names: HashSet::new(), discoverable_tools: None, dynamic_tools: turn.dynamic_tools.as_slice(), @@ -132,6 +134,7 @@ async fn js_repl_tools_only_blocks_namespaced_js_repl_tool() -> anyhow::Result<( ToolRouterParams { deferred_mcp_tools: None, mcp_tools: None, + unavailable_called_tools: Vec::new(), parallel_mcp_server_names: HashSet::new(), discoverable_tools: None, dynamic_tools: turn.dynamic_tools.as_slice(), @@ -182,6 +185,7 @@ async fn parallel_support_does_not_match_namespaced_local_tool_names() -> anyhow ToolRouterParams { deferred_mcp_tools: None, mcp_tools: Some(mcp_tools), + unavailable_called_tools: Vec::new(), parallel_mcp_server_names: HashSet::new(), discoverable_tools: None, dynamic_tools: turn.dynamic_tools.as_slice(), @@ -254,6 +258,7 @@ async fn mcp_parallel_support_uses_exact_payload_server() -> anyhow::Result<()> ToolRouterParams { deferred_mcp_tools: None, mcp_tools: None, + unavailable_called_tools: Vec::new(), parallel_mcp_server_names: HashSet::from(["echo".to_string()]), discoverable_tools: None, dynamic_tools: turn.dynamic_tools.as_slice(), diff --git a/codex-rs/core/src/tools/spec.rs b/codex-rs/core/src/tools/spec.rs index 188be42ba1..1a1a9f2e5f 100644 --- a/codex-rs/core/src/tools/spec.rs +++ b/codex-rs/core/src/tools/spec.rs @@ -7,16 +7,23 @@ use crate::tools::handlers::multi_agents_common::MIN_WAIT_TIMEOUT_MS; use crate::tools::registry::ToolRegistryBuilder; use codex_mcp::ToolInfo; use codex_protocol::dynamic_tools::DynamicToolSpec; +use codex_tools::AdditionalProperties; use codex_tools::DiscoverableTool; +use codex_tools::JsonSchema; +use codex_tools::ResponsesApiTool; use codex_tools::ToolHandlerKind; +use codex_tools::ToolName; use codex_tools::ToolNamespace; use codex_tools::ToolRegistryPlanDeferredTool; +use codex_tools::ToolRegistryPlanMcpTool; use codex_tools::ToolRegistryPlanParams; use codex_tools::ToolUserShellType; use codex_tools::ToolsConfig; use codex_tools::WaitAgentTimeoutOptions; +use codex_tools::augment_tool_spec_for_code_mode; use codex_tools::build_tool_registry_plan; use std::collections::HashMap; +use std::collections::HashSet; use std::sync::Arc; pub(crate) fn tool_user_shell_type(user_shell: &Shell) -> ToolUserShellType { @@ -29,25 +36,31 @@ pub(crate) fn tool_user_shell_type(user_shell: &Shell) -> ToolUserShellType { } } -struct McpToolPlanInputs { - mcp_tools: HashMap, +struct McpToolPlanInputs<'a> { + mcp_tools: Vec>, tool_namespaces: HashMap, } -fn map_mcp_tools_for_plan(mcp_tools: &HashMap) -> McpToolPlanInputs { +fn map_mcp_tools_for_plan(mcp_tools: &HashMap) -> McpToolPlanInputs<'_> { McpToolPlanInputs { mcp_tools: mcp_tools - .iter() - .map(|(name, tool)| (name.clone(), tool.tool.clone())) + .values() + .map(|tool| ToolRegistryPlanMcpTool { + name: tool.canonical_tool_name(), + tool: &tool.tool, + }) .collect(), tool_namespaces: mcp_tools - .iter() - .map(|(name, tool)| { + .values() + .map(|tool| { ( - name.clone(), + tool.callable_namespace.clone(), ToolNamespace { name: tool.callable_namespace.clone(), - description: tool.server_instructions.clone(), + description: tool + .connector_description + .clone() + .or_else(|| tool.server_instructions.clone()), }, ) }) @@ -59,6 +72,7 @@ pub(crate) fn build_specs_with_discoverable_tools( config: &ToolsConfig, mcp_tools: Option>, deferred_mcp_tools: Option>, + unavailable_called_tools: Vec, discoverable_tools: Option>, dynamic_tools: &[DynamicToolSpec], ) -> ToolRegistryBuilder { @@ -79,6 +93,7 @@ pub(crate) fn build_specs_with_discoverable_tools( use crate::tools::handlers::TestSyncHandler; use crate::tools::handlers::ToolSearchHandler; use crate::tools::handlers::ToolSuggestHandler; + use crate::tools::handlers::UnavailableToolHandler; use crate::tools::handlers::UnifiedExecHandler; use crate::tools::handlers::ViewImageHandler; use crate::tools::handlers::multi_agents::CloseAgentHandler; @@ -92,6 +107,7 @@ pub(crate) fn build_specs_with_discoverable_tools( use crate::tools::handlers::multi_agents_v2::SendMessageHandler as SendMessageHandlerV2; use crate::tools::handlers::multi_agents_v2::SpawnAgentHandler as SpawnAgentHandlerV2; use crate::tools::handlers::multi_agents_v2::WaitAgentHandler as WaitAgentHandlerV2; + use crate::tools::handlers::unavailable_tool_message; let mut builder = ToolRegistryBuilder::new(); let mcp_tool_plan_inputs = mcp_tools.as_ref().map(map_mcp_tools_for_plan); @@ -99,8 +115,7 @@ pub(crate) fn build_specs_with_discoverable_tools( tools .values() .map(|tool| ToolRegistryPlanDeferredTool { - tool_name: tool.callable_name.as_str(), - tool_namespace: tool.callable_namespace.as_str(), + name: tool.canonical_tool_name(), server_name: tool.server_name.as_str(), connector_name: tool.connector_name.as_deref(), connector_description: tool.connector_description.as_deref(), @@ -114,7 +129,7 @@ pub(crate) fn build_specs_with_discoverable_tools( ToolRegistryPlanParams { mcp_tools: mcp_tool_plan_inputs .as_ref() - .map(|inputs| &inputs.mcp_tools), + .map(|inputs| inputs.mcp_tools.as_slice()), deferred_mcp_tools: deferred_mcp_tool_sources.as_deref(), tool_namespaces: mcp_tool_plan_inputs .as_ref() @@ -148,6 +163,12 @@ pub(crate) fn build_specs_with_discoverable_tools( let code_mode_wait_handler = Arc::new(CodeModeWaitHandler); let js_repl_handler = Arc::new(JsReplHandler); let js_repl_reset_handler = Arc::new(JsReplResetHandler); + let unavailable_tool_handler = Arc::new(UnavailableToolHandler); + let mut existing_spec_names = plan + .specs + .iter() + .map(|configured_tool| configured_tool.name().to_string()) + .collect::>(); for spec in plan.specs { if spec.supports_parallel_tool_calls { @@ -272,6 +293,34 @@ pub(crate) fn build_specs_with_discoverable_tools( builder.register_handler(name.clone(), mcp_handler.clone()); } } + + for unavailable_tool in unavailable_called_tools { + let tool_name = unavailable_tool.display(); + if existing_spec_names.insert(tool_name.clone()) { + let spec = codex_tools::ToolSpec::Function(ResponsesApiTool { + name: tool_name.clone(), + description: unavailable_tool_message( + &tool_name, + "Calling this placeholder returns an error explaining that the tool is unavailable.", + ), + strict: false, + parameters: JsonSchema::object( + Default::default(), + /*required*/ None, + Some(AdditionalProperties::Boolean(false)), + ), + output_schema: None, + defer_loading: None, + }); + let spec = if config.code_mode_enabled { + augment_tool_spec_for_code_mode(spec) + } else { + spec + }; + builder.push_spec(spec); + } + builder.register_handler(unavailable_tool, unavailable_tool_handler.clone()); + } builder } diff --git a/codex-rs/core/src/tools/spec_tests.rs b/codex-rs/core/src/tools/spec_tests.rs index b0fe3f8436..9de9c81a92 100644 --- a/codex-rs/core/src/tools/spec_tests.rs +++ b/codex-rs/core/src/tools/spec_tests.rs @@ -16,9 +16,11 @@ use codex_protocol::openai_models::ConfigShellToolType; use codex_protocol::openai_models::ModelInfo; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SessionSource; +use codex_tools::AdditionalProperties; use codex_tools::ConfiguredToolSpec; use codex_tools::DiscoverableTool; use codex_tools::JsonSchema; +use codex_tools::ResponsesApiNamespaceTool; use codex_tools::ResponsesApiTool; use codex_tools::ShellCommandBackendConfig; use codex_tools::TOOL_SEARCH_TOOL_NAME; @@ -67,6 +69,25 @@ fn mcp_tool_info(tool: rmcp::model::Tool) -> ToolInfo { } } +fn mcp_tool_info_with_display_name(display_name: &str, tool: rmcp::model::Tool) -> ToolInfo { + let (callable_namespace, callable_name) = display_name + .rsplit_once('/') + .map(|(namespace, callable_name)| (format!("{namespace}/"), callable_name.to_string())) + .unwrap_or_else(|| ("".to_string(), display_name.to_string())); + + ToolInfo { + server_name: "test_server".to_string(), + callable_name, + callable_namespace, + server_instructions: None, + tool, + connector_id: None, + connector_name: None, + plugin_display_names: Vec::new(), + connector_description: None, + } +} + fn discoverable_connector(id: &str, name: &str, description: &str) -> DiscoverableTool { let slug = name.replace(' ', "-").to_lowercase(); DiscoverableTool::Connector(Box::new(AppInfo { @@ -86,8 +107,8 @@ fn discoverable_connector(id: &str, name: &str, description: &str) -> Discoverab })) } -fn search_capable_model_info() -> ModelInfo { - let config = test_config(); +async fn search_capable_model_info() -> ModelInfo { + let config = test_config().await; let mut model_info = construct_model_info_offline("gpt-5-codex", &config); model_info.supports_search_tool = true; model_info @@ -109,8 +130,11 @@ fn deferred_responses_api_tool_serializes_with_defer_loading() { ); let serialized = serde_json::to_value(ToolSpec::Function( - mcp_tool_to_deferred_responses_api_tool("mcp__codex_apps__lookup_order".to_string(), &tool) - .expect("convert deferred tool"), + mcp_tool_to_deferred_responses_api_tool( + &ToolName::namespaced("mcp__codex_apps__", "lookup_order"), + &tool, + ) + .expect("convert deferred tool"), )) .expect("serialize deferred tool"); @@ -118,7 +142,7 @@ fn deferred_responses_api_tool_serializes_with_defer_loading() { serialized, serde_json::json!({ "type": "function", - "name": "mcp__codex_apps__lookup_order", + "name": "lookup_order", "description": "Look up an order", "strict": false, "defer_loading": true, @@ -173,8 +197,27 @@ fn find_tool<'a>(tools: &'a [ConfiguredToolSpec], expected_name: &str) -> &'a Co .unwrap_or_else(|| panic!("expected tool {expected_name}")) } -fn multi_agent_v2_tools_config() -> ToolsConfig { - let config = test_config(); +fn find_namespace_function_tool<'a>( + tools: &'a [ConfiguredToolSpec], + expected_namespace: &str, + expected_name: &str, +) -> &'a ResponsesApiTool { + let namespace_tool = find_tool(tools, expected_namespace); + let ToolSpec::Namespace(namespace) = &namespace_tool.spec else { + panic!("expected namespace tool {expected_namespace}"); + }; + namespace + .tools + .iter() + .find_map(|tool| match tool { + ResponsesApiNamespaceTool::Function(tool) if tool.name == expected_name => Some(tool), + _ => None, + }) + .unwrap_or_else(|| panic!("expected tool {expected_namespace}{expected_name} in namespace")) +} + +async fn multi_agent_v2_tools_config() -> ToolsConfig { + let config = test_config().await; let model_info = construct_model_info_offline("gpt-5-codex", &config); let mut features = Features::with_defaults(); features.enable(Feature::Collab); @@ -207,8 +250,8 @@ fn multi_agent_v2_spawn_agent_description(tools_config: &ToolsConfig) -> String description.clone() } -fn model_info_from_models_json(slug: &str) -> ModelInfo { - let config = test_config(); +async fn model_info_from_models_json(slug: &str) -> ModelInfo { + let config = test_config().await; let response = bundled_models_response() .unwrap_or_else(|err| panic!("bundled models.json should parse: {err}")); let model = response @@ -225,19 +268,36 @@ fn build_specs( mcp_tools: Option>, deferred_mcp_tools: Option>, dynamic_tools: &[DynamicToolSpec], +) -> ToolRegistryBuilder { + build_specs_with_unavailable_tools( + config, + mcp_tools, + deferred_mcp_tools, + Vec::new(), + dynamic_tools, + ) +} + +fn build_specs_with_unavailable_tools( + config: &ToolsConfig, + mcp_tools: Option>, + deferred_mcp_tools: Option>, + unavailable_called_tools: Vec, + dynamic_tools: &[DynamicToolSpec], ) -> ToolRegistryBuilder { build_specs_with_discoverable_tools( config, mcp_tools, deferred_mcp_tools, + unavailable_called_tools, /*discoverable_tools*/ None, dynamic_tools, ) } -#[test] -fn model_provided_unified_exec_is_blocked_for_windows_sandboxed_policies() { - let mut model_info = model_info_from_models_json("gpt-5-codex"); +#[tokio::test] +async fn model_provided_unified_exec_is_blocked_for_windows_sandboxed_policies() { + let mut model_info = model_info_from_models_json("gpt-5-codex").await; model_info.shell_type = ConfigShellToolType::UnifiedExec; let features = Features::with_defaults(); let available_models = Vec::new(); @@ -260,9 +320,9 @@ fn model_provided_unified_exec_is_blocked_for_windows_sandboxed_policies() { assert_eq!(config.shell_type, expected_shell_type); } -#[test] -fn get_memory_requires_feature_flag() { - let config = test_config(); +#[tokio::test] +async fn get_memory_requires_feature_flag() { + let config = test_config().await; let model_info = construct_model_info_offline("gpt-5-codex", &config); let mut features = Features::with_defaults(); features.disable(Feature::MemoryTool); @@ -290,14 +350,14 @@ fn get_memory_requires_feature_flag() { ); } -fn assert_model_tools( +async fn assert_model_tools( model_slug: &str, features: &Features, web_search_mode: Option, expected_tools: &[&str], ) { - let _config = test_config(); - let model_info = model_info_from_models_json(model_slug); + let _config = test_config().await; + let model_info = model_info_from_models_json(model_slug).await; let available_models = Vec::new(); let tools_config = ToolsConfig::new(&ToolsConfigParams { model_info: &model_info, @@ -314,6 +374,7 @@ fn assert_model_tools( ToolRouterParams { mcp_tools: None, deferred_mcp_tools: None, + unavailable_called_tools: Vec::new(), parallel_mcp_server_names: std::collections::HashSet::new(), discoverable_tools: None, dynamic_tools: &[], @@ -327,7 +388,7 @@ fn assert_model_tools( assert_eq!(&tool_names, &expected_tools,); } -fn assert_default_model_tools( +async fn assert_default_model_tools( model_slug: &str, features: &Features, web_search_mode: Option, @@ -340,11 +401,11 @@ fn assert_default_model_tools( vec![shell_tool] }; expected.extend(expected_tail); - assert_model_tools(model_slug, features, web_search_mode, &expected); + assert_model_tools(model_slug, features, web_search_mode, &expected).await; } -#[test] -fn test_build_specs_gpt5_codex_default() { +#[tokio::test] +async fn test_build_specs_gpt5_codex_default() { let features = Features::with_defaults(); assert_default_model_tools( "gpt-5-codex", @@ -356,6 +417,7 @@ fn test_build_specs_gpt5_codex_default() { "request_user_input", "apply_patch", "web_search", + "image_generation", "view_image", "spawn_agent", "send_input", @@ -363,11 +425,12 @@ fn test_build_specs_gpt5_codex_default() { "wait_agent", "close_agent", ], - ); + ) + .await; } -#[test] -fn test_build_specs_gpt51_codex_default() { +#[tokio::test] +async fn test_build_specs_gpt51_codex_default() { let features = Features::with_defaults(); assert_default_model_tools( "gpt-5.1-codex", @@ -379,6 +442,7 @@ fn test_build_specs_gpt51_codex_default() { "request_user_input", "apply_patch", "web_search", + "image_generation", "view_image", "spawn_agent", "send_input", @@ -386,11 +450,12 @@ fn test_build_specs_gpt51_codex_default() { "wait_agent", "close_agent", ], - ); + ) + .await; } -#[test] -fn test_build_specs_gpt5_codex_unified_exec_web_search() { +#[tokio::test] +async fn test_build_specs_gpt5_codex_unified_exec_web_search() { let mut features = Features::with_defaults(); features.enable(Feature::UnifiedExec); assert_model_tools( @@ -404,6 +469,7 @@ fn test_build_specs_gpt5_codex_unified_exec_web_search() { "request_user_input", "apply_patch", "web_search", + "image_generation", "view_image", "spawn_agent", "send_input", @@ -411,11 +477,12 @@ fn test_build_specs_gpt5_codex_unified_exec_web_search() { "wait_agent", "close_agent", ], - ); + ) + .await; } -#[test] -fn test_build_specs_gpt51_codex_unified_exec_web_search() { +#[tokio::test] +async fn test_build_specs_gpt51_codex_unified_exec_web_search() { let mut features = Features::with_defaults(); features.enable(Feature::UnifiedExec); assert_model_tools( @@ -429,6 +496,7 @@ fn test_build_specs_gpt51_codex_unified_exec_web_search() { "request_user_input", "apply_patch", "web_search", + "image_generation", "view_image", "spawn_agent", "send_input", @@ -436,11 +504,12 @@ fn test_build_specs_gpt51_codex_unified_exec_web_search() { "wait_agent", "close_agent", ], - ); + ) + .await; } -#[test] -fn test_gpt_5_1_codex_max_defaults() { +#[tokio::test] +async fn test_gpt_5_1_codex_max_defaults() { let features = Features::with_defaults(); assert_default_model_tools( "gpt-5.1-codex-max", @@ -452,6 +521,7 @@ fn test_gpt_5_1_codex_max_defaults() { "request_user_input", "apply_patch", "web_search", + "image_generation", "view_image", "spawn_agent", "send_input", @@ -459,11 +529,12 @@ fn test_gpt_5_1_codex_max_defaults() { "wait_agent", "close_agent", ], - ); + ) + .await; } -#[test] -fn test_codex_5_1_mini_defaults() { +#[tokio::test] +async fn test_codex_5_1_mini_defaults() { let features = Features::with_defaults(); assert_default_model_tools( "gpt-5.1-codex-mini", @@ -475,6 +546,7 @@ fn test_codex_5_1_mini_defaults() { "request_user_input", "apply_patch", "web_search", + "image_generation", "view_image", "spawn_agent", "send_input", @@ -482,11 +554,12 @@ fn test_codex_5_1_mini_defaults() { "wait_agent", "close_agent", ], - ); + ) + .await; } -#[test] -fn test_gpt_5_defaults() { +#[tokio::test] +async fn test_gpt_5_defaults() { let features = Features::with_defaults(); assert_default_model_tools( "gpt-5", @@ -497,6 +570,7 @@ fn test_gpt_5_defaults() { "update_plan", "request_user_input", "web_search", + "image_generation", "view_image", "spawn_agent", "send_input", @@ -504,11 +578,12 @@ fn test_gpt_5_defaults() { "wait_agent", "close_agent", ], - ); + ) + .await; } -#[test] -fn test_gpt_5_1_defaults() { +#[tokio::test] +async fn test_gpt_5_1_defaults() { let features = Features::with_defaults(); assert_default_model_tools( "gpt-5.1", @@ -520,6 +595,7 @@ fn test_gpt_5_1_defaults() { "request_user_input", "apply_patch", "web_search", + "image_generation", "view_image", "spawn_agent", "send_input", @@ -527,11 +603,12 @@ fn test_gpt_5_1_defaults() { "wait_agent", "close_agent", ], - ); + ) + .await; } -#[test] -fn test_gpt_5_1_codex_max_unified_exec_web_search() { +#[tokio::test] +async fn test_gpt_5_1_codex_max_unified_exec_web_search() { let mut features = Features::with_defaults(); features.enable(Feature::UnifiedExec); assert_model_tools( @@ -545,6 +622,7 @@ fn test_gpt_5_1_codex_max_unified_exec_web_search() { "request_user_input", "apply_patch", "web_search", + "image_generation", "view_image", "spawn_agent", "send_input", @@ -552,12 +630,13 @@ fn test_gpt_5_1_codex_max_unified_exec_web_search() { "wait_agent", "close_agent", ], - ); + ) + .await; } -#[test] -fn test_build_specs_default_shell_present() { - let config = test_config(); +#[tokio::test] +async fn test_build_specs_default_shell_present() { + let config = test_config().await; let model_info = construct_model_info_offline("o3", &config); let mut features = Features::with_defaults(); features.enable(Feature::UnifiedExec); @@ -588,9 +667,9 @@ fn test_build_specs_default_shell_present() { assert_contains_tool_names(&tools, &subset); } -#[test] -fn shell_zsh_fork_prefers_shell_command_over_unified_exec() { - let config = test_config(); +#[tokio::test] +async fn shell_zsh_fork_prefers_shell_command_over_unified_exec() { + let config = test_config().await; let model_info = construct_model_info_offline("o3", &config); let mut features = Features::with_defaults(); features.enable(Feature::UnifiedExec); @@ -652,9 +731,10 @@ fn shell_zsh_fork_prefers_shell_command_over_unified_exec() { ); } -#[test] -fn spawn_agent_description_omits_usage_hint_when_disabled() { +#[tokio::test] +async fn spawn_agent_description_omits_usage_hint_when_disabled() { let tools_config = multi_agent_v2_tools_config() + .await .with_spawn_agent_usage_hint(/*spawn_agent_usage_hint*/ false); let description = multi_agent_v2_spawn_agent_description(&tools_config); @@ -673,11 +753,13 @@ fn spawn_agent_description_omits_usage_hint_when_disabled() { ); } -#[test] -fn spawn_agent_description_uses_configured_usage_hint_text() { - let tools_config = multi_agent_v2_tools_config().with_spawn_agent_usage_hint_text(Some( - /*spawn_agent_usage_hint_text*/ "Custom delegation guidance only.".to_string(), - )); +#[tokio::test] +async fn spawn_agent_description_uses_configured_usage_hint_text() { + let tools_config = multi_agent_v2_tools_config() + .await + .with_spawn_agent_usage_hint_text(Some( + /*spawn_agent_usage_hint_text*/ "Custom delegation guidance only.".to_string(), + )); let description = multi_agent_v2_spawn_agent_description(&tools_config); assert_regex_match( @@ -696,9 +778,9 @@ fn spawn_agent_description_uses_configured_usage_hint_text() { ); } -#[test] -fn tool_suggest_requires_apps_and_plugins_features() { - let model_info = search_capable_model_info(); +#[tokio::test] +async fn tool_suggest_requires_apps_and_plugins_features() { + let model_info = search_capable_model_info().await; let discoverable_tools = Some(vec![discoverable_connector( "connector_2128aebfecb84f64a069897515042a44", "Google Calendar", @@ -728,6 +810,7 @@ fn tool_suggest_requires_apps_and_plugins_features() { &tools_config, /*mcp_tools*/ None, /*deferred_mcp_tools*/ None, + Vec::new(), discoverable_tools.clone(), &[], ) @@ -742,9 +825,9 @@ fn tool_suggest_requires_apps_and_plugins_features() { } } -#[test] -fn search_tool_description_handles_no_enabled_mcp_tools() { - let model_info = search_capable_model_info(); +#[tokio::test] +async fn search_tool_description_handles_no_enabled_mcp_tools() { + let model_info = search_capable_model_info().await; let mut features = Features::with_defaults(); features.enable(Feature::Apps); features.enable(Feature::ToolSearch); @@ -776,9 +859,9 @@ fn search_tool_description_handles_no_enabled_mcp_tools() { assert!(!description.contains("{{source_descriptions}}")); } -#[test] -fn search_tool_description_falls_back_to_connector_name_without_description() { - let model_info = search_capable_model_info(); +#[tokio::test] +async fn search_tool_description_falls_back_to_connector_name_without_description() { + let model_info = search_capable_model_info().await; let mut features = Features::with_defaults(); features.enable(Feature::Apps); features.enable(Feature::ToolSearch); @@ -827,9 +910,9 @@ fn search_tool_description_falls_back_to_connector_name_without_description() { assert!(!description.contains("- Calendar:")); } -#[test] -fn search_tool_registers_namespaced_mcp_tool_aliases() { - let model_info = search_capable_model_info(); +#[tokio::test] +async fn search_tool_registers_namespaced_mcp_tool_aliases() { + let model_info = search_capable_model_info().await; let mut features = Features::with_defaults(); features.enable(Feature::Apps); features.enable(Feature::ToolSearch); @@ -910,13 +993,97 @@ fn search_tool_registers_namespaced_mcp_tool_aliases() { assert!(registry.has_handler(&ToolName::plain(TOOL_SEARCH_TOOL_NAME))); assert!(registry.has_handler(&app_alias)); assert!(registry.has_handler(&mcp_alias)); - assert!(registry.has_handler(&ToolName::plain("mcp__codex_apps__calendar_create_event"))); - assert!(registry.has_handler(&ToolName::plain("mcp__rmcp__echo"))); } -#[test] -fn test_mcp_tool_property_missing_type_defaults_to_string() { - let config = test_config(); +#[tokio::test] +async fn direct_mcp_tools_register_namespaced_handlers() { + let config = test_config().await; + let model_info = construct_model_info_offline("gpt-5-codex", &config); + let mut features = Features::with_defaults(); + features.enable(Feature::UnifiedExec); + let available_models = Vec::new(); + let tools_config = ToolsConfig::new(&ToolsConfigParams { + model_info: &model_info, + available_models: &available_models, + features: &features, + image_generation_tool_auth_allowed: true, + web_search_mode: Some(WebSearchMode::Cached), + session_source: SessionSource::Cli, + sandbox_policy: &SandboxPolicy::DangerFullAccess, + windows_sandbox_level: WindowsSandboxLevel::Disabled, + }); + + let (_, registry) = build_specs( + &tools_config, + Some(HashMap::from([( + "mcp__test_server__echo".to_string(), + mcp_tool_info(mcp_tool( + "echo", + "Echo", + serde_json::json!({"type": "object"}), + )), + )])), + /*deferred_mcp_tools*/ None, + &[], + ) + .build(); + + assert!(registry.has_handler(&ToolName::namespaced("mcp__test_server__", "echo"))); + assert!(!registry.has_handler(&ToolName::plain("mcp__test_server__echo"))); +} + +#[tokio::test] +async fn unavailable_mcp_tools_are_exposed_as_dummy_function_tools() { + let config = test_config().await; + let model_info = construct_model_info_offline("gpt-5-codex", &config); + let mut features = Features::with_defaults(); + features.enable(Feature::UnifiedExec); + let available_models = Vec::new(); + let tools_config = ToolsConfig::new(&ToolsConfigParams { + model_info: &model_info, + available_models: &available_models, + features: &features, + image_generation_tool_auth_allowed: true, + web_search_mode: Some(WebSearchMode::Cached), + session_source: SessionSource::Cli, + sandbox_policy: &SandboxPolicy::DangerFullAccess, + windows_sandbox_level: WindowsSandboxLevel::Disabled, + }); + + let unavailable_tool = ToolName::namespaced("mcp__codex_apps__calendar", "_create_event"); + let (tools, registry) = build_specs_with_unavailable_tools( + &tools_config, + /*mcp_tools*/ None, + /*deferred_mcp_tools*/ None, + vec![unavailable_tool], + &[], + ) + .build(); + + let tool = find_tool(&tools, "mcp__codex_apps__calendar_create_event"); + let ToolSpec::Function(ResponsesApiTool { + description, + parameters, + .. + }) = &tool.spec + else { + panic!("unavailable MCP tool should be exposed as a function tool"); + }; + assert!(description.contains("not currently available")); + assert_eq!( + parameters.additional_properties, + Some(AdditionalProperties::Boolean(false)) + ); + assert!(registry.has_handler(&ToolName::namespaced( + "mcp__codex_apps__calendar", + "_create_event" + ))); + assert!(!registry.has_handler(&ToolName::plain("mcp__codex_apps__calendar_create_event"))); +} + +#[tokio::test] +async fn test_mcp_tool_property_missing_type_defaults_to_string() { + let config = test_config().await; let model_info = construct_model_info_offline("gpt-5-codex", &config); let mut features = Features::with_defaults(); features.enable(Feature::UnifiedExec); @@ -936,27 +1103,30 @@ fn test_mcp_tool_property_missing_type_defaults_to_string() { &tools_config, Some(HashMap::from([( "dash/search".to_string(), - mcp_tool_info(mcp_tool( - "search", - "Search docs", - serde_json::json!({ - "type": "object", - "properties": { - "query": {"description": "search query"} - } - }), - )), + mcp_tool_info_with_display_name( + "dash/search", + mcp_tool( + "search", + "Search docs", + serde_json::json!({ + "type": "object", + "properties": { + "query": {"description": "search query"} + } + }), + ), + ), )])), /*deferred_mcp_tools*/ None, &[], ) .build(); - let tool = find_tool(&tools, "dash/search"); + let tool = find_namespace_function_tool(&tools, "dash/", "search"); assert_eq!( - tool.spec, - ToolSpec::Function(ResponsesApiTool { - name: "dash/search".to_string(), + *tool, + ResponsesApiTool { + name: "search".to_string(), parameters: JsonSchema::object( /*properties*/ BTreeMap::from([( @@ -970,13 +1140,13 @@ fn test_mcp_tool_property_missing_type_defaults_to_string() { strict: false, output_schema: Some(mcp_call_tool_result_output_schema(serde_json::json!({}))), defer_loading: None, - }) + } ); } -#[test] -fn test_mcp_tool_preserves_integer_schema() { - let config = test_config(); +#[tokio::test] +async fn test_mcp_tool_preserves_integer_schema() { + let config = test_config().await; let model_info = construct_model_info_offline("gpt-5-codex", &config); let mut features = Features::with_defaults(); features.enable(Feature::UnifiedExec); @@ -996,25 +1166,28 @@ fn test_mcp_tool_preserves_integer_schema() { &tools_config, Some(HashMap::from([( "dash/paginate".to_string(), - mcp_tool_info(mcp_tool( - "paginate", - "Pagination", - serde_json::json!({ - "type": "object", - "properties": {"page": {"type": "integer"}} - }), - )), + mcp_tool_info_with_display_name( + "dash/paginate", + mcp_tool( + "paginate", + "Pagination", + serde_json::json!({ + "type": "object", + "properties": {"page": {"type": "integer"}} + }), + ), + ), )])), /*deferred_mcp_tools*/ None, &[], ) .build(); - let tool = find_tool(&tools, "dash/paginate"); + let tool = find_namespace_function_tool(&tools, "dash/", "paginate"); assert_eq!( - tool.spec, - ToolSpec::Function(ResponsesApiTool { - name: "dash/paginate".to_string(), + *tool, + ResponsesApiTool { + name: "paginate".to_string(), parameters: JsonSchema::object( /*properties*/ BTreeMap::from([( @@ -1028,13 +1201,13 @@ fn test_mcp_tool_preserves_integer_schema() { strict: false, output_schema: Some(mcp_call_tool_result_output_schema(serde_json::json!({}))), defer_loading: None, - }) + } ); } -#[test] -fn test_mcp_tool_array_without_items_gets_default_string_items() { - let config = test_config(); +#[tokio::test] +async fn test_mcp_tool_array_without_items_gets_default_string_items() { + let config = test_config().await; let model_info = construct_model_info_offline("gpt-5-codex", &config); let mut features = Features::with_defaults(); features.enable(Feature::UnifiedExec); @@ -1055,25 +1228,28 @@ fn test_mcp_tool_array_without_items_gets_default_string_items() { &tools_config, Some(HashMap::from([( "dash/tags".to_string(), - mcp_tool_info(mcp_tool( - "tags", - "Tags", - serde_json::json!({ - "type": "object", - "properties": {"tags": {"type": "array"}} - }), - )), + mcp_tool_info_with_display_name( + "dash/tags", + mcp_tool( + "tags", + "Tags", + serde_json::json!({ + "type": "object", + "properties": {"tags": {"type": "array"}} + }), + ), + ), )])), /*deferred_mcp_tools*/ None, &[], ) .build(); - let tool = find_tool(&tools, "dash/tags"); + let tool = find_namespace_function_tool(&tools, "dash/", "tags"); assert_eq!( - tool.spec, - ToolSpec::Function(ResponsesApiTool { - name: "dash/tags".to_string(), + *tool, + ResponsesApiTool { + name: "tags".to_string(), parameters: JsonSchema::object( /*properties*/ BTreeMap::from([( @@ -1090,13 +1266,13 @@ fn test_mcp_tool_array_without_items_gets_default_string_items() { strict: false, output_schema: Some(mcp_call_tool_result_output_schema(serde_json::json!({}))), defer_loading: None, - }) + } ); } -#[test] -fn test_mcp_tool_anyof_defaults_to_string() { - let config = test_config(); +#[tokio::test] +async fn test_mcp_tool_anyof_defaults_to_string() { + let config = test_config().await; let model_info = construct_model_info_offline("gpt-5-codex", &config); let mut features = Features::with_defaults(); features.enable(Feature::UnifiedExec); @@ -1116,27 +1292,30 @@ fn test_mcp_tool_anyof_defaults_to_string() { &tools_config, Some(HashMap::from([( "dash/value".to_string(), - mcp_tool_info(mcp_tool( - "value", - "AnyOf Value", - serde_json::json!({ - "type": "object", - "properties": { - "value": {"anyOf": [{"type": "string"}, {"type": "number"}]} - } - }), - )), + mcp_tool_info_with_display_name( + "dash/value", + mcp_tool( + "value", + "AnyOf Value", + serde_json::json!({ + "type": "object", + "properties": { + "value": {"anyOf": [{"type": "string"}, {"type": "number"}]} + } + }), + ), + ), )])), /*deferred_mcp_tools*/ None, &[], ) .build(); - let tool = find_tool(&tools, "dash/value"); + let tool = find_namespace_function_tool(&tools, "dash/", "value"); assert_eq!( - tool.spec, - ToolSpec::Function(ResponsesApiTool { - name: "dash/value".to_string(), + *tool, + ResponsesApiTool { + name: "value".to_string(), parameters: JsonSchema::object( /*properties*/ BTreeMap::from([( @@ -1156,13 +1335,13 @@ fn test_mcp_tool_anyof_defaults_to_string() { strict: false, output_schema: Some(mcp_call_tool_result_output_schema(serde_json::json!({}))), defer_loading: None, - }) + } ); } -#[test] -fn test_get_openai_tools_mcp_tools_with_additional_properties_schema() { - let config = test_config(); +#[tokio::test] +async fn test_get_openai_tools_mcp_tools_with_additional_properties_schema() { + let config = test_config().await; let model_info = construct_model_info_offline("gpt-5-codex", &config); let mut features = Features::with_defaults(); features.enable(Feature::UnifiedExec); @@ -1181,12 +1360,14 @@ fn test_get_openai_tools_mcp_tools_with_additional_properties_schema() { &tools_config, Some(HashMap::from([( "test_server/do_something_cool".to_string(), - mcp_tool_info(mcp_tool( - "do_something_cool", - "Do something cool", - serde_json::json!({ - "type": "object", - "properties": { + mcp_tool_info_with_display_name( + "test_server/do_something_cool", + mcp_tool( + "do_something_cool", + "Do something cool", + serde_json::json!({ + "type": "object", + "properties": { "string_argument": {"type": "string"}, "number_argument": {"type": "number"}, "object_argument": { @@ -1203,22 +1384,23 @@ fn test_get_openai_tools_mcp_tools_with_additional_properties_schema() { }, "required": ["addtl_prop"], "additionalProperties": false + } } } - } - }), - )), + }), + ), + ), )])), /*deferred_mcp_tools*/ None, &[], ) .build(); - let tool = find_tool(&tools, "test_server/do_something_cool"); + let tool = find_namespace_function_tool(&tools, "test_server/", "do_something_cool"); assert_eq!( - tool.spec, - ToolSpec::Function(ResponsesApiTool { - name: "test_server/do_something_cool".to_string(), + *tool, + ResponsesApiTool { + name: "do_something_cool".to_string(), parameters: JsonSchema::object( /*properties*/ BTreeMap::from([ @@ -1268,12 +1450,12 @@ fn test_get_openai_tools_mcp_tools_with_additional_properties_schema() { strict: false, output_schema: Some(mcp_call_tool_result_output_schema(serde_json::json!({}))), defer_loading: None, - }) + } ); } -#[test] -fn code_mode_only_restricts_model_tools_to_exec_tools() { +#[tokio::test] +async fn code_mode_only_restricts_model_tools_to_exec_tools() { let mut features = Features::with_defaults(); features.enable(Feature::CodeMode); features.enable(Feature::CodeModeOnly); @@ -1283,5 +1465,6 @@ fn code_mode_only_restricts_model_tools_to_exec_tools() { &features, Some(WebSearchMode::Live), &["exec", "wait"], - ); + ) + .await; } diff --git a/codex-rs/core/src/unavailable_tool.rs b/codex-rs/core/src/unavailable_tool.rs new file mode 100644 index 0000000000..39ba21f9b8 --- /dev/null +++ b/codex-rs/core/src/unavailable_tool.rs @@ -0,0 +1,92 @@ +use std::collections::BTreeMap; +use std::collections::HashSet; + +use codex_protocol::models::ResponseItem; +use codex_tools::ToolName; + +pub(crate) fn collect_unavailable_called_tools( + input: &[ResponseItem], + exposed_tool_names: &HashSet<&str>, +) -> Vec { + let mut unavailable_tools = BTreeMap::new(); + + for item in input { + let ResponseItem::FunctionCall { + name, namespace, .. + } = item + else { + continue; + }; + if !should_collect_unavailable_tool(name, namespace.as_deref()) { + continue; + } + + let tool_name = match namespace { + Some(namespace) => ToolName::namespaced(namespace.clone(), name.clone()), + None => ToolName::plain(name.clone()), + }; + let display_name = tool_name.display(); + if exposed_tool_names.contains(display_name.as_str()) { + continue; + } + + unavailable_tools + .entry(display_name) + .or_insert_with(|| tool_name); + } + + unavailable_tools.into_values().collect() +} + +fn should_collect_unavailable_tool(name: &str, namespace: Option<&str>) -> bool { + namespace.is_some_and(|namespace| namespace.starts_with("mcp__")) || name.starts_with("mcp__") +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + + fn function_call(name: &str, namespace: Option<&str>) -> ResponseItem { + ResponseItem::FunctionCall { + id: None, + name: name.to_string(), + namespace: namespace.map(str::to_string), + arguments: "{}".to_string(), + call_id: format!("call-{name}"), + } + } + + #[test] + fn collect_unavailable_called_tools_detects_mcp_function_calls() { + let input = vec![ + function_call("shell", /*namespace*/ None), + function_call("mcp__server__lookup", /*namespace*/ None), + function_call("_create_event", Some("mcp__codex_apps__calendar")), + ]; + + let tools = collect_unavailable_called_tools(&input, &HashSet::new()); + + assert_eq!( + tools, + vec![ + ToolName::namespaced("mcp__codex_apps__calendar", "_create_event"), + ToolName::plain("mcp__server__lookup"), + ] + ); + } + + #[test] + fn collect_unavailable_called_tools_skips_currently_available_tools() { + let exposed_tool_names = HashSet::from(["mcp__server__lookup", "mcp__server__search"]); + let input = vec![ + function_call("mcp__server__lookup", /*namespace*/ None), + function_call("mcp__server__search", /*namespace*/ None), + function_call("mcp__server__missing", /*namespace*/ None), + ]; + + let tools = collect_unavailable_called_tools(&input, &exposed_tool_names); + + assert_eq!(tools, vec![ToolName::plain("mcp__server__missing")]); + } +} diff --git a/codex-rs/core/src/unified_exec/process_manager.rs b/codex-rs/core/src/unified_exec/process_manager.rs index 5214892681..aaaa7076f4 100644 --- a/codex-rs/core/src/unified_exec/process_manager.rs +++ b/codex-rs/core/src/unified_exec/process_manager.rs @@ -152,6 +152,7 @@ fn exec_server_params_for_request( env_policy, env, tty, + pipe_stdin: false, arg0: request.arg0.clone(), } } diff --git a/codex-rs/core/tests/common/apps_test_server.rs b/codex-rs/core/tests/common/apps_test_server.rs index 5b039aa149..757fc146f2 100644 --- a/codex-rs/core/tests/common/apps_test_server.rs +++ b/codex-rs/core/tests/common/apps_test_server.rs @@ -21,6 +21,8 @@ const SERVER_VERSION: &str = "1.0.0"; const SEARCHABLE_TOOL_COUNT: usize = 100; pub const CALENDAR_CREATE_EVENT_RESOURCE_URI: &str = "connector://calendar/tools/calendar_create_event"; +pub const CALENDAR_CREATE_EVENT_MCP_APP_RESOURCE_URI: &str = + "ui://widget/calendar-create-event.html"; const CALENDAR_LIST_EVENTS_RESOURCE_URI: &str = "connector://calendar/tools/calendar_list_events"; pub const DOCUMENT_EXTRACT_TEXT_RESOURCE_URI: &str = "connector://calendar/tools/calendar_extract_text"; @@ -206,6 +208,7 @@ impl Respond for CodexAppsJsonRpcResponder { "connector_id": CONNECTOR_ID, "connector_name": self.connector_name.clone(), "connector_description": self.connector_description.clone(), + "openai/outputTemplate": CALENDAR_CREATE_EVENT_MCP_APP_RESOURCE_URI, "_codex_apps": { "resource_uri": CALENDAR_CREATE_EVENT_RESOURCE_URI, "contains_mcp_source": true, diff --git a/codex-rs/core/tests/common/responses.rs b/codex-rs/core/tests/common/responses.rs index 2e2155ebdd..8def3ac6ea 100644 --- a/codex-rs/core/tests/common/responses.rs +++ b/codex-rs/core/tests/common/responses.rs @@ -125,6 +125,10 @@ impl ResponsesRequest { self.body_json().to_string().contains(&json_fragment) } + pub fn tool_by_name(&self, namespace: &str, tool_name: &str) -> Option { + namespace_child_tool(&self.body_json(), namespace, tool_name).cloned() + } + pub fn instructions_text(&self) -> String { self.body_json()["instructions"] .as_str() @@ -315,6 +319,31 @@ pub(crate) fn output_value_to_text(value: &Value) -> Option { } } +pub fn namespace_child_tool<'a>( + body: &'a Value, + namespace: &str, + tool_name: &str, +) -> Option<&'a Value> { + let tools = body.get("tools")?.as_array()?; + for tool in tools { + if tool.get("name").and_then(Value::as_str) != Some(namespace) + || tool.get("type").and_then(Value::as_str) != Some("namespace") + { + continue; + } + + let child_tools = tool.get("tools")?.as_array()?; + if let Some(child_tool) = child_tools + .iter() + .find(|tool| tool.get("name").and_then(Value::as_str) == Some(tool_name)) + { + return Some(child_tool); + } + } + + None +} + #[cfg(test)] mod tests { use super::*; @@ -780,6 +809,24 @@ pub fn ev_function_call(call_id: &str, name: &str, arguments: &str) -> Value { }) } +pub fn ev_function_call_with_namespace( + call_id: &str, + namespace: &str, + name: &str, + arguments: &str, +) -> Value { + serde_json::json!({ + "type": "response.output_item.done", + "item": { + "type": "function_call", + "call_id": call_id, + "namespace": namespace, + "name": name, + "arguments": arguments + } + }) +} + pub fn ev_tool_search_call(call_id: &str, arguments: &serde_json::Value) -> Value { serde_json::json!({ "type": "response.output_item.done", diff --git a/codex-rs/core/tests/common/test_codex.rs b/codex-rs/core/tests/common/test_codex.rs index b1274d5a94..a5aba950c9 100644 --- a/codex-rs/core/tests/common/test_codex.rs +++ b/codex-rs/core/tests/common/test_codex.rs @@ -8,7 +8,6 @@ use std::sync::Arc; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; use std::time::Duration; -use std::time::Instant; use anyhow::Context; use anyhow::Result; @@ -44,7 +43,6 @@ use tempfile::TempDir; use wiremock::MockServer; use crate::PathBufExt; -use crate::RemoteEnvConfig; use crate::TempDirExt; use crate::get_remote_test_env; use crate::load_default_config_for_test; @@ -52,8 +50,8 @@ use crate::responses::WebSocketTestServer; use crate::responses::output_value_to_text; use crate::responses::start_mock_server; use crate::streaming_sse::StreamingSseServer; -use crate::wait_for_event; use crate::wait_for_event_match; +use crate::wait_for_event_with_timeout; use wiremock::Match; use wiremock::matchers::path_regex; @@ -62,50 +60,16 @@ type PreBuildHook = dyn FnOnce(&Path) + Send + 'static; type WorkspaceSetup = dyn FnOnce(AbsolutePathBuf, Arc) -> BoxFuture<'static, Result<()>> + Send; const TEST_MODEL_WITH_EXPERIMENTAL_TOOLS: &str = "test-gpt-5.1-codex"; -const REMOTE_EXEC_SERVER_START_TIMEOUT: Duration = Duration::from_secs(5); -const REMOTE_EXEC_SERVER_POLL_INTERVAL: Duration = Duration::from_millis(25); -static REMOTE_EXEC_SERVER_INSTANCE_COUNTER: AtomicU64 = AtomicU64::new(0); - -#[derive(Debug)] -struct RemoteExecServerProcess { - container_name: String, - pid: u32, - remote_exec_server_path: String, - stdout_path: String, - cleanup_paths: Vec, -} - -impl Drop for RemoteExecServerProcess { - fn drop(&mut self) { - let cleanup_paths = self.cleanup_paths.join(" "); - let cleanup_paths_script = if cleanup_paths.is_empty() { - String::new() - } else { - format!("rm -rf {cleanup_paths}; ") - }; - let script = format!( - "if kill -0 {pid} 2>/dev/null; then kill {pid}; fi; {cleanup_paths_script}rm -f {remote_exec_server_path} {stdout_path}", - pid = self.pid, - cleanup_paths_script = cleanup_paths_script, - remote_exec_server_path = self.remote_exec_server_path, - stdout_path = self.stdout_path - ); - let _ = docker_command_capture_stdout(["exec", &self.container_name, "sh", "-lc", &script]); - } -} - -impl RemoteExecServerProcess { - fn register_cleanup_path(&mut self, path: &Path) { - self.cleanup_paths.push(path.display().to_string()); - } -} +const REMOTE_EXEC_SERVER_URL_ENV_VAR: &str = "CODEX_TEST_REMOTE_EXEC_SERVER_URL"; +static REMOTE_TEST_INSTANCE_COUNTER: AtomicU64 = AtomicU64::new(0); +const SUBMIT_TURN_COMPLETE_TIMEOUT: Duration = Duration::from_secs(30); #[derive(Debug)] pub struct TestEnv { environment: codex_exec_server::Environment, cwd: AbsolutePathBuf, local_cwd_temp_dir: Option>, - _remote_exec_server_process: Option, + remote_container_name: Option, } impl TestEnv { @@ -117,7 +81,7 @@ impl TestEnv { environment, cwd, local_cwd_temp_dir: Some(local_cwd_temp_dir), - _remote_exec_server_process: None, + remote_container_name: None, }) } @@ -138,12 +102,19 @@ impl TestEnv { } } +impl Drop for TestEnv { + fn drop(&mut self) { + if let Some(container_name) = &self.remote_container_name { + let script = format!("rm -rf {}", self.cwd.as_path().display()); + let _ = docker_command_capture_stdout(["exec", container_name, "sh", "-lc", &script]); + } + } +} + pub async fn test_env() -> Result { match get_remote_test_env() { Some(remote_env) => { - let mut remote_process = start_remote_exec_server(&remote_env)?; - let remote_ip = remote_container_ip(&remote_env.container_name)?; - let websocket_url = rewrite_websocket_host(&remote_process.listen_url, &remote_ip)?; + let websocket_url = remote_exec_server_url()?; let environment = codex_exec_server::Environment::create(Some(websocket_url)).await?; let cwd = remote_aware_cwd_path(); environment @@ -154,184 +125,43 @@ pub async fn test_env() -> Result { /*sandbox*/ None, ) .await?; - remote_process.process.register_cleanup_path(cwd.as_path()); Ok(TestEnv { environment, cwd, local_cwd_temp_dir: None, - _remote_exec_server_process: Some(remote_process.process), + remote_container_name: Some(remote_env.container_name), }) } None => TestEnv::local().await, } } -struct RemoteExecServerStart { - process: RemoteExecServerProcess, - listen_url: String, -} - -fn start_remote_exec_server(remote_env: &RemoteEnvConfig) -> Result { - let container_name = remote_env.container_name.as_str(); - let instance_id = remote_exec_server_instance_id(); - let remote_exec_server_path = format!("/tmp/codex-{instance_id}"); - let remote_linux_sandbox_path = format!("/tmp/codex-linux-sandbox-{instance_id}"); - let stdout_path = format!("/tmp/codex-exec-server-{instance_id}.stdout"); - let local_binary = codex_utils_cargo_bin::cargo_bin("codex").context("resolve codex binary")?; - let local_linux_sandbox = codex_utils_cargo_bin::cargo_bin("codex-linux-sandbox") - .context("resolve codex-linux-sandbox binary")?; - let local_binary = local_binary.to_string_lossy().to_string(); - let local_linux_sandbox = local_linux_sandbox.to_string_lossy().to_string(); - let remote_binary = format!("{container_name}:{remote_exec_server_path}"); - let remote_linux_sandbox = format!("{container_name}:{remote_linux_sandbox_path}"); - - docker_command_success(["cp", &local_binary, &remote_binary])?; - docker_command_success(["cp", &local_linux_sandbox, &remote_linux_sandbox])?; - docker_command_success([ - "exec", - container_name, - "chmod", - "+x", - &remote_exec_server_path, - ])?; - docker_command_success([ - "exec", - container_name, - "chmod", - "+x", - &remote_linux_sandbox_path, - ])?; - probe_remote_linux_sandbox(container_name, &remote_linux_sandbox_path)?; - - let start_script = format!( - "rm -f {stdout_path}; \ -nohup {remote_exec_server_path} exec-server --listen ws://0.0.0.0:0 > {stdout_path} 2>&1 & \ -echo $!" - ); - let pid_output = - docker_command_capture_stdout(["exec", container_name, "sh", "-lc", &start_script])?; - let pid = pid_output - .trim() - .parse::() - .with_context(|| format!("parse remote exec-server PID from {pid_output:?}"))?; - - let listen_url = wait_for_remote_listen_url(container_name, &stdout_path)?; - - Ok(RemoteExecServerStart { - process: RemoteExecServerProcess { - container_name: container_name.to_string(), - pid, - remote_exec_server_path, - stdout_path, - cleanup_paths: vec![remote_linux_sandbox_path], - }, - listen_url, - }) -} - -fn probe_remote_linux_sandbox(container_name: &str, remote_linux_sandbox_path: &str) -> Result<()> { - let policy = serde_json::to_string(&SandboxPolicy::new_read_only_policy()) - .context("serialize remote sandbox probe policy")?; - let probe_script = format!( - "{remote_linux_sandbox_path} --sandbox-policy-cwd /tmp --sandbox-policy '{policy}' -- /bin/true" - ); - let output = Command::new("docker") - .args(["exec", container_name, "sh", "-lc", &probe_script]) - .output() - .with_context(|| format!("probe remote linux sandbox in container `{container_name}`"))?; - if !output.status.success() { - return Err(anyhow!( - "remote linux sandbox probe failed in container `{container_name}`: stdout={} stderr={}", - String::from_utf8_lossy(&output.stdout).trim(), - String::from_utf8_lossy(&output.stderr).trim() - )); - } - Ok(()) -} - fn remote_aware_cwd_path() -> AbsolutePathBuf { PathBuf::from(format!( "/tmp/codex-core-test-cwd-{}", - remote_exec_server_instance_id() + remote_test_instance_id() )) .abs() } -fn wait_for_remote_listen_url(container_name: &str, stdout_path: &str) -> Result { - let deadline = Instant::now() + REMOTE_EXEC_SERVER_START_TIMEOUT; - loop { - let line = docker_command_capture_stdout([ - "exec", - container_name, - "sh", - "-lc", - &format!("head -n 1 {stdout_path} 2>/dev/null || true"), - ])?; - let listen_url = line.trim(); - if listen_url.starts_with("ws://") { - return Ok(listen_url.to_string()); - } - - if Instant::now() >= deadline { - return Err(anyhow!( - "timed out waiting for remote exec-server listen URL in container `{container_name}` after {REMOTE_EXEC_SERVER_START_TIMEOUT:?}" - )); - } - std::thread::sleep(REMOTE_EXEC_SERVER_POLL_INTERVAL); +fn remote_exec_server_url() -> Result { + let listen_url = std::env::var(REMOTE_EXEC_SERVER_URL_ENV_VAR).with_context(|| { + format!("{REMOTE_EXEC_SERVER_URL_ENV_VAR} must be set for remote tests") + })?; + let listen_url = listen_url.trim(); + if listen_url.is_empty() { + return Err(anyhow!( + "{REMOTE_EXEC_SERVER_URL_ENV_VAR} must not be empty" + )); } + Ok(listen_url.to_string()) } -fn remote_exec_server_instance_id() -> String { - let instance = REMOTE_EXEC_SERVER_INSTANCE_COUNTER.fetch_add(1, Ordering::Relaxed); +fn remote_test_instance_id() -> String { + let instance = REMOTE_TEST_INSTANCE_COUNTER.fetch_add(1, Ordering::Relaxed); format!("{}-{instance}", std::process::id()) } -fn remote_container_ip(container_name: &str) -> Result { - let ip = docker_command_capture_stdout([ - "inspect", - "-f", - "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", - container_name, - ])?; - let ip = ip.trim(); - if ip.is_empty() { - return Err(anyhow!( - "container `{container_name}` has no IP address; cannot connect to remote exec-server" - )); - } - Ok(ip.to_string()) -} - -fn rewrite_websocket_host(listen_url: &str, host: &str) -> Result { - let Some(address) = listen_url.strip_prefix("ws://") else { - return Err(anyhow!( - "unexpected websocket listen URL `{listen_url}`; expected ws://IP:PORT" - )); - }; - let Some((_, port)) = address.rsplit_once(':') else { - return Err(anyhow!( - "unexpected websocket listen URL `{listen_url}`; expected ws://IP:PORT" - )); - }; - Ok(format!("ws://{host}:{port}")) -} - -fn docker_command_success(args: [&str; N]) -> Result<()> { - let output = Command::new("docker") - .args(args) - .output() - .with_context(|| format!("run docker {args:?}"))?; - if !output.status.success() { - return Err(anyhow!( - "docker {:?} failed: stdout={} stderr={}", - args, - String::from_utf8_lossy(&output.stdout).trim(), - String::from_utf8_lossy(&output.stderr).trim() - )); - } - Ok(()) -} - fn docker_command_capture_stdout(args: [&str; N]) -> Result { let output = Command::new("docker") .args(args) @@ -809,10 +639,14 @@ impl TestCodex { _ => None, }) .await; - wait_for_event(&self.codex, |event| match event { - EventMsg::TurnComplete(event) => event.turn_id == turn_id, - _ => false, - }) + wait_for_event_with_timeout( + &self.codex, + |event| match event { + EventMsg::TurnComplete(event) => event.turn_id == turn_id, + _ => false, + }, + SUBMIT_TURN_COMPLETE_TIMEOUT, + ) .await; Ok(()) } diff --git a/codex-rs/core/tests/responses_headers.rs b/codex-rs/core/tests/responses_headers.rs index 8498956517..db4dc794be 100644 --- a/codex-rs/core/tests/responses_headers.rs +++ b/codex-rs/core/tests/responses_headers.rs @@ -575,16 +575,17 @@ async fn responses_stream_includes_turn_metadata_header_for_git_workspace_e2e() .and_then(serde_json::Value::as_str), Some(expected_head.as_str()) ); - let actual_origin = workspace + if let Some(actual_origin) = workspace .get("associated_remote_urls") .and_then(serde_json::Value::as_object) .and_then(|remotes| remotes.get("origin")) .and_then(serde_json::Value::as_str) - .expect("origin remote should be present"); - assert_eq!( - normalize_git_remote_url(actual_origin), - normalize_git_remote_url(&expected_origin) - ); + { + assert_eq!( + normalize_git_remote_url(actual_origin), + normalize_git_remote_url(&expected_origin) + ); + } assert_eq!( workspace .get("has_changes") diff --git a/codex-rs/core/tests/suite/approvals.rs b/codex-rs/core/tests/suite/approvals.rs index 84066ff6e6..208b45d594 100644 --- a/codex-rs/core/tests/suite/approvals.rs +++ b/codex-rs/core/tests/suite/approvals.rs @@ -226,12 +226,11 @@ impl ActionKind { let _ = fs::remove_file(&path); let patch = build_add_file_patch(&patch_path, content); let command = shell_apply_patch_command(&patch); - let event = shell_event( - call_id, - &command, - /*timeout_ms*/ 30_000, - sandbox_permissions, - )?; + // Bazel may need to launch the configured Codex helper binary + // to apply the verified patch, which can exceed the normal + // short command timeout on slower CI runners. + let timeout_ms = 30_000; + let event = shell_event(call_id, &command, timeout_ms, sandbox_permissions)?; Ok((event, Some(command))) } } @@ -2803,6 +2802,169 @@ allow_local_binding = true Ok(()) } +#[tokio::test(flavor = "current_thread")] +async fn network_approval_flow_survives_danger_full_access_session_start() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let home = Arc::new(TempDir::new()?); + fs::write( + home.path().join("config.toml"), + r#"default_permissions = "workspace" + +[permissions.workspace.filesystem] +":minimal" = "read" + +[permissions.workspace.network] +enabled = true +mode = "limited" +allow_local_binding = true +"#, + )?; + let approval_policy = AskForApproval::OnFailure; + let turn_sandbox_policy = SandboxPolicy::WorkspaceWrite { + writable_roots: vec![], + read_only_access: Default::default(), + network_access: true, + exclude_tmpdir_env_var: false, + exclude_slash_tmp: false, + }; + let mut builder = test_codex().with_home(home).with_config(move |config| { + config.permissions.approval_policy = Constrained::allow_any(approval_policy); + config.permissions.sandbox_policy = Constrained::allow_any(SandboxPolicy::DangerFullAccess); + let layers = config + .config_layer_stack + .get_layers( + ConfigLayerStackOrdering::LowestPrecedenceFirst, + /*include_disabled*/ true, + ) + .into_iter() + .cloned() + .collect(); + let mut requirements = config.config_layer_stack.requirements().clone(); + requirements.network = Some(Sourced::new( + NetworkConstraints { + enabled: Some(true), + allow_local_binding: Some(true), + ..Default::default() + }, + RequirementSource::CloudRequirements, + )); + let mut requirements_toml = config.config_layer_stack.requirements_toml().clone(); + requirements_toml.network = Some(NetworkRequirementsToml { + enabled: Some(true), + allow_local_binding: Some(true), + ..Default::default() + }); + config.config_layer_stack = ConfigLayerStack::new(layers, requirements, requirements_toml) + .expect("rebuild config layer stack with network requirements"); + }); + let test = builder.build(&server).await?; + assert!( + !test.config.managed_network_requirements_enabled(), + "expected managed network requirements to stay inactive in danger-full-access" + ); + assert!( + test.config.permissions.network.is_some(), + "expected managed network proxy config to be present" + ); + assert!( + test.session_configured.network_proxy.is_none(), + "expected session configured event to hide managed network proxy in danger-full-access" + ); + + let call_id = "allow-network-after-yolo"; + let fetch_command = r#"python3 -c "import urllib.request; opener = urllib.request.build_opener(urllib.request.ProxyHandler()); print('OK:' + opener.open('http://codex-network-test.invalid', timeout=30).read().decode(errors='replace'))""# + .to_string(); + let event = shell_event( + call_id, + &fetch_command, + /*timeout_ms*/ 30_000, + SandboxPermissions::UseDefault, + )?; + + let _ = mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-network-after-yolo-1"), + event, + ev_completed("resp-network-after-yolo-1"), + ]), + ) + .await; + let _ = mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-network-after-yolo-1", "done"), + ev_completed("resp-network-after-yolo-2"), + ]), + ) + .await; + + submit_turn( + &test, + "allow-network-after-yolo", + approval_policy, + turn_sandbox_policy, + ) + .await?; + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + let approval = loop { + let remaining = deadline + .checked_duration_since(std::time::Instant::now()) + .expect("timed out waiting for network approval request"); + let event = wait_for_event_with_timeout( + &test.codex, + |event| { + matches!( + event, + EventMsg::ExecApprovalRequest(_) | EventMsg::TurnComplete(_) + ) + }, + remaining, + ) + .await; + match event { + EventMsg::ExecApprovalRequest(approval) => { + if approval.command.first().map(std::string::String::as_str) + == Some("network-access") + { + break approval; + } + test.codex + .submit(Op::ExecApproval { + id: approval.effective_approval_id(), + turn_id: None, + decision: ReviewDecision::Approved, + }) + .await?; + } + EventMsg::TurnComplete(_) => { + panic!("expected network approval request before completion"); + } + other => panic!("unexpected event: {other:?}"), + } + }; + + let network_context = approval + .network_approval_context + .clone() + .expect("expected network approval context"); + assert_eq!(network_context.protocol, NetworkApprovalProtocol::Http); + + test.codex + .submit(Op::ExecApproval { + id: approval.effective_approval_id(), + turn_id: None, + decision: ReviewDecision::Denied, + }) + .await?; + wait_for_completion(&test).await; + + Ok(()) +} + // todo(dylan) add ScenarioSpec support for rules #[tokio::test(flavor = "current_thread")] #[cfg(unix)] diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index 76a70d5076..474d221d27 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -6,6 +6,8 @@ use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use codex_config::types::McpServerConfig; use codex_config::types::McpServerTransportConfig; use codex_features::Feature; +use codex_login::CodexAuth; +use codex_models_manager::bundled_models_response; use codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem; use codex_protocol::dynamic_tools::DynamicToolResponse; use codex_protocol::dynamic_tools::DynamicToolSpec; @@ -14,6 +16,7 @@ use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::user_input::UserInput; +use core_test_support::apps_test_server::AppsTestServer; use core_test_support::assert_regex_match; use core_test_support::responses; use core_test_support::responses::ResponseMock; @@ -178,44 +181,77 @@ async fn run_code_mode_turn_with_rmcp( prompt: &str, code: &str, ) -> Result<(TestCodex, ResponseMock)> { - let rmcp_test_server_bin = stdio_server_bin()?; - let mut builder = test_codex() - .with_model("test-gpt-5.1-codex") - .with_config(move |config| { - let _ = config.features.enable(Feature::CodeMode); + run_code_mode_turn_with_rmcp_model(server, prompt, code, "test-gpt-5.1-codex").await +} - let mut servers = config.mcp_servers.get().clone(); - servers.insert( - "rmcp".to_string(), - McpServerConfig { - transport: McpServerTransportConfig::Stdio { - command: rmcp_test_server_bin, - args: Vec::new(), - env: Some(HashMap::from([( - "MCP_TEST_VALUE".to_string(), - "propagated-env".to_string(), - )])), - env_vars: Vec::new(), - cwd: None, - }, - enabled: true, - required: false, - supports_parallel_tool_calls: false, - disabled_reason: None, - startup_timeout_sec: Some(Duration::from_secs(10)), - tool_timeout_sec: None, - enabled_tools: None, - disabled_tools: None, - scopes: None, - oauth_resource: None, - tools: HashMap::new(), +async fn run_code_mode_turn_with_rmcp_model( + server: &MockServer, + prompt: &str, + code: &str, + model: &'static str, +) -> Result<(TestCodex, ResponseMock)> { + run_code_mode_turn_with_rmcp_config(server, prompt, code, model, /*code_mode_only*/ false).await +} + +async fn run_code_mode_turn_with_rmcp_mode( + server: &MockServer, + prompt: &str, + code: &str, + code_mode_only: bool, +) -> Result<(TestCodex, ResponseMock)> { + run_code_mode_turn_with_rmcp_config(server, prompt, code, "test-gpt-5.1-codex", code_mode_only) + .await +} + +async fn run_code_mode_turn_with_rmcp_config( + server: &MockServer, + prompt: &str, + code: &str, + model: &'static str, + code_mode_only: bool, +) -> Result<(TestCodex, ResponseMock)> { + let rmcp_test_server_bin = stdio_server_bin()?; + let mut builder = test_codex().with_model(model).with_config(move |config| { + let _ = if code_mode_only { + config.features.enable(Feature::CodeModeOnly) + } else { + config.features.enable(Feature::CodeMode) + }; + + let mut servers = config.mcp_servers.get().clone(); + servers.insert( + "rmcp".to_string(), + McpServerConfig { + transport: McpServerTransportConfig::Stdio { + command: rmcp_test_server_bin, + args: Vec::new(), + env: Some(HashMap::from([( + "MCP_TEST_VALUE".to_string(), + "propagated-env".to_string(), + )])), + env_vars: Vec::new(), + cwd: None, }, - ); - config - .mcp_servers - .set(servers) - .expect("test mcp servers should accept any configuration"); - }); + experimental_environment: None, + enabled: true, + required: false, + supports_parallel_tool_calls: false, + disabled_reason: None, + startup_timeout_sec: Some(Duration::from_secs(10)), + tool_timeout_sec: None, + default_tools_approval_mode: None, + enabled_tools: None, + disabled_tools: None, + scopes: None, + oauth_resource: None, + tools: HashMap::new(), + }, + ); + config + .mcp_servers + .set(servers) + .expect("test mcp servers should accept any configuration"); + }); let test = builder.build(server).await?; responses::mount_sse_once( @@ -315,6 +351,131 @@ async fn code_mode_only_restricts_prompt_tools() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn code_mode_only_guides_all_tools_search_and_calls_deferred_app_tools() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let apps_server = AppsTestServer::mount_searchable(&server).await?; + let resp_mock = responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-1"), + ev_custom_tool_call( + "call-1", + "exec", + r#" +const tool = ALL_TOOLS.find( + ({ name }) => name === "mcp__codex_apps__calendar_timezone_option_99" +); +if (!tool) { + text(JSON.stringify({ found: false })); +} else { + const result = await tools[tool.name]({ timezone: "UTC" }); + text(JSON.stringify({ + found: true, + isError: Boolean(result.isError), + text: result.content?.[0]?.text ?? "", + })); +} +"#, + ), + ev_completed("resp-1"), + ]), + ) + .await; + let follow_up_mock = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]), + ) + .await; + + let apps_base_url = apps_server.chatgpt_base_url.clone(); + let mut builder = test_codex() + .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) + .with_config(move |config| { + config + .features + .enable(Feature::Apps) + .expect("test config should allow feature update"); + config + .features + .enable(Feature::ToolSearch) + .expect("test config should allow feature update"); + config + .features + .enable(Feature::CodeMode) + .expect("test config should allow feature update"); + config + .features + .enable(Feature::CodeModeOnly) + .expect("test config should allow feature update"); + config.chatgpt_base_url = apps_base_url; + config.model = Some("gpt-5-codex".to_string()); + + let mut model_catalog = bundled_models_response() + .unwrap_or_else(|err| panic!("bundled models.json should parse: {err}")); + let model = model_catalog + .models + .iter_mut() + .find(|model| model.slug == "gpt-5-codex") + .expect("gpt-5-codex exists in bundled models.json"); + model.supports_search_tool = true; + config.model_catalog = Some(model_catalog); + }); + let test = builder.build(&server).await?; + test.submit_turn("inspect tools in code mode only").await?; + + let first_body = resp_mock.single_request().body_json(); + assert_eq!( + tool_names(&first_body), + vec!["exec".to_string(), "wait".to_string()] + ); + + let exec_description = first_body + .get("tools") + .and_then(Value::as_array) + .and_then(|tools| { + tools.iter().find_map(|tool| { + if tool + .get("name") + .or_else(|| tool.get("type")) + .and_then(Value::as_str) + == Some("exec") + { + tool.get("description").and_then(Value::as_str) + } else { + None + } + }) + }) + .expect("exec description should be present"); + assert!(exec_description.contains("filter `ALL_TOOLS` by `name` and `description`")); + assert!(!exec_description.contains("calendar_timezone_option_99")); + + let request = follow_up_mock.single_request(); + let (output, success) = custom_tool_output_body_and_success(&request, "call-1"); + assert_ne!( + success, + Some(false), + "code_mode_only deferred app tool call failed unexpectedly: {output}" + ); + let parsed: Value = serde_json::from_str(&output)?; + assert_eq!( + parsed, + serde_json::json!({ + "found": true, + "isError": false, + "text": "called calendar_timezone_option_99 for at with ", + }) + ); + + Ok(()) +} + #[cfg_attr(windows, ignore = "no exec_command on Windows")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn code_mode_only_can_call_nested_tools() -> Result<()> { @@ -1906,6 +2067,62 @@ image(out); Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn code_mode_can_use_mcp_image_result_with_image_helper() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let code = r#" +const out = await tools.mcp__rmcp__image_scenario({ + scenario: "image_only_original_detail", +}); +const imageItem = out.content.find((item) => item.type === "image"); +image(imageItem); +"#; + + let (_test, second_mock) = run_code_mode_turn_with_rmcp_model( + &server, + "use exec to call the rmcp image scenario tool and emit its image output", + code, + "gpt-5.3-codex", + ) + .await?; + + let req = second_mock.single_request(); + let items = custom_tool_output_items(&req, "call-1"); + let (_, success) = custom_tool_output_body_and_success(&req, "call-1"); + assert_ne!( + success, + Some(false), + "code_mode mcp image scenario call failed unexpectedly" + ); + assert_eq!(items.len(), 2); + assert_regex_match( + concat!( + r"(?s)\A", + r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z" + ), + text_item(&items, /*index*/ 0), + ); + + assert_eq!( + items[1].get("type").and_then(Value::as_str), + Some("input_image") + ); + + let emitted_image_url = items[1] + .get("image_url") + .and_then(Value::as_str) + .expect("image helper should emit an input_image item with image_url"); + assert!(emitted_image_url.starts_with("data:image/png;base64,")); + assert_eq!( + items[1].get("detail").and_then(Value::as_str), + Some("original") + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn code_mode_can_apply_patch_via_nested_tool() -> Result<()> { skip_if_no_network!(Ok(())); @@ -1989,6 +2206,36 @@ contentLength=0" Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn code_mode_only_can_call_mcp_tool() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let code = r#" +const result = await tools.mcp__rmcp__echo({ message: "ping" }); +text(`echo=${result.structuredContent?.echo ?? "missing"}`); +"#; + + let (_test, second_mock) = run_code_mode_turn_with_rmcp_mode( + &server, + "use exec to run the rmcp echo tool in code mode only", + code, + /*code_mode_only*/ true, + ) + .await?; + + let req = second_mock.single_request(); + let (output, success) = custom_tool_output_body_and_success(&req, "call-1"); + assert_ne!( + success, + Some(false), + "code_mode_only rmcp tool call failed unexpectedly: {output}" + ); + assert_eq!(output, "echo=ECHOING: ping"); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn code_mode_exposes_mcp_tools_on_global_tools_object() -> Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/core/tests/suite/compact.rs b/codex-rs/core/tests/suite/compact.rs index dc5f127df5..bdbc1edbd7 100644 --- a/codex-rs/core/tests/suite/compact.rs +++ b/codex-rs/core/tests/suite/compact.rs @@ -2838,10 +2838,12 @@ async fn auto_compact_counts_encrypted_reasoning_before_last_user() { ]; let compact_mock = mount_compact_json_once(&server, serde_json::json!({ "output": compacted_history })).await; + let chatgpt_base_url = format!("{}/backend-api", server.uri()); let codex = test_codex() .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) - .with_config(|config| { + .with_config(move |config| { + config.chatgpt_base_url = chatgpt_base_url; set_test_compact_prompt(config); config.model_auto_compact_token_limit = Some(300); }) diff --git a/codex-rs/core/tests/suite/compact_remote.rs b/codex-rs/core/tests/suite/compact_remote.rs index 8322046d10..4b1eaa44d2 100644 --- a/codex-rs/core/tests/suite/compact_remote.rs +++ b/codex-rs/core/tests/suite/compact_remote.rs @@ -34,8 +34,10 @@ use core_test_support::test_codex::TestCodexHarness; use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event; use core_test_support::wait_for_event_match; +use core_test_support::wait_for_event_with_timeout; use pretty_assertions::assert_eq; use serde_json::json; +use tokio::time::Duration; use wiremock::ResponseTemplate; fn approx_token_count(text: &str) -> i64 { @@ -55,6 +57,7 @@ fn estimate_compact_payload_tokens(request: &responses::ResponsesRequest) -> i64 const PRETURN_CONTEXT_DIFF_CWD: &str = "/tmp/PRETURN_CONTEXT_DIFF_CWD"; const DUMMY_FUNCTION_NAME: &str = "test_tool"; +const REMOTE_COMPACT_TURN_COMPLETE_TIMEOUT: Duration = Duration::from_secs(30); fn summary_with_prefix(summary: &str) -> String { format!("{SUMMARY_PREFIX}\n{summary}") @@ -197,6 +200,15 @@ fn assert_request_contains_realtime_end(request: &responses::ResponsesRequest) { ); } +async fn wait_for_turn_complete(codex: &codex_core::CodexThread) { + wait_for_event_with_timeout( + codex, + |ev| matches!(ev, EventMsg::TurnComplete(_)), + REMOTE_COMPACT_TURN_COMPLETE_TIMEOUT, + ) + .await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn remote_compact_replaces_history_for_followups() -> Result<()> { skip_if_no_network!(Ok(())); @@ -242,10 +254,10 @@ async fn remote_compact_replaces_history_for_followups() -> Result<()> { responsesapi_client_metadata: None, }) .await?; - wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + wait_for_turn_complete(&codex).await; codex.submit(Op::Compact).await?; - wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + wait_for_turn_complete(&codex).await; codex .submit(Op::UserInput { @@ -257,7 +269,7 @@ async fn remote_compact_replaces_history_for_followups() -> Result<()> { responsesapi_client_metadata: None, }) .await?; - wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + wait_for_turn_complete(&codex).await; let compact_request = compact_mock.single_request(); assert_eq!(compact_request.path(), "/v1/responses/compact"); diff --git a/codex-rs/core/tests/suite/exec.rs b/codex-rs/core/tests/suite/exec.rs index 19869ebb2b..c80c8a3340 100644 --- a/codex-rs/core/tests/suite/exec.rs +++ b/codex-rs/core/tests/suite/exec.rs @@ -1,8 +1,5 @@ #![cfg(target_os = "macos")] -use std::collections::HashMap; -use std::string::ToString; - use codex_core::exec::ExecCapturePolicy; use codex_core::exec::ExecParams; use codex_core::exec::process_exec_tool_call; @@ -17,6 +14,7 @@ use codex_protocol::protocol::SandboxPolicy; use codex_sandboxing::SandboxType; use codex_sandboxing::get_platform_sandbox; use core_test_support::PathExt; +use std::collections::HashMap; use tempfile::TempDir; fn skip_test() -> bool { @@ -29,14 +27,18 @@ fn skip_test() -> bool { } #[expect(clippy::expect_used)] -async fn run_test_cmd(tmp: TempDir, cmd: Vec<&str>) -> Result { +async fn run_test_cmd(tmp: TempDir, command: I) -> Result +where + I: IntoIterator, + S: Into, +{ let sandbox_type = get_platform_sandbox(/*windows_sandbox_enabled*/ false) .expect("should be able to get sandbox type"); assert_eq!(sandbox_type, SandboxType::MacosSeatbelt); let cwd = tmp.path().abs(); let params = ExecParams { - command: cmd.iter().map(ToString::to_string).collect(), + command: command.into_iter().map(Into::into).collect(), cwd: cwd.clone(), expiration: 1000.into(), capture_policy: ExecCapturePolicy::ShellTool, @@ -129,6 +131,37 @@ async fn exit_command_not_found_is_ok() { run_test_cmd(tmp, cmd).await.unwrap(); } +#[tokio::test] +async fn openpty_works_under_real_exec_seatbelt_path() { + if skip_test() { + return; + } + + let python = match which::which("python3") { + Ok(path) => path, + Err(_) => { + eprintln!("python3 not found in PATH, skipping test."); + return; + } + }; + + let tmp = TempDir::new().expect("should be able to create temp dir"); + let cmd = vec![ + python.to_string_lossy().into_owned(), + "-c".to_string(), + r#"import os + +master, slave = os.openpty() +os.write(slave, b"ping") +assert os.read(master, 4) == b"ping""# + .to_string(), + ]; + + let output = run_test_cmd(tmp, cmd).await.unwrap(); + assert_eq!(output.stdout.text, ""); + assert_eq!(output.stderr.text, ""); +} + /// Writing a file fails and should be considered a sandbox error #[tokio::test] async fn write_file_fails_as_sandbox_error() { @@ -139,7 +172,7 @@ async fn write_file_fails_as_sandbox_error() { let tmp = TempDir::new().expect("should be able to create temp dir"); let path = tmp.path().join("test.txt"); let cmd = vec![ - "/user/bin/touch", + "/usr/bin/touch", path.to_str().expect("should be able to get path"), ]; diff --git a/codex-rs/core/tests/suite/js_repl.rs b/codex-rs/core/tests/suite/js_repl.rs index 5f52c24e19..450aa08f0b 100644 --- a/codex-rs/core/tests/suite/js_repl.rs +++ b/codex-rs/core/tests/suite/js_repl.rs @@ -1,6 +1,8 @@ #![allow(clippy::expect_used, clippy::unwrap_used)] use anyhow::Result; +use codex_config::types::McpServerConfig; +use codex_config::types::McpServerTransportConfig; use codex_features::Feature; use codex_protocol::protocol::EventMsg; use core_test_support::responses; @@ -12,12 +14,15 @@ use core_test_support::responses::ev_custom_tool_call; use core_test_support::responses::ev_response_created; use core_test_support::responses::sse; use core_test_support::skip_if_no_network; +use core_test_support::stdio_server_bin; use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event_match; +use std::collections::HashMap; use std::fs; #[cfg(unix)] use std::os::unix::fs::PermissionsExt; use std::path::Path; +use std::time::Duration; use tempfile::tempdir; use wiremock::MockServer; @@ -594,6 +599,84 @@ async fn js_repl_can_invoke_builtin_tools() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn js_repl_can_invoke_mcp_tools_by_display_name() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let rmcp_test_server_bin = stdio_server_bin()?; + let mut builder = test_codex().with_config(move |config| { + config + .features + .enable(Feature::JsRepl) + .expect("test config should allow feature update"); + + let mut servers = config.mcp_servers.get().clone(); + servers.insert( + "rmcp".to_string(), + McpServerConfig { + transport: McpServerTransportConfig::Stdio { + command: rmcp_test_server_bin, + args: Vec::new(), + env: None, + env_vars: Vec::new(), + cwd: None, + }, + experimental_environment: None, + enabled: true, + required: false, + supports_parallel_tool_calls: false, + disabled_reason: None, + startup_timeout_sec: Some(Duration::from_secs(10)), + tool_timeout_sec: None, + default_tools_approval_mode: None, + enabled_tools: None, + disabled_tools: None, + scopes: None, + oauth_resource: None, + tools: HashMap::new(), + }, + ); + config + .mcp_servers + .set(servers) + .expect("test mcp servers should accept any configuration"); + }); + let test = builder.build(&server).await?; + + responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-1"), + ev_custom_tool_call( + "call-1", + "js_repl", + r#" +const result = await codex.tool("mcp__rmcp__echo", { message: "ping" }); +console.log(result.output); +"#, + ), + ev_completed("resp-1"), + ]), + ) + .await; + let final_mock = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]), + ) + .await; + + test.submit_turn("use js_repl to call an MCP tool").await?; + + let req = final_mock.single_request(); + assert_js_repl_ok(&req, "call-1", "ECHOING: ping"); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn js_repl_tool_call_rejects_recursive_js_repl_invocation() -> Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/core/tests/suite/memories.rs b/codex-rs/core/tests/suite/memories.rs index 98e6186437..60685592f6 100644 --- a/codex-rs/core/tests/suite/memories.rs +++ b/codex-rs/core/tests/suite/memories.rs @@ -159,6 +159,145 @@ async fn memories_startup_phase2_tracks_added_and_removed_inputs_across_runs() - Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn memories_startup_phase2_prunes_old_extension_resources_and_reports_them() -> Result<()> { + let server = start_mock_server().await; + let home = Arc::new(TempDir::new()?); + let db = init_state_db(&home).await?; + let now = Utc::now(); + let thread_id = seed_stage1_output( + db.as_ref(), + home.path(), + now - ChronoDuration::hours(1), + "raw memory", + "rollout summary", + "rollout", + ) + .await?; + + let telepathy_resources = home.path().join("memories_extensions/telepathy/resources"); + tokio::fs::create_dir_all(&telepathy_resources).await?; + tokio::fs::write( + home.path() + .join("memories_extensions/telepathy/instructions.md"), + "instructions", + ) + .await?; + let old_file_name = format!( + "{}-abcd-10min-old.md", + (now - ChronoDuration::days(8)).format("%Y-%m-%dT%H-%M-%S") + ); + let old_file = telepathy_resources.join(&old_file_name); + tokio::fs::write(&old_file, "old resource").await?; + let recent_file = telepathy_resources.join(format!( + "{}-abcd-10min-recent.md", + (now - ChronoDuration::days(6)).format("%Y-%m-%dT%H-%M-%S") + )); + tokio::fs::write(&recent_file, "recent resource").await?; + + let phase2 = mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-phase2"), + ev_assistant_message("msg-phase2", "phase2 complete"), + ev_completed("resp-phase2"), + ]), + ) + .await; + + let codex = build_test_codex(&server, home.clone()).await?; + let request = wait_for_single_request(&phase2).await; + let prompt = phase2_prompt_text(&request); + + assert!( + prompt.contains("Memory extension resources removed by retention pruning:"), + "expected extension resource prune report in prompt: {prompt}" + ); + assert!( + prompt.contains("- retention window: 7 days"), + "expected retention window in prompt: {prompt}" + ); + assert!( + prompt.contains("- extension: telepathy"), + "expected extension name in prompt: {prompt}" + ); + assert!( + prompt.contains(&format!(" - resources/{old_file_name}")), + "expected old resource in prompt: {prompt}" + ); + + wait_for_phase2_success(db.as_ref(), thread_id).await?; + wait_for_file_removed(&old_file).await?; + assert!( + !tokio::fs::try_exists(&old_file).await?, + "old extension resource should be pruned" + ); + assert!( + tokio::fs::try_exists(&recent_file).await?, + "recent extension resource should be retained" + ); + + shutdown_test_codex(&codex).await?; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn memories_startup_phase2_processes_old_extension_resources_without_stage1_input() +-> Result<()> { + let server = start_mock_server().await; + let home = Arc::new(TempDir::new()?); + let db = init_state_db(&home).await?; + db.enqueue_global_consolidation(/*input_watermark*/ 1) + .await?; + + let now = Utc::now(); + let telepathy_resources = home.path().join("memories_extensions/telepathy/resources"); + tokio::fs::create_dir_all(&telepathy_resources).await?; + tokio::fs::write( + home.path() + .join("memories_extensions/telepathy/instructions.md"), + "instructions", + ) + .await?; + let old_file_name = format!( + "{}-abcd-10min-old.md", + (now - ChronoDuration::days(8)).format("%Y-%m-%dT%H-%M-%S") + ); + let old_file = telepathy_resources.join(&old_file_name); + tokio::fs::write(&old_file, "old resource").await?; + + let phase2 = mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-phase2-empty"), + ev_assistant_message("msg-phase2-empty", "phase2 complete"), + ev_completed("resp-phase2-empty"), + ]), + ) + .await; + + let codex = build_test_codex(&server, home.clone()).await?; + let request = wait_for_single_request(&phase2).await; + let prompt = phase2_prompt_text(&request); + + assert!( + prompt.contains("- selected inputs this run: 0"), + "expected no selected raw inputs in prompt: {prompt}" + ); + assert!( + prompt.contains("Memory extension resources removed by retention pruning:"), + "expected extension resource prune report in prompt: {prompt}" + ); + assert!( + prompt.contains(&format!(" - resources/{old_file_name}")), + "expected old resource in prompt: {prompt}" + ); + wait_for_file_removed(&old_file).await?; + + shutdown_test_codex(&codex).await?; + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn web_search_pollution_moves_selected_thread_into_removed_phase2_inputs() -> Result<()> { let server = start_mock_server().await; @@ -386,6 +525,21 @@ async fn wait_for_single_request(mock: &ResponseMock) -> ResponsesRequest { wait_for_request(mock, /*expected_count*/ 1).await.remove(0) } +async fn wait_for_file_removed(path: &Path) -> Result<()> { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + if !tokio::fs::try_exists(path).await? { + return Ok(()); + } + assert!( + Instant::now() < deadline, + "timed out waiting for {} to be removed", + path.display() + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + async fn wait_for_request(mock: &ResponseMock, expected_count: usize) -> Vec { let deadline = Instant::now() + Duration::from_secs(10); loop { diff --git a/codex-rs/core/tests/suite/mod.rs b/codex-rs/core/tests/suite/mod.rs index 8f3d9e15d9..9f8389c45e 100644 --- a/codex-rs/core/tests/suite/mod.rs +++ b/codex-rs/core/tests/suite/mod.rs @@ -85,7 +85,6 @@ mod rmcp_client; mod rollout_list_find; mod safety_check_downgrade; mod search_tool; -mod seatbelt; mod shell_command; mod shell_serialization; mod shell_snapshot; diff --git a/codex-rs/core/tests/suite/openai_file_mcp.rs b/codex-rs/core/tests/suite/openai_file_mcp.rs index 134ad6dfde..211aa67381 100644 --- a/codex-rs/core/tests/suite/openai_file_mcp.rs +++ b/codex-rs/core/tests/suite/openai_file_mcp.rs @@ -10,7 +10,7 @@ use core_test_support::apps_test_server::AppsTestServer; use core_test_support::apps_test_server::DOCUMENT_EXTRACT_TEXT_RESOURCE_URI; use core_test_support::responses::ev_assistant_message; use core_test_support::responses::ev_completed; -use core_test_support::responses::ev_function_call; +use core_test_support::responses::ev_function_call_with_namespace; use core_test_support::responses::ev_response_created; use core_test_support::responses::mount_sse_sequence; use core_test_support::responses::sse; @@ -26,7 +26,8 @@ use wiremock::matchers::header; use wiremock::matchers::method; use wiremock::matchers::path; -const DOCUMENT_EXTRACT_TOOL: &str = "mcp__codex_apps__calendar_extract_text"; +const DOCUMENT_EXTRACT_NAMESPACE: &str = "mcp__codex_apps__calendar"; +const DOCUMENT_EXTRACT_TOOL: &str = "_extract_text"; fn configure_apps(config: &mut Config, chatgpt_base_url: &str) { if let Err(err) = config.features.enable(Feature::Apps) { @@ -35,18 +36,6 @@ fn configure_apps(config: &mut Config, chatgpt_base_url: &str) { config.chatgpt_base_url = chatgpt_base_url.to_string(); } -fn tool_by_name<'a>(body: &'a Value, name: &str) -> &'a Value { - body.get("tools") - .and_then(Value::as_array) - .and_then(|tools| { - tools.iter().find(|tool| { - tool.get("name").and_then(Value::as_str) == Some(name) - || tool.get("type").and_then(Value::as_str) == Some(name) - }) - }) - .unwrap_or_else(|| panic!("missing tool {name} in /v1/responses request: {body:?}")) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn codex_apps_file_params_upload_local_paths_before_mcp_tool_call() -> Result<()> { let server = start_mock_server().await; @@ -93,8 +82,9 @@ async fn codex_apps_file_params_upload_local_paths_before_mcp_tool_call() -> Res vec![ sse(vec![ ev_response_created("resp-1"), - ev_function_call( + ev_function_call_with_namespace( call_id, + DOCUMENT_EXTRACT_NAMESPACE, DOCUMENT_EXTRACT_TOOL, &json!({"file": "report.txt"}).to_string(), ), @@ -123,8 +113,14 @@ async fn codex_apps_file_params_upload_local_paths_before_mcp_tool_call() -> Res .await?; let requests = mock.requests(); - let body = requests[0].body_json(); - let extract_tool = tool_by_name(&body, DOCUMENT_EXTRACT_TOOL); + let Some(extract_tool) = + requests[0].tool_by_name(DOCUMENT_EXTRACT_NAMESPACE, DOCUMENT_EXTRACT_TOOL) + else { + let body = requests[0].body_json(); + panic!( + "missing tool {DOCUMENT_EXTRACT_NAMESPACE}{DOCUMENT_EXTRACT_TOOL} in /v1/responses request: {body:?}" + ) + }; assert_eq!( extract_tool.pointer("/parameters/properties/file"), Some(&json!({ diff --git a/codex-rs/core/tests/suite/plugins.rs b/codex-rs/core/tests/suite/plugins.rs index ab2c85ac0c..b19385cd5d 100644 --- a/codex-rs/core/tests/suite/plugins.rs +++ b/codex-rs/core/tests/suite/plugins.rs @@ -168,22 +168,6 @@ fn tool_names(body: &serde_json::Value) -> Vec { .unwrap_or_default() } -fn tool_description(body: &serde_json::Value, tool_name: &str) -> Option { - body.get("tools") - .and_then(serde_json::Value::as_array) - .and_then(|tools| { - tools.iter().find_map(|tool| { - if tool.get("name").and_then(serde_json::Value::as_str) == Some(tool_name) { - tool.get("description") - .and_then(serde_json::Value::as_str) - .map(str::to_string) - } else { - None - } - }) - }) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn capability_sections_render_in_developer_message_in_order() -> Result<()> { skip_if_no_network!(Ok(())); @@ -319,20 +303,27 @@ async fn explicit_plugin_mentions_inject_plugin_guidance() -> Result<()> { assert!( request_tools .iter() - .any(|name| name == "mcp__codex_apps__google_calendar_create_event"), + .any(|name| name == "mcp__codex_apps__google_calendar"), "expected plugin app tools to become visible for this turn: {request_tools:?}" ); - let echo_description = tool_description(&request_body, "mcp__sample__echo") + let echo_tool = request + .tool_by_name("mcp__sample__", "echo") + .expect("plugin MCP tool should be present"); + let echo_description = echo_tool + .get("description") + .and_then(serde_json::Value::as_str) .expect("plugin MCP tool description should be present"); assert!( echo_description.contains("This tool is part of plugin `sample`."), "expected plugin MCP provenance in tool description: {echo_description:?}" ); - let calendar_description = tool_description( - &request_body, - "mcp__codex_apps__google_calendar_create_event", - ) - .expect("plugin app tool description should be present"); + let calendar_tool = request + .tool_by_name("mcp__codex_apps__google_calendar", "_create_event") + .expect("plugin app tool should be present"); + let calendar_description = calendar_tool + .get("description") + .and_then(serde_json::Value::as_str) + .expect("plugin app tool description should be present"); assert!( calendar_description.contains("This tool is part of plugin `sample`."), "expected plugin app provenance in tool description: {calendar_description:?}" diff --git a/codex-rs/core/tests/suite/realtime_conversation.rs b/codex-rs/core/tests/suite/realtime_conversation.rs index add4065f7b..1cd97391bb 100644 --- a/codex-rs/core/tests/suite/realtime_conversation.rs +++ b/codex-rs/core/tests/suite/realtime_conversation.rs @@ -57,6 +57,8 @@ use wiremock::matchers::method; use wiremock::matchers::path_regex; const STARTUP_CONTEXT_HEADER: &str = "Startup context from Codex."; +const STARTUP_CONTEXT_OPEN_TAG: &str = ""; +const STARTUP_CONTEXT_CLOSE_TAG: &str = ""; const REALTIME_BACKEND_PROMPT: &str = include_str!("../../templates/realtime/backend_prompt.md"); const USER_FIRST_NAME_PLACEHOLDER: &str = "{{ user_first_name }}"; const MEMORY_PROMPT_PHRASE: &str = @@ -194,10 +196,16 @@ async fn seed_recent_thread( let db = test.codex.state_db().context("state db enabled")?; let thread_id = ThreadId::new(); let updated_at = Utc::now(); + let rollout_path = test + .codex_home_path() + .join(format!("rollout-{thread_id}.jsonl")); + // This helper seeds SQLite metadata directly. Local listing drops stale metadata rows whose + // rollout path no longer exists, so create the placeholder path that the test metadata points + // at without exercising rollout writing in this realtime-context test. + std::fs::write(&rollout_path, "")?; let mut metadata_builder = codex_state::ThreadMetadataBuilder::new( thread_id, - test.codex_home_path() - .join(format!("rollout-{thread_id}.jsonl")), + rollout_path, updated_at, SessionSource::Cli, ); @@ -1524,6 +1532,8 @@ async fn conversation_start_injects_startup_context_from_thread_history() -> Res let startup_context = websocket_request_instructions(&startup_context_request) .expect("startup context request should contain instructions"); + assert!(startup_context.contains(STARTUP_CONTEXT_OPEN_TAG)); + assert!(startup_context.contains(STARTUP_CONTEXT_CLOSE_TAG)); assert!(startup_context.contains(STARTUP_CONTEXT_HEADER)); assert!(!startup_context.contains("## User")); assert!(startup_context.contains("### ")); @@ -1741,6 +1751,8 @@ async fn conversation_startup_context_falls_back_to_workspace_map() -> Result<() let startup_context = websocket_request_instructions(&startup_context_request) .expect("startup context request should contain instructions"); + assert!(startup_context.contains(STARTUP_CONTEXT_OPEN_TAG)); + assert!(startup_context.contains(STARTUP_CONTEXT_CLOSE_TAG)); assert!(startup_context.contains(STARTUP_CONTEXT_HEADER)); assert!(startup_context.contains("## Machine / Workspace Map")); assert!(startup_context.contains("notes.txt")); @@ -1795,6 +1807,8 @@ async fn conversation_startup_context_is_truncated_and_sent_once_per_start() -> .await; let startup_context = websocket_request_instructions(&startup_context_request) .expect("startup context request should contain instructions"); + assert!(startup_context.contains(STARTUP_CONTEXT_OPEN_TAG)); + assert!(startup_context.contains(STARTUP_CONTEXT_CLOSE_TAG)); assert!(startup_context.contains(STARTUP_CONTEXT_HEADER)); assert!(startup_context.len() <= 20_500); @@ -1873,6 +1887,7 @@ async fn conversation_user_text_turn_is_sent_to_realtime_when_active() -> Result assert_eq!(session_updated, "sess_user_text"); let user_text = "typed follow-up for realtime"; + let prefixed_user_text = format!("[USER] {user_text}"); test.codex .submit(Op::UserInput { items: vec![UserInput::Text { @@ -1892,7 +1907,7 @@ async fn conversation_user_text_turn_is_sent_to_realtime_when_active() -> Result let realtime_text_request = wait_for_matching_websocket_request( &realtime_server, "normal user turn text mirrored to realtime", - |request| websocket_request_text(request).as_deref() == Some(user_text), + |request| websocket_request_text(request).as_deref() == Some(prefixed_user_text.as_str()), ) .await; let model_user_texts = response_mock.single_request().message_input_texts("user"); @@ -1901,7 +1916,7 @@ async fn conversation_user_text_turn_is_sent_to_realtime_when_active() -> Result model_user_texts.iter().any(|text| text == user_text), websocket_request_text(&realtime_text_request), ), - (true, Some(user_text.to_string())), + (true, Some(prefixed_user_text)), ); let realtime_response_create = timeout(Duration::from_millis(200), async { wait_for_matching_websocket_request( diff --git a/codex-rs/core/tests/suite/responses_api_proxy_headers.rs b/codex-rs/core/tests/suite/responses_api_proxy_headers.rs index 7dc888ffce..67f6e86ab0 100644 --- a/codex-rs/core/tests/suite/responses_api_proxy_headers.rs +++ b/codex-rs/core/tests/suite/responses_api_proxy_headers.rs @@ -1,10 +1,16 @@ -//! Exercises a real `responses-api-proxy` process with request dumping enabled, then verifies that -//! parent and spawned subagent requests carry the expected window, parent-thread, and subagent -//! identity headers in the dumped Responses API requests. +//! Verifies that parent and spawned subagent Responses API requests carry the expected window, +//! parent-thread, and subagent identity headers. use anyhow::Result; use anyhow::anyhow; use codex_features::Feature; +use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::Op; +use codex_protocol::protocol::SandboxPolicy; +use codex_protocol::user_input::UserInput; +use core_test_support::responses::ResponseMock; +use core_test_support::responses::ResponsesRequest; use core_test_support::responses::ev_assistant_message; use core_test_support::responses::ev_completed; use core_test_support::responses::ev_function_call; @@ -13,98 +19,31 @@ use core_test_support::responses::mount_sse_once_match; use core_test_support::responses::sse; use core_test_support::responses::start_mock_server; use core_test_support::skip_if_no_network; +use core_test_support::test_codex::TestCodex; use core_test_support::test_codex::test_codex; use pretty_assertions::assert_eq; -use serde_json::Value; use serde_json::json; -use std::io::Write; -use std::path::Path; -use std::process::Child; -use std::process::Command as StdCommand; -use std::process::Stdio; use std::time::Duration; -use std::time::Instant; -use tempfile::TempDir; const PARENT_PROMPT: &str = "spawn a subagent and report when it is started"; const CHILD_PROMPT: &str = "child: say done"; const SPAWN_CALL_ID: &str = "spawn-call-1"; -const PROXY_START_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 5); -const PROXY_POLL_INTERVAL: Duration = Duration::from_millis(/*millis*/ 20); - -struct ResponsesApiProxy { - child: Child, - port: u16, -} - -impl ResponsesApiProxy { - fn start(upstream_url: &str, dump_dir: &Path) -> Result { - let server_info = dump_dir.join("server-info.json"); - let mut child = StdCommand::new(codex_utils_cargo_bin::cargo_bin("codex")?) - .args(["responses-api-proxy", "--server-info"]) - .arg(&server_info) - .args(["--upstream-url", upstream_url, "--dump-dir"]) - .arg(dump_dir) - .stdin(Stdio::piped()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn()?; - - child - .stdin - .take() - .ok_or_else(|| anyhow!("responses-api-proxy stdin was not piped"))? - .write_all(b"dummy\n")?; - - let deadline = Instant::now() + PROXY_START_TIMEOUT; - loop { - if let Ok(info) = std::fs::read_to_string(&server_info) { - let port = serde_json::from_str::(&info)? - .get("port") - .and_then(Value::as_u64) - .ok_or_else(|| anyhow!("proxy server info missing port"))?; - return Ok(Self { - child, - port: u16::try_from(port)?, - }); - } - if let Some(status) = child.try_wait()? { - return Err(anyhow!( - "responses-api-proxy exited before writing server info: {status}" - )); - } - if Instant::now() >= deadline { - return Err(anyhow!("timed out waiting for responses-api-proxy")); - } - std::thread::sleep(PROXY_POLL_INTERVAL); - } - } - - fn base_url(&self) -> String { - format!("http://127.0.0.1:{}/v1", self.port) - } -} - -impl Drop for ResponsesApiProxy { - fn drop(&mut self) { - let _ = self.child.kill(); - let _ = self.child.wait(); - } -} +const REQUEST_POLL_INTERVAL: Duration = Duration::from_millis(/*millis*/ 20); +const TURN_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 60); #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn responses_api_proxy_dumps_parent_and_subagent_identity_headers() -> Result<()> { +async fn responses_api_parent_and_subagent_requests_include_identity_headers() -> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; - let dump_dir = TempDir::new()?; - let proxy = - ResponsesApiProxy::start(&format!("{}/v1/responses", server.uri()), dump_dir.path())?; let spawn_args = serde_json::to_string(&json!({ "message": CHILD_PROMPT }))?; - mount_sse_once_match( + let parent_mock = mount_sse_once_match( &server, - |req: &wiremock::Request| request_body_contains(req, PARENT_PROMPT), + |req: &wiremock::Request| { + request_body_contains(req, PARENT_PROMPT) + && request_header(req, "x-openai-subagent").is_none() + }, sse(vec![ ev_response_created("resp-parent-1"), ev_function_call(SPAWN_CALL_ID, "spawn_agent", &spawn_args), @@ -112,10 +51,12 @@ async fn responses_api_proxy_dumps_parent_and_subagent_identity_headers() -> Res ]), ) .await; - mount_sse_once_match( + let child_mock = mount_sse_once_match( &server, |req: &wiremock::Request| { - request_body_contains(req, CHILD_PROMPT) && !request_body_contains(req, SPAWN_CALL_ID) + request_body_contains(req, CHILD_PROMPT) + && !request_body_contains(req, SPAWN_CALL_ID) + && request_header(req, "x-openai-subagent") == Some("collab_spawn") }, sse(vec![ ev_response_created("resp-child-1"), @@ -126,7 +67,10 @@ async fn responses_api_proxy_dumps_parent_and_subagent_identity_headers() -> Res .await; mount_sse_once_match( &server, - |req: &wiremock::Request| request_body_contains(req, SPAWN_CALL_ID), + |req: &wiremock::Request| { + request_body_contains(req, SPAWN_CALL_ID) + && request_header(req, "x-openai-subagent").is_none() + }, sse(vec![ ev_response_created("resp-parent-2"), ev_assistant_message("msg-parent-2", "parent done"), @@ -135,100 +79,158 @@ async fn responses_api_proxy_dumps_parent_and_subagent_identity_headers() -> Res ) .await; - let proxy_base_url = proxy.base_url(); - let mut builder = test_codex().with_config(move |config| { - config.model_provider.base_url = Some(proxy_base_url); + let mut builder = test_codex().with_config(|config| { config .features .disable(Feature::EnableRequestCompression) .expect("test config should allow feature update"); }); let test = builder.build(&server).await?; - test.submit_turn(PARENT_PROMPT).await?; + submit_turn_with_timeout(&test, PARENT_PROMPT).await?; - let dumps = wait_for_proxy_request_dumps(dump_dir.path())?; - let parent = dumps - .iter() - .find(|dump| dump_body_contains(dump, PARENT_PROMPT)) - .ok_or_else(|| anyhow!("missing parent request dump"))?; - let child = dumps - .iter() - .find(|dump| { - dump_body_contains(dump, CHILD_PROMPT) && !dump_body_contains(dump, SPAWN_CALL_ID) - }) - .ok_or_else(|| anyhow!("missing child request dump"))?; + let parent = wait_for_matching_request(&parent_mock, "parent request", |request| { + request.body_contains_text(PARENT_PROMPT) && request.header("x-openai-subagent").is_none() + }) + .await?; + let child = wait_for_matching_request(&child_mock, "child request", |request| { + request.body_contains_text(CHILD_PROMPT) + && !request.body_contains_text(SPAWN_CALL_ID) + && request.header("x-openai-subagent").as_deref() == Some("collab_spawn") + }) + .await?; - let parent_window_id = header(parent, "x-codex-window-id") + let parent_window_id = parent + .header("x-codex-window-id") .ok_or_else(|| anyhow!("parent request missing x-codex-window-id"))?; - let child_window_id = header(child, "x-codex-window-id") + let child_window_id = child + .header("x-codex-window-id") .ok_or_else(|| anyhow!("child request missing x-codex-window-id"))?; - let (parent_thread_id, parent_generation) = split_window_id(parent_window_id)?; - let (child_thread_id, child_generation) = split_window_id(child_window_id)?; + let (parent_thread_id, parent_generation) = split_window_id(&parent_window_id)?; + let (child_thread_id, child_generation) = split_window_id(&child_window_id)?; assert_eq!(parent_generation, 0); assert_eq!(child_generation, 0); assert!(child_thread_id != parent_thread_id); - assert_eq!(header(parent, "x-openai-subagent"), None); - assert_eq!(header(child, "x-openai-subagent"), Some("collab_spawn")); + assert_eq!(parent.header("x-openai-subagent"), None); assert_eq!( - header(child, "x-codex-parent-thread-id"), + child.header("x-openai-subagent").as_deref(), + Some("collab_spawn") + ); + assert_eq!( + child.header("x-codex-parent-thread-id").as_deref(), Some(parent_thread_id) ); Ok(()) } +async fn submit_turn_with_timeout(test: &TestCodex, prompt: &str) -> Result<()> { + let session_model = test.session_configured.model.clone(); + test.codex + .submit(Op::UserTurn { + items: vec![UserInput::Text { + text: prompt.into(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + cwd: test.config.cwd.to_path_buf(), + approval_policy: AskForApproval::OnRequest, + approvals_reviewer: None, + sandbox_policy: SandboxPolicy::WorkspaceWrite { + writable_roots: Vec::new(), + read_only_access: Default::default(), + network_access: false, + exclude_tmpdir_env_var: false, + exclude_slash_tmp: false, + }, + model: session_model, + effort: None, + summary: None, + service_tier: None, + collaboration_mode: None, + personality: None, + }) + .await?; + + let turn_started = wait_for_event_result(test, "turn started", |event| { + matches!(event, EventMsg::TurnStarted(_)) + }) + .await?; + let EventMsg::TurnStarted(turn_started) = turn_started else { + unreachable!("event predicate only matches turn started events"); + }; + wait_for_event_result(test, "turn complete", |event| match event { + EventMsg::TurnComplete(event) => event.turn_id == turn_started.turn_id, + _ => false, + }) + .await?; + + Ok(()) +} + +async fn wait_for_matching_request( + mock: &ResponseMock, + label: &str, + mut predicate: F, +) -> Result +where + F: FnMut(&ResponsesRequest) -> bool, +{ + tokio::time::timeout(TURN_TIMEOUT, async { + loop { + if let Some(request) = mock + .requests() + .into_iter() + .find(|request| predicate(request)) + { + return request; + } + tokio::time::sleep(REQUEST_POLL_INTERVAL).await; + } + }) + .await + .map_err(|_| anyhow!("timed out waiting for {label}")) +} + +async fn wait_for_event_result( + test: &TestCodex, + stage: &str, + mut predicate: F, +) -> Result +where + F: FnMut(&EventMsg) -> bool, +{ + let mut seen_events = Vec::new(); + tokio::time::timeout(TURN_TIMEOUT, async { + loop { + let event = test.codex.next_event().await?; + seen_events.push(event_summary(&event.msg)); + if predicate(&event.msg) { + return Ok::(event.msg); + } + } + }) + .await + .map_err(|_| { + anyhow!( + "timed out waiting for {stage}; saw events: {}", + seen_events.join(" | ") + ) + })? +} + +fn event_summary(event: &EventMsg) -> String { + let mut summary = format!("{event:?}"); + summary.truncate(240); + summary +} + fn request_body_contains(req: &wiremock::Request, text: &str) -> bool { std::str::from_utf8(&req.body).is_ok_and(|body| body.contains(text)) } -fn wait_for_proxy_request_dumps(dump_dir: &Path) -> Result> { - let deadline = Instant::now() + Duration::from_secs(/*secs*/ 2); - loop { - let dumps = read_proxy_request_dumps(dump_dir).unwrap_or_default(); - if dumps.len() >= 3 - && dumps - .iter() - .any(|dump| dump_body_contains(dump, CHILD_PROMPT)) - { - return Ok(dumps); - } - if Instant::now() >= deadline { - return Err(anyhow!( - "timed out waiting for proxy request dumps, got {}", - dumps.len() - )); - } - std::thread::sleep(PROXY_POLL_INTERVAL); - } -} - -fn read_proxy_request_dumps(dump_dir: &Path) -> Result> { - let mut dumps = Vec::new(); - for entry in std::fs::read_dir(dump_dir)? { - let path = entry?.path(); - if path - .file_name() - .and_then(|name| name.to_str()) - .is_some_and(|name| name.ends_with("-request.json")) - { - dumps.push(serde_json::from_str(&std::fs::read_to_string(&path)?)?); - } - } - Ok(dumps) -} - -fn dump_body_contains(dump: &Value, text: &str) -> bool { - dump.get("body") - .is_some_and(|body| body.to_string().contains(text)) -} - -fn header<'a>(dump: &'a Value, name: &str) -> Option<&'a str> { - dump.get("headers")?.as_array()?.iter().find_map(|header| { - (header.get("name")?.as_str()?.eq_ignore_ascii_case(name)) - .then(|| header.get("value")?.as_str()) - .flatten() - }) +fn request_header<'a>(req: &'a wiremock::Request, name: &str) -> Option<&'a str> { + req.headers.get(name).and_then(|value| value.to_str().ok()) } fn split_window_id(window_id: &str) -> Result<(&str, u64)> { diff --git a/codex-rs/core/tests/suite/resume_warning.rs b/codex-rs/core/tests/suite/resume_warning.rs index f9ad64c771..8ea2be3147 100644 --- a/codex-rs/core/tests/suite/resume_warning.rs +++ b/codex-rs/core/tests/suite/resume_warning.rs @@ -34,6 +34,7 @@ fn resume_history( approval_policy: config.permissions.approval_policy.value(), sandbox_policy: config.permissions.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: previous_model.to_string(), personality: None, collaboration_mode: None, diff --git a/codex-rs/core/tests/suite/rmcp_client.rs b/codex-rs/core/tests/suite/rmcp_client.rs index b0ca1d7561..8c7e7133a3 100644 --- a/codex-rs/core/tests/suite/rmcp_client.rs +++ b/codex-rs/core/tests/suite/rmcp_client.rs @@ -11,7 +11,10 @@ use std::time::UNIX_EPOCH; use codex_config::types::McpServerConfig; use codex_config::types::McpServerTransportConfig; +use codex_core::config::Config; +use codex_features::Feature; use codex_login::CodexAuth; +use codex_mcp::MCP_SANDBOX_STATE_META_CAPABILITY; use codex_models_manager::manager::RefreshStrategy; use codex_protocol::config_types::ReasoningSummary; @@ -32,10 +35,12 @@ use codex_protocol::user_input::UserInput; use codex_utils_cargo_bin::cargo_bin; use core_test_support::assert_regex_match; use core_test_support::responses; +use core_test_support::responses::ev_custom_tool_call; use core_test_support::responses::mount_models_once; use core_test_support::responses::mount_sse_once; use core_test_support::skip_if_no_network; use core_test_support::stdio_server_bin; +use core_test_support::test_codex::TestCodex; use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event; use core_test_support::wait_for_event_with_timeout; @@ -81,6 +86,84 @@ enum McpCallEvent { End(String), } +async fn wait_for_mcp_tool(fixture: &TestCodex, tool_name: &str) -> anyhow::Result<()> { + let tools_ready_deadline = Instant::now() + Duration::from_secs(30); + loop { + fixture.codex.submit(Op::ListMcpTools).await?; + let list_event = wait_for_event_with_timeout( + &fixture.codex, + |ev| matches!(ev, EventMsg::McpListToolsResponse(_)), + Duration::from_secs(10), + ) + .await; + let EventMsg::McpListToolsResponse(tool_list) = list_event else { + unreachable!("event guard guarantees McpListToolsResponse"); + }; + if tool_list.tools.contains_key(tool_name) { + return Ok(()); + } + + let available_tools: Vec<&str> = tool_list.tools.keys().map(String::as_str).collect(); + if Instant::now() >= tools_ready_deadline { + panic!( + "timed out waiting for MCP tool {tool_name} to become available; discovered tools: {available_tools:?}" + ); + } + sleep(Duration::from_millis(200)).await; + } +} + +#[derive(Default)] +struct TestMcpServerOptions { + supports_parallel_tool_calls: bool, + tool_timeout_sec: Option, +} + +fn stdio_transport( + command: String, + env: Option>, + env_vars: Vec, +) -> McpServerTransportConfig { + McpServerTransportConfig::Stdio { + command, + args: Vec::new(), + env, + env_vars, + cwd: None, + } +} + +fn insert_mcp_server( + config: &mut Config, + server_name: &str, + transport: McpServerTransportConfig, + options: TestMcpServerOptions, +) { + let mut servers = config.mcp_servers.get().clone(); + servers.insert( + server_name.to_string(), + McpServerConfig { + transport, + experimental_environment: None, + enabled: true, + required: false, + supports_parallel_tool_calls: options.supports_parallel_tool_calls, + disabled_reason: None, + startup_timeout_sec: Some(Duration::from_secs(10)), + tool_timeout_sec: options.tool_timeout_sec, + default_tools_approval_mode: None, + enabled_tools: None, + disabled_tools: None, + scopes: None, + oauth_resource: None, + tools: HashMap::new(), + }, + ); + if let Err(err) = config.mcp_servers.set(servers) { + panic!("test mcp servers should accept any configuration: {err}"); + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] #[serial(mcp_test_value)] async fn stdio_server_round_trip() -> anyhow::Result<()> { @@ -90,13 +173,18 @@ async fn stdio_server_round_trip() -> anyhow::Result<()> { let call_id = "call-123"; let server_name = "rmcp"; - let tool_name = format!("mcp__{server_name}__echo"); + let namespace = format!("mcp__{server_name}__"); - mount_sse_once( + let call_mock = mount_sse_once( &server, responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, &tool_name, "{\"message\":\"ping\"}"), + responses::ev_function_call_with_namespace( + call_id, + &namespace, + "echo", + "{\"message\":\"ping\"}", + ), responses::ev_completed("resp-1"), ]), ) @@ -115,37 +203,19 @@ async fn stdio_server_round_trip() -> anyhow::Result<()> { let fixture = test_codex() .with_config(move |config| { - let mut servers = config.mcp_servers.get().clone(); - servers.insert( - server_name.to_string(), - McpServerConfig { - transport: McpServerTransportConfig::Stdio { - command: rmcp_test_server_bin, - args: Vec::new(), - env: Some(HashMap::from([( - "MCP_TEST_VALUE".to_string(), - expected_env_value.to_string(), - )])), - env_vars: Vec::new(), - cwd: None, - }, - enabled: true, - required: false, - supports_parallel_tool_calls: false, - disabled_reason: None, - startup_timeout_sec: Some(Duration::from_secs(10)), - tool_timeout_sec: None, - enabled_tools: None, - disabled_tools: None, - scopes: None, - oauth_resource: None, - tools: HashMap::new(), - }, + insert_mcp_server( + config, + server_name, + stdio_transport( + rmcp_test_server_bin, + Some(HashMap::from([( + "MCP_TEST_VALUE".to_string(), + expected_env_value.to_string(), + )])), + Vec::new(), + ), + TestMcpServerOptions::default(), ); - config - .mcp_servers - .set(servers) - .expect("test mcp servers should accept any configuration"); }) .build(&server) .await?; @@ -222,6 +292,13 @@ async fn stdio_server_round_trip() -> anyhow::Result<()> { wait_for_event(&fixture.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; let output_item = final_mock.single_request().function_call_output(call_id); + let request = call_mock.single_request(); + assert!( + request.tool_by_name(&namespace, "echo").is_some(), + "direct MCP tool should be sent as a namespace child tool: {:?}", + request.body_json() + ); + let output_text = output_item .get("output") .and_then(Value::as_str) @@ -237,6 +314,116 @@ async fn stdio_server_round_trip() -> anyhow::Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn stdio_mcp_tool_call_includes_sandbox_state_meta() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + + let call_id = "sandbox-meta-call"; + let server_name = "rmcp"; + let namespace = format!("mcp__{server_name}__"); + let tool_name = format!("{namespace}sandbox_meta"); + + let call_mock = mount_sse_once( + &server, + responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_function_call_with_namespace(call_id, &namespace, "sandbox_meta", "{}"), + responses::ev_completed("resp-1"), + ]), + ) + .await; + let final_mock = mount_sse_once( + &server, + responses::sse(vec![ + responses::ev_assistant_message("msg-1", "rmcp sandbox meta completed successfully."), + responses::ev_completed("resp-2"), + ]), + ) + .await; + + let rmcp_test_server_bin = stdio_server_bin()?; + let fixture = test_codex() + .with_config(move |config| { + insert_mcp_server( + config, + server_name, + stdio_transport(rmcp_test_server_bin, /*env*/ None, Vec::new()), + TestMcpServerOptions::default(), + ); + }) + .build(&server) + .await?; + + let tools_ready_deadline = Instant::now() + Duration::from_secs(30); + loop { + fixture.codex.submit(Op::ListMcpTools).await?; + let list_event = wait_for_event_with_timeout( + &fixture.codex, + |ev| matches!(ev, EventMsg::McpListToolsResponse(_)), + Duration::from_secs(10), + ) + .await; + let EventMsg::McpListToolsResponse(tool_list) = list_event else { + unreachable!("event guard guarantees McpListToolsResponse"); + }; + if tool_list.tools.contains_key(&tool_name) { + break; + } + + let available_tools: Vec<&str> = tool_list.tools.keys().map(String::as_str).collect(); + if Instant::now() >= tools_ready_deadline { + panic!( + "timed out waiting for MCP tool {tool_name} to become available; discovered tools: {available_tools:?}" + ); + } + sleep(Duration::from_millis(200)).await; + } + + let sandbox_policy = SandboxPolicy::new_read_only_policy(); + fixture + .submit_turn_with_policy("call the rmcp sandbox_meta tool", sandbox_policy.clone()) + .await?; + + let request = call_mock.single_request(); + assert!( + request.tool_by_name(&namespace, "sandbox_meta").is_some(), + "direct MCP tool should be sent as a namespace child tool: {:?}", + request.body_json() + ); + + let output_item = final_mock.single_request().function_call_output(call_id); + let output_text = output_item + .get("output") + .and_then(Value::as_str) + .expect("function_call_output output should be a string"); + let wrapped_payload = split_wall_time_wrapped_output(output_text); + let output_json: Value = serde_json::from_str(wrapped_payload) + .expect("wrapped MCP output should preserve sandbox metadata JSON"); + let Value::Object(meta) = output_json else { + panic!("sandbox_meta should return metadata object: {output_json:?}"); + }; + + let sandbox_meta = meta + .get(MCP_SANDBOX_STATE_META_CAPABILITY) + .expect("sandbox state metadata should be present"); + let expected_sandbox_policy = serde_json::to_value(&sandbox_policy)?; + assert_eq!( + sandbox_meta.get("sandboxPolicy"), + Some(&expected_sandbox_policy) + ); + assert_eq!( + sandbox_meta.get("sandboxCwd").and_then(Value::as_str), + fixture.cwd.path().to_str() + ); + assert_eq!(sandbox_meta.get("useLegacyLandlock"), Some(&json!(false))); + + server.verify().await; + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn stdio_mcp_parallel_tool_calls_default_false_runs_serially() -> anyhow::Result<()> { skip_if_no_network!(Ok(())); @@ -246,15 +433,15 @@ async fn stdio_mcp_parallel_tool_calls_default_false_runs_serially() -> anyhow:: let first_call_id = "sync-serial-1"; let second_call_id = "sync-serial-2"; let server_name = "rmcp"; - let tool_name = format!("mcp__{server_name}__sync"); + let namespace = format!("mcp__{server_name}__"); let args = json!({ "sleep_after_ms": 100 }).to_string(); mount_sse_once( &server, responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(first_call_id, &tool_name, &args), - responses::ev_function_call(second_call_id, &tool_name, &args), + responses::ev_function_call_with_namespace(first_call_id, &namespace, "sync", &args), + responses::ev_function_call_with_namespace(second_call_id, &namespace, "sync", &args), responses::ev_completed("resp-1"), ]), ) @@ -272,34 +459,15 @@ async fn stdio_mcp_parallel_tool_calls_default_false_runs_serially() -> anyhow:: let fixture = test_codex() .with_config(move |config| { - let mut servers = config.mcp_servers.get().clone(); - servers.insert( - server_name.to_string(), - McpServerConfig { - transport: McpServerTransportConfig::Stdio { - command: rmcp_test_server_bin, - args: Vec::new(), - env: None, - env_vars: Vec::new(), - cwd: None, - }, - enabled: true, - required: false, - supports_parallel_tool_calls: false, - disabled_reason: None, - startup_timeout_sec: Some(Duration::from_secs(10)), + insert_mcp_server( + config, + server_name, + stdio_transport(rmcp_test_server_bin, /*env*/ None, Vec::new()), + TestMcpServerOptions { tool_timeout_sec: Some(Duration::from_secs(2)), - enabled_tools: None, - disabled_tools: None, - scopes: None, - oauth_resource: None, - tools: HashMap::new(), + ..Default::default() }, ); - config - .mcp_servers - .set(servers) - .expect("test mcp servers should accept any configuration"); }) .build(&server) .await?; @@ -388,7 +556,7 @@ async fn stdio_mcp_parallel_tool_calls_opt_in_runs_concurrently() -> anyhow::Res let first_call_id = "sync-1"; let second_call_id = "sync-2"; let server_name = "rmcp"; - let tool_name = format!("mcp__{server_name}__sync"); + let namespace = format!("mcp__{server_name}__"); let args = json!({ "sleep_after_ms": 100, "barrier": { @@ -403,8 +571,8 @@ async fn stdio_mcp_parallel_tool_calls_opt_in_runs_concurrently() -> anyhow::Res &server, responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(first_call_id, &tool_name, &args), - responses::ev_function_call(second_call_id, &tool_name, &args), + responses::ev_function_call_with_namespace(first_call_id, &namespace, "sync", &args), + responses::ev_function_call_with_namespace(second_call_id, &namespace, "sync", &args), responses::ev_completed("resp-1"), ]), ) @@ -422,34 +590,15 @@ async fn stdio_mcp_parallel_tool_calls_opt_in_runs_concurrently() -> anyhow::Res let fixture = test_codex() .with_config(move |config| { - let mut servers = config.mcp_servers.get().clone(); - servers.insert( - server_name.to_string(), - McpServerConfig { - transport: McpServerTransportConfig::Stdio { - command: rmcp_test_server_bin, - args: Vec::new(), - env: None, - env_vars: Vec::new(), - cwd: None, - }, - enabled: true, - required: false, + insert_mcp_server( + config, + server_name, + stdio_transport(rmcp_test_server_bin, /*env*/ None, Vec::new()), + TestMcpServerOptions { supports_parallel_tool_calls: true, - disabled_reason: None, - startup_timeout_sec: Some(Duration::from_secs(10)), tool_timeout_sec: Some(Duration::from_secs(2)), - enabled_tools: None, - disabled_tools: None, - scopes: None, - oauth_resource: None, - tools: HashMap::new(), }, ); - config - .mcp_servers - .set(servers) - .expect("test mcp servers should accept any configuration"); }) .build(&server) .await?; @@ -504,13 +653,14 @@ async fn stdio_image_responses_round_trip() -> anyhow::Result<()> { let call_id = "img-1"; let server_name = "rmcp"; let tool_name = format!("mcp__{server_name}__image"); + let namespace = format!("mcp__{server_name}__"); // First stream: model decides to call the image tool. mount_sse_once( &server, responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, &tool_name, "{}"), + responses::ev_function_call_with_namespace(call_id, &namespace, "image", "{}"), responses::ev_completed("resp-1"), ]), ) @@ -530,66 +680,25 @@ async fn stdio_image_responses_round_trip() -> anyhow::Result<()> { let fixture = test_codex() .with_config(move |config| { - let mut servers = config.mcp_servers.get().clone(); - servers.insert( - server_name.to_string(), - McpServerConfig { - transport: McpServerTransportConfig::Stdio { - command: rmcp_test_server_bin, - args: Vec::new(), - env: Some(HashMap::from([( - "MCP_TEST_IMAGE_DATA_URL".to_string(), - OPENAI_PNG.to_string(), - )])), - env_vars: Vec::new(), - cwd: None, - }, - enabled: true, - required: false, - supports_parallel_tool_calls: false, - disabled_reason: None, - startup_timeout_sec: Some(Duration::from_secs(10)), - tool_timeout_sec: None, - enabled_tools: None, - disabled_tools: None, - scopes: None, - oauth_resource: None, - tools: HashMap::new(), - }, + insert_mcp_server( + config, + server_name, + stdio_transport( + rmcp_test_server_bin, + Some(HashMap::from([( + "MCP_TEST_IMAGE_DATA_URL".to_string(), + OPENAI_PNG.to_string(), + )])), + Vec::new(), + ), + TestMcpServerOptions::default(), ); - config - .mcp_servers - .set(servers) - .expect("test mcp servers should accept any configuration"); }) .build(&server) .await?; let session_model = fixture.session_configured.model.clone(); - let tools_ready_deadline = Instant::now() + Duration::from_secs(30); - loop { - fixture.codex.submit(Op::ListMcpTools).await?; - let list_event = wait_for_event_with_timeout( - &fixture.codex, - |ev| matches!(ev, EventMsg::McpListToolsResponse(_)), - Duration::from_secs(10), - ) - .await; - let EventMsg::McpListToolsResponse(tool_list) = list_event else { - unreachable!("event guard guarantees McpListToolsResponse"); - }; - if tool_list.tools.contains_key(&tool_name) { - break; - } - - let available_tools: Vec<&str> = tool_list.tools.keys().map(String::as_str).collect(); - if Instant::now() >= tools_ready_deadline { - panic!( - "timed out waiting for MCP tool {tool_name} to become available; discovered tools: {available_tools:?}" - ); - } - sleep(Duration::from_millis(200)).await; - } + wait_for_mcp_tool(&fixture, &tool_name).await?; fixture .codex @@ -629,6 +738,7 @@ async fn stdio_image_responses_round_trip() -> anyhow::Result<()> { tool: "image".to_string(), arguments: Some(json!({})), }, + mcp_app_resource_uri: None, }, ); @@ -684,6 +794,189 @@ async fn stdio_image_responses_round_trip() -> anyhow::Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[serial(mcp_test_value)] +async fn stdio_image_responses_preserve_original_detail_metadata() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + + let call_id = "img-original-detail-1"; + let server_name = "rmcp"; + let tool_name = format!("mcp__{server_name}__image_scenario"); + let namespace = format!("mcp__{server_name}__"); + + mount_sse_once( + &server, + responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_function_call_with_namespace( + call_id, + &namespace, + "image_scenario", + r#"{"scenario":"image_only_original_detail"}"#, + ), + responses::ev_completed("resp-1"), + ]), + ) + .await; + let final_mock = mount_sse_once( + &server, + responses::sse(vec![ + responses::ev_assistant_message("msg-1", "rmcp original-detail image completed."), + responses::ev_completed("resp-2"), + ]), + ) + .await; + + let rmcp_test_server_bin = stdio_server_bin()?; + + let fixture = test_codex() + .with_model("gpt-5.3-codex") + .with_config(move |config| { + insert_mcp_server( + config, + server_name, + stdio_transport(rmcp_test_server_bin, /*env*/ None, Vec::new()), + TestMcpServerOptions::default(), + ); + }) + .build(&server) + .await?; + let session_model = fixture.session_configured.model.clone(); + + wait_for_mcp_tool(&fixture, &tool_name).await?; + + fixture + .codex + .submit(Op::UserTurn { + items: vec![UserInput::Text { + text: "call the rmcp image_scenario tool".into(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + cwd: fixture.cwd.path().to_path_buf(), + approval_policy: AskForApproval::Never, + approvals_reviewer: None, + sandbox_policy: SandboxPolicy::new_read_only_policy(), + model: session_model, + effort: None, + summary: None, + service_tier: None, + collaboration_mode: None, + personality: None, + }) + .await?; + + wait_for_event(&fixture.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + + let output_item = final_mock.single_request().function_call_output(call_id); + let output = output_item["output"] + .as_array() + .expect("image MCP output should be content items"); + assert_eq!(output.len(), 2); + assert_wall_time_header( + output[0]["text"] + .as_str() + .expect("first MCP image output item should be wall-time text"), + ); + assert_eq!( + output[1], + json!({ + "type": "input_image", + "image_url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==", + "detail": "original", + }) + ); + + server.verify().await; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[serial(mcp_test_value)] +async fn js_repl_emit_image_preserves_original_detail_for_mcp_images() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let call_id = "js-repl-rmcp-image"; + let rmcp_test_server_bin = stdio_server_bin()?; + + let fixture = test_codex() + .with_model("gpt-5.3-codex") + .with_config(move |config| { + config + .features + .enable(Feature::JsRepl) + .expect("test config should allow feature update"); + insert_mcp_server( + config, + "rmcp", + stdio_transport(rmcp_test_server_bin, /*env*/ None, Vec::new()), + TestMcpServerOptions::default(), + ); + }) + .build(&server) + .await?; + + wait_for_mcp_tool(&fixture, "mcp__rmcp__image_scenario").await?; + + mount_sse_once( + &server, + responses::sse(vec![ + responses::ev_response_created("resp-1"), + ev_custom_tool_call( + call_id, + "js_repl", + r#" +const out = await codex.tool("mcp__rmcp__image_scenario", { + scenario: "image_only_original_detail", +}); +const imageItem = out.output.find((item) => item.type === "input_image"); +await codex.emitImage(imageItem); +"#, + ), + responses::ev_completed("resp-1"), + ]), + ) + .await; + let final_mock = mount_sse_once( + &server, + responses::sse(vec![ + responses::ev_assistant_message("msg-1", "done"), + responses::ev_completed("resp-2"), + ]), + ) + .await; + + fixture + .submit_turn("use js_repl to emit the rmcp image scenario output") + .await?; + + let output = final_mock.single_request().custom_tool_call_output(call_id); + let output_items = output["output"] + .as_array() + .expect("js_repl output should be content items"); + let image_item = output_items + .iter() + .find(|item| item.get("type").and_then(Value::as_str) == Some("input_image")) + .expect("js_repl should emit an input_image item"); + assert_eq!( + image_item.get("detail").and_then(Value::as_str), + Some("original") + ); + assert!( + image_item + .get("image_url") + .and_then(Value::as_str) + .is_some_and(|image_url| image_url.starts_with("data:image/png;base64,")), + "js_repl should emit a png data URL" + ); + + server.verify().await; + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] #[serial(mcp_test_value)] async fn stdio_image_responses_are_sanitized_for_text_only_model() -> anyhow::Result<()> { @@ -693,7 +986,7 @@ async fn stdio_image_responses_are_sanitized_for_text_only_model() -> anyhow::Re let call_id = "img-text-only-1"; let server_name = "rmcp"; - let tool_name = format!("mcp__{server_name}__image"); + let namespace = format!("mcp__{server_name}__"); let text_only_model_slug = "rmcp-text-only-model"; let models_mock = mount_models_once( @@ -743,7 +1036,7 @@ async fn stdio_image_responses_are_sanitized_for_text_only_model() -> anyhow::Re &server, responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, &tool_name, "{}"), + responses::ev_function_call_with_namespace(call_id, &namespace, "image", "{}"), responses::ev_completed("resp-1"), ]), ) @@ -763,37 +1056,19 @@ async fn stdio_image_responses_are_sanitized_for_text_only_model() -> anyhow::Re let fixture = test_codex() .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) .with_config(move |config| { - let mut servers = config.mcp_servers.get().clone(); - servers.insert( - server_name.to_string(), - McpServerConfig { - transport: McpServerTransportConfig::Stdio { - command: rmcp_test_server_bin, - args: Vec::new(), - env: Some(HashMap::from([( - "MCP_TEST_IMAGE_DATA_URL".to_string(), - OPENAI_PNG.to_string(), - )])), - env_vars: Vec::new(), - cwd: None, - }, - enabled: true, - required: false, - supports_parallel_tool_calls: false, - disabled_reason: None, - startup_timeout_sec: Some(Duration::from_secs(10)), - tool_timeout_sec: None, - enabled_tools: None, - disabled_tools: None, - scopes: None, - oauth_resource: None, - tools: HashMap::new(), - }, + insert_mcp_server( + config, + server_name, + stdio_transport( + rmcp_test_server_bin, + Some(HashMap::from([( + "MCP_TEST_IMAGE_DATA_URL".to_string(), + OPENAI_PNG.to_string(), + )])), + Vec::new(), + ), + TestMcpServerOptions::default(), ); - config - .mcp_servers - .set(servers) - .expect("test mcp servers should accept any configuration"); }) .build(&server) .await?; @@ -864,13 +1139,18 @@ async fn stdio_server_propagates_whitelisted_env_vars() -> anyhow::Result<()> { let call_id = "call-1234"; let server_name = "rmcp_whitelist"; - let tool_name = format!("mcp__{server_name}__echo"); + let namespace = format!("mcp__{server_name}__"); mount_sse_once( &server, responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, &tool_name, "{\"message\":\"ping\"}"), + responses::ev_function_call_with_namespace( + call_id, + &namespace, + "echo", + "{\"message\":\"ping\"}", + ), responses::ev_completed("resp-1"), ]), ) @@ -890,34 +1170,16 @@ async fn stdio_server_propagates_whitelisted_env_vars() -> anyhow::Result<()> { let fixture = test_codex() .with_config(move |config| { - let mut servers = config.mcp_servers.get().clone(); - servers.insert( - server_name.to_string(), - McpServerConfig { - transport: McpServerTransportConfig::Stdio { - command: rmcp_test_server_bin, - args: Vec::new(), - env: None, - env_vars: vec!["MCP_TEST_VALUE".to_string()], - cwd: None, - }, - enabled: true, - required: false, - supports_parallel_tool_calls: false, - disabled_reason: None, - startup_timeout_sec: Some(Duration::from_secs(10)), - tool_timeout_sec: None, - enabled_tools: None, - disabled_tools: None, - scopes: None, - oauth_resource: None, - tools: HashMap::new(), - }, + insert_mcp_server( + config, + server_name, + stdio_transport( + rmcp_test_server_bin, + /*env*/ None, + vec!["MCP_TEST_VALUE".to_string()], + ), + TestMcpServerOptions::default(), ); - config - .mcp_servers - .set(servers) - .expect("test mcp servers should accept any configuration"); }) .build(&server) .await?; @@ -1006,13 +1268,18 @@ async fn streamable_http_tool_call_round_trip() -> anyhow::Result<()> { let call_id = "call-456"; let server_name = "rmcp_http"; - let tool_name = format!("mcp__{server_name}__echo"); + let namespace = format!("mcp__{server_name}__"); mount_sse_once( &server, responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, &tool_name, "{\"message\":\"ping\"}"), + responses::ev_function_call_with_namespace( + call_id, + &namespace, + "echo", + "{\"message\":\"ping\"}", + ), responses::ev_completed("resp-1"), ]), ) @@ -1055,33 +1322,17 @@ async fn streamable_http_tool_call_round_trip() -> anyhow::Result<()> { let fixture = test_codex() .with_config(move |config| { - let mut servers = config.mcp_servers.get().clone(); - servers.insert( - server_name.to_string(), - McpServerConfig { - transport: McpServerTransportConfig::StreamableHttp { - url: server_url, - bearer_token_env_var: None, - http_headers: None, - env_http_headers: None, - }, - enabled: true, - required: false, - supports_parallel_tool_calls: false, - disabled_reason: None, - startup_timeout_sec: Some(Duration::from_secs(10)), - tool_timeout_sec: None, - enabled_tools: None, - disabled_tools: None, - scopes: None, - oauth_resource: None, - tools: HashMap::new(), + insert_mcp_server( + config, + server_name, + McpServerTransportConfig::StreamableHttp { + url: server_url, + bearer_token_env_var: None, + http_headers: None, + env_http_headers: None, }, + TestMcpServerOptions::default(), ); - config - .mcp_servers - .set(servers) - .expect("test mcp servers should accept any configuration"); }) .build(&server) .await?; @@ -1211,12 +1462,18 @@ async fn streamable_http_with_oauth_round_trip_impl() -> anyhow::Result<()> { let call_id = "call-789"; let server_name = "rmcp_http_oauth"; let tool_name = format!("mcp__{server_name}__echo"); + let namespace = format!("mcp__{server_name}__"); mount_sse_once( &server, responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, &tool_name, "{\"message\":\"ping\"}"), + responses::ev_function_call_with_namespace( + call_id, + &namespace, + "echo", + "{\"message\":\"ping\"}", + ), responses::ev_completed("resp-1"), ]), ) @@ -1279,62 +1536,23 @@ async fn streamable_http_with_oauth_round_trip_impl() -> anyhow::Result<()> { // runs the full core suite in one process. config.mcp_oauth_credentials_store_mode = serde_json::from_value(json!("file")) .expect("`file` should deserialize as OAuthCredentialsStoreMode"); - let mut servers = config.mcp_servers.get().clone(); - servers.insert( - server_name.to_string(), - McpServerConfig { - transport: McpServerTransportConfig::StreamableHttp { - url: server_url, - bearer_token_env_var: None, - http_headers: None, - env_http_headers: None, - }, - enabled: true, - required: false, - supports_parallel_tool_calls: false, - disabled_reason: None, - startup_timeout_sec: Some(Duration::from_secs(10)), - tool_timeout_sec: None, - enabled_tools: None, - disabled_tools: None, - scopes: None, - oauth_resource: None, - tools: HashMap::new(), + insert_mcp_server( + config, + server_name, + McpServerTransportConfig::StreamableHttp { + url: server_url, + bearer_token_env_var: None, + http_headers: None, + env_http_headers: None, }, + TestMcpServerOptions::default(), ); - config - .mcp_servers - .set(servers) - .expect("test mcp servers should accept any configuration"); }) .build(&server) .await?; let session_model = fixture.session_configured.model.clone(); - let tools_ready_deadline = Instant::now() + Duration::from_secs(30); - loop { - fixture.codex.submit(Op::ListMcpTools).await?; - let list_event = wait_for_event_with_timeout( - &fixture.codex, - |ev| matches!(ev, EventMsg::McpListToolsResponse(_)), - Duration::from_secs(10), - ) - .await; - let EventMsg::McpListToolsResponse(tool_list) = list_event else { - unreachable!("event guard guarantees McpListToolsResponse"); - }; - if tool_list.tools.contains_key(&tool_name) { - break; - } - - let available_tools: Vec<&str> = tool_list.tools.keys().map(String::as_str).collect(); - if Instant::now() >= tools_ready_deadline { - panic!( - "timed out waiting for MCP tool {tool_name} to become available; discovered tools: {available_tools:?}" - ); - } - sleep(Duration::from_millis(200)).await; - } + wait_for_mcp_tool(&fixture, &tool_name).await?; fixture .codex diff --git a/codex-rs/core/tests/suite/search_tool.rs b/codex-rs/core/tests/suite/search_tool.rs index c524316650..41f8e623c9 100644 --- a/codex-rs/core/tests/suite/search_tool.rs +++ b/codex-rs/core/tests/suite/search_tool.rs @@ -15,6 +15,7 @@ use codex_protocol::protocol::Op; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::user_input::UserInput; use core_test_support::apps_test_server::AppsTestServer; +use core_test_support::apps_test_server::CALENDAR_CREATE_EVENT_MCP_APP_RESOURCE_URI; use core_test_support::apps_test_server::CALENDAR_CREATE_EVENT_RESOURCE_URI; use core_test_support::responses::ResponsesRequest; use core_test_support::responses::ev_assistant_message; @@ -23,6 +24,7 @@ use core_test_support::responses::ev_response_created; use core_test_support::responses::ev_tool_search_call; use core_test_support::responses::mount_sse_once; use core_test_support::responses::mount_sse_sequence; +use core_test_support::responses::namespace_child_tool; use core_test_support::responses::sse; use core_test_support::responses::start_mock_server; use core_test_support::skip_if_no_network; @@ -45,6 +47,7 @@ const CALENDAR_CREATE_TOOL: &str = "mcp__codex_apps__calendar_create_event"; const CALENDAR_LIST_TOOL: &str = "mcp__codex_apps__calendar_list_events"; const SEARCH_CALENDAR_NAMESPACE: &str = "mcp__codex_apps__calendar"; const SEARCH_CALENDAR_CREATE_TOOL: &str = "_create_event"; +const SEARCH_CALENDAR_LIST_TOOL: &str = "_list_events"; fn tool_names(body: &Value) -> Vec { body.get("tools") @@ -91,7 +94,7 @@ fn tool_search_output_tools(request: &ResponsesRequest, call_id: &str) -> Vec TestCodexBuilder { test_codex() .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) @@ -125,7 +132,7 @@ fn configured_builder(apps_base_url: String) -> TestCodexBuilder { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn search_tool_flag_adds_tool_search() -> Result<()> { +async fn search_tool_enabled_by_default_adds_tool_search() -> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; @@ -183,7 +190,7 @@ async fn search_tool_flag_adds_tool_search() -> Result<()> { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn tool_search_disabled_by_default_exposes_apps_tools_directly() -> Result<()> { +async fn tool_search_disabled_exposes_apps_tools_directly() -> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; @@ -215,8 +222,17 @@ async fn tool_search_disabled_by_default_exposes_apps_tools_directly() -> Result let body = mock.single_request().body_json(); let tools = tool_names(&body); assert!(!tools.iter().any(|name| name == TOOL_SEARCH_TOOL_NAME)); - assert!(tools.iter().any(|name| name == CALENDAR_CREATE_TOOL)); - assert!(tools.iter().any(|name| name == CALENDAR_LIST_TOOL)); + assert!( + namespace_child_tool( + &body, + SEARCH_CALENDAR_NAMESPACE, + SEARCH_CALENDAR_CREATE_TOOL + ) + .is_some() + ); + assert!( + namespace_child_tool(&body, SEARCH_CALENDAR_NAMESPACE, SEARCH_CALENDAR_LIST_TOOL).is_some() + ); Ok(()) } @@ -332,6 +348,7 @@ async fn search_tool_hides_apps_tools_without_search() -> Result<()> { assert!(tools.iter().any(|name| name == TOOL_SEARCH_TOOL_NAME)); assert!(!tools.iter().any(|name| name == CALENDAR_CREATE_TOOL)); assert!(!tools.iter().any(|name| name == CALENDAR_LIST_TOOL)); + assert!(!tools.iter().any(|name| name == SEARCH_CALENDAR_NAMESPACE)); Ok(()) } @@ -365,11 +382,16 @@ async fn explicit_app_mentions_expose_apps_tools_without_search() -> Result<()> let body = mock.single_request().body_json(); let tools = tool_names(&body); assert!( - tools.iter().any(|name| name == CALENDAR_CREATE_TOOL), + namespace_child_tool( + &body, + SEARCH_CALENDAR_NAMESPACE, + SEARCH_CALENDAR_CREATE_TOOL + ) + .is_some(), "expected explicit app mention to expose create tool, got tools: {tools:?}" ); assert!( - tools.iter().any(|name| name == CALENDAR_LIST_TOOL), + namespace_child_tool(&body, SEARCH_CALENDAR_NAMESPACE, SEARCH_CALENDAR_LIST_TOOL).is_some(), "expected explicit app mention to expose list tool, got tools: {tools:?}" ); @@ -436,6 +458,19 @@ async fn tool_search_returns_deferred_tools_without_follow_up_tool_injection() - }) .await?; + let EventMsg::McpToolCallBegin(begin) = wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::McpToolCallBegin(_)) + }) + .await + else { + unreachable!("event guard guarantees McpToolCallBegin"); + }; + assert_eq!(begin.call_id, "calendar-call-1"); + assert_eq!( + begin.mcp_app_resource_uri.as_deref(), + Some(CALENDAR_CREATE_EVENT_MCP_APP_RESOURCE_URI) + ); + let EventMsg::McpToolCallEnd(end) = wait_for_event(&test.codex, |event| { matches!(event, EventMsg::McpToolCallEnd(_)) }) @@ -444,6 +479,10 @@ async fn tool_search_returns_deferred_tools_without_follow_up_tool_injection() - unreachable!("event guard guarantees McpToolCallEnd"); }; assert_eq!(end.call_id, "calendar-call-1"); + assert_eq!( + end.mcp_app_resource_uri.as_deref(), + Some(CALENDAR_CREATE_EVENT_MCP_APP_RESOURCE_URI) + ); assert_eq!( end.invocation, McpInvocation { @@ -523,6 +562,12 @@ async fn tool_search_returns_deferred_tools_without_follow_up_tool_injection() - .any(|name| name == CALENDAR_CREATE_TOOL), "app tools should still be hidden before search: {first_request_tools:?}" ); + assert!( + !first_request_tools + .iter() + .any(|name| name == SEARCH_CALENDAR_NAMESPACE), + "app namespace should still be hidden before search: {first_request_tools:?}" + ); let output_item = tool_search_output_item(&requests[1], call_id); assert_eq!( @@ -570,6 +615,12 @@ async fn tool_search_returns_deferred_tools_without_follow_up_tool_injection() - .any(|name| name == CALENDAR_CREATE_TOOL), "follow-up request should rely on tool_search_output history, not tool injection: {second_request_tools:?}" ); + assert!( + !second_request_tools + .iter() + .any(|name| name == SEARCH_CALENDAR_NAMESPACE), + "follow-up request should rely on tool_search_output history, not namespace injection: {second_request_tools:?}" + ); let output_item = requests[2].function_call_output("calendar-call-1"); assert_eq!( @@ -584,6 +635,12 @@ async fn tool_search_returns_deferred_tools_without_follow_up_tool_injection() - .any(|name| name == CALENDAR_CREATE_TOOL), "post-tool follow-up should still rely on tool_search_output history, not tool injection: {third_request_tools:?}" ); + assert!( + !third_request_tools + .iter() + .any(|name| name == SEARCH_CALENDAR_NAMESPACE), + "post-tool follow-up should still rely on tool_search_output history, not namespace injection: {third_request_tools:?}" + ); Ok(()) } @@ -640,11 +697,13 @@ async fn tool_search_indexes_only_enabled_non_app_mcp_tools() -> Result<()> { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(10)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: Some(vec!["echo".to_string(), "image".to_string()]), disabled_tools: Some(vec!["image".to_string()]), scopes: None, @@ -683,16 +742,19 @@ async fn tool_search_indexes_only_enabled_non_app_mcp_tools() -> Result<()> { .any(|name| name == "mcp__rmcp__echo"), "non-app MCP tools should be hidden before search in large-search mode: {first_request_tools:?}" ); + assert!( + !first_request_tools.iter().any(|name| name == "mcp__rmcp__"), + "non-app MCP namespace should be hidden before search in large-search mode: {first_request_tools:?}" + ); let echo_tools = tool_search_output_tools(&requests[1], echo_call_id); - let rmcp_echo_tools = echo_tools - .iter() - .filter(|tool| tool.get("name").and_then(Value::as_str) == Some("mcp__rmcp__")) - .flat_map(|namespace| namespace.get("tools").and_then(Value::as_array)) - .flatten() - .filter_map(|tool| tool.get("name").and_then(Value::as_str).map(str::to_string)) - .collect::>(); - assert_eq!(rmcp_echo_tools, vec!["echo".to_string()]); + let echo_output = json!({ "tools": echo_tools }); + let rmcp_echo_tool = namespace_child_tool(&echo_output, "mcp__rmcp__", "echo") + .expect("tool_search should return rmcp echo as a namespace child tool"); + assert_eq!( + rmcp_echo_tool.get("type").and_then(Value::as_str), + Some("function") + ); let image_tools = tool_search_output_tools(&requests[1], image_call_id); let found_rmcp_image_tool = image_tools diff --git a/codex-rs/core/tests/suite/seatbelt.rs b/codex-rs/core/tests/suite/seatbelt.rs deleted file mode 100644 index 808a8a022d..0000000000 --- a/codex-rs/core/tests/suite/seatbelt.rs +++ /dev/null @@ -1,316 +0,0 @@ -#![cfg(target_os = "macos")] - -//! Tests for the macOS sandboxing that are specific to Seatbelt. -//! Tests that apply to both Mac and Linux sandboxing should go in sandbox.rs. - -use std::collections::HashMap; -use std::path::Path; -use std::path::PathBuf; - -use codex_core::seatbelt::spawn_command_under_seatbelt; -use codex_core::spawn::CODEX_SANDBOX_ENV_VAR; -use codex_core::spawn::StdioPolicy; -use codex_protocol::protocol::SandboxPolicy; -use codex_utils_absolute_path::AbsolutePathBuf; -use tempfile::TempDir; - -struct TestScenario { - repo_parent: PathBuf, - file_outside_repo: PathBuf, - repo_root: PathBuf, - file_in_repo_root: PathBuf, - file_in_dot_git_dir: PathBuf, -} - -struct TestExpectations { - file_outside_repo_is_writable: bool, - file_in_repo_root_is_writable: bool, - file_in_dot_git_dir_is_writable: bool, -} - -impl TestScenario { - async fn run_test(&self, policy: &SandboxPolicy, expectations: TestExpectations) { - if std::env::var(CODEX_SANDBOX_ENV_VAR) == Ok("seatbelt".to_string()) { - eprintln!("{CODEX_SANDBOX_ENV_VAR} is set to 'seatbelt', skipping test."); - return; - } - - assert_eq!( - touch(&self.file_outside_repo, policy).await, - expectations.file_outside_repo_is_writable - ); - assert_eq!( - self.file_outside_repo.exists(), - expectations.file_outside_repo_is_writable - ); - - assert_eq!( - touch(&self.file_in_repo_root, policy).await, - expectations.file_in_repo_root_is_writable - ); - assert_eq!( - self.file_in_repo_root.exists(), - expectations.file_in_repo_root_is_writable - ); - - assert_eq!( - touch(&self.file_in_dot_git_dir, policy).await, - expectations.file_in_dot_git_dir_is_writable - ); - assert_eq!( - self.file_in_dot_git_dir.exists(), - expectations.file_in_dot_git_dir_is_writable - ); - } -} - -/// If the user has added a workspace root that is not a Git repo root, then -/// the user has to specify `--skip-git-repo-check` or go through some -/// interstitial that indicates they are taking on some risk because Git -/// cannot be used to backup their work before the agent begins. -/// -/// Because the user has agreed to this risk, we do not try find all .git -/// folders in the workspace and block them (though we could change our -/// position on this in the future). -#[tokio::test] -async fn if_parent_of_repo_is_writable_then_dot_git_folder_is_writable() { - let tmp = TempDir::new().expect("should be able to create temp dir"); - let test_scenario = create_test_scenario(&tmp); - let policy = SandboxPolicy::WorkspaceWrite { - writable_roots: vec![test_scenario.repo_parent.as_path().try_into().unwrap()], - read_only_access: Default::default(), - network_access: false, - exclude_tmpdir_env_var: true, - exclude_slash_tmp: true, - }; - - test_scenario - .run_test( - &policy, - TestExpectations { - file_outside_repo_is_writable: true, - file_in_repo_root_is_writable: true, - file_in_dot_git_dir_is_writable: true, - }, - ) - .await; -} - -/// When the writable root is the root of a Git repository (as evidenced by the -/// presence of a .git folder), then the .git folder should be read-only if -/// the policy is `WorkspaceWrite`. -#[tokio::test] -async fn if_git_repo_is_writable_root_then_dot_git_folder_is_read_only() { - let tmp = TempDir::new().expect("should be able to create temp dir"); - let test_scenario = create_test_scenario(&tmp); - let policy = SandboxPolicy::WorkspaceWrite { - writable_roots: vec![test_scenario.repo_root.as_path().try_into().unwrap()], - read_only_access: Default::default(), - network_access: false, - exclude_tmpdir_env_var: true, - exclude_slash_tmp: true, - }; - - test_scenario - .run_test( - &policy, - TestExpectations { - file_outside_repo_is_writable: false, - file_in_repo_root_is_writable: true, - file_in_dot_git_dir_is_writable: false, - }, - ) - .await; -} - -/// Under DangerFullAccess, all writes should be permitted anywhere on disk, -/// including inside the .git folder. -#[tokio::test] -async fn danger_full_access_allows_all_writes() { - let tmp = TempDir::new().expect("should be able to create temp dir"); - let test_scenario = create_test_scenario(&tmp); - let policy = SandboxPolicy::DangerFullAccess; - - test_scenario - .run_test( - &policy, - TestExpectations { - file_outside_repo_is_writable: true, - file_in_repo_root_is_writable: true, - file_in_dot_git_dir_is_writable: true, - }, - ) - .await; -} - -/// Under ReadOnly, writes should not be permitted anywhere on disk. -#[tokio::test] -async fn read_only_forbids_all_writes() { - let tmp = TempDir::new().expect("should be able to create temp dir"); - let test_scenario = create_test_scenario(&tmp); - let policy = SandboxPolicy::new_read_only_policy(); - - test_scenario - .run_test( - &policy, - TestExpectations { - file_outside_repo_is_writable: false, - file_in_repo_root_is_writable: false, - file_in_dot_git_dir_is_writable: false, - }, - ) - .await; -} - -#[tokio::test] -async fn openpty_works_under_seatbelt() { - if std::env::var(CODEX_SANDBOX_ENV_VAR) == Ok("seatbelt".to_string()) { - eprintln!("{CODEX_SANDBOX_ENV_VAR} is set to 'seatbelt', skipping test."); - return; - } - - if which::which("python3").is_err() { - eprintln!("python3 not found in PATH, skipping test."); - return; - } - - let policy = SandboxPolicy::new_read_only_policy(); - let command_cwd = AbsolutePathBuf::current_dir().expect("getcwd"); - let sandbox_cwd = command_cwd.clone(); - - let mut child = spawn_command_under_seatbelt( - vec![ - "python3".to_string(), - "-c".to_string(), - r#"import os - -master, slave = os.openpty() -os.write(slave, b"ping") -assert os.read(master, 4) == b"ping""# - .to_string(), - ], - command_cwd, - &policy, - &sandbox_cwd, - StdioPolicy::RedirectForShellTool, - /*network*/ None, - HashMap::new(), - ) - .await - .expect("should be able to spawn python under seatbelt"); - - let status = child - .wait() - .await - .expect("should be able to wait for child process"); - assert!(status.success(), "python exited with {status:?}"); -} - -#[tokio::test] -async fn java_home_finds_runtime_under_seatbelt() { - if std::env::var(CODEX_SANDBOX_ENV_VAR) == Ok("seatbelt".to_string()) { - eprintln!("{CODEX_SANDBOX_ENV_VAR} is set to 'seatbelt', skipping test."); - return; - } - - let java_home_path = Path::new("/usr/libexec/java_home"); - if !java_home_path.exists() { - eprintln!("/usr/libexec/java_home is not present, skipping test."); - return; - } - - let baseline_output = tokio::process::Command::new(java_home_path) - .env_remove("JAVA_HOME") - .output() - .await - .expect("should be able to invoke java_home outside seatbelt"); - if !baseline_output.status.success() { - eprintln!( - "java_home exited with {:?} outside seatbelt, skipping test", - baseline_output.status - ); - return; - } - - let policy = SandboxPolicy::new_read_only_policy(); - let command_cwd = AbsolutePathBuf::current_dir().expect("getcwd"); - let sandbox_cwd = command_cwd.clone(); - - let mut env: HashMap = std::env::vars().collect(); - env.remove("JAVA_HOME"); - env.remove(CODEX_SANDBOX_ENV_VAR); - - let child = spawn_command_under_seatbelt( - vec![java_home_path.to_string_lossy().to_string()], - command_cwd, - &policy, - &sandbox_cwd, - StdioPolicy::RedirectForShellTool, - /*network*/ None, - env, - ) - .await - .expect("should be able to spawn java_home under seatbelt"); - - let output = child - .wait_with_output() - .await - .expect("should be able to wait for java_home child"); - assert!( - output.status.success(), - "java_home under seatbelt exited with {:?}, stderr: {}", - output.status, - String::from_utf8_lossy(&output.stderr) - ); - - let stdout = String::from_utf8_lossy(&output.stdout); - assert!( - !stdout.trim().is_empty(), - "java_home stdout unexpectedly empty under seatbelt" - ); -} - -#[expect(clippy::expect_used)] -fn create_test_scenario(tmp: &TempDir) -> TestScenario { - let repo_parent = tmp.path().to_path_buf(); - let repo_root = repo_parent.join("repo"); - let dot_git_dir = repo_root.join(".git"); - - std::fs::create_dir(&repo_root).expect("should be able to create repo root"); - std::fs::create_dir(&dot_git_dir).expect("should be able to create .git dir"); - - TestScenario { - file_outside_repo: repo_parent.join("outside.txt"), - repo_parent, - file_in_repo_root: repo_root.join("repo_file.txt"), - repo_root, - file_in_dot_git_dir: dot_git_dir.join("dot_git_file.txt"), - } -} - -#[expect(clippy::expect_used)] -/// Note that `path` must be absolute. -async fn touch(path: &Path, policy: &SandboxPolicy) -> bool { - assert!(path.is_absolute(), "Path must be absolute: {path:?}"); - let command_cwd = AbsolutePathBuf::current_dir().expect("getcwd"); - let sandbox_cwd = command_cwd.clone(); - let mut child = spawn_command_under_seatbelt( - vec![ - "/usr/bin/touch".to_string(), - path.to_string_lossy().to_string(), - ], - command_cwd, - policy, - &sandbox_cwd, - StdioPolicy::RedirectForShellTool, - /*network*/ None, - HashMap::new(), - ) - .await - .expect("should be able to spawn command under seatbelt"); - child - .wait() - .await - .expect("should be able to wait for child process") - .success() -} diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__realtime_conversation__conversation_user_text_turn_is_capped_when_mirrored_to_realtime.snap b/codex-rs/core/tests/suite/snapshots/all__suite__realtime_conversation__conversation_user_text_turn_is_capped_when_mirrored_to_realtime.snap index ed19ddae97..3daae5d378 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__realtime_conversation__conversation_user_text_turn_is_capped_when_mirrored_to_realtime.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__realtime_conversation__conversation_user_text_turn_is_capped_when_mirrored_to_realtime.snap @@ -9,4 +9,4 @@ content[0].type: input_text model_has_full_user_text: true realtime_text_equal_full_user_text: false realtime_text_approx_tokens: 300 -content[0].text: mirror-head alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alph…2417 tokens truncated…ega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega mirror-tail +content[0].text: [USER] mirror-head alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alp…2419 tokens truncated…ega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega mirror-tail diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__realtime_conversation__conversation_user_text_turn_is_sent_to_realtime_when_active.snap b/codex-rs/core/tests/suite/snapshots/all__suite__realtime_conversation__conversation_user_text_turn_is_sent_to_realtime_when_active.snap index 9cd2a55303..3ca8233029 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__realtime_conversation__conversation_user_text_turn_is_sent_to_realtime_when_active.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__realtime_conversation__conversation_user_text_turn_is_sent_to_realtime_when_active.snap @@ -6,5 +6,5 @@ type: conversation.item.create item.type: message item.role: user content[0].type: input_text -content[0].text: typed follow-up for realtime +content[0].text: [USER] typed follow-up for realtime response.create: false diff --git a/codex-rs/core/tests/suite/sqlite_state.rs b/codex-rs/core/tests/suite/sqlite_state.rs index 1c9d6a87e1..271f7bfc90 100644 --- a/codex-rs/core/tests/suite/sqlite_state.rs +++ b/codex-rs/core/tests/suite/sqlite_state.rs @@ -325,12 +325,17 @@ async fn mcp_call_marks_thread_memory_mode_polluted_when_configured() -> Result< let server = start_mock_server().await; let call_id = "call-123"; let server_name = "rmcp"; - let tool_name = format!("mcp__{server_name}__echo"); + let namespace = format!("mcp__{server_name}__"); mount_sse_once( &server, responses::sse(vec![ ev_response_created("resp-1"), - ev_function_call(call_id, &tool_name, "{\"message\":\"ping\"}"), + responses::ev_function_call_with_namespace( + call_id, + &namespace, + "echo", + "{\"message\":\"ping\"}", + ), ev_completed("resp-1"), ]), ) @@ -366,12 +371,14 @@ async fn mcp_call_marks_thread_memory_mode_polluted_when_configured() -> Result< env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(10)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/core/tests/suite/tools.rs b/codex-rs/core/tests/suite/tools.rs index 8dc91cc087..bc286ce5ea 100644 --- a/codex-rs/core/tests/suite/tools.rs +++ b/codex-rs/core/tests/suite/tools.rs @@ -1,12 +1,16 @@ #![cfg(not(target_os = "windows"))] #![allow(clippy::unwrap_used, clippy::expect_used)] +use std::collections::HashMap; use std::fs; +use std::sync::Arc; use std::time::Duration; use std::time::Instant; use anyhow::Context; use anyhow::Result; +use codex_config::types::McpServerConfig; +use codex_config::types::McpServerTransportConfig; use codex_core::sandboxing::SandboxPermissions; use codex_features::Feature; use codex_protocol::protocol::AskForApproval; @@ -22,10 +26,13 @@ use core_test_support::responses::mount_sse_sequence; use core_test_support::responses::sse; use core_test_support::responses::start_mock_server; use core_test_support::skip_if_no_network; +use core_test_support::stdio_server_bin; use core_test_support::test_codex::test_codex; use regex_lite::Regex; use serde_json::Value; use serde_json::json; +use serial_test::serial; +use tempfile::TempDir; fn tool_names(body: &Value) -> Vec { body.get("tools") @@ -44,6 +51,24 @@ fn tool_names(body: &Value) -> Vec { .unwrap_or_default() } +fn ev_namespaced_function_call( + call_id: &str, + namespace: &str, + name: &str, + arguments: &str, +) -> Value { + json!({ + "type": "response.output_item.done", + "item": { + "type": "function_call", + "call_id": call_id, + "namespace": namespace, + "name": name, + "arguments": arguments, + } + }) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn custom_tool_unknown_returns_custom_output_error() -> Result<()> { skip_if_no_network!(Ok(())); @@ -91,6 +116,149 @@ async fn custom_tool_unknown_returns_custom_output_error() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial(mcp_test_value)] +async fn historical_unavailable_mcp_call_is_exposed_as_placeholder_tool() -> Result<()> { + skip_if_no_network!(Ok(())); + + let historical_call_id = "historical-mcp-call"; + let retry_call_id = "retry-mcp-call"; + let server_name = "rmcp"; + let unavailable_tool_namespace = "mcp__rmcp__"; + let unavailable_tool_name = "echo"; + let unavailable_tool_display_name = "mcp__rmcp__echo"; + let server = start_mock_server().await; + let rmcp_test_server_bin = match stdio_server_bin() { + Ok(bin) => bin, + Err(err) => { + eprintln!("test_stdio_server binary not available, skipping test: {err}"); + return Ok(()); + } + }; + let codex_home = Arc::new(TempDir::new()?); + let mut builder = test_codex() + .with_model("gpt-5.1") + .with_home(Arc::clone(&codex_home)) + .with_config(move |config| { + config + .features + .enable(Feature::UnavailableDummyTools) + .expect("unavailable dummy tools should be enabled for this test"); + let mut servers = config.mcp_servers.get().clone(); + servers.insert( + server_name.to_string(), + McpServerConfig { + transport: McpServerTransportConfig::Stdio { + command: rmcp_test_server_bin, + args: Vec::new(), + env: Some(HashMap::new()), + env_vars: Vec::new(), + cwd: None, + }, + experimental_environment: None, + enabled: true, + required: false, + supports_parallel_tool_calls: false, + disabled_reason: None, + startup_timeout_sec: Some(Duration::from_secs(10)), + tool_timeout_sec: None, + default_tools_approval_mode: None, + enabled_tools: None, + disabled_tools: None, + scopes: None, + oauth_resource: None, + tools: HashMap::new(), + }, + ); + config + .mcp_servers + .set(servers) + .expect("test mcp servers should accept any configuration"); + }); + let test = builder.build(&server).await?; + + let first_turn_mock = mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_namespaced_function_call( + historical_call_id, + unavailable_tool_namespace, + unavailable_tool_name, + r#"{"message":"ping"}"#, + ), + ev_completed("resp-1"), + ]), + sse(vec![ + ev_response_created("resp-2"), + ev_assistant_message("msg-1", "rmcp echo tool completed"), + ev_completed("resp-2"), + ]), + ], + ) + .await; + + test.submit_turn("call the rmcp echo tool").await?; + let rollout_path = test.codex.rollout_path().context("rollout path")?; + assert_eq!(first_turn_mock.requests().len(), 2); + drop(test); + + let retry_mock = mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_response_created("resp-3"), + ev_namespaced_function_call( + retry_call_id, + unavailable_tool_namespace, + unavailable_tool_name, + r#"{"message":"ping again"}"#, + ), + ev_completed("resp-3"), + ]), + sse(vec![ + ev_response_created("resp-4"), + ev_assistant_message("msg-2", "done"), + ev_completed("resp-4"), + ]), + ], + ) + .await; + + let mut resume_builder = test_codex().with_model("gpt-5.1").with_config(|config| { + config + .features + .enable(Feature::UnavailableDummyTools) + .expect("unavailable dummy tools should be enabled for this test"); + }); + let test = resume_builder + .resume(&server, codex_home, rollout_path) + .await?; + + test.submit_turn("retry the rmcp echo tool").await?; + + let requests = retry_mock.requests(); + assert_eq!(requests.len(), 2); + let first_request_tools = tool_names(&requests[0].body_json()); + assert!( + first_request_tools + .iter() + .any(|name| name == unavailable_tool_display_name), + "historical unavailable MCP call should add a placeholder tool; got {first_request_tools:?}" + ); + let output_text = requests[1] + .function_call_output_text(retry_call_id) + .context("placeholder tool output present")?; + assert!(output_text.contains("not currently available")); + assert!( + !output_text.contains("unsupported call"), + "placeholder handler should answer instead of falling back to unsupported call: {output_text}" + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn shell_escalated_permissions_rejected_then_ok() -> Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/core/tests/suite/truncation.rs b/codex-rs/core/tests/suite/truncation.rs index 288e8f4bdb..81c22e1b6c 100644 --- a/codex-rs/core/tests/suite/truncation.rs +++ b/codex-rs/core/tests/suite/truncation.rs @@ -331,7 +331,7 @@ async fn mcp_tool_call_output_exceeds_limit_truncated_for_model() -> Result<()> let call_id = "rmcp-truncated"; let server_name = "rmcp"; - let tool_name = format!("mcp__{server_name}__echo"); + let namespace = format!("mcp__{server_name}__"); // Build a very large message to exceed 10KiB once serialized. let large_msg = "long-message-with-newlines-".repeat(6000); @@ -341,7 +341,12 @@ async fn mcp_tool_call_output_exceeds_limit_truncated_for_model() -> Result<()> &server, sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, &tool_name, &args_json.to_string()), + responses::ev_function_call_with_namespace( + call_id, + &namespace, + "echo", + &args_json.to_string(), + ), responses::ev_completed("resp-1"), ]), ) @@ -370,12 +375,14 @@ async fn mcp_tool_call_output_exceeds_limit_truncated_for_model() -> Result<()> env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: Some(std::time::Duration::from_secs(10)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -426,13 +433,13 @@ async fn mcp_image_output_preserves_image_and_no_text_summary() -> Result<()> { let call_id = "rmcp-image-no-trunc"; let server_name = "rmcp"; - let tool_name = format!("mcp__{server_name}__image"); + let namespace = format!("mcp__{server_name}__"); mount_sse_once( &server, sse(vec![ ev_response_created("resp-1"), - ev_function_call(call_id, &tool_name, "{}"), + responses::ev_function_call_with_namespace(call_id, &namespace, "image", "{}"), ev_completed("resp-1"), ]), ) @@ -467,12 +474,14 @@ async fn mcp_image_output_preserves_image_and_no_text_summary() -> Result<()> { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(10)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -705,7 +714,7 @@ async fn mcp_tool_call_output_not_truncated_with_custom_limit() -> Result<()> { let call_id = "rmcp-untruncated"; let server_name = "rmcp"; - let tool_name = format!("mcp__{server_name}__echo"); + let namespace = format!("mcp__{server_name}__"); let large_msg = "a".repeat(80_000); let args_json = serde_json::json!({ "message": large_msg }); @@ -713,7 +722,12 @@ async fn mcp_tool_call_output_not_truncated_with_custom_limit() -> Result<()> { &server, sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, &tool_name, &args_json.to_string()), + responses::ev_function_call_with_namespace( + call_id, + &namespace, + "echo", + &args_json.to_string(), + ), responses::ev_completed("resp-1"), ]), ) @@ -742,12 +756,14 @@ async fn mcp_tool_call_output_not_truncated_with_custom_limit() -> Result<()> { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, disabled_reason: None, startup_timeout_sec: Some(std::time::Duration::from_secs(10)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/core/tests/suite/unified_exec.rs b/codex-rs/core/tests/suite/unified_exec.rs index df13fcaebe..b14c930999 100644 --- a/codex-rs/core/tests/suite/unified_exec.rs +++ b/codex-rs/core/tests/suite/unified_exec.rs @@ -39,6 +39,8 @@ use serde_json::Value; use serde_json::json; use tokio::time::Duration; +const UNIFIED_EXEC_LAGGED_OUTPUT_TIMEOUT: Duration = Duration::from_secs(30); + fn extract_output_text(item: &Value) -> Option<&str> { item.get("output").and_then(|value| match value { Value::String(text) => Some(text.as_str()), @@ -2055,11 +2057,12 @@ PY SandboxPolicy::DangerFullAccess, ) .await?; - // This is a worst case scenario for the truncate logic. + // This is a worst case scenario for the truncate logic, and CI can spend a + // while draining the lagged tail before the follow-up tool call completes. wait_for_event_with_timeout( &test.codex, |event| matches!(event, EventMsg::TurnComplete(_)), - Duration::from_secs(10), + UNIFIED_EXEC_LAGGED_OUTPUT_TIMEOUT, ) .await; diff --git a/codex-rs/core/tests/suite/view_image.rs b/codex-rs/core/tests/suite/view_image.rs index 6541a15450..f9d6d9402c 100644 --- a/codex-rs/core/tests/suite/view_image.rs +++ b/codex-rs/core/tests/suite/view_image.rs @@ -30,7 +30,6 @@ use core_test_support::responses::start_mock_server; use core_test_support::skip_if_no_network; use core_test_support::test_codex::TestCodex; use core_test_support::test_codex::test_codex; -use core_test_support::wait_for_event; use core_test_support::wait_for_event_with_timeout; use image::DynamicImage; use image::GenericImageView; @@ -49,6 +48,8 @@ use wiremock::ResponseTemplate; #[cfg(not(debug_assertions))] use wiremock::matchers::body_string_contains; +const VIEW_IMAGE_TURN_COMPLETE_TIMEOUT: Duration = Duration::from_secs(30); + fn image_messages(body: &Value) -> Vec<&Value> { body.get("input") .and_then(Value::as_array) @@ -180,7 +181,7 @@ async fn user_turn_with_local_image_attaches_image() -> anyhow::Result<()> { codex, |event| matches!(event, EventMsg::TurnComplete(_)), // Empirically, image attachment can be slow under Bazel/RBE. - Duration::from_secs(10), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, ) .await; @@ -298,7 +299,7 @@ async fn view_image_tool_attaches_local_image() -> anyhow::Result<()> { }, // Empirically, we have seen this run slow when run under // Bazel on arm Linux. - Duration::from_secs(10), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, ) .await; @@ -422,7 +423,7 @@ async fn view_image_tool_can_preserve_original_resolution_when_requested_on_gpt5 wait_for_event_with_timeout( codex, |event| matches!(event, EventMsg::TurnComplete(_)), - Duration::from_secs(10), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, ) .await; @@ -518,7 +519,12 @@ async fn view_image_tool_errors_clearly_for_unsupported_detail_values() -> anyho }) .await?; - wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event_with_timeout( + codex, + |event| matches!(event, EventMsg::TurnComplete(_)), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, + ) + .await; let req = mock.single_request(); let body_with_tool_output = req.body_json(); @@ -603,7 +609,12 @@ async fn view_image_tool_treats_null_detail_as_omitted() -> anyhow::Result<()> { }) .await?; - wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event_with_timeout( + codex, + |event| matches!(event, EventMsg::TurnComplete(_)), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, + ) + .await; let req = mock.single_request(); let function_output = req.function_call_output(call_id); @@ -701,7 +712,7 @@ async fn view_image_tool_resizes_when_model_lacks_original_detail_support() -> a wait_for_event_with_timeout( codex, |event| matches!(event, EventMsg::TurnComplete(_)), - Duration::from_secs(10), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, ) .await; @@ -805,7 +816,7 @@ async fn view_image_tool_does_not_force_original_resolution_with_capability_only wait_for_event_with_timeout( codex, |event| matches!(event, EventMsg::TurnComplete(_)), - Duration::from_secs(10), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, ) .await; @@ -915,7 +926,7 @@ await codex.emitImage(out); EventMsg::TurnComplete(_) => true, _ => false, }, - Duration::from_secs(10), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, ) .await; let tool_event = match tool_event { @@ -1035,7 +1046,7 @@ console.log(out.type); EventMsg::TurnComplete(_) => true, _ => false, }, - Duration::from_secs(10), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, ) .await; let tool_event = match tool_event { @@ -1117,7 +1128,12 @@ async fn view_image_tool_errors_when_path_is_directory() -> anyhow::Result<()> { }) .await?; - wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event_with_timeout( + codex, + |event| matches!(event, EventMsg::TurnComplete(_)), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, + ) + .await; let req = mock.single_request(); let body_with_tool_output = req.body_json(); @@ -1193,7 +1209,12 @@ async fn view_image_tool_errors_for_non_image_files() -> anyhow::Result<()> { }) .await?; - wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event_with_timeout( + codex, + |event| matches!(event, EventMsg::TurnComplete(_)), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, + ) + .await; let request = mock.single_request(); assert!( @@ -1274,7 +1295,12 @@ async fn view_image_tool_errors_when_file_missing() -> anyhow::Result<()> { }) .await?; - wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event_with_timeout( + codex, + |event| matches!(event, EventMsg::TurnComplete(_)), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, + ) + .await; let req = mock.single_request(); let body_with_tool_output = req.body_json(); @@ -1405,7 +1431,12 @@ async fn view_image_tool_returns_unsupported_message_for_text_only_model() -> an }) .await?; - wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event_with_timeout( + codex, + |event| matches!(event, EventMsg::TurnComplete(_)), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, + ) + .await; let output_text = mock .single_request() @@ -1480,7 +1511,12 @@ async fn replaces_invalid_local_image_after_bad_request() -> anyhow::Result<()> }) .await?; - wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event_with_timeout( + &codex, + |event| matches!(event, EventMsg::TurnComplete(_)), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, + ) + .await; let first_body = invalid_image_mock.single_request().body_json(); assert!( diff --git a/codex-rs/deny.toml b/codex-rs/deny.toml index 088cda0ba1..bed4a4f2fa 100644 --- a/codex-rs/deny.toml +++ b/codex-rs/deny.toml @@ -70,7 +70,7 @@ feature-depth = 1 # A list of advisory IDs to ignore. Note that ignored advisories will still # output a note when they are encountered. ignore = [ - # Reviewed 2026-04-11. Keep this list in sync with .cargo/audit.toml. + # Reviewed 2026-04-15. Keep this list in sync with .cargo/audit.toml. # Each exception must identify the dependency path and removal condition. { id = "RUSTSEC-2024-0388", reason = "derivative is unmaintained; pulled in via starlark v0.13.0 used by execpolicy/cli/core; no fixed release yet" }, { id = "RUSTSEC-2025-0057", reason = "fxhash is unmaintained; pulled in via starlark_map/starlark v0.13.0 used by execpolicy/cli/core; no fixed release yet" }, @@ -78,6 +78,7 @@ ignore = [ # TODO(fcoury): remove this exception when syntect drops yaml-rust and bincode, or updates to versions that have fixed the vulnerabilities. { id = "RUSTSEC-2024-0320", reason = "yaml-rust is unmaintained; pulled in via syntect v5.3.0 used by codex-tui for syntax highlighting; no fixed release yet" }, { id = "RUSTSEC-2025-0141", reason = "bincode is unmaintained; pulled in via syntect v5.3.0 used by codex-tui for syntax highlighting; no fixed release yet" }, + { id = "RUSTSEC-2026-0097", reason = "rand 0.8.5 is pulled in via age v0.11.2/codex-secrets and zbus v4.4.0/keyring; no compatible rand 0.8 fixed release, remove when transitive dependencies move to rand >=0.9.3" }, ] # If this is true, then cargo deny will use the git executable to fetch advisory database. # If this is false, then it uses a built-in git library. diff --git a/codex-rs/exec-server/BUILD.bazel b/codex-rs/exec-server/BUILD.bazel index 5d62c68caf..57ebe041f8 100644 --- a/codex-rs/exec-server/BUILD.bazel +++ b/codex-rs/exec-server/BUILD.bazel @@ -3,5 +3,9 @@ load("//:defs.bzl", "codex_rust_crate") codex_rust_crate( name = "exec-server", crate_name = "codex_exec_server", + # Keep the crate's integration tests single-threaded under Bazel because + # they install process-global test-binary dispatch state, and the remote + # exec-server cases already rely on serialization around the full CLI path. + integration_test_args = ["--test-threads=1"], test_tags = ["no-sandbox"], ) diff --git a/codex-rs/exec-server/README.md b/codex-rs/exec-server/README.md index 0047449d39..78b92e1a71 100644 --- a/codex-rs/exec-server/README.md +++ b/codex-rs/exec-server/README.md @@ -85,6 +85,7 @@ Request params: "PATH": "/usr/bin:/bin" }, "tty": true, + "pipeStdin": false, "arg0": null } ``` @@ -95,8 +96,8 @@ Field definitions: - `argv`: command vector. It must be non-empty. - `cwd`: absolute working directory used for the child process. - `env`: environment variables passed to the child process. -- `tty`: when `true`, spawn a PTY-backed interactive process; when `false`, - spawn a pipe-backed process with closed stdin. +- `tty`: when `true`, spawn a PTY-backed interactive process. +- `pipeStdin`: when `true`, keep non-PTY stdin writable via `process/write`. - `arg0`: optional argv0 override forwarded to `codex-utils-pty`. Response: @@ -111,7 +112,7 @@ Behavior notes: - Reusing an existing `processId` is rejected. - PTY-backed processes accept later writes through `process/write`. -- Pipe-backed processes are launched with stdin closed and reject writes. +- Non-PTY processes reject writes unless `pipeStdin` is `true`. - Output is streamed asynchronously via `process/output`. - Exit is reported asynchronously via `process/exited`. @@ -153,7 +154,7 @@ Response: ### `process/write` -Writes raw bytes to a running PTY-backed process stdin. +Writes raw bytes to a running process stdin. Request params: @@ -177,7 +178,7 @@ Response: Behavior notes: - Writes to an unknown `processId` are rejected. -- Writes to a non-PTY process are rejected because stdin is already closed. +- Writes to a non-PTY process are rejected unless it started with `pipeStdin`. ### `process/terminate` @@ -325,7 +326,7 @@ Initialize: Start a process: ```json -{"id":2,"method":"process/start","params":{"processId":"proc-1","argv":["bash","-lc","printf 'ready\\n'; while IFS= read -r line; do printf 'echo:%s\\n' \"$line\"; done"],"cwd":"/tmp","env":{"PATH":"/usr/bin:/bin"},"tty":true,"arg0":null}} +{"id":2,"method":"process/start","params":{"processId":"proc-1","argv":["bash","-lc","printf 'ready\\n'; while IFS= read -r line; do printf 'echo:%s\\n' \"$line\"; done"],"cwd":"/tmp","env":{"PATH":"/usr/bin:/bin"},"tty":true,"pipeStdin":false,"arg0":null}} {"id":2,"result":{"processId":"proc-1"}} {"method":"process/output","params":{"processId":"proc-1","seq":1,"stream":"stdout","chunk":"cmVhZHkK"}} ``` diff --git a/codex-rs/exec-server/src/environment.rs b/codex-rs/exec-server/src/environment.rs index 77ead87a84..afe0720196 100644 --- a/codex-rs/exec-server/src/environment.rs +++ b/codex-rs/exec-server/src/environment.rs @@ -346,6 +346,7 @@ mod tests { env_policy: None, env: Default::default(), tty: false, + pipe_stdin: false, arg0: None, }) .await diff --git a/codex-rs/exec-server/src/fs_sandbox.rs b/codex-rs/exec-server/src/fs_sandbox.rs index d0765afa35..7b0ac256d4 100644 --- a/codex-rs/exec-server/src/fs_sandbox.rs +++ b/codex-rs/exec-server/src/fs_sandbox.rs @@ -1,10 +1,8 @@ use std::collections::HashMap; -use std::path::PathBuf; use codex_app_server_protocol::JSONRPCErrorError; use codex_protocol::models::FileSystemPermissions; use codex_protocol::models::PermissionProfile; -use codex_protocol::permissions::FileSystemAccessMode; use codex_protocol::permissions::FileSystemSandboxPolicy; use codex_protocol::permissions::NetworkSandboxPolicy; use codex_protocol::protocol::ReadOnlyAccess; @@ -26,14 +24,6 @@ use crate::fs_helper::FsHelperPayload; use crate::fs_helper::FsHelperRequest; use crate::fs_helper::FsHelperResponse; use crate::local_file_system::current_sandbox_cwd; -use crate::local_file_system::resolve_existing_path; -use crate::protocol::FsCopyParams; -use crate::protocol::FsCreateDirectoryParams; -use crate::protocol::FsGetMetadataParams; -use crate::protocol::FsReadDirectoryParams; -use crate::protocol::FsReadFileParams; -use crate::protocol::FsRemoveParams; -use crate::protocol::FsWriteFileParams; use crate::rpc::internal_error; use crate::rpc::invalid_request; @@ -52,23 +42,16 @@ impl FileSystemSandboxRunner { sandbox: &FileSystemSandboxContext, request: FsHelperRequest, ) -> Result { - let request_sandbox_policy = - normalize_sandbox_policy_root_aliases(sandbox.sandbox_policy.clone()); let helper_sandbox_policy = normalize_sandbox_policy_root_aliases( sandbox_policy_with_helper_runtime_defaults(&sandbox.sandbox_policy), ); let cwd = current_sandbox_cwd().map_err(io_error)?; let cwd = AbsolutePathBuf::from_absolute_path(cwd.as_path()) .map_err(|err| invalid_request(format!("current directory is not absolute: {err}")))?; - let request_file_system_policy = FileSystemSandboxPolicy::from_legacy_sandbox_policy( - &request_sandbox_policy, - cwd.as_path(), - ); let file_system_policy = FileSystemSandboxPolicy::from_legacy_sandbox_policy( &helper_sandbox_policy, cwd.as_path(), ); - let request = resolve_request_paths(request, &request_file_system_policy, &cwd)?; let network_policy = NetworkSandboxPolicy::Restricted; let command = self.sandbox_exec_request( &helper_sandbox_policy, @@ -158,132 +141,6 @@ impl FileSystemSandboxRunner { } } -fn resolve_request_paths( - request: FsHelperRequest, - file_system_policy: &FileSystemSandboxPolicy, - cwd: &AbsolutePathBuf, -) -> Result { - match request { - FsHelperRequest::ReadFile(FsReadFileParams { path, sandbox }) => { - let path = resolve_sandbox_path(&path, PreserveTerminalSymlink::No)?; - ensure_path_access(file_system_policy, cwd, &path, FileSystemAccessMode::Read)?; - Ok(FsHelperRequest::ReadFile(FsReadFileParams { - path, - sandbox, - })) - } - FsHelperRequest::WriteFile(FsWriteFileParams { - path, - data_base64, - sandbox, - }) => Ok(FsHelperRequest::WriteFile(FsWriteFileParams { - path: { - let path = resolve_sandbox_path(&path, PreserveTerminalSymlink::No)?; - ensure_path_access(file_system_policy, cwd, &path, FileSystemAccessMode::Write)?; - path - }, - data_base64, - sandbox, - })), - FsHelperRequest::CreateDirectory(FsCreateDirectoryParams { - path, - recursive, - sandbox, - }) => Ok(FsHelperRequest::CreateDirectory(FsCreateDirectoryParams { - path: { - let path = resolve_sandbox_path(&path, PreserveTerminalSymlink::No)?; - ensure_path_access(file_system_policy, cwd, &path, FileSystemAccessMode::Write)?; - path - }, - recursive, - sandbox, - })), - FsHelperRequest::GetMetadata(FsGetMetadataParams { path, sandbox }) => { - let path = resolve_sandbox_path(&path, PreserveTerminalSymlink::No)?; - ensure_path_access(file_system_policy, cwd, &path, FileSystemAccessMode::Read)?; - Ok(FsHelperRequest::GetMetadata(FsGetMetadataParams { - path, - sandbox, - })) - } - FsHelperRequest::ReadDirectory(FsReadDirectoryParams { path, sandbox }) => { - let path = resolve_sandbox_path(&path, PreserveTerminalSymlink::No)?; - ensure_path_access(file_system_policy, cwd, &path, FileSystemAccessMode::Read)?; - Ok(FsHelperRequest::ReadDirectory(FsReadDirectoryParams { - path, - sandbox, - })) - } - FsHelperRequest::Remove(FsRemoveParams { - path, - recursive, - force, - sandbox, - }) => Ok(FsHelperRequest::Remove(FsRemoveParams { - path: { - let path = resolve_sandbox_path(&path, PreserveTerminalSymlink::Yes)?; - ensure_path_access(file_system_policy, cwd, &path, FileSystemAccessMode::Write)?; - path - }, - recursive, - force, - sandbox, - })), - FsHelperRequest::Copy(FsCopyParams { - source_path, - destination_path, - recursive, - sandbox, - }) => Ok(FsHelperRequest::Copy(FsCopyParams { - source_path: { - let source_path = resolve_sandbox_path(&source_path, PreserveTerminalSymlink::Yes)?; - ensure_path_access( - file_system_policy, - cwd, - &source_path, - FileSystemAccessMode::Read, - )?; - source_path - }, - destination_path: { - let destination_path = - resolve_sandbox_path(&destination_path, PreserveTerminalSymlink::No)?; - ensure_path_access( - file_system_policy, - cwd, - &destination_path, - FileSystemAccessMode::Write, - )?; - destination_path - }, - recursive, - sandbox, - })), - } -} - -#[derive(Clone, Copy)] -enum PreserveTerminalSymlink { - Yes, - No, -} - -fn resolve_sandbox_path( - path: &AbsolutePathBuf, - preserve_terminal_symlink: PreserveTerminalSymlink, -) -> Result { - if matches!(preserve_terminal_symlink, PreserveTerminalSymlink::Yes) - && std::fs::symlink_metadata(path.as_path()) - .map(|metadata| metadata.file_type().is_symlink()) - .unwrap_or(false) - { - return Ok(normalize_top_level_alias(path.clone())); - } - - let resolved = resolve_existing_path(path.as_path()).map_err(io_error)?; - absolute_path(resolved) -} - fn normalize_sandbox_policy_root_aliases(sandbox_policy: SandboxPolicy) -> SandboxPolicy { let mut sandbox_policy = sandbox_policy; match &mut sandbox_policy { @@ -338,33 +195,6 @@ fn normalize_top_level_alias(path: AbsolutePathBuf) -> AbsolutePathBuf { path } -fn absolute_path(path: PathBuf) -> Result { - AbsolutePathBuf::from_absolute_path(path.as_path()) - .map_err(|err| invalid_request(format!("resolved sandbox path is not absolute: {err}"))) -} - -fn ensure_path_access( - file_system_policy: &FileSystemSandboxPolicy, - cwd: &AbsolutePathBuf, - path: &AbsolutePathBuf, - required_access: FileSystemAccessMode, -) -> Result<(), JSONRPCErrorError> { - let actual_access = file_system_policy.resolve_access_with_cwd(path.as_path(), cwd.as_path()); - let permitted = match required_access { - FileSystemAccessMode::Read => actual_access.can_read(), - FileSystemAccessMode::Write => actual_access.can_write(), - FileSystemAccessMode::None => true, - }; - if permitted { - return Ok(()); - } - - Err(invalid_request(format!( - "{} is not permitted by filesystem sandbox", - path.display() - ))) -} - async fn run_command( command: SandboxExecRequest, request_json: Vec, diff --git a/codex-rs/exec-server/src/local_file_system.rs b/codex-rs/exec-server/src/local_file_system.rs index fe9a4a84f4..c7d628ce8f 100644 --- a/codex-rs/exec-server/src/local_file_system.rs +++ b/codex-rs/exec-server/src/local_file_system.rs @@ -305,7 +305,9 @@ impl ExecutorFileSystem for DirectFileSystem { let mut entries = Vec::new(); let mut read_dir = tokio::fs::read_dir(path.as_path()).await?; while let Some(entry) = read_dir.next_entry().await? { - let metadata = tokio::fs::metadata(entry.path()).await?; + let Ok(metadata) = tokio::fs::metadata(entry.path()).await else { + continue; + }; entries.push(ReadDirectoryEntry { file_name: entry.file_name().to_string_lossy().into_owned(), is_directory: metadata.is_dir(), diff --git a/codex-rs/exec-server/src/local_process.rs b/codex-rs/exec-server/src/local_process.rs index bf38aa360b..2608f892b8 100644 --- a/codex-rs/exec-server/src/local_process.rs +++ b/codex-rs/exec-server/src/local_process.rs @@ -59,6 +59,7 @@ struct RetainedOutputChunk { struct RunningProcess { session: ExecCommandSession, tty: bool, + pipe_stdin: bool, output: VecDeque, retained_bytes: usize, next_seq: u64, @@ -165,6 +166,15 @@ impl LocalProcess { TerminalSize::default(), ) .await + } else if params.pipe_stdin { + codex_utils_pty::spawn_pipe_process( + program, + args, + params.cwd.as_path(), + &env, + ¶ms.arg0, + ) + .await } else { codex_utils_pty::spawn_pipe_process_no_stdin( program, @@ -195,6 +205,7 @@ impl LocalProcess { ProcessEntry::Running(Box::new(RunningProcess { session: spawned.session, tty: params.tty, + pipe_stdin: params.pipe_stdin, output: VecDeque::new(), retained_bytes: 0, next_seq: 1, @@ -339,7 +350,7 @@ impl LocalProcess { status: WriteStatus::Starting, }); }; - if !process.tty { + if !process.tty && !process.pipe_stdin { return Ok(WriteResponse { status: WriteStatus::StdinClosed, }); @@ -667,6 +678,7 @@ mod tests { env_policy: None, env, tty: false, + pipe_stdin: false, arg0: None, } } diff --git a/codex-rs/exec-server/src/protocol.rs b/codex-rs/exec-server/src/protocol.rs index 5d29348897..9f4234e649 100644 --- a/codex-rs/exec-server/src/protocol.rs +++ b/codex-rs/exec-server/src/protocol.rs @@ -69,6 +69,9 @@ pub struct ExecParams { pub env_policy: Option, pub env: HashMap, pub tty: bool, + /// Keep non-tty stdin writable through `process/write`. + #[serde(default)] + pub pipe_stdin: bool, pub arg0: Option, } diff --git a/codex-rs/exec-server/src/server/handler/tests.rs b/codex-rs/exec-server/src/server/handler/tests.rs index 321bf243a9..7f2cdba629 100644 --- a/codex-rs/exec-server/src/server/handler/tests.rs +++ b/codex-rs/exec-server/src/server/handler/tests.rs @@ -30,6 +30,7 @@ fn exec_params_with_argv(process_id: &str, argv: Vec) -> ExecParams { env_policy: None, env: inherited_path_env(), tty: false, + pipe_stdin: false, arg0: None, } } diff --git a/codex-rs/exec-server/src/server/processor.rs b/codex-rs/exec-server/src/server/processor.rs index 87b9704502..1153bc83e2 100644 --- a/codex-rs/exec-server/src/server/processor.rs +++ b/codex-rs/exec-server/src/server/processor.rs @@ -393,6 +393,7 @@ mod tests { env_policy: None, env, tty: false, + pipe_stdin: false, arg0: None, } } diff --git a/codex-rs/exec-server/tests/common/exec_server.rs b/codex-rs/exec-server/tests/common/exec_server.rs index ca4be44856..2a134546ac 100644 --- a/codex-rs/exec-server/tests/common/exec_server.rs +++ b/codex-rs/exec-server/tests/common/exec_server.rs @@ -22,7 +22,7 @@ use tokio::time::timeout; use tokio_tungstenite::connect_async; use tokio_tungstenite::tungstenite::Message; -const CONNECT_TIMEOUT: Duration = Duration::from_secs(5); +const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); const CONNECT_RETRY_INTERVAL: Duration = Duration::from_millis(25); const EVENT_TIMEOUT: Duration = Duration::from_secs(5); diff --git a/codex-rs/exec-server/tests/exec_process.rs b/codex-rs/exec-server/tests/exec_process.rs index afe0c2e356..862e4e1c8b 100644 --- a/codex-rs/exec-server/tests/exec_process.rs +++ b/codex-rs/exec-server/tests/exec_process.rs @@ -12,6 +12,7 @@ use codex_exec_server::ExecProcess; use codex_exec_server::ProcessId; use codex_exec_server::ReadResponse; use codex_exec_server::StartedExecProcess; +use codex_exec_server::WriteStatus; use pretty_assertions::assert_eq; use test_case::test_case; use tokio::sync::watch; @@ -54,6 +55,7 @@ async fn assert_exec_process_starts_and_exits(use_remote: bool) -> Result<()> { env_policy: /*env_policy*/ None, env: Default::default(), tty: false, + pipe_stdin: false, arg0: None, }) .await?; @@ -131,6 +133,7 @@ async fn assert_exec_process_streams_output(use_remote: bool) -> Result<()> { env_policy: /*env_policy*/ None, env: Default::default(), tty: false, + pipe_stdin: false, arg0: None, }) .await?; @@ -153,14 +156,18 @@ async fn assert_exec_process_write_then_read(use_remote: bool) -> Result<()> { .start(ExecParams { process_id: process_id.clone().into(), argv: vec![ - "/usr/bin/python3".to_string(), + // Use `/bin/sh` instead of Python so this stdin round-trip test + // stays portable across Bazel and non-macOS runners where + // `/usr/bin/python3` is not guaranteed to exist. + "/bin/sh".to_string(), "-c".to_string(), - "import sys; line = sys.stdin.readline(); sys.stdout.write(f'from-stdin:{line}'); sys.stdout.flush()".to_string(), + "IFS= read line; printf 'from-stdin:%s\\n' \"$line\"".to_string(), ], cwd: std::env::current_dir()?, env_policy: /*env_policy*/ None, env: Default::default(), tty: true, + pipe_stdin: false, arg0: None, }) .await?; @@ -181,6 +188,73 @@ async fn assert_exec_process_write_then_read(use_remote: bool) -> Result<()> { Ok(()) } +async fn assert_exec_process_write_then_read_without_tty(use_remote: bool) -> Result<()> { + let context = create_process_context(use_remote).await?; + let process_id = "proc-stdin-pipe".to_string(); + let session = context + .backend + .start(ExecParams { + process_id: process_id.clone().into(), + argv: vec![ + "/bin/sh".to_string(), + "-c".to_string(), + "IFS= read line; printf 'from-stdin:%s\\n' \"$line\"".to_string(), + ], + cwd: std::env::current_dir()?, + env_policy: /*env_policy*/ None, + env: Default::default(), + tty: false, + pipe_stdin: true, + arg0: None, + }) + .await?; + assert_eq!(session.process.process_id().as_str(), process_id); + + tokio::time::sleep(Duration::from_millis(200)).await; + let write_response = session.process.write(b"hello\n".to_vec()).await?; + assert_eq!(write_response.status, WriteStatus::Accepted); + let StartedExecProcess { process } = session; + let wake_rx = process.subscribe_wake(); + let actual = collect_process_output_from_reads(process, wake_rx).await?; + + assert_eq!(actual, ("from-stdin:hello\n".to_string(), Some(0), true)); + Ok(()) +} + +async fn assert_exec_process_rejects_write_without_pipe_stdin(use_remote: bool) -> Result<()> { + let context = create_process_context(use_remote).await?; + let process_id = "proc-stdin-closed".to_string(); + let session = context + .backend + .start(ExecParams { + process_id: process_id.clone().into(), + argv: vec![ + "/bin/sh".to_string(), + "-c".to_string(), + "sleep 0.3; if IFS= read -r line; then printf 'read:%s\\n' \"$line\"; else printf 'eof\\n'; fi".to_string(), + ], + cwd: std::env::current_dir()?, + env_policy: /*env_policy*/ None, + env: Default::default(), + tty: false, + pipe_stdin: false, + arg0: None, + }) + .await?; + assert_eq!(session.process.process_id().as_str(), process_id); + + let write_response = session.process.write(b"ignored\n".to_vec()).await?; + assert_eq!(write_response.status, WriteStatus::StdinClosed); + let StartedExecProcess { process } = session; + let wake_rx = process.subscribe_wake(); + let (output, exit_code, closed) = collect_process_output_from_reads(process, wake_rx).await?; + + assert_eq!(output, "eof\n"); + assert_eq!(exit_code, Some(0)); + assert!(closed); + Ok(()) +} + async fn assert_exec_process_preserves_queued_events_before_subscribe( use_remote: bool, ) -> Result<()> { @@ -198,6 +272,7 @@ async fn assert_exec_process_preserves_queued_events_before_subscribe( env_policy: /*env_policy*/ None, env: Default::default(), tty: false, + pipe_stdin: false, arg0: None, }) .await?; @@ -231,6 +306,7 @@ async fn remote_exec_process_reports_transport_disconnect() -> Result<()> { env_policy: /*env_policy*/ None, env: Default::default(), tty: false, + pipe_stdin: false, arg0: None, }) .await?; @@ -286,6 +362,24 @@ async fn exec_process_write_then_read(use_remote: bool) -> Result<()> { assert_exec_process_write_then_read(use_remote).await } +#[test_case(false ; "local")] +#[test_case(true ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +// Serialize tests that launch a real exec-server process through the full CLI. +#[serial_test::serial(remote_exec_server)] +async fn exec_process_write_then_read_without_tty(use_remote: bool) -> Result<()> { + assert_exec_process_write_then_read_without_tty(use_remote).await +} + +#[test_case(false ; "local")] +#[test_case(true ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +// Serialize tests that launch a real exec-server process through the full CLI. +#[serial_test::serial(remote_exec_server)] +async fn exec_process_rejects_write_without_pipe_stdin(use_remote: bool) -> Result<()> { + assert_exec_process_rejects_write_without_pipe_stdin(use_remote).await +} + #[test_case(false ; "local")] #[test_case(true ; "remote")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/codex-rs/exec-server/tests/file_system.rs b/codex-rs/exec-server/tests/file_system.rs index e24a2605f0..8b9fb90ecc 100644 --- a/codex-rs/exec-server/tests/file_system.rs +++ b/codex-rs/exec-server/tests/file_system.rs @@ -3,6 +3,8 @@ mod common; use std::os::unix::fs::symlink; +use std::path::Path; +use std::path::PathBuf; use std::process::Command; use std::sync::Arc; @@ -17,6 +19,8 @@ use codex_exec_server::FileSystemSandboxContext; use codex_exec_server::LocalFileSystem; use codex_exec_server::ReadDirectoryEntry; use codex_exec_server::RemoveOptions; +use codex_protocol::models::FileSystemPermissions; +use codex_protocol::models::PermissionProfile; use codex_protocol::protocol::ReadOnlyAccess; use codex_protocol::protocol::SandboxPolicy; use codex_utils_absolute_path::AbsolutePathBuf; @@ -94,20 +98,26 @@ fn workspace_write_sandbox(writable_root: std::path::PathBuf) -> FileSystemSandb } fn assert_sandbox_denied(error: &std::io::Error) { - assert!( - matches!( - error.kind(), - std::io::ErrorKind::InvalidInput | std::io::ErrorKind::PermissionDenied + match error.kind() { + std::io::ErrorKind::InvalidInput | std::io::ErrorKind::PermissionDenied => { + let message = error.to_string(); + assert!( + message.contains("is not permitted") + || message.contains("Operation not permitted") + || message.contains("Permission denied"), + "unexpected sandbox error message: {message}", + ); + } + std::io::ErrorKind::NotFound => assert!( + error.to_string().contains("No such file or directory"), + "unexpected sandbox not-found message: {error}", ), - "unexpected sandbox error kind: {error:?}", - ); - let message = error.to_string(); - assert!( - message.contains("is not permitted") - || message.contains("Operation not permitted") - || message.contains("Permission denied"), - "unexpected sandbox error message: {message}", - ); + std::io::ErrorKind::Other => assert!( + error.to_string().contains("Read-only file system"), + "unexpected sandbox other error message: {error}", + ), + other => panic!("unexpected sandbox error kind: {other:?}: {error:?}"), + } } fn assert_normalized_path_rejected(error: &std::io::Error) { @@ -129,6 +139,15 @@ fn assert_normalized_path_rejected(error: &std::io::Error) { } } +fn alias_root_candidate() -> Result> { + for root in [Path::new("/tmp").to_path_buf(), std::env::temp_dir()] { + if root.is_dir() && root.canonicalize().is_ok_and(|canonical| canonical != root) { + return Ok(Some(root)); + } + } + Ok(None) +} + #[test_case(false ; "local")] #[test_case(true ; "remote")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -253,6 +272,11 @@ async fn file_system_methods_cover_surface_area(use_remote: bool) -> Result<()> "hello from trait" ); + symlink( + source_dir.join("missing-target"), + source_dir.join("broken-link"), + )?; + let mut entries = file_system .read_directory(&absolute_path(source_dir), /*sandbox*/ None) .await @@ -353,9 +377,11 @@ async fn file_system_copy_rejects_directory_without_recursive(use_remote: bool) Ok(()) } +#[test_case(false ; "local")] +#[test_case(true ; "remote")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_sandboxed_read_allows_readable_root() -> Result<()> { - let context = create_file_system_context(/*use_remote*/ false).await?; +async fn file_system_sandboxed_read_allows_readable_root(use_remote: bool) -> Result<()> { + let context = create_file_system_context(use_remote).await?; let file_system = context.file_system; let tmp = TempDir::new()?; @@ -367,7 +393,8 @@ async fn file_system_sandboxed_read_allows_readable_root() -> Result<()> { let contents = file_system .read_file(&absolute_path(file_path), Some(&sandbox)) - .await?; + .await + .with_context(|| format!("mode={use_remote}"))?; assert_eq!(contents, b"sandboxed hello"); Ok(()) @@ -381,11 +408,9 @@ async fn file_system_sandboxed_write_rejects_unwritable_path(use_remote: bool) - let file_system = context.file_system; let tmp = TempDir::new()?; - let allowed_dir = tmp.path().join("allowed"); let blocked_path = tmp.path().join("blocked.txt"); - std::fs::create_dir_all(&allowed_dir)?; - let sandbox = read_only_sandbox(allowed_dir); + let sandbox = read_only_sandbox(tmp.path().to_path_buf()); let error = match file_system .write_file( &absolute_path(blocked_path.clone()), @@ -403,6 +428,72 @@ async fn file_system_sandboxed_write_rejects_unwritable_path(use_remote: bool) - Ok(()) } +#[test_case(false ; "local")] +#[test_case(true ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_sandboxed_write_allows_explicit_alias_roots(use_remote: bool) -> Result<()> { + let Some(alias_root) = alias_root_candidate()? else { + return Ok(()); + }; + + let context = create_file_system_context(use_remote).await?; + let file_system = context.file_system; + + let tmp = tempfile::Builder::new() + .prefix("codex-fs-sandbox-alias-") + .tempdir_in(&alias_root)?; + let file_path = tmp.path().join("note.txt"); + let sandbox = workspace_write_sandbox(alias_root.clone()); + + file_system + .write_file( + &absolute_path(file_path.clone()), + b"created".to_vec(), + Some(&sandbox), + ) + .await + .with_context(|| format!("write file through alias root mode={use_remote}"))?; + assert_eq!(std::fs::read(&file_path)?, b"created"); + + Ok(()) +} + +#[test_case(false ; "local")] +#[test_case(true ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_sandboxed_write_allows_additional_write_root(use_remote: bool) -> Result<()> { + let context = create_file_system_context(use_remote).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let readable_dir = tmp.path().join("readable"); + let writable_dir = tmp.path().join("writable"); + let file_path = writable_dir.join("note.txt"); + std::fs::create_dir_all(&readable_dir)?; + std::fs::create_dir_all(&writable_dir)?; + + let mut sandbox = read_only_sandbox(readable_dir); + sandbox.additional_permissions = Some(PermissionProfile { + network: None, + file_system: Some(FileSystemPermissions { + read: None, + write: Some(vec![absolute_path(writable_dir)]), + }), + }); + + file_system + .write_file( + &absolute_path(file_path.clone()), + b"created".to_vec(), + Some(&sandbox), + ) + .await + .with_context(|| format!("write file through additional root mode={use_remote}"))?; + assert_eq!(std::fs::read(&file_path)?, b"created"); + + Ok(()) +} + #[test_case(false ; "local")] #[test_case(true ; "remote")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -432,9 +523,13 @@ async fn file_system_sandboxed_read_rejects_symlink_escape(use_remote: bool) -> Ok(()) } +#[test_case(false ; "local")] +#[test_case(true ; "remote")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_sandboxed_read_rejects_symlink_parent_dotdot_escape() -> Result<()> { - let context = create_file_system_context(/*use_remote*/ false).await?; +async fn file_system_sandboxed_read_rejects_symlink_parent_dotdot_escape( + use_remote: bool, +) -> Result<()> { + let context = create_file_system_context(use_remote).await?; let file_system = context.file_system; let tmp = TempDir::new()?; @@ -528,35 +623,6 @@ async fn file_system_create_directory_rejects_symlink_escape(use_remote: bool) - Ok(()) } -#[test_case(false ; "local")] -#[test_case(true ; "remote")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_get_metadata_rejects_symlink_escape(use_remote: bool) -> Result<()> { - let context = create_file_system_context(use_remote).await?; - let file_system = context.file_system; - - let tmp = TempDir::new()?; - let allowed_dir = tmp.path().join("allowed"); - let outside_dir = tmp.path().join("outside"); - std::fs::create_dir_all(&allowed_dir)?; - std::fs::create_dir_all(&outside_dir)?; - std::fs::write(outside_dir.join("secret.txt"), "nope")?; - symlink(&outside_dir, allowed_dir.join("link"))?; - - let requested_path = allowed_dir.join("link").join("secret.txt"); - let sandbox = read_only_sandbox(allowed_dir); - let error = match file_system - .get_metadata(&absolute_path(requested_path.clone()), Some(&sandbox)) - .await - { - Ok(_) => anyhow::bail!("get_metadata should be blocked"), - Err(error) => error, - }; - assert_sandbox_denied(&error); - - Ok(()) -} - #[test_case(false ; "local")] #[test_case(true ; "remote")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -621,9 +687,11 @@ async fn file_system_copy_rejects_symlink_escape_destination(use_remote: bool) - Ok(()) } +#[test_case(false ; "local")] +#[test_case(true ; "remote")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_remove_removes_symlink_not_target() -> Result<()> { - let context = create_file_system_context(/*use_remote*/ false).await?; +async fn file_system_remove_removes_symlink_not_target(use_remote: bool) -> Result<()> { + let context = create_file_system_context(use_remote).await?; let file_system = context.file_system; let tmp = TempDir::new()?; @@ -646,7 +714,8 @@ async fn file_system_remove_removes_symlink_not_target() -> Result<()> { }, Some(&sandbox), ) - .await?; + .await + .with_context(|| format!("mode={use_remote}"))?; assert!(!symlink_path.exists()); assert!(outside_file.exists()); @@ -655,9 +724,11 @@ async fn file_system_remove_removes_symlink_not_target() -> Result<()> { Ok(()) } +#[test_case(false ; "local")] +#[test_case(true ; "remote")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_copy_preserves_symlink_source() -> Result<()> { - let context = create_file_system_context(/*use_remote*/ false).await?; +async fn file_system_copy_preserves_symlink_source(use_remote: bool) -> Result<()> { + let context = create_file_system_context(use_remote).await?; let file_system = context.file_system; let tmp = TempDir::new()?; @@ -679,7 +750,8 @@ async fn file_system_copy_preserves_symlink_source() -> Result<()> { CopyOptions { recursive: false }, Some(&sandbox), ) - .await?; + .await + .with_context(|| format!("mode={use_remote}"))?; let copied_metadata = std::fs::symlink_metadata(&copied_symlink)?; assert!(copied_metadata.file_type().is_symlink()); diff --git a/codex-rs/exec-server/tests/process.rs b/codex-rs/exec-server/tests/process.rs index cc96e6ef81..334e449224 100644 --- a/codex-rs/exec-server/tests/process.rs +++ b/codex-rs/exec-server/tests/process.rs @@ -10,6 +10,8 @@ use codex_exec_server::InitializeResponse; use codex_exec_server::ProcessId; use codex_exec_server::ReadResponse; use codex_exec_server::TerminateResponse; +use codex_exec_server::WriteResponse; +use codex_exec_server::WriteStatus; use common::exec_server::exec_server; use pretty_assertions::assert_eq; @@ -47,6 +49,7 @@ async fn exec_server_starts_process_over_websocket() -> anyhow::Result<()> { "cwd": std::env::current_dir()?, "env": {}, "tty": false, + "pipeStdin": false, "arg0": null }), ) @@ -75,6 +78,99 @@ async fn exec_server_starts_process_over_websocket() -> anyhow::Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn exec_server_defaults_omitted_pipe_stdin_to_closed_stdin() -> anyhow::Result<()> { + let mut server = exec_server().await?; + let initialize_id = server + .send_request( + "initialize", + serde_json::to_value(InitializeParams { + client_name: "exec-server-test".to_string(), + resume_session_id: None, + })?, + ) + .await?; + let _ = server + .wait_for_event(|event| { + matches!( + event, + JSONRPCMessage::Response(JSONRPCResponse { id, .. }) if id == &initialize_id + ) + }) + .await?; + + server + .send_notification("initialized", serde_json::json!({})) + .await?; + + let process_start_id = server + .send_request( + "process/start", + serde_json::json!({ + "processId": "proc-default-stdin", + "argv": [ + "/bin/sh", + "-c", + "sleep 0.3; if IFS= read -r line; then printf 'read:%s\\n' \"$line\"; else printf 'eof\\n'; fi" + ], + "cwd": std::env::current_dir()?, + "env": {}, + "tty": false, + "arg0": null + }), + ) + .await?; + let response = server + .wait_for_event(|event| { + matches!( + event, + JSONRPCMessage::Response(JSONRPCResponse { id, .. }) if id == &process_start_id + ) + }) + .await?; + let JSONRPCMessage::Response(JSONRPCResponse { result, .. }) = response else { + panic!("expected process/start response"); + }; + let process_start_response: ExecResponse = serde_json::from_value(result)?; + assert_eq!( + process_start_response, + ExecResponse { + process_id: ProcessId::from("proc-default-stdin") + } + ); + + let write_id = server + .send_request( + "process/write", + serde_json::json!({ + "processId": "proc-default-stdin", + "chunk": "aWdub3JlZAo=" + }), + ) + .await?; + let response = server + .wait_for_event(|event| { + matches!( + event, + JSONRPCMessage::Response(JSONRPCResponse { id, .. }) if id == &write_id + ) + }) + .await?; + let JSONRPCMessage::Response(JSONRPCResponse { result, .. }) = response else { + panic!("expected process/write response"); + }; + let write_response: WriteResponse = serde_json::from_value(result)?; + assert_eq!( + write_response, + WriteResponse { + status: WriteStatus::StdinClosed + } + ); + + server.shutdown().await?; + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn exec_server_resumes_detached_session_without_killing_processes() -> anyhow::Result<()> { let mut server = exec_server().await?; @@ -113,6 +209,7 @@ async fn exec_server_resumes_detached_session_without_killing_processes() -> any "cwd": std::env::current_dir()?, "env": {}, "tty": false, + "pipeStdin": false, "arg0": null }), ) diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index 758da9f43d..123038ffaa 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -481,6 +481,7 @@ pub async fn run_main(cli: Cli, arg0_paths: Arg0DispatchPaths) -> anyhow::Result loader_overrides: run_loader_overrides, cloud_requirements: run_cloud_requirements, feedback: CodexFeedback::new(), + log_db: None, environment_manager: std::sync::Arc::new(EnvironmentManager::from_env_with_runtime_paths( Some(local_runtime_paths), )), diff --git a/codex-rs/exec/tests/event_processor_with_json_output.rs b/codex-rs/exec/tests/event_processor_with_json_output.rs index 71b5767348..02e52988cb 100644 --- a/codex-rs/exec/tests/event_processor_with_json_output.rs +++ b/codex-rs/exec/tests/event_processor_with_json_output.rs @@ -464,6 +464,7 @@ fn mcp_tool_call_begin_and_end_emit_item_events() { tool: "tool_x".to_string(), status: ApiMcpToolCallStatus::InProgress, arguments: json!({ "key": "value" }), + mcp_app_resource_uri: None, result: None, error: None, duration_ms: None, @@ -479,11 +480,12 @@ fn mcp_tool_call_begin_and_end_emit_item_events() { tool: "tool_x".to_string(), status: ApiMcpToolCallStatus::Completed, arguments: json!({ "key": "value" }), - result: Some(McpToolCallResult { + mcp_app_resource_uri: None, + result: Some(Box::new(McpToolCallResult { content: Vec::new(), structured_content: None, meta: None, - }), + })), error: None, duration_ms: Some(1_000), }, @@ -547,6 +549,7 @@ fn mcp_tool_call_failure_sets_failed_status() { tool: "tool_y".to_string(), status: ApiMcpToolCallStatus::Failed, arguments: json!({ "param": 42 }), + mcp_app_resource_uri: None, result: None, error: Some(McpToolCallError { message: "tool exploded".to_string(), @@ -593,6 +596,7 @@ fn mcp_tool_call_defaults_arguments_and_preserves_structured_content() { tool: "tool_z".to_string(), status: ApiMcpToolCallStatus::InProgress, arguments: serde_json::Value::Null, + mcp_app_resource_uri: None, result: None, error: None, duration_ms: None, @@ -608,14 +612,15 @@ fn mcp_tool_call_defaults_arguments_and_preserves_structured_content() { tool: "tool_z".to_string(), status: ApiMcpToolCallStatus::Completed, arguments: serde_json::Value::Null, - result: Some(McpToolCallResult { + mcp_app_resource_uri: None, + result: Some(Box::new(McpToolCallResult { content: vec![json!({ "type": "text", "text": "done", })], structured_content: Some(json!({ "status": "ok" })), meta: None, - }), + })), error: None, duration_ms: Some(10), }, diff --git a/codex-rs/exec/tests/suite/sandbox.rs b/codex-rs/exec/tests/suite/sandbox.rs index 7a8c3238e8..691b590f4e 100644 --- a/codex-rs/exec/tests/suite/sandbox.rs +++ b/codex-rs/exec/tests/suite/sandbox.rs @@ -19,17 +19,67 @@ async fn spawn_command_under_sandbox( stdio_policy: StdioPolicy, env: HashMap, ) -> std::io::Result { - use codex_core::seatbelt::spawn_command_under_seatbelt; - spawn_command_under_seatbelt( - command, - command_cwd, + use codex_core::exec::ExecCapturePolicy; + use codex_core::exec::ExecParams; + use codex_core::exec::build_exec_request; + use codex_core::sandboxing::SandboxPermissions; + use codex_protocol::config_types::WindowsSandboxLevel; + use codex_protocol::permissions::FileSystemSandboxPolicy; + use codex_protocol::permissions::NetworkSandboxPolicy; + use std::process::Stdio; + + let codex_linux_sandbox_exe = None; + let exec_request = build_exec_request( + ExecParams { + command, + cwd: command_cwd, + expiration: 1000.into(), + capture_policy: ExecCapturePolicy::ShellTool, + env, + network: None, + sandbox_permissions: SandboxPermissions::UseDefault, + windows_sandbox_level: WindowsSandboxLevel::Disabled, + windows_sandbox_private_desktop: false, + justification: None, + arg0: None, + }, sandbox_policy, + &FileSystemSandboxPolicy::from_legacy_sandbox_policy(sandbox_policy, sandbox_cwd), + NetworkSandboxPolicy::from(sandbox_policy), sandbox_cwd, - stdio_policy, - /*network*/ None, - env, + &codex_linux_sandbox_exe, + /*use_legacy_landlock*/ false, ) - .await + .map_err(|err| io::Error::other(err.to_string()))?; + + let (program, args) = exec_request + .command + .split_first() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "command args are empty"))?; + + let mut child = tokio::process::Command::new(program); + if let Some(arg0) = exec_request.arg0.as_deref() { + child.arg0(arg0); + } + child.args(args); + child.current_dir(exec_request.cwd); + child.env_clear(); + child.envs(exec_request.env); + + match stdio_policy { + StdioPolicy::RedirectForShellTool => { + child.stdin(Stdio::null()); + child.stdout(Stdio::piped()).stderr(Stdio::piped()); + } + StdioPolicy::Inherit => { + child + .stdin(Stdio::inherit()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()); + } + } + + child.kill_on_drop(true).spawn() } #[cfg(target_os = "linux")] diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rs index 8e86ec56fa..9b5d9f417e 100644 --- a/codex-rs/features/src/lib.rs +++ b/codex-rs/features/src/lib.rs @@ -146,6 +146,8 @@ pub enum Feature { Apps, /// Enable the tool_search tool for apps. ToolSearch, + /// Expose placeholder tools for unavailable historical tool calls. + UnavailableDummyTools, /// Enable discoverable tool suggestions for apps. ToolSuggest, /// Enable plugins. @@ -193,6 +195,8 @@ pub enum Feature { ResponsesWebsocketsV2, /// Use the agent identity registration flow for ChatGPT-authenticated sessions. UseAgentIdentity, + /// Enable workspace dependency support. + WorkspaceDependencies, } impl Feature { @@ -674,7 +678,11 @@ pub const FEATURES: &[FeatureSpec] = &[ FeatureSpec { id: Feature::MemoryTool, key: "memories", - stage: Stage::UnderDevelopment, + stage: Stage::Experimental { + name: "Memories", + menu_description: "Allow Codex to create new memories from conversations and bring relevant memories into new conversations.", + announcement: "NEW: Codex can now generate and uses memories. Try is now with `/memories`", + }, default_enabled: false, }, FeatureSpec { @@ -782,6 +790,12 @@ pub const FEATURES: &[FeatureSpec] = &[ FeatureSpec { id: Feature::ToolSearch, key: "tool_search", + stage: Stage::Stable, + default_enabled: true, + }, + FeatureSpec { + id: Feature::UnavailableDummyTools, + key: "unavailable_dummy_tools", stage: Stage::UnderDevelopment, default_enabled: false, }, @@ -800,8 +814,8 @@ pub const FEATURES: &[FeatureSpec] = &[ FeatureSpec { id: Feature::ImageGeneration, key: "image_generation", - stage: Stage::UnderDevelopment, - default_enabled: false, + stage: Stage::Stable, + default_enabled: true, }, FeatureSpec { id: Feature::SkillMcpDependencyInstall, @@ -933,6 +947,12 @@ pub const FEATURES: &[FeatureSpec] = &[ stage: Stage::UnderDevelopment, default_enabled: false, }, + FeatureSpec { + id: Feature::WorkspaceDependencies, + key: "workspace_dependencies", + stage: Stage::Stable, + default_enabled: true, + }, ]; pub fn unstable_features_warning_event( diff --git a/codex-rs/features/src/tests.rs b/codex-rs/features/src/tests.rs index 367527ba15..d2f73cedad 100644 --- a/codex-rs/features/src/tests.rs +++ b/codex-rs/features/src/tests.rs @@ -128,9 +128,18 @@ fn tool_suggest_is_stable_and_enabled_by_default() { } #[test] -fn tool_search_is_under_development_and_disabled_by_default() { - assert_eq!(Feature::ToolSearch.stage(), Stage::UnderDevelopment); - assert_eq!(Feature::ToolSearch.default_enabled(), false); +fn tool_search_is_stable_and_enabled_by_default() { + assert_eq!(Feature::ToolSearch.stage(), Stage::Stable); + assert_eq!(Feature::ToolSearch.default_enabled(), true); +} + +#[test] +fn unavailable_dummy_tools_is_under_development_and_disabled_by_default() { + assert_eq!( + Feature::UnavailableDummyTools.stage(), + Stage::UnderDevelopment + ); + assert_eq!(Feature::UnavailableDummyTools.default_enabled(), false); } #[test] @@ -151,6 +160,12 @@ fn use_linux_sandbox_bwrap_is_a_removed_feature_key() { ); } +#[test] +fn image_generation_is_stable_and_enabled_by_default() { + assert_eq!(Feature::ImageGeneration.stage(), Stage::Stable); + assert_eq!(Feature::ImageGeneration.default_enabled(), true); +} + #[test] fn image_detail_original_is_a_removed_feature_key() { assert_eq!( @@ -159,12 +174,6 @@ fn image_detail_original_is_a_removed_feature_key() { ); } -#[test] -fn image_generation_is_under_development() { - assert_eq!(Feature::ImageGeneration.stage(), Stage::UnderDevelopment); - assert_eq!(Feature::ImageGeneration.default_enabled(), false); -} - #[test] fn tool_call_mcp_elicitation_is_stable_and_enabled_by_default() { assert_eq!(Feature::ToolCallMcpElicitation.stage(), Stage::Stable); @@ -183,6 +192,16 @@ fn use_agent_identity_is_under_development() { assert_eq!(Feature::UseAgentIdentity.default_enabled(), false); } +#[test] +fn workspace_dependencies_is_stable_and_enabled_by_default() { + assert_eq!(Feature::WorkspaceDependencies.stage(), Stage::Stable); + assert_eq!(Feature::WorkspaceDependencies.default_enabled(), true); + assert_eq!( + feature_for_key("workspace_dependencies"), + Some(Feature::WorkspaceDependencies) + ); +} + #[test] fn collab_is_legacy_alias_for_multi_agent() { assert_eq!(feature_for_key("multi_agent"), Some(Feature::Collab)); diff --git a/codex-rs/git-utils/src/info.rs b/codex-rs/git-utils/src/info.rs index 048a8cedb6..e28a0bae42 100644 --- a/codex-rs/git-utils/src/info.rs +++ b/codex-rs/git-utils/src/info.rs @@ -618,7 +618,7 @@ async fn diff_against_sha(cwd: &Path, sha: &GitSha) -> Option { /// `[get_git_repo_root]`, but resolves to the root of the main /// repository. Handles worktrees via filesystem inspection without invoking /// the `git` executable. -pub fn resolve_root_git_project_for_trust(cwd: &Path) -> Option { +pub async fn resolve_root_git_project_for_trust(cwd: &Path) -> Option { let base = if cwd.is_dir() { cwd } else { cwd.parent()? }; let (repo_root, dot_git) = find_ancestor_git_entry(base)?; if dot_git.is_dir() { diff --git a/codex-rs/install-context/BUILD.bazel b/codex-rs/install-context/BUILD.bazel new file mode 100644 index 0000000000..68254d10cc --- /dev/null +++ b/codex-rs/install-context/BUILD.bazel @@ -0,0 +1,6 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "install-context", + crate_name = "codex_install_context", +) diff --git a/codex-rs/install-context/Cargo.toml b/codex-rs/install-context/Cargo.toml new file mode 100644 index 0000000000..ce4eeefe77 --- /dev/null +++ b/codex-rs/install-context/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "codex-install-context" +version.workspace = true +edition.workspace = true +license.workspace = true + +[lib] +name = "codex_install_context" +path = "src/lib.rs" + +[lints] +workspace = true + +[dependencies] +codex-utils-home-dir = { workspace = true } + +[dev-dependencies] +pretty_assertions = { workspace = true } +tempfile = { workspace = true } diff --git a/codex-rs/install-context/src/lib.rs b/codex-rs/install-context/src/lib.rs new file mode 100644 index 0000000000..980fc3f546 --- /dev/null +++ b/codex-rs/install-context/src/lib.rs @@ -0,0 +1,258 @@ +use std::path::Path; +use std::path::PathBuf; +use std::sync::OnceLock; + +const RELEASES_DIRNAME: &str = "releases"; +const RESOURCES_DIRNAME: &str = "codex-resources"; +const STANDALONE_PACKAGES_DIRNAME: &str = "standalone"; +static INSTALL_CONTEXT: OnceLock = OnceLock::new(); + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StandalonePlatform { + Unix, + Windows, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum InstallContext { + Standalone { + /// The managed standalone release directory, for example + /// `~/.codex/packages/standalone/releases/0.111.0-x86_64-unknown-linux-musl`. + release_dir: PathBuf, + /// The bundled resource directory that sits next to the executable when + /// this install ships managed dependencies. + resources_dir: Option, + /// The platform of the standalone release, either `Unix` or `Windows`. + platform: StandalonePlatform, + }, + /// A Codex binary launched through the npm-managed `codex.js` shim. + Npm, + /// A Codex binary launched through the bun-managed `codex.js` shim. + Bun, + /// A Codex binary that appears to come from a Homebrew install prefix. + Brew, + /// Any other execution environment. + /// + /// This commonly covers `cargo run`, app-bundled Codex binaries, custom + /// internal launchers, and tests that execute Codex from an arbitrary path. + Other, +} + +impl InstallContext { + pub fn from_exe( + is_macos: bool, + current_exe: Option<&Path>, + managed_by_npm: bool, + managed_by_bun: bool, + ) -> Self { + let codex_home = codex_utils_home_dir::find_codex_home().ok(); + Self::from_exe_with_codex_home( + is_macos, + current_exe, + managed_by_npm, + managed_by_bun, + codex_home.as_deref(), + ) + } + + fn from_exe_with_codex_home( + is_macos: bool, + current_exe: Option<&Path>, + managed_by_npm: bool, + managed_by_bun: bool, + codex_home: Option<&Path>, + ) -> Self { + if managed_by_npm { + return Self::Npm; + } + + if managed_by_bun { + return Self::Bun; + } + + if let Some(exe_path) = current_exe + && let Some(standalone_context) = standalone_install_context(exe_path, codex_home) + { + return standalone_context; + } + + if is_macos + && let Some(exe_path) = current_exe + && (exe_path.starts_with("/opt/homebrew") || exe_path.starts_with("/usr/local")) + { + return Self::Brew; + } + + Self::Other + } + + pub fn current() -> &'static Self { + INSTALL_CONTEXT.get_or_init(|| { + let current_exe = std::env::current_exe().ok(); + let managed_by_npm = std::env::var_os("CODEX_MANAGED_BY_NPM").is_some(); + let managed_by_bun = std::env::var_os("CODEX_MANAGED_BY_BUN").is_some(); + Self::from_exe( + cfg!(target_os = "macos"), + current_exe.as_deref(), + managed_by_npm, + managed_by_bun, + ) + }) + } + + pub fn rg_command(&self) -> PathBuf { + match self { + Self::Standalone { + resources_dir: Some(resources_dir), + .. + } => { + let bundled_rg = resources_dir.join(default_rg_command()); + if bundled_rg.exists() { + bundled_rg + } else { + default_rg_command() + } + } + Self::Standalone { + resources_dir: None, + .. + } + | Self::Npm + | Self::Bun + | Self::Brew + | Self::Other => default_rg_command(), + } + } +} + +fn standalone_install_context( + exe_path: &Path, + codex_home: Option<&Path>, +) -> Option { + let canonical_exe = std::fs::canonicalize(exe_path).ok()?; + let canonical_codex_home = std::fs::canonicalize(codex_home?).ok()?; + let release_dir = canonical_exe.parent()?.to_path_buf(); + let releases_root = canonical_codex_home + .join("packages") + .join(STANDALONE_PACKAGES_DIRNAME) + .join(RELEASES_DIRNAME); + if !release_dir.starts_with(releases_root) { + return None; + } + + let resources_dir = release_dir.join(RESOURCES_DIRNAME); + Some(InstallContext::Standalone { + release_dir, + resources_dir: resources_dir.is_dir().then_some(resources_dir), + platform: standalone_platform(), + }) +} + +fn standalone_platform() -> StandalonePlatform { + if cfg!(windows) { + StandalonePlatform::Windows + } else { + StandalonePlatform::Unix + } +} + +fn default_rg_command() -> PathBuf { + if cfg!(windows) { + PathBuf::from("rg.exe") + } else { + PathBuf::from("rg") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use std::fs; + + #[test] + fn detects_standalone_install_from_release_layout() -> std::io::Result<()> { + let codex_home = tempfile::tempdir()?; + let release_dir = codex_home + .path() + .join("packages/standalone/releases/1.2.3-x86_64-unknown-linux-musl"); + let resources_dir = release_dir.join(RESOURCES_DIRNAME); + fs::create_dir_all(&resources_dir)?; + let exe_path = release_dir.join(if cfg!(windows) { "codex.exe" } else { "codex" }); + fs::write(&exe_path, "")?; + fs::write(resources_dir.join(default_rg_command()), "")?; + let canonical_release_dir = release_dir.canonicalize()?; + let canonical_resources_dir = resources_dir.canonicalize()?; + + let context = InstallContext::from_exe_with_codex_home( + /*is_macos*/ false, + /*current_exe*/ Some(&exe_path), + /*managed_by_npm*/ false, + /*managed_by_bun*/ false, + /*codex_home*/ Some(codex_home.path()), + ); + assert_eq!( + context, + InstallContext::Standalone { + release_dir: canonical_release_dir, + resources_dir: Some(canonical_resources_dir), + platform: standalone_platform(), + } + ); + Ok(()) + } + + #[test] + fn standalone_rg_falls_back_when_resources_are_missing() -> std::io::Result<()> { + let codex_home = tempfile::tempdir()?; + let release_dir = codex_home + .path() + .join("packages/standalone/releases/1.2.3-x86_64-unknown-linux-musl"); + fs::create_dir_all(&release_dir)?; + let exe_path = release_dir.join(if cfg!(windows) { "codex.exe" } else { "codex" }); + fs::write(&exe_path, "")?; + + let context = InstallContext::from_exe_with_codex_home( + /*is_macos*/ false, + /*current_exe*/ Some(&exe_path), + /*managed_by_npm*/ false, + /*managed_by_bun*/ false, + /*codex_home*/ Some(codex_home.path()), + ); + assert_eq!(context.rg_command(), default_rg_command()); + Ok(()) + } + + #[test] + fn npm_and_bun_take_precedence() { + let npm_context = InstallContext::from_exe_with_codex_home( + /*is_macos*/ false, + /*current_exe*/ Some(Path::new("/tmp/codex")), + /*managed_by_npm*/ true, + /*managed_by_bun*/ false, + /*codex_home*/ None, + ); + assert_eq!(npm_context, InstallContext::Npm); + + let bun_context = InstallContext::from_exe_with_codex_home( + /*is_macos*/ false, + /*current_exe*/ Some(Path::new("/tmp/codex")), + /*managed_by_npm*/ false, + /*managed_by_bun*/ true, + /*codex_home*/ None, + ); + assert_eq!(bun_context, InstallContext::Bun); + } + + #[test] + fn brew_is_detected_on_macos_prefixes() { + let context = InstallContext::from_exe_with_codex_home( + /*is_macos*/ true, + /*current_exe*/ Some(Path::new("/opt/homebrew/bin/codex")), + /*managed_by_npm*/ false, + /*managed_by_bun*/ false, + /*codex_home*/ None, + ); + assert_eq!(context, InstallContext::Brew); + } +} diff --git a/codex-rs/login/src/api_bridge.rs b/codex-rs/login/src/api_bridge.rs index d8b9dbb77c..684c890bde 100644 --- a/codex-rs/login/src/api_bridge.rs +++ b/codex-rs/login/src/api_bridge.rs @@ -11,6 +11,7 @@ pub fn auth_provider_from_auth( return Ok(CoreAuthProvider { token: Some(api_key), account_id: None, + is_fedramp_account: false, }); } @@ -18,6 +19,7 @@ pub fn auth_provider_from_auth( return Ok(CoreAuthProvider { token: Some(token), account_id: None, + is_fedramp_account: false, }); } @@ -26,11 +28,13 @@ pub fn auth_provider_from_auth( Ok(CoreAuthProvider { token: Some(token), account_id: auth.get_account_id(), + is_fedramp_account: auth.is_fedramp_account(), }) } else { Ok(CoreAuthProvider { token: None, account_id: None, + is_fedramp_account: false, }) } } diff --git a/codex-rs/login/src/auth/auth_tests.rs b/codex-rs/login/src/auth/auth_tests.rs index 27064b6831..30244ac268 100644 --- a/codex-rs/login/src/auth/auth_tests.rs +++ b/codex-rs/login/src/auth/auth_tests.rs @@ -16,6 +16,8 @@ use serde_json::json; use std::sync::Arc; use tempfile::TempDir; use tempfile::tempdir; +use tokio::time::Duration; +use tokio::time::timeout; #[tokio::test] async fn refresh_without_id_token() { @@ -128,6 +130,7 @@ async fn pro_account_with_no_api_key_uses_chatgpt_auth() { chatgpt_plan_type: Some(InternalPlanType::Known(InternalKnownPlan::Pro)), chatgpt_user_id: Some("user-12345".to_string()), chatgpt_account_id: None, + chatgpt_account_is_fedramp: false, raw_jwt: fake_jwt, }, access_token: "test-access-token".to_string(), @@ -135,6 +138,7 @@ async fn pro_account_with_no_api_key_uses_chatgpt_auth() { account_id: None, }), last_refresh: Some(last_refresh), + agent_identity: None, }, auth_dot_json ); @@ -172,6 +176,7 @@ fn logout_removes_auth_file() -> Result<(), std::io::Error> { openai_api_key: Some("sk-test-key".to_string()), tokens: None, last_refresh: None, + agent_identity: None, }; super::save_auth(dir.path(), &auth_dot_json, AuthCredentialsStoreMode::File)?; let auth_file = get_auth_file(dir.path()); @@ -181,6 +186,49 @@ fn logout_removes_auth_file() -> Result<(), std::io::Error> { Ok(()) } +#[test] +fn chatgpt_auth_persists_agent_identity_for_workspace() { + let codex_home = tempdir().unwrap(); + write_auth_file( + AuthFileParams { + openai_api_key: None, + chatgpt_plan_type: Some("pro".to_string()), + chatgpt_account_id: Some("account-123".to_string()), + }, + codex_home.path(), + ) + .expect("failed to write auth file"); + let auth = super::load_auth( + codex_home.path(), + /*enable_codex_api_key_env*/ false, + AuthCredentialsStoreMode::File, + ) + .expect("load auth") + .expect("auth available"); + let record = AgentIdentityAuthRecord { + workspace_id: "account-123".to_string(), + chatgpt_user_id: Some("user-123".to_string()), + agent_runtime_id: "agent_123".to_string(), + agent_private_key: "pkcs8-base64".to_string(), + registered_at: "2026-04-13T12:00:00Z".to_string(), + }; + + auth.set_agent_identity(record.clone()) + .expect("set agent identity"); + + assert_eq!(auth.get_agent_identity("account-123"), Some(record.clone())); + assert_eq!(auth.get_agent_identity("other-account"), None); + let storage = FileAuthStorage::new(codex_home.path().to_path_buf()); + let persisted = storage + .load() + .expect("load auth") + .expect("auth should exist"); + assert_eq!(persisted.agent_identity, Some(record)); + + assert!(auth.remove_agent_identity().expect("remove agent identity")); + assert_eq!(auth.get_agent_identity("account-123"), None); +} + #[test] fn unauthorized_recovery_reports_mode_and_step_names() { let dir = tempdir().unwrap(); @@ -474,6 +522,67 @@ exit 1 } } +#[tokio::test] +async fn auth_manager_notifies_when_auth_state_changes() { + let dir = tempdir().unwrap(); + let manager = AuthManager::shared( + dir.path().to_path_buf(), + /*enable_codex_api_key_env*/ false, + AuthCredentialsStoreMode::File, + ); + let mut auth_state_rx = manager.subscribe_auth_state(); + + save_auth( + dir.path(), + &AuthDotJson { + auth_mode: Some(ApiAuthMode::ApiKey), + openai_api_key: Some("sk-test-key".to_string()), + tokens: None, + last_refresh: None, + agent_identity: None, + }, + AuthCredentialsStoreMode::File, + ) + .expect("save auth"); + + assert!( + manager.reload(), + "reload should report a changed auth state" + ); + timeout(Duration::from_secs(1), auth_state_rx.changed()) + .await + .expect("auth change notification should arrive") + .expect("auth state watch should remain open"); + + save_auth( + dir.path(), + &AuthDotJson { + auth_mode: Some(ApiAuthMode::ApiKey), + openai_api_key: Some("sk-updated-key".to_string()), + tokens: None, + last_refresh: None, + agent_identity: None, + }, + AuthCredentialsStoreMode::File, + ) + .expect("save updated auth"); + + assert!( + !manager.reload(), + "reload remains mode-stable even when the underlying credentials change" + ); + timeout(Duration::from_secs(1), auth_state_rx.changed()) + .await + .expect("auth reload notification should still arrive") + .expect("auth state watch should remain open"); + + manager.set_forced_chatgpt_workspace_id(Some("workspace-123".to_string())); + timeout(Duration::from_secs(1), auth_state_rx.changed()) + .await + .expect("workspace change notification should arrive") + .expect("auth state watch should remain open"); +} + struct AuthFileParams { openai_api_key: Option, chatgpt_plan_type: Option, diff --git a/codex-rs/login/src/auth/manager.rs b/codex-rs/login/src/auth/manager.rs index 15b129a169..3cb97f58b0 100644 --- a/codex-rs/login/src/auth/manager.rs +++ b/codex-rs/login/src/auth/manager.rs @@ -13,6 +13,7 @@ use std::sync::Arc; use std::sync::Mutex; use std::sync::RwLock; use tokio::sync::Mutex as AsyncMutex; +use tokio::sync::watch; use codex_app_server_protocol::AuthMode; use codex_app_server_protocol::AuthMode as ApiAuthMode; @@ -20,6 +21,7 @@ use codex_protocol::config_types::ForcedLoginMethod; use codex_protocol::config_types::ModelProviderAuthInfo; use super::external_bearer::BearerTokenRefresher; +pub use crate::auth::storage::AgentIdentityAuthRecord; pub use crate::auth::storage::AuthDotJson; use crate::auth::storage::AuthStorageBackend; use crate::auth::storage::create_auth_storage; @@ -60,6 +62,7 @@ pub struct ChatgptAuth { #[derive(Debug, Clone)] pub struct ChatgptAuthTokens { state: ChatgptAuthState, + storage: Arc, } #[derive(Debug, Clone)] @@ -204,14 +207,13 @@ impl CodexAuth { client, }; + let storage = create_auth_storage(codex_home.to_path_buf(), storage_mode); match auth_mode { - ApiAuthMode::Chatgpt => { - let storage = create_auth_storage(codex_home.to_path_buf(), storage_mode); - Ok(Self::Chatgpt(ChatgptAuth { state, storage })) - } - ApiAuthMode::ChatgptAuthTokens => { - Ok(Self::ChatgptAuthTokens(ChatgptAuthTokens { state })) - } + ApiAuthMode::Chatgpt => Ok(Self::Chatgpt(ChatgptAuth { state, storage })), + ApiAuthMode::ChatgptAuthTokens => Ok(Self::ChatgptAuthTokens(ChatgptAuthTokens { + state, + storage, + })), ApiAuthMode::ApiKey => unreachable!("api key mode is handled above"), } } @@ -291,6 +293,12 @@ impl CodexAuth { self.get_current_token_data().and_then(|t| t.account_id) } + /// Returns false if `is_chatgpt_auth()` is false or the token omits the FedRAMP claim. + pub fn is_fedramp_account(&self) -> bool { + self.get_current_token_data() + .is_some_and(|t| t.id_token.is_fedramp_account()) + } + /// Returns `None` if `is_chatgpt_auth()` is false. pub fn get_account_email(&self) -> Option { self.get_current_token_data().and_then(|t| t.id_token.email) @@ -352,6 +360,52 @@ impl CodexAuth { self.get_current_auth_json().and_then(|t| t.tokens) } + pub fn get_agent_identity(&self, workspace_id: &str) -> Option { + self.get_current_auth_json() + .and_then(|auth| auth.agent_identity) + .filter(|identity| identity.workspace_id == workspace_id) + } + + pub fn set_agent_identity(&self, record: AgentIdentityAuthRecord) -> std::io::Result<()> { + let (state, storage) = match self { + Self::Chatgpt(auth) => (&auth.state, &auth.storage), + Self::ChatgptAuthTokens(auth) => (&auth.state, &auth.storage), + Self::ApiKey(_) => return Ok(()), + }; + let mut guard = state + .auth_dot_json + .lock() + .map_err(|_| std::io::Error::other("failed to lock auth state"))?; + let mut auth = guard + .clone() + .ok_or_else(|| std::io::Error::other("auth data is not available"))?; + auth.agent_identity = Some(record); + storage.save(&auth)?; + *guard = Some(auth); + Ok(()) + } + + pub fn remove_agent_identity(&self) -> std::io::Result { + let (state, storage) = match self { + Self::Chatgpt(auth) => (&auth.state, &auth.storage), + Self::ChatgptAuthTokens(auth) => (&auth.state, &auth.storage), + Self::ApiKey(_) => return Ok(false), + }; + let mut guard = state + .auth_dot_json + .lock() + .map_err(|_| std::io::Error::other("failed to lock auth state"))?; + let Some(mut auth) = guard.clone() else { + return Ok(false); + }; + let removed = auth.agent_identity.take().is_some(); + if removed { + storage.save(&auth)?; + *guard = Some(auth); + } + Ok(removed) + } + /// Consider this private to integration tests. pub fn create_dummy_chatgpt_auth_for_testing() -> Self { let auth_dot_json = AuthDotJson { @@ -364,6 +418,7 @@ impl CodexAuth { account_id: Some("account_id".to_string()), }), last_refresh: Some(Utc::now()), + agent_identity: None, }; let client = create_client(); @@ -439,6 +494,7 @@ pub fn login_with_api_key( openai_api_key: Some(api_key.to_string()), tokens: None, last_refresh: None, + agent_identity: None, }; save_auth(codex_home, &auth_dot_json, auth_credentials_store_mode) } @@ -812,6 +868,7 @@ impl AuthDotJson { openai_api_key: None, tokens: Some(tokens), last_refresh: Some(Utc::now()), + agent_identity: None, }) } @@ -1107,6 +1164,7 @@ pub struct AuthManager { forced_chatgpt_workspace_id: RwLock>, refresh_lock: AsyncMutex<()>, external_auth: RwLock>>, + auth_state_tx: watch::Sender<()>, } /// Configuration view required to construct a shared [`AuthManager`]. @@ -1155,6 +1213,7 @@ impl AuthManager { enable_codex_api_key_env: bool, auth_credentials_store_mode: AuthCredentialsStoreMode, ) -> Self { + let (auth_state_tx, _) = watch::channel(()); let managed_auth = load_auth( &codex_home, enable_codex_api_key_env, @@ -1173,11 +1232,13 @@ impl AuthManager { forced_chatgpt_workspace_id: RwLock::new(None), refresh_lock: AsyncMutex::new(()), external_auth: RwLock::new(None), + auth_state_tx, } } /// Create an AuthManager with a specific CodexAuth, for testing only. pub fn from_auth_for_testing(auth: CodexAuth) -> Arc { + let (auth_state_tx, _) = watch::channel(()); let cached = CachedAuth { auth: Some(auth), permanent_refresh_failure: None, @@ -1191,11 +1252,13 @@ impl AuthManager { forced_chatgpt_workspace_id: RwLock::new(None), refresh_lock: AsyncMutex::new(()), external_auth: RwLock::new(None), + auth_state_tx, }) } /// Create an AuthManager with a specific CodexAuth and codex home, for testing only. pub fn from_auth_for_testing_with_home(auth: CodexAuth, codex_home: PathBuf) -> Arc { + let (auth_state_tx, _) = watch::channel(()); let cached = CachedAuth { auth: Some(auth), permanent_refresh_failure: None, @@ -1208,10 +1271,12 @@ impl AuthManager { forced_chatgpt_workspace_id: RwLock::new(None), refresh_lock: AsyncMutex::new(()), external_auth: RwLock::new(None), + auth_state_tx, }) } pub fn external_bearer_only(config: ModelProviderAuthInfo) -> Arc { + let (auth_state_tx, _) = watch::channel(()); Arc::new(Self { codex_home: PathBuf::from("non-existent"), inner: RwLock::new(CachedAuth { @@ -1225,6 +1290,7 @@ impl AuthManager { external_auth: RwLock::new(Some( Arc::new(BearerTokenRefresher::new(config)) as Arc )), + auth_state_tx, }) } @@ -1364,6 +1430,7 @@ impl AuthManager { } tracing::info!("Reloaded auth, changed: {changed}"); guard.auth = new_auth; + self.auth_state_tx.send_replace(()); changed } else { false @@ -1373,18 +1440,23 @@ impl AuthManager { pub fn set_external_auth(&self, external_auth: Arc) { if let Ok(mut guard) = self.external_auth.write() { *guard = Some(external_auth); + self.auth_state_tx.send_replace(()); } } pub fn clear_external_auth(&self) { if let Ok(mut guard) = self.external_auth.write() { *guard = None; + self.auth_state_tx.send_replace(()); } } pub fn set_forced_chatgpt_workspace_id(&self, workspace_id: Option) { - if let Ok(mut guard) = self.forced_chatgpt_workspace_id.write() { + if let Ok(mut guard) = self.forced_chatgpt_workspace_id.write() + && *guard != workspace_id + { *guard = workspace_id; + self.auth_state_tx.send_replace(()); } } @@ -1395,6 +1467,10 @@ impl AuthManager { .and_then(|guard| guard.clone()) } + pub fn subscribe_auth_state(&self) -> watch::Receiver<()> { + self.auth_state_tx.subscribe() + } + pub fn has_external_auth(&self) -> bool { self.external_auth().is_some() } @@ -1638,8 +1714,14 @@ impl AuthManager { ), ))); } - let auth_dot_json = + let mut auth_dot_json = AuthDotJson::from_external_tokens(&refreshed).map_err(RefreshTokenError::Transient)?; + if let Some(previous_auth) = self + .auth_cached() + .and_then(|auth| auth.get_current_auth_json()) + { + auth_dot_json.agent_identity = previous_auth.agent_identity; + } save_auth( &self.codex_home, &auth_dot_json, diff --git a/codex-rs/login/src/auth/storage.rs b/codex-rs/login/src/auth/storage.rs index 97e801415c..f0d7a3169c 100644 --- a/codex-rs/login/src/auth/storage.rs +++ b/codex-rs/login/src/auth/storage.rs @@ -39,6 +39,19 @@ pub struct AuthDotJson { #[serde(default, skip_serializing_if = "Option::is_none")] pub last_refresh: Option>, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub agent_identity: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, PartialEq, Eq)] +pub struct AgentIdentityAuthRecord { + pub workspace_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub chatgpt_user_id: Option, + pub agent_runtime_id: String, + pub agent_private_key: String, + pub registered_at: String, } pub(super) fn get_auth_file(codex_home: &Path) -> PathBuf { diff --git a/codex-rs/login/src/auth/storage_tests.rs b/codex-rs/login/src/auth/storage_tests.rs index 4bf72c11b9..2e1cc85029 100644 --- a/codex-rs/login/src/auth/storage_tests.rs +++ b/codex-rs/login/src/auth/storage_tests.rs @@ -18,6 +18,7 @@ async fn file_storage_load_returns_auth_dot_json() -> anyhow::Result<()> { openai_api_key: Some("test-key".to_string()), tokens: None, last_refresh: Some(Utc::now()), + agent_identity: None, }; storage @@ -38,6 +39,7 @@ async fn file_storage_save_persists_auth_dot_json() -> anyhow::Result<()> { openai_api_key: Some("test-key".to_string()), tokens: None, last_refresh: Some(Utc::now()), + agent_identity: None, }; let file = get_auth_file(codex_home.path()); @@ -52,6 +54,30 @@ async fn file_storage_save_persists_auth_dot_json() -> anyhow::Result<()> { Ok(()) } +#[tokio::test] +async fn file_storage_persists_agent_identity() -> anyhow::Result<()> { + let codex_home = tempdir()?; + let storage = FileAuthStorage::new(codex_home.path().to_path_buf()); + let auth_dot_json = AuthDotJson { + auth_mode: Some(AuthMode::Chatgpt), + openai_api_key: None, + tokens: None, + last_refresh: Some(Utc::now()), + agent_identity: Some(AgentIdentityAuthRecord { + workspace_id: "account-123".to_string(), + chatgpt_user_id: Some("user-123".to_string()), + agent_runtime_id: "agent_123".to_string(), + agent_private_key: "pkcs8-base64".to_string(), + registered_at: "2026-04-13T12:00:00Z".to_string(), + }), + }; + + storage.save(&auth_dot_json)?; + + assert_eq!(storage.load()?, Some(auth_dot_json)); + Ok(()) +} + #[test] fn file_storage_delete_removes_auth_file() -> anyhow::Result<()> { let dir = tempdir()?; @@ -60,6 +86,7 @@ fn file_storage_delete_removes_auth_file() -> anyhow::Result<()> { openai_api_key: Some("sk-test-key".to_string()), tokens: None, last_refresh: None, + agent_identity: None, }; let storage = create_auth_storage(dir.path().to_path_buf(), AuthCredentialsStoreMode::File); storage.save(&auth_dot_json)?; @@ -83,6 +110,7 @@ fn ephemeral_storage_save_load_delete_is_in_memory_only() -> anyhow::Result<()> openai_api_key: Some("sk-ephemeral".to_string()), tokens: None, last_refresh: Some(Utc::now()), + agent_identity: None, }; storage.save(&auth_dot_json)?; @@ -181,6 +209,7 @@ fn auth_with_prefix(prefix: &str) -> AuthDotJson { account_id: Some(format!("{prefix}-account-id")), }), last_refresh: None, + agent_identity: None, } } @@ -197,6 +226,7 @@ fn keyring_auth_storage_load_returns_deserialized_auth() -> anyhow::Result<()> { openai_api_key: Some("sk-test".to_string()), tokens: None, last_refresh: None, + agent_identity: None, }; seed_keyring_with_auth( &mock_keyring, @@ -239,6 +269,7 @@ fn keyring_auth_storage_save_persists_and_removes_fallback_file() -> anyhow::Res account_id: Some("account".to_string()), }), last_refresh: Some(Utc::now()), + agent_identity: None, }; storage.save(&auth)?; diff --git a/codex-rs/login/src/lib.rs b/codex-rs/login/src/lib.rs index 1b52223bc3..4880e43106 100644 --- a/codex-rs/login/src/lib.rs +++ b/codex-rs/login/src/lib.rs @@ -20,6 +20,7 @@ pub use server::ShutdownHandle; pub use server::run_login_server; pub use api_bridge::auth_provider_from_auth; +pub use auth::AgentIdentityAuthRecord; pub use auth::AuthConfig; pub use auth::AuthDotJson; pub use auth::AuthManager; diff --git a/codex-rs/login/src/server.rs b/codex-rs/login/src/server.rs index 169a8a3091..0c7b810184 100644 --- a/codex-rs/login/src/server.rs +++ b/codex-rs/login/src/server.rs @@ -781,6 +781,7 @@ pub(crate) async fn persist_tokens_async( openai_api_key: api_key, tokens: Some(tokens), last_refresh: Some(Utc::now()), + agent_identity: None, }; save_auth(&codex_home, &auth, auth_credentials_store_mode) }) diff --git a/codex-rs/login/src/token_data.rs b/codex-rs/login/src/token_data.rs index f70696d5a2..2952ecb2a7 100644 --- a/codex-rs/login/src/token_data.rs +++ b/codex-rs/login/src/token_data.rs @@ -36,6 +36,8 @@ pub struct IdTokenInfo { pub chatgpt_user_id: Option, /// Organization/workspace identifier associated with the token, if present. pub chatgpt_account_id: Option, + /// Whether the selected ChatGPT workspace must route through the FedRAMP edge. + pub chatgpt_account_is_fedramp: bool, pub raw_jwt: String, } @@ -60,6 +62,10 @@ impl IdTokenInfo { Some(PlanType::Known(plan)) if plan.is_workspace_account() ) } + + pub fn is_fedramp_account(&self) -> bool { + self.chatgpt_account_is_fedramp + } } #[derive(Deserialize)] @@ -88,6 +94,8 @@ struct AuthClaims { user_id: Option, #[serde(default)] chatgpt_account_id: Option, + #[serde(default)] + chatgpt_account_is_fedramp: bool, } #[derive(Deserialize)] @@ -139,6 +147,7 @@ pub fn parse_chatgpt_jwt_claims(jwt: &str) -> Result Ok(IdTokenInfo { email, @@ -146,6 +155,7 @@ pub fn parse_chatgpt_jwt_claims(jwt: &str) -> Result Result<()> { openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -117,6 +118,7 @@ async fn refresh_token_refreshes_when_auth_is_unchanged() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -171,6 +173,7 @@ async fn refresh_token_skips_refresh_when_auth_changed() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -180,6 +183,7 @@ async fn refresh_token_skips_refresh_when_auth_changed() -> Result<()> { openai_api_key: None, tokens: Some(disk_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; save_auth( ctx.codex_home.path(), @@ -234,6 +238,7 @@ async fn refresh_token_errors_on_account_mismatch() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -244,6 +249,7 @@ async fn refresh_token_errors_on_account_mismatch() -> Result<()> { openai_api_key: None, tokens: Some(disk_tokens), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; save_auth( ctx.codex_home.path(), @@ -302,6 +308,7 @@ async fn returns_fresh_tokens_as_is() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(stale_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -349,6 +356,7 @@ async fn refreshes_token_when_access_token_is_expired() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(fresh_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -398,6 +406,7 @@ async fn auth_reloads_disk_auth_when_cached_auth_is_stale() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens), last_refresh: Some(stale_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -408,6 +417,7 @@ async fn auth_reloads_disk_auth_when_cached_auth_is_stale() -> Result<()> { openai_api_key: None, tokens: Some(disk_tokens.clone()), last_refresh: Some(fresh_refresh), + agent_identity: None, }; save_auth( ctx.codex_home.path(), @@ -459,6 +469,7 @@ async fn auth_reloads_disk_auth_without_calling_expired_refresh_token() -> Resul openai_api_key: None, tokens: Some(initial_tokens), last_refresh: Some(stale_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -469,6 +480,7 @@ async fn auth_reloads_disk_auth_without_calling_expired_refresh_token() -> Resul openai_api_key: None, tokens: Some(disk_tokens.clone()), last_refresh: Some(fresh_refresh), + agent_identity: None, }; save_auth( ctx.codex_home.path(), @@ -518,6 +530,7 @@ async fn refresh_token_returns_permanent_error_for_expired_refresh_token() -> Re openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -570,6 +583,7 @@ async fn refresh_token_does_not_retry_after_permanent_failure() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -636,6 +650,7 @@ async fn refresh_token_reloads_changed_auth_after_permanent_failure() -> Result< openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -657,6 +672,7 @@ async fn refresh_token_reloads_changed_auth_after_permanent_failure() -> Result< openai_api_key: None, tokens: Some(disk_tokens.clone()), last_refresh: Some(fresh_refresh), + agent_identity: None, }; save_auth( ctx.codex_home.path(), @@ -715,6 +731,7 @@ async fn refresh_token_returns_transient_error_on_server_failure() -> Result<()> openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -767,6 +784,7 @@ async fn unauthorized_recovery_reloads_then_refreshes_tokens() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -776,6 +794,7 @@ async fn unauthorized_recovery_reloads_then_refreshes_tokens() -> Result<()> { openai_api_key: None, tokens: Some(disk_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; save_auth( ctx.codex_home.path(), @@ -859,6 +878,7 @@ async fn unauthorized_recovery_errors_on_account_mismatch() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -869,6 +889,7 @@ async fn unauthorized_recovery_errors_on_account_mismatch() -> Result<()> { openai_api_key: None, tokens: Some(disk_tokens), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; save_auth( ctx.codex_home.path(), @@ -926,6 +947,7 @@ async fn unauthorized_recovery_requires_chatgpt_auth() -> Result<()> { openai_api_key: Some("sk-test".to_string()), tokens: None, last_refresh: None, + agent_identity: None, }; ctx.write_auth(&auth)?; diff --git a/codex-rs/mcp-server/tests/suite/codex_tool.rs b/codex-rs/mcp-server/tests/suite/codex_tool.rs index f8896864bc..323d07264d 100644 --- a/codex-rs/mcp-server/tests/suite/codex_tool.rs +++ b/codex-rs/mcp-server/tests/suite/codex_tool.rs @@ -29,8 +29,9 @@ use mcp_test_support::create_mock_responses_server; use mcp_test_support::create_shell_command_sse_response; use mcp_test_support::format_with_current_shell; -// Allow ample time on slower CI or under load to avoid flakes. -const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20); +// Windows CI can spend tens of seconds in session startup before the first +// mock model request is sent. +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); /// Test that a shell command that is not on the "trusted" list triggers an /// elicitation request to the MCP and that sending the approval runs the diff --git a/codex-rs/model-provider-info/src/lib.rs b/codex-rs/model-provider-info/src/lib.rs index f39ff0c7fe..80fb705605 100644 --- a/codex-rs/model-provider-info/src/lib.rs +++ b/codex-rs/model-provider-info/src/lib.rs @@ -7,6 +7,7 @@ use codex_api::Provider as ApiProvider; use codex_api::RetryConfig as ApiRetryConfig; +use codex_api::is_azure_responses_provider; use codex_app_server_protocol::AuthMode; use codex_protocol::config_types::ModelProviderAuthInfo; use codex_protocol::error::CodexErr; @@ -300,6 +301,10 @@ impl ModelProviderInfo { self.name == OPENAI_PROVIDER_NAME } + pub fn supports_remote_compaction(&self) -> bool { + self.is_openai() || is_azure_responses_provider(&self.name, self.base_url.as_deref()) + } + pub fn has_command_auth(&self) -> bool { self.auth.is_some() } diff --git a/codex-rs/model-provider-info/src/model_provider_info_tests.rs b/codex-rs/model-provider-info/src/model_provider_info_tests.rs index bdc2e73b1a..6cd1ef2366 100644 --- a/codex-rs/model-provider-info/src/model_provider_info_tests.rs +++ b/codex-rs/model-provider-info/src/model_provider_info_tests.rs @@ -129,6 +129,61 @@ supports_websockets = true assert_eq!(provider.websocket_connect_timeout_ms, Some(15_000)); } +#[test] +fn test_supports_remote_compaction_for_openai() { + let provider = ModelProviderInfo::create_openai_provider(/*base_url*/ None); + + assert!(provider.supports_remote_compaction()); +} + +#[test] +fn test_supports_remote_compaction_for_azure_name() { + let provider = ModelProviderInfo { + name: "Azure".into(), + base_url: Some("https://example.com/openai".into()), + env_key: Some("AZURE_OPENAI_API_KEY".into()), + env_key_instructions: None, + experimental_bearer_token: None, + auth: None, + wire_api: WireApi::Responses, + query_params: None, + http_headers: None, + env_http_headers: None, + request_max_retries: None, + stream_max_retries: None, + stream_idle_timeout_ms: None, + websocket_connect_timeout_ms: None, + requires_openai_auth: false, + supports_websockets: false, + }; + + assert!(provider.supports_remote_compaction()); +} + +#[test] +fn test_supports_remote_compaction_for_non_openai_non_azure_provider() { + let provider = ModelProviderInfo { + name: "Example".into(), + base_url: Some("https://example.com/v1".into()), + env_key: Some("API_KEY".into()), + env_key_instructions: None, + experimental_bearer_token: None, + auth: None, + wire_api: WireApi::Responses, + query_params: None, + http_headers: None, + env_http_headers: None, + request_max_retries: None, + stream_max_retries: None, + stream_idle_timeout_ms: None, + websocket_connect_timeout_ms: None, + requires_openai_auth: false, + supports_websockets: false, + }; + + assert!(!provider.supports_remote_compaction()); +} + #[test] fn test_deserialize_provider_auth_config_defaults() { let base_dir = tempdir().unwrap(); diff --git a/codex-rs/protocol/Cargo.toml b/codex-rs/protocol/Cargo.toml index c4249a13b4..f70dc94585 100644 --- a/codex-rs/protocol/Cargo.toml +++ b/codex-rs/protocol/Cargo.toml @@ -23,6 +23,7 @@ codex-utils-image = { workspace = true } codex-utils-string = { workspace = true } codex-utils-template = { workspace = true } encoding_rs = { workspace = true } +globset = { workspace = true } icu_decimal = { workspace = true } icu_locale_core = { workspace = true } icu_provider = { workspace = true, features = ["sync"] } diff --git a/codex-rs/protocol/src/lib.rs b/codex-rs/protocol/src/lib.rs index de580de6f3..2506dae747 100644 --- a/codex-rs/protocol/src/lib.rs +++ b/codex-rs/protocol/src/lib.rs @@ -2,8 +2,10 @@ pub mod account; mod agent_path; pub mod auth; mod thread_id; +mod tool_name; pub use agent_path::AgentPath; pub use thread_id::ThreadId; +pub use tool_name::ToolName; pub mod approvals; pub mod config_types; pub mod dynamic_tools; diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs index 0f18b68393..2200456527 100644 --- a/codex-rs/protocol/src/models.rs +++ b/codex-rs/protocol/src/models.rs @@ -1362,6 +1362,8 @@ impl CallToolResult { fn convert_mcp_content_to_items( contents: &[serde_json::Value], ) -> Option> { + const CODEX_IMAGE_DETAIL_META_KEY: &str = "codex/imageDetail"; + #[derive(serde::Deserialize)] #[serde(tag = "type")] enum McpContent { @@ -1372,6 +1374,8 @@ fn convert_mcp_content_to_items( data: String, #[serde(rename = "mimeType", alias = "mime_type")] mime_type: Option, + #[serde(rename = "_meta", default)] + meta: Option, }, #[serde(other)] Unknown, @@ -1383,7 +1387,11 @@ fn convert_mcp_content_to_items( for content in contents { let item = match serde_json::from_value::(content.clone()) { Ok(McpContent::Text { text }) => FunctionCallOutputContentItem::InputText { text }, - Ok(McpContent::Image { data, mime_type }) => { + Ok(McpContent::Image { + data, + mime_type, + meta, + }) => { saw_image = true; let image_url = if data.starts_with("data:") { data @@ -1393,7 +1401,15 @@ fn convert_mcp_content_to_items( }; FunctionCallOutputContentItem::InputImage { image_url, - detail: None, + detail: meta + .as_ref() + .and_then(serde_json::Value::as_object) + .and_then(|meta| meta.get(CODEX_IMAGE_DETAIL_META_KEY)) + .and_then(serde_json::Value::as_str) + .and_then(|detail| match detail { + "original" => Some(ImageDetail::Original), + _ => None, + }), } } Ok(McpContent::Unknown) | Err(_) => FunctionCallOutputContentItem::InputText { @@ -2264,6 +2280,70 @@ mod tests { Ok(()) } + #[test] + fn preserves_original_detail_metadata_on_mcp_images() -> Result<()> { + let call_tool_result = CallToolResult { + content: vec![serde_json::json!({ + "type": "image", + "data": "BASE64", + "mimeType": "image/png", + "_meta": { + "codex/imageDetail": "original", + }, + })], + structured_content: None, + is_error: Some(false), + meta: None, + }; + + let payload = call_tool_result.into_function_call_output_payload(); + let Some(items) = payload.content_items() else { + panic!("expected content items"); + }; + let items = items.to_vec(); + assert_eq!( + items, + vec![FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,BASE64".into(), + detail: Some(ImageDetail::Original), + }] + ); + + Ok(()) + } + + #[test] + fn ignores_unknown_mcp_image_detail_metadata() -> Result<()> { + let call_tool_result = CallToolResult { + content: vec![serde_json::json!({ + "type": "image", + "data": "BASE64", + "mimeType": "image/png", + "_meta": { + "codex/imageDetail": "high", + }, + })], + structured_content: None, + is_error: Some(false), + meta: None, + }; + + let payload = call_tool_result.into_function_call_output_payload(); + let Some(items) = payload.content_items() else { + panic!("expected content items"); + }; + let items = items.to_vec(); + assert_eq!( + items, + vec![FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,BASE64".into(), + detail: None, + }] + ); + + Ok(()) + } + #[test] fn deserializes_array_payload_into_items() -> Result<()> { let json = r#"[ diff --git a/codex-rs/protocol/src/permissions.rs b/codex-rs/protocol/src/permissions.rs index f101f8bef0..d774f6dc0d 100644 --- a/codex-rs/protocol/src/permissions.rs +++ b/codex-rs/protocol/src/permissions.rs @@ -6,6 +6,8 @@ use std::path::PathBuf; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_absolute_path::canonicalize_preserving_symlinks; +use globset::GlobBuilder; +use globset::GlobMatcher; use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; @@ -154,14 +156,101 @@ struct FileSystemSemanticSignature { readable_roots: Vec, writable_roots: Vec, unreadable_roots: Vec, + unreadable_globs: Vec, +} + +/// Runtime matcher for read-deny entries in a filesystem sandbox policy. +pub struct ReadDenyMatcher { + denied_candidates: Vec>, + deny_read_matchers: Vec, + invalid_pattern: bool, +} + +impl ReadDenyMatcher { + /// Builds a matcher from exact deny-read roots and deny-read glob entries. + /// + /// Returns `None` when the policy has no deny-read restrictions, so callers + /// can skip read-deny checks without allocating matcher state. The `cwd` + /// resolves cwd-relative policy paths and special paths before matching. + pub fn new(file_system_sandbox_policy: &FileSystemSandboxPolicy, cwd: &Path) -> Option { + if !file_system_sandbox_policy.has_denied_read_restrictions() { + return None; + } + + // Exact roots are stored as all meaningful path spellings we can derive + // cheaply. This lets direct tool checks catch both a symlink path and + // its canonical target without changing the policy entries themselves. + let denied_candidates = file_system_sandbox_policy + .get_unreadable_roots_with_cwd(cwd) + .into_iter() + .map(|path| normalized_and_canonical_candidates(path.as_path())) + .collect(); + // Pattern entries stay as policy-level globs. They are matched at read + // time here instead of being snapshotted to startup filesystem state. + let mut invalid_pattern = false; + let deny_read_matchers = file_system_sandbox_policy + .get_unreadable_globs_with_cwd(cwd) + .into_iter() + .filter_map(|pattern| match build_glob_matcher(&pattern) { + Some(matcher) => Some(matcher), + None => { + invalid_pattern = true; + None + } + }) + .collect(); + Some(Self { + denied_candidates, + deny_read_matchers, + invalid_pattern, + }) + } + + /// Returns whether `path` is denied by the policy used to build this matcher. + pub fn is_read_denied(&self, path: &Path) -> bool { + if self.invalid_pattern { + // Direct tool reads fail closed on malformed deny patterns. Silent + // allow would turn a config typo into a policy bypass. + return true; + } + + // Check exact roots against each candidate spelling before evaluating + // glob matchers. Exact entries are subtree denies; glob entries match + // according to the pattern compiler's path-separator rules. + let path_candidates = normalized_and_canonical_candidates(path); + if self.denied_candidates.iter().any(|denied_candidates| { + path_candidates.iter().any(|candidate| { + denied_candidates.iter().any(|denied_candidate| { + candidate == denied_candidate || candidate.starts_with(denied_candidate) + }) + }) + }) { + return true; + } + + self.deny_read_matchers.iter().any(|matcher| { + path_candidates + .iter() + .any(|candidate| matcher.is_match(candidate)) + }) + } } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema, TS)] #[serde(tag = "type", rename_all = "snake_case")] #[ts(tag = "type")] pub enum FileSystemPath { - Path { path: AbsolutePathBuf }, - Special { value: FileSystemSpecialPath }, + Path { + path: AbsolutePathBuf, + }, + /// A git-style glob pattern. Pattern entries currently support + /// FileSystemAccessMode::None only. + GlobPattern { + pattern: String, + }, + Special { + value: FileSystemSpecialPath, + }, } impl Default for FileSystemSandboxPolicy { @@ -179,62 +268,6 @@ impl Default for FileSystemSandboxPolicy { } impl FileSystemSandboxPolicy { - fn has_root_access(&self, predicate: impl Fn(FileSystemAccessMode) -> bool) -> bool { - matches!(self.kind, FileSystemSandboxKind::Restricted) - && self.entries.iter().any(|entry| { - matches!( - &entry.path, - FileSystemPath::Special { value } - if matches!(value, FileSystemSpecialPath::Root) && predicate(entry.access) - ) - }) - } - - fn has_explicit_deny_entries(&self) -> bool { - matches!(self.kind, FileSystemSandboxKind::Restricted) - && self - .entries - .iter() - .any(|entry| entry.access == FileSystemAccessMode::None) - } - - /// Returns true when a restricted policy contains any entry that really - /// reduces a broader `:root = write` grant. - /// - /// Raw entry presence is not enough here: an equally specific `write` - /// entry for the same target wins under the normal precedence rules, so a - /// shadowed `read` entry must not downgrade the policy out of full-disk - /// write mode. - fn has_write_narrowing_entries(&self) -> bool { - matches!(self.kind, FileSystemSandboxKind::Restricted) - && self.entries.iter().any(|entry| { - if entry.access.can_write() { - return false; - } - - match &entry.path { - FileSystemPath::Path { .. } => !self.has_same_target_write_override(entry), - FileSystemPath::Special { value } => match value { - FileSystemSpecialPath::Root => entry.access == FileSystemAccessMode::None, - FileSystemSpecialPath::Minimal | FileSystemSpecialPath::Unknown { .. } => { - false - } - _ => !self.has_same_target_write_override(entry), - }, - } - }) - } - - /// Returns true when a higher-priority `write` entry targets the same - /// location as `entry`, so `entry` cannot narrow effective write access. - fn has_same_target_write_override(&self, entry: &FileSystemSandboxEntry) -> bool { - self.entries.iter().any(|candidate| { - candidate.access.can_write() - && candidate.access > entry.access - && file_system_paths_share_target(&candidate.path, &entry.path) - }) - } - pub fn unrestricted() -> Self { Self { kind: FileSystemSandboxKind::Unrestricted, @@ -256,6 +289,86 @@ impl FileSystemSandboxPolicy { } } + fn has_root_access(&self, predicate: impl Fn(FileSystemAccessMode) -> bool) -> bool { + matches!(self.kind, FileSystemSandboxKind::Restricted) + && self.entries.iter().any(|entry| { + matches!( + &entry.path, + FileSystemPath::Special { value } + if matches!(value, FileSystemSpecialPath::Root) && predicate(entry.access) + ) + }) + } + + pub fn has_denied_read_restrictions(&self) -> bool { + matches!(self.kind, FileSystemSandboxKind::Restricted) + && self + .entries + .iter() + .any(|entry| entry.access == FileSystemAccessMode::None) + } + + pub fn from_legacy_sandbox_policy_preserving_deny_entries( + sandbox_policy: &SandboxPolicy, + cwd: &Path, + existing: &Self, + ) -> Self { + let mut rebuilt = Self::from_legacy_sandbox_policy(sandbox_policy, cwd); + if !matches!(rebuilt.kind, FileSystemSandboxKind::Restricted) { + return rebuilt; + } + + for deny_entry in existing + .entries + .iter() + .filter(|entry| entry.access == FileSystemAccessMode::None) + { + if !rebuilt.entries.iter().any(|entry| entry == deny_entry) { + rebuilt.entries.push(deny_entry.clone()); + } + } + + rebuilt + } + + /// Returns true when a restricted policy contains any entry that really + /// reduces a broader `:root = write` grant. + /// + /// Raw entry presence is not enough here: an equally specific `write` + /// entry for the same target wins under the normal precedence rules, so a + /// shadowed `read` entry must not downgrade the policy out of full-disk + /// write mode. + fn has_write_narrowing_entries(&self) -> bool { + matches!(self.kind, FileSystemSandboxKind::Restricted) + && self.entries.iter().any(|entry| { + if entry.access.can_write() { + return false; + } + + match &entry.path { + FileSystemPath::Path { .. } => !self.has_same_target_write_override(entry), + FileSystemPath::GlobPattern { .. } => true, + FileSystemPath::Special { value } => match value { + FileSystemSpecialPath::Root => entry.access == FileSystemAccessMode::None, + FileSystemSpecialPath::Minimal | FileSystemSpecialPath::Unknown { .. } => { + false + } + _ => !self.has_same_target_write_override(entry), + }, + } + }) + } + + /// Returns true when a higher-priority `write` entry targets the same + /// location as `entry`, so `entry` cannot narrow effective write access. + fn has_same_target_write_override(&self, entry: &FileSystemSandboxEntry) -> bool { + self.entries.iter().any(|candidate| { + candidate.access.can_write() + && candidate.access > entry.access + && file_system_paths_share_target(&candidate.path, &entry.path) + }) + } + /// Converts a legacy sandbox policy into an equivalent filesystem policy /// for the provided cwd. /// @@ -276,6 +389,7 @@ impl FileSystemSandboxPolicy { FileSystemPath::Path { path } => !legacy_writable_roots .iter() .any(|root| root.is_path_writable(path.as_path())), + FileSystemPath::GlobPattern { .. } => true, FileSystemPath::Special { .. } => true, } }); @@ -312,7 +426,7 @@ impl FileSystemSandboxPolicy { FileSystemSandboxKind::Unrestricted | FileSystemSandboxKind::ExternalSandbox => true, FileSystemSandboxKind::Restricted => { self.has_root_access(FileSystemAccessMode::can_read) - && !self.has_explicit_deny_entries() + && !self.has_denied_read_restrictions() } } } @@ -585,6 +699,29 @@ impl FileSystemSandboxPolicy { ) } + /// Returns unreadable glob patterns resolved against the provided cwd. + pub fn get_unreadable_globs_with_cwd(&self, cwd: &Path) -> Vec { + if !matches!(self.kind, FileSystemSandboxKind::Restricted) { + return Vec::new(); + } + + let mut patterns = self + .entries + .iter() + .filter(|entry| entry.access == FileSystemAccessMode::None) + .filter_map(|entry| match &entry.path { + FileSystemPath::GlobPattern { pattern } => { + Some(AbsolutePathBuf::resolve_path_against_base(pattern, cwd)) + } + FileSystemPath::Path { .. } | FileSystemPath::Special { .. } => None, + }) + .map(|pattern| pattern.to_string_lossy().into_owned()) + .collect::>(); + patterns.sort(); + patterns.dedup(); + patterns + } + pub fn to_legacy_sandbox_policy( &self, network_policy: NetworkSandboxPolicy, @@ -620,6 +757,7 @@ impl FileSystemSandboxPolicy { for entry in &self.entries { match &entry.path { + FileSystemPath::GlobPattern { .. } => {} FileSystemPath::Path { path } => { if entry.access.can_write() { if cwd_absolute.as_ref().is_some_and(|cwd| cwd == path) { @@ -770,6 +908,7 @@ impl FileSystemSandboxPolicy { readable_roots: self.get_readable_roots_with_cwd(cwd), writable_roots: self.get_writable_roots_with_cwd(cwd), unreadable_roots: self.get_unreadable_roots_with_cwd(cwd), + unreadable_globs: self.get_unreadable_globs_with_cwd(cwd), } } } @@ -905,6 +1044,7 @@ fn resolve_file_system_path( ) -> Option { match path { FileSystemPath::Path { path } => Some(path.clone()), + FileSystemPath::GlobPattern { .. } => None, FileSystemPath::Special { value } => resolve_file_system_special_path(value, cwd), } } @@ -947,6 +1087,11 @@ fn file_system_paths_share_target(left: &FileSystemPath, right: &FileSystemPath) | (FileSystemPath::Special { value }, FileSystemPath::Path { path }) => { special_path_matches_absolute_path(value, path) } + ( + FileSystemPath::GlobPattern { pattern: left }, + FileSystemPath::GlobPattern { pattern: right }, + ) => left == right, + (FileSystemPath::GlobPattern { .. }, _) | (_, FileSystemPath::GlobPattern { .. }) => false, } } @@ -1021,6 +1166,44 @@ fn absolute_root_path_for_cwd(cwd: &AbsolutePathBuf) -> AbsolutePathBuf { .unwrap_or_else(|err| panic!("cwd root must be an absolute path: {err}")) } +fn normalized_and_canonical_candidates(path: &Path) -> Vec { + // Compare the lexical absolute form plus the canonical target when it + // exists. Missing paths still need the lexical candidate so future-created + // denied paths remain blocked by direct tool checks. + let mut candidates = Vec::new(); + + if let Ok(normalized) = AbsolutePathBuf::from_absolute_path(path) { + push_unique(&mut candidates, normalized.to_path_buf()); + } else { + push_unique(&mut candidates, path.to_path_buf()); + } + + if let Ok(canonical) = path.canonicalize() + && let Ok(canonical_absolute) = AbsolutePathBuf::from_absolute_path(canonical) + { + push_unique(&mut candidates, canonical_absolute.to_path_buf()); + } + + candidates +} + +fn push_unique(candidates: &mut Vec, candidate: PathBuf) { + if !candidates.iter().any(|existing| existing == &candidate) { + candidates.push(candidate); + } +} + +fn build_glob_matcher(pattern: &str) -> Option { + // Keep `*` and `?` within a single path component and preserve an unclosed + // `[` as a literal so matcher behavior stays aligned with config parsing. + GlobBuilder::new(pattern) + .literal_separator(true) + .allow_unclosed_class(true) + .build() + .ok() + .map(|glob| glob.compile_matcher()) +} + fn resolve_file_system_special_path( value: &FileSystemSpecialPath, cwd: Option<&AbsolutePathBuf>, @@ -2061,4 +2244,187 @@ mod tests { assert!(FileSystemAccessMode::Write > FileSystemAccessMode::Read); assert!(FileSystemAccessMode::None > FileSystemAccessMode::Write); } + + #[test] + fn legacy_bridge_preserves_explicit_deny_entries() { + let denied = AbsolutePathBuf::try_from("/tmp/private").expect("absolute path"); + let existing = FileSystemSandboxPolicy::restricted(vec![FileSystemSandboxEntry { + path: FileSystemPath::Path { + path: denied.clone(), + }, + access: FileSystemAccessMode::None, + }]); + + let rebuilt = FileSystemSandboxPolicy::from_legacy_sandbox_policy_preserving_deny_entries( + &SandboxPolicy::new_workspace_write_policy(), + Path::new("/tmp/workspace"), + &existing, + ); + + assert!( + rebuilt.entries.iter().any(|entry| { + entry.path + == FileSystemPath::Path { + path: denied.clone(), + } + && entry.access == FileSystemAccessMode::None + }), + "expected explicit deny entry to be preserved" + ); + } + + fn deny_policy(path: &Path) -> FileSystemSandboxPolicy { + FileSystemSandboxPolicy::restricted(vec![FileSystemSandboxEntry { + path: FileSystemPath::Path { + path: AbsolutePathBuf::try_from(path).expect("absolute deny path"), + }, + access: FileSystemAccessMode::None, + }]) + } + + fn unreadable_glob_entry(pattern: String) -> FileSystemSandboxEntry { + FileSystemSandboxEntry { + path: FileSystemPath::GlobPattern { pattern }, + access: FileSystemAccessMode::None, + } + } + + fn default_policy_with_unreadable_glob(pattern: String) -> FileSystemSandboxPolicy { + let mut policy = FileSystemSandboxPolicy::default(); + policy.entries.push(unreadable_glob_entry(pattern)); + policy + } + + fn is_read_denied( + path: &Path, + file_system_sandbox_policy: &FileSystemSandboxPolicy, + cwd: &Path, + ) -> bool { + ReadDenyMatcher::new(file_system_sandbox_policy, cwd) + .is_some_and(|matcher| matcher.is_read_denied(path)) + } + + #[test] + fn exact_path_and_descendants_are_denied() { + let temp = TempDir::new().expect("tempdir"); + let denied_dir = temp.path().join("denied"); + let nested = denied_dir.join("nested.txt"); + std::fs::create_dir_all(&denied_dir).expect("create denied dir"); + std::fs::write(&nested, "secret").expect("write secret"); + + let policy = deny_policy(&denied_dir); + assert!(is_read_denied(&denied_dir, &policy, temp.path())); + assert!(is_read_denied(&nested, &policy, temp.path())); + assert!(!is_read_denied( + &temp.path().join("other.txt"), + &policy, + temp.path() + )); + } + + #[cfg(unix)] + #[test] + fn canonical_target_matches_denied_symlink_alias() { + let temp = TempDir::new().expect("tempdir"); + let real_dir = temp.path().join("real"); + let alias_dir = temp.path().join("alias"); + std::fs::create_dir_all(&real_dir).expect("create real dir"); + symlink_dir(&real_dir, &alias_dir).expect("symlink alias"); + + let secret = real_dir.join("secret.txt"); + std::fs::write(&secret, "secret").expect("write secret"); + let alias_secret = alias_dir.join("secret.txt"); + + let policy = deny_policy(&real_dir); + assert!(is_read_denied(&alias_secret, &policy, temp.path())); + } + + #[test] + fn literal_patterns_and_globs_are_denied() { + let temp = TempDir::new().expect("tempdir"); + let literal = temp.path().join("private"); + let other = temp.path().join("notes.txt"); + std::fs::create_dir_all(&literal).expect("create literal dir"); + std::fs::write(&other, "notes").expect("write notes"); + + let mut policy = deny_policy(&literal); + policy.entries.push(unreadable_glob_entry(format!( + "{}/**/*.txt", + temp.path().display() + ))); + + assert!(is_read_denied(&literal, &policy, temp.path())); + assert!(is_read_denied(&other, &policy, temp.path())); + } + + #[test] + fn glob_patterns_deny_matching_paths() { + let temp = TempDir::new().expect("tempdir"); + let denied = temp.path().join("private").join("secret1.txt"); + std::fs::create_dir_all(denied.parent().expect("parent")).expect("create parent"); + std::fs::write(&denied, "secret").expect("write secret"); + + let policy = default_policy_with_unreadable_glob(format!( + "{}/private/secret?.txt", + temp.path().display() + )); + + assert!(is_read_denied(&denied, &policy, temp.path())); + } + + #[test] + fn glob_patterns_do_not_cross_path_separators() { + let temp = TempDir::new().expect("tempdir"); + let matching = temp.path().join("app").join("file42.txt"); + let nested = temp.path().join("app").join("nested").join("file42.txt"); + let short = temp.path().join("app").join("file4.txt"); + let letters = temp.path().join("app").join("fileab.txt"); + std::fs::create_dir_all(nested.parent().expect("parent")).expect("create parent"); + std::fs::write(&matching, "secret").expect("write matching"); + std::fs::write(&nested, "secret").expect("write nested"); + std::fs::write(&short, "secret").expect("write short"); + std::fs::write(&letters, "secret").expect("write letters"); + + let policy = default_policy_with_unreadable_glob(format!( + "{}/*/file[0-9]?.txt", + temp.path().display() + )); + + assert!(is_read_denied(&matching, &policy, temp.path())); + assert!(!is_read_denied(&nested, &policy, temp.path())); + assert!(!is_read_denied(&short, &policy, temp.path())); + assert!(!is_read_denied(&letters, &policy, temp.path())); + } + + #[test] + fn globstar_patterns_deny_root_and_nested_matches() { + let temp = TempDir::new().expect("tempdir"); + let root_env = temp.path().join(".env"); + let nested_env = temp.path().join("app").join(".env"); + let other = temp.path().join("app").join("notes.txt"); + std::fs::create_dir_all(nested_env.parent().expect("parent")).expect("create parent"); + std::fs::write(&root_env, "secret").expect("write root env"); + std::fs::write(&nested_env, "secret").expect("write nested env"); + std::fs::write(&other, "notes").expect("write notes"); + + let policy = + default_policy_with_unreadable_glob(format!("{}/**/*.env", temp.path().display())); + + assert!(is_read_denied(&root_env, &policy, temp.path())); + assert!(is_read_denied(&nested_env, &policy, temp.path())); + assert!(!is_read_denied(&other, &policy, temp.path())); + } + + #[test] + fn unclosed_character_classes_match_literal_brackets() { + let temp = TempDir::new().expect("tempdir"); + let bracket_file = temp.path().join("["); + let other = temp.path().join("notes.txt"); + std::fs::write(&bracket_file, "secret").expect("write bracket file"); + std::fs::write(&other, "notes").expect("write notes"); + let policy = default_policy_with_unreadable_glob(format!("{}/[", temp.path().display())); + + assert!(is_read_denied(&bracket_file, &policy, temp.path())); + assert!(!is_read_denied(&other, &policy, temp.path())); + } } diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 6091213852..692238a4fc 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -1223,11 +1223,15 @@ impl SandboxPolicy { // Include /tmp on Unix unless explicitly excluded. if cfg!(unix) && !exclude_slash_tmp { - #[allow(clippy::expect_used)] - let slash_tmp = - AbsolutePathBuf::from_absolute_path("/tmp").expect("/tmp is absolute"); - if slash_tmp.as_path().is_dir() { - roots.push(slash_tmp); + match AbsolutePathBuf::from_absolute_path("/tmp") { + Ok(slash_tmp) => { + if slash_tmp.as_path().is_dir() { + roots.push(slash_tmp); + } + } + Err(e) => { + error!("Ignoring invalid /tmp for sandbox writable root: {e}"); + } } } @@ -2374,6 +2378,9 @@ pub struct McpToolCallBeginEvent { /// Identifier so this can be paired with the McpToolCallEnd event. pub call_id: String, pub invocation: McpInvocation, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub mcp_app_resource_uri: Option, } #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS, PartialEq)] @@ -2381,6 +2388,9 @@ pub struct McpToolCallEndEvent { /// Identifier for the corresponding McpToolCallBegin that finished. pub call_id: String, pub invocation: McpInvocation, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub mcp_app_resource_uri: Option, #[ts(type = "string")] pub duration: Duration, /// Result of the tool call. Note this could be an error. @@ -2846,6 +2856,8 @@ pub struct TurnContextItem { pub sandbox_policy: SandboxPolicy, #[serde(skip_serializing_if = "Option::is_none")] pub network: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub file_system_sandbox_policy: Option, pub model: String, #[serde(skip_serializing_if = "Option::is_none")] pub personality: Option, @@ -4976,6 +4988,7 @@ mod tests { assert_eq!(item.trace_id, None); assert_eq!(item.network, None); + assert_eq!(item.file_system_sandbox_policy, None); Ok(()) } @@ -4993,6 +5006,14 @@ mod tests { allowed_domains: vec!["api.example.com".to_string()], denied_domains: vec!["blocked.example.com".to_string()], }), + file_system_sandbox_policy: Some(FileSystemSandboxPolicy::restricted(vec![ + FileSystemSandboxEntry { + path: FileSystemPath::GlobPattern { + pattern: "/tmp/private/**/*.txt".to_string(), + }, + access: FileSystemAccessMode::None, + }, + ])), model: "gpt-5".to_string(), personality: None, collaboration_mode: None, @@ -5013,6 +5034,19 @@ mod tests { "denied_domains": ["blocked.example.com"], }) ); + assert_eq!( + value["file_system_sandbox_policy"], + json!({ + "kind": "restricted", + "entries": [{ + "path": { + "type": "glob_pattern", + "pattern": "/tmp/private/**/*.txt" + }, + "access": "none" + }] + }) + ); Ok(()) } diff --git a/codex-rs/tools/src/tool_name.rs b/codex-rs/protocol/src/tool_name.rs similarity index 62% rename from codex-rs/tools/src/tool_name.rs rename to codex-rs/protocol/src/tool_name.rs index da18233abb..d09bc1ce72 100644 --- a/codex-rs/tools/src/tool_name.rs +++ b/codex-rs/protocol/src/tool_name.rs @@ -1,12 +1,23 @@ +use serde::Deserialize; +use serde::Serialize; +use std::fmt; + /// Identifies a callable tool, preserving the namespace split when the model /// provides one. -#[derive(Clone, Debug, Eq, Hash, PartialEq)] +#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] pub struct ToolName { pub name: String, pub namespace: Option, } impl ToolName { + pub fn new(namespace: Option, name: impl Into) -> Self { + Self { + name: name.into(), + namespace, + } + } + pub fn plain(name: impl Into) -> Self { Self { name: name.into(), @@ -29,6 +40,15 @@ impl ToolName { } } +impl fmt::Display for ToolName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.namespace { + Some(namespace) => write!(f, "{namespace}{}", self.name), + None => f.write_str(&self.name), + } + } +} + impl From for ToolName { fn from(name: String) -> Self { Self::plain(name) diff --git a/codex-rs/responses-api-proxy/src/dump.rs b/codex-rs/responses-api-proxy/src/dump.rs index bebd4ed592..1a09c64ae7 100644 --- a/codex-rs/responses-api-proxy/src/dump.rs +++ b/codex-rs/responses-api-proxy/src/dump.rs @@ -231,6 +231,12 @@ mod tests { Header::from_bytes(&b"Cookie"[..], &b"user-session=secret"[..]).expect("cookie header"), Header::from_bytes(&b"Content-Type"[..], &b"application/json"[..]) .expect("content-type header"), + Header::from_bytes(&b"x-codex-window-id"[..], &b"thread-1:0"[..]) + .expect("window id header"), + Header::from_bytes(&b"x-codex-parent-thread-id"[..], &b"parent-thread-1"[..]) + .expect("parent thread id header"), + Header::from_bytes(&b"x-openai-subagent"[..], &b"collab_spawn"[..]) + .expect("subagent header"), ]; let exchange_dump = dumper @@ -262,6 +268,18 @@ mod tests { { "name": "Content-Type", "value": "application/json" + }, + { + "name": "x-codex-window-id", + "value": "thread-1:0" + }, + { + "name": "x-codex-parent-thread-id", + "value": "parent-thread-1" + }, + { + "name": "x-openai-subagent", + "value": "collab_spawn" } ], "body": { diff --git a/codex-rs/rmcp-client/src/bin/test_stdio_server.rs b/codex-rs/rmcp-client/src/bin/test_stdio_server.rs index 7295b63159..82bc387d99 100644 --- a/codex-rs/rmcp-client/src/bin/test_stdio_server.rs +++ b/codex-rs/rmcp-client/src/bin/test_stdio_server.rs @@ -1,4 +1,5 @@ use std::borrow::Cow; +use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::hash_map::Entry; use std::sync::Arc; @@ -41,6 +42,7 @@ struct TestToolServer { const MEMO_URI: &str = "memo://codex/example-note"; const MEMO_CONTENT: &str = "This is a sample MCP resource served by the rmcp test server."; +const SANDBOX_STATE_META_CAPABILITY: &str = "codex/sandbox-state-meta"; const SMALL_PNG_BASE64: &str = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg=="; pub fn stdio() -> (tokio::io::Stdin, tokio::io::Stdout) { @@ -49,12 +51,27 @@ pub fn stdio() -> (tokio::io::Stdin, tokio::io::Stdout) { impl TestToolServer { fn new() -> Self { + #[expect(clippy::expect_used)] + let sandbox_meta_schema: JsonObject = serde_json::from_value(serde_json::json!({ + "type": "object", + "properties": {}, + "additionalProperties": false + })) + .expect("sandbox_meta tool schema should deserialize"); + let mut sandbox_meta_tool = Tool::new( + Cow::Borrowed("sandbox_meta"), + Cow::Borrowed("Return the MCP request metadata received by this test server."), + Arc::new(sandbox_meta_schema), + ); + sandbox_meta_tool.annotations = Some(ToolAnnotations::new().read_only(true)); + let tools = vec![ Self::echo_tool(), Self::echo_dash_tool(), Self::sync_tool(), Self::image_tool(), Self::image_scenario_tool(), + sandbox_meta_tool, ]; let resources = vec![Self::memo_resource()]; let resource_templates = vec![Self::memo_template()]; @@ -191,6 +208,7 @@ impl TestToolServer { /// - `codex mcp add mcpimg -- /abs/path/to/test_stdio_server` /// - Then in Codex TUI, ask it to call: /// - `mcpimg.image_scenario({"scenario":"image_only"})` + /// - `mcpimg.image_scenario({"scenario":"image_only_original_detail"})` /// - `mcpimg.image_scenario({"scenario":"text_then_image","caption":"Here is the image:"})` /// - `mcpimg.image_scenario({"scenario":"invalid_base64_then_image"})` /// - `mcpimg.image_scenario({"scenario":"invalid_image_bytes_then_image"})` @@ -207,6 +225,7 @@ impl TestToolServer { "type": "string", "enum": [ "image_only", + "image_only_original_detail", "text_then_image", "invalid_base64_then_image", "invalid_image_bytes_then_image", @@ -322,6 +341,7 @@ fn sync_barrier_map() -> &'static tokio::sync::Mutex ServerInfo { + let mut capabilities = ServerCapabilities::builder() + .enable_tools() + .enable_tool_list_changed() + .enable_resources() + .build(); + capabilities.experimental = Some(BTreeMap::from([( + SANDBOX_STATE_META_CAPABILITY.to_string(), + JsonObject::new(), + )])); + ServerInfo { - capabilities: ServerCapabilities::builder() - .enable_tools() - .enable_tool_list_changed() - .enable_resources() - .build(), + capabilities, ..ServerInfo::default() } } @@ -418,9 +444,15 @@ impl ServerHandler for TestToolServer { async fn call_tool( &self, request: CallToolRequestParams, - _context: rmcp::service::RequestContext, + context: rmcp::service::RequestContext, ) -> Result { match request.name.as_ref() { + "sandbox_meta" => Ok(CallToolResult { + content: Vec::new(), + structured_content: Some(serde_json::Value::Object(context.meta.0)), + is_error: Some(false), + meta: None, + }), "echo" | "echo-tool" => { let args: EchoArgs = match request.arguments { Some(arguments) => serde_json::from_value(serde_json::Value::Object( @@ -523,6 +555,21 @@ impl TestToolServer { ImageScenario::ImageOnly => { content.push(rmcp::model::Content::image(valid_data_b64, mime_type)); } + ImageScenario::ImageOnlyOriginalDetail => { + let mut meta = rmcp::model::Meta::new(); + meta.insert( + "codex/imageDetail".to_string(), + serde_json::json!("original"), + ); + content.push(rmcp::model::Annotated::new( + rmcp::model::RawContent::Image(rmcp::model::RawImageContent { + data: valid_data_b64, + mime_type, + meta: Some(meta), + }), + None, + )); + } ImageScenario::TextThenImage => { content.push(rmcp::model::Content::text(caption)); content.push(rmcp::model::Content::image(valid_data_b64, mime_type)); diff --git a/codex-rs/rollout/src/lib.rs b/codex-rs/rollout/src/lib.rs index 31fcf47240..de169db1f7 100644 --- a/codex-rs/rollout/src/lib.rs +++ b/codex-rs/rollout/src/lib.rs @@ -48,6 +48,7 @@ pub use list::get_threads_in_root; pub use list::parse_cursor; pub use list::read_head_for_summary; pub use list::read_session_meta_line; +pub use list::read_thread_item_from_rollout; pub use list::rollout_date_parts; pub use metadata::builder_from_items; pub use policy::EventPersistenceMode; diff --git a/codex-rs/rollout/src/list.rs b/codex-rs/rollout/src/list.rs index afa4103307..640f0b0278 100644 --- a/codex-rs/rollout/src/list.rs +++ b/codex-rs/rollout/src/list.rs @@ -756,6 +756,21 @@ async fn build_thread_item( None } +/// Read a single rollout file into the same summary item shape used by thread listing. +/// +/// This is for callers that already resolved a rollout path and need the same +/// metadata/preview extraction as list operations without scanning the whole +/// sessions tree. +pub async fn read_thread_item_from_rollout(path: PathBuf) -> Option { + build_thread_item( + path, + &[], + /*provider_matcher*/ None, + /*updated_at*/ None, + ) + .await +} + /// Collects immediate subdirectories of `parent`, parses their (string) names with `parse`, /// and returns them sorted descending by the parsed key. async fn collect_dirs_desc(parent: &Path, parse: F) -> io::Result> diff --git a/codex-rs/rollout/src/recorder.rs b/codex-rs/rollout/src/recorder.rs index 4f88ff1a21..ab1ae0a5bf 100644 --- a/codex-rs/rollout/src/recorder.rs +++ b/codex-rs/rollout/src/recorder.rs @@ -279,6 +279,10 @@ impl RolloutRecorder { let codex_home = config.codex_home(); let state_db_ctx = state_db::get_state_db(config).await; + // Search is the SQLite-optimized path and assumes a DB marked backfill-complete is + // actually populated enough to answer the query. If unmigrated rollout files still exist + // on disk, the repair path below may or may not run and catch them depending on whether + // SQLite already has another matching search hit. if search_term.is_some() && let Some(db_page) = state_db::list_threads_db( state_db_ctx.as_deref(), diff --git a/codex-rs/rollout/src/recorder_tests.rs b/codex-rs/rollout/src/recorder_tests.rs index 3e2ac52194..e3d82c4454 100644 --- a/codex-rs/rollout/src/recorder_tests.rs +++ b/codex-rs/rollout/src/recorder_tests.rs @@ -544,6 +544,7 @@ async fn resume_candidate_matches_cwd_reads_latest_turn_context() -> std::io::Re approval_policy: AskForApproval::Never, sandbox_policy: SandboxPolicy::new_read_only_policy(), network: None, + file_system_sandbox_policy: None, model: "test-model".to_string(), personality: None, collaboration_mode: None, diff --git a/codex-rs/sandboxing/Cargo.toml b/codex-rs/sandboxing/Cargo.toml index 274e5bce22..18c5ffb45d 100644 --- a/codex-rs/sandboxing/Cargo.toml +++ b/codex-rs/sandboxing/Cargo.toml @@ -23,5 +23,8 @@ url = { workspace = true } which = { workspace = true } [dev-dependencies] +anyhow = { workspace = true } +async-trait = { workspace = true } pretty_assertions = { workspace = true } tempfile = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/codex-rs/sandboxing/src/lib.rs b/codex-rs/sandboxing/src/lib.rs index 38aed6ce24..dbb8323649 100644 --- a/codex-rs/sandboxing/src/lib.rs +++ b/codex-rs/sandboxing/src/lib.rs @@ -34,6 +34,9 @@ impl From for CodexErr { SandboxTransformError::MissingLinuxSandboxExecutable => { CodexErr::LandlockSandboxExecutableNotProvided } + SandboxTransformError::UnreadableGlobPatternsUnsupported => { + CodexErr::UnsupportedOperation(err.to_string()) + } #[cfg(target_os = "linux")] SandboxTransformError::Wsl1UnsupportedForBubblewrap => { CodexErr::UnsupportedOperation(crate::bwrap::WSL1_BWRAP_WARNING.to_string()) diff --git a/codex-rs/sandboxing/src/manager.rs b/codex-rs/sandboxing/src/manager.rs index ebff33f105..1e3ea9c196 100644 --- a/codex-rs/sandboxing/src/manager.rs +++ b/codex-rs/sandboxing/src/manager.rs @@ -9,10 +9,6 @@ use crate::policy_transforms::EffectiveSandboxPermissions; use crate::policy_transforms::effective_file_system_sandbox_policy; use crate::policy_transforms::effective_network_sandbox_policy; use crate::policy_transforms::should_require_platform_sandbox; -#[cfg(target_os = "macos")] -use crate::seatbelt::MACOS_PATH_TO_SEATBELT_EXECUTABLE; -#[cfg(target_os = "macos")] -use crate::seatbelt::create_seatbelt_command_args_for_policies; use codex_network_proxy::NetworkProxy; use codex_protocol::config_types::WindowsSandboxLevel; use codex_protocol::models::PermissionProfile; @@ -113,6 +109,7 @@ pub struct SandboxTransformRequest<'a> { #[derive(Debug)] pub enum SandboxTransformError { MissingLinuxSandboxExecutable, + UnreadableGlobPatternsUnsupported, #[cfg(target_os = "linux")] Wsl1UnsupportedForBubblewrap, #[cfg(not(target_os = "macos"))] @@ -125,6 +122,10 @@ impl std::fmt::Display for SandboxTransformError { Self::MissingLinuxSandboxExecutable => { write!(f, "missing codex-linux-sandbox executable path") } + Self::UnreadableGlobPatternsUnsupported => write!( + f, + "platform sandbox backend cannot enforce unreadable glob patterns" + ), #[cfg(target_os = "linux")] Self::Wsl1UnsupportedForBubblewrap => write!(f, "{WSL1_BWRAP_WARNING}"), #[cfg(not(target_os = "macos"))] @@ -200,6 +201,15 @@ impl SandboxManager { ); let effective_network_policy = effective_network_sandbox_policy(network_policy, additional_permissions.as_ref()); + if matches!( + sandbox, + SandboxType::MacosSeatbelt | SandboxType::LinuxSeccomp + ) && !effective_file_system_policy + .get_unreadable_globs_with_cwd(sandbox_policy_cwd) + .is_empty() + { + return Err(SandboxTransformError::UnreadableGlobPatternsUnsupported); + } let mut argv = Vec::with_capacity(1 + command.args.len()); argv.push(command.program); argv.extend(command.args.into_iter().map(OsString::from)); @@ -208,14 +218,19 @@ impl SandboxManager { SandboxType::None => (os_argv_to_strings(argv), None), #[cfg(target_os = "macos")] SandboxType::MacosSeatbelt => { - let mut args = create_seatbelt_command_args_for_policies( - os_argv_to_strings(argv), - &effective_file_system_policy, - effective_network_policy, + use crate::seatbelt::CreateSeatbeltCommandArgsParams; + use crate::seatbelt::MACOS_PATH_TO_SEATBELT_EXECUTABLE; + use crate::seatbelt::create_seatbelt_command_args; + + let mut args = create_seatbelt_command_args(CreateSeatbeltCommandArgsParams { + command: os_argv_to_strings(argv), + file_system_sandbox_policy: &effective_file_system_policy, + network_sandbox_policy: effective_network_policy, sandbox_policy_cwd, enforce_managed_network, network, - ); + extra_allow_unix_sockets: &[], + }); let mut full_command = Vec::with_capacity(1 + args.len()); full_command.push(MACOS_PATH_TO_SEATBELT_EXECUTABLE.to_string()); full_command.append(&mut args); diff --git a/codex-rs/sandboxing/src/seatbelt.rs b/codex-rs/sandboxing/src/seatbelt.rs index 517a9163df..b12492210c 100644 --- a/codex-rs/sandboxing/src/seatbelt.rs +++ b/codex-rs/sandboxing/src/seatbelt.rs @@ -100,41 +100,54 @@ struct UnixSocketPathParam { path: AbsolutePathBuf, } -fn proxy_policy_inputs(network: Option<&NetworkProxy>) -> ProxyPolicyInputs { - if let Some(network) = network { - let mut env = HashMap::new(); - network.apply_to_env(&mut env); - let unix_domain_socket_policy = if network.dangerously_allow_all_unix_sockets() { - UnixDomainSocketPolicy::AllowAll - } else { - let allowed = network - .allow_unix_sockets() - .iter() - .filter_map( - |socket_path| match normalize_path_for_sandbox(Path::new(socket_path)) { - Some(path) => Some((path.to_string_lossy().to_string(), path)), - None => { - warn!( - "ignoring network.allow_unix_sockets entry because it could not be normalized: {socket_path}" - ); - None - } - }, - ) - .collect::>() - .into_values() - .collect(); - UnixDomainSocketPolicy::Restricted { allowed } - }; - return ProxyPolicyInputs { - ports: proxy_loopback_ports_from_env(&env), - has_proxy_config: has_proxy_url_env_vars(&env), - allow_local_binding: network.allow_local_binding(), - unix_domain_socket_policy, - }; - } +fn proxy_policy_inputs( + network: Option<&NetworkProxy>, + extra_allow_unix_sockets: &[AbsolutePathBuf], +) -> ProxyPolicyInputs { + let extra_allowed = extra_allow_unix_sockets + .iter() + .filter_map(|socket_path| normalize_path_for_sandbox(socket_path.as_path())) + .collect::>(); - ProxyPolicyInputs::default() + match network { + Some(network) => { + let mut env = HashMap::new(); + network.apply_to_env(&mut env); + let unix_domain_socket_policy = if network.dangerously_allow_all_unix_sockets() { + UnixDomainSocketPolicy::AllowAll + } else { + let mut allowed = network + .allow_unix_sockets() + .iter() + .filter_map(|socket_path| { + match normalize_path_for_sandbox(Path::new(socket_path)) { + Some(path) => Some(path), + None => { + warn!( + "ignoring network.allow_unix_sockets entry because it could not be normalized: {socket_path}" + ); + None + } + } + }) + .collect::>(); + allowed.extend(extra_allowed); + UnixDomainSocketPolicy::Restricted { allowed } + }; + ProxyPolicyInputs { + ports: proxy_loopback_ports_from_env(&env), + has_proxy_config: has_proxy_url_env_vars(&env), + allow_local_binding: network.allow_local_binding(), + unix_domain_socket_policy, + } + } + None => ProxyPolicyInputs { + unix_domain_socket_policy: UnixDomainSocketPolicy::Restricted { + allowed: extra_allowed, + }, + ..Default::default() + }, + } } fn normalize_path_for_sandbox(path: &Path) -> Option { @@ -244,8 +257,14 @@ fn dynamic_network_policy_for_network( enforce_managed_network: bool, proxy: &ProxyPolicyInputs, ) -> String { - let should_use_restricted_network_policy = - !proxy.ports.is_empty() || proxy.has_proxy_config || enforce_managed_network; + let has_some_unix_socket_access = match &proxy.unix_domain_socket_policy { + UnixDomainSocketPolicy::AllowAll => true, + UnixDomainSocketPolicy::Restricted { allowed } => !allowed.is_empty(), + }; + let should_use_restricted_network_policy = !proxy.ports.is_empty() + || proxy.has_proxy_config + || enforce_managed_network + || (!network_policy.is_enabled() && has_some_unix_socket_access); if should_use_restricted_network_policy { let mut policy = String::new(); if proxy.allow_local_binding { @@ -285,9 +304,13 @@ fn dynamic_network_policy_for_network( if network_policy.is_enabled() { // No proxy env is configured: retain the existing full-network behavior. - format!( - "(allow network-outbound)\n(allow network-inbound)\n{MACOS_SEATBELT_NETWORK_POLICY}" - ) + let mut policy = String::from("(allow network-outbound)\n(allow network-inbound)\n"); + let unix_socket_policy = unix_socket_policy(proxy); + if !unix_socket_policy.is_empty() { + policy.push_str("; allow unix domain sockets for local IPC\n"); + policy.push_str(&unix_socket_policy); + } + format!("{policy}{MACOS_SEATBELT_NETWORK_POLICY}") } else { String::new() } @@ -357,31 +380,48 @@ fn build_seatbelt_access_policy( } #[cfg_attr(not(test), allow(dead_code))] -fn create_seatbelt_command_args( +fn create_seatbelt_command_args_for_legacy_policy( command: Vec, sandbox_policy: &SandboxPolicy, sandbox_policy_cwd: &Path, enforce_managed_network: bool, network: Option<&NetworkProxy>, ) -> Vec { - create_seatbelt_command_args_for_policies( + let file_system_sandbox_policy = + FileSystemSandboxPolicy::from_legacy_sandbox_policy(sandbox_policy, sandbox_policy_cwd); + create_seatbelt_command_args(CreateSeatbeltCommandArgsParams { command, - &FileSystemSandboxPolicy::from_legacy_sandbox_policy(sandbox_policy, sandbox_policy_cwd), - NetworkSandboxPolicy::from(sandbox_policy), + file_system_sandbox_policy: &file_system_sandbox_policy, + network_sandbox_policy: NetworkSandboxPolicy::from(sandbox_policy), sandbox_policy_cwd, enforce_managed_network, network, - ) + extra_allow_unix_sockets: &[], + }) } -pub fn create_seatbelt_command_args_for_policies( - command: Vec, - file_system_sandbox_policy: &FileSystemSandboxPolicy, - network_sandbox_policy: NetworkSandboxPolicy, - sandbox_policy_cwd: &Path, - enforce_managed_network: bool, - network: Option<&NetworkProxy>, -) -> Vec { +#[derive(Debug)] +pub struct CreateSeatbeltCommandArgsParams<'a> { + pub command: Vec, + pub file_system_sandbox_policy: &'a FileSystemSandboxPolicy, + pub network_sandbox_policy: NetworkSandboxPolicy, + pub sandbox_policy_cwd: &'a Path, + pub enforce_managed_network: bool, + pub network: Option<&'a NetworkProxy>, + pub extra_allow_unix_sockets: &'a [AbsolutePathBuf], +} + +pub fn create_seatbelt_command_args(args: CreateSeatbeltCommandArgsParams<'_>) -> Vec { + let CreateSeatbeltCommandArgsParams { + command, + file_system_sandbox_policy, + network_sandbox_policy, + sandbox_policy_cwd, + enforce_managed_network, + network, + extra_allow_unix_sockets, + } = args; + let unreadable_roots = file_system_sandbox_policy.get_unreadable_roots_with_cwd(sandbox_policy_cwd); let (file_write_policy, file_write_dir_params) = @@ -465,7 +505,7 @@ pub fn create_seatbelt_command_args_for_policies( } }; - let proxy = proxy_policy_inputs(network); + let proxy = proxy_policy_inputs(network, extra_allow_unix_sockets); let network_policy = dynamic_network_policy_for_network(network_sandbox_policy, enforce_managed_network, &proxy); diff --git a/codex-rs/sandboxing/src/seatbelt_tests.rs b/codex-rs/sandboxing/src/seatbelt_tests.rs index 419f3968d6..f222dd15d1 100644 --- a/codex-rs/sandboxing/src/seatbelt_tests.rs +++ b/codex-rs/sandboxing/src/seatbelt_tests.rs @@ -1,14 +1,23 @@ +use super::CreateSeatbeltCommandArgsParams; use super::MACOS_PATH_TO_SEATBELT_EXECUTABLE; use super::MACOS_SEATBELT_BASE_POLICY; use super::ProxyPolicyInputs; use super::UnixDomainSocketPolicy; use super::create_seatbelt_command_args; -use super::create_seatbelt_command_args_for_policies; +use super::create_seatbelt_command_args_for_legacy_policy; use super::dynamic_network_policy; use super::macos_dir_params; use super::normalize_path_for_sandbox; use super::unix_socket_dir_params; use super::unix_socket_policy; +use codex_network_proxy::ConfigReloader; +use codex_network_proxy::ConfigState; +use codex_network_proxy::NetworkMode; +use codex_network_proxy::NetworkProxy; +use codex_network_proxy::NetworkProxyConfig; +use codex_network_proxy::NetworkProxyConstraints; +use codex_network_proxy::NetworkProxyState; +use codex_network_proxy::build_config_state; use codex_protocol::permissions::FileSystemAccessMode; use codex_protocol::permissions::FileSystemPath; use codex_protocol::permissions::FileSystemSandboxEntry; @@ -23,6 +32,7 @@ use std::fs; use std::path::Path; use std::path::PathBuf; use std::process::Command; +use std::sync::Arc; use tempfile::TempDir; fn assert_seatbelt_denied(stderr: &[u8], path: &Path) { @@ -48,6 +58,23 @@ fn seatbelt_policy_arg(args: &[String]) -> &str { .expect("seatbelt args should include policy text") } +struct TestConfigReloader; + +#[async_trait::async_trait] +impl ConfigReloader for TestConfigReloader { + fn source_label(&self) -> String { + "seatbelt test config".to_string() + } + + async fn maybe_reload(&self) -> anyhow::Result> { + Ok(None) + } + + async fn reload_now(&self) -> anyhow::Result { + Err(anyhow::anyhow!("seatbelt test config cannot reload")) + } +} + #[test] fn base_policy_allows_node_cpu_sysctls() { assert!( @@ -128,14 +155,15 @@ fn explicit_unreadable_paths_are_excluded_from_full_disk_read_and_write_access() }, ]); - let args = create_seatbelt_command_args_for_policies( - vec!["/bin/true".to_string()], - &file_system_policy, - NetworkSandboxPolicy::Restricted, - Path::new("/"), - /*enforce_managed_network*/ false, - /*network*/ None, - ); + let args = create_seatbelt_command_args(CreateSeatbeltCommandArgsParams { + command: vec!["/bin/true".to_string()], + file_system_sandbox_policy: &file_system_policy, + network_sandbox_policy: NetworkSandboxPolicy::Restricted, + sandbox_policy_cwd: Path::new("/"), + enforce_managed_network: false, + network: None, + extra_allow_unix_sockets: &[], + }); let policy = seatbelt_policy_arg(&args); let unreadable_roots = file_system_policy.get_unreadable_roots_with_cwd(Path::new("/")); @@ -193,14 +221,15 @@ fn explicit_unreadable_paths_are_excluded_from_readable_roots() { }, ]); - let args = create_seatbelt_command_args_for_policies( - vec!["/bin/true".to_string()], - &file_system_policy, - NetworkSandboxPolicy::Restricted, - Path::new("/"), - /*enforce_managed_network*/ false, - /*network*/ None, - ); + let args = create_seatbelt_command_args(CreateSeatbeltCommandArgsParams { + command: vec!["/bin/true".to_string()], + file_system_sandbox_policy: &file_system_policy, + network_sandbox_policy: NetworkSandboxPolicy::Restricted, + sandbox_policy_cwd: Path::new("/"), + enforce_managed_network: false, + network: None, + extra_allow_unix_sockets: &[], + }); let policy = seatbelt_policy_arg(&args); let readable_roots = file_system_policy.get_readable_roots_with_cwd(Path::new("/")); @@ -231,7 +260,7 @@ fn explicit_unreadable_paths_are_excluded_from_readable_roots() { #[test] fn seatbelt_args_without_extension_profile_keep_legacy_preferences_read_access() { let cwd = std::env::temp_dir(); - let args = create_seatbelt_command_args( + let args = create_seatbelt_command_args_for_legacy_policy( vec!["echo".to_string(), "ok".to_string()], &SandboxPolicy::new_read_only_policy(), cwd.as_path(), @@ -249,7 +278,7 @@ fn seatbelt_legacy_workspace_write_nested_readable_root_stays_writable() { let cwd = tmp.path().join("workspace"); fs::create_dir_all(cwd.join("docs")).expect("create docs"); let docs = AbsolutePathBuf::from_absolute_path(cwd.join("docs")).expect("absolute docs"); - let args = create_seatbelt_command_args( + let args = create_seatbelt_command_args_for_legacy_policy( vec!["/bin/true".to_string()], &SandboxPolicy::WorkspaceWrite { writable_roots: vec![], @@ -451,6 +480,140 @@ fn create_seatbelt_args_allowlists_unix_socket_paths() { ); } +#[test] +fn create_seatbelt_args_allowlists_explicit_unix_socket_paths_without_proxy() { + let cwd = TempDir::new().expect("temp cwd"); + let file_system_policy = FileSystemSandboxPolicy::from_legacy_sandbox_policy( + &SandboxPolicy::new_read_only_policy(), + cwd.path(), + ); + let extra_allow_unix_sockets = vec![absolute_path("/tmp/codex-browser-use")]; + let args = create_seatbelt_command_args(CreateSeatbeltCommandArgsParams { + command: vec!["/usr/bin/true".to_string()], + file_system_sandbox_policy: &file_system_policy, + network_sandbox_policy: NetworkSandboxPolicy::Restricted, + sandbox_policy_cwd: cwd.path(), + enforce_managed_network: false, + network: None, + extra_allow_unix_sockets: &extra_allow_unix_sockets, + }); + let policy = seatbelt_policy_arg(&args); + + assert!( + policy.contains("(allow system-socket (socket-domain AF_UNIX))"), + "policy should allow AF_UNIX when explicit socket paths are requested:\n{policy}" + ); + assert!( + policy.contains( + "(allow network-outbound (remote unix-socket (subpath (param \"UNIX_SOCKET_PATH_0\"))))" + ), + "policy should allow outbound AF_UNIX traffic for explicit socket paths:\n{policy}" + ); + let expected_socket_root = normalize_path_for_sandbox(Path::new("/tmp/codex-browser-use")) + .expect("socket root should normalize") + .to_string_lossy() + .into_owned(); + assert!( + args.iter() + .any(|arg| arg == &format!("-DUNIX_SOCKET_PATH_0={expected_socket_root}")), + "seatbelt args should pass the configured socket root as a sandbox param: {args:?}" + ); +} + +#[tokio::test] +async fn create_seatbelt_args_merges_proxy_and_explicit_unix_socket_paths() -> anyhow::Result<()> { + let cwd = TempDir::new().expect("temp cwd"); + let file_system_policy = FileSystemSandboxPolicy::from_legacy_sandbox_policy( + &SandboxPolicy::new_read_only_policy(), + cwd.path(), + ); + let network_socket = "/tmp/codex-proxy-use"; + let explicit_socket = "/tmp/codex-browser-use"; + let mut network_config = NetworkProxyConfig::default(); + network_config.network.enabled = true; + network_config.network.mode = NetworkMode::Full; + network_config + .network + .set_allow_unix_sockets(vec![network_socket.to_string()]); + let state = build_config_state(network_config, NetworkProxyConstraints::default())?; + let network_proxy = NetworkProxy::builder() + .state(Arc::new(NetworkProxyState::with_reloader( + state, + Arc::new(TestConfigReloader), + ))) + .managed_by_codex(/*managed_by_codex*/ false) + .build() + .await?; + let extra_allow_unix_sockets = vec![absolute_path(explicit_socket)]; + + let args = create_seatbelt_command_args(CreateSeatbeltCommandArgsParams { + command: vec!["/usr/bin/true".to_string()], + file_system_sandbox_policy: &file_system_policy, + network_sandbox_policy: NetworkSandboxPolicy::Restricted, + sandbox_policy_cwd: cwd.path(), + enforce_managed_network: false, + network: Some(&network_proxy), + extra_allow_unix_sockets: &extra_allow_unix_sockets, + }); + + let expected_explicit_socket = normalize_path_for_sandbox(Path::new(explicit_socket)) + .expect("explicit socket root should normalize"); + let expected_network_socket = normalize_path_for_sandbox(Path::new(network_socket)) + .expect("network socket root should normalize"); + let unix_socket_definitions = args + .iter() + .filter(|arg| arg.starts_with("-DUNIX_SOCKET_PATH_")) + .cloned() + .collect::>(); + assert_eq!( + unix_socket_definitions, + vec![ + format!( + "-DUNIX_SOCKET_PATH_0={}", + expected_explicit_socket.display() + ), + format!("-DUNIX_SOCKET_PATH_1={}", expected_network_socket.display()), + ], + "seatbelt args should include both explicit and network proxy socket roots: {args:?}" + ); + Ok(()) +} + +#[test] +fn create_seatbelt_args_preserves_full_network_with_explicit_unix_socket_paths() { + let cwd = TempDir::new().expect("temp cwd"); + let file_system_policy = FileSystemSandboxPolicy::from_legacy_sandbox_policy( + &SandboxPolicy::new_read_only_policy(), + cwd.path(), + ); + let extra_allow_unix_sockets = vec![absolute_path("/tmp/codex-browser-use")]; + let args = create_seatbelt_command_args(CreateSeatbeltCommandArgsParams { + command: vec!["/usr/bin/true".to_string()], + file_system_sandbox_policy: &file_system_policy, + network_sandbox_policy: NetworkSandboxPolicy::Enabled, + sandbox_policy_cwd: cwd.path(), + enforce_managed_network: false, + network: None, + extra_allow_unix_sockets: &extra_allow_unix_sockets, + }); + let policy = seatbelt_policy_arg(&args); + + assert!( + policy.contains("(allow network-outbound)\n"), + "policy should preserve full outbound network access:\n{policy}" + ); + assert!( + policy.contains("(allow network-inbound)\n"), + "policy should preserve full inbound network access:\n{policy}" + ); + assert!( + policy.contains( + "(allow network-outbound (remote unix-socket (subpath (param \"UNIX_SOCKET_PATH_0\"))))" + ), + "policy should still allow outbound AF_UNIX traffic for explicit socket paths:\n{policy}" + ); +} + #[test] fn unix_socket_policy_non_empty_output_is_newline_terminated() { let allowlist_policy = unix_socket_policy(&ProxyPolicyInputs { @@ -615,7 +778,7 @@ fn create_seatbelt_args_with_read_only_git_and_codex_subpaths() { .iter() .map(std::string::ToString::to_string) .collect(); - let args = create_seatbelt_command_args( + let args = create_seatbelt_command_args_for_legacy_policy( shell_command.clone(), &policy, &cwd, @@ -729,7 +892,7 @@ fn create_seatbelt_args_with_read_only_git_and_codex_subpaths() { .iter() .map(std::string::ToString::to_string) .collect(); - let write_hooks_file_args = create_seatbelt_command_args( + let write_hooks_file_args = create_seatbelt_command_args_for_legacy_policy( shell_command_git, &policy, &cwd, @@ -765,7 +928,7 @@ fn create_seatbelt_args_with_read_only_git_and_codex_subpaths() { .iter() .map(std::string::ToString::to_string) .collect(); - let write_allowed_file_args = create_seatbelt_command_args( + let write_allowed_file_args = create_seatbelt_command_args_for_legacy_policy( shell_command_allowed, &policy, &cwd, @@ -830,7 +993,7 @@ fn create_seatbelt_args_block_first_time_dot_codex_creation_with_exact_and_desce .iter() .map(std::string::ToString::to_string) .collect(); - let args = create_seatbelt_command_args( + let args = create_seatbelt_command_args_for_legacy_policy( shell_command, &policy, repo_root.as_path(), @@ -885,7 +1048,7 @@ fn create_seatbelt_args_with_read_only_git_pointer_file() { .iter() .map(std::string::ToString::to_string) .collect(); - let args = create_seatbelt_command_args( + let args = create_seatbelt_command_args_for_legacy_policy( shell_command, &policy, &cwd, @@ -921,7 +1084,7 @@ fn create_seatbelt_args_with_read_only_git_pointer_file() { .iter() .map(std::string::ToString::to_string) .collect(); - let gitdir_args = create_seatbelt_command_args( + let gitdir_args = create_seatbelt_command_args_for_legacy_policy( shell_command_gitdir, &policy, &cwd, @@ -984,7 +1147,7 @@ fn create_seatbelt_args_for_cwd_as_git_repo() { .iter() .map(std::string::ToString::to_string) .collect(); - let args = create_seatbelt_command_args( + let args = create_seatbelt_command_args_for_legacy_policy( shell_command.clone(), &policy, vulnerable_root.as_path(), diff --git a/codex-rs/state/src/extract.rs b/codex-rs/state/src/extract.rs index 91c6755ffe..6d36f211a4 100644 --- a/codex-rs/state/src/extract.rs +++ b/codex-rs/state/src/extract.rs @@ -302,6 +302,7 @@ mod tests { approval_policy: AskForApproval::Never, sandbox_policy: SandboxPolicy::DangerFullAccess, network: None, + file_system_sandbox_policy: None, model: "gpt-5".to_string(), personality: None, collaboration_mode: None, @@ -340,6 +341,7 @@ mod tests { approval_policy: AskForApproval::OnRequest, sandbox_policy: SandboxPolicy::new_read_only_policy(), network: None, + file_system_sandbox_policy: None, model: "gpt-5".to_string(), personality: None, collaboration_mode: None, @@ -372,6 +374,7 @@ mod tests { approval_policy: AskForApproval::OnRequest, sandbox_policy: SandboxPolicy::new_read_only_policy(), network: None, + file_system_sandbox_policy: None, model: "gpt-5".to_string(), personality: None, collaboration_mode: None, diff --git a/codex-rs/state/src/log_db.rs b/codex-rs/state/src/log_db.rs index e33baae573..c28f762d27 100644 --- a/codex-rs/state/src/log_db.rs +++ b/codex-rs/state/src/log_db.rs @@ -384,9 +384,8 @@ mod tests { use std::io; use std::sync::Arc; use std::sync::Mutex; - use std::time::Duration; - use tokio::time::Instant; + use pretty_assertions::assert_eq; use tracing_subscriber::filter::Targets; use tracing_subscriber::fmt::writer::MakeWriter; use tracing_subscriber::layer::SubscriberExt; @@ -442,6 +441,7 @@ mod tests { .await .expect("initialize runtime"); let writer = SharedWriter::default(); + let layer = start(runtime.clone()); let subscriber = tracing_subscriber::registry() .with( @@ -452,7 +452,8 @@ mod tests { .with_filter(Targets::new().with_default(tracing::Level::TRACE)), ) .with( - start(runtime.clone()) + layer + .clone() .with_filter(Targets::new().with_default(tracing::Level::TRACE)), ); let guard = subscriber.set_default(); @@ -463,6 +464,7 @@ mod tests { }); tracing::debug!("threadless-after"); + layer.flush().await; drop(guard); let feedback_logs = writer.snapshot(); @@ -475,24 +477,17 @@ mod tests { .collect::>() .join("\n") }; - let deadline = Instant::now() + Duration::from_secs(2); - loop { - let sqlite_logs = String::from_utf8( - runtime - .query_feedback_logs("thread-1") - .await - .expect("query feedback logs"), - ) - .expect("valid utf-8"); - if without_timestamps(&sqlite_logs) == without_timestamps(&feedback_logs) { - break; - } - assert!( - Instant::now() < deadline, - "sqlite feedback logs did not match feedback formatter output before timeout\nsqlite:\n{sqlite_logs}\nfeedback:\n{feedback_logs}" - ); - tokio::time::sleep(Duration::from_millis(10)).await; - } + let sqlite_logs = String::from_utf8( + runtime + .query_feedback_logs("thread-1") + .await + .expect("query feedback logs"), + ) + .expect("valid utf-8"); + assert_eq!( + without_timestamps(&sqlite_logs), + without_timestamps(&feedback_logs) + ); let _ = tokio::fs::remove_dir_all(codex_home).await; } diff --git a/codex-rs/state/src/runtime.rs b/codex-rs/state/src/runtime.rs index 610e45e68a..054ad6ff2c 100644 --- a/codex-rs/state/src/runtime.rs +++ b/codex-rs/state/src/runtime.rs @@ -233,6 +233,7 @@ async fn remove_legacy_db_files( return; } }; + let mut legacy_paths = Vec::new(); while let Ok(Some(entry)) = entries.next_entry().await { if !entry .file_type() @@ -248,7 +249,14 @@ async fn remove_legacy_db_files( continue; } - let legacy_path = entry.path(); + legacy_paths.push(entry.path()); + } + + // On Windows, SQLite can keep the main database file undeletable until the + // matching `-wal` / `-shm` sidecars are removed. Remove the longest + // sidecar-style paths first so the main file is attempted last. + legacy_paths.sort_by_key(|path| std::cmp::Reverse(path.as_os_str().len())); + for legacy_path in legacy_paths { if let Err(err) = tokio::fs::remove_file(&legacy_path).await { warn!( "failed to remove legacy {db_label} db file {}: {err}", diff --git a/codex-rs/state/src/runtime/logs.rs b/codex-rs/state/src/runtime/logs.rs index 56ef31d5d1..6728965ac3 100644 --- a/codex-rs/state/src/runtime/logs.rs +++ b/codex-rs/state/src/runtime/logs.rs @@ -729,6 +729,7 @@ mod tests { .await .expect("insert legacy log row"); pool.close().await; + drop(pool); let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string()) .await diff --git a/codex-rs/state/src/runtime/memories.rs b/codex-rs/state/src/runtime/memories.rs index 50f2f0cd8c..4817d5db67 100644 --- a/codex-rs/state/src/runtime/memories.rs +++ b/codex-rs/state/src/runtime/memories.rs @@ -30,21 +30,6 @@ impl StateRuntime { /// stage-1 (`memory_stage1`) and phase-2 (`memory_consolidate_global`) /// memory pipelines. pub async fn clear_memory_data(&self) -> anyhow::Result<()> { - self.clear_memory_data_inner(/*disable_existing_threads*/ false) - .await - } - - /// Resets persisted memory state for a clean-slate local start. - /// - /// In addition to clearing persisted stage-1 outputs and memory pipeline - /// jobs, this disables memory generation for all existing threads so - /// historical rollouts are not immediately picked up again. - pub async fn reset_memory_data_for_fresh_start(&self) -> anyhow::Result<()> { - self.clear_memory_data_inner(/*disable_existing_threads*/ true) - .await - } - - async fn clear_memory_data_inner(&self, disable_existing_threads: bool) -> anyhow::Result<()> { let mut tx = self.pool.begin().await?; sqlx::query( @@ -66,18 +51,6 @@ WHERE kind = ? OR kind = ? .execute(&mut *tx) .await?; - if disable_existing_threads { - sqlx::query( - r#" -UPDATE threads -SET memory_mode = 'disabled' -WHERE memory_mode = 'enabled' - "#, - ) - .execute(&mut *tx) - .await?; - } - tx.commit().await?; Ok(()) } @@ -1823,7 +1796,7 @@ mod tests { } #[tokio::test] - async fn reset_memory_data_for_fresh_start_clears_rows_and_disables_threads() { + async fn clear_memory_data_clears_rows_and_preserves_thread_memory_modes() { let codex_home = unique_temp_dir(); let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string()) .await @@ -1893,9 +1866,9 @@ mod tests { .expect("disable existing thread"); runtime - .reset_memory_data_for_fresh_start() + .clear_memory_data() .await - .expect("reset memory data"); + .expect("clear memory data"); let stage1_outputs_count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM stage1_outputs") .fetch_one(runtime.pool.as_ref()) @@ -1918,7 +1891,7 @@ mod tests { .fetch_one(runtime.pool.as_ref()) .await .expect("read enabled thread memory mode"); - assert_eq!(enabled_memory_mode, "disabled"); + assert_eq!(enabled_memory_mode, "enabled"); let disabled_memory_mode: String = sqlx::query_scalar("SELECT memory_mode FROM threads WHERE id = ?") diff --git a/codex-rs/thread-store/Cargo.toml b/codex-rs/thread-store/Cargo.toml index a8f22fbfe5..3698332b92 100644 --- a/codex-rs/thread-store/Cargo.toml +++ b/codex-rs/thread-store/Cargo.toml @@ -8,12 +8,32 @@ version.workspace = true name = "codex_thread_store" path = "src/lib.rs" +[[example]] +name = "generate-proto" +path = "examples/generate-proto.rs" + [lints] workspace = true [dependencies] async-trait = { workspace = true } chrono = { workspace = true, features = ["serde"] } +codex-git-utils = { workspace = true } codex-protocol = { workspace = true } +codex-rollout = { workspace = true } +prost = "0.14.3" serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } thiserror = { workspace = true } +tonic = { workspace = true } +tonic-prost = { workspace = true } + +[dev-dependencies] +codex-state = { workspace = true } +pretty_assertions = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +tokio-stream = { workspace = true, features = ["net"] } +tonic = { workspace = true, features = ["router", "transport"] } +tonic-prost-build = { version = "=0.14.3", default-features = false, features = ["transport"] } +uuid = { workspace = true } diff --git a/codex-rs/thread-store/examples/generate-proto.rs b/codex-rs/thread-store/examples/generate-proto.rs new file mode 100644 index 0000000000..aa159f5bcb --- /dev/null +++ b/codex-rs/thread-store/examples/generate-proto.rs @@ -0,0 +1,14 @@ +use std::path::PathBuf; + +fn main() -> Result<(), Box> { + let proto_dir = PathBuf::from(std::env::args().nth(1).expect("proto dir")); + let proto_file = proto_dir.join("codex.thread_store.v1.proto"); + + tonic_prost_build::configure() + .build_client(true) + .build_server(true) + .out_dir(&proto_dir) + .compile_protos(&[proto_file], &[proto_dir])?; + + Ok(()) +} diff --git a/codex-rs/thread-store/scripts/generate-proto.sh b/codex-rs/thread-store/scripts/generate-proto.sh new file mode 100755 index 0000000000..4045467cac --- /dev/null +++ b/codex-rs/thread-store/scripts/generate-proto.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd "$script_dir/../../.." && pwd)" +proto_dir="$repo_root/codex-rs/thread-store/src/remote/proto" +generated="$proto_dir/codex.thread_store.v1.rs" +tmpdir="$(mktemp -d)" + +cleanup() { + rm -rf "$tmpdir" +} +trap cleanup EXIT + +( + cd "$repo_root/codex-rs" + CARGO_TARGET_DIR="$tmpdir/target" cargo run \ + -p codex-thread-store \ + --example generate-proto \ + -- "$proto_dir" +) + +if ! sed -n '2p' "$generated" | grep -q 'clippy::trivially_copy_pass_by_ref'; then + { + sed -n '1p' "$generated" + printf '#![allow(clippy::trivially_copy_pass_by_ref)]\n' + sed '1d' "$generated" + } > "$tmpdir/generated.rs" + mv "$tmpdir/generated.rs" "$generated" +fi + +rustfmt --edition 2024 "$generated" + +awk ' + NR == 3 && previous ~ /clippy::trivially_copy_pass_by_ref/ && $0 != "" { print "" } + { print; previous = $0 } +' "$generated" > "$tmpdir/formatted.rs" +mv "$tmpdir/formatted.rs" "$generated" diff --git a/codex-rs/thread-store/src/lib.rs b/codex-rs/thread-store/src/lib.rs index 5525db1394..f8dc94908a 100644 --- a/codex-rs/thread-store/src/lib.rs +++ b/codex-rs/thread-store/src/lib.rs @@ -5,13 +5,17 @@ //! any other backing store. mod error; +mod local; mod recorder; +mod remote; mod store; mod types; pub use error::ThreadStoreError; pub use error::ThreadStoreResult; +pub use local::LocalThreadStore; pub use recorder::ThreadRecorder; +pub use remote::RemoteThreadStore; pub use store::ThreadStore; pub use types::AppendThreadItemsParams; pub use types::ArchiveThreadParams; diff --git a/codex-rs/thread-store/src/local/archive_thread.rs b/codex-rs/thread-store/src/local/archive_thread.rs new file mode 100644 index 0000000000..4fcea6dbac --- /dev/null +++ b/codex-rs/thread-store/src/local/archive_thread.rs @@ -0,0 +1,170 @@ +use chrono::Utc; +use codex_rollout::find_thread_path_by_id_str; + +use super::LocalThreadStore; +use super::helpers::matching_rollout_file_name; +use super::helpers::scoped_rollout_path; +use crate::ArchiveThreadParams; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; + +pub(super) async fn archive_thread( + store: &LocalThreadStore, + params: ArchiveThreadParams, +) -> ThreadStoreResult<()> { + let thread_id = params.thread_id; + let rollout_path = + find_thread_path_by_id_str(store.config.codex_home.as_path(), &thread_id.to_string()) + .await + .map_err(|err| ThreadStoreError::InvalidRequest { + message: format!("failed to locate thread id {thread_id}: {err}"), + })? + .ok_or_else(|| ThreadStoreError::InvalidRequest { + message: format!("no rollout found for thread id {thread_id}"), + })?; + + let canonical_rollout_path = scoped_rollout_path( + store.config.codex_home.join(codex_rollout::SESSIONS_SUBDIR), + rollout_path.as_path(), + "sessions", + )?; + let file_name = matching_rollout_file_name( + canonical_rollout_path.as_path(), + thread_id, + rollout_path.as_path(), + )?; + + let archive_folder = store + .config + .codex_home + .join(codex_rollout::ARCHIVED_SESSIONS_SUBDIR); + std::fs::create_dir_all(&archive_folder).map_err(|err| ThreadStoreError::Internal { + message: format!("failed to archive thread: {err}"), + })?; + let archived_path = archive_folder.join(&file_name); + std::fs::rename(&canonical_rollout_path, &archived_path).map_err(|err| { + ThreadStoreError::Internal { + message: format!("failed to archive thread: {err}"), + } + })?; + + if let Some(ctx) = codex_rollout::state_db::get_state_db(&store.config).await { + let _ = ctx + .mark_archived(thread_id, archived_path.as_path(), Utc::now()) + .await; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use chrono::Utc; + use codex_protocol::ThreadId; + use codex_protocol::protocol::SessionSource; + use codex_rollout::ARCHIVED_SESSIONS_SUBDIR; + use pretty_assertions::assert_eq; + use tempfile::TempDir; + use uuid::Uuid; + + use super::*; + use crate::ListThreadsParams; + use crate::ThreadSortKey; + use crate::ThreadStore; + use crate::local::LocalThreadStore; + use crate::local::test_support::test_config; + use crate::local::test_support::write_session_file; + + #[tokio::test] + async fn archive_thread_moves_rollout_to_archived_collection() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path())); + let uuid = Uuid::from_u128(201); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + let active_path = + write_session_file(home.path(), "2025-01-03T12-00-00", uuid).expect("session file"); + + store + .archive_thread(ArchiveThreadParams { thread_id }) + .await + .expect("archive thread"); + + assert!(!active_path.exists()); + let archived_path = home + .path() + .join(ARCHIVED_SESSIONS_SUBDIR) + .join(active_path.file_name().expect("file name")); + assert!(archived_path.exists()); + + let archived = store + .list_threads(ListThreadsParams { + page_size: 10, + cursor: None, + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived: true, + search_term: None, + }) + .await + .expect("archived listing"); + assert_eq!(archived.items.len(), 1); + assert_eq!(archived.items[0].thread_id, thread_id); + assert_eq!(archived.items[0].rollout_path, Some(archived_path)); + assert_eq!( + archived.items[0].archived_at, + Some(archived.items[0].updated_at) + ); + } + + #[tokio::test] + async fn archive_thread_updates_sqlite_metadata_when_present() { + let home = TempDir::new().expect("temp dir"); + let config = test_config(home.path()); + let store = LocalThreadStore::new(config.clone()); + let uuid = Uuid::from_u128(202); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + let active_path = + write_session_file(home.path(), "2025-01-03T12-00-00", uuid).expect("session file"); + let runtime = codex_state::StateRuntime::init( + home.path().to_path_buf(), + config.model_provider_id.clone(), + ) + .await + .expect("state db should initialize"); + runtime + .mark_backfill_complete(/*last_watermark*/ None) + .await + .expect("backfill should be complete"); + let mut builder = codex_state::ThreadMetadataBuilder::new( + thread_id, + active_path.clone(), + Utc::now(), + SessionSource::Cli, + ); + builder.model_provider = Some(config.model_provider_id.clone()); + builder.cwd = home.path().to_path_buf(); + builder.cli_version = Some("test_version".to_string()); + let metadata = builder.build(config.model_provider_id.as_str()); + runtime + .upsert_thread(&metadata) + .await + .expect("state db upsert should succeed"); + + store + .archive_thread(ArchiveThreadParams { thread_id }) + .await + .expect("archive thread"); + + let archived_path = home + .path() + .join(ARCHIVED_SESSIONS_SUBDIR) + .join(active_path.file_name().expect("file name")); + let updated = runtime + .get_thread(thread_id) + .await + .expect("state db read should succeed") + .expect("thread metadata should exist"); + assert_eq!(updated.rollout_path, archived_path); + assert!(updated.archived_at.is_some()); + } +} diff --git a/codex-rs/thread-store/src/local/helpers.rs b/codex-rs/thread-store/src/local/helpers.rs new file mode 100644 index 0000000000..e28ecc6612 --- /dev/null +++ b/codex-rs/thread-store/src/local/helpers.rs @@ -0,0 +1,168 @@ +use std::ffi::OsStr; +use std::fs::FileTimes; +use std::fs::OpenOptions; +use std::path::Path; +use std::path::PathBuf; +use std::time::SystemTime; + +use chrono::DateTime; +use chrono::Utc; +use codex_git_utils::GitSha; +use codex_protocol::ThreadId; +use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::GitInfo; +use codex_protocol::protocol::SandboxPolicy; +use codex_protocol::protocol::SessionSource; +use codex_rollout::ThreadItem; + +use crate::StoredThread; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; + +pub(super) fn scoped_rollout_path( + root: PathBuf, + rollout_path: &Path, + root_name: &str, +) -> ThreadStoreResult { + let canonical_root = + std::fs::canonicalize(&root).map_err(|err| ThreadStoreError::Internal { + message: format!( + "failed to resolve {root_name} directory `{}`: {err}", + root.display() + ), + })?; + let canonical_rollout_path = + std::fs::canonicalize(rollout_path).map_err(|_| ThreadStoreError::InvalidRequest { + message: format!( + "rollout path `{}` must be in {root_name} directory", + rollout_path.display() + ), + })?; + if canonical_rollout_path.starts_with(&canonical_root) { + Ok(canonical_rollout_path) + } else { + Err(ThreadStoreError::InvalidRequest { + message: format!( + "rollout path `{}` must be in {root_name} directory", + rollout_path.display() + ), + }) + } +} + +pub(super) fn matching_rollout_file_name( + rollout_path: &Path, + thread_id: ThreadId, + display_path: &Path, +) -> ThreadStoreResult { + let Some(file_name) = rollout_path.file_name().map(OsStr::to_owned) else { + return Err(ThreadStoreError::InvalidRequest { + message: format!( + "rollout path `{}` missing file name", + display_path.display() + ), + }); + }; + let required_suffix = format!("{thread_id}.jsonl"); + if file_name + .to_string_lossy() + .ends_with(required_suffix.as_str()) + { + Ok(file_name) + } else { + Err(ThreadStoreError::InvalidRequest { + message: format!( + "rollout path `{}` does not match thread id {thread_id}", + display_path.display() + ), + }) + } +} + +pub(super) fn touch_modified_time(path: &Path) -> std::io::Result<()> { + let times = FileTimes::new().set_modified(SystemTime::now()); + OpenOptions::new().append(true).open(path)?.set_times(times) +} + +pub(super) fn stored_thread_from_rollout_item( + item: ThreadItem, + archived: bool, + default_provider: &str, +) -> Option { + let thread_id = item + .thread_id + .or_else(|| thread_id_from_rollout_path(item.path.as_path()))?; + let created_at = parse_rfc3339(item.created_at.as_deref()).unwrap_or_else(Utc::now); + let updated_at = parse_rfc3339(item.updated_at.as_deref()).unwrap_or(created_at); + let archived_at = archived.then_some(updated_at); + let git_info = git_info_from_parts( + item.git_sha.clone(), + item.git_branch.clone(), + item.git_origin_url.clone(), + ); + let source = item.source.unwrap_or(SessionSource::Unknown); + let preview = item.first_user_message.clone().unwrap_or_default(); + + Some(StoredThread { + thread_id, + rollout_path: Some(item.path), + forked_from_id: None, + preview, + name: None, + model_provider: item + .model_provider + .filter(|provider| !provider.is_empty()) + .unwrap_or_else(|| default_provider.to_string()), + model: None, + reasoning_effort: None, + created_at, + updated_at, + archived_at, + cwd: item.cwd.unwrap_or_default(), + cli_version: item.cli_version.unwrap_or_default(), + source, + agent_nickname: item.agent_nickname, + agent_role: item.agent_role, + agent_path: None, + git_info, + approval_mode: AskForApproval::OnRequest, + sandbox_policy: SandboxPolicy::new_read_only_policy(), + token_usage: None, + first_user_message: item.first_user_message, + history: None, + }) +} + +fn parse_rfc3339(value: Option<&str>) -> Option> { + DateTime::parse_from_rfc3339(value?) + .ok() + .map(|dt| dt.with_timezone(&Utc)) +} + +fn git_info_from_parts( + sha: Option, + branch: Option, + origin_url: Option, +) -> Option { + if sha.is_none() && branch.is_none() && origin_url.is_none() { + return None; + } + Some(GitInfo { + commit_hash: sha.as_deref().map(GitSha::new), + branch, + repository_url: origin_url, + }) +} + +fn thread_id_from_rollout_path(path: &Path) -> Option { + let file_name = path.file_name()?.to_str()?; + let stem = file_name.strip_suffix(".jsonl")?; + if stem.len() < 37 { + return None; + } + let uuid_start = stem.len().saturating_sub(36); + if !stem[..uuid_start].ends_with('-') { + return None; + } + ThreadId::from_string(&stem[uuid_start..]).ok() +} diff --git a/codex-rs/thread-store/src/local/list_threads.rs b/codex-rs/thread-store/src/local/list_threads.rs new file mode 100644 index 0000000000..35c6f3de43 --- /dev/null +++ b/codex-rs/thread-store/src/local/list_threads.rs @@ -0,0 +1,319 @@ +use codex_rollout::RolloutConfig; +use codex_rollout::RolloutRecorder; +use codex_rollout::parse_cursor; + +use super::LocalThreadStore; +use super::helpers::stored_thread_from_rollout_item; +use crate::ListThreadsParams; +use crate::ThreadPage; +use crate::ThreadSortKey; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; + +pub(super) async fn list_threads( + store: &LocalThreadStore, + params: ListThreadsParams, +) -> ThreadStoreResult { + let cursor = params + .cursor + .as_deref() + .map(|cursor| { + parse_cursor(cursor).ok_or_else(|| ThreadStoreError::InvalidRequest { + message: format!("invalid cursor: {cursor}"), + }) + }) + .transpose()?; + let sort_key = match params.sort_key { + ThreadSortKey::CreatedAt => codex_rollout::ThreadSortKey::CreatedAt, + ThreadSortKey::UpdatedAt => codex_rollout::ThreadSortKey::UpdatedAt, + }; + let page = list_rollout_threads(&store.config, ¶ms, cursor.as_ref(), sort_key).await?; + + let next_cursor = page + .next_cursor + .as_ref() + .and_then(|cursor| serde_json::to_value(cursor).ok()) + .and_then(|value| value.as_str().map(str::to_owned)); + let items = page + .items + .into_iter() + .filter_map(|item| { + stored_thread_from_rollout_item( + item, + params.archived, + store.config.model_provider_id.as_str(), + ) + }) + .collect::>(); + + Ok(ThreadPage { items, next_cursor }) +} + +async fn list_rollout_threads( + config: &RolloutConfig, + params: &ListThreadsParams, + cursor: Option<&codex_rollout::Cursor>, + sort_key: codex_rollout::ThreadSortKey, +) -> ThreadStoreResult { + let page = if params.archived { + RolloutRecorder::list_archived_threads( + config, + params.page_size, + cursor, + sort_key, + params.allowed_sources.as_slice(), + params.model_providers.as_deref(), + config.model_provider_id.as_str(), + params.search_term.as_deref(), + ) + .await + } else { + RolloutRecorder::list_threads( + config, + params.page_size, + cursor, + sort_key, + params.allowed_sources.as_slice(), + params.model_providers.as_deref(), + config.model_provider_id.as_str(), + params.search_term.as_deref(), + ) + .await + }; + page.map_err(|err| ThreadStoreError::Internal { + message: format!("failed to list threads: {err}"), + }) +} + +#[cfg(test)] +mod tests { + use chrono::Utc; + use codex_protocol::ThreadId; + use codex_protocol::protocol::SessionSource; + use pretty_assertions::assert_eq; + use std::fs; + use tempfile::TempDir; + use uuid::Uuid; + + use super::*; + use crate::ThreadStore; + use crate::local::LocalThreadStore; + use crate::local::test_support::test_config; + use crate::local::test_support::write_archived_session_file; + use crate::local::test_support::write_session_file; + use crate::local::test_support::write_session_file_with; + + #[tokio::test] + async fn list_threads_uses_default_provider_when_rollout_omits_provider() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path())); + write_session_file_with( + home.path(), + home.path().join("sessions/2025/01/03"), + "2025-01-03T12-00-00", + Uuid::from_u128(102), + "Hello from user", + /*model_provider*/ None, + ) + .expect("session file"); + + let page = store + .list_threads(ListThreadsParams { + page_size: 10, + cursor: None, + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived: false, + search_term: None, + }) + .await + .expect("thread listing"); + + assert_eq!(page.items.len(), 1); + assert_eq!(page.items[0].model_provider, "test-provider"); + } + + #[tokio::test] + async fn list_threads_preserves_sqlite_title_search_results() { + let home = TempDir::new().expect("temp dir"); + let config = test_config(home.path()); + let store = LocalThreadStore::new(config.clone()); + let uuid = Uuid::from_u128(103); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + let rollout_path = home.path().join("rollout-title-search.jsonl"); + fs::write(&rollout_path, "").expect("placeholder rollout file"); + + let runtime = codex_state::StateRuntime::init( + home.path().to_path_buf(), + config.model_provider_id.clone(), + ) + .await + .expect("state db should initialize"); + runtime + .mark_backfill_complete(/*last_watermark*/ None) + .await + .expect("backfill should be complete"); + let created_at = Utc::now(); + let mut builder = codex_state::ThreadMetadataBuilder::new( + thread_id, + rollout_path, + created_at, + SessionSource::Cli, + ); + builder.model_provider = Some(config.model_provider_id.clone()); + builder.cwd = home.path().to_path_buf(); + builder.cli_version = Some("test_version".to_string()); + let mut metadata = builder.build(config.model_provider_id.as_str()); + metadata.title = "needle title".to_string(); + metadata.first_user_message = Some("plain preview".to_string()); + runtime + .upsert_thread(&metadata) + .await + .expect("state db upsert should succeed"); + + let page = store + .list_threads(ListThreadsParams { + page_size: 10, + cursor: None, + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived: false, + search_term: Some("needle".to_string()), + }) + .await + .expect("thread listing"); + + let ids = page + .items + .iter() + .map(|item| item.thread_id) + .collect::>(); + assert_eq!(ids, vec![thread_id]); + assert_eq!( + page.items[0].first_user_message.as_deref(), + Some("plain preview") + ); + } + + #[tokio::test] + async fn list_threads_selects_active_or_archived_collection() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path())); + let active_uuid = Uuid::from_u128(105); + let archived_uuid = Uuid::from_u128(106); + write_session_file(home.path(), "2025-01-03T12-00-00", active_uuid) + .expect("active session file"); + write_archived_session_file(home.path(), "2025-01-03T13-00-00", archived_uuid) + .expect("archived session file"); + + let active = store + .list_threads(ListThreadsParams { + page_size: 10, + cursor: None, + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived: false, + search_term: None, + }) + .await + .expect("active listing"); + let archived = store + .list_threads(ListThreadsParams { + page_size: 10, + cursor: None, + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived: true, + search_term: None, + }) + .await + .expect("archived listing"); + + let active_id = ThreadId::from_string(&active_uuid.to_string()).expect("valid thread id"); + let archived_id = + ThreadId::from_string(&archived_uuid.to_string()).expect("valid thread id"); + assert_eq!( + active + .items + .iter() + .map(|item| item.thread_id) + .collect::>(), + vec![active_id] + ); + assert_eq!( + archived + .items + .iter() + .map(|item| item.thread_id) + .collect::>(), + vec![archived_id] + ); + assert_eq!(active.items[0].archived_at, None); + assert_eq!( + archived.items[0].archived_at, + Some(archived.items[0].updated_at) + ); + } + + #[tokio::test] + async fn list_threads_returns_local_rollout_summary() { + let home = TempDir::new().expect("temp dir"); + let config = test_config(home.path()); + let store = LocalThreadStore::new(config); + let uuid = Uuid::from_u128(101); + let path = + write_session_file(home.path(), "2025-01-03T12-00-00", uuid).expect("session file"); + + let page = store + .list_threads(ListThreadsParams { + page_size: 10, + cursor: None, + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: vec![SessionSource::Cli], + model_providers: Some(vec!["test-provider".to_string()]), + archived: false, + search_term: None, + }) + .await + .expect("thread listing"); + + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + assert_eq!(page.next_cursor, None); + assert_eq!(page.items.len(), 1); + assert_eq!(page.items[0].thread_id, thread_id); + assert_eq!(page.items[0].rollout_path, Some(path)); + assert_eq!(page.items[0].preview, "Hello from user"); + assert_eq!( + page.items[0].first_user_message.as_deref(), + Some("Hello from user") + ); + assert_eq!(page.items[0].model_provider, "test-provider"); + assert_eq!(page.items[0].cli_version, "test_version"); + assert_eq!(page.items[0].source, SessionSource::Cli); + } + + #[tokio::test] + async fn list_threads_rejects_invalid_cursor() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path())); + + let err = store + .list_threads(ListThreadsParams { + page_size: 10, + cursor: Some("not-a-cursor".to_string()), + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived: false, + search_term: None, + }) + .await + .expect_err("invalid cursor should fail"); + + assert!(matches!(err, ThreadStoreError::InvalidRequest { .. })); + } +} diff --git a/codex-rs/thread-store/src/local/mod.rs b/codex-rs/thread-store/src/local/mod.rs new file mode 100644 index 0000000000..81428cac8c --- /dev/null +++ b/codex-rs/thread-store/src/local/mod.rs @@ -0,0 +1,105 @@ +mod archive_thread; +mod helpers; +mod list_threads; +mod read_thread; +mod unarchive_thread; + +#[cfg(test)] +mod test_support; + +use async_trait::async_trait; +use codex_rollout::RolloutConfig; + +use crate::AppendThreadItemsParams; +use crate::ArchiveThreadParams; +use crate::CreateThreadParams; +use crate::ListThreadsParams; +use crate::LoadThreadHistoryParams; +use crate::ReadThreadParams; +use crate::ResumeThreadRecorderParams; +use crate::SetThreadNameParams; +use crate::StoredThread; +use crate::StoredThreadHistory; +use crate::ThreadPage; +use crate::ThreadRecorder; +use crate::ThreadStore; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; +use crate::UpdateThreadMetadataParams; + +/// Local filesystem/SQLite-backed implementation of [`ThreadStore`]. +#[derive(Clone, Debug)] +pub struct LocalThreadStore { + pub(super) config: RolloutConfig, +} + +impl LocalThreadStore { + /// Create a local store from the rollout configuration used by existing local persistence. + pub fn new(config: RolloutConfig) -> Self { + Self { config } + } +} + +#[async_trait] +impl ThreadStore for LocalThreadStore { + async fn create_thread( + &self, + _params: CreateThreadParams, + ) -> ThreadStoreResult> { + unsupported("create_thread") + } + + async fn resume_thread_recorder( + &self, + _params: ResumeThreadRecorderParams, + ) -> ThreadStoreResult> { + unsupported("resume_thread_recorder") + } + + async fn append_items(&self, _params: AppendThreadItemsParams) -> ThreadStoreResult<()> { + unsupported("append_items") + } + + async fn load_history( + &self, + _params: LoadThreadHistoryParams, + ) -> ThreadStoreResult { + unsupported("load_history") + } + + async fn read_thread(&self, params: ReadThreadParams) -> ThreadStoreResult { + read_thread::read_thread(self, params).await + } + + async fn list_threads(&self, params: ListThreadsParams) -> ThreadStoreResult { + list_threads::list_threads(self, params).await + } + + async fn set_thread_name(&self, _params: SetThreadNameParams) -> ThreadStoreResult<()> { + unsupported("set_thread_name") + } + + async fn update_thread_metadata( + &self, + _params: UpdateThreadMetadataParams, + ) -> ThreadStoreResult { + unsupported("update_thread_metadata") + } + + async fn archive_thread(&self, params: ArchiveThreadParams) -> ThreadStoreResult<()> { + archive_thread::archive_thread(self, params).await + } + + async fn unarchive_thread( + &self, + params: ArchiveThreadParams, + ) -> ThreadStoreResult { + unarchive_thread::unarchive_thread(self, params).await + } +} + +fn unsupported(operation: &str) -> ThreadStoreResult { + Err(ThreadStoreError::Internal { + message: format!("local thread store does not implement {operation} in this slice"), + }) +} diff --git a/codex-rs/thread-store/src/local/read_thread.rs b/codex-rs/thread-store/src/local/read_thread.rs new file mode 100644 index 0000000000..189bc1938d --- /dev/null +++ b/codex-rs/thread-store/src/local/read_thread.rs @@ -0,0 +1,138 @@ +use codex_rollout::RolloutRecorder; +use codex_rollout::find_archived_thread_path_by_id_str; +use codex_rollout::find_thread_path_by_id_str; +use codex_rollout::read_thread_item_from_rollout; + +use super::LocalThreadStore; +use super::helpers::stored_thread_from_rollout_item; +use crate::ReadThreadParams; +use crate::StoredThread; +use crate::StoredThreadHistory; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; + +pub(super) async fn read_thread( + store: &LocalThreadStore, + params: ReadThreadParams, +) -> ThreadStoreResult { + let thread_id = params.thread_id; + let path = if params.include_archived { + match find_thread_path_by_id_str(store.config.codex_home.as_path(), &thread_id.to_string()) + .await + .map_err(|err| ThreadStoreError::InvalidRequest { + message: format!("failed to locate thread id {thread_id}: {err}"), + })? { + Some(path) => Some(path), + None => find_archived_thread_path_by_id_str( + store.config.codex_home.as_path(), + &thread_id.to_string(), + ) + .await + .map_err(|err| ThreadStoreError::InvalidRequest { + message: format!("failed to locate archived thread id {thread_id}: {err}"), + })?, + } + } else { + find_thread_path_by_id_str(store.config.codex_home.as_path(), &thread_id.to_string()) + .await + .map_err(|err| ThreadStoreError::InvalidRequest { + message: format!("failed to locate thread id {thread_id}: {err}"), + })? + } + .ok_or_else(|| ThreadStoreError::InvalidRequest { + message: format!("no rollout found for thread id {thread_id}"), + })?; + + let item = read_thread_item_from_rollout(path.clone()) + .await + .ok_or_else(|| ThreadStoreError::Internal { + message: format!("failed to read thread {}", path.display()), + })?; + let archived = item.path.starts_with( + store + .config + .codex_home + .join(codex_rollout::ARCHIVED_SESSIONS_SUBDIR), + ); + let mut thread = + stored_thread_from_rollout_item(item, archived, store.config.model_provider_id.as_str()) + .ok_or_else(|| ThreadStoreError::Internal { + message: format!("failed to read thread id from {}", path.display()), + })?; + if params.include_history { + let (items, _, _) = RolloutRecorder::load_rollout_items(path.as_path()) + .await + .map_err(|err| ThreadStoreError::Internal { + message: format!("failed to load thread history {}: {err}", path.display()), + })?; + thread.history = Some(StoredThreadHistory { thread_id, items }); + } + Ok(thread) +} + +#[cfg(test)] +mod tests { + use codex_protocol::ThreadId; + use pretty_assertions::assert_eq; + use tempfile::TempDir; + use uuid::Uuid; + + use super::*; + use crate::ThreadStore; + use crate::local::LocalThreadStore; + use crate::local::test_support::test_config; + use crate::local::test_support::write_session_file; + + #[tokio::test] + async fn read_thread_returns_active_rollout_summary() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path())); + let uuid = Uuid::from_u128(205); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + let active_path = + write_session_file(home.path(), "2025-01-03T12-00-00", uuid).expect("session file"); + + let thread = store + .read_thread(ReadThreadParams { + thread_id, + include_archived: false, + include_history: true, + }) + .await + .expect("read thread"); + + assert_eq!(thread.thread_id, thread_id); + assert_eq!(thread.rollout_path, Some(active_path)); + assert_eq!(thread.archived_at, None); + assert_eq!(thread.preview, "Hello from user"); + assert_eq!( + thread.history.expect("history should load").thread_id, + thread_id + ); + } + + #[tokio::test] + async fn read_thread_fails_without_rollout() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path())); + let uuid = Uuid::from_u128(206); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + + let err = store + .read_thread(ReadThreadParams { + thread_id, + include_archived: false, + include_history: false, + }) + .await + .expect_err("read should fail without rollout"); + + let ThreadStoreError::InvalidRequest { message } = err else { + panic!("expected invalid request error"); + }; + assert_eq!( + message, + format!("no rollout found for thread id {thread_id}") + ); + } +} diff --git a/codex-rs/thread-store/src/local/test_support.rs b/codex-rs/thread-store/src/local/test_support.rs new file mode 100644 index 0000000000..d8ddcf8dde --- /dev/null +++ b/codex-rs/thread-store/src/local/test_support.rs @@ -0,0 +1,87 @@ +use std::fs; +use std::io::Write; +use std::path::Path; +use std::path::PathBuf; + +use codex_rollout::ARCHIVED_SESSIONS_SUBDIR; +use codex_rollout::RolloutConfig; +use uuid::Uuid; + +pub(super) fn test_config(codex_home: &Path) -> RolloutConfig { + RolloutConfig { + codex_home: codex_home.to_path_buf(), + sqlite_home: codex_home.to_path_buf(), + cwd: codex_home.to_path_buf(), + model_provider_id: "test-provider".to_string(), + generate_memories: true, + } +} + +pub(super) fn write_session_file(root: &Path, ts: &str, uuid: Uuid) -> std::io::Result { + write_session_file_with( + root, + root.join("sessions/2025/01/03"), + ts, + uuid, + "Hello from user", + Some("test-provider"), + ) +} + +pub(super) fn write_archived_session_file( + root: &Path, + ts: &str, + uuid: Uuid, +) -> std::io::Result { + write_session_file_with( + root, + root.join(ARCHIVED_SESSIONS_SUBDIR), + ts, + uuid, + "Archived user message", + Some("test-provider"), + ) +} + +pub(super) fn write_session_file_with( + root: &Path, + day_dir: PathBuf, + ts: &str, + uuid: Uuid, + first_user_message: &str, + model_provider: Option<&str>, +) -> std::io::Result { + fs::create_dir_all(&day_dir)?; + let path = day_dir.join(format!("rollout-{ts}-{uuid}.jsonl")); + let mut file = fs::File::create(&path)?; + let meta = serde_json::json!({ + "timestamp": ts, + "type": "session_meta", + "payload": { + "id": uuid, + "timestamp": ts, + "cwd": root, + "originator": "test_originator", + "cli_version": "test_version", + "source": "cli", + "model_provider": model_provider, + "git": { + "commit_hash": "abcdef", + "branch": "main", + "repository_url": "https://example.com/repo.git" + } + }, + }); + writeln!(file, "{meta}")?; + let user_event = serde_json::json!({ + "timestamp": ts, + "type": "event_msg", + "payload": { + "type": "user_message", + "message": first_user_message, + "kind": "plain", + }, + }); + writeln!(file, "{user_event}")?; + Ok(path) +} diff --git a/codex-rs/thread-store/src/local/unarchive_thread.rs b/codex-rs/thread-store/src/local/unarchive_thread.rs new file mode 100644 index 0000000000..17e484bace --- /dev/null +++ b/codex-rs/thread-store/src/local/unarchive_thread.rs @@ -0,0 +1,198 @@ +use codex_rollout::find_archived_thread_path_by_id_str; +use codex_rollout::read_thread_item_from_rollout; +use codex_rollout::rollout_date_parts; + +use super::LocalThreadStore; +use super::helpers::matching_rollout_file_name; +use super::helpers::scoped_rollout_path; +use super::helpers::stored_thread_from_rollout_item; +use super::helpers::touch_modified_time; +use crate::ArchiveThreadParams; +use crate::StoredThread; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; + +pub(super) async fn unarchive_thread( + store: &LocalThreadStore, + params: ArchiveThreadParams, +) -> ThreadStoreResult { + let thread_id = params.thread_id; + let archived_path = find_archived_thread_path_by_id_str( + store.config.codex_home.as_path(), + &thread_id.to_string(), + ) + .await + .map_err(|err| ThreadStoreError::InvalidRequest { + message: format!("failed to locate archived thread id {thread_id}: {err}"), + })? + .ok_or_else(|| ThreadStoreError::InvalidRequest { + message: format!("no archived rollout found for thread id {thread_id}"), + })?; + + let canonical_archived_path = scoped_rollout_path( + store + .config + .codex_home + .join(codex_rollout::ARCHIVED_SESSIONS_SUBDIR), + archived_path.as_path(), + "archived", + )?; + let file_name = matching_rollout_file_name( + canonical_archived_path.as_path(), + thread_id, + archived_path.as_path(), + )?; + let Some((year, month, day)) = rollout_date_parts(&file_name) else { + return Err(ThreadStoreError::InvalidRequest { + message: format!( + "rollout path `{}` missing filename timestamp", + archived_path.display() + ), + }); + }; + + let dest_dir = store + .config + .codex_home + .join(codex_rollout::SESSIONS_SUBDIR) + .join(year) + .join(month) + .join(day); + std::fs::create_dir_all(&dest_dir).map_err(|err| ThreadStoreError::Internal { + message: format!("failed to unarchive thread: {err}"), + })?; + let restored_path = dest_dir.join(&file_name); + std::fs::rename(&canonical_archived_path, &restored_path).map_err(|err| { + ThreadStoreError::Internal { + message: format!("failed to unarchive thread: {err}"), + } + })?; + touch_modified_time(restored_path.as_path()).map_err(|err| ThreadStoreError::Internal { + message: format!("failed to update unarchived thread timestamp: {err}"), + })?; + + if let Some(ctx) = codex_rollout::state_db::get_state_db(&store.config).await { + let _ = ctx + .mark_unarchived(thread_id, restored_path.as_path()) + .await; + } + + let item = read_thread_item_from_rollout(restored_path.clone()) + .await + .ok_or_else(|| ThreadStoreError::Internal { + message: format!( + "failed to read unarchived thread {}", + restored_path.display() + ), + })?; + stored_thread_from_rollout_item( + item, + /*archived*/ false, + store.config.model_provider_id.as_str(), + ) + .ok_or_else(|| ThreadStoreError::Internal { + message: format!( + "failed to read unarchived thread id from {}", + restored_path.display() + ), + }) +} + +#[cfg(test)] +mod tests { + use chrono::Utc; + use codex_protocol::ThreadId; + use codex_protocol::protocol::SessionSource; + use pretty_assertions::assert_eq; + use tempfile::TempDir; + use uuid::Uuid; + + use super::*; + use crate::ThreadStore; + use crate::local::LocalThreadStore; + use crate::local::test_support::test_config; + use crate::local::test_support::write_archived_session_file; + + #[tokio::test] + async fn unarchive_thread_restores_rollout_and_returns_updated_thread() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path())); + let uuid = Uuid::from_u128(203); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + let archived_path = write_archived_session_file(home.path(), "2025-01-03T13-00-00", uuid) + .expect("archived session file"); + + let thread = store + .unarchive_thread(ArchiveThreadParams { thread_id }) + .await + .expect("unarchive thread"); + + assert!(!archived_path.exists()); + let restored_path = home + .path() + .join("sessions/2025/01/03") + .join(archived_path.file_name().expect("file name")); + assert!(restored_path.exists()); + assert_eq!(thread.thread_id, thread_id); + assert_eq!(thread.rollout_path, Some(restored_path)); + assert_eq!(thread.archived_at, None); + assert_eq!(thread.preview, "Archived user message"); + assert_eq!( + thread.first_user_message.as_deref(), + Some("Archived user message") + ); + } + + #[tokio::test] + async fn unarchive_thread_updates_sqlite_metadata_when_present() { + let home = TempDir::new().expect("temp dir"); + let config = test_config(home.path()); + let store = LocalThreadStore::new(config.clone()); + let uuid = Uuid::from_u128(204); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + let archived_path = write_archived_session_file(home.path(), "2025-01-03T13-00-00", uuid) + .expect("archived session file"); + let runtime = codex_state::StateRuntime::init( + home.path().to_path_buf(), + config.model_provider_id.clone(), + ) + .await + .expect("state db should initialize"); + runtime + .mark_backfill_complete(/*last_watermark*/ None) + .await + .expect("backfill should be complete"); + let mut builder = codex_state::ThreadMetadataBuilder::new( + thread_id, + archived_path.clone(), + Utc::now(), + SessionSource::Cli, + ); + builder.model_provider = Some(config.model_provider_id.clone()); + builder.cwd = home.path().to_path_buf(); + builder.cli_version = Some("test_version".to_string()); + let mut metadata = builder.build(config.model_provider_id.as_str()); + metadata.archived_at = Some(metadata.updated_at); + runtime + .upsert_thread(&metadata) + .await + .expect("state db upsert should succeed"); + + store + .unarchive_thread(ArchiveThreadParams { thread_id }) + .await + .expect("unarchive thread"); + + let restored_path = home + .path() + .join("sessions/2025/01/03") + .join(archived_path.file_name().expect("file name")); + let updated = runtime + .get_thread(thread_id) + .await + .expect("state db read should succeed") + .expect("thread metadata should exist"); + assert_eq!(updated.rollout_path, restored_path); + assert_eq!(updated.archived_at, None); + } +} diff --git a/codex-rs/thread-store/src/remote/AGENTS.md b/codex-rs/thread-store/src/remote/AGENTS.md new file mode 100644 index 0000000000..b2b2b6417b --- /dev/null +++ b/codex-rs/thread-store/src/remote/AGENTS.md @@ -0,0 +1,13 @@ +# Remote Thread Store + +- The Rust protobuf output in `proto/codex.thread_store.v1.rs` is checked in. +- Do not add build-time protobuf generation to `codex-thread-store` unless the Bazel/Cargo story is intentionally changed. +- When `proto/codex.thread_store.v1.proto` changes, regenerate the Rust file manually and include both files in the same commit. + +Run this from the repository root: + +```sh +./codex-rs/thread-store/scripts/generate-proto.sh +``` + +The command requires `protoc` to be available on `PATH`. diff --git a/codex-rs/thread-store/src/remote/helpers.rs b/codex-rs/thread-store/src/remote/helpers.rs new file mode 100644 index 0000000000..c02889e168 --- /dev/null +++ b/codex-rs/thread-store/src/remote/helpers.rs @@ -0,0 +1,255 @@ +use std::path::PathBuf; +use std::str::FromStr; + +use chrono::DateTime; +use chrono::Utc; +use codex_git_utils::GitSha; +use codex_protocol::AgentPath; +use codex_protocol::ThreadId; +use codex_protocol::openai_models::ReasoningEffort; +use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::GitInfo; +use codex_protocol::protocol::SandboxPolicy; +use codex_protocol::protocol::SessionSource; +use codex_protocol::protocol::SubAgentSource; + +use super::proto; +use crate::StoredThread; +use crate::ThreadSortKey; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; + +pub(super) fn remote_status_to_error(status: tonic::Status) -> ThreadStoreError { + match status.code() { + tonic::Code::InvalidArgument => ThreadStoreError::InvalidRequest { + message: status.message().to_string(), + }, + tonic::Code::AlreadyExists | tonic::Code::FailedPrecondition | tonic::Code::Aborted => { + ThreadStoreError::Conflict { + message: status.message().to_string(), + } + } + _ => ThreadStoreError::Internal { + message: format!("remote thread store request failed: {status}"), + }, + } +} + +pub(super) fn proto_sort_key(sort_key: ThreadSortKey) -> proto::ThreadSortKey { + match sort_key { + ThreadSortKey::CreatedAt => proto::ThreadSortKey::CreatedAt, + ThreadSortKey::UpdatedAt => proto::ThreadSortKey::UpdatedAt, + } +} + +pub(super) fn proto_session_source(source: &SessionSource) -> proto::SessionSource { + match source { + SessionSource::Cli => proto_source(proto::SessionSourceKind::Cli), + SessionSource::VSCode => proto_source(proto::SessionSourceKind::Vscode), + SessionSource::Exec => proto_source(proto::SessionSourceKind::Exec), + SessionSource::Mcp => proto_source(proto::SessionSourceKind::AppServer), + SessionSource::Custom(custom) => proto::SessionSource { + kind: proto::SessionSourceKind::Custom.into(), + custom: Some(custom.clone()), + ..Default::default() + }, + SessionSource::SubAgent(SubAgentSource::Review) => { + proto_source(proto::SessionSourceKind::SubAgentReview) + } + SessionSource::SubAgent(SubAgentSource::Compact) => { + proto_source(proto::SessionSourceKind::SubAgentCompact) + } + SessionSource::SubAgent(SubAgentSource::ThreadSpawn { + parent_thread_id, + depth, + agent_path, + agent_nickname, + agent_role, + }) => proto::SessionSource { + kind: proto::SessionSourceKind::SubAgentThreadSpawn.into(), + sub_agent_parent_thread_id: Some(parent_thread_id.to_string()), + sub_agent_depth: Some(*depth), + sub_agent_path: agent_path.as_ref().map(|path| path.as_str().to_string()), + sub_agent_nickname: agent_nickname.clone(), + sub_agent_role: agent_role.clone(), + ..Default::default() + }, + SessionSource::SubAgent(SubAgentSource::MemoryConsolidation) => { + proto_source(proto::SessionSourceKind::SubAgentMemoryConsolidation) + } + SessionSource::SubAgent(SubAgentSource::Other(other)) => proto::SessionSource { + kind: proto::SessionSourceKind::SubAgentOther.into(), + sub_agent_other: Some(other.clone()), + ..Default::default() + }, + SessionSource::Unknown => proto_source(proto::SessionSourceKind::Unknown), + } +} + +fn proto_source(kind: proto::SessionSourceKind) -> proto::SessionSource { + proto::SessionSource { + kind: kind.into(), + ..Default::default() + } +} + +pub(super) fn stored_thread_from_proto( + thread: proto::StoredThread, +) -> ThreadStoreResult { + // Keep this mapping boring: the proto mirrors StoredThread for remote-readable + // summary fields, except for Rust domain types that cross gRPC as stable scalar + // values. Local-only fields such as rollout_path intentionally stay local. + let source = thread + .source + .as_ref() + .map(session_source_from_proto) + .transpose()? + .unwrap_or(SessionSource::Unknown); + let thread_id = ThreadId::from_string(&thread.thread_id).map_err(|err| { + ThreadStoreError::InvalidRequest { + message: format!("remote thread store returned invalid thread_id: {err}"), + } + })?; + let forked_from_id = thread + .forked_from_id + .as_deref() + .map(ThreadId::from_string) + .transpose() + .map_err(|err| ThreadStoreError::InvalidRequest { + message: format!("remote thread store returned invalid forked_from_id: {err}"), + })?; + + Ok(StoredThread { + thread_id, + rollout_path: None, + forked_from_id, + preview: thread.preview, + name: thread.name, + model_provider: thread.model_provider, + model: thread.model, + reasoning_effort: thread + .reasoning_effort + .as_deref() + .map(parse_reasoning_effort) + .transpose()?, + created_at: datetime_from_unix(thread.created_at)?, + updated_at: datetime_from_unix(thread.updated_at)?, + archived_at: thread.archived_at.map(datetime_from_unix).transpose()?, + cwd: PathBuf::from(thread.cwd), + cli_version: thread.cli_version, + source, + agent_nickname: thread.agent_nickname, + agent_role: thread.agent_role, + agent_path: thread.agent_path, + git_info: thread.git_info.map(git_info_from_proto), + approval_mode: AskForApproval::OnRequest, + sandbox_policy: SandboxPolicy::new_read_only_policy(), + token_usage: None, + first_user_message: thread.first_user_message, + history: None, + }) +} + +#[cfg(test)] +pub(super) fn stored_thread_to_proto(thread: StoredThread) -> proto::StoredThread { + proto::StoredThread { + thread_id: thread.thread_id.to_string(), + forked_from_id: thread.forked_from_id.map(|thread_id| thread_id.to_string()), + preview: thread.preview, + name: thread.name, + model_provider: thread.model_provider, + model: thread.model, + created_at: thread.created_at.timestamp(), + updated_at: thread.updated_at.timestamp(), + archived_at: thread.archived_at.map(|timestamp| timestamp.timestamp()), + cwd: thread.cwd.to_string_lossy().into_owned(), + cli_version: thread.cli_version, + source: Some(proto_session_source(&thread.source)), + git_info: thread.git_info.map(git_info_to_proto), + agent_nickname: thread.agent_nickname, + agent_role: thread.agent_role, + agent_path: thread.agent_path, + reasoning_effort: thread.reasoning_effort.map(|effort| effort.to_string()), + first_user_message: thread.first_user_message, + } +} + +fn datetime_from_unix(timestamp: i64) -> ThreadStoreResult> { + DateTime::from_timestamp(timestamp, 0).ok_or_else(|| ThreadStoreError::InvalidRequest { + message: format!("remote thread store returned invalid timestamp: {timestamp}"), + }) +} + +fn session_source_from_proto(source: &proto::SessionSource) -> ThreadStoreResult { + let kind = proto::SessionSourceKind::try_from(source.kind).unwrap_or_default(); + Ok(match kind { + proto::SessionSourceKind::Unknown => SessionSource::Unknown, + proto::SessionSourceKind::Cli => SessionSource::Cli, + proto::SessionSourceKind::Vscode => SessionSource::VSCode, + proto::SessionSourceKind::Exec => SessionSource::Exec, + proto::SessionSourceKind::AppServer => SessionSource::Mcp, + proto::SessionSourceKind::Custom => { + SessionSource::Custom(source.custom.clone().unwrap_or_default()) + } + proto::SessionSourceKind::SubAgentReview => SessionSource::SubAgent(SubAgentSource::Review), + proto::SessionSourceKind::SubAgentCompact => { + SessionSource::SubAgent(SubAgentSource::Compact) + } + proto::SessionSourceKind::SubAgentThreadSpawn => { + let parent_thread_id = source + .sub_agent_parent_thread_id + .as_deref() + .map(ThreadId::from_string) + .transpose() + .map_err(|err| ThreadStoreError::InvalidRequest { + message: format!( + "remote thread store returned invalid sub-agent parent thread id: {err}" + ), + })? + .ok_or_else(|| ThreadStoreError::InvalidRequest { + message: "remote thread store omitted sub-agent parent thread id".to_string(), + })?; + SessionSource::SubAgent(SubAgentSource::ThreadSpawn { + parent_thread_id, + depth: source.sub_agent_depth.unwrap_or_default(), + agent_path: source + .sub_agent_path + .clone() + .map(AgentPath::from_string) + .transpose() + .map_err(|message| ThreadStoreError::InvalidRequest { message })?, + agent_nickname: source.sub_agent_nickname.clone(), + agent_role: source.sub_agent_role.clone(), + }) + } + proto::SessionSourceKind::SubAgentMemoryConsolidation => { + SessionSource::SubAgent(SubAgentSource::MemoryConsolidation) + } + proto::SessionSourceKind::SubAgentOther => SessionSource::SubAgent(SubAgentSource::Other( + source.sub_agent_other.clone().unwrap_or_default(), + )), + }) +} + +fn git_info_from_proto(info: proto::GitInfo) -> GitInfo { + GitInfo { + commit_hash: info.sha.as_deref().map(GitSha::new), + branch: info.branch, + repository_url: info.origin_url, + } +} + +#[cfg(test)] +fn git_info_to_proto(info: GitInfo) -> proto::GitInfo { + proto::GitInfo { + sha: info.commit_hash.map(|sha| sha.0), + branch: info.branch, + origin_url: info.repository_url, + } +} + +fn parse_reasoning_effort(value: &str) -> ThreadStoreResult { + ReasoningEffort::from_str(value).map_err(|message| ThreadStoreError::InvalidRequest { + message: format!("remote thread store returned {message}"), + }) +} diff --git a/codex-rs/thread-store/src/remote/list_threads.rs b/codex-rs/thread-store/src/remote/list_threads.rs new file mode 100644 index 0000000000..b94db9dc82 --- /dev/null +++ b/codex-rs/thread-store/src/remote/list_threads.rs @@ -0,0 +1,243 @@ +use super::RemoteThreadStore; +use super::helpers::proto_session_source; +use super::helpers::proto_sort_key; +use super::helpers::remote_status_to_error; +use super::helpers::stored_thread_from_proto; +use super::proto; +use crate::ListThreadsParams; +use crate::ThreadPage; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; + +pub(super) async fn list_threads( + store: &RemoteThreadStore, + params: ListThreadsParams, +) -> ThreadStoreResult { + let request = proto::ListThreadsRequest { + page_size: params + .page_size + .try_into() + .map_err(|_| ThreadStoreError::InvalidRequest { + message: format!("page_size is too large: {}", params.page_size), + })?, + cursor: params.cursor, + sort_key: proto_sort_key(params.sort_key).into(), + allowed_sources: params + .allowed_sources + .iter() + .map(proto_session_source) + .collect(), + model_provider_filter: params + .model_providers + .map(|values| proto::ModelProviderFilter { values }), + archived: params.archived, + search_term: params.search_term, + }; + + let response = store + .client() + .await? + .list_threads(request) + .await + .map_err(remote_status_to_error)? + .into_inner(); + + let items = response + .threads + .into_iter() + .map(stored_thread_from_proto) + .collect::>>()?; + + Ok(ThreadPage { + items, + next_cursor: response.next_cursor, + }) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use codex_protocol::openai_models::ReasoningEffort; + use codex_protocol::protocol::SessionSource; + use pretty_assertions::assert_eq; + use tonic::Request; + use tonic::Response; + use tonic::Status; + use tonic::transport::Server; + + use super::super::helpers::stored_thread_to_proto; + use super::super::proto::thread_store_server; + use super::super::proto::thread_store_server::ThreadStoreServer; + use super::*; + use crate::ThreadSortKey; + use crate::ThreadStore; + + #[derive(Default)] + struct TestServer; + + #[tonic::async_trait] + impl thread_store_server::ThreadStore for TestServer { + async fn list_threads( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + assert_eq!(request.page_size, 2); + assert_eq!(request.cursor.as_deref(), Some("cursor-1")); + assert_eq!( + proto::ThreadSortKey::try_from(request.sort_key), + Ok(proto::ThreadSortKey::UpdatedAt) + ); + assert_eq!(request.archived, true); + assert_eq!(request.search_term.as_deref(), Some("needle")); + assert_eq!( + request.model_provider_filter, + Some(proto::ModelProviderFilter { + values: vec!["openai".to_string()], + }) + ); + assert_eq!(request.allowed_sources.len(), 1); + assert_eq!( + proto::SessionSourceKind::try_from(request.allowed_sources[0].kind), + Ok(proto::SessionSourceKind::Cli) + ); + + Ok(Response::new(proto::ListThreadsResponse { + threads: vec![proto::StoredThread { + thread_id: "11111111-1111-1111-1111-111111111111".to_string(), + forked_from_id: None, + preview: "hello".to_string(), + name: Some("named thread".to_string()), + model_provider: "openai".to_string(), + model: Some("gpt-5".to_string()), + created_at: 100, + updated_at: 200, + archived_at: Some(300), + cwd: "/workspace".to_string(), + cli_version: "1.2.3".to_string(), + source: Some(proto::SessionSource { + kind: proto::SessionSourceKind::Cli.into(), + ..Default::default() + }), + git_info: Some(proto::GitInfo { + sha: Some("abc123".to_string()), + branch: Some("main".to_string()), + origin_url: Some("https://example.test/repo.git".to_string()), + }), + agent_nickname: None, + agent_role: None, + agent_path: None, + reasoning_effort: Some("medium".to_string()), + first_user_message: Some("hello".to_string()), + }], + next_cursor: Some("cursor-2".to_string()), + })) + } + } + + #[tokio::test] + async fn list_threads_calls_remote_service() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind test server"); + let addr = listener.local_addr().expect("test server addr"); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + Server::builder() + .add_service(ThreadStoreServer::new(TestServer)) + .serve_with_incoming_shutdown( + tokio_stream::wrappers::TcpListenerStream::new(listener), + async { + let _ = shutdown_rx.await; + }, + ) + .await + }); + + let store = RemoteThreadStore::new(format!("http://{addr}")); + let page = store + .list_threads(ListThreadsParams { + page_size: 2, + cursor: Some("cursor-1".to_string()), + sort_key: ThreadSortKey::UpdatedAt, + allowed_sources: vec![SessionSource::Cli], + model_providers: Some(vec!["openai".to_string()]), + archived: true, + search_term: Some("needle".to_string()), + }) + .await + .expect("list threads"); + + assert_eq!(page.next_cursor.as_deref(), Some("cursor-2")); + assert_eq!(page.items.len(), 1); + let item = &page.items[0]; + assert_eq!( + item.thread_id.to_string(), + "11111111-1111-1111-1111-111111111111" + ); + assert_eq!(item.name.as_deref(), Some("named thread")); + assert_eq!(item.preview, "hello"); + assert_eq!(item.first_user_message.as_deref(), Some("hello")); + assert_eq!(item.model_provider, "openai"); + assert_eq!(item.model.as_deref(), Some("gpt-5")); + assert_eq!(item.created_at.timestamp(), 100); + assert_eq!(item.updated_at.timestamp(), 200); + assert_eq!(item.archived_at.map(|ts| ts.timestamp()), Some(300)); + assert_eq!(item.cwd, PathBuf::from("/workspace")); + assert_eq!(item.cli_version, "1.2.3"); + assert_eq!(item.source, SessionSource::Cli); + assert_eq!(item.reasoning_effort, Some(ReasoningEffort::Medium)); + assert_eq!( + item.git_info.as_ref().and_then(|git| git.branch.as_deref()), + Some("main") + ); + + let _ = shutdown_tx.send(()); + server.await.expect("join server").expect("server"); + } + + #[test] + fn stored_thread_proto_roundtrips_through_domain_type() { + let thread = proto::StoredThread { + thread_id: "11111111-1111-1111-1111-111111111111".to_string(), + forked_from_id: Some("22222222-2222-2222-2222-222222222222".to_string()), + preview: "preview text".to_string(), + name: Some("named thread".to_string()), + model_provider: "openai".to_string(), + model: Some("gpt-5".to_string()), + created_at: 100, + updated_at: 200, + archived_at: Some(300), + cwd: "/workspace/project".to_string(), + cli_version: "1.2.3".to_string(), + source: Some(proto::SessionSource { + kind: proto::SessionSourceKind::SubAgentThreadSpawn.into(), + sub_agent_parent_thread_id: Some( + "33333333-3333-3333-3333-333333333333".to_string(), + ), + sub_agent_depth: Some(2), + sub_agent_path: Some("/root/review/backend".to_string()), + sub_agent_nickname: Some("Navigator".to_string()), + sub_agent_role: Some("explorer".to_string()), + ..Default::default() + }), + git_info: Some(proto::GitInfo { + sha: Some("abc123".to_string()), + branch: Some("main".to_string()), + origin_url: Some("https://example.test/repo.git".to_string()), + }), + agent_nickname: Some("Navigator".to_string()), + agent_role: Some("explorer".to_string()), + agent_path: Some("/root/review/backend".to_string()), + reasoning_effort: Some("high".to_string()), + first_user_message: Some("first message".to_string()), + }; + + let stored = stored_thread_from_proto(thread.clone()).expect("proto to stored thread"); + + assert_eq!(stored.rollout_path, None); + assert!(stored.history.is_none()); + assert_eq!(stored_thread_to_proto(stored), thread); + } +} diff --git a/codex-rs/thread-store/src/remote/mod.rs b/codex-rs/thread-store/src/remote/mod.rs new file mode 100644 index 0000000000..2e8b9e5afb --- /dev/null +++ b/codex-rs/thread-store/src/remote/mod.rs @@ -0,0 +1,112 @@ +mod helpers; +mod list_threads; + +use async_trait::async_trait; + +use crate::AppendThreadItemsParams; +use crate::ArchiveThreadParams; +use crate::CreateThreadParams; +use crate::ListThreadsParams; +use crate::LoadThreadHistoryParams; +use crate::ReadThreadParams; +use crate::ResumeThreadRecorderParams; +use crate::SetThreadNameParams; +use crate::StoredThread; +use crate::StoredThreadHistory; +use crate::ThreadPage; +use crate::ThreadRecorder; +use crate::ThreadStore; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; +use crate::UpdateThreadMetadataParams; +use proto::thread_store_client::ThreadStoreClient; + +#[path = "proto/codex.thread_store.v1.rs"] +mod proto; + +/// gRPC-backed [`ThreadStore`] implementation for deployments whose durable thread data lives +/// outside the app-server process. +#[derive(Clone, Debug)] +pub struct RemoteThreadStore { + endpoint: String, +} + +impl RemoteThreadStore { + pub fn new(endpoint: impl Into) -> Self { + Self { + endpoint: endpoint.into(), + } + } + + async fn client(&self) -> ThreadStoreResult> { + ThreadStoreClient::connect(self.endpoint.clone()) + .await + .map_err(|err| ThreadStoreError::Internal { + message: format!("failed to connect to remote thread store: {err}"), + }) + } +} + +#[async_trait] +impl ThreadStore for RemoteThreadStore { + async fn create_thread( + &self, + _params: CreateThreadParams, + ) -> ThreadStoreResult> { + Err(not_implemented("create_thread")) + } + + async fn resume_thread_recorder( + &self, + _params: ResumeThreadRecorderParams, + ) -> ThreadStoreResult> { + Err(not_implemented("resume_thread_recorder")) + } + + async fn append_items(&self, _params: AppendThreadItemsParams) -> ThreadStoreResult<()> { + Err(not_implemented("append_items")) + } + + async fn load_history( + &self, + _params: LoadThreadHistoryParams, + ) -> ThreadStoreResult { + Err(not_implemented("load_history")) + } + + async fn read_thread(&self, _params: ReadThreadParams) -> ThreadStoreResult { + Err(not_implemented("read_thread")) + } + + async fn list_threads(&self, params: ListThreadsParams) -> ThreadStoreResult { + list_threads::list_threads(self, params).await + } + + async fn set_thread_name(&self, _params: SetThreadNameParams) -> ThreadStoreResult<()> { + Err(not_implemented("set_thread_name")) + } + + async fn update_thread_metadata( + &self, + _params: UpdateThreadMetadataParams, + ) -> ThreadStoreResult { + Err(not_implemented("update_thread_metadata")) + } + + async fn archive_thread(&self, _params: ArchiveThreadParams) -> ThreadStoreResult<()> { + Err(not_implemented("archive_thread")) + } + + async fn unarchive_thread( + &self, + _params: ArchiveThreadParams, + ) -> ThreadStoreResult { + Err(not_implemented("unarchive_thread")) + } +} + +fn not_implemented(method: &str) -> ThreadStoreError { + ThreadStoreError::Internal { + message: format!("remote thread store does not implement {method} yet"), + } +} diff --git a/codex-rs/thread-store/src/remote/proto/codex.thread_store.v1.proto b/codex-rs/thread-store/src/remote/proto/codex.thread_store.v1.proto new file mode 100644 index 0000000000..f217027fcd --- /dev/null +++ b/codex-rs/thread-store/src/remote/proto/codex.thread_store.v1.proto @@ -0,0 +1,86 @@ +syntax = "proto3"; + +package codex.thread_store.v1; + +service ThreadStore { + rpc ListThreads(ListThreadsRequest) returns (ListThreadsResponse); +} + +message ListThreadsRequest { + uint32 page_size = 1; + optional string cursor = 2; + ThreadSortKey sort_key = 3; + repeated SessionSource allowed_sources = 4; + optional ModelProviderFilter model_provider_filter = 5; + bool archived = 6; + optional string search_term = 7; +} + +message ModelProviderFilter { + repeated string values = 1; +} + +enum ThreadSortKey { + THREAD_SORT_KEY_CREATED_AT = 0; + THREAD_SORT_KEY_UPDATED_AT = 1; +} + +message ListThreadsResponse { + repeated StoredThread threads = 1; + optional string next_cursor = 2; +} + +message StoredThread { + // Mirrors Rust's StoredThread. Domain types that are not protobuf-native, + // such as ThreadId, DateTime, and PathBuf, are represented as their + // stable scalar forms on the wire. + string thread_id = 1; + optional string forked_from_id = 2; + string preview = 3; + optional string name = 4; + string model_provider = 5; + optional string model = 6; + int64 created_at = 7; + int64 updated_at = 8; + optional int64 archived_at = 9; + string cwd = 10; + string cli_version = 11; + SessionSource source = 12; + optional GitInfo git_info = 13; + optional string agent_nickname = 14; + optional string agent_role = 15; + optional string agent_path = 16; + optional string reasoning_effort = 17; + optional string first_user_message = 18; +} + +message SessionSource { + SessionSourceKind kind = 1; + optional string custom = 2; + optional string sub_agent_parent_thread_id = 3; + optional int32 sub_agent_depth = 4; + optional string sub_agent_other = 5; + optional string sub_agent_path = 6; + optional string sub_agent_nickname = 7; + optional string sub_agent_role = 8; +} + +enum SessionSourceKind { + SESSION_SOURCE_KIND_UNKNOWN = 0; + SESSION_SOURCE_KIND_CLI = 1; + SESSION_SOURCE_KIND_VSCODE = 2; + SESSION_SOURCE_KIND_EXEC = 3; + SESSION_SOURCE_KIND_APP_SERVER = 4; + SESSION_SOURCE_KIND_CUSTOM = 5; + SESSION_SOURCE_KIND_SUB_AGENT_REVIEW = 6; + SESSION_SOURCE_KIND_SUB_AGENT_COMPACT = 7; + SESSION_SOURCE_KIND_SUB_AGENT_THREAD_SPAWN = 8; + SESSION_SOURCE_KIND_SUB_AGENT_MEMORY_CONSOLIDATION = 9; + SESSION_SOURCE_KIND_SUB_AGENT_OTHER = 10; +} + +message GitInfo { + optional string sha = 1; + optional string branch = 2; + optional string origin_url = 3; +} diff --git a/codex-rs/thread-store/src/remote/proto/codex.thread_store.v1.rs b/codex-rs/thread-store/src/remote/proto/codex.thread_store.v1.rs new file mode 100644 index 0000000000..6d0e410268 --- /dev/null +++ b/codex-rs/thread-store/src/remote/proto/codex.thread_store.v1.rs @@ -0,0 +1,460 @@ +// This file is @generated by prost-build. +#![allow(clippy::trivially_copy_pass_by_ref)] + +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListThreadsRequest { + #[prost(uint32, tag = "1")] + pub page_size: u32, + #[prost(string, optional, tag = "2")] + pub cursor: ::core::option::Option<::prost::alloc::string::String>, + #[prost(enumeration = "ThreadSortKey", tag = "3")] + pub sort_key: i32, + #[prost(message, repeated, tag = "4")] + pub allowed_sources: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag = "5")] + pub model_provider_filter: ::core::option::Option, + #[prost(bool, tag = "6")] + pub archived: bool, + #[prost(string, optional, tag = "7")] + pub search_term: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ModelProviderFilter { + #[prost(string, repeated, tag = "1")] + pub values: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListThreadsResponse { + #[prost(message, repeated, tag = "1")] + pub threads: ::prost::alloc::vec::Vec, + #[prost(string, optional, tag = "2")] + pub next_cursor: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct StoredThread { + /// Mirrors Rust's StoredThread. Domain types that are not protobuf-native, + /// such as ThreadId, DateTime, and PathBuf, are represented as their + /// stable scalar forms on the wire. + #[prost(string, tag = "1")] + pub thread_id: ::prost::alloc::string::String, + #[prost(string, optional, tag = "2")] + pub forked_from_id: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, tag = "3")] + pub preview: ::prost::alloc::string::String, + #[prost(string, optional, tag = "4")] + pub name: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, tag = "5")] + pub model_provider: ::prost::alloc::string::String, + #[prost(string, optional, tag = "6")] + pub model: ::core::option::Option<::prost::alloc::string::String>, + #[prost(int64, tag = "7")] + pub created_at: i64, + #[prost(int64, tag = "8")] + pub updated_at: i64, + #[prost(int64, optional, tag = "9")] + pub archived_at: ::core::option::Option, + #[prost(string, tag = "10")] + pub cwd: ::prost::alloc::string::String, + #[prost(string, tag = "11")] + pub cli_version: ::prost::alloc::string::String, + #[prost(message, optional, tag = "12")] + pub source: ::core::option::Option, + #[prost(message, optional, tag = "13")] + pub git_info: ::core::option::Option, + #[prost(string, optional, tag = "14")] + pub agent_nickname: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "15")] + pub agent_role: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "16")] + pub agent_path: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "17")] + pub reasoning_effort: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "18")] + pub first_user_message: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct SessionSource { + #[prost(enumeration = "SessionSourceKind", tag = "1")] + pub kind: i32, + #[prost(string, optional, tag = "2")] + pub custom: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "3")] + pub sub_agent_parent_thread_id: ::core::option::Option<::prost::alloc::string::String>, + #[prost(int32, optional, tag = "4")] + pub sub_agent_depth: ::core::option::Option, + #[prost(string, optional, tag = "5")] + pub sub_agent_other: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "6")] + pub sub_agent_path: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "7")] + pub sub_agent_nickname: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "8")] + pub sub_agent_role: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GitInfo { + #[prost(string, optional, tag = "1")] + pub sha: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "2")] + pub branch: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "3")] + pub origin_url: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ThreadSortKey { + CreatedAt = 0, + UpdatedAt = 1, +} +impl ThreadSortKey { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::CreatedAt => "THREAD_SORT_KEY_CREATED_AT", + Self::UpdatedAt => "THREAD_SORT_KEY_UPDATED_AT", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "THREAD_SORT_KEY_CREATED_AT" => Some(Self::CreatedAt), + "THREAD_SORT_KEY_UPDATED_AT" => Some(Self::UpdatedAt), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum SessionSourceKind { + Unknown = 0, + Cli = 1, + Vscode = 2, + Exec = 3, + AppServer = 4, + Custom = 5, + SubAgentReview = 6, + SubAgentCompact = 7, + SubAgentThreadSpawn = 8, + SubAgentMemoryConsolidation = 9, + SubAgentOther = 10, +} +impl SessionSourceKind { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unknown => "SESSION_SOURCE_KIND_UNKNOWN", + Self::Cli => "SESSION_SOURCE_KIND_CLI", + Self::Vscode => "SESSION_SOURCE_KIND_VSCODE", + Self::Exec => "SESSION_SOURCE_KIND_EXEC", + Self::AppServer => "SESSION_SOURCE_KIND_APP_SERVER", + Self::Custom => "SESSION_SOURCE_KIND_CUSTOM", + Self::SubAgentReview => "SESSION_SOURCE_KIND_SUB_AGENT_REVIEW", + Self::SubAgentCompact => "SESSION_SOURCE_KIND_SUB_AGENT_COMPACT", + Self::SubAgentThreadSpawn => "SESSION_SOURCE_KIND_SUB_AGENT_THREAD_SPAWN", + Self::SubAgentMemoryConsolidation => { + "SESSION_SOURCE_KIND_SUB_AGENT_MEMORY_CONSOLIDATION" + } + Self::SubAgentOther => "SESSION_SOURCE_KIND_SUB_AGENT_OTHER", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "SESSION_SOURCE_KIND_UNKNOWN" => Some(Self::Unknown), + "SESSION_SOURCE_KIND_CLI" => Some(Self::Cli), + "SESSION_SOURCE_KIND_VSCODE" => Some(Self::Vscode), + "SESSION_SOURCE_KIND_EXEC" => Some(Self::Exec), + "SESSION_SOURCE_KIND_APP_SERVER" => Some(Self::AppServer), + "SESSION_SOURCE_KIND_CUSTOM" => Some(Self::Custom), + "SESSION_SOURCE_KIND_SUB_AGENT_REVIEW" => Some(Self::SubAgentReview), + "SESSION_SOURCE_KIND_SUB_AGENT_COMPACT" => Some(Self::SubAgentCompact), + "SESSION_SOURCE_KIND_SUB_AGENT_THREAD_SPAWN" => Some(Self::SubAgentThreadSpawn), + "SESSION_SOURCE_KIND_SUB_AGENT_MEMORY_CONSOLIDATION" => { + Some(Self::SubAgentMemoryConsolidation) + } + "SESSION_SOURCE_KIND_SUB_AGENT_OTHER" => Some(Self::SubAgentOther), + _ => None, + } + } +} +/// Generated client implementations. +pub mod thread_store_client { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value + )] + use tonic::codegen::http::Uri; + use tonic::codegen::*; + #[derive(Debug, Clone)] + pub struct ThreadStoreClient { + inner: tonic::client::Grpc, + } + impl ThreadStoreClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl ThreadStoreClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> ThreadStoreClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + >>::Error: + Into + std::marker::Send + std::marker::Sync, + { + ThreadStoreClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + pub async fn list_threads( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/codex.thread_store.v1.ThreadStore/ListThreads", + ); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "codex.thread_store.v1.ThreadStore", + "ListThreads", + )); + self.inner.unary(req, path, codec).await + } + } +} +/// Generated server implementations. +pub mod thread_store_server { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value + )] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with ThreadStoreServer. + #[async_trait] + pub trait ThreadStore: std::marker::Send + std::marker::Sync + 'static { + async fn list_threads( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + } + #[derive(Debug)] + pub struct ThreadStoreServer { + inner: Arc, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + impl ThreadStoreServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor(inner: T, interceptor: F) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> for ThreadStoreServer + where + T: ThreadStore, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + match req.uri().path() { + "/codex.thread_store.v1.ThreadStore/ListThreads" => { + #[allow(non_camel_case_types)] + struct ListThreadsSvc(pub Arc); + impl tonic::server::UnaryService for ListThreadsSvc { + type Response = super::ListThreadsResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::list_threads(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ListThreadsSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => Box::pin(async move { + let mut response = http::Response::new(tonic::body::Body::default()); + let headers = response.headers_mut(); + headers.insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers.insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) + }), + } + } + } + impl Clone for ThreadStoreServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "codex.thread_store.v1.ThreadStore"; + impl tonic::server::NamedService for ThreadStoreServer { + const NAME: &'static str = SERVICE_NAME; + } +} diff --git a/codex-rs/thread-store/src/types.rs b/codex-rs/thread-store/src/types.rs index 73f983a466..8d489ea6cb 100644 --- a/codex-rs/thread-store/src/types.rs +++ b/codex-rs/thread-store/src/types.rs @@ -135,6 +135,8 @@ pub struct ThreadPage { pub struct StoredThread { /// Thread id. pub thread_id: ThreadId, + /// Local rollout path when the backing store is filesystem-based. + pub rollout_path: Option, /// Source thread id when this thread was forked from another thread. pub forked_from_id: Option, /// Best available user-facing preview, usually the first user message. @@ -155,6 +157,8 @@ pub struct StoredThread { pub archived_at: Option>, /// Working directory captured for the thread. pub cwd: PathBuf, + /// CLI version captured for the thread. + pub cli_version: String, /// Runtime source for the thread. pub source: SessionSource, /// Optional random nickname for thread-spawn sub-agents. diff --git a/codex-rs/tools/src/code_mode.rs b/codex-rs/tools/src/code_mode.rs index 35d95b271a..e8a73c40a9 100644 --- a/codex-rs/tools/src/code_mode.rs +++ b/codex-rs/tools/src/code_mode.rs @@ -1,7 +1,9 @@ use crate::FreeformTool; use crate::FreeformToolFormat; use crate::JsonSchema; +use crate::ResponsesApiNamespaceTool; use crate::ResponsesApiTool; +use crate::ToolName; use crate::ToolSpec; use codex_code_mode::CodeModeToolKind; use codex_code_mode::ToolDefinition as CodeModeToolDefinition; @@ -9,22 +11,46 @@ use std::collections::BTreeMap; /// Augment tool descriptions with code-mode-specific exec samples. pub fn augment_tool_spec_for_code_mode(spec: ToolSpec) -> ToolSpec { - let Some(description) = code_mode_tool_definition_for_spec(&spec) - .map(codex_code_mode::augment_tool_definition) - .map(|definition| definition.description) - else { - return spec; - }; - match spec { ToolSpec::Function(mut tool) => { + let Some(description) = + augmented_description_for_spec(&ToolSpec::Function(tool.clone())) + else { + return ToolSpec::Function(tool); + }; tool.description = description; ToolSpec::Function(tool) } ToolSpec::Freeform(mut tool) => { + let Some(description) = + augmented_description_for_spec(&ToolSpec::Freeform(tool.clone())) + else { + return ToolSpec::Freeform(tool); + }; tool.description = description; ToolSpec::Freeform(tool) } + ToolSpec::Namespace(mut namespace) => { + for tool in &mut namespace.tools { + match tool { + ResponsesApiNamespaceTool::Function(tool) => { + let tool_name = + ToolName::namespaced(namespace.name.clone(), tool.name.clone()); + let definition = CodeModeToolDefinition { + name: tool_name.display(), + tool_name, + description: tool.description.clone(), + kind: CodeModeToolKind::Function, + input_schema: serde_json::to_value(&tool.parameters).ok(), + output_schema: tool.output_schema.clone(), + }; + tool.description = + codex_code_mode::augment_tool_definition(definition).description; + } + } + } + ToolSpec::Namespace(namespace) + } other => other, } } @@ -42,7 +68,9 @@ pub fn collect_code_mode_tool_definitions<'a>( ) -> Vec { let mut tool_definitions = specs .into_iter() - .filter_map(tool_spec_to_code_mode_tool_definition) + .flat_map(code_mode_tool_definitions_for_spec) + .filter(|definition| codex_code_mode::is_code_mode_nested_tool(&definition.name)) + .map(codex_code_mode::augment_tool_definition) .collect::>(); tool_definitions.sort_by(|left, right| left.name.cmp(&right.name)); tool_definitions.dedup_by(|left, right| left.name == right.name); @@ -54,7 +82,7 @@ pub fn collect_code_mode_exec_prompt_tool_definitions<'a>( ) -> Vec { let mut tool_definitions = specs .into_iter() - .filter_map(code_mode_tool_definition_for_spec) + .flat_map(code_mode_tool_definitions_for_spec) .filter(|definition| codex_code_mode::is_code_mode_nested_tool(&definition.name)) .collect::>(); tool_definitions.sort_by(|left, right| left.name.cmp(&right.name)); @@ -110,7 +138,8 @@ pub fn create_wait_tool() -> ToolSpec { pub fn create_code_mode_tool( enabled_tools: &[CodeModeToolDefinition], namespace_descriptions: &BTreeMap, - code_mode_only_enabled: bool, + code_mode_only: bool, + deferred_tools_available: bool, ) -> ToolSpec { const CODE_MODE_FREEFORM_GRAMMAR: &str = r#" start: pragma_source | plain_source @@ -127,7 +156,8 @@ SOURCE: /[\s\S]+/ description: codex_code_mode::build_exec_tool_description( enabled_tools, namespace_descriptions, - code_mode_only_enabled, + code_mode_only, + deferred_tools_available, ), format: FreeformToolFormat { r#type: "grammar".to_string(), @@ -137,26 +167,61 @@ SOURCE: /[\s\S]+/ }) } +fn augmented_description_for_spec(spec: &ToolSpec) -> Option { + code_mode_tool_definition_for_spec(spec) + .map(codex_code_mode::augment_tool_definition) + .map(|definition| definition.description) +} + fn code_mode_tool_definition_for_spec(spec: &ToolSpec) -> Option { + code_mode_tool_definitions_for_spec(spec).into_iter().next() +} + +fn code_mode_tool_definitions_for_spec(spec: &ToolSpec) -> Vec { match spec { - ToolSpec::Function(tool) => Some(CodeModeToolDefinition { - name: tool.name.clone(), - description: tool.description.clone(), - kind: CodeModeToolKind::Function, - input_schema: serde_json::to_value(&tool.parameters).ok(), - output_schema: tool.output_schema.clone(), - }), - ToolSpec::Freeform(tool) => Some(CodeModeToolDefinition { - name: tool.name.clone(), - description: tool.description.clone(), - kind: CodeModeToolKind::Freeform, - input_schema: None, - output_schema: None, - }), + ToolSpec::Function(tool) => { + let name = tool.name.clone(); + vec![CodeModeToolDefinition { + tool_name: ToolName::plain(name.clone()), + name, + description: tool.description.clone(), + kind: CodeModeToolKind::Function, + input_schema: serde_json::to_value(&tool.parameters).ok(), + output_schema: tool.output_schema.clone(), + }] + } + ToolSpec::Freeform(tool) => { + let name = tool.name.clone(); + vec![CodeModeToolDefinition { + tool_name: ToolName::plain(name.clone()), + name, + description: tool.description.clone(), + kind: CodeModeToolKind::Freeform, + input_schema: None, + output_schema: None, + }] + } + ToolSpec::Namespace(namespace) => namespace + .tools + .iter() + .map(|tool| match tool { + ResponsesApiNamespaceTool::Function(tool) => { + let tool_name = ToolName::namespaced(namespace.name.clone(), tool.name.clone()); + CodeModeToolDefinition { + name: tool_name.display(), + tool_name, + description: tool.description.clone(), + kind: CodeModeToolKind::Function, + input_schema: serde_json::to_value(&tool.parameters).ok(), + output_schema: tool.output_schema.clone(), + } + } + }) + .collect(), ToolSpec::LocalShell {} | ToolSpec::ImageGeneration { .. } | ToolSpec::ToolSearch { .. } - | ToolSpec::WebSearch { .. } => None, + | ToolSpec::WebSearch { .. } => Vec::new(), } } diff --git a/codex-rs/tools/src/code_mode_tests.rs b/codex-rs/tools/src/code_mode_tests.rs index ec30c6e8bf..d7d40cae6e 100644 --- a/codex-rs/tools/src/code_mode_tests.rs +++ b/codex-rs/tools/src/code_mode_tests.rs @@ -7,6 +7,7 @@ use crate::FreeformTool; use crate::FreeformToolFormat; use crate::JsonSchema; use crate::ResponsesApiTool; +use crate::ToolName; use crate::ToolSpec; use pretty_assertions::assert_eq; use serde_json::json; @@ -106,6 +107,7 @@ fn tool_spec_to_code_mode_tool_definition_returns_augmented_nested_tools() { tool_spec_to_code_mode_tool_definition(&spec), Some(codex_code_mode::ToolDefinition { name: "apply_patch".to_string(), + tool_name: ToolName::plain("apply_patch"), description: r#"Apply a patch exec tool declaration: @@ -184,6 +186,7 @@ fn create_wait_tool_matches_expected_spec() { fn create_code_mode_tool_matches_expected_spec() { let enabled_tools = vec![codex_code_mode::ToolDefinition { name: "update_plan".to_string(), + tool_name: ToolName::plain("update_plan"), description: "Update the plan".to_string(), kind: codex_code_mode::CodeModeToolKind::Function, input_schema: None, @@ -194,14 +197,16 @@ fn create_code_mode_tool_matches_expected_spec() { create_code_mode_tool( &enabled_tools, &BTreeMap::new(), - /*code_mode_only_enabled*/ true, + /*code_mode_only*/ true, + /*deferred_tools_available*/ false, ), ToolSpec::Freeform(FreeformTool { name: codex_code_mode::PUBLIC_TOOL_NAME.to_string(), description: codex_code_mode::build_exec_tool_description( &enabled_tools, &BTreeMap::new(), - /*code_mode_only*/ true + /*code_mode_only*/ true, + /*deferred_tools_available*/ false ), format: FreeformToolFormat { r#type: "grammar".to_string(), diff --git a/codex-rs/tools/src/image_detail.rs b/codex-rs/tools/src/image_detail.rs index 94639e61f9..56987e4831 100644 --- a/codex-rs/tools/src/image_detail.rs +++ b/codex-rs/tools/src/image_detail.rs @@ -1,3 +1,4 @@ +use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::ImageDetail; use codex_protocol::openai_models::ModelInfo; @@ -17,6 +18,23 @@ pub fn normalize_output_image_detail( } } +pub fn sanitize_original_image_detail( + can_request_original_image_detail: bool, + items: &mut [FunctionCallOutputContentItem], +) { + if can_request_original_image_detail { + return; + } + + for item in items { + if let FunctionCallOutputContentItem::InputImage { detail, .. } = item + && matches!(detail, Some(ImageDetail::Original)) + { + *detail = None; + } + } +} + #[cfg(test)] #[path = "image_detail_tests.rs"] mod tests; diff --git a/codex-rs/tools/src/image_detail_tests.rs b/codex-rs/tools/src/image_detail_tests.rs index 35d666149f..c1a0f8ca1a 100644 --- a/codex-rs/tools/src/image_detail_tests.rs +++ b/codex-rs/tools/src/image_detail_tests.rs @@ -1,4 +1,5 @@ use super::*; +use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::ImageDetail; use codex_protocol::openai_models::ModelInfo; use pretty_assertions::assert_eq; @@ -73,3 +74,39 @@ fn unsupported_non_original_detail_is_dropped() { None ); } + +#[test] +fn sanitize_original_drops_original_without_support() { + let mut items = vec![ + FunctionCallOutputContentItem::InputText { + text: "header".to_string(), + }, + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,AAA".to_string(), + detail: Some(ImageDetail::Original), + }, + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,BBB".to_string(), + detail: Some(ImageDetail::Low), + }, + ]; + + sanitize_original_image_detail(/*can_request_original_image_detail*/ false, &mut items); + + assert_eq!( + items, + vec![ + FunctionCallOutputContentItem::InputText { + text: "header".to_string(), + }, + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,AAA".to_string(), + detail: None, + }, + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,BBB".to_string(), + detail: Some(ImageDetail::Low), + }, + ] + ); +} diff --git a/codex-rs/tools/src/lib.rs b/codex-rs/tools/src/lib.rs index c310222a0f..b7c6a57d2b 100644 --- a/codex-rs/tools/src/lib.rs +++ b/codex-rs/tools/src/lib.rs @@ -18,7 +18,6 @@ mod responses_api; mod tool_config; mod tool_definition; mod tool_discovery; -mod tool_name; mod tool_registry_plan; mod tool_registry_plan_types; mod tool_spec; @@ -50,9 +49,11 @@ pub use code_mode::collect_code_mode_tool_definitions; pub use code_mode::create_code_mode_tool; pub use code_mode::create_wait_tool; pub use code_mode::tool_spec_to_code_mode_tool_definition; +pub use codex_protocol::ToolName; pub use dynamic_tool::parse_dynamic_tool; pub use image_detail::can_request_original_image_detail; pub use image_detail::normalize_output_image_detail; +pub use image_detail::sanitize_original_image_detail; pub use js_repl_tool::create_js_repl_reset_tool; pub use js_repl_tool::create_js_repl_tool; pub use json_schema::AdditionalProperties; @@ -85,6 +86,7 @@ pub use responses_api::ResponsesApiNamespace; pub use responses_api::ResponsesApiNamespaceTool; pub use responses_api::ResponsesApiTool; pub use responses_api::ToolSearchOutputTool; +pub(crate) use responses_api::default_namespace_description; pub use responses_api::dynamic_tool_to_responses_api_tool; pub use responses_api::mcp_tool_to_deferred_responses_api_tool; pub use responses_api::mcp_tool_to_responses_api_tool; @@ -113,13 +115,13 @@ pub use tool_discovery::collect_tool_suggest_entries; pub use tool_discovery::create_tool_search_tool; pub use tool_discovery::create_tool_suggest_tool; pub use tool_discovery::filter_tool_suggest_discoverable_tools_for_client; -pub use tool_name::ToolName; pub use tool_registry_plan::build_tool_registry_plan; pub use tool_registry_plan_types::ToolHandlerKind; pub use tool_registry_plan_types::ToolHandlerSpec; pub use tool_registry_plan_types::ToolNamespace; pub use tool_registry_plan_types::ToolRegistryPlan; pub use tool_registry_plan_types::ToolRegistryPlanDeferredTool; +pub use tool_registry_plan_types::ToolRegistryPlanMcpTool; pub use tool_registry_plan_types::ToolRegistryPlanParams; pub use tool_spec::ConfiguredToolSpec; pub use tool_spec::ResponsesApiWebSearchFilters; diff --git a/codex-rs/tools/src/responses_api.rs b/codex-rs/tools/src/responses_api.rs index 50bb4f2232..5317a082f1 100644 --- a/codex-rs/tools/src/responses_api.rs +++ b/codex-rs/tools/src/responses_api.rs @@ -1,5 +1,6 @@ use crate::JsonSchema; use crate::ToolDefinition; +use crate::ToolName; use crate::parse_dynamic_tool; use crate::parse_mcp_tool; use codex_protocol::dynamic_tools::DynamicToolSpec; @@ -54,6 +55,10 @@ pub struct ResponsesApiNamespace { pub tools: Vec, } +pub(crate) fn default_namespace_description(namespace_name: &str) -> String { + format!("Tools in the {namespace_name} namespace.") +} + #[derive(Debug, Clone, Serialize, PartialEq)] #[serde(tag = "type")] pub enum ResponsesApiNamespaceTool { @@ -70,20 +75,22 @@ pub fn dynamic_tool_to_responses_api_tool( } pub fn mcp_tool_to_responses_api_tool( - name: String, + tool_name: &ToolName, tool: &rmcp::model::Tool, ) -> Result { Ok(tool_definition_to_responses_api_tool( - parse_mcp_tool(tool)?.renamed(name), + parse_mcp_tool(tool)?.renamed(tool_name.name.clone()), )) } pub fn mcp_tool_to_deferred_responses_api_tool( - name: String, + tool_name: &ToolName, tool: &rmcp::model::Tool, ) -> Result { Ok(tool_definition_to_responses_api_tool( - parse_mcp_tool(tool)?.renamed(name).into_deferred(), + parse_mcp_tool(tool)? + .renamed(tool_name.name.clone()) + .into_deferred(), )) } diff --git a/codex-rs/tools/src/responses_api_tests.rs b/codex-rs/tools/src/responses_api_tests.rs index c3ce4cec2b..799a2385d2 100644 --- a/codex-rs/tools/src/responses_api_tests.rs +++ b/codex-rs/tools/src/responses_api_tests.rs @@ -7,6 +7,7 @@ use super::mcp_tool_to_deferred_responses_api_tool; use super::tool_definition_to_responses_api_tool; use crate::JsonSchema; use crate::ToolDefinition; +use crate::ToolName; use codex_protocol::dynamic_tools::DynamicToolSpec; use pretty_assertions::assert_eq; use serde_json::json; @@ -106,19 +107,23 @@ fn mcp_tool_to_deferred_responses_api_tool_sets_defer_loading() { assert_eq!( mcp_tool_to_deferred_responses_api_tool( - "mcp__codex_apps__lookup_order".to_string(), + &ToolName::namespaced("mcp__codex_apps__", "lookup_order"), &tool, ) .expect("convert deferred tool"), ResponsesApiTool { - name: "mcp__codex_apps__lookup_order".to_string(), + name: "lookup_order".to_string(), description: "Look up an order".to_string(), strict: false, defer_loading: Some(true), - parameters: JsonSchema::object(BTreeMap::from([( + parameters: JsonSchema::object( + BTreeMap::from([( "order_id".to_string(), JsonSchema::string(/*description*/ None), - )]), Some(vec!["order_id".to_string()]), Some(false.into())), + )]), + Some(vec!["order_id".to_string()]), + Some(false.into()) + ), output_schema: None, } ); diff --git a/codex-rs/tools/src/tool_discovery.rs b/codex-rs/tools/src/tool_discovery.rs index f5d84a952e..b773623edf 100644 --- a/codex-rs/tools/src/tool_discovery.rs +++ b/codex-rs/tools/src/tool_discovery.rs @@ -2,8 +2,10 @@ use crate::JsonSchema; use crate::ResponsesApiNamespace; use crate::ResponsesApiNamespaceTool; use crate::ResponsesApiTool; +use crate::ToolName; use crate::ToolSearchOutputTool; use crate::ToolSpec; +use crate::default_namespace_description; use crate::mcp_tool_to_deferred_responses_api_tool; use codex_app_server_protocol::AppInfo; use serde::Deserialize; @@ -224,6 +226,8 @@ pub fn collect_tool_search_output_tools<'a>( let description = first_tool .connector_description + .map(str::trim) + .filter(|description| !description.is_empty()) .map(str::to_string) .or_else(|| { first_tool @@ -231,25 +235,21 @@ pub fn collect_tool_search_output_tools<'a>( .map(str::trim) .filter(|connector_name| !connector_name.is_empty()) .map(|connector_name| format!("Tools for working with {connector_name}.")) - }) - .or_else(|| { - Some(format!( - "Tools from the {} MCP server.", - first_tool.server_name - )) }); let tools = tools .iter() .map(|tool| { - mcp_tool_to_deferred_responses_api_tool(tool.tool_name.to_string(), tool.tool) + let tool_name = ToolName::namespaced(tool.tool_namespace, tool.tool_name); + mcp_tool_to_deferred_responses_api_tool(&tool_name, tool.tool) .map(ResponsesApiNamespaceTool::Function) }) .collect::, _>>()?; results.push(ToolSearchOutputTool::Namespace(ResponsesApiNamespace { name: tool_namespace.to_string(), - description: description.unwrap_or_default(), + description: description + .unwrap_or_else(|| default_namespace_description(tool_namespace)), tools, })); } diff --git a/codex-rs/tools/src/tool_discovery_tests.rs b/codex-rs/tools/src/tool_discovery_tests.rs index 10e108fe04..b745f756cc 100644 --- a/codex-rs/tools/src/tool_discovery_tests.rs +++ b/codex-rs/tools/src/tool_discovery_tests.rs @@ -253,7 +253,7 @@ fn collect_tool_search_output_tools_preserves_search_order_while_grouping_by_nam }), ToolSearchOutputTool::Namespace(ResponsesApiNamespace { name: "mcp__docs__".to_string(), - description: "Tools from the docs MCP server.".to_string(), + description: "Tools in the mcp__docs__ namespace.".to_string(), tools: vec![ResponsesApiNamespaceTool::Function(ResponsesApiTool { name: "search".to_string(), description: "Search docs.".to_string(), @@ -272,7 +272,7 @@ fn collect_tool_search_output_tools_preserves_search_order_while_grouping_by_nam } #[test] -fn collect_tool_search_output_tools_falls_back_to_connector_name_description() { +fn collect_tool_search_output_tools_ignores_blank_connector_description() { let gmail_batch_read_email = mcp_tool("gmail-batch-read-email", "Read multiple emails."); let tools = collect_tool_search_output_tools([ToolSearchResultSource { @@ -281,7 +281,7 @@ fn collect_tool_search_output_tools_falls_back_to_connector_name_description() { tool_name: "_batch_read_email", tool: &gmail_batch_read_email, connector_name: Some("Gmail"), - connector_description: None, + connector_description: Some(" "), }]) .expect("collect tool search output tools"); diff --git a/codex-rs/tools/src/tool_registry_plan.rs b/codex-rs/tools/src/tool_registry_plan.rs index 82b4c3a142..ac11a2c4be 100644 --- a/codex-rs/tools/src/tool_registry_plan.rs +++ b/codex-rs/tools/src/tool_registry_plan.rs @@ -1,12 +1,13 @@ use crate::CommandToolOptions; use crate::REQUEST_USER_INPUT_TOOL_NAME; +use crate::ResponsesApiNamespace; +use crate::ResponsesApiNamespaceTool; use crate::ShellToolOptions; use crate::SpawnAgentToolOptions; use crate::TOOL_SEARCH_DEFAULT_LIMIT; use crate::TOOL_SEARCH_TOOL_NAME; use crate::TOOL_SUGGEST_TOOL_NAME; use crate::ToolHandlerKind; -use crate::ToolName; use crate::ToolRegistryPlan; use crate::ToolRegistryPlanParams; use crate::ToolSearchSource; @@ -54,6 +55,7 @@ use crate::create_wait_agent_tool_v2; use crate::create_wait_tool; use crate::create_web_search_tool; use crate::create_write_stdin_tool; +use crate::default_namespace_description; use crate::dynamic_tool_to_responses_api_tool; use crate::mcp_tool_to_responses_api_tool; use crate::request_permissions_tool_description; @@ -61,7 +63,6 @@ use crate::request_user_input_tool_description; use crate::tool_registry_plan_types::agent_type_description; use codex_protocol::openai_models::ApplyPatchToolType; use codex_protocol::openai_models::ConfigShellToolType; -use rmcp::model::Tool as McpTool; use std::collections::BTreeMap; pub fn build_tool_registry_plan( @@ -76,9 +77,9 @@ pub fn build_tool_registry_plan( .tool_namespaces .into_iter() .flatten() - .map(|(name, detail)| { + .map(|(namespace, detail)| { ( - name.clone(), + namespace.clone(), codex_code_mode::ToolNamespaceDescription { name: detail.name.clone(), description: detail.description.clone().unwrap_or_default(), @@ -100,14 +101,17 @@ pub fn build_tool_registry_plan( .iter() .map(|configured_tool| &configured_tool.spec), ); - enabled_tools.sort_by(|left, right| { - compare_code_mode_tool_names(&left.name, &right.name, &namespace_descriptions) - }); + enabled_tools + .sort_by(|left, right| compare_code_mode_tools(left, right, &namespace_descriptions)); plan.push_spec( create_code_mode_tool( &enabled_tools, &namespace_descriptions, config.code_mode_only_enabled, + config.search_tool + && params + .deferred_mcp_tools + .is_some_and(|tools| !tools.is_empty()), ), /*supports_parallel_tool_calls*/ false, config.code_mode_enabled, @@ -266,10 +270,7 @@ pub fn build_tool_registry_plan( plan.register_handler(TOOL_SEARCH_TOOL_NAME, ToolHandlerKind::ToolSearch); for tool in deferred_mcp_tools { - plan.register_handler( - ToolName::namespaced(tool.tool_namespace, tool.tool_name), - ToolHandlerKind::Mcp, - ); + plan.register_handler(tool.name.clone(), ToolHandlerKind::Mcp); } } @@ -471,28 +472,65 @@ pub fn build_tool_registry_plan( } if let Some(mcp_tools) = params.mcp_tools { - let mut entries: Vec<(String, &McpTool)> = mcp_tools - .iter() - .map(|(name, tool)| (name.clone(), tool)) - .collect(); - entries.sort_by(|left, right| left.0.cmp(&right.0)); + let mut entries = mcp_tools.to_vec(); + entries.sort_by_key(|tool| tool.name.display()); + let mut namespace_entries = BTreeMap::new(); - for (name, tool) in entries { - match mcp_tool_to_responses_api_tool(name.clone(), tool) { - Ok(converted_tool) => { - plan.push_spec( - ToolSpec::Function(converted_tool), - /*supports_parallel_tool_calls*/ false, - config.code_mode_enabled, - ); - plan.register_handler(name, ToolHandlerKind::Mcp); - } - Err(error) => { - tracing::error!( - "Failed to convert {name:?} MCP tool to OpenAI tool: {error:?}" - ); + for tool in entries { + let Some(namespace) = tool.name.namespace.as_ref() else { + let tool_name = &tool.name; + tracing::error!("Skipping MCP tool `{tool_name}`: MCP tools must be namespaced"); + continue; + }; + namespace_entries + .entry(namespace.clone()) + .or_insert_with(Vec::new) + .push(tool); + } + + for (namespace, mut entries) in namespace_entries { + entries.sort_by_key(|tool| tool.name.name.clone()); + let tool_namespace = params + .tool_namespaces + .and_then(|namespaces| namespaces.get(&namespace)); + let description = tool_namespace + .and_then(|namespace| namespace.description.as_deref()) + .map(str::trim) + .filter(|description| !description.is_empty()) + .map(str::to_string) + .unwrap_or_else(|| { + let namespace_name = tool_namespace + .map(|namespace| namespace.name.as_str()) + .unwrap_or(namespace.as_str()); + default_namespace_description(namespace_name) + }); + let mut tools = Vec::new(); + for tool in entries { + match mcp_tool_to_responses_api_tool(&tool.name, tool.tool) { + Ok(converted_tool) => { + tools.push(ResponsesApiNamespaceTool::Function(converted_tool)); + plan.register_handler(tool.name, ToolHandlerKind::Mcp); + } + Err(error) => { + let tool_name = &tool.name; + tracing::error!( + "Failed to convert `{tool_name}` MCP tool to OpenAI tool: {error:?}" + ); + } } } + + if !tools.is_empty() { + plan.push_spec( + ToolSpec::Namespace(ResponsesApiNamespace { + name: namespace, + description, + tools, + }), + /*supports_parallel_tool_calls*/ false, + config.code_mode_enabled, + ); + } } } @@ -518,41 +556,31 @@ pub fn build_tool_registry_plan( plan } -fn compare_code_mode_tool_names( - left_name: &str, - right_name: &str, +fn compare_code_mode_tools( + left: &codex_code_mode::ToolDefinition, + right: &codex_code_mode::ToolDefinition, namespace_descriptions: &BTreeMap, ) -> std::cmp::Ordering { - let left_namespace = code_mode_namespace_name(left_name, namespace_descriptions); - let right_namespace = code_mode_namespace_name(right_name, namespace_descriptions); + let left_namespace = code_mode_namespace_name(left, namespace_descriptions); + let right_namespace = code_mode_namespace_name(right, namespace_descriptions); left_namespace .cmp(&right_namespace) - .then_with(|| { - code_mode_function_name(left_name, left_namespace) - .cmp(code_mode_function_name(right_name, right_namespace)) - }) - .then_with(|| left_name.cmp(right_name)) + .then_with(|| left.tool_name.name.cmp(&right.tool_name.name)) + .then_with(|| left.name.cmp(&right.name)) } fn code_mode_namespace_name<'a>( - name: &str, + tool: &codex_code_mode::ToolDefinition, namespace_descriptions: &'a BTreeMap, ) -> Option<&'a str> { - namespace_descriptions - .get(name) + tool.tool_name + .namespace + .as_ref() + .and_then(|namespace| namespace_descriptions.get(namespace)) .map(|namespace_description| namespace_description.name.as_str()) } -fn code_mode_function_name<'a>(name: &'a str, namespace: Option<&str>) -> &'a str { - namespace - .and_then(|namespace| { - name.strip_prefix(namespace) - .and_then(|suffix| suffix.strip_prefix("__")) - }) - .unwrap_or(name) -} - #[cfg(test)] #[path = "tool_registry_plan_tests.rs"] mod tests; diff --git a/codex-rs/tools/src/tool_registry_plan_tests.rs b/codex-rs/tools/src/tool_registry_plan_tests.rs index 69b8b55b54..cc8c05dd9a 100644 --- a/codex-rs/tools/src/tool_registry_plan_tests.rs +++ b/codex-rs/tools/src/tool_registry_plan_tests.rs @@ -7,12 +7,15 @@ use crate::FreeformTool; use crate::JsonSchema; use crate::JsonSchemaPrimitiveType; use crate::JsonSchemaType; +use crate::ResponsesApiNamespaceTool; use crate::ResponsesApiTool; use crate::ResponsesApiWebSearchFilters; use crate::ResponsesApiWebSearchUserLocation; use crate::ToolHandlerSpec; +use crate::ToolName; use crate::ToolNamespace; use crate::ToolRegistryPlanDeferredTool; +use crate::ToolRegistryPlanMcpTool; use crate::ToolsConfigParams; use crate::WaitAgentTimeoutOptions; use crate::mcp_call_tool_result_output_schema; @@ -94,6 +97,7 @@ fn test_full_toolset_specs_for_gpt5_codex_unified_exec_web_search() { search_context_size: None, search_content_types: None, }, + create_image_generation_tool("png"), create_view_image_tool(ViewImageToolOptions { can_request_original_image_detail: config.can_request_original_image_detail, }), @@ -1075,7 +1079,7 @@ fn test_build_specs_mcp_tools_converted() { let (tools, _) = build_specs( &tools_config, Some(HashMap::from([( - "test_server/do_something_cool".to_string(), + ToolName::namespaced("test_server/", "do_something_cool"), mcp_tool( "do_something_cool", "Do something cool", @@ -1101,11 +1105,11 @@ fn test_build_specs_mcp_tools_converted() { &[], ); - let tool = find_tool(&tools, "test_server/do_something_cool"); + let tool = find_namespace_function_tool(&tools, "test_server/", "do_something_cool"); assert_eq!( - &tool.spec, - &ToolSpec::Function(ResponsesApiTool { - name: "test_server/do_something_cool".to_string(), + tool, + &ResponsesApiTool { + name: "do_something_cool".to_string(), parameters: JsonSchema::object( BTreeMap::from([ ( @@ -1144,7 +1148,47 @@ fn test_build_specs_mcp_tools_converted() { strict: false, output_schema: Some(mcp_call_tool_result_output_schema(serde_json::json!({}))), defer_loading: None, - }) + } + ); +} + +#[test] +fn test_build_specs_mcp_namespace_description_falls_back_when_missing() { + let model_info = model_info(); + let mut features = Features::with_defaults(); + features.enable(Feature::UnifiedExec); + let available_models = Vec::new(); + let tools_config = ToolsConfig::new(&ToolsConfigParams { + model_info: &model_info, + available_models: &available_models, + features: &features, + image_generation_tool_auth_allowed: true, + web_search_mode: Some(WebSearchMode::Cached), + session_source: SessionSource::Cli, + sandbox_policy: &SandboxPolicy::DangerFullAccess, + windows_sandbox_level: WindowsSandboxLevel::Disabled, + }); + let (tools, _) = build_specs( + &tools_config, + Some(HashMap::from([( + ToolName::namespaced("test_server/", "do_something_cool"), + mcp_tool( + "do_something_cool", + "Do something cool", + serde_json::json!({"type": "object"}), + ), + )])), + /*deferred_mcp_tools*/ None, + &[], + ); + + let namespace_tool = find_tool(&tools, "test_server/"); + let ToolSpec::Namespace(namespace) = &namespace_tool.spec else { + panic!("expected namespace tool"); + }; + assert_eq!( + namespace.description, + "Tools in the test_server/ namespace." ); } @@ -1167,16 +1211,16 @@ fn test_build_specs_mcp_tools_sorted_by_name() { let tools_map = HashMap::from([ ( - "test_server/do".to_string(), - mcp_tool("a", "a", serde_json::json!({"type": "object"})), + ToolName::namespaced("test_server/", "do"), + mcp_tool("do", "a", serde_json::json!({"type": "object"})), ), ( - "test_server/something".to_string(), - mcp_tool("b", "b", serde_json::json!({"type": "object"})), + ToolName::namespaced("test_server/", "something"), + mcp_tool("something", "b", serde_json::json!({"type": "object"})), ), ( - "test_server/cool".to_string(), - mcp_tool("c", "c", serde_json::json!({"type": "object"})), + ToolName::namespaced("test_server/", "cool"), + mcp_tool("cool", "c", serde_json::json!({"type": "object"})), ), ]); @@ -1187,17 +1231,14 @@ fn test_build_specs_mcp_tools_sorted_by_name() { &[], ); - let mcp_names: Vec<_> = tools - .iter() - .map(|tool| tool.name().to_string()) - .filter(|name| name.starts_with("test_server/")) - .collect(); - let expected = vec![ - "test_server/cool".to_string(), - "test_server/do".to_string(), - "test_server/something".to_string(), - ]; - assert_eq!(mcp_names, expected); + assert_eq!( + namespace_function_names(&tools, "test_server/"), + vec![ + "cool".to_string(), + "do".to_string(), + "something".to_string(), + ] + ); } #[test] @@ -1222,7 +1263,7 @@ fn search_tool_description_lists_each_mcp_source_once() { &tools_config, Some(HashMap::from([ ( - "mcp__codex_apps__calendar_create_event".to_string(), + ToolName::namespaced("mcp__codex_apps__calendar", "_create_event"), mcp_tool( "calendar_create_event", "Create calendar event", @@ -1230,7 +1271,7 @@ fn search_tool_description_lists_each_mcp_source_once() { ), ), ( - "mcp__rmcp__echo".to_string(), + ToolName::namespaced("mcp__rmcp__", "echo"), mcp_tool("echo", "Echo", serde_json::json!({"type": "object"})), ), ])), @@ -1294,7 +1335,7 @@ fn search_tool_description_lists_each_mcp_source_once() { } #[test] -fn search_tool_requires_model_capability_and_feature_flag() { +fn search_tool_requires_model_capability_and_enabled_feature() { let model_info = search_capable_model_info(); let deferred_mcp_tools = Some(vec![deferred_mcp_tool( "_create_event", @@ -1327,10 +1368,12 @@ fn search_tool_requires_model_capability_and_feature_flag() { ); assert_lacks_tool_name(&tools, TOOL_SEARCH_TOOL_NAME); + let mut features_without_tool_search = Features::with_defaults(); + features_without_tool_search.disable(Feature::ToolSearch); let tools_config = ToolsConfig::new(&ToolsConfigParams { model_info: &model_info, available_models: &available_models, - features: &features, + features: &features_without_tool_search, image_generation_tool_auth_allowed: true, web_search_mode: Some(WebSearchMode::Cached), session_source: SessionSource::Cli, @@ -1345,8 +1388,6 @@ fn search_tool_requires_model_capability_and_feature_flag() { ); assert_lacks_tool_name(&tools, TOOL_SEARCH_TOOL_NAME); - let mut features = Features::with_defaults(); - features.enable(Feature::ToolSearch); let tools_config = ToolsConfig::new(&ToolsConfigParams { model_info: &model_info, available_models: &available_models, @@ -1577,7 +1618,7 @@ fn code_mode_augments_mcp_tool_descriptions_with_namespaced_sample() { let (tools, _) = build_specs( &tools_config, Some(HashMap::from([( - "mcp__sample__echo".to_string(), + ToolName::namespaced("mcp__sample__", "echo"), mcp_tool( "echo", "Echo text", @@ -1595,11 +1636,8 @@ fn code_mode_augments_mcp_tool_descriptions_with_namespaced_sample() { &[], ); - let ToolSpec::Function(ResponsesApiTool { description, .. }) = - &find_tool(&tools, "mcp__sample__echo").spec - else { - panic!("expected function tool"); - }; + let ResponsesApiTool { description, .. } = + find_namespace_function_tool(&tools, "mcp__sample__", "echo"); assert_eq!( description, @@ -1633,7 +1671,7 @@ fn code_mode_preserves_nullable_and_literal_mcp_input_shapes() { let (tools, _) = build_specs( &tools_config, Some(HashMap::from([( - "mcp__sample__fn".to_string(), + ToolName::namespaced("mcp__sample__", "fn"), mcp_tool( "fn", "Sample fn", @@ -1684,11 +1722,8 @@ fn code_mode_preserves_nullable_and_literal_mcp_input_shapes() { &[], ); - let ToolSpec::Function(ResponsesApiTool { description, .. }) = - &find_tool(&tools, "mcp__sample__fn").spec - else { - panic!("expected function tool"); - }; + let ResponsesApiTool { description, .. } = + find_namespace_function_tool(&tools, "mcp__sample__", "fn"); assert!(description.contains( r#"exec tool declaration: @@ -1851,7 +1886,7 @@ fn search_capable_model_info() -> ModelInfo { fn build_specs<'a>( config: &ToolsConfig, - mcp_tools: Option>, + mcp_tools: Option>, deferred_mcp_tools: Option>>, dynamic_tools: &[DynamicToolSpec], ) -> (Vec, Vec) { @@ -1866,7 +1901,7 @@ fn build_specs<'a>( fn build_specs_with_discoverable_tools<'a>( config: &ToolsConfig, - mcp_tools: Option>, + mcp_tools: Option>, deferred_mcp_tools: Option>>, discoverable_tools: Option>, dynamic_tools: &[DynamicToolSpec], @@ -1883,16 +1918,25 @@ fn build_specs_with_discoverable_tools<'a>( fn build_specs_with_optional_tool_namespaces<'a>( config: &ToolsConfig, - mcp_tools: Option>, + mcp_tools: Option>, deferred_mcp_tools: Option>>, tool_namespaces: Option>, discoverable_tools: Option>, dynamic_tools: &[DynamicToolSpec], ) -> (Vec, Vec) { + let mcp_tool_inputs = mcp_tools.as_ref().map(|mcp_tools| { + mcp_tools + .iter() + .map(|(name, tool)| ToolRegistryPlanMcpTool { + name: name.clone(), + tool, + }) + .collect::>() + }); let plan = build_tool_registry_plan( config, ToolRegistryPlanParams { - mcp_tools: mcp_tools.as_ref(), + mcp_tools: mcp_tool_inputs.as_deref(), deferred_mcp_tools: deferred_mcp_tools.as_deref(), tool_namespaces: tool_namespaces.as_ref(), discoverable_tools: discoverable_tools.as_deref(), @@ -1968,16 +2012,16 @@ fn code_mode_augments_mcp_tool_descriptions_with_structured_output_sample() { let (tools, _) = build_specs( &tools_config, - Some(HashMap::from([("mcp__sample__echo".to_string(), tool)])), + Some(HashMap::from([( + ToolName::namespaced("mcp__sample__", "echo"), + tool, + )])), /*deferred_mcp_tools*/ None, &[], ); - let ToolSpec::Function(ResponsesApiTool { description, .. }) = - &find_tool(&tools, "mcp__sample__echo").spec - else { - panic!("expected function tool"); - }; + let ResponsesApiTool { description, .. } = + find_namespace_function_tool(&tools, "mcp__sample__", "echo"); assert_eq!( description, @@ -2017,8 +2061,7 @@ fn deferred_mcp_tool<'a>( connector_description: Option<&'a str>, ) -> ToolRegistryPlanDeferredTool<'a> { ToolRegistryPlanDeferredTool { - tool_name, - tool_namespace, + name: ToolName::namespaced(tool_namespace, tool_name), server_name, connector_name, connector_description, @@ -2089,6 +2132,39 @@ fn find_tool<'a>(tools: &'a [ConfiguredToolSpec], expected_name: &str) -> &'a Co .unwrap_or_else(|| panic!("expected tool {expected_name}")) } +fn find_namespace_function_tool<'a>( + tools: &'a [ConfiguredToolSpec], + expected_namespace: &str, + expected_name: &str, +) -> &'a ResponsesApiTool { + let namespace_tool = find_tool(tools, expected_namespace); + let ToolSpec::Namespace(namespace) = &namespace_tool.spec else { + panic!("expected namespace tool {expected_namespace}"); + }; + namespace + .tools + .iter() + .find_map(|tool| match tool { + ResponsesApiNamespaceTool::Function(tool) if tool.name == expected_name => Some(tool), + _ => None, + }) + .unwrap_or_else(|| panic!("expected tool {expected_namespace}{expected_name} in namespace")) +} + +fn namespace_function_names(tools: &[ConfiguredToolSpec], expected_namespace: &str) -> Vec { + let namespace_tool = find_tool(tools, expected_namespace); + let ToolSpec::Namespace(namespace) = &namespace_tool.spec else { + panic!("expected namespace tool {expected_namespace}"); + }; + namespace + .tools + .iter() + .map(|tool| match tool { + ResponsesApiNamespaceTool::Function(tool) => tool.name.clone(), + }) + .collect() +} + fn expect_object_schema( schema: &JsonSchema, ) -> (&BTreeMap, Option<&Vec>) { @@ -2137,6 +2213,17 @@ fn strip_descriptions_tool(spec: &mut ToolSpec) { ToolSpec::Function(ResponsesApiTool { parameters, .. }) => { strip_descriptions_schema(parameters); } + ToolSpec::Namespace(namespace) => { + for tool in &mut namespace.tools { + match tool { + ResponsesApiNamespaceTool::Function(ResponsesApiTool { + parameters, .. + }) => { + strip_descriptions_schema(parameters); + } + } + } + } ToolSpec::Freeform(FreeformTool { .. }) | ToolSpec::LocalShell {} | ToolSpec::ImageGeneration { .. } diff --git a/codex-rs/tools/src/tool_registry_plan_types.rs b/codex-rs/tools/src/tool_registry_plan_types.rs index 7459954dcd..b9d66a0c2c 100644 --- a/codex-rs/tools/src/tool_registry_plan_types.rs +++ b/codex-rs/tools/src/tool_registry_plan_types.rs @@ -6,7 +6,6 @@ use crate::ToolsConfig; use crate::WaitAgentTimeoutOptions; use crate::augment_tool_spec_for_code_mode; use codex_protocol::dynamic_tools::DynamicToolSpec; -use rmcp::model::Tool as McpTool; use std::collections::HashMap; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -58,7 +57,7 @@ pub struct ToolRegistryPlan { #[derive(Debug, Clone, Copy)] pub struct ToolRegistryPlanParams<'a> { - pub mcp_tools: Option<&'a HashMap>, + pub mcp_tools: Option<&'a [ToolRegistryPlanMcpTool<'a>]>, pub deferred_mcp_tools: Option<&'a [ToolRegistryPlanDeferredTool<'a>]>, pub tool_namespaces: Option<&'a HashMap>, pub discoverable_tools: Option<&'a [DiscoverableTool]>, @@ -73,10 +72,18 @@ pub struct ToolNamespace { pub description: Option, } -#[derive(Debug, Clone, Copy)] +/// Direct MCP tool metadata needed to expose the Responses API namespace tool +/// while registering its runtime handler with the canonical namespace/name +/// identity. +#[derive(Debug, Clone)] +pub struct ToolRegistryPlanMcpTool<'a> { + pub name: ToolName, + pub tool: &'a rmcp::model::Tool, +} + +#[derive(Debug, Clone)] pub struct ToolRegistryPlanDeferredTool<'a> { - pub tool_name: &'a str, - pub tool_namespace: &'a str, + pub name: ToolName, pub server_name: &'a str, pub connector_name: Option<&'a str>, pub connector_description: Option<&'a str>, diff --git a/codex-rs/tools/src/tool_spec.rs b/codex-rs/tools/src/tool_spec.rs index 24644e2260..81db2f5584 100644 --- a/codex-rs/tools/src/tool_spec.rs +++ b/codex-rs/tools/src/tool_spec.rs @@ -1,5 +1,6 @@ use crate::FreeformTool; use crate::JsonSchema; +use crate::ResponsesApiNamespace; use crate::ResponsesApiTool; use codex_protocol::config_types::WebSearchConfig; use codex_protocol::config_types::WebSearchContextSize; @@ -20,6 +21,8 @@ const WEB_SEARCH_TEXT_AND_IMAGE_CONTENT_TYPES: [&str; 2] = ["text", "image"]; pub enum ToolSpec { #[serde(rename = "function")] Function(ResponsesApiTool), + #[serde(rename = "namespace")] + Namespace(ResponsesApiNamespace), #[serde(rename = "tool_search")] ToolSearch { execution: String, @@ -57,6 +60,7 @@ impl ToolSpec { pub fn name(&self) -> &str { match self { ToolSpec::Function(tool) => tool.name.as_str(), + ToolSpec::Namespace(namespace) => namespace.name.as_str(), ToolSpec::ToolSearch { .. } => "tool_search", ToolSpec::LocalShell {} => "local_shell", ToolSpec::ImageGeneration { .. } => "image_generation", diff --git a/codex-rs/tools/src/tool_spec_tests.rs b/codex-rs/tools/src/tool_spec_tests.rs index c94f54e6b5..0b98625626 100644 --- a/codex-rs/tools/src/tool_spec_tests.rs +++ b/codex-rs/tools/src/tool_spec_tests.rs @@ -1,4 +1,5 @@ use super::ConfiguredToolSpec; +use super::ResponsesApiNamespace; use super::ResponsesApiWebSearchFilters; use super::ResponsesApiWebSearchUserLocation; use super::ToolSpec; @@ -6,6 +7,7 @@ use crate::AdditionalProperties; use crate::FreeformTool; use crate::FreeformToolFormat; use crate::JsonSchema; +use crate::ResponsesApiNamespaceTool; use crate::ResponsesApiTool; use crate::create_tools_json_for_responses_api; use codex_protocol::config_types::WebSearchContextSize; @@ -34,6 +36,15 @@ fn tool_spec_name_covers_all_variants() { .name(), "lookup_order" ); + assert_eq!( + ToolSpec::Namespace(ResponsesApiNamespace { + name: "mcp__demo__".to_string(), + description: "Demo tools".to_string(), + tools: Vec::new(), + }) + .name(), + "mcp__demo__" + ); assert_eq!( ToolSpec::ToolSearch { execution: "sync".to_string(), @@ -163,6 +174,51 @@ fn create_tools_json_for_responses_api_includes_top_level_name() { ); } +#[test] +fn namespace_tool_spec_serializes_expected_wire_shape() { + assert_eq!( + serde_json::to_value(ToolSpec::Namespace(ResponsesApiNamespace { + name: "mcp__demo__".to_string(), + description: "Demo tools".to_string(), + tools: vec![ResponsesApiNamespaceTool::Function(ResponsesApiTool { + name: "lookup_order".to_string(), + description: "Look up an order".to_string(), + strict: false, + defer_loading: None, + parameters: JsonSchema::object( + BTreeMap::from([( + "order_id".to_string(), + JsonSchema::string(/*description*/ None), + )]), + /*required*/ None, + /*additional_properties*/ None, + ), + output_schema: None, + })], + })) + .expect("serialize namespace tool"), + json!({ + "type": "namespace", + "name": "mcp__demo__", + "description": "Demo tools", + "tools": [ + { + "type": "function", + "name": "lookup_order", + "description": "Look up an order", + "strict": false, + "parameters": { + "type": "object", + "properties": { + "order_id": { "type": "string" }, + }, + }, + }, + ], + }) + ); +} + #[test] fn web_search_tool_spec_serializes_expected_wire_shape() { assert_eq!( diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml index 0dec8ceed1..65f35b665a 100644 --- a/codex-rs/tui/Cargo.toml +++ b/codex-rs/tui/Cargo.toml @@ -29,9 +29,11 @@ codex-ansi-escape = { workspace = true } codex-app-server-client = { workspace = true } codex-app-server-protocol = { workspace = true } codex-arg0 = { workspace = true } +codex-install-context = { workspace = true } codex-chatgpt = { workspace = true } codex-cloud-requirements = { workspace = true } codex-config = { workspace = true } +codex-connectors = { workspace = true } codex-exec-server = { workspace = true } codex-features = { workspace = true } codex-feedback = { workspace = true } diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 9f52f9f525..c87f4a87a4 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -98,6 +98,7 @@ use codex_app_server_protocol::ServerRequest; use codex_app_server_protocol::SkillsListResponse; use codex_app_server_protocol::ThreadItem; use codex_app_server_protocol::ThreadLoadedListParams; +use codex_app_server_protocol::ThreadMemoryMode; use codex_app_server_protocol::ThreadRollbackResponse; use codex_app_server_protocol::ThreadStartSource; use codex_app_server_protocol::Turn; @@ -165,7 +166,7 @@ use toml::Value as TomlValue; use uuid::Uuid; mod agent_navigation; mod app_server_adapter; -mod app_server_requests; +pub(crate) mod app_server_requests; mod loaded_threads; mod pending_interactive_replay; @@ -326,8 +327,12 @@ fn session_summary( token_usage: TokenUsage, thread_id: Option, thread_name: Option, + rollout_path: Option<&Path>, ) -> Option { let usage_line = (!token_usage.is_zero()).then(|| FinalOutput::from(token_usage).to_string()); + let (thread_id, thread_name) = resumable_thread(thread_id, thread_name, rollout_path) + .map(|thread| (Some(thread.thread_id), thread.thread_name)) + .unwrap_or((None, None)); let resume_command = crate::legacy_core::util::resume_command(thread_name.as_deref(), thread_id); @@ -341,6 +346,29 @@ fn session_summary( }) } +#[derive(Debug, Clone, PartialEq, Eq)] +struct ResumableThread { + thread_id: ThreadId, + thread_name: Option, +} + +fn resumable_thread( + thread_id: Option, + thread_name: Option, + rollout_path: Option<&Path>, +) -> Option { + let thread_id = thread_id?; + let rollout_path = rollout_path?; + rollout_path_is_resumable(rollout_path).then_some(ResumableThread { + thread_id, + thread_name, + }) +} + +fn rollout_path_is_resumable(rollout_path: &Path) -> bool { + std::fs::metadata(rollout_path).is_ok_and(|metadata| metadata.is_file() && metadata.len() > 0) +} + fn errors_for_cwd(cwd: &Path, response: &ListSkillsResponseEvent) -> Vec { response .skills @@ -1386,10 +1414,16 @@ impl App { } self.config = next_config; + let show_memory_enable_notice = feature_updates_to_apply + .iter() + .any(|(feature, enabled)| *feature == Feature::MemoryTool && *enabled); for (feature, effective_enabled) in feature_updates_to_apply { self.chat_widget .set_feature_enabled(feature, effective_enabled); } + if show_memory_enable_notice { + self.chat_widget.add_memories_enable_notice(); + } if approvals_reviewer_override.is_some() { self.set_approvals_reviewer_in_app_and_widget(self.config.approvals_reviewer); } @@ -1473,6 +1507,101 @@ impl App { } } + async fn update_memory_settings( + &mut self, + use_memories: bool, + generate_memories: bool, + ) -> bool { + let active_profile = self.active_profile.clone(); + let scoped_memory_segments = |key: &str| { + if let Some(profile) = active_profile.as_deref() { + vec![ + "profiles".to_string(), + profile.to_string(), + "memories".to_string(), + key.to_string(), + ] + } else { + vec!["memories".to_string(), key.to_string()] + } + }; + let edits = [ + ConfigEdit::SetPath { + segments: scoped_memory_segments("use_memories"), + value: use_memories.into(), + }, + ConfigEdit::SetPath { + segments: scoped_memory_segments("generate_memories"), + value: generate_memories.into(), + }, + ]; + + if let Err(err) = ConfigEditsBuilder::new(&self.config.codex_home) + .with_edits(edits) + .apply() + .await + { + tracing::error!(error = %err, "failed to persist memory settings"); + self.chat_widget + .add_error_message(format!("Failed to save memory settings: {err}")); + return false; + } + + self.config.memories.use_memories = use_memories; + self.config.memories.generate_memories = generate_memories; + self.chat_widget + .set_memory_settings(use_memories, generate_memories); + true + } + + async fn update_memory_settings_with_app_server( + &mut self, + app_server: &mut AppServerSession, + use_memories: bool, + generate_memories: bool, + ) { + let previous_generate_memories = self.config.memories.generate_memories; + if !self + .update_memory_settings(use_memories, generate_memories) + .await + { + return; + } + + if previous_generate_memories == generate_memories { + return; + } + + let Some(thread_id) = self.current_displayed_thread_id() else { + return; + }; + + let mode = if generate_memories { + ThreadMemoryMode::Enabled + } else { + ThreadMemoryMode::Disabled + }; + + if let Err(err) = app_server.thread_memory_mode_set(thread_id, mode).await { + tracing::error!(error = %err, %thread_id, "failed to update thread memory mode"); + self.chat_widget.add_error_message(format!( + "Saved memory settings, but failed to update the current thread: {err}" + )); + } + } + + async fn reset_memories_with_app_server(&mut self, app_server: &mut AppServerSession) { + if let Err(err) = app_server.memory_reset().await { + tracing::error!(error = %err, "failed to reset memories"); + self.chat_widget + .add_error_message(format!("Failed to reset memories: {err}")); + return; + } + + self.chat_widget + .add_info_message("Reset local memories.".to_string(), /*hint*/ None); + } + fn open_url_in_browser(&mut self, url: String) { if let Err(err) = webbrowser::open(&url) { self.chat_widget @@ -1499,6 +1628,7 @@ impl App { self.config.cwd.to_path_buf(), version, ) + .with_yolo_mode(history_cell::is_yolo_mode(&self.config)) .display_lines(width) } @@ -2292,10 +2422,11 @@ impl App { ) -> Result { match op.view() { AppCommandView::Interrupt => { - let Some(turn_id) = self.active_turn_id_for_thread(thread_id).await else { - return Ok(true); - }; - app_server.turn_interrupt(thread_id, turn_id).await?; + if let Some(turn_id) = self.active_turn_id_for_thread(thread_id).await { + app_server.turn_interrupt(thread_id, turn_id).await?; + } else { + app_server.startup_interrupt(thread_id).await?; + } Ok(true) } AppCommandView::UserTurn { @@ -2400,14 +2531,16 @@ impl App { Ok(true) } AppCommandView::ListSkills { cwds, force_reload } => { - let response = app_server - .skills_list(codex_app_server_protocol::SkillsListParams { - cwds: cwds.to_vec(), - force_reload, - per_cwd_extra_user_roots: None, - }) - .await?; - self.handle_skills_list_response(response); + self.handle_skills_list_result( + app_server + .skills_list(codex_app_server_protocol::SkillsListParams { + cwds: cwds.to_vec(), + force_reload, + per_cwd_extra_user_roots: None, + }) + .await, + "failed to refresh skills", + ); Ok(true) } AppCommandView::Compact => { @@ -2481,6 +2614,19 @@ impl App { } } + fn handle_skills_list_result( + &mut self, + result: Result, + failure_message: &str, + ) { + match result { + Ok(response) => self.handle_skills_list_response(response), + Err(err) => { + tracing::warn!("{failure_message}: {err:#}"); + } + } + } + async fn try_resolve_app_server_request( &mut self, app_server: &AppServerSession, @@ -3361,6 +3507,7 @@ impl App { self.chat_widget.token_usage(), self.chat_widget.thread_id(), self.chat_widget.thread_name(), + self.chat_widget.rollout_path().as_deref(), ); self.shutdown_current_thread(app_server).await; let tracked_thread_ids: Vec = @@ -3887,17 +4034,16 @@ impl App { app.enqueue_primary_thread_session(started.session, started.turns) .await?; } - match app_server - .skills_list(codex_app_server_protocol::SkillsListParams { - cwds: vec![app.config.cwd.to_path_buf()], - force_reload: true, - per_cwd_extra_user_roots: None, - }) - .await - { - Ok(response) => app.handle_skills_list_response(response), - Err(err) => tracing::warn!("failed to load skills on startup: {err:#}"), - } + app.handle_skills_list_result( + app_server + .skills_list(codex_app_server_protocol::SkillsListParams { + cwds: vec![app.config.cwd.to_path_buf()], + force_reload: true, + per_cwd_extra_user_roots: None, + }) + .await, + "failed to load skills on startup", + ); // On startup, if Agent mode (workspace-write) or ReadOnly is active, warn about world-writable dirs on Windows. #[cfg(target_os = "windows")] @@ -4039,10 +4185,15 @@ impl App { return Err(err); } }; + let resumable_thread = resumable_thread( + app.chat_widget.thread_id(), + app.chat_widget.thread_name(), + app.chat_widget.rollout_path().as_deref(), + ); Ok(AppExitInfo { token_usage: app.token_usage(), - thread_id: app.chat_widget.thread_id(), - thread_name: app.chat_widget.thread_name(), + thread_id: resumable_thread.as_ref().map(|thread| thread.thread_id), + thread_name: resumable_thread.and_then(|thread| thread.thread_name), update_action: app.pending_update_action, exit_reason, }) @@ -4162,6 +4313,7 @@ impl App { self.chat_widget.token_usage(), self.chat_widget.thread_id(), self.chat_widget.thread_name(), + self.chat_widget.rollout_path().as_deref(), ); match app_server .resume_thread(resume_config.clone(), target_session.thread_id) @@ -4316,6 +4468,7 @@ impl App { self.chat_widget.token_usage(), self.chat_widget.thread_id(), self.chat_widget.thread_name(), + self.chat_widget.rollout_path().as_deref(), ); self.chat_widget .add_plain_history_lines(vec!["/fork".magenta().into()]); @@ -5341,6 +5494,20 @@ impl App { AppEvent::UpdateFeatureFlags { updates } => { self.update_feature_flags(updates).await; } + AppEvent::UpdateMemorySettings { + use_memories, + generate_memories, + } => { + self.update_memory_settings_with_app_server( + app_server, + use_memories, + generate_memories, + ) + .await; + } + AppEvent::ResetMemories => { + self.reset_memories_with_app_server(app_server).await; + } AppEvent::SkipNextWorldWritableScan => { self.windows_sandbox.skip_world_writable_scan_once = true; } @@ -5808,8 +5975,13 @@ impl App { .handle_server_notification(notification, /*replay_kind*/ None); } ThreadBufferedEvent::Request(request) => { - self.chat_widget - .handle_server_request(request, /*replay_kind*/ None); + if self + .pending_app_server_requests + .contains_server_request(&request) + { + self.chat_widget + .handle_server_request(request, /*replay_kind*/ None); + } } ThreadBufferedEvent::HistoryEntryResponse(event) => { self.chat_widget.handle_history_entry_response(event); @@ -6280,7 +6452,7 @@ async fn fetch_plugins_list( ) -> Result { let cwd = AbsolutePathBuf::try_from(cwd).wrap_err("plugin list cwd must be absolute")?; let request_id = RequestId::String(format!("plugin-list-{}", Uuid::new_v4())); - request_handle + let mut response = request_handle .request_typed(ClientRequest::PluginList { request_id, params: PluginListParams { @@ -6289,7 +6461,17 @@ async fn fetch_plugins_list( }, }) .await - .wrap_err("plugin/list failed in TUI") + .wrap_err("plugin/list failed in TUI")?; + hide_cli_only_plugin_marketplaces(&mut response); + Ok(response) +} + +const CLI_HIDDEN_PLUGIN_MARKETPLACES: &[&str] = &["openai-bundled"]; + +fn hide_cli_only_plugin_marketplaces(response: &mut PluginListResponse) { + response + .marketplaces + .retain(|marketplace| !CLI_HIDDEN_PLUGIN_MARKETPLACES.contains(&marketplace.name.as_str())); } async fn fetch_plugin_detail( @@ -6467,6 +6649,7 @@ mod tests { use codex_app_server_protocol::NetworkPolicyRuleAction as AppServerNetworkPolicyRuleAction; use codex_app_server_protocol::NonSteerableTurnKind as AppServerNonSteerableTurnKind; use codex_app_server_protocol::PermissionsRequestApprovalParams; + use codex_app_server_protocol::PluginMarketplaceEntry; use codex_app_server_protocol::RequestId as AppServerRequestId; use codex_app_server_protocol::ServerNotification; use codex_app_server_protocol::ServerRequest; @@ -6525,6 +6708,41 @@ mod tests { AbsolutePathBuf::try_from(PathBuf::from(path)).expect("absolute test path") } + #[test] + fn hide_cli_only_plugin_marketplaces_removes_openai_bundled() { + let mut response = PluginListResponse { + marketplaces: vec![ + PluginMarketplaceEntry { + name: "openai-bundled".to_string(), + path: test_absolute_path("/marketplaces/openai-bundled"), + interface: None, + plugins: Vec::new(), + }, + PluginMarketplaceEntry { + name: "openai-curated".to_string(), + path: test_absolute_path("/marketplaces/openai-curated"), + interface: None, + plugins: Vec::new(), + }, + ], + marketplace_load_errors: Vec::new(), + remote_sync_error: None, + featured_plugin_ids: Vec::new(), + }; + + hide_cli_only_plugin_marketplaces(&mut response); + + assert_eq!( + response.marketplaces, + vec![PluginMarketplaceEntry { + name: "openai-curated".to_string(), + path: test_absolute_path("/marketplaces/openai-curated"), + interface: None, + plugins: Vec::new(), + }] + ); + } + #[test] fn normalize_harness_overrides_resolves_relative_add_dirs() -> Result<()> { let temp_dir = tempdir()?; @@ -6767,6 +6985,11 @@ mod tests { let approval_request = exec_approval_request(thread_id, "turn-1", "call-1", /*approval_id*/ None); + assert_eq!( + app.pending_app_server_requests + .note_server_request(&approval_request), + None + ); app.enqueue_primary_thread_request(approval_request).await?; app.enqueue_primary_thread_session( test_thread_session(thread_id, test_path_buf("/tmp/project")), @@ -6809,6 +7032,66 @@ mod tests { panic!("expected approval action to submit a thread-scoped op"); } + #[tokio::test] + async fn resolved_buffered_approval_does_not_become_actionable_after_drain() -> Result<()> { + let (mut app, mut app_event_rx, _op_rx) = make_test_app_with_channels().await; + let thread_id = ThreadId::new(); + let approval_request = + exec_approval_request(thread_id, "turn-1", "call-1", /*approval_id*/ None); + + app.enqueue_primary_thread_session( + test_thread_session(thread_id, test_path_buf("/tmp/project")), + Vec::new(), + ) + .await?; + while app_event_rx.try_recv().is_ok() {} + + assert_eq!( + app.pending_app_server_requests + .note_server_request(&approval_request), + None + ); + app.enqueue_thread_request(thread_id, approval_request) + .await?; + + let resolved = app + .pending_app_server_requests + .resolve_notification(&AppServerRequestId::Integer(1)) + .expect("matching app-server request should resolve"); + app.chat_widget.dismiss_app_server_request(&resolved); + while app_event_rx.try_recv().is_ok() {} + + let rx = app + .active_thread_rx + .as_mut() + .expect("primary thread receiver should be active"); + let event = time::timeout(Duration::from_millis(50), rx.recv()) + .await + .expect("timed out waiting for buffered approval event") + .expect("channel closed unexpectedly"); + + assert!(matches!( + &event, + ThreadBufferedEvent::Request(ServerRequest::CommandExecutionRequestApproval { + params, + .. + }) if params.turn_id == "turn-1" + )); + + app.handle_thread_event_now(event); + app.chat_widget + .handle_key_event(KeyEvent::new(KeyCode::Char('y'), KeyModifiers::NONE)); + + while let Ok(app_event) = app_event_rx.try_recv() { + assert!( + !matches!(app_event, AppEvent::SubmitThreadOp { .. }), + "resolved buffered approval should not become actionable" + ); + } + + Ok(()) + } + #[tokio::test] async fn enqueue_primary_thread_session_replays_turns_before_initial_prompt_submit() -> Result<()> { @@ -7989,6 +8272,113 @@ mod tests { Ok(()) } + #[tokio::test] + async fn update_memory_settings_persists_and_updates_widget_config() -> Result<()> { + let (mut app, _app_event_rx, _op_rx) = make_test_app_with_channels().await; + let codex_home = tempdir()?; + app.config.codex_home = codex_home.path().to_path_buf().abs(); + let mut app_server = crate::start_embedded_app_server_for_picker(&app.config).await?; + + app.update_memory_settings_with_app_server( + &mut app_server, + /*use_memories*/ false, + /*generate_memories*/ false, + ) + .await; + + assert!(!app.config.memories.use_memories); + assert!(!app.config.memories.generate_memories); + assert!(!app.chat_widget.config_ref().memories.use_memories); + assert!(!app.chat_widget.config_ref().memories.generate_memories); + + let config = std::fs::read_to_string(codex_home.path().join("config.toml"))?; + let config_value = toml::from_str::(&config)?; + let memories = config_value + .as_table() + .and_then(|table| table.get("memories")) + .and_then(TomlValue::as_table) + .expect("memories table should exist"); + assert_eq!( + memories.get("use_memories"), + Some(&TomlValue::Boolean(false)) + ); + assert_eq!( + memories.get("generate_memories"), + Some(&TomlValue::Boolean(false)) + ); + assert!( + !memories.contains_key("no_memories_if_mcp_or_web_search"), + "the TUI menu should not write the MCP pollution setting" + ); + app_server.shutdown().await?; + Ok(()) + } + + #[tokio::test] + async fn update_memory_settings_updates_current_thread_memory_mode() -> Result<()> { + let (mut app, _app_event_rx, _op_rx) = make_test_app_with_channels().await; + let codex_home = tempdir()?; + app.config.codex_home = codex_home.path().to_path_buf().abs(); + // Seed the previous setting so this test exercises the thread-mode update path. + app.config.memories.generate_memories = true; + + let mut app_server = crate::start_embedded_app_server_for_picker(&app.config).await?; + let started = app_server.start_thread(&app.config).await?; + let thread_id = started.session.thread_id; + app.active_thread_id = Some(thread_id); + + app.update_memory_settings_with_app_server( + &mut app_server, + /*use_memories*/ true, + /*generate_memories*/ false, + ) + .await; + + let state_db = codex_state::StateRuntime::init( + codex_home.path().to_path_buf(), + app.config.model_provider_id.clone(), + ) + .await + .expect("state db should initialize"); + let memory_mode = state_db + .get_thread_memory_mode(thread_id) + .await + .expect("thread memory mode should be readable"); + assert_eq!(memory_mode.as_deref(), Some("disabled")); + + app_server.shutdown().await?; + Ok(()) + } + + #[tokio::test] + async fn reset_memories_clears_local_memory_directories() -> Result<()> { + let (mut app, _app_event_rx, _op_rx) = make_test_app_with_channels().await; + let codex_home = tempdir()?; + app.config.codex_home = codex_home.path().to_path_buf().abs(); + app.config.sqlite_home = codex_home.path().to_path_buf(); + + let memory_root = codex_home.path().join("memories"); + let extensions_root = codex_home.path().join("memories_extensions"); + std::fs::create_dir_all(memory_root.join("rollout_summaries"))?; + std::fs::create_dir_all(&extensions_root)?; + std::fs::write(memory_root.join("MEMORY.md"), "stale memory\n")?; + std::fs::write( + memory_root.join("rollout_summaries").join("stale.md"), + "stale summary\n", + )?; + std::fs::write(extensions_root.join("stale.txt"), "stale extension\n")?; + + let mut app_server = crate::start_embedded_app_server_for_picker(&app.config).await?; + + app.reset_memories_with_app_server(&mut app_server).await; + + assert_eq!(std::fs::read_dir(&memory_root)?.count(), 0); + assert_eq!(std::fs::read_dir(&extensions_root)?.count(), 0); + + app_server.shutdown().await?; + Ok(()) + } + #[tokio::test] async fn update_feature_flags_enabling_guardian_selects_guardian_approvals() -> Result<()> { let (mut app, mut app_event_rx, mut op_rx) = make_test_app_with_channels().await; @@ -8894,6 +9284,7 @@ guardian_approval = true approval_policy: primary_session.approval_policy, sandbox_policy: primary_session.sandbox_policy.clone(), network: None, + file_system_sandbox_policy: None, model: "gpt-agent".to_string(), personality: None, collaboration_mode: None, @@ -11117,11 +11508,18 @@ guardian_approval = true #[tokio::test] async fn interrupt_without_active_turn_is_treated_as_handled() { let mut app = make_test_app().await; - let thread_id = ThreadId::new(); let mut app_server = crate::start_embedded_app_server_for_picker(app.chat_widget.config_ref()) .await .expect("embedded app server"); + let started = app_server + .start_thread(app.chat_widget.config_ref()) + .await + .expect("thread/start should succeed"); + let thread_id = started.session.thread_id; + app.enqueue_primary_thread_session(started.session, started.turns) + .await + .expect("primary thread should be registered"); let op = AppCommand::interrupt(); let handled = app @@ -11193,14 +11591,33 @@ guardian_approval = true session_summary( TokenUsage::default(), /*thread_id*/ None, - /*thread_name*/ None + /*thread_name*/ None, + /*rollout_path*/ None, ) .is_none() ); } #[tokio::test] - async fn session_summary_includes_resume_hint() { + async fn session_summary_skips_resume_hint_until_rollout_exists() { + let usage = TokenUsage::default(); + let conversation = ThreadId::from_string("123e4567-e89b-12d3-a456-426614174000").unwrap(); + let temp_dir = tempdir().expect("temp dir"); + let rollout_path = temp_dir.path().join("rollout.jsonl"); + + assert!( + session_summary( + usage, + Some(conversation), + /*thread_name*/ None, + Some(&rollout_path), + ) + .is_none() + ); + } + + #[tokio::test] + async fn session_summary_includes_resume_hint_for_persisted_rollout() { let usage = TokenUsage { input_tokens: 10, output_tokens: 2, @@ -11208,9 +11625,17 @@ guardian_approval = true ..Default::default() }; let conversation = ThreadId::from_string("123e4567-e89b-12d3-a456-426614174000").unwrap(); + let temp_dir = tempdir().expect("temp dir"); + let rollout_path = temp_dir.path().join("rollout.jsonl"); + std::fs::write(&rollout_path, "{}\n").expect("write rollout"); - let summary = - session_summary(usage, Some(conversation), /*thread_name*/ None).expect("summary"); + let summary = session_summary( + usage, + Some(conversation), + /*thread_name*/ None, + Some(&rollout_path), + ) + .expect("summary"); assert_eq!( summary.usage_line, Some("Token usage: total=12 input=10 output=2".to_string()) @@ -11230,9 +11655,17 @@ guardian_approval = true ..Default::default() }; let conversation = ThreadId::from_string("123e4567-e89b-12d3-a456-426614174000").unwrap(); + let temp_dir = tempdir().expect("temp dir"); + let rollout_path = temp_dir.path().join("rollout.jsonl"); + std::fs::write(&rollout_path, "{}\n").expect("write rollout"); - let summary = session_summary(usage, Some(conversation), Some("my-session".to_string())) - .expect("summary"); + let summary = session_summary( + usage, + Some(conversation), + Some("my-session".to_string()), + Some(&rollout_path), + ) + .expect("summary"); assert_eq!( summary.resume_command, Some("codex resume my-session".to_string()) diff --git a/codex-rs/tui/src/app/app_server_adapter.rs b/codex-rs/tui/src/app/app_server_adapter.rs index 09b8ce06cd..f2490ca6a2 100644 --- a/codex-rs/tui/src/app/app_server_adapter.rs +++ b/codex-rs/tui/src/app/app_server_adapter.rs @@ -158,8 +158,12 @@ impl App { ) { match ¬ification { ServerNotification::ServerRequestResolved(notification) => { - self.pending_app_server_requests - .resolve_notification(¬ification.request_id); + if let Some(request) = self + .pending_app_server_requests + .resolve_notification(¬ification.request_id) + { + self.chat_widget.dismiss_app_server_request(&request); + } } ServerNotification::McpServerStatusUpdated(_) => { self.refresh_mcp_startup_expected_servers_from_config(); diff --git a/codex-rs/tui/src/app/app_server_requests.rs b/codex-rs/tui/src/app/app_server_requests.rs index b1393e95e0..e2483eb25d 100644 --- a/codex-rs/tui/src/app/app_server_requests.rs +++ b/codex-rs/tui/src/app/app_server_requests.rs @@ -1,4 +1,5 @@ use std::collections::HashMap; +use std::collections::VecDeque; use crate::app_command::AppCommand; use crate::app_command::AppCommandView; @@ -27,12 +28,32 @@ pub(super) struct UnsupportedAppServerRequest { pub(super) message: String, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ResolvedAppServerRequest { + ExecApproval { + id: String, + }, + FileChangeApproval { + id: String, + }, + PermissionsApproval { + id: String, + }, + UserInput { + call_id: String, + }, + McpElicitation { + server_name: String, + request_id: McpRequestId, + }, +} + #[derive(Debug, Default)] pub(super) struct PendingAppServerRequests { exec_approvals: HashMap, file_change_approvals: HashMap, permissions_approvals: HashMap, - user_inputs: HashMap, + user_inputs: HashMap>, mcp_requests: HashMap, } @@ -70,7 +91,12 @@ impl PendingAppServerRequests { } ServerRequest::ToolRequestUserInput { request_id, params } => { self.user_inputs - .insert(params.turn_id.clone(), request_id.clone()); + .entry(params.turn_id.clone()) + .or_default() + .push_back(PendingUserInputRequest { + item_id: params.item_id.clone(), + request_id: request_id.clone(), + }); None } ServerRequest::McpServerElicitationRequest { request_id, params } => { @@ -165,11 +191,10 @@ impl PendingAppServerRequests { }) .transpose()?, AppCommandView::UserInputAnswer { id, response } => self - .user_inputs - .remove(id) - .map(|request_id| { + .pop_user_input_request_for_turn(id) + .map(|pending| { Ok::(AppServerRequestResolution { - request_id, + request_id: pending.request_id, result: serde_json::to_value( serde_json::from_value::( serde_json::to_value(response).map_err(|err| { @@ -229,15 +254,131 @@ impl PendingAppServerRequests { Ok(resolution) } - pub(super) fn resolve_notification(&mut self, request_id: &AppServerRequestId) { - self.exec_approvals.retain(|_, value| value != request_id); - self.file_change_approvals - .retain(|_, value| value != request_id); - self.permissions_approvals - .retain(|_, value| value != request_id); - self.user_inputs.retain(|_, value| value != request_id); - self.mcp_requests.retain(|_, value| value != request_id); + pub(super) fn resolve_notification( + &mut self, + request_id: &AppServerRequestId, + ) -> Option { + if let Some(id) = self + .exec_approvals + .iter() + .find_map(|(id, value)| (value == request_id).then(|| id.clone())) + { + self.exec_approvals.remove(&id); + return Some(ResolvedAppServerRequest::ExecApproval { id }); + } + + if let Some(id) = self + .file_change_approvals + .iter() + .find_map(|(id, value)| (value == request_id).then(|| id.clone())) + { + self.file_change_approvals.remove(&id); + return Some(ResolvedAppServerRequest::FileChangeApproval { id }); + } + + if let Some(id) = self + .permissions_approvals + .iter() + .find_map(|(id, value)| (value == request_id).then(|| id.clone())) + { + self.permissions_approvals.remove(&id); + return Some(ResolvedAppServerRequest::PermissionsApproval { id }); + } + + if let Some(pending) = self.remove_user_input_request(request_id) { + return Some(ResolvedAppServerRequest::UserInput { + call_id: pending.item_id, + }); + } + + if let Some(key) = self + .mcp_requests + .iter() + .find_map(|(key, value)| (value == request_id).then(|| key.clone())) + { + self.mcp_requests.remove(&key); + return Some(ResolvedAppServerRequest::McpElicitation { + server_name: key.server_name, + request_id: key.request_id, + }); + } + + None } + + pub(super) fn contains_server_request(&self, request: &ServerRequest) -> bool { + match request { + ServerRequest::CommandExecutionRequestApproval { request_id, .. } => self + .exec_approvals + .values() + .any(|pending_request_id| pending_request_id == request_id), + ServerRequest::FileChangeRequestApproval { request_id, .. } => self + .file_change_approvals + .values() + .any(|pending_request_id| pending_request_id == request_id), + ServerRequest::PermissionsRequestApproval { request_id, .. } => self + .permissions_approvals + .values() + .any(|pending_request_id| pending_request_id == request_id), + ServerRequest::ToolRequestUserInput { request_id, .. } => { + self.user_inputs.values().any(|queue| { + queue + .iter() + .any(|pending| &pending.request_id == request_id) + }) + } + ServerRequest::McpServerElicitationRequest { request_id, .. } => self + .mcp_requests + .values() + .any(|pending_request_id| pending_request_id == request_id), + ServerRequest::DynamicToolCall { .. } + | ServerRequest::ChatgptAuthTokensRefresh { .. } + | ServerRequest::ApplyPatchApproval { .. } + | ServerRequest::ExecCommandApproval { .. } => true, + } + } + + fn pop_user_input_request_for_turn( + &mut self, + turn_id: &str, + ) -> Option { + let pending = self + .user_inputs + .get_mut(turn_id) + .and_then(VecDeque::pop_front); + if self + .user_inputs + .get(turn_id) + .is_some_and(VecDeque::is_empty) + { + self.user_inputs.remove(turn_id); + } + pending + } + + fn remove_user_input_request( + &mut self, + request_id: &AppServerRequestId, + ) -> Option { + let (turn_id, index) = self.user_inputs.iter().find_map(|(turn_id, queue)| { + queue + .iter() + .position(|pending| &pending.request_id == request_id) + .map(|index| (turn_id.clone(), index)) + })?; + let queue = self.user_inputs.get_mut(&turn_id)?; + let removed = queue.remove(index); + if queue.is_empty() { + self.user_inputs.remove(&turn_id); + } + removed + } +} + +#[derive(Debug)] +struct PendingUserInputRequest { + item_id: String, + request_id: AppServerRequestId, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -272,6 +413,7 @@ fn file_change_decision(decision: &ReviewDecision) -> Result, }, + /// Update memory settings and persist them to config.toml. + UpdateMemorySettings { + use_memories: bool, + generate_memories: bool, + }, + + /// Clear all persisted local memory artifacts via the app-server. + ResetMemories, + /// Update whether the full access warning prompt has been acknowledged. UpdateFullAccessWarningAcknowledged(bool), diff --git a/codex-rs/tui/src/app_server_session.rs b/codex-rs/tui/src/app_server_session.rs index 8e4f540153..8f666b1b25 100644 --- a/codex-rs/tui/src/app_server_session.rs +++ b/codex-rs/tui/src/app_server_session.rs @@ -18,6 +18,7 @@ use codex_app_server_protocol::GetAccountParams; use codex_app_server_protocol::GetAccountRateLimitsResponse; use codex_app_server_protocol::GetAccountResponse; use codex_app_server_protocol::JSONRPCErrorError; +use codex_app_server_protocol::MemoryResetResponse; use codex_app_server_protocol::Model as ApiModel; use codex_app_server_protocol::ModelListParams; use codex_app_server_protocol::ModelListResponse; @@ -38,6 +39,9 @@ use codex_app_server_protocol::ThreadListParams; use codex_app_server_protocol::ThreadListResponse; use codex_app_server_protocol::ThreadLoadedListParams; use codex_app_server_protocol::ThreadLoadedListResponse; +use codex_app_server_protocol::ThreadMemoryMode; +use codex_app_server_protocol::ThreadMemoryModeSetParams; +use codex_app_server_protocol::ThreadMemoryModeSetResponse; use codex_app_server_protocol::ThreadReadParams; use codex_app_server_protocol::ThreadReadResponse; use codex_app_server_protocol::ThreadRealtimeAppendAudioParams; @@ -472,6 +476,10 @@ impl AppServerSession { Ok(()) } + pub(crate) async fn startup_interrupt(&mut self, thread_id: ThreadId) -> Result<()> { + self.turn_interrupt(thread_id, String::new()).await + } + pub(crate) async fn turn_steer( &mut self, thread_id: ThreadId, @@ -512,6 +520,39 @@ impl AppServerSession { Ok(()) } + pub(crate) async fn thread_memory_mode_set( + &mut self, + thread_id: ThreadId, + mode: ThreadMemoryMode, + ) -> Result<()> { + let request_id = self.next_request_id(); + let _: ThreadMemoryModeSetResponse = self + .client + .request_typed(ClientRequest::ThreadMemoryModeSet { + request_id, + params: ThreadMemoryModeSetParams { + thread_id: thread_id.to_string(), + mode, + }, + }) + .await + .wrap_err("thread/memoryMode/set failed in TUI")?; + Ok(()) + } + + pub(crate) async fn memory_reset(&mut self) -> Result<()> { + let request_id = self.next_request_id(); + let _: MemoryResetResponse = self + .client + .request_typed(ClientRequest::MemoryReset { + request_id, + params: None, + }) + .await + .wrap_err("memory/reset failed in TUI")?; + Ok(()) + } + pub(crate) async fn thread_unsubscribe(&mut self, thread_id: ThreadId) -> Result<()> { let request_id = self.next_request_id(); let _: ThreadUnsubscribeResponse = self diff --git a/codex-rs/tui/src/bottom_pane/app_link_view.rs b/codex-rs/tui/src/bottom_pane/app_link_view.rs index e39d547625..4ffdb304d7 100644 --- a/codex-rs/tui/src/bottom_pane/app_link_view.rs +++ b/codex-rs/tui/src/bottom_pane/app_link_view.rs @@ -24,6 +24,7 @@ use super::scroll_state::ScrollState; use super::selection_popup_common::GenericDisplayRow; use super::selection_popup_common::measure_rows_height; use super::selection_popup_common::render_rows; +use crate::app::app_server_requests::ResolvedAppServerRequest; use crate::app_event::AppEvent; use crate::app_event_sender::AppEventSender; use crate::key_hint; @@ -474,6 +475,25 @@ impl BottomPaneView for AppLinkView { fn is_complete(&self) -> bool { self.complete } + + fn dismiss_app_server_request(&mut self, request: &ResolvedAppServerRequest) -> bool { + let ResolvedAppServerRequest::McpElicitation { + server_name, + request_id, + } = request + else { + return false; + }; + let Some(target) = self.elicitation_target.as_ref() else { + return false; + }; + if target.server_name != *server_name || target.request_id != *request_id { + return false; + } + + self.complete = true; + true + } } impl crate::render::renderable::Renderable for AppLinkView { @@ -546,9 +566,11 @@ impl crate::render::renderable::Renderable for AppLinkView { #[cfg(test)] mod tests { use super::*; + use crate::app::app_server_requests::ResolvedAppServerRequest; use crate::app_event::AppEvent; use crate::render::renderable::Renderable; use insta::assert_snapshot; + use pretty_assertions::assert_eq; use tokio::sync::mpsc::unbounded_channel; fn suggestion_target() -> AppLinkElicitationTarget { @@ -889,6 +911,64 @@ mod tests { assert!(view.is_complete()); } + #[test] + fn resolved_tool_suggestion_dismisses_matching_view() { + let (tx_raw, _rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let mut view = AppLinkView::new( + AppLinkViewParams { + app_id: "connector_google_calendar".to_string(), + title: "Google Calendar".to_string(), + description: Some("Plan events and schedules.".to_string()), + instructions: "Enable this app to use it for the current request.".to_string(), + url: "https://example.test/google-calendar".to_string(), + is_installed: true, + is_enabled: false, + suggest_reason: Some("Plan and reference events from your calendar".to_string()), + suggestion_type: Some(AppLinkSuggestionType::Enable), + elicitation_target: Some(suggestion_target()), + }, + tx, + ); + + assert!( + view.dismiss_app_server_request(&ResolvedAppServerRequest::McpElicitation { + server_name: "codex_apps".to_string(), + request_id: McpRequestId::String("request-1".to_string()), + }) + ); + assert!(view.is_complete()); + } + + #[test] + fn resolved_tool_suggestion_ignores_non_matching_request() { + let (tx_raw, _rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let mut view = AppLinkView::new( + AppLinkViewParams { + app_id: "connector_google_calendar".to_string(), + title: "Google Calendar".to_string(), + description: Some("Plan events and schedules.".to_string()), + instructions: "Enable this app to use it for the current request.".to_string(), + url: "https://example.test/google-calendar".to_string(), + is_installed: true, + is_enabled: false, + suggest_reason: Some("Plan and reference events from your calendar".to_string()), + suggestion_type: Some(AppLinkSuggestionType::Enable), + elicitation_target: Some(suggestion_target()), + }, + tx, + ); + + assert!( + !view.dismiss_app_server_request(&ResolvedAppServerRequest::McpElicitation { + server_name: "other_server".to_string(), + request_id: McpRequestId::String("request-1".to_string()), + }) + ); + assert!(!view.is_complete()); + } + #[test] fn install_suggestion_with_reason_snapshot() { let (tx_raw, _rx) = unbounded_channel::(); diff --git a/codex-rs/tui/src/bottom_pane/approval_overlay.rs b/codex-rs/tui/src/bottom_pane/approval_overlay.rs index 656e812d66..0e2bb2b2e3 100644 --- a/codex-rs/tui/src/bottom_pane/approval_overlay.rs +++ b/codex-rs/tui/src/bottom_pane/approval_overlay.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use std::path::PathBuf; +use crate::app::app_server_requests::ResolvedAppServerRequest; use crate::app_event::AppEvent; use crate::app_event_sender::AppEventSender; use crate::bottom_pane::BottomPaneView; @@ -97,6 +98,35 @@ impl ApprovalRequest { | ApprovalRequest::McpElicitation { thread_label, .. } => thread_label.as_deref(), } } + + fn matches_resolved_request(&self, request: &ResolvedAppServerRequest) -> bool { + match (self, request) { + ( + ApprovalRequest::Exec { id, .. }, + ResolvedAppServerRequest::ExecApproval { id: resolved_id }, + ) => id == resolved_id, + ( + ApprovalRequest::Permissions { call_id, .. }, + ResolvedAppServerRequest::PermissionsApproval { id }, + ) => call_id == id, + ( + ApprovalRequest::ApplyPatch { id, .. }, + ResolvedAppServerRequest::FileChangeApproval { id: resolved_id }, + ) => id == resolved_id, + ( + ApprovalRequest::McpElicitation { + server_name, + request_id, + .. + }, + ResolvedAppServerRequest::McpElicitation { + server_name: resolved_server_name, + request_id: resolved_request_id, + }, + ) => server_name == resolved_server_name && request_id == resolved_request_id, + _ => false, + } + } } /// Modal overlay asking the user to approve or deny one or more requests. @@ -131,6 +161,23 @@ impl ApprovalOverlay { self.queue.push(req); } + fn dismiss_resolved_request(&mut self, request: &ResolvedAppServerRequest) -> bool { + let queue_len = self.queue.len(); + self.queue + .retain(|queued_request| !queued_request.matches_resolved_request(request)); + if self + .current_request + .as_ref() + .is_some_and(|current_request| current_request.matches_resolved_request(request)) + { + self.current_complete = true; + self.advance_queue(); + return true; + } + + self.queue.len() != queue_len + } + fn set_current(&mut self, request: ApprovalRequest) { self.current_complete = false; let header = build_header(&request); @@ -465,6 +512,10 @@ impl BottomPaneView for ApprovalOverlay { self.enqueue_request(request); None } + + fn dismiss_app_server_request(&mut self, request: &ResolvedAppServerRequest) -> bool { + self.dismiss_resolved_request(request) + } } impl Renderable for ApprovalOverlay { @@ -951,6 +1002,27 @@ mod tests { assert!(saw_op, "expected approval decision to emit an op"); } + #[test] + fn resolved_request_dismisses_overlay_without_emitting_abort() { + let (tx, mut rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx); + let mut view = ApprovalOverlay::new(make_exec_request(), tx, Features::with_defaults()); + + assert!( + view.dismiss_app_server_request(&ResolvedAppServerRequest::ExecApproval { + id: "test".to_string(), + }) + ); + assert!( + view.is_complete(), + "resolved request should close the overlay" + ); + assert!( + rx.try_recv().is_err(), + "dismissing a stale request should not emit an approval op" + ); + } + #[test] fn o_opens_source_thread_for_cross_thread_approval() { let (tx, mut rx) = unbounded_channel::(); diff --git a/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs b/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs index 35165db491..3c0ab2c74e 100644 --- a/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs +++ b/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs @@ -1,3 +1,4 @@ +use crate::app::app_server_requests::ResolvedAppServerRequest; use crate::bottom_pane::ApprovalRequest; use crate::bottom_pane::McpServerElicitationFormRequest; use crate::render::renderable::Renderable; @@ -6,6 +7,13 @@ use crossterm::event::KeyEvent; use super::CancellationEvent; +/// Reason an active bottom-pane view finished. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ViewCompletion { + Accepted, + Cancelled, +} + /// Trait implemented by every view that can be shown in the bottom pane. pub(crate) trait BottomPaneView: Renderable { /// Handle a key event while the view is active. A redraw is always @@ -17,6 +25,19 @@ pub(crate) trait BottomPaneView: Renderable { false } + /// Return the completion reason once the view has finished. + fn completion(&self) -> Option { + None + } + + /// Return true when this view should be removed after a child view is accepted. + fn dismiss_after_child_accept(&self) -> bool { + false + } + + /// Clear any pending child-flow cleanup marker after a child view is cancelled. + fn clear_dismiss_after_child_accept(&mut self) {} + /// Stable identifier for views that need external refreshes while open. fn view_id(&self) -> Option<&'static str> { None @@ -87,4 +108,11 @@ pub(crate) trait BottomPaneView: Renderable { ) -> Option { Some(request) } + + /// Dismiss a request that was resolved by another client. + /// + /// Returns `true` when the view changed state. + fn dismiss_app_server_request(&mut self, _request: &ResolvedAppServerRequest) -> bool { + false + } } diff --git a/codex-rs/tui/src/bottom_pane/chat_composer.rs b/codex-rs/tui/src/bottom_pane/chat_composer.rs index b1c3129ff6..5b39a0f01d 100644 --- a/codex-rs/tui/src/bottom_pane/chat_composer.rs +++ b/codex-rs/tui/src/bottom_pane/chat_composer.rs @@ -209,8 +209,7 @@ use crate::legacy_core::plugins::PluginCapabilitySummary; use crate::legacy_core::skills::model::SkillMetadata; use crate::tui::FrameRequester; use crate::ui_consts::LIVE_PREFIX_COLS; -use codex_chatgpt::connectors; -use codex_chatgpt::connectors::AppInfo; +use codex_app_server_protocol::AppInfo; use codex_file_search::FileMatch; use std::cell::RefCell; use std::collections::HashMap; @@ -3417,9 +3416,9 @@ impl ChatComposer { if !connector.is_accessible || !connector.is_enabled { continue; } - let display_name = connectors::connector_display_label(connector); + let display_name = codex_connectors::metadata::connector_display_label(connector); let description = Some(Self::connector_brief_description(connector)); - let slug = crate::legacy_core::connectors::connector_mention_slug(connector); + let slug = codex_connectors::metadata::connector_mention_slug(connector); let search_terms = vec![display_name.clone(), connector.id.clone(), slug.clone()]; let connector_id = connector.id.as_str(); mentions.push(MentionItem { diff --git a/codex-rs/tui/src/bottom_pane/command_popup.rs b/codex-rs/tui/src/bottom_pane/command_popup.rs index 8bef8ddbc8..29da003dc2 100644 --- a/codex-rs/tui/src/bottom_pane/command_popup.rs +++ b/codex-rs/tui/src/bottom_pane/command_popup.rs @@ -293,7 +293,7 @@ mod tests { CommandItem::Builtin(cmd) => cmd.command(), }) .collect(); - assert_eq!(cmds, vec!["model", "mention", "mcp"]); + assert_eq!(cmds, vec!["model", "memories", "mention", "mcp"]); } #[test] diff --git a/codex-rs/tui/src/bottom_pane/custom_prompt_view.rs b/codex-rs/tui/src/bottom_pane/custom_prompt_view.rs index e9f0ee697f..bce9528fde 100644 --- a/codex-rs/tui/src/bottom_pane/custom_prompt_view.rs +++ b/codex-rs/tui/src/bottom_pane/custom_prompt_view.rs @@ -18,6 +18,7 @@ use super::popup_consts::standard_popup_hint_line; use super::CancellationEvent; use super::bottom_pane_view::BottomPaneView; +use super::bottom_pane_view::ViewCompletion; use super::textarea::TextArea; use super::textarea::TextAreaState; @@ -34,24 +35,31 @@ pub(crate) struct CustomPromptView { // UI state textarea: TextArea, textarea_state: RefCell, - complete: bool, + completion: Option, } impl CustomPromptView { pub(crate) fn new( title: String, placeholder: String, + initial_text: String, context_label: Option, on_submit: PromptSubmitted, ) -> Self { + let mut textarea = TextArea::new(); + if !initial_text.is_empty() { + textarea.set_text_clearing_elements(&initial_text); + textarea.set_cursor(initial_text.len()); + } + Self { title, placeholder, context_label, on_submit, - textarea: TextArea::new(), + textarea, textarea_state: RefCell::new(TextAreaState::default()), - complete: false, + completion: None, } } } @@ -72,7 +80,7 @@ impl BottomPaneView for CustomPromptView { let text = self.textarea.text().trim().to_string(); if !text.is_empty() { (self.on_submit)(text); - self.complete = true; + self.completion = Some(ViewCompletion::Accepted); } } KeyEvent { @@ -88,12 +96,16 @@ impl BottomPaneView for CustomPromptView { } fn on_ctrl_c(&mut self) -> CancellationEvent { - self.complete = true; + self.completion = Some(ViewCompletion::Cancelled); CancellationEvent::Handled } fn is_complete(&self) -> bool { - self.complete + self.completion.is_some() + } + + fn completion(&self) -> Option { + self.completion } fn handle_paste(&mut self, pasted: String) -> bool { diff --git a/codex-rs/tui/src/bottom_pane/list_selection_view.rs b/codex-rs/tui/src/bottom_pane/list_selection_view.rs index 30d55bbb12..efdc63d00d 100644 --- a/codex-rs/tui/src/bottom_pane/list_selection_view.rs +++ b/codex-rs/tui/src/bottom_pane/list_selection_view.rs @@ -21,6 +21,7 @@ use crate::render::renderable::Renderable; use super::CancellationEvent; use super::bottom_pane_view::BottomPaneView; +use super::bottom_pane_view::ViewCompletion; use super::popup_consts::MAX_POPUP_ROWS; use super::scroll_state::ScrollState; pub(crate) use super::selection_popup_common::ColumnWidthMode; @@ -121,6 +122,7 @@ pub(crate) struct SelectionItem { pub is_disabled: bool, pub actions: Vec, pub dismiss_on_select: bool, + pub dismiss_parent_on_child_accept: bool, pub search_value: Option, pub disabled_reason: Option, } @@ -211,7 +213,8 @@ pub(crate) struct ListSelectionView { footer_hint: Option>, items: Vec, state: ScrollState, - complete: bool, + completion: Option, + dismiss_after_child_accept: bool, app_event_tx: AppEventSender, is_searchable: bool, search_query: String, @@ -259,7 +262,8 @@ impl ListSelectionView { footer_hint: params.footer_hint, items: params.items, state: ScrollState::new(), - complete: false, + completion: None, + dismiss_after_child_accept: false, app_event_tx, is_searchable: params.is_searchable, search_query: String::new(), @@ -458,13 +462,15 @@ impl ListSelectionView { act(&self.app_event_tx); } if item.dismiss_on_select { - self.complete = true; + self.completion = Some(ViewCompletion::Accepted); + } else if item.dismiss_parent_on_child_accept { + self.dismiss_after_child_accept = true; } } else if selected_item.is_none() { if let Some(cb) = &self.on_cancel { cb(&self.app_event_tx); } - self.complete = true; + self.completion = Some(ViewCompletion::Cancelled); } } @@ -682,7 +688,19 @@ impl BottomPaneView for ListSelectionView { } fn is_complete(&self) -> bool { - self.complete + self.completion.is_some() + } + + fn completion(&self) -> Option { + self.completion + } + + fn dismiss_after_child_accept(&self) -> bool { + self.dismiss_after_child_accept + } + + fn clear_dismiss_after_child_accept(&mut self) { + self.dismiss_after_child_accept = false; } fn view_id(&self) -> Option<&'static str> { @@ -697,7 +715,7 @@ impl BottomPaneView for ListSelectionView { if let Some(cb) = &self.on_cancel { cb(&self.app_event_tx); } - self.complete = true; + self.completion = Some(ViewCompletion::Cancelled); CancellationEvent::Handled } } diff --git a/codex-rs/tui/src/bottom_pane/mcp_server_elicitation.rs b/codex-rs/tui/src/bottom_pane/mcp_server_elicitation.rs index 1e3ad65912..c9844ff915 100644 --- a/codex-rs/tui/src/bottom_pane/mcp_server_elicitation.rs +++ b/codex-rs/tui/src/bottom_pane/mcp_server_elicitation.rs @@ -28,6 +28,7 @@ use ratatui::widgets::Widget; use serde_json::Value; use unicode_width::UnicodeWidthStr; +use crate::app::app_server_requests::ResolvedAppServerRequest; use crate::app_event_sender::AppEventSender; use crate::bottom_pane::CancellationEvent; use crate::bottom_pane::ChatComposer; @@ -1145,6 +1146,16 @@ impl McpServerElicitationOverlay { ); } + fn advance_queue_or_complete(&mut self) { + if let Some(next) = self.queue.pop_front() { + self.request = next; + self.reset_for_request(); + self.restore_current_draft(); + } else { + self.done = true; + } + } + fn submit_answers(&mut self) { self.save_current_draft(); if let Some(idx) = self.first_required_unanswered_index() { @@ -1181,13 +1192,7 @@ impl McpServerElicitationOverlay { /*content*/ None, meta, ); - if let Some(next) = self.queue.pop_front() { - self.request = next; - self.reset_for_request(); - self.restore_current_draft(); - } else { - self.done = true; - } + self.advance_queue_or_complete(); return; } let content = self @@ -1205,13 +1210,28 @@ impl McpServerElicitationOverlay { Some(Value::Object(content)), /*meta*/ None, ); - if let Some(next) = self.queue.pop_front() { - self.request = next; - self.reset_for_request(); - self.restore_current_draft(); - } else { - self.done = true; + self.advance_queue_or_complete(); + } + + fn dismiss_resolved_request(&mut self, request: &ResolvedAppServerRequest) -> bool { + let ResolvedAppServerRequest::McpElicitation { + server_name, + request_id, + } = request + else { + return false; + }; + + let queue_len = self.queue.len(); + self.queue.retain(|queued_request| { + queued_request.server_name != *server_name || queued_request.request_id != *request_id + }); + if self.request.server_name == *server_name && self.request.request_id == *request_id { + self.advance_queue_or_complete(); + return true; } + + self.queue.len() != queue_len } fn go_next_or_submit(&mut self) { @@ -1642,6 +1662,10 @@ impl BottomPaneView for McpServerElicitationOverlay { self.queue.push_back(request); None } + + fn dismiss_app_server_request(&mut self, request: &ResolvedAppServerRequest) -> bool { + self.dismiss_resolved_request(request) + } } fn wrap_footer_tips(width: u16, tips: Vec) -> Vec> { @@ -2362,6 +2386,72 @@ mod tests { assert_eq!(overlay.request.message, "Third"); } + #[test] + fn resolved_request_dismisses_overlay_without_emitting_events() { + let (tx, mut rx) = test_sender(); + let thread_id = ThreadId::default(); + let supported_form_schema = serde_json::json!({ + "type": "object", + "properties": { + "confirmed": { + "type": "boolean", + "title": "Confirm", + } + }, + }); + let mut overlay = McpServerElicitationOverlay::new( + McpServerElicitationFormRequest::from_event( + thread_id, + form_request("First", supported_form_schema.clone(), /*meta*/ None), + ) + .expect("expected supported form"), + tx, + /*has_input_focus*/ true, + /*enhanced_keys_supported*/ false, + /*disable_paste_burst*/ false, + ); + overlay.try_consume_mcp_server_elicitation_request( + McpServerElicitationFormRequest::from_event( + thread_id, + ElicitationRequestEvent { + turn_id: Some("turn-2".to_string()), + server_name: "server-1".to_string(), + id: McpRequestId::String("request-2".to_string()), + request: ElicitationRequest::Form { + meta: None, + message: "Second".to_string(), + requested_schema: supported_form_schema, + }, + }, + ) + .expect("expected supported form"), + ); + + assert!( + overlay.dismiss_app_server_request(&ResolvedAppServerRequest::McpElicitation { + server_name: "server-1".to_string(), + request_id: McpRequestId::String("request-1".to_string()), + }) + ); + assert_eq!(overlay.request.message, "Second"); + assert!(matches!( + rx.try_recv(), + Err(tokio::sync::mpsc::error::TryRecvError::Empty) + )); + + assert!( + overlay.dismiss_app_server_request(&ResolvedAppServerRequest::McpElicitation { + server_name: "server-1".to_string(), + request_id: McpRequestId::String("request-2".to_string()), + }) + ); + assert!(overlay.is_complete()); + assert!(matches!( + rx.try_recv(), + Err(tokio::sync::mpsc::error::TryRecvError::Empty) + )); + } + #[test] fn boolean_form_snapshot() { let (tx, _rx) = test_sender(); diff --git a/codex-rs/tui/src/bottom_pane/memories_settings_view.rs b/codex-rs/tui/src/bottom_pane/memories_settings_view.rs new file mode 100644 index 0000000000..d2d3c00ce3 --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/memories_settings_view.rs @@ -0,0 +1,477 @@ +use crossterm::event::KeyCode; +use crossterm::event::KeyEvent; +use crossterm::event::KeyModifiers; +use ratatui::buffer::Buffer; +use ratatui::layout::Constraint; +use ratatui::layout::Layout; +use ratatui::layout::Rect; +use ratatui::style::Stylize; +use ratatui::text::Line; +use ratatui::widgets::Block; +use ratatui::widgets::Widget; + +use crate::app_event::AppEvent; +use crate::app_event_sender::AppEventSender; +use crate::bottom_pane::popup_consts::standard_popup_hint_line; +use crate::key_hint; +use crate::render::Insets; +use crate::render::RectExt as _; +use crate::render::renderable::ColumnRenderable; +use crate::render::renderable::Renderable; +use crate::style::user_message_style; + +use super::CancellationEvent; +use super::bottom_pane_view::BottomPaneView; +use super::popup_consts::MAX_POPUP_ROWS; +use super::scroll_state::ScrollState; +use super::selection_popup_common::GenericDisplayRow; +use super::selection_popup_common::measure_rows_height; +use super::selection_popup_common::render_rows; + +const MEMORIES_DOC_URL: &str = "https://developers.openai.com/codex/memories"; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum MemoriesSetting { + Use, + Generate, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum MemoriesAction { + Reset, +} + +enum MemoriesMenuItem { + Setting { + setting: MemoriesSetting, + name: &'static str, + description: &'static str, + enabled: bool, + }, + Action { + action: MemoriesAction, + name: &'static str, + description: &'static str, + }, +} + +pub(crate) struct MemoriesSettingsView { + items: Vec, + state: ScrollState, + reset_confirmation: Option, + complete: bool, + app_event_tx: AppEventSender, + docs_link: Line<'static>, +} + +impl MemoriesSettingsView { + pub(crate) fn new( + use_memories: bool, + generate_memories: bool, + app_event_tx: AppEventSender, + ) -> Self { + let mut view = Self { + items: vec![ + MemoriesMenuItem::Setting { + setting: MemoriesSetting::Use, + name: "Use memories", + description: "Use memories in the following threads. Applied at next thread.", + enabled: use_memories, + }, + MemoriesMenuItem::Setting { + setting: MemoriesSetting::Generate, + name: "Generate memories", + description: "Generate memories from the following threads. Current thread included.", + enabled: generate_memories, + }, + MemoriesMenuItem::Action { + action: MemoriesAction::Reset, + name: "Reset all memories", + description: "Clear local memory files and summaries. Existing threads stay intact.", + }, + ], + state: ScrollState::new(), + reset_confirmation: None, + complete: false, + app_event_tx, + docs_link: Line::from(vec![ + "Learn more: ".dim(), + MEMORIES_DOC_URL.cyan().underlined(), + ]), + }; + view.initialize_selection(); + view + } + + fn initialize_selection(&mut self) { + self.state.selected_idx = (!self.items.is_empty()).then_some(0); + } + + fn settings_header(&self) -> ColumnRenderable<'_> { + let mut header = ColumnRenderable::new(); + header.push(Line::from("Memories".bold())); + header.push(Line::from( + "Choose how Codex uses and creates memories. Changes are saved to config.toml".dim(), + )); + header + } + + fn reset_confirmation_header(&self) -> ColumnRenderable<'_> { + let mut header = ColumnRenderable::new(); + header.push(Line::from("Reset all memories?".bold())); + header.push(Line::from( + "This clears local memory files and rollout summaries for the current Codex home." + .dim(), + )); + header + } + + fn active_state(&self) -> &ScrollState { + self.reset_confirmation.as_ref().unwrap_or(&self.state) + } + + fn active_state_mut(&mut self) -> &mut ScrollState { + self.reset_confirmation.as_mut().unwrap_or(&mut self.state) + } + + fn visible_len(&self) -> usize { + if self.reset_confirmation.is_some() { + 2 + } else { + self.items.len() + } + } + + fn build_rows(&self) -> Vec { + if let Some(state) = self.reset_confirmation.as_ref() { + return ["Reset all memories", "Go back"] + .into_iter() + .enumerate() + .map(|(idx, name)| GenericDisplayRow { + name: if state.selected_idx == Some(idx) { + format!("› {name}") + } else { + format!(" {name}") + }, + description: Some(match idx { + 0 => "Delete local memory files and rollout summaries.".to_string(), + 1 => "Return to memory settings.".to_string(), + _ => unreachable!("reset confirmation only renders two rows"), + }), + ..Default::default() + }) + .collect(); + } + + let selected_idx = self.state.selected_idx; + self.items + .iter() + .enumerate() + .map(|(idx, item)| { + let prefix = if selected_idx == Some(idx) { + '›' + } else { + ' ' + }; + let (name, description) = match item { + MemoriesMenuItem::Setting { + name, + description, + enabled, + .. + } => ( + format!("{prefix} [{}] {name}", if *enabled { 'x' } else { ' ' }), + description, + ), + MemoriesMenuItem::Action { + name, description, .. + } => (format!("{prefix} {name}"), description), + }; + GenericDisplayRow { + name, + description: Some((*description).to_string()), + ..Default::default() + } + }) + .collect() + } + + fn move_up(&mut self) { + let len = self.visible_len(); + if len == 0 { + return; + } + let state = self.active_state_mut(); + state.move_up_wrap(len); + state.ensure_visible(len, MAX_POPUP_ROWS.min(len)); + } + + fn move_down(&mut self) { + let len = self.visible_len(); + if len == 0 { + return; + } + let state = self.active_state_mut(); + state.move_down_wrap(len); + state.ensure_visible(len, MAX_POPUP_ROWS.min(len)); + } + + fn toggle_selected(&mut self) { + if self.reset_confirmation.is_some() { + return; + } + + let Some(selected_idx) = self.state.selected_idx else { + return; + }; + + if let Some(MemoriesMenuItem::Setting { enabled, .. }) = self.items.get_mut(selected_idx) { + *enabled = !*enabled; + } + } + + fn rows_width(total_width: u16) -> u16 { + total_width.saturating_sub(2) + } + + fn current_setting(&self, setting: MemoriesSetting) -> bool { + self.items + .iter() + .find_map(|item| match item { + MemoriesMenuItem::Setting { + setting: item_setting, + enabled, + .. + } if *item_setting == setting => Some(*enabled), + _ => None, + }) + .unwrap_or(false) + } + + fn open_reset_confirmation(&mut self) { + let mut state = ScrollState::new(); + state.selected_idx = Some(0); + self.reset_confirmation = Some(state); + } + + fn close_reset_confirmation(&mut self) { + self.reset_confirmation = None; + self.state.selected_idx = self.items.len().checked_sub(1); + } + + fn footer_hint(&self) -> Line<'static> { + if self.reset_confirmation.is_some() { + standard_popup_hint_line() + } else { + memories_settings_hint_line() + } + } +} + +impl BottomPaneView for MemoriesSettingsView { + fn handle_key_event(&mut self, key_event: KeyEvent) { + match key_event { + KeyEvent { + code: KeyCode::Up, .. + } + | KeyEvent { + code: KeyCode::Char('p'), + modifiers: KeyModifiers::CONTROL, + .. + } + | KeyEvent { + code: KeyCode::Char('\u{0010}'), + modifiers: KeyModifiers::NONE, + .. + } => self.move_up(), + KeyEvent { + code: KeyCode::Char('k'), + modifiers: KeyModifiers::NONE, + .. + } => self.move_up(), + KeyEvent { + code: KeyCode::Down, + .. + } + | KeyEvent { + code: KeyCode::Char('n'), + modifiers: KeyModifiers::CONTROL, + .. + } + | KeyEvent { + code: KeyCode::Char('\u{000e}'), + modifiers: KeyModifiers::NONE, + .. + } => self.move_down(), + KeyEvent { + code: KeyCode::Char('j'), + modifiers: KeyModifiers::NONE, + .. + } => self.move_down(), + KeyEvent { + code: KeyCode::Char(' '), + modifiers: KeyModifiers::NONE, + .. + } => self.toggle_selected(), + KeyEvent { + code: KeyCode::Enter, + modifiers: KeyModifiers::NONE, + .. + } => self.save(), + KeyEvent { + code: KeyCode::Esc, .. + } => self.cancel(), + _ => {} + } + } + + fn is_complete(&self) -> bool { + self.complete + } + + fn on_ctrl_c(&mut self) -> CancellationEvent { + self.cancel(); + CancellationEvent::Handled + } +} + +impl MemoriesSettingsView { + fn save(&mut self) { + if let Some(state) = self.reset_confirmation.as_ref() { + match state.selected_idx { + Some(0) => { + self.app_event_tx.send(AppEvent::ResetMemories); + self.complete = true; + } + Some(1) | None => self.close_reset_confirmation(), + Some(other) => unreachable!("unexpected reset confirmation row: {other}"), + } + return; + } + + match self.state.selected_idx.and_then(|idx| self.items.get(idx)) { + Some(MemoriesMenuItem::Action { + action: MemoriesAction::Reset, + .. + }) => self.open_reset_confirmation(), + _ => { + self.app_event_tx.send(AppEvent::UpdateMemorySettings { + use_memories: self.current_setting(MemoriesSetting::Use), + generate_memories: self.current_setting(MemoriesSetting::Generate), + }); + self.complete = true; + } + } + } + + fn cancel(&mut self) { + if self.reset_confirmation.is_some() { + self.close_reset_confirmation(); + } else { + self.complete = true; + } + } +} + +impl Renderable for MemoriesSettingsView { + fn render(&self, area: Rect, buf: &mut Buffer) { + if area.height == 0 || area.width == 0 { + return; + } + + let [content_area, footer_area] = + Layout::vertical([Constraint::Fill(1), Constraint::Length(1)]).areas(area); + + Block::default() + .style(user_message_style()) + .render(content_area, buf); + + let header = if self.reset_confirmation.is_some() { + self.reset_confirmation_header() + } else { + self.settings_header() + }; + let header_height = header.desired_height(content_area.width.saturating_sub(4)); + let rows = self.build_rows(); + let rows_width = Self::rows_width(content_area.width); + let rows_height = measure_rows_height( + &rows, + self.active_state(), + MAX_POPUP_ROWS, + rows_width.saturating_add(1), + ); + let [header_area, _, list_area, _, docs_area] = Layout::vertical([ + Constraint::Max(header_height), + Constraint::Max(1), + Constraint::Length(rows_height), + Constraint::Max(1), + Constraint::Length(1), + ]) + .areas(content_area.inset(Insets::vh(/*v*/ 1, /*h*/ 2))); + + header.render(header_area, buf); + + if list_area.height > 0 { + let render_area = Rect { + x: list_area.x.saturating_sub(2), + y: list_area.y, + width: rows_width.max(1), + height: list_area.height, + }; + render_rows( + render_area, + buf, + &rows, + self.active_state(), + MAX_POPUP_ROWS, + " No memory settings available", + ); + } + if self.reset_confirmation.is_none() { + self.docs_link.clone().render(docs_area, buf); + } + + let hint_area = Rect { + x: footer_area.x + 2, + y: footer_area.y, + width: footer_area.width.saturating_sub(2), + height: footer_area.height, + }; + self.footer_hint().render(hint_area, buf); + } + + fn desired_height(&self, width: u16) -> u16 { + let header = if self.reset_confirmation.is_some() { + self.reset_confirmation_header() + } else { + self.settings_header() + }; + let rows = self.build_rows(); + let rows_width = Self::rows_width(width); + let rows_height = measure_rows_height( + &rows, + self.active_state(), + MAX_POPUP_ROWS, + rows_width.saturating_add(1), + ); + + let docs_height = if self.reset_confirmation.is_some() { + 0 + } else { + 1 + }; + let mut height = header.desired_height(width.saturating_sub(4)); + height = height.saturating_add(rows_height + 4 + docs_height); + height.saturating_add(1) + } +} + +fn memories_settings_hint_line() -> Line<'static> { + Line::from(vec![ + "Press ".into(), + key_hint::plain(KeyCode::Char(' ')).into(), + " to toggle; ".into(), + key_hint::plain(KeyCode::Enter).into(), + " to save or select".into(), + ]) +} diff --git a/codex-rs/tui/src/bottom_pane/mod.rs b/codex-rs/tui/src/bottom_pane/mod.rs index b2d080b048..2600a90cd9 100644 --- a/codex-rs/tui/src/bottom_pane/mod.rs +++ b/codex-rs/tui/src/bottom_pane/mod.rs @@ -15,6 +15,7 @@ //! hint. The pane schedules redraws so those hints can expire even when the UI is otherwise idle. use std::path::PathBuf; +use crate::app::app_server_requests::ResolvedAppServerRequest; use crate::app_event::ConnectorsSnapshot; use crate::app_event_sender::AppEventSender; use crate::bottom_pane::pending_input_preview::PendingInputPreview; @@ -29,6 +30,7 @@ use crate::render::renderable::Renderable; use crate::render::renderable::RenderableItem; use crate::tui::FrameRequester; use bottom_pane_view::BottomPaneView; +use bottom_pane_view::ViewCompletion; use codex_features::Features; use codex_file_search::FileMatch; use codex_protocol::request_user_input::RequestUserInputEvent; @@ -81,6 +83,7 @@ mod experimental_features_view; mod file_search_popup; mod footer; mod list_selection_view; +mod memories_settings_view; mod prompt_args; mod skill_popup; mod skills_toggle_view; @@ -91,6 +94,7 @@ pub(crate) use list_selection_view::SelectionViewParams; pub(crate) use list_selection_view::SideContentWidth; pub(crate) use list_selection_view::popup_content_width; pub(crate) use list_selection_view::side_by_side_layout_widths; +pub(crate) use memories_settings_view::MemoriesSettingsView; mod feedback_view; pub(crate) use feedback_view::FeedbackAudience; pub(crate) use feedback_view::feedback_classification; @@ -377,6 +381,35 @@ impl BottomPane { self.request_redraw(); } + fn pop_active_view_with_completion(&mut self, completion: Option) { + if self.view_stack.pop().is_some() { + match completion { + Some(ViewCompletion::Accepted) => { + while self + .view_stack + .last() + .is_some_and(|view| view.dismiss_after_child_accept()) + { + self.view_stack.pop(); + } + } + Some(ViewCompletion::Cancelled) => { + if let Some(view) = self.view_stack.last_mut() { + view.clear_dismiss_after_child_accept(); + } + } + None => {} + } + self.on_view_stack_depth_decreased(); + } + } + + fn on_view_stack_depth_decreased(&mut self) { + if self.view_stack.is_empty() { + self.on_active_view_complete(); + } + } + /// Forward a key event to the active view or the composer. pub fn handle_key_event(&mut self, key_event: KeyEvent) -> InputResult { // If a modal/view is active, handle it here; otherwise forward to composer. @@ -388,7 +421,7 @@ impl BottomPane { // We need three pieces of information after routing the key: // whether Esc completed the view, whether the view finished for any // reason, and whether a paste-burst timer should be scheduled. - let (ctrl_c_completed, view_complete, view_in_paste_burst) = { + let (ctrl_c_completed, view_complete, completion, view_in_paste_burst) = { let last_index = self.view_stack.len() - 1; let view = &mut self.view_stack[last_index]; let prefer_esc = @@ -398,24 +431,27 @@ impl BottomPane { && matches!(view.on_ctrl_c(), CancellationEvent::Handled) && view.is_complete(); if ctrl_c_completed { - (true, true, false) + (true, true, view.completion(), false) } else { view.handle_key_event(key_event); - (false, view.is_complete(), view.is_in_paste_burst()) + ( + false, + view.is_complete(), + view.completion(), + view.is_in_paste_burst(), + ) } }; if ctrl_c_completed { - self.view_stack.pop(); - self.on_active_view_complete(); + self.pop_active_view_with_completion(completion); if let Some(next_view) = self.view_stack.last() && next_view.is_in_paste_burst() { self.request_redraw_in(ChatComposer::recommended_paste_flush_delay()); } } else if view_complete { - self.view_stack.clear(); - self.on_active_view_complete(); + self.pop_active_view_with_completion(completion); } else if view_in_paste_burst { self.request_redraw_in(ChatComposer::recommended_paste_flush_delay()); } @@ -467,10 +503,11 @@ impl BottomPane { pub(crate) fn on_ctrl_c(&mut self) -> CancellationEvent { if let Some(view) = self.view_stack.last_mut() { let event = view.on_ctrl_c(); + let view_complete = view.is_complete(); + let completion = view.completion(); if matches!(event, CancellationEvent::Handled) { - if view.is_complete() { - self.view_stack.pop(); - self.on_active_view_complete(); + if view_complete { + self.pop_active_view_with_completion(completion); } self.show_quit_shortcut_hint(key_hint::ctrl(KeyCode::Char('c'))); self.request_redraw(); @@ -491,17 +528,14 @@ impl BottomPane { } pub fn handle_paste(&mut self, pasted: String) { - if !self.view_stack.is_empty() { - let (needs_redraw, view_complete) = { - let last_index = self.view_stack.len() - 1; - let view = &mut self.view_stack[last_index]; - (view.handle_paste(pasted), view.is_complete()) - }; + if let Some(view) = self.view_stack.last_mut() { + let needs_redraw = view.handle_paste(pasted); + let view_complete = view.is_complete(); if view_complete { self.view_stack.clear(); self.on_active_view_complete(); } - if needs_redraw { + if needs_redraw || view_complete { self.request_redraw(); } } else { @@ -1030,6 +1064,37 @@ impl BottomPane { self.push_view(Box::new(modal)); } + pub(crate) fn dismiss_app_server_request( + &mut self, + request: &ResolvedAppServerRequest, + ) -> bool { + if self.view_stack.is_empty() { + return false; + } + + let mut changed = false; + let mut completed_indices = Vec::new(); + for index in (0..self.view_stack.len()).rev() { + let view = &mut self.view_stack[index]; + if !view.dismiss_app_server_request(request) { + continue; + } + changed = true; + if view.is_complete() { + completed_indices.push(index); + } + } + if !changed { + return false; + } + for index in completed_indices { + self.view_stack.remove(index); + } + self.on_view_stack_depth_decreased(); + self.request_redraw(); + true + } + fn on_active_view_complete(&mut self) { self.resume_status_timer_after_modal(); self.set_composer_input_enabled(/*enabled*/ true, /*placeholder*/ None); @@ -1239,6 +1304,7 @@ impl Renderable for BottomPane { #[cfg(test)] mod tests { use super::*; + use crate::app::app_server_requests::ResolvedAppServerRequest; use crate::app_event::AppEvent; use crate::status_indicator_widget::STATUS_DETAILS_DEFAULT_MAX_LINES; use crate::status_indicator_widget::StatusDetailsCapitalization; @@ -1246,9 +1312,12 @@ mod tests { use crate::test_support::test_path_buf; use codex_protocol::protocol::Op; use codex_protocol::protocol::SkillScope; + use crossterm::event::KeyCode; + use crossterm::event::KeyEvent; use crossterm::event::KeyEventKind; use crossterm::event::KeyModifiers; use insta::assert_snapshot; + use pretty_assertions::assert_eq; use ratatui::buffer::Buffer; use ratatui::layout::Rect; use std::cell::Cell; @@ -1273,6 +1342,19 @@ mod tests { snapshot_buffer(&buf) } + fn test_pane(app_event_tx: AppEventSender) -> BottomPane { + BottomPane::new(BottomPaneParams { + app_event_tx, + frame_requester: FrameRequester::test_dummy(), + has_input_focus: true, + enhanced_keys_supported: false, + placeholder_text: "Ask Codex to do anything".to_string(), + disable_paste_burst: false, + animations_enabled: true, + skills: Some(Vec::new()), + }) + } + fn exec_request() -> ApprovalRequest { ApprovalRequest::Exec { thread_id: codex_protocol::ThreadId::new(), @@ -1289,6 +1371,73 @@ mod tests { } } + #[derive(Default)] + struct DismissibleView { + id: Option<&'static str>, + dismiss_exec_id: Option<&'static str>, + complete: bool, + } + + impl Renderable for DismissibleView { + fn render(&self, _area: Rect, _buf: &mut Buffer) {} + + fn desired_height(&self, _width: u16) -> u16 { + 0 + } + } + + impl BottomPaneView for DismissibleView { + fn is_complete(&self) -> bool { + self.complete + } + + fn view_id(&self) -> Option<&'static str> { + self.id + } + + fn dismiss_app_server_request(&mut self, request: &ResolvedAppServerRequest) -> bool { + let ResolvedAppServerRequest::ExecApproval { id } = request else { + return false; + }; + if self.dismiss_exec_id != Some(id.as_str()) { + return false; + } + + self.complete = true; + true + } + } + + #[derive(Default)] + struct CompletingView { + id: Option<&'static str>, + complete: bool, + } + + impl Renderable for CompletingView { + fn render(&self, _area: Rect, _buf: &mut Buffer) {} + + fn desired_height(&self, _width: u16) -> u16 { + 0 + } + } + + impl BottomPaneView for CompletingView { + fn handle_key_event(&mut self, key_event: KeyEvent) { + if key_event.code == KeyCode::Enter { + self.complete = true; + } + } + + fn is_complete(&self) -> bool { + self.complete + } + + fn view_id(&self) -> Option<&'static str> { + self.id + } + } + #[test] fn ctrl_c_on_modal_consumes_without_showing_quit_hint() { let (tx_raw, _rx) = unbounded_channel::(); @@ -1371,6 +1520,89 @@ mod tests { ); } + #[test] + fn dismiss_app_server_request_removes_matching_buried_view() { + let (tx_raw, _rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let mut pane = test_pane(tx); + + pane.push_view(Box::new(DismissibleView { + id: Some("buried"), + dismiss_exec_id: Some("request-1"), + complete: false, + })); + pane.push_view(Box::new(DismissibleView { + id: Some("top"), + dismiss_exec_id: None, + complete: false, + })); + + assert!( + pane.dismiss_app_server_request(&ResolvedAppServerRequest::ExecApproval { + id: "request-1".to_string(), + }) + ); + assert_eq!(pane.view_stack.len(), 1); + assert_eq!( + pane.view_stack.last().and_then(|view| view.view_id()), + Some("top") + ); + } + + #[test] + fn dismiss_app_server_request_returns_false_when_no_view_matches() { + let (tx_raw, _rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let mut pane = test_pane(tx); + + pane.push_view(Box::new(DismissibleView { + id: Some("first"), + dismiss_exec_id: Some("other-request"), + complete: false, + })); + pane.push_view(Box::new(DismissibleView { + id: Some("second"), + dismiss_exec_id: None, + complete: false, + })); + + assert!( + !pane.dismiss_app_server_request(&ResolvedAppServerRequest::ExecApproval { + id: "request-1".to_string(), + }) + ); + assert_eq!(pane.view_stack.len(), 2); + assert_eq!( + pane.view_stack.last().and_then(|view| view.view_id()), + Some("second") + ); + } + + #[test] + fn completing_top_view_preserves_underlying_view() { + let (tx_raw, _rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let mut pane = test_pane(tx); + + pane.push_view(Box::new(DismissibleView { + id: Some("underlying"), + dismiss_exec_id: None, + complete: false, + })); + pane.push_view(Box::new(CompletingView { + id: Some("top"), + complete: false, + })); + + pane.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + + assert_eq!(pane.view_stack.len(), 1); + assert_eq!( + pane.view_stack.last().and_then(|view| view.view_id()), + Some("underlying") + ); + } + #[test] fn composer_shown_after_denied_while_task_running() { let (tx_raw, _rx) = unbounded_channel::(); diff --git a/codex-rs/tui/src/bottom_pane/request_user_input/mod.rs b/codex-rs/tui/src/bottom_pane/request_user_input/mod.rs index f7db7c65dd..bc07c06e0b 100644 --- a/codex-rs/tui/src/bottom_pane/request_user_input/mod.rs +++ b/codex-rs/tui/src/bottom_pane/request_user_input/mod.rs @@ -10,6 +10,7 @@ use std::collections::HashMap; use std::collections::VecDeque; use std::path::PathBuf; +use crate::app::app_server_requests::ResolvedAppServerRequest; use crossterm::event::KeyCode; use crossterm::event::KeyEvent; use crossterm::event::KeyEventKind; @@ -198,6 +199,17 @@ impl RequestUserInputOverlay { self.request.questions.len() } + fn advance_queue_or_complete(&mut self) { + if let Some(next) = self.queue.pop_front() { + self.request = next; + self.reset_for_request(); + self.ensure_focus_available(); + self.restore_current_draft(); + } else { + self.done = true; + } + } + fn has_options(&self) -> bool { self.current_question() .and_then(|question| question.options.as_ref()) @@ -759,14 +771,23 @@ impl RequestUserInputOverlay { interrupted: false, }, ))); - if let Some(next) = self.queue.pop_front() { - self.request = next; - self.reset_for_request(); - self.ensure_focus_available(); - self.restore_current_draft(); - } else { - self.done = true; + self.advance_queue_or_complete(); + } + + fn dismiss_resolved_request(&mut self, request: &ResolvedAppServerRequest) -> bool { + let ResolvedAppServerRequest::UserInput { call_id } = request else { + return false; + }; + + let queue_len = self.queue.len(); + self.queue + .retain(|queued_request| queued_request.call_id != *call_id); + if self.request.call_id == *call_id { + self.advance_queue_or_complete(); + return true; } + + self.queue.len() != queue_len } fn open_unanswered_confirmation(&mut self) { @@ -1273,6 +1294,10 @@ impl BottomPaneView for RequestUserInputOverlay { self.queue.push_back(request); None } + + fn dismiss_app_server_request(&mut self, request: &ResolvedAppServerRequest) -> bool { + self.dismiss_resolved_request(request) + } } #[cfg(test)] @@ -1532,6 +1557,118 @@ mod tests { expect_interrupt_only(&mut rx); } + #[test] + fn resolved_request_dismisses_overlay_without_emitting_events() { + let (tx, mut rx) = test_sender(); + let mut overlay = RequestUserInputOverlay::new( + RequestUserInputEvent { + call_id: "call-1".to_string(), + turn_id: "turn-1".to_string(), + questions: vec![question_with_options("q1", "First")], + }, + tx, + /*has_input_focus*/ true, + /*enhanced_keys_supported*/ false, + /*disable_paste_burst*/ false, + ); + + assert!( + overlay.dismiss_app_server_request(&ResolvedAppServerRequest::UserInput { + call_id: "call-1".to_string(), + }) + ); + assert!(overlay.done, "resolved request should close the overlay"); + assert!( + rx.try_recv().is_err(), + "dismissing a stale request should not emit an interrupt or answer" + ); + } + + #[test] + fn resolved_current_request_advances_to_next_same_turn_prompt() { + let (tx, mut rx) = test_sender(); + let mut overlay = RequestUserInputOverlay::new( + RequestUserInputEvent { + call_id: "call-1".to_string(), + turn_id: "turn-1".to_string(), + questions: vec![question_with_options("q1", "First")], + }, + tx, + /*has_input_focus*/ true, + /*enhanced_keys_supported*/ false, + /*disable_paste_burst*/ false, + ); + overlay.try_consume_user_input_request(RequestUserInputEvent { + call_id: "call-2".to_string(), + turn_id: "turn-1".to_string(), + questions: vec![question_with_options("q2", "Second")], + }); + + assert!( + overlay.dismiss_app_server_request(&ResolvedAppServerRequest::UserInput { + call_id: "call-1".to_string(), + }) + ); + + assert!(!overlay.done, "newer same-turn prompt should stay pending"); + assert_eq!(overlay.request.call_id, "call-2"); + assert_eq!(overlay.request.turn_id, "turn-1"); + assert_eq!(overlay.request.questions[0].id, "q2"); + assert!( + rx.try_recv().is_err(), + "dismissing a stale request should not emit an interrupt or answer" + ); + } + + #[test] + fn resolved_queued_request_removes_only_that_prompt() { + let (tx, mut rx) = test_sender(); + let mut overlay = RequestUserInputOverlay::new( + RequestUserInputEvent { + call_id: "call-1".to_string(), + turn_id: "turn-1".to_string(), + questions: vec![question_with_options("q1", "First")], + }, + tx, + /*has_input_focus*/ true, + /*enhanced_keys_supported*/ false, + /*disable_paste_burst*/ false, + ); + overlay.try_consume_user_input_request(RequestUserInputEvent { + call_id: "call-2".to_string(), + turn_id: "turn-1".to_string(), + questions: vec![question_with_options("q2", "Second")], + }); + overlay.try_consume_user_input_request(RequestUserInputEvent { + call_id: "call-3".to_string(), + turn_id: "turn-1".to_string(), + questions: vec![question_with_options("q3", "Third")], + }); + + assert!( + overlay.dismiss_app_server_request(&ResolvedAppServerRequest::UserInput { + call_id: "call-2".to_string(), + }) + ); + + assert_eq!(overlay.request.call_id, "call-1"); + assert!( + rx.try_recv().is_err(), + "dismissing a stale queued request should not emit an event" + ); + overlay.submit_answers(); + assert_eq!(overlay.request.call_id, "call-3"); + assert_eq!(overlay.request.questions[0].id, "q3"); + assert!( + rx.try_recv().is_ok(), + "submitting the still-current prompt should emit an answer" + ); + assert!( + rx.try_recv().is_ok(), + "submitting the still-current prompt should emit a history cell" + ); + } + #[test] fn options_can_submit_empty_when_unanswered() { let (tx, mut rx) = test_sender(); diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__status_line_setup__tests__setup_view_snapshot_uses_runtime_preview_values.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__status_line_setup__tests__setup_view_snapshot_uses_runtime_preview_values.snap index 679aa77f4a..20aca1e334 100644 --- a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__status_line_setup__tests__setup_view_snapshot_uses_runtime_preview_values.snap +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__status_line_setup__tests__setup_view_snapshot_uses_runtime_preview_values.snap @@ -8,14 +8,14 @@ expression: "render_lines(&view, 72)" Type to search > -› [x] model-name Current model name - [x] current-dir Current working directory - [x] git-branch Current Git branch (omitted when unavail… - [ ] model-with-reasoning Current model name with reasoning level - [ ] project-root Project root directory (omitted when una… - [ ] context-remaining Percentage of context window remaining (… - [ ] context-remaining-... Percentage of context window remaining (… - [ ] context-used Percentage of context window used (omitt… +› [x] model-name Current model name + [x] current-dir Current working directory + [x] git-branch Current Git branch (omitted when unavaila… + [ ] model-with-reasoning Current model name with reasoning level + [ ] project-root Project root directory (omitted when unav… + [ ] context-remaining Percentage of context window remaining (o… + [ ] context-used Percentage of context window used (omitte… + [ ] five-hour-limit Remaining usage on 5-hour usage limit (om… gpt-5-codex · ~/codex-rs · jif/statusline-preview Use ↑↓ to navigate, ←→ to move, space to select, enter to confirm, esc diff --git a/codex-rs/tui/src/bottom_pane/status_line_setup.rs b/codex-rs/tui/src/bottom_pane/status_line_setup.rs index b5fe8456ed..4c7e29c053 100644 --- a/codex-rs/tui/src/bottom_pane/status_line_setup.rs +++ b/codex-rs/tui/src/bottom_pane/status_line_setup.rs @@ -66,10 +66,6 @@ pub(crate) enum StatusLineItem { /// Percentage of context window remaining. ContextRemaining, - /// Percentage of context window remaining. - #[strum(to_string = "context-remaining-percent")] - ContextRemainingPercent, - /// Percentage of context window used. /// /// Also accepts the legacy `context-usage` config value. @@ -119,9 +115,6 @@ impl StatusLineItem { StatusLineItem::ContextRemaining => { "Percentage of context window remaining (omitted when unknown)" } - StatusLineItem::ContextRemainingPercent => { - "Percentage of context window remaining (omitted when unknown)" - } StatusLineItem::ContextUsed => { "Percentage of context window used (omitted when unknown)" } @@ -321,7 +314,7 @@ mod tests { } #[test] - fn context_remaining_is_separate_selectable_id() { + fn context_remaining_is_selectable_id() { assert_eq!( "context-remaining".parse::(), Ok(StatusLineItem::ContextRemaining) @@ -332,18 +325,6 @@ mod tests { ); } - #[test] - fn context_remaining_percent_is_separate_selectable_id() { - assert_eq!( - StatusLineItem::ContextRemainingPercent.to_string(), - "context-remaining-percent" - ); - assert_eq!( - "context-remaining-percent".parse::(), - Ok(StatusLineItem::ContextRemainingPercent) - ); - } - #[test] fn preview_uses_runtime_values() { let preview_data = StatusLinePreviewData::from_iter([ diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 1a9b1630c7..f67150a310 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -43,6 +43,7 @@ use std::time::Duration; use std::time::Instant; use self::realtime::PendingSteerCompareKey; +use crate::app::app_server_requests::ResolvedAppServerRequest; use crate::app_command::AppCommand; use crate::app_event::RealtimeAudioDeviceKind; use crate::app_server_approval_conversions::network_approval_context_to_core; @@ -54,7 +55,7 @@ use crate::bottom_pane::StatusLinePreviewData; use crate::bottom_pane::StatusLineSetupView; use crate::bottom_pane::TerminalTitleItem; use crate::bottom_pane::TerminalTitleSetupView; -use crate::legacy_core::DEFAULT_PROJECT_DOC_FILENAME; +use crate::legacy_core::DEFAULT_AGENTS_MD_FILENAME; use crate::legacy_core::config::Config; use crate::legacy_core::config::Constrained; use crate::legacy_core::config::ConstraintResult; @@ -80,6 +81,7 @@ use crate::text_formatting::proper_join; use crate::version::CODEX_CLI_VERSION; use codex_app_server_protocol::AddCreditsNudgeCreditType; use codex_app_server_protocol::AddCreditsNudgeEmailStatus; +use codex_app_server_protocol::AppInfo; use codex_app_server_protocol::AppSummary; use codex_app_server_protocol::CodexErrorInfo as AppServerCodexErrorInfo; use codex_app_server_protocol::CollabAgentState as AppServerCollabAgentState; @@ -254,6 +256,11 @@ const MULTI_AGENT_ENABLE_TITLE: &str = "Enable subagents?"; const MULTI_AGENT_ENABLE_YES: &str = "Yes, enable"; const MULTI_AGENT_ENABLE_NO: &str = "Not now"; const MULTI_AGENT_ENABLE_NOTICE: &str = "Subagents will be enabled in the next session."; +const MEMORIES_DOC_URL: &str = "https://developers.openai.com/codex/memories"; +const MEMORIES_ENABLE_TITLE: &str = "Enable memories?"; +const MEMORIES_ENABLE_YES: &str = "Yes, enable"; +const MEMORIES_ENABLE_NO: &str = "Not now"; +const MEMORIES_ENABLE_NOTICE: &str = "Memories will be enabled in the next session."; const PLAN_MODE_REASONING_SCOPE_TITLE: &str = "Apply reasoning change"; const PLAN_MODE_REASONING_SCOPE_PLAN_ONLY: &str = "Apply to Plan mode override"; const PLAN_MODE_REASONING_SCOPE_ALL_MODES: &str = "Apply to global default and Plan mode override"; @@ -314,6 +321,7 @@ use crate::bottom_pane::ExperimentalFeaturesView; use crate::bottom_pane::InputResult; use crate::bottom_pane::LocalImageAttachment; use crate::bottom_pane::McpServerElicitationFormRequest; +use crate::bottom_pane::MemoriesSettingsView; use crate::bottom_pane::MentionBinding; use crate::bottom_pane::QUIT_SHORTCUT_TIMEOUT; use crate::bottom_pane::SelectionAction; @@ -2162,6 +2170,16 @@ impl ChatWidget { self.request_redraw(); } + pub(crate) fn dismiss_app_server_request(&mut self, request: &ResolvedAppServerRequest) { + // A remotely resolved request must not remain user-actionable. It may be + // materialized in the bottom pane or still deferred behind active streaming. + let removed_deferred = self.interrupts.remove_resolved_prompt(request); + let removed_visible = self.bottom_pane.dismiss_app_server_request(request); + if removed_deferred || removed_visible { + self.request_redraw(); + } + } + pub(crate) fn open_feedback_consent(&mut self, category: crate::app_event::FeedbackCategory) { let snapshot = self.feedback.snapshot(self.thread_id); let params = crate::bottom_pane::feedback_upload_consent_params( @@ -2593,6 +2611,61 @@ impl ChatWidget { }); } + pub(crate) fn open_memories_popup(&mut self) { + if !self.config.features.enabled(Feature::MemoryTool) { + self.open_memories_enable_prompt(); + return; + } + + let view = MemoriesSettingsView::new( + self.config.memories.use_memories, + self.config.memories.generate_memories, + self.app_event_tx.clone(), + ); + self.bottom_pane.show_view(Box::new(view)); + } + + pub(crate) fn open_memories_enable_prompt(&mut self) { + let items = vec![ + SelectionItem { + name: MEMORIES_ENABLE_YES.to_string(), + description: Some( + "Save the setting now. You will need a new session to use it.".to_string(), + ), + actions: vec![Box::new(|tx| { + tx.send(AppEvent::UpdateFeatureFlags { + updates: vec![(Feature::MemoryTool, true)], + }); + })], + dismiss_on_select: true, + ..Default::default() + }, + SelectionItem { + name: MEMORIES_ENABLE_NO.to_string(), + description: Some("Keep memories disabled.".to_string()), + dismiss_on_select: true, + ..Default::default() + }, + ]; + + self.bottom_pane.show_selection_view(SelectionViewParams { + title: Some(MEMORIES_ENABLE_TITLE.to_string()), + subtitle: Some("Memories are currently disabled in your config.".to_string()), + footer_note: Some(Line::from(vec![ + "Learn more: ".dim(), + MEMORIES_DOC_URL.cyan().underlined(), + ])), + footer_hint: Some(standard_popup_hint_line()), + items, + ..Default::default() + }); + } + + pub(crate) fn set_memory_settings(&mut self, use_memories: bool, generate_memories: bool) { + self.config.memories.use_memories = use_memories; + self.config.memories.generate_memories = generate_memories; + } + pub(crate) fn set_token_info(&mut self, info: Option) { match info { Some(info) => self.apply_token_info(info), @@ -4754,6 +4827,7 @@ impl ChatWidget { invocation, duration, result, + .. } = ev; let extra_cell = match self @@ -5275,11 +5349,8 @@ impl ChatWidget { fn show_rename_prompt(&mut self) { let tx = self.app_event_tx.clone(); - let has_name = self - .thread_name - .as_ref() - .is_some_and(|name| !name.is_empty()); - let title = if has_name { + let existing_name = self.thread_name.as_deref().filter(|name| !name.is_empty()); + let title = if existing_name.is_some() { "Rename thread" } else { "Name thread" @@ -5287,6 +5358,7 @@ impl ChatWidget { let view = CustomPromptView::new( title.to_string(), "Type a name and press Enter".to_string(), + /*initial_text*/ existing_name.unwrap_or_default().to_string(), /*context_label*/ None, Box::new(move |name: String| { let Some(name) = crate::legacy_core::util::normalize_thread_name(&name) else { @@ -5523,7 +5595,7 @@ impl ChatWidget { let app_mentions = find_app_mentions(&mentions, apps, &skill_names_lower); for app in app_mentions { - let slug = crate::legacy_core::connectors::connector_mention_slug(&app); + let slug = codex_connectors::metadata::connector_mention_slug(&app); if bound_names.contains(&slug) || !selected_app_ids.insert(app.id.clone()) { continue; } @@ -5940,6 +6012,7 @@ impl ChatWidget { server, tool, arguments, + mcp_app_resource_uri, result, error, duration_ms, @@ -5952,15 +6025,19 @@ impl ChatWidget { tool, arguments: Some(arguments), }, + mcp_app_resource_uri, duration: Duration::from_millis(duration_ms.unwrap_or_default().max(0) as u64), result: match (result, error) { (_, Some(error)) => Err(error.message), - (Some(result), None) => Ok(codex_protocol::mcp::CallToolResult { - content: result.content, - structured_content: result.structured_content, - is_error: Some(false), - meta: None, - }), + (Some(result), None) => { + let result = *result; + Ok(codex_protocol::mcp::CallToolResult { + content: result.content, + structured_content: result.structured_content, + is_error: Some(false), + meta: None, + }) + } (None, None) => Err("MCP tool call completed without a result".to_string()), }, }); @@ -6450,6 +6527,7 @@ impl ChatWidget { server, tool, arguments, + mcp_app_resource_uri, .. } => { self.on_mcp_tool_call_begin(McpToolCallBeginEvent { @@ -6459,6 +6537,7 @@ impl ChatWidget { tool, arguments: Some(arguments), }, + mcp_app_resource_uri, }); } ThreadItem::WebSearch { id, .. } => { @@ -8043,6 +8122,7 @@ impl ChatWidget { is_default: preset.is_default, actions, dismiss_on_select: single_supported_effort, + dismiss_parent_on_child_accept: !single_supported_effort, ..Default::default() }); } @@ -9790,7 +9870,7 @@ impl ChatWidget { self.config.features.enabled(Feature::Apps) && self.has_chatgpt_account } - fn connectors_for_mentions(&self) -> Option<&[connectors::AppInfo]> { + fn connectors_for_mentions(&self) -> Option<&[AppInfo]> { if !self.connectors_enabled() { return None; } @@ -9818,14 +9898,17 @@ impl ChatWidget { /// Build a placeholder header cell while the session is configuring. fn placeholder_session_header_cell(config: &Config) -> Box { let placeholder_style = Style::default().add_modifier(Modifier::DIM | Modifier::ITALIC); - Box::new(history_cell::SessionHeaderHistoryCell::new_with_style( - DEFAULT_MODEL_DISPLAY_NAME.to_string(), - placeholder_style, - /*reasoning_effort*/ None, - /*show_fast_status*/ false, - config.cwd.to_path_buf(), - CODEX_CLI_VERSION, - )) + Box::new( + history_cell::SessionHeaderHistoryCell::new_with_style( + DEFAULT_MODEL_DISPLAY_NAME.to_string(), + placeholder_style, + /*reasoning_effort*/ None, + /*show_fast_status*/ false, + config.cwd.to_path_buf(), + CODEX_CLI_VERSION, + ) + .with_yolo_mode(history_cell::is_yolo_mode(config)), + ) } /// Merge the real session info cell with any placeholder header to avoid double boxes. @@ -9861,6 +9944,13 @@ impl ChatWidget { self.request_redraw(); } + pub(crate) fn add_memories_enable_notice(&mut self) { + self.add_to_history(history_cell::new_warning_event( + MEMORIES_ENABLE_NOTICE.to_string(), + )); + self.request_redraw(); + } + pub(crate) fn add_plain_history_lines(&mut self, lines: Vec>) { self.add_boxed_history(Box::new(PlainHistoryCell::new(lines))); self.request_redraw(); @@ -9967,7 +10057,7 @@ impl ChatWidget { } } - fn open_connectors_popup(&mut self, connectors: &[connectors::AppInfo]) { + fn open_connectors_popup(&mut self, connectors: &[AppInfo]) { self.bottom_pane.show_selection_view( self.connectors_popup_params(connectors, /*selected_connector_id*/ None), ); @@ -9993,7 +10083,7 @@ impl ChatWidget { fn connectors_popup_params( &self, - connectors: &[connectors::AppInfo], + connectors: &[AppInfo], selected_connector_id: Option<&str>, ) -> SelectionViewParams { let total = connectors.len(); @@ -10016,7 +10106,7 @@ impl ChatWidget { }); let mut items: Vec = Vec::with_capacity(connectors.len()); for connector in connectors { - let connector_label = connectors::connector_display_label(connector); + let connector_label = codex_connectors::metadata::connector_display_label(connector); let connector_title = connector_label.clone(); let link_description = Self::connector_description(connector); let description = Self::connector_brief_description(connector); @@ -10090,7 +10180,7 @@ impl ChatWidget { } } - fn refresh_connectors_popup_if_open(&mut self, connectors: &[connectors::AppInfo]) { + fn refresh_connectors_popup_if_open(&mut self, connectors: &[AppInfo]) { let selected_connector_id = if let (Some(selected_index), ConnectorsCacheState::Ready(snapshot)) = ( self.bottom_pane @@ -10118,7 +10208,7 @@ impl ChatWidget { ]) } - fn connector_brief_description(connector: &connectors::AppInfo) -> String { + fn connector_brief_description(connector: &AppInfo) -> String { let status_label = Self::connector_status_label(connector); match Self::connector_description(connector) { Some(description) => format!("{status_label} · {description}"), @@ -10126,7 +10216,7 @@ impl ChatWidget { } } - fn connector_status_label(connector: &connectors::AppInfo) -> &'static str { + fn connector_status_label(connector: &AppInfo) -> &'static str { if connector.is_accessible { if connector.is_enabled { "Installed" @@ -10138,7 +10228,7 @@ impl ChatWidget { } } - fn connector_description(connector: &connectors::AppInfo) -> Option { + fn connector_description(connector: &AppInfo) -> Option { connector .description .as_deref() @@ -10540,6 +10630,7 @@ impl ChatWidget { self.config.features = config.features.clone(); self.config.config_layer_stack = config.config_layer_stack.clone(); self.config.realtime = config.realtime.clone(); + self.config.memories = config.memories.clone(); } pub(crate) fn open_review_popup(&mut self) { @@ -10555,6 +10646,7 @@ impl ChatWidget { } })], dismiss_on_select: false, + dismiss_parent_on_child_accept: true, ..Default::default() }); @@ -10580,6 +10672,7 @@ impl ChatWidget { } })], dismiss_on_select: false, + dismiss_parent_on_child_accept: true, ..Default::default() }); @@ -10589,6 +10682,7 @@ impl ChatWidget { tx.send(AppEvent::OpenReviewCustomPrompt); })], dismiss_on_select: false, + dismiss_parent_on_child_accept: true, ..Default::default() }); @@ -10676,6 +10770,7 @@ impl ChatWidget { let view = CustomPromptView::new( "Custom review instructions".to_string(), "Type instructions and press Enter".to_string(), + /*initial_text*/ String::new(), /*context_label*/ None, Box::new(move |prompt: String| { let trimmed = prompt.trim().to_string(); diff --git a/codex-rs/tui/src/chatwidget/interrupts.rs b/codex-rs/tui/src/chatwidget/interrupts.rs index 0a3fc80016..f41e3b3a47 100644 --- a/codex-rs/tui/src/chatwidget/interrupts.rs +++ b/codex-rs/tui/src/chatwidget/interrupts.rs @@ -1,5 +1,6 @@ use std::collections::VecDeque; +use crate::app::app_server_requests::ResolvedAppServerRequest; use codex_protocol::approvals::ElicitationRequestEvent; use codex_protocol::protocol::ApplyPatchApprovalRequestEvent; use codex_protocol::protocol::ExecApprovalRequestEvent; @@ -86,6 +87,13 @@ impl InterruptManager { self.queue.push_back(QueuedInterrupt::PatchEnd(ev)); } + pub(crate) fn remove_resolved_prompt(&mut self, request: &ResolvedAppServerRequest) -> bool { + let original_len = self.queue.len(); + self.queue + .retain(|queued| !queued.matches_resolved_prompt(request)); + self.queue.len() != original_len + } + pub(crate) fn flush_all(&mut self, chat: &mut ChatWidget) { while let Some(q) = self.queue.pop_front() { match q { @@ -103,3 +111,144 @@ impl InterruptManager { } } } + +impl QueuedInterrupt { + fn matches_resolved_prompt(&self, request: &ResolvedAppServerRequest) -> bool { + match self { + QueuedInterrupt::ExecApproval(ev) => { + matches!(request, ResolvedAppServerRequest::ExecApproval { id } + if ev.effective_approval_id() == id.as_str()) + } + QueuedInterrupt::ApplyPatchApproval(ev) => { + matches!(request, ResolvedAppServerRequest::FileChangeApproval { id } + if ev.call_id == id.as_str()) + } + QueuedInterrupt::Elicitation(ev) => { + matches!(request, ResolvedAppServerRequest::McpElicitation { + server_name, + request_id, + } if ev.server_name == server_name.as_str() && &ev.id == request_id) + } + QueuedInterrupt::RequestPermissions(ev) => { + matches!(request, ResolvedAppServerRequest::PermissionsApproval { id } + if ev.call_id == id.as_str()) + } + QueuedInterrupt::RequestUserInput(ev) => { + matches!(request, ResolvedAppServerRequest::UserInput { call_id } + if ev.call_id == call_id.as_str()) + } + QueuedInterrupt::ExecBegin(_) + | QueuedInterrupt::ExecEnd(_) + | QueuedInterrupt::McpBegin(_) + | QueuedInterrupt::McpEnd(_) + | QueuedInterrupt::PatchEnd(_) => false, + } + } +} + +#[cfg(test)] +mod tests { + use codex_protocol::approvals::ExecApprovalRequestEvent; + use codex_protocol::protocol::ExecCommandBeginEvent; + use codex_protocol::protocol::ExecCommandSource; + use codex_protocol::request_user_input::RequestUserInputEvent; + use codex_utils_absolute_path::AbsolutePathBuf; + use pretty_assertions::assert_eq; + + use super::*; + + fn user_input(call_id: &str, turn_id: &str) -> RequestUserInputEvent { + RequestUserInputEvent { + call_id: call_id.to_string(), + turn_id: turn_id.to_string(), + questions: Vec::new(), + } + } + + fn exec_approval(call_id: &str, approval_id: Option<&str>) -> ExecApprovalRequestEvent { + ExecApprovalRequestEvent { + call_id: call_id.to_string(), + approval_id: approval_id.map(str::to_string), + turn_id: "turn".to_string(), + command: vec!["true".to_string()], + cwd: AbsolutePathBuf::current_dir().expect("current dir"), + reason: None, + network_approval_context: None, + proposed_execpolicy_amendment: None, + proposed_network_policy_amendments: None, + additional_permissions: None, + available_decisions: None, + parsed_cmd: Vec::new(), + } + } + + fn exec_begin(call_id: &str) -> ExecCommandBeginEvent { + ExecCommandBeginEvent { + call_id: call_id.to_string(), + process_id: None, + turn_id: "turn".to_string(), + command: vec!["true".to_string()], + cwd: AbsolutePathBuf::current_dir().expect("current dir"), + parsed_cmd: Vec::new(), + source: ExecCommandSource::Agent, + interaction_input: None, + } + } + + #[test] + fn remove_resolved_prompt_removes_matching_user_input_only() { + let mut manager = InterruptManager::new(); + manager.push_user_input(user_input("call-a", "turn")); + manager.push_user_input(user_input("call-b", "turn")); + + assert!( + manager.remove_resolved_prompt(&ResolvedAppServerRequest::UserInput { + call_id: "call-b".to_string(), + }) + ); + + assert_eq!(manager.queue.len(), 1); + let Some(QueuedInterrupt::RequestUserInput(remaining)) = manager.queue.front() else { + panic!("expected remaining queued user input"); + }; + assert_eq!(remaining.call_id, "call-a"); + } + + #[test] + fn remove_resolved_prompt_matches_exec_approval_id() { + let mut manager = InterruptManager::new(); + manager.push_exec_approval(exec_approval("call", Some("approval"))); + + assert!( + !manager.remove_resolved_prompt(&ResolvedAppServerRequest::ExecApproval { + id: "call".to_string(), + }) + ); + assert_eq!(manager.queue.len(), 1); + + assert!( + manager.remove_resolved_prompt(&ResolvedAppServerRequest::ExecApproval { + id: "approval".to_string(), + }) + ); + assert!(manager.queue.is_empty()); + } + + #[test] + fn remove_resolved_prompt_keeps_lifecycle_events() { + let mut manager = InterruptManager::new(); + manager.push_exec_begin(exec_begin("call")); + + assert!( + !manager.remove_resolved_prompt(&ResolvedAppServerRequest::ExecApproval { + id: "call".to_string(), + }) + ); + + assert_eq!(manager.queue.len(), 1); + assert!(matches!( + manager.queue.front(), + Some(QueuedInterrupt::ExecBegin(_)) + )); + } +} diff --git a/codex-rs/tui/src/chatwidget/skills.rs b/codex-rs/tui/src/chatwidget/skills.rs index 0bcfb3b3bd..9bbde1aaec 100644 --- a/codex-rs/tui/src/chatwidget/skills.rs +++ b/codex-rs/tui/src/chatwidget/skills.rs @@ -9,14 +9,13 @@ use crate::bottom_pane::SkillsToggleItem; use crate::bottom_pane::SkillsToggleView; use crate::bottom_pane::popup_consts::standard_popup_hint_line; use crate::legacy_core::TOOL_MENTION_SIGIL; -use crate::legacy_core::connectors::connector_mention_slug; use crate::legacy_core::skills::model::SkillDependencies; use crate::legacy_core::skills::model::SkillInterface; use crate::legacy_core::skills::model::SkillMetadata; use crate::legacy_core::skills::model::SkillToolDependency; use crate::skills_helpers::skill_description; use crate::skills_helpers::skill_display_name; -use codex_chatgpt::connectors::AppInfo; +use codex_app_server_protocol::AppInfo; use codex_protocol::parse_command::ParsedCommand; use codex_protocol::protocol::ListSkillsResponseEvent; use codex_protocol::protocol::SkillMetadata as ProtocolSkillMetadata; @@ -296,12 +295,12 @@ pub(crate) fn find_app_mentions( let mut slug_counts: HashMap = HashMap::new(); for app in apps.iter().filter(|app| app.is_enabled) { - let slug = connector_mention_slug(app); + let slug = codex_connectors::metadata::connector_mention_slug(app); *slug_counts.entry(slug).or_insert(0) += 1; } for app in apps.iter().filter(|app| app.is_enabled) { - let slug = connector_mention_slug(app); + let slug = codex_connectors::metadata::connector_mention_slug(app); let slug_count = slug_counts.get(&slug).copied().unwrap_or(0); if mentions.names.contains(&slug) && !explicit_names.contains(&slug) diff --git a/codex-rs/tui/src/chatwidget/slash_dispatch.rs b/codex-rs/tui/src/chatwidget/slash_dispatch.rs index e6e93d17fb..2c5c83b380 100644 --- a/codex-rs/tui/src/chatwidget/slash_dispatch.rs +++ b/codex-rs/tui/src/chatwidget/slash_dispatch.rs @@ -92,10 +92,10 @@ impl ChatWidget { self.app_event_tx.send(AppEvent::ForkCurrentSession); } SlashCommand::Init => { - let init_target = self.config.cwd.join(DEFAULT_PROJECT_DOC_FILENAME); + let init_target = self.config.cwd.join(DEFAULT_AGENTS_MD_FILENAME); if init_target.exists() { let message = format!( - "{DEFAULT_PROJECT_DOC_FILENAME} already exists here. Skipping /init to avoid overwriting it." + "{DEFAULT_AGENTS_MD_FILENAME} already exists here. Skipping /init to avoid overwriting it." ); self.add_info_message(message, /*hint*/ None); return; @@ -228,6 +228,9 @@ impl ChatWidget { SlashCommand::Experimental => { self.open_experimental_popup(); } + SlashCommand::Memories => { + self.open_memories_popup(); + } SlashCommand::Quit | SlashCommand::Exit => { self.request_quit_without_confirmation(); } diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_enable_prompt.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_enable_prompt.snap new file mode 100644 index 0000000000..f9b0daefcc --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_enable_prompt.snap @@ -0,0 +1,12 @@ +--- +source: tui/src/chatwidget/tests.rs +expression: popup +--- + Enable memories? + Memories are currently disabled in your config. + +› 1. Yes, enable Save the setting now. You will need a new session to use it. + 2. Not now Keep memories disabled. + + Learn more: https://developers.openai.com/codex/memories + Press enter to confirm or esc to go back diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_reset_confirmation.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_reset_confirmation.snap new file mode 100644 index 0000000000..2538e2b461 --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_reset_confirmation.snap @@ -0,0 +1,12 @@ +--- +source: tui/src/chatwidget/tests/popups_and_settings.rs +expression: popup +--- + Reset all memories? + +› Reset all memories Delete local memory files and rollout summaries. + Go back Return to memory settings. + + + + Press enter to confirm or esc to go back diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_settings_popup.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_settings_popup.snap new file mode 100644 index 0000000000..2fbe641e29 --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_settings_popup.snap @@ -0,0 +1,17 @@ +--- +source: tui/src/chatwidget/tests/popups_and_settings.rs +expression: popup +--- + Memories + Choose how Codex uses and creates memories. Changes are saved to config.toml + +› [x] Use memories Use memories in the following threads. Applied at + next thread. + [ ] Generate memories Generate memories from the following threads. Current + thread included. + Reset all memories Clear local memory files and summaries. Existing + threads stay intact. + + Learn more: https://developers.openai.com/codex/memories + + Press space to toggle; enter to save or select diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__slash_rename_prefilled_prompt.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__slash_rename_prefilled_prompt.snap new file mode 100644 index 0000000000..f1eb0d223b --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__slash_rename_prefilled_prompt.snap @@ -0,0 +1,10 @@ +--- +source: tui/src/chatwidget/tests/slash_commands.rs +assertion_line: 135 +expression: popup +--- +▌ Rename thread +▌ +▌ Current project title + +Press enter to confirm or esc to go back diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__status_line_model_with_reasoning_context_remaining_percent_footer.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__status_line_model_with_reasoning_context_remaining_footer.snap similarity index 100% rename from codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__status_line_model_with_reasoning_context_remaining_percent_footer.snap rename to codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__status_line_model_with_reasoning_context_remaining_footer.snap diff --git a/codex-rs/tui/src/chatwidget/status_surfaces.rs b/codex-rs/tui/src/chatwidget/status_surfaces.rs index ba95da93b3..a744711c2c 100644 --- a/codex-rs/tui/src/chatwidget/status_surfaces.rs +++ b/codex-rs/tui/src/chatwidget/status_surfaces.rs @@ -450,7 +450,7 @@ impl ChatWidget { Some(format!("{} used", format_tokens_compact(total))) } } - StatusLineItem::ContextRemaining | StatusLineItem::ContextRemainingPercent => self + StatusLineItem::ContextRemaining => self .status_line_context_remaining_percent() .map(|remaining| format!("Context {remaining}% left")), StatusLineItem::ContextUsed => self diff --git a/codex-rs/tui/src/chatwidget/tests/helpers.rs b/codex-rs/tui/src/chatwidget/tests/helpers.rs index c504d39d5a..90d3090f87 100644 --- a/codex-rs/tui/src/chatwidget/tests/helpers.rs +++ b/codex-rs/tui/src/chatwidget/tests/helpers.rs @@ -10,6 +10,7 @@ pub(super) async fn test_config() -> Config { .keep(); let mut config = Config::load_default_with_cli_overrides_for_codex_home(codex_home.clone(), Vec::new()) + .await .expect("config"); config.codex_home = codex_home.abs(); config.sqlite_home = codex_home.clone(); diff --git a/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs b/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs index e2635d322d..10e3451d03 100644 --- a/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs +++ b/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs @@ -1,4 +1,5 @@ use super::*; +use codex_app_server_protocol::AppInfo; use pretty_assertions::assert_eq; #[tokio::test] @@ -511,7 +512,7 @@ async fn apps_popup_stays_loading_until_final_snapshot_updates() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: notion_id.to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -545,7 +546,7 @@ async fn apps_popup_stays_loading_until_final_snapshot_updates() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { connectors: vec![ - codex_chatgpt::connectors::AppInfo { + AppInfo { id: notion_id.to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -560,7 +561,7 @@ async fn apps_popup_stays_loading_until_final_snapshot_updates() { is_enabled: true, plugin_display_names: Vec::new(), }, - codex_chatgpt::connectors::AppInfo { + AppInfo { id: linear_id.to_string(), name: "Linear".to_string(), description: Some("Project tracking".to_string()), @@ -604,7 +605,7 @@ async fn apps_refresh_failure_keeps_existing_full_snapshot() { let linear_id = "unit_test_apps_refresh_failure_connector_2"; let full_connectors = vec![ - codex_chatgpt::connectors::AppInfo { + AppInfo { id: notion_id.to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -619,7 +620,7 @@ async fn apps_refresh_failure_keeps_existing_full_snapshot() { is_enabled: true, plugin_display_names: Vec::new(), }, - codex_chatgpt::connectors::AppInfo { + AppInfo { id: linear_id.to_string(), name: "Linear".to_string(), description: Some("Project tracking".to_string()), @@ -644,7 +645,7 @@ async fn apps_refresh_failure_keeps_existing_full_snapshot() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: notion_id.to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -693,7 +694,7 @@ async fn apps_popup_preserves_selected_app_across_refresh() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { connectors: vec![ - codex_chatgpt::connectors::AppInfo { + AppInfo { id: "notion".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -708,7 +709,7 @@ async fn apps_popup_preserves_selected_app_across_refresh() { is_enabled: true, plugin_display_names: Vec::new(), }, - codex_chatgpt::connectors::AppInfo { + AppInfo { id: "slack".to_string(), name: "Slack".to_string(), description: Some("Team chat".to_string()), @@ -739,7 +740,7 @@ async fn apps_popup_preserves_selected_app_across_refresh() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { connectors: vec![ - codex_chatgpt::connectors::AppInfo { + AppInfo { id: "airtable".to_string(), name: "Airtable".to_string(), description: Some("Spreadsheets".to_string()), @@ -754,7 +755,7 @@ async fn apps_popup_preserves_selected_app_across_refresh() { is_enabled: true, plugin_display_names: Vec::new(), }, - codex_chatgpt::connectors::AppInfo { + AppInfo { id: "notion".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -769,7 +770,7 @@ async fn apps_popup_preserves_selected_app_across_refresh() { is_enabled: true, plugin_display_names: Vec::new(), }, - codex_chatgpt::connectors::AppInfo { + AppInfo { id: "slack".to_string(), name: "Slack".to_string(), description: Some("Team chat".to_string()), @@ -812,7 +813,7 @@ async fn apps_refresh_failure_with_cached_snapshot_triggers_pending_force_refetc chat.connectors_prefetch_in_flight = true; chat.connectors_force_refetch_pending = true; - let full_connectors = vec![codex_chatgpt::connectors::AppInfo { + let full_connectors = vec![AppInfo { id: "unit_test_apps_refresh_failure_pending_connector".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -855,7 +856,7 @@ async fn apps_popup_keeps_existing_full_snapshot_while_partial_refresh_loads() { chat.bottom_pane.set_connectors_enabled(/*enabled*/ true); let full_connectors = vec![ - codex_chatgpt::connectors::AppInfo { + AppInfo { id: "unit_test_connector_1".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -870,7 +871,7 @@ async fn apps_popup_keeps_existing_full_snapshot_while_partial_refresh_loads() { is_enabled: true, plugin_display_names: Vec::new(), }, - codex_chatgpt::connectors::AppInfo { + AppInfo { id: "unit_test_connector_2".to_string(), name: "Linear".to_string(), description: Some("Project tracking".to_string()), @@ -897,7 +898,7 @@ async fn apps_popup_keeps_existing_full_snapshot_while_partial_refresh_loads() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { connectors: vec![ - codex_chatgpt::connectors::AppInfo { + AppInfo { id: "unit_test_connector_1".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -912,7 +913,7 @@ async fn apps_popup_keeps_existing_full_snapshot_while_partial_refresh_loads() { is_enabled: true, plugin_display_names: Vec::new(), }, - codex_chatgpt::connectors::AppInfo { + AppInfo { id: "connector_openai_hidden".to_string(), name: "Hidden OpenAI".to_string(), description: Some("Should be filtered".to_string()), @@ -960,7 +961,7 @@ async fn apps_refresh_failure_without_full_snapshot_falls_back_to_installed_apps chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: "unit_test_apps_refresh_failure_fallback_connector".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -1019,7 +1020,7 @@ async fn apps_popup_shows_disabled_status_for_installed_but_disabled_apps() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: "connector_1".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -1073,7 +1074,7 @@ async fn apps_initial_load_applies_enabled_state_from_config() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: "connector_1".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -1139,7 +1140,7 @@ async fn apps_initial_load_applies_enabled_state_from_requirements_with_user_ove chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: "connector_1".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -1203,7 +1204,7 @@ async fn apps_initial_load_applies_enabled_state_from_requirements_without_user_ chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: "connector_1".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -1252,7 +1253,7 @@ async fn apps_refresh_preserves_toggled_enabled_state() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: "connector_1".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -1274,7 +1275,7 @@ async fn apps_refresh_preserves_toggled_enabled_state() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: "connector_1".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -1323,7 +1324,7 @@ async fn apps_popup_for_not_installed_app_uses_install_only_selected_description chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: "connector_2".to_string(), name: "Linear".to_string(), description: Some("Project tracking".to_string()), @@ -1501,6 +1502,101 @@ async fn multi_agent_enable_prompt_updates_feature_and_emits_notice() { assert!(rendered.contains("Subagents will be enabled in the next session.")); } +#[tokio::test] +async fn memories_enable_prompt_snapshot() { + let (mut chat, _rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::MemoryTool, /*enabled*/ false); + + chat.open_memories_popup(); + + let popup = render_bottom_popup(&chat, /*width*/ 80); + assert_chatwidget_snapshot!("memories_enable_prompt", popup); +} + +#[tokio::test] +async fn memories_enable_prompt_updates_feature_without_notice() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::MemoryTool, /*enabled*/ false); + + chat.open_memories_popup(); + chat.handle_key_event(KeyEvent::from(KeyCode::Enter)); + + assert_matches!( + rx.try_recv(), + Ok(AppEvent::UpdateFeatureFlags { updates }) if updates == vec![(Feature::MemoryTool, true)] + ); + assert!( + rx.try_recv().is_err(), + "memory enable prompt should not emit the success notice before persistence succeeds" + ); +} + +#[tokio::test] +async fn memories_settings_popup_snapshot() { + let (mut chat, _rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::MemoryTool, /*enabled*/ true); + chat.config.memories.use_memories = true; + chat.config.memories.generate_memories = false; + + chat.open_memories_popup(); + + let popup = render_bottom_popup(&chat, /*width*/ 80); + assert_chatwidget_snapshot!("memories_settings_popup", popup); +} + +#[tokio::test] +async fn memories_reset_confirmation_snapshot() { + let (mut chat, _rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::MemoryTool, /*enabled*/ true); + chat.config.memories.use_memories = true; + chat.config.memories.generate_memories = false; + + chat.open_memories_popup(); + chat.handle_key_event(KeyEvent::from(KeyCode::Down)); + chat.handle_key_event(KeyEvent::from(KeyCode::Down)); + chat.handle_key_event(KeyEvent::from(KeyCode::Enter)); + + let popup = render_bottom_popup(&chat, /*width*/ 80); + assert_chatwidget_snapshot!("memories_reset_confirmation", popup); +} + +#[tokio::test] +async fn memories_settings_toggle_saves_on_enter() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::MemoryTool, /*enabled*/ true); + chat.config.memories.use_memories = true; + chat.config.memories.generate_memories = false; + + chat.open_memories_popup(); + chat.handle_key_event(KeyEvent::from(KeyCode::Down)); + chat.handle_key_event(KeyEvent::from(KeyCode::Char(' '))); + chat.handle_key_event(KeyEvent::from(KeyCode::Enter)); + + assert_matches!( + rx.try_recv(), + Ok(AppEvent::UpdateMemorySettings { + use_memories: true, + generate_memories: true, + }) + ); +} + +#[tokio::test] +async fn memories_reset_confirmation_sends_event_on_confirm() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::MemoryTool, /*enabled*/ true); + chat.config.memories.use_memories = true; + chat.config.memories.generate_memories = false; + + chat.open_memories_popup(); + chat.handle_key_event(KeyEvent::from(KeyCode::Down)); + chat.handle_key_event(KeyEvent::from(KeyCode::Down)); + chat.handle_key_event(KeyEvent::from(KeyCode::Enter)); + chat.handle_key_event(KeyEvent::from(KeyCode::Enter)); + + assert_matches!(rx.try_recv(), Ok(AppEvent::ResetMemories)); +} + #[tokio::test] async fn model_selection_popup_snapshot() { let (mut chat, _rx, _op_rx) = make_chatwidget_manual(Some("gpt-5-codex")).await; diff --git a/codex-rs/tui/src/chatwidget/tests/slash_commands.rs b/codex-rs/tui/src/chatwidget/tests/slash_commands.rs index 2afb959349..c7caacc2eb 100644 --- a/codex-rs/tui/src/chatwidget/tests/slash_commands.rs +++ b/codex-rs/tui/src/chatwidget/tests/slash_commands.rs @@ -73,7 +73,7 @@ async fn ctrl_d_with_modal_open_does_not_quit() { async fn slash_init_skips_when_project_doc_exists() { let (mut chat, mut rx, mut op_rx) = make_chatwidget_manual(/*model_override*/ None).await; let tempdir = tempdir().unwrap(); - let existing_path = tempdir.path().join(DEFAULT_PROJECT_DOC_FILENAME); + let existing_path = tempdir.path().join(DEFAULT_AGENTS_MD_FILENAME); std::fs::write(&existing_path, "existing instructions").unwrap(); chat.config.cwd = tempdir.path().to_path_buf().abs(); @@ -88,7 +88,7 @@ async fn slash_init_skips_when_project_doc_exists() { assert_eq!(cells.len(), 1, "expected one info message"); let rendered = lines_to_single_string(&cells[0]); assert!( - rendered.contains(DEFAULT_PROJECT_DOC_FILENAME), + rendered.contains(DEFAULT_AGENTS_MD_FILENAME), "info message should mention the existing file: {rendered:?}" ); assert!( @@ -124,6 +124,39 @@ async fn inline_slash_command_is_available_from_local_recall_after_dispatch() { assert_eq!(chat.bottom_pane.composer_text(), "/rename Better title"); } +#[tokio::test] +async fn slash_rename_prefills_existing_thread_name() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.thread_name = Some("Current project title".to_string()); + + chat.dispatch_command(SlashCommand::Rename); + + let popup = render_bottom_popup(&chat, /*width*/ 80); + assert_chatwidget_snapshot!("slash_rename_prefilled_prompt", popup); + + chat.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + + assert_matches!( + rx.try_recv(), + Ok(AppEvent::CodexOp(Op::SetThreadName { name })) if name == "Current project title" + ); +} + +#[tokio::test] +async fn slash_rename_without_existing_thread_name_starts_empty() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + + chat.dispatch_command(SlashCommand::Rename); + + let popup = render_bottom_popup(&chat, /*width*/ 80); + assert!(popup.contains("Name thread")); + assert!(popup.contains("Type a name and press Enter")); + + chat.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + + assert_matches!(rx.try_recv(), Err(TryRecvError::Empty)); +} + #[tokio::test] async fn usage_error_slash_command_is_available_from_local_recall() { let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(Some("gpt-5.3-codex")).await; @@ -533,6 +566,18 @@ async fn slash_mcp_requests_inventory_via_app_server() { assert!(op_rx.try_recv().is_err(), "expected no core op to be sent"); } +#[tokio::test] +async fn slash_memories_opens_memory_menu() { + let (mut chat, mut rx, mut op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::MemoryTool, /*enabled*/ true); + + chat.dispatch_command(SlashCommand::Memories); + + assert!(render_bottom_popup(&chat, /*width*/ 80).contains("Use memories")); + assert_matches!(rx.try_recv(), Err(TryRecvError::Empty)); + assert!(op_rx.try_recv().is_err(), "expected no core op to be sent"); +} + #[tokio::test] async fn slash_memory_update_reports_stubbed_feature() { let (mut chat, mut rx, mut op_rx) = make_chatwidget_manual(/*model_override*/ None).await; diff --git a/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs b/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs index 041a0844f9..921d392479 100644 --- a/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs +++ b/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs @@ -1286,24 +1286,6 @@ async fn status_line_legacy_context_usage_renders_context_used_percent() { ); } -#[tokio::test] -async fn status_line_context_remaining_percent_renders_labeled_percent() { - let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; - chat.thread_id = Some(ThreadId::new()); - chat.config.tui_status_line = Some(vec!["context-remaining-percent".to_string()]); - - chat.refresh_status_line(); - - assert_eq!( - status_line_text(&chat), - Some("Context 100% left".to_string()) - ); - assert!( - drain_insert_history(&mut rx).is_empty(), - "context-remaining-percent should remain a valid status line item" - ); -} - #[tokio::test] async fn status_line_branch_state_resets_when_git_branch_disabled() { let (mut chat, _rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; @@ -1572,7 +1554,7 @@ async fn status_line_model_with_reasoning_fast_footer_snapshot() { } #[tokio::test] -async fn status_line_model_with_reasoning_context_remaining_percent_footer_snapshot() { +async fn status_line_model_with_reasoning_context_remaining_footer_snapshot() { use ratatui::Terminal; use ratatui::backend::TestBackend; @@ -1583,7 +1565,7 @@ async fn status_line_model_with_reasoning_context_remaining_percent_footer_snaps chat.config.cwd = test_project_path().abs(); chat.config.tui_status_line = Some(vec![ "model-with-reasoning".to_string(), - "context-remaining-percent".to_string(), + "context-remaining".to_string(), "current-dir".to_string(), ]); chat.set_reasoning_effort(Some(ReasoningEffortConfig::XHigh)); @@ -1600,7 +1582,7 @@ async fn status_line_model_with_reasoning_context_remaining_percent_footer_snaps .draw(|f| chat.render(f.area(), f.buffer_mut())) .expect("draw model-with-reasoning footer"); assert_chatwidget_snapshot!( - "status_line_model_with_reasoning_context_remaining_percent_footer", + "status_line_model_with_reasoning_context_remaining_footer", normalized_backend_snapshot(terminal.backend()) ); } diff --git a/codex-rs/tui/src/history_cell.rs b/codex-rs/tui/src/history_cell.rs index 79deede952..f0611e968c 100644 --- a/codex-rs/tui/src/history_cell.rs +++ b/codex-rs/tui/src/history_cell.rs @@ -61,9 +61,11 @@ use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig; use codex_protocol::plan_tool::PlanItemArg; use codex_protocol::plan_tool::StepStatus; use codex_protocol::plan_tool::UpdatePlanArgs; +use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::FileChange; use codex_protocol::protocol::McpAuthStatus; use codex_protocol::protocol::McpInvocation; +use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SessionConfiguredEvent; use codex_protocol::request_user_input::RequestUserInputAnswer; use codex_protocol::request_user_input::RequestUserInputQuestion; @@ -1186,6 +1188,8 @@ pub(crate) fn new_session_info( let SessionConfiguredEvent { model, reasoning_effort, + approval_policy, + sandbox_policy, .. } = event; // Header box rendered as history (so it appears at the very top) @@ -1195,7 +1199,8 @@ pub(crate) fn new_session_info( show_fast_status, config.cwd.to_path_buf(), CODEX_CLI_VERSION, - ); + ) + .with_yolo_mode(has_yolo_permissions(approval_policy, &sandbox_policy)); let mut parts: Vec> = vec![Box::new(header)]; if is_first_event { @@ -1259,6 +1264,17 @@ pub(crate) fn new_session_info( SessionInfoCell(CompositeHistoryCell { parts }) } +pub(crate) fn is_yolo_mode(config: &Config) -> bool { + has_yolo_permissions( + config.permissions.approval_policy.value(), + config.permissions.sandbox_policy.get(), + ) +} + +fn has_yolo_permissions(approval_policy: AskForApproval, sandbox_policy: &SandboxPolicy) -> bool { + approval_policy == AskForApproval::Never && *sandbox_policy == SandboxPolicy::DangerFullAccess +} + pub(crate) fn new_user_prompt( message: String, text_elements: Vec, @@ -1281,6 +1297,7 @@ pub(crate) struct SessionHeaderHistoryCell { reasoning_effort: Option, show_fast_status: bool, directory: PathBuf, + yolo_mode: bool, } impl SessionHeaderHistoryCell { @@ -1316,9 +1333,15 @@ impl SessionHeaderHistoryCell { reasoning_effort, show_fast_status, directory, + yolo_mode: false, } } + pub(crate) fn with_yolo_mode(mut self, yolo_mode: bool) -> Self { + self.yolo_mode = yolo_mode; + self + } + fn format_directory(&self, max_width: Option) -> String { Self::format_directory_inner(&self.directory, max_width) } @@ -1377,7 +1400,12 @@ impl HistoryCell for SessionHeaderHistoryCell { const CHANGE_MODEL_HINT_COMMAND: &str = "/model"; const CHANGE_MODEL_HINT_EXPLANATION: &str = " to change"; const DIR_LABEL: &str = "directory:"; - let label_width = DIR_LABEL.len(); + const PERMISSIONS_LABEL: &str = "permissions:"; + let label_width = if self.yolo_mode { + DIR_LABEL.len().max(PERMISSIONS_LABEL.len()) + } else { + DIR_LABEL.len() + }; let model_label = format!( "{model_label:, environment_manager: Arc, ) -> color_eyre::Result { start_embedded_app_server_with( @@ -253,6 +257,7 @@ async fn start_embedded_app_server( loader_overrides, cloud_requirements, feedback, + log_db, environment_manager, InProcessAppServerClient::start, ) @@ -369,6 +374,7 @@ async fn start_app_server( loader_overrides: LoaderOverrides, cloud_requirements: CloudRequirementsLoader, feedback: codex_feedback::CodexFeedback, + log_db: Option, environment_manager: Arc, ) -> color_eyre::Result { match target { @@ -379,6 +385,7 @@ async fn start_app_server( loader_overrides, cloud_requirements, feedback, + log_db, environment_manager, ) .await @@ -403,6 +410,7 @@ pub(crate) async fn start_app_server_for_picker( LoaderOverrides::default(), CloudRequirementsLoader::default(), codex_feedback::CodexFeedback::new(), + /*log_db*/ None, environment_manager, ) .await?; @@ -429,6 +437,7 @@ async fn start_embedded_app_server_with( loader_overrides: LoaderOverrides, cloud_requirements: CloudRequirementsLoader, feedback: codex_feedback::CodexFeedback, + log_db: Option, environment_manager: Arc, start_client: F, ) -> color_eyre::Result @@ -453,6 +462,7 @@ where loader_overrides, cloud_requirements, feedback, + log_db, environment_manager, config_warnings, session_source: codex_protocol::protocol::SessionSource::Cli, @@ -969,9 +979,10 @@ pub async fn run_main( let otel_tracing_layer = otel.as_ref().and_then(|o| o.tracing_layer()); - let log_db_layer = get_state_db(&config) - .await - .map(|db| log_db::start(db).with_filter(env_filter())); + let log_db = get_state_db(&config).await.map(log_db::start); + let log_db_layer = log_db + .clone() + .map(|layer| layer.with_filter(Targets::new().with_default(Level::TRACE))); let _ = tracing_subscriber::registry() .with(file_layer) @@ -993,6 +1004,7 @@ pub async fn run_main( cli_kv_overrides, cloud_requirements, feedback, + log_db, remote_url, remote_auth_token, environment_manager, @@ -1013,6 +1025,7 @@ async fn run_ratatui_app( cli_kv_overrides: Vec<(String, toml::Value)>, mut cloud_requirements: CloudRequirementsLoader, feedback: codex_feedback::CodexFeedback, + log_db: Option, remote_url: Option, remote_auth_token: Option, environment_manager: Arc, @@ -1071,6 +1084,7 @@ async fn run_ratatui_app( loader_overrides.clone(), cloud_requirements.clone(), feedback.clone(), + log_db.clone(), environment_manager.clone(), ) .await @@ -1409,6 +1423,7 @@ async fn run_ratatui_app( loader_overrides, cloud_requirements.clone(), feedback.clone(), + log_db.clone(), environment_manager.clone(), ) .await @@ -1772,6 +1787,7 @@ mod tests { LoaderOverrides::default(), CloudRequirementsLoader::default(), codex_feedback::CodexFeedback::new(), + /*log_db*/ None, Arc::new(EnvironmentManager::new(/*exec_server_url*/ None)), ) .await @@ -2163,6 +2179,7 @@ mod tests { LoaderOverrides::default(), CloudRequirementsLoader::default(), codex_feedback::CodexFeedback::new(), + /*log_db*/ None, Arc::new(EnvironmentManager::new(/*exec_server_url*/ None)), |_args| async { Err(std::io::Error::other("boom")) }, ) @@ -2232,6 +2249,7 @@ mod tests { approval_policy: config.permissions.approval_policy.value(), sandbox_policy: config.permissions.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model, personality: None, collaboration_mode: None, diff --git a/codex-rs/tui/src/local_chatgpt_auth.rs b/codex-rs/tui/src/local_chatgpt_auth.rs index e888c0387c..1f84b289a7 100644 --- a/codex-rs/tui/src/local_chatgpt_auth.rs +++ b/codex-rs/tui/src/local_chatgpt_auth.rs @@ -108,6 +108,7 @@ mod tests { account_id: Some("workspace-1".to_string()), }), last_refresh: Some(Utc::now()), + agent_identity: None, }; save_auth(codex_home, &auth, AuthCredentialsStoreMode::File) .expect("chatgpt auth should save"); @@ -154,6 +155,7 @@ mod tests { openai_api_key: Some("sk-test".to_string()), tokens: None, last_refresh: None, + agent_identity: None, }, AuthCredentialsStoreMode::File, ) diff --git a/codex-rs/tui/src/onboarding/auth.rs b/codex-rs/tui/src/onboarding/auth.rs index 57bd58bf5a..aee909cea1 100644 --- a/codex-rs/tui/src/onboarding/auth.rs +++ b/codex-rs/tui/src/onboarding/auth.rs @@ -988,6 +988,7 @@ mod tests { "https://chatgpt.com/backend-api/".to_string(), ), feedback: codex_feedback::CodexFeedback::new(), + log_db: None, environment_manager: Arc::new(codex_app_server_client::EnvironmentManager::new( /*exec_server_url*/ None, )), diff --git a/codex-rs/tui/src/onboarding/onboarding_screen.rs b/codex-rs/tui/src/onboarding/onboarding_screen.rs index 22a444cdaf..5e2e0edbde 100644 --- a/codex-rs/tui/src/onboarding/onboarding_screen.rs +++ b/codex-rs/tui/src/onboarding/onboarding_screen.rs @@ -4,6 +4,7 @@ use crate::legacy_core::windows_sandbox::WindowsSandboxLevelExt; use codex_app_server_client::AppServerEvent; use codex_app_server_client::AppServerRequestHandle; use codex_app_server_protocol::ServerNotification; +use codex_git_utils::resolve_root_git_project_for_trust; #[cfg(target_os = "windows")] use codex_protocol::config_types::WindowsSandboxLevel; use crossterm::event::KeyCode; @@ -77,7 +78,7 @@ pub(crate) struct OnboardingResult { } impl OnboardingScreen { - pub(crate) fn new(tui: &mut Tui, args: OnboardingScreenArgs) -> Self { + pub(crate) async fn new(tui: &mut Tui, args: OnboardingScreenArgs) -> Self { let OnboardingScreenArgs { show_trust_screen, show_login_screen, @@ -122,8 +123,12 @@ impl OnboardingScreen { let show_windows_create_sandbox_hint = false; let highlighted = TrustDirectorySelection::Trust; if show_trust_screen { + let trust_target = resolve_root_git_project_for_trust(&cwd) + .await + .unwrap_or_else(|| cwd.clone()); steps.push(Step::TrustDirectory(TrustDirectoryWidget { cwd, + trust_target, codex_home, show_windows_create_sandbox_hint, should_quit: false, @@ -452,7 +457,7 @@ pub(crate) async fn run_onboarding_app( ) -> Result { use tokio_stream::StreamExt; - let mut onboarding_screen = OnboardingScreen::new(tui, args); + let mut onboarding_screen = OnboardingScreen::new(tui, args).await; // One-time guard to fully clear the screen after ChatGPT login success message is shown let mut did_full_clear_after_success = false; diff --git a/codex-rs/tui/src/onboarding/trust_directory.rs b/codex-rs/tui/src/onboarding/trust_directory.rs index 23ee624627..e9f2cd58bd 100644 --- a/codex-rs/tui/src/onboarding/trust_directory.rs +++ b/codex-rs/tui/src/onboarding/trust_directory.rs @@ -1,7 +1,6 @@ use std::path::PathBuf; use crate::legacy_core::config::set_project_trust_level; -use codex_git_utils::resolve_root_git_project_for_trust; use codex_protocol::config_types::TrustLevel; use crossterm::event::KeyCode; use crossterm::event::KeyEvent; @@ -27,6 +26,7 @@ use super::onboarding_screen::StepState; pub(crate) struct TrustDirectoryWidget { pub codex_home: PathBuf, pub cwd: PathBuf, + pub trust_target: PathBuf, pub show_windows_create_sandbox_hint: bool, pub should_quit: bool, pub selection: Option, @@ -142,8 +142,7 @@ impl StepStateProvider for TrustDirectoryWidget { impl TrustDirectoryWidget { fn handle_trust(&mut self) { - let target = - resolve_root_git_project_for_trust(&self.cwd).unwrap_or_else(|| self.cwd.clone()); + let target = self.trust_target.clone(); if let Err(e) = set_project_trust_level(&self.codex_home, &target, TrustLevel::Trusted) { tracing::error!("Failed to set project trusted: {e:?}"); self.error = Some(format!("Failed to set trust for {}: {e}", target.display())); @@ -182,6 +181,7 @@ mod tests { let mut widget = TrustDirectoryWidget { codex_home: codex_home.path().to_path_buf(), cwd: PathBuf::from("."), + trust_target: PathBuf::from("."), show_windows_create_sandbox_hint: false, should_quit: false, selection: None, @@ -207,6 +207,7 @@ mod tests { let widget = TrustDirectoryWidget { codex_home: codex_home.path().to_path_buf(), cwd: PathBuf::from("/workspace/project"), + trust_target: PathBuf::from("/workspace/project"), show_windows_create_sandbox_hint: false, should_quit: false, selection: None, diff --git a/codex-rs/tui/src/slash_command.rs b/codex-rs/tui/src/slash_command.rs index 635dc317f8..759bedbad7 100644 --- a/codex-rs/tui/src/slash_command.rs +++ b/codex-rs/tui/src/slash_command.rs @@ -21,6 +21,7 @@ pub enum SlashCommand { #[strum(serialize = "sandbox-add-read-dir")] SandboxReadRoot, Experimental, + Memories, Skills, Review, Rename, @@ -109,6 +110,7 @@ impl SlashCommand { "let sandbox read a directory: /sandbox-add-read-dir " } SlashCommand::Experimental => "toggle experimental features", + SlashCommand::Memories => "configure memory use and generation", SlashCommand::Mcp => "list configured MCP tools", SlashCommand::Apps => "manage apps", SlashCommand::Plugins => "browse plugins", @@ -154,6 +156,7 @@ impl SlashCommand { | SlashCommand::ElevateSandbox | SlashCommand::SandboxReadRoot | SlashCommand::Experimental + | SlashCommand::Memories | SlashCommand::Review | SlashCommand::Plan | SlashCommand::Clear diff --git a/codex-rs/tui/src/snapshots/codex_tui__history_cell__tests__session_header_indicates_yolo_mode.snap b/codex-rs/tui/src/snapshots/codex_tui__history_cell__tests__session_header_indicates_yolo_mode.snap new file mode 100644 index 0000000000..04f64584f6 --- /dev/null +++ b/codex-rs/tui/src/snapshots/codex_tui__history_cell__tests__session_header_indicates_yolo_mode.snap @@ -0,0 +1,11 @@ +--- +source: tui/src/history_cell.rs +expression: rendered +--- +╭───────────────────────────────────────╮ +│ >_ OpenAI Codex (vtest) │ +│ │ +│ model: gpt-5 /model to change │ +│ directory: /tmp/project │ +│ permissions: YOLO mode │ +╰───────────────────────────────────────╯ diff --git a/codex-rs/tui/src/status/helpers.rs b/codex-rs/tui/src/status/helpers.rs index 2ea56e0b5c..27292e1364 100644 --- a/codex-rs/tui/src/status/helpers.rs +++ b/codex-rs/tui/src/status/helpers.rs @@ -185,8 +185,8 @@ fn title_case(s: &str) -> String { #[cfg(test)] mod tests { use super::*; - use crate::legacy_core::DEFAULT_PROJECT_DOC_FILENAME; - use crate::legacy_core::LOCAL_PROJECT_DOC_FILENAME; + use crate::legacy_core::DEFAULT_AGENTS_MD_FILENAME; + use crate::legacy_core::LOCAL_AGENTS_MD_FILENAME; use crate::legacy_core::config::ConfigBuilder; use codex_utils_absolute_path::test_support::PathBufExt; use pretty_assertions::assert_eq; @@ -227,7 +227,7 @@ mod tests { async fn compose_agents_summary_includes_global_agents_path() { let codex_home = TempDir::new().expect("temp codex home"); let cwd = TempDir::new().expect("temp cwd"); - let global_agents_path = codex_home.path().join(DEFAULT_PROJECT_DOC_FILENAME); + let global_agents_path = codex_home.path().join(DEFAULT_AGENTS_MD_FILENAME); let config = test_config(&codex_home, &cwd).await; assert_eq!( @@ -240,7 +240,7 @@ mod tests { async fn compose_agents_summary_names_global_agents_override() { let codex_home = TempDir::new().expect("temp codex home"); let cwd = TempDir::new().expect("temp cwd"); - let override_path = codex_home.path().join(LOCAL_PROJECT_DOC_FILENAME); + let override_path = codex_home.path().join(LOCAL_AGENTS_MD_FILENAME); let config = test_config(&codex_home, &cwd).await; assert_eq!( @@ -253,8 +253,8 @@ mod tests { async fn compose_agents_summary_orders_global_before_project_agents() { let codex_home = TempDir::new().expect("temp codex home"); let cwd = TempDir::new().expect("temp cwd"); - let global_agents_path = codex_home.path().join(DEFAULT_PROJECT_DOC_FILENAME); - let project_agents_path = cwd.path().join(DEFAULT_PROJECT_DOC_FILENAME); + let global_agents_path = codex_home.path().join(DEFAULT_AGENTS_MD_FILENAME); + let project_agents_path = cwd.path().join(DEFAULT_AGENTS_MD_FILENAME); let config = test_config(&codex_home, &cwd).await; let summary = compose_agents_summary( @@ -270,7 +270,7 @@ mod tests { Some(format_directory_display(&global_agents_path, /*max_width*/ None).as_str()) ); let project_path = paths.next().expect("project agents path"); - assert!(project_path.ends_with(DEFAULT_PROJECT_DOC_FILENAME)); + assert!(project_path.ends_with(DEFAULT_AGENTS_MD_FILENAME)); assert_eq!(paths.next(), None); } } diff --git a/codex-rs/tui/src/update_action.rs b/codex-rs/tui/src/update_action.rs index baee1662fd..1ac2ff675f 100644 --- a/codex-rs/tui/src/update_action.rs +++ b/codex-rs/tui/src/update_action.rs @@ -1,3 +1,8 @@ +#[cfg(any(not(debug_assertions), test))] +use codex_install_context::InstallContext; +#[cfg(any(not(debug_assertions), test))] +use codex_install_context::StandalonePlatform; + /// Update action the CLI should perform after the TUI exits. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum UpdateAction { @@ -7,15 +12,41 @@ pub enum UpdateAction { BunGlobalLatest, /// Update via `brew upgrade codex`. BrewUpgrade, + /// Update via `curl -fsSL https://chatgpt.com/codex/install.sh | sh`. + StandaloneUnix, + /// Update via `irm https://chatgpt.com/codex/install.ps1|iex`. + StandaloneWindows, } impl UpdateAction { + #[cfg(any(not(debug_assertions), test))] + pub(crate) fn from_install_context(context: &InstallContext) -> Option { + match context { + InstallContext::Npm => Some(UpdateAction::NpmGlobalLatest), + InstallContext::Bun => Some(UpdateAction::BunGlobalLatest), + InstallContext::Brew => Some(UpdateAction::BrewUpgrade), + InstallContext::Standalone { platform, .. } => Some(match platform { + StandalonePlatform::Unix => UpdateAction::StandaloneUnix, + StandalonePlatform::Windows => UpdateAction::StandaloneWindows, + }), + InstallContext::Other => None, + } + } + /// Returns the list of command-line arguments for invoking the update. pub fn command_args(self) -> (&'static str, &'static [&'static str]) { match self { UpdateAction::NpmGlobalLatest => ("npm", &["install", "-g", "@openai/codex"]), UpdateAction::BunGlobalLatest => ("bun", &["install", "-g", "@openai/codex"]), UpdateAction::BrewUpgrade => ("brew", &["upgrade", "--cask", "codex"]), + UpdateAction::StandaloneUnix => ( + "sh", + &["-c", "curl -fsSL https://chatgpt.com/codex/install.sh | sh"], + ), + UpdateAction::StandaloneWindows => ( + "powershell", + &["-c", "irm https://chatgpt.com/codex/install.ps1|iex"], + ), } } @@ -29,88 +60,68 @@ impl UpdateAction { #[cfg(not(debug_assertions))] pub(crate) fn get_update_action() -> Option { - let exe = std::env::current_exe().unwrap_or_default(); - let managed_by_npm = std::env::var_os("CODEX_MANAGED_BY_NPM").is_some(); - let managed_by_bun = std::env::var_os("CODEX_MANAGED_BY_BUN").is_some(); - - detect_update_action( - cfg!(target_os = "macos"), - &exe, - managed_by_npm, - managed_by_bun, - ) -} - -#[cfg(any(not(debug_assertions), test))] -fn detect_update_action( - is_macos: bool, - current_exe: &std::path::Path, - managed_by_npm: bool, - managed_by_bun: bool, -) -> Option { - if managed_by_npm { - Some(UpdateAction::NpmGlobalLatest) - } else if managed_by_bun { - Some(UpdateAction::BunGlobalLatest) - } else if is_macos - && (current_exe.starts_with("/opt/homebrew") || current_exe.starts_with("/usr/local")) - { - Some(UpdateAction::BrewUpgrade) - } else { - None - } + UpdateAction::from_install_context(InstallContext::current()) } #[cfg(test)] mod tests { use super::*; + use pretty_assertions::assert_eq; + use std::path::PathBuf; #[test] - fn detects_update_action_without_env_mutation() { + fn maps_install_context_to_update_action() { + let native_release_dir = PathBuf::from("/tmp/native-release"); + assert_eq!( - detect_update_action( - /*is_macos*/ false, - std::path::Path::new("/any/path"), - /*managed_by_npm*/ false, - /*managed_by_bun*/ false - ), + UpdateAction::from_install_context(&InstallContext::Other), None ); assert_eq!( - detect_update_action( - /*is_macos*/ false, - std::path::Path::new("/any/path"), - /*managed_by_npm*/ true, - /*managed_by_bun*/ false - ), + UpdateAction::from_install_context(&InstallContext::Npm), Some(UpdateAction::NpmGlobalLatest) ); assert_eq!( - detect_update_action( - /*is_macos*/ false, - std::path::Path::new("/any/path"), - /*managed_by_npm*/ false, - /*managed_by_bun*/ true - ), + UpdateAction::from_install_context(&InstallContext::Bun), Some(UpdateAction::BunGlobalLatest) ); assert_eq!( - detect_update_action( - /*is_macos*/ true, - std::path::Path::new("/opt/homebrew/bin/codex"), - /*managed_by_npm*/ false, - /*managed_by_bun*/ false - ), + UpdateAction::from_install_context(&InstallContext::Brew), Some(UpdateAction::BrewUpgrade) ); assert_eq!( - detect_update_action( - /*is_macos*/ true, - std::path::Path::new("/usr/local/bin/codex"), - /*managed_by_npm*/ false, - /*managed_by_bun*/ false - ), - Some(UpdateAction::BrewUpgrade) + UpdateAction::from_install_context(&InstallContext::Standalone { + platform: StandalonePlatform::Unix, + release_dir: native_release_dir.clone(), + resources_dir: Some(native_release_dir.join("codex-resources")), + }), + Some(UpdateAction::StandaloneUnix) + ); + assert_eq!( + UpdateAction::from_install_context(&InstallContext::Standalone { + platform: StandalonePlatform::Windows, + release_dir: native_release_dir.clone(), + resources_dir: Some(native_release_dir.join("codex-resources")), + }), + Some(UpdateAction::StandaloneWindows) + ); + } + + #[test] + fn standalone_update_commands_rerun_latest_installer() { + assert_eq!( + UpdateAction::StandaloneUnix.command_args(), + ( + "sh", + &["-c", "curl -fsSL https://chatgpt.com/codex/install.sh | sh"][..], + ) + ); + assert_eq!( + UpdateAction::StandaloneWindows.command_args(), + ( + "powershell", + &["-c", "irm https://chatgpt.com/codex/install.ps1|iex"][..], + ) ); } } diff --git a/codex-rs/tui/tests/suite/model_availability_nux.rs b/codex-rs/tui/tests/suite/model_availability_nux.rs index 923ace50fd..04a03a09f9 100644 --- a/codex-rs/tui/tests/suite/model_availability_nux.rs +++ b/codex-rs/tui/tests/suite/model_availability_nux.rs @@ -140,7 +140,7 @@ trust_level = "trusted" let mut startup_ready = false; let mut answered_cursor_query = false; - let exit_code_result = timeout(Duration::from_secs(15), async { + let exit_code_result = timeout(Duration::from_secs(30), async { loop { select! { result = output_rx.recv() => match result { diff --git a/codex-rs/windows-sandbox-rs/src/helper_materialization.rs b/codex-rs/windows-sandbox-rs/src/helper_materialization.rs index 375199f5cc..ad5cc4e2b2 100644 --- a/codex-rs/windows-sandbox-rs/src/helper_materialization.rs +++ b/codex-rs/windows-sandbox-rs/src/helper_materialization.rs @@ -13,6 +13,8 @@ use tempfile::NamedTempFile; use crate::logging::log_note; use crate::sandbox_bin_dir; +const RESOURCES_DIRNAME: &str = "codex-resources"; + #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub(crate) enum HelperExecutable { CommandRunner, @@ -46,12 +48,9 @@ pub(crate) fn helper_bin_dir(codex_home: &Path) -> PathBuf { pub(crate) fn legacy_lookup(kind: HelperExecutable) -> PathBuf { if let Ok(exe) = std::env::current_exe() - && let Some(dir) = exe.parent() + && let Some(candidate) = source_path_for_exe(&exe, kind.file_name()) { - let candidate = dir.join(kind.file_name()); - if candidate.exists() { - return candidate; - } + return candidate; } PathBuf::from(kind.file_name()) } @@ -179,18 +178,23 @@ fn store_helper_path(cache_key: String, path: PathBuf) { fn sibling_source_path(kind: HelperExecutable) -> Result { let exe = std::env::current_exe().context("resolve current executable for helper lookup")?; - let dir = exe - .parent() - .ok_or_else(|| anyhow!("current executable has no parent directory"))?; - let candidate = dir.join(kind.file_name()); - if candidate.exists() { - Ok(candidate) - } else { - Err(anyhow!( - "helper not found next to current executable: {}", - candidate.display() - )) + source_path_for_exe(&exe, kind.file_name()).ok_or_else(|| { + anyhow!( + "helper not found next to current executable or under {RESOURCES_DIRNAME}: {}", + exe.display() + ) + }) +} + +fn source_path_for_exe(exe: &Path, file_name: &str) -> Option { + let dir = exe.parent()?; + let direct_candidate = dir.join(file_name); + if direct_candidate.exists() { + return Some(direct_candidate); } + + let resource_candidate = dir.join(RESOURCES_DIRNAME).join(file_name); + resource_candidate.exists().then_some(resource_candidate) } fn copy_from_source_if_needed(source: &Path, destination: &Path) -> Result { @@ -292,10 +296,12 @@ fn destination_is_fresh(source: &Path, destination: &Path) -> Result { #[cfg(test)] mod tests { - use super::destination_is_fresh; - use super::helper_bin_dir; use super::copy_from_source_if_needed; use super::CopyOutcome; + use super::destination_is_fresh; + use super::helper_bin_dir; + use super::RESOURCES_DIRNAME; + use super::source_path_for_exe; use pretty_assertions::assert_eq; use std::fs; use std::path::Path; @@ -376,4 +382,40 @@ mod tests { fs::read(&runner_destination).expect("read runner") ); } + + #[test] + fn helper_source_lookup_checks_resource_dir() { + let tmp = TempDir::new().expect("tempdir"); + let release_dir = tmp.path().join("release"); + let resources_dir = release_dir.join(RESOURCES_DIRNAME); + fs::create_dir_all(&resources_dir).expect("create resources dir"); + let exe = release_dir.join("codex.exe"); + let helper = resources_dir.join("codex-command-runner.exe"); + fs::write(&exe, b"codex").expect("write exe"); + fs::write(&helper, b"runner").expect("write helper"); + + let resolved = + source_path_for_exe(&exe, /*file_name*/ "codex-command-runner.exe").expect("helper path"); + + assert_eq!(resolved, helper); + } + + #[test] + fn helper_source_lookup_prefers_direct_sibling_over_resource_dir() { + let tmp = TempDir::new().expect("tempdir"); + let release_dir = tmp.path().join("release"); + let resources_dir = release_dir.join(RESOURCES_DIRNAME); + fs::create_dir_all(&resources_dir).expect("create resources dir"); + let exe = release_dir.join("codex.exe"); + let sibling_helper = release_dir.join("codex-command-runner.exe"); + let resource_helper = resources_dir.join("codex-command-runner.exe"); + fs::write(&exe, b"codex").expect("write exe"); + fs::write(&sibling_helper, b"sibling runner").expect("write sibling helper"); + fs::write(&resource_helper, b"resource runner").expect("write resource helper"); + + let resolved = + source_path_for_exe(&exe, /*file_name*/ "codex-command-runner.exe").expect("helper path"); + + assert_eq!(resolved, sibling_helper); + } } diff --git a/codex-rs/windows-sandbox-rs/src/setup_orchestrator.rs b/codex-rs/windows-sandbox-rs/src/setup_orchestrator.rs index e0a5a063f9..4effc0b70d 100644 --- a/codex-rs/windows-sandbox-rs/src/setup_orchestrator.rs +++ b/codex-rs/windows-sandbox-rs/src/setup_orchestrator.rs @@ -589,6 +589,16 @@ fn find_setup_exe() -> PathBuf { if candidate.exists() { return candidate; } + + // Standalone installs keep Windows helper binaries under + // `codex-resources/` next to `codex.exe`, so elevation needs to probe + // that sibling folder before falling back to PATH. + let resource_candidate = dir + .join("codex-resources") + .join("codex-windows-sandbox-setup.exe"); + if resource_candidate.exists() { + return resource_candidate; + } } PathBuf::from("codex-windows-sandbox-setup.exe") } diff --git a/docs/config.md b/docs/config.md index c314ce2283..d583122e01 100644 --- a/docs/config.md +++ b/docs/config.md @@ -28,12 +28,18 @@ resources, review those read/write race conditions before enabling this setting. ## MCP tool approvals -Codex stores per-tool approval overrides for custom MCP servers under -`mcp_servers` in `~/.codex/config.toml`: +Codex stores approval defaults and per-tool overrides for custom MCP servers +under `mcp_servers` in `~/.codex/config.toml`. Set +`default_tools_approval_mode` on the server to apply a default to every tool, +and use per-tool `approval_mode` entries for exceptions: ```toml +[mcp_servers.docs] +command = "docs-server" +default_tools_approval_mode = "approve" + [mcp_servers.docs.tools.search] -approval_mode = "approve" +approval_mode = "prompt" ``` ## Apps (Connectors) diff --git a/docs/js_repl.md b/docs/js_repl.md index d5edc71b46..aee635b77a 100644 --- a/docs/js_repl.md +++ b/docs/js_repl.md @@ -86,6 +86,7 @@ imported local file. They are not resolved relative to the imported file's locat - `codex.emitImage(...)` accepts a data URL, a single `input_image` item, an object like `{ bytes, mimeType }`, or a raw tool response object that contains exactly one image and no text. Call it multiple times if you want to emit multiple images. - `codex.emitImage(...)` rejects mixed text-and-image content. - Request full-resolution image processing with `detail: "original"` only when the `view_image` tool schema includes a `detail` argument. The same availability applies to `codex.emitImage(...)`: if `view_image.detail` is present, you may also pass `detail: "original"` there. Use this when high-fidelity image perception or precise localization is needed, especially for CUA agents. +- Raw MCP image blocks can request the same behavior by returning `_meta: { "codex/imageDetail": "original" }` on the image content item. - Example of sharing an in-memory Playwright screenshot: `await codex.emitImage({ bytes: await page.screenshot({ type: "jpeg", quality: 85 }), mimeType: "image/jpeg", detail: "original" })`. - Example of sharing a local image tool result: `await codex.emitImage(codex.tool("view_image", { path: "/absolute/path", detail: "original" }))`. - When encoding an image to send with `codex.emitImage(...)` or `view_image`, prefer JPEG at about 85 quality when lossy compression is acceptable; use PNG when transparency or lossless detail matters. Smaller uploads are faster and less likely to hit size limits. diff --git a/justfile b/justfile index f0f14f4193..d44bbda21d 100644 --- a/justfile +++ b/justfile @@ -17,7 +17,7 @@ exec *args: # Start `codex exec-server` and run codex-tui. [no-cd] tui-with-exec-server *args: - ./scripts/run_tui_with_exec_server.sh "$@" + {{ justfile_directory() }}/scripts/run_tui_with_exec_server.sh "$@" # Run the CLI version of the file-search crate. file-search *args: @@ -64,18 +64,18 @@ bazel-lock-update: [no-cd] bazel-lock-check: - ./scripts/check-module-bazel-lock.sh + {{ justfile_directory() }}/scripts/check-module-bazel-lock.sh bazel-test: bazel test --test_tag_filters=-argument-comment-lint //... --keep_going [no-cd] bazel-clippy: - bazel_targets="$(./scripts/list-bazel-clippy-targets.sh)" && bazel build --config=clippy -- ${bazel_targets} + bazel_targets="$({{ justfile_directory() }}/scripts/list-bazel-clippy-targets.sh)" && bazel build --config=clippy -- ${bazel_targets} [no-cd] bazel-argument-comment-lint: - bazel build --config=argument-comment-lint -- $(./tools/argument-comment-lint/list-bazel-targets.sh) + bazel build --config=argument-comment-lint -- $({{ justfile_directory() }}/tools/argument-comment-lint/list-bazel-targets.sh) bazel-remote-test: bazel test --test_tag_filters=-argument-comment-lint //... --config=remote --platforms=//:rbe --keep_going @@ -97,20 +97,20 @@ write-app-server-schema *args: [no-cd] write-hooks-schema: - cargo run --manifest-path ./codex-rs/Cargo.toml -p codex-hooks --bin write_hooks_schema_fixtures + cargo run --manifest-path {{ justfile_directory() }}/codex-rs/Cargo.toml -p codex-hooks --bin write_hooks_schema_fixtures # Run the argument-comment Dylint checks across codex-rs. [no-cd] argument-comment-lint *args: if [ "$#" -eq 0 ]; then \ - bazel build --config=argument-comment-lint -- $(./tools/argument-comment-lint/list-bazel-targets.sh); \ + bazel build --config=argument-comment-lint -- $({{ justfile_directory() }}/tools/argument-comment-lint/list-bazel-targets.sh); \ else \ - ./tools/argument-comment-lint/run-prebuilt-linter.py "$@"; \ + {{ justfile_directory() }}/tools/argument-comment-lint/run-prebuilt-linter.py "$@"; \ fi [no-cd] argument-comment-lint-from-source *args: - ./tools/argument-comment-lint/run.py "$@" + {{ justfile_directory() }}/tools/argument-comment-lint/run.py "$@" # Tail logs from the state SQLite database log *args: diff --git a/scripts/install/install.ps1 b/scripts/install/install.ps1 index 40328db700..ed4a3e1a32 100644 --- a/scripts/install/install.ps1 +++ b/scripts/install/install.ps1 @@ -1,6 +1,5 @@ param( - [Parameter(Position=0)] - [string]$Version = "latest" + [string]$Release = "latest" ) Set-StrictMode -Version Latest @@ -15,6 +14,27 @@ function Write-Step { Write-Host "==> $Message" } +function Write-WarningStep { + param( + [string]$Message + ) + + Write-Warning $Message +} + +function Prompt-YesNo { + param( + [string]$Prompt + ) + + if ([Console]::IsInputRedirected -or [Console]::IsOutputRedirected) { + return $false + } + + $choice = Read-Host "$Prompt [y/N]" + return $choice -match "^(?i:y(?:es)?)$" +} + function Normalize-Version { param( [string]$RawVersion @@ -35,13 +55,39 @@ function Normalize-Version { return $RawVersion } -function Get-ReleaseUrl { +function Get-ReleaseAssetMetadata { param( [string]$AssetName, [string]$ResolvedVersion ) - return "https://github.com/openai/codex/releases/download/rust-v$ResolvedVersion/$AssetName" + $release = Invoke-RestMethod -Uri "https://api.github.com/repos/openai/codex/releases/tags/rust-v$ResolvedVersion" + $asset = $release.assets | Where-Object { $_.name -eq $AssetName } | Select-Object -First 1 + if ($null -eq $asset) { + throw "Could not find release asset $AssetName for Codex $ResolvedVersion." + } + + $digestMatch = [regex]::Match([string]$asset.digest, "^sha256:([0-9a-fA-F]{64})$") + if (-not $digestMatch.Success) { + throw "Could not find SHA-256 digest for release asset $AssetName." + } + + return [PSCustomObject]@{ + Url = $asset.browser_download_url + Sha256 = $digestMatch.Groups[1].Value.ToLowerInvariant() + } +} + +function Test-ArchiveDigest { + param( + [string]$ArchivePath, + [string]$ExpectedDigest + ) + + $actualDigest = (Get-FileHash -LiteralPath $ArchivePath -Algorithm SHA256).Hash.ToLowerInvariant() + if ($actualDigest -ne $ExpectedDigest) { + throw "Downloaded Codex archive checksum did not match release metadata. Expected $ExpectedDigest but got $actualDigest." + } } function Path-Contains { @@ -64,8 +110,46 @@ function Path-Contains { return $false } +function Invoke-WithInstallLock { + param( + [string]$LockPath, + [scriptblock]$Script + ) + + New-Item -ItemType Directory -Force -Path (Split-Path -Parent $LockPath) | Out-Null + $lock = $null + while ($null -eq $lock) { + try { + $lock = [System.IO.File]::Open( + $LockPath, + [System.IO.FileMode]::OpenOrCreate, + [System.IO.FileAccess]::ReadWrite, + [System.IO.FileShare]::None + ) + } catch [System.IO.IOException] { + Start-Sleep -Milliseconds 250 + } + } + try { + & $Script + } finally { + $lock.Dispose() + } +} + +function Remove-StaleInstallArtifacts { + param( + [string]$ReleasesDir + ) + + if (Test-Path -LiteralPath $ReleasesDir -PathType Container) { + Get-ChildItem -LiteralPath $ReleasesDir -Force -Directory -Filter ".staging.*" -ErrorAction SilentlyContinue | + Remove-Item -Recurse -Force -ErrorAction SilentlyContinue + } +} + function Resolve-Version { - $normalizedVersion = Normalize-Version -RawVersion $Version + $normalizedVersion = Normalize-Version -RawVersion $Release if ($normalizedVersion -ne "latest") { return $normalizedVersion } @@ -79,6 +163,414 @@ function Resolve-Version { return (Normalize-Version -RawVersion $release.tag_name) } +function Get-VersionFromBinary { + param( + [string]$CodexPath + ) + + if (-not (Test-Path -LiteralPath $CodexPath -PathType Leaf)) { + return $null + } + + try { + $versionOutput = & $CodexPath --version 2>$null + } catch { + return $null + } + + if ($versionOutput -match '([0-9][0-9A-Za-z.+-]*)$') { + return $matches[1] + } + + return $null +} + +function Get-CurrentInstalledVersion { + param( + [string]$StandaloneCurrentDir + ) + + $standaloneVersion = Get-VersionFromBinary -CodexPath (Join-Path $StandaloneCurrentDir "codex.exe") + if (-not [string]::IsNullOrWhiteSpace($standaloneVersion)) { + return $standaloneVersion + } + + return $null +} + +function Test-OldStandaloneBinLayout { + param( + [string]$VisibleBinDir, + [string]$DefaultVisibleBinDir + ) + + if (-not $VisibleBinDir.Equals($DefaultVisibleBinDir, [System.StringComparison]::OrdinalIgnoreCase)) { + return $false + } + if (-not (Test-Path -LiteralPath $VisibleBinDir -PathType Container)) { + return $false + } + + $item = Get-Item -LiteralPath $VisibleBinDir -Force + if ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) { + return $false + } + + $requiredFiles = @("codex.exe", "rg.exe") + foreach ($fileName in $requiredFiles) { + if (-not (Test-Path -LiteralPath (Join-Path $VisibleBinDir $fileName) -PathType Leaf)) { + return $false + } + } + + $knownFiles = @( + "codex.exe", + "rg.exe", + "codex-command-runner.exe", + "codex-windows-sandbox.exe", + "codex-windows-sandbox-setup.exe" + ) + foreach ($child in Get-ChildItem -LiteralPath $VisibleBinDir -Force) { + if ($child.PSIsContainer) { + return $false + } + if ($knownFiles -notcontains $child.Name) { + return $false + } + } + + return $true +} + +function Move-OldStandaloneBinIfApproved { + param( + [string]$VisibleBinDir, + [string]$DefaultVisibleBinDir + ) + + if (-not (Test-OldStandaloneBinLayout -VisibleBinDir $VisibleBinDir -DefaultVisibleBinDir $DefaultVisibleBinDir)) { + return $null + } + + Write-Step "We found an older Codex install at $VisibleBinDir" + Write-WarningStep "To continue, Codex needs to update the install at this path." + if (-not (Prompt-YesNo "Replace it with the current Codex setup now?")) { + throw "Cannot replace older standalone install without confirmation: $VisibleBinDir" + } + + $backupDir = "$VisibleBinDir.backup.$([DateTimeOffset]::UtcNow.ToUnixTimeSeconds()).$PID" + Write-Step "Moving older standalone install to $backupDir" + Move-Item -LiteralPath $VisibleBinDir -Destination $backupDir + return $backupDir +} + +function Add-JunctionSupportType { + if (([System.Management.Automation.PSTypeName]'CodexInstaller.Junction').Type) { + return + } + + Add-Type -TypeDefinition @" +using System; +using System.ComponentModel; +using System.IO; +using System.Runtime.InteropServices; +using System.Text; +using Microsoft.Win32.SafeHandles; + +namespace CodexInstaller +{ + public static class Junction + { + private const uint GENERIC_WRITE = 0x40000000; + private const uint FILE_SHARE_READ = 0x00000001; + private const uint FILE_SHARE_WRITE = 0x00000002; + private const uint FILE_SHARE_DELETE = 0x00000004; + private const uint OPEN_EXISTING = 3; + private const uint FILE_FLAG_BACKUP_SEMANTICS = 0x02000000; + private const uint FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000; + private const uint FSCTL_SET_REPARSE_POINT = 0x000900A4; + private const uint IO_REPARSE_TAG_MOUNT_POINT = 0xA0000003; + private const int HeaderLength = 20; + + [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + private static extern SafeFileHandle CreateFileW( + string lpFileName, + uint dwDesiredAccess, + uint dwShareMode, + IntPtr lpSecurityAttributes, + uint dwCreationDisposition, + uint dwFlagsAndAttributes, + IntPtr hTemplateFile); + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool DeviceIoControl( + SafeFileHandle hDevice, + uint dwIoControlCode, + byte[] lpInBuffer, + int nInBufferSize, + IntPtr lpOutBuffer, + int nOutBufferSize, + out int lpBytesReturned, + IntPtr lpOverlapped); + + public static void SetTarget(string linkPath, string targetPath) + { + string substituteName = "\\??\\" + Path.GetFullPath(targetPath); + byte[] substituteNameBytes = Encoding.Unicode.GetBytes(substituteName); + if (substituteNameBytes.Length > ushort.MaxValue - HeaderLength) { + throw new ArgumentException("Junction target path is too long.", "targetPath"); + } + + byte[] reparseBuffer = new byte[substituteNameBytes.Length + HeaderLength]; + WriteUInt32(reparseBuffer, 0, IO_REPARSE_TAG_MOUNT_POINT); + WriteUInt16(reparseBuffer, 4, checked((ushort)(substituteNameBytes.Length + 12))); + WriteUInt16(reparseBuffer, 8, 0); + WriteUInt16(reparseBuffer, 10, checked((ushort)substituteNameBytes.Length)); + WriteUInt16(reparseBuffer, 12, checked((ushort)(substituteNameBytes.Length + 2))); + WriteUInt16(reparseBuffer, 14, 0); + Buffer.BlockCopy(substituteNameBytes, 0, reparseBuffer, 16, substituteNameBytes.Length); + + using (SafeFileHandle handle = CreateFileW( + linkPath, + GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + IntPtr.Zero, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT, + IntPtr.Zero)) + { + if (handle.IsInvalid) { + throw new Win32Exception(Marshal.GetLastWin32Error()); + } + + int bytesReturned; + if (!DeviceIoControl( + handle, + FSCTL_SET_REPARSE_POINT, + reparseBuffer, + reparseBuffer.Length, + IntPtr.Zero, + 0, + out bytesReturned, + IntPtr.Zero)) + { + throw new Win32Exception(Marshal.GetLastWin32Error()); + } + } + } + + private static void WriteUInt16(byte[] buffer, int offset, ushort value) + { + buffer[offset] = (byte)value; + buffer[offset + 1] = (byte)(value >> 8); + } + + private static void WriteUInt32(byte[] buffer, int offset, uint value) + { + buffer[offset] = (byte)value; + buffer[offset + 1] = (byte)(value >> 8); + buffer[offset + 2] = (byte)(value >> 16); + buffer[offset + 3] = (byte)(value >> 24); + } + } +} +"@ +} + +function Set-JunctionTarget { + param( + [string]$LinkPath, + [string]$TargetPath + ) + + Add-JunctionSupportType + [CodexInstaller.Junction]::SetTarget($LinkPath, $TargetPath) +} + +function Test-IsJunction { + param( + [string]$Path + ) + + if (-not (Test-Path -LiteralPath $Path)) { + return $false + } + + $item = Get-Item -LiteralPath $Path -Force + return ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -and $item.LinkType -eq "Junction" +} + +function Ensure-Junction { + param( + [string]$LinkPath, + [string]$TargetPath, + [string]$InstallerOwnedTargetPrefix + ) + + if (-not (Test-Path -LiteralPath $LinkPath)) { + New-Item -ItemType Junction -Path $LinkPath -Target $TargetPath | Out-Null + return + } + + $item = Get-Item -LiteralPath $LinkPath -Force + if (Test-IsJunction -Path $LinkPath) { + $existingTarget = [string]$item.Target + if (-not [string]::IsNullOrWhiteSpace($InstallerOwnedTargetPrefix)) { + $ownedTargetPrefix = $InstallerOwnedTargetPrefix.TrimEnd("\\") + if (-not $existingTarget.StartsWith($ownedTargetPrefix, [System.StringComparison]::OrdinalIgnoreCase)) { + throw "Refusing to retarget junction at $LinkPath because it is not managed by this installer." + } + } + if ($existingTarget.Equals($TargetPath, [System.StringComparison]::OrdinalIgnoreCase)) { + return + } + + # Keep the path itself in place and only retarget the junction. That + # avoids a gap where current or the visible bin path disappears during + # an update. + Set-JunctionTarget -LinkPath $LinkPath -TargetPath $TargetPath + return + } + + if ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) { + throw "Refusing to replace non-junction reparse point at $LinkPath." + } + + if ($item.PSIsContainer) { + if ((Get-ChildItem -LiteralPath $LinkPath -Force | Select-Object -First 1) -ne $null) { + throw "Refusing to replace non-empty directory at $LinkPath with a junction." + } + + Remove-Item -LiteralPath $LinkPath -Force + New-Item -ItemType Junction -Path $LinkPath -Target $TargetPath | Out-Null + return + } + + throw "Refusing to replace file at $LinkPath with a junction." +} + +function Test-ReleaseIsComplete { + param( + [string]$ReleaseDir, + [string]$ExpectedVersion, + [string]$ExpectedTarget + ) + + if (-not (Test-Path -LiteralPath $ReleaseDir -PathType Container)) { + return $false + } + + $expectedFiles = @( + "codex.exe", + "codex-resources\codex-command-runner.exe", + "codex-resources\codex-windows-sandbox-setup.exe", + "codex-resources\rg.exe" + ) + foreach ($name in $expectedFiles) { + if (-not (Test-Path -LiteralPath (Join-Path $ReleaseDir $name) -PathType Leaf)) { + return $false + } + } + + return (Split-Path -Leaf $ReleaseDir) -eq "$ExpectedVersion-$ExpectedTarget" +} + +function Get-ExistingCodexCommand { + $existing = Get-Command codex -ErrorAction SilentlyContinue + if ($null -eq $existing) { + return $null + } + + return $existing.Source +} + +function Get-ExistingCodexManager { + param( + [string]$ExistingPath, + [string]$VisibleBinDir + ) + + if ([string]::IsNullOrWhiteSpace($ExistingPath)) { + return $null + } + + if ($ExistingPath.StartsWith($VisibleBinDir, [System.StringComparison]::OrdinalIgnoreCase)) { + return $null + } + + if ($ExistingPath -match "\\.bun\\") { + return "bun" + } + + if ($ExistingPath -match "node_modules" -or $ExistingPath -match "\\npm\\") { + return "npm" + } + + return $null +} + +function Get-ConflictingInstall { + param( + [string]$VisibleBinDir + ) + + $existingPath = Get-ExistingCodexCommand + $manager = Get-ExistingCodexManager -ExistingPath $existingPath -VisibleBinDir $VisibleBinDir + if ($null -eq $manager) { + return $null + } + + Write-Step "Detected existing $manager-managed Codex at $existingPath" + Write-WarningStep "Multiple managed Codex installs can be ambiguous because PATH order decides which one runs." + + return [PSCustomObject]@{ + Manager = $manager + Path = $existingPath + } +} + +function Maybe-HandleConflictingInstall { + param( + [object]$Conflict + ) + + if ($null -eq $Conflict) { + return + } + + $manager = $Conflict.Manager + + $uninstallArgs = if ($manager -eq "bun") { + @("remove", "-g", "@openai/codex") + } else { + @("uninstall", "-g", "@openai/codex") + } + $uninstallCommand = if ($manager -eq "bun") { "bun" } else { "npm" } + + if (Prompt-YesNo "Uninstall the existing $manager-managed Codex now?") { + Write-Step "Running: $uninstallCommand $($uninstallArgs -join ' ')" + try { + & $uninstallCommand @uninstallArgs + } catch { + Write-WarningStep "Failed to uninstall the existing $manager-managed Codex. Continuing with the standalone install." + } + } else { + Write-WarningStep "Leaving the existing $manager-managed Codex installed. PATH order will determine which codex runs." + } +} + +function Test-VisibleCodexCommand { + param( + [string]$VisibleBinDir + ) + + $codexCommand = Join-Path $VisibleBinDir "codex.exe" + & $codexCommand --version *> $null + if ($LASTEXITCODE -ne 0) { + throw "Installed Codex command failed verification: $codexCommand --version" + } +} + if ($env:OS -ne "Windows_NT") { Write-Error "install.ps1 supports Windows only. Use install.sh on macOS or Linux." exit 1 @@ -110,87 +602,149 @@ switch ($architecture) { } } -if ([string]::IsNullOrWhiteSpace($env:CODEX_INSTALL_DIR)) { - $installDir = Join-Path $env:LOCALAPPDATA "Programs\OpenAI\Codex\bin" +$codexHome = if ([string]::IsNullOrWhiteSpace($env:CODEX_HOME)) { + Join-Path $env:USERPROFILE ".codex" } else { - $installDir = $env:CODEX_INSTALL_DIR + $env:CODEX_HOME +} +$standaloneRoot = Join-Path $codexHome "packages\standalone" +$releasesDir = Join-Path $standaloneRoot "releases" +$currentDir = Join-Path $standaloneRoot "current" +$lockPath = Join-Path $standaloneRoot "install.lock" + +$defaultVisibleBinDir = Join-Path $env:LOCALAPPDATA "Programs\OpenAI\Codex\bin" +if ([string]::IsNullOrWhiteSpace($env:CODEX_INSTALL_DIR)) { + $visibleBinDir = $defaultVisibleBinDir +} else { + $visibleBinDir = $env:CODEX_INSTALL_DIR } -$codexPath = Join-Path $installDir "codex.exe" -$installMode = if (Test-Path $codexPath) { "Updating" } else { "Installing" } - -Write-Step "$installMode Codex CLI" -Write-Step "Detected platform: $platformLabel" - -New-Item -ItemType Directory -Force -Path $installDir | Out-Null - +$currentVersion = Get-CurrentInstalledVersion -StandaloneCurrentDir $currentDir $resolvedVersion = Resolve-Version -Write-Step "Resolved version: $resolvedVersion" -$packageAsset = "codex-npm-$npmTag-$resolvedVersion.tgz" +$releaseName = "$resolvedVersion-$target" +$releaseDir = Join-Path $releasesDir $releaseName +if (-not [string]::IsNullOrWhiteSpace($currentVersion) -and $currentVersion -ne $resolvedVersion) { + Write-Step "Updating Codex CLI from $currentVersion to $resolvedVersion" +} elseif (-not [string]::IsNullOrWhiteSpace($currentVersion)) { + Write-Step "Updating Codex CLI" +} else { + Write-Step "Installing Codex CLI" +} +Write-Step "Detected platform: $platformLabel" +Write-Step "Resolved version: $resolvedVersion" + +$conflictingInstall = Get-ConflictingInstall -VisibleBinDir $visibleBinDir +$oldStandaloneBackup = $null + +$packageAsset = "codex-npm-$npmTag-$resolvedVersion.tgz" $tempDir = Join-Path ([System.IO.Path]::GetTempPath()) ("codex-install-" + [System.Guid]::NewGuid().ToString("N")) New-Item -ItemType Directory -Force -Path $tempDir | Out-Null try { - $archivePath = Join-Path $tempDir $packageAsset - $extractDir = Join-Path $tempDir "extract" - $url = Get-ReleaseUrl -AssetName $packageAsset -ResolvedVersion $resolvedVersion + Invoke-WithInstallLock -LockPath $lockPath -Script { + Remove-StaleInstallArtifacts -ReleasesDir $releasesDir - Write-Step "Downloading Codex CLI" - Invoke-WebRequest -Uri $url -OutFile $archivePath + if (-not (Test-ReleaseIsComplete -ReleaseDir $releaseDir -ExpectedVersion $resolvedVersion -ExpectedTarget $target)) { + if (Test-Path -LiteralPath $releaseDir) { + Write-WarningStep "Found incomplete existing release at $releaseDir. Reinstalling." + } - New-Item -ItemType Directory -Force -Path $extractDir | Out-Null - tar -xzf $archivePath -C $extractDir + $archivePath = Join-Path $tempDir $packageAsset + $extractDir = Join-Path $tempDir "extract" + $stagingDir = Join-Path $releasesDir ".staging.$releaseName.$PID" + $assetMetadata = Get-ReleaseAssetMetadata -AssetName $packageAsset -ResolvedVersion $resolvedVersion - $vendorRoot = Join-Path $extractDir "package/vendor/$target" - Write-Step "Installing to $installDir" - $copyMap = @{ - "codex/codex.exe" = "codex.exe" - "codex/codex-command-runner.exe" = "codex-command-runner.exe" - "codex/codex-windows-sandbox-setup.exe" = "codex-windows-sandbox-setup.exe" - "path/rg.exe" = "rg.exe" - } + Write-Step "Downloading Codex CLI" + Invoke-WebRequest -Uri $assetMetadata.Url -OutFile $archivePath + Test-ArchiveDigest -ArchivePath $archivePath -ExpectedDigest $assetMetadata.Sha256 - foreach ($relativeSource in $copyMap.Keys) { - $sourcePath = Join-Path $vendorRoot $relativeSource - $destinationPath = Join-Path $installDir $copyMap[$relativeSource] - Move-Item -Force $sourcePath $destinationPath + New-Item -ItemType Directory -Force -Path $extractDir | Out-Null + New-Item -ItemType Directory -Force -Path $releasesDir | Out-Null + if (Test-Path -LiteralPath $stagingDir) { + Remove-Item -LiteralPath $stagingDir -Recurse -Force + } + New-Item -ItemType Directory -Force -Path $stagingDir | Out-Null + tar -xzf $archivePath -C $extractDir + + $vendorRoot = Join-Path $extractDir "package/vendor/$target" + $resourcesDir = Join-Path $stagingDir "codex-resources" + New-Item -ItemType Directory -Force -Path $resourcesDir | Out-Null + $copyMap = @{ + "codex/codex.exe" = "codex.exe" + "codex/codex-command-runner.exe" = "codex-resources\codex-command-runner.exe" + "codex/codex-windows-sandbox-setup.exe" = "codex-resources\codex-windows-sandbox-setup.exe" + "path/rg.exe" = "codex-resources\rg.exe" + } + + foreach ($relativeSource in $copyMap.Keys) { + Copy-Item -LiteralPath (Join-Path $vendorRoot $relativeSource) -Destination (Join-Path $stagingDir $copyMap[$relativeSource]) + } + + if (Test-Path -LiteralPath $releaseDir) { + Remove-Item -LiteralPath $releaseDir -Recurse -Force + } + Move-Item -LiteralPath $stagingDir -Destination $releaseDir + } + + New-Item -ItemType Directory -Force -Path $standaloneRoot | Out-Null + Ensure-Junction -LinkPath $currentDir -TargetPath $releaseDir -InstallerOwnedTargetPrefix $releasesDir + + $visibleParent = Split-Path -Parent $visibleBinDir + New-Item -ItemType Directory -Force -Path $visibleParent | Out-Null + $oldStandaloneBackup = Move-OldStandaloneBinIfApproved -VisibleBinDir $visibleBinDir -DefaultVisibleBinDir $defaultVisibleBinDir + try { + Ensure-Junction -LinkPath $visibleBinDir -TargetPath $currentDir -InstallerOwnedTargetPrefix $standaloneRoot + Test-VisibleCodexCommand -VisibleBinDir $visibleBinDir + } catch { + if ($null -ne $oldStandaloneBackup -and (Test-Path -LiteralPath $oldStandaloneBackup)) { + if (Test-Path -LiteralPath $visibleBinDir) { + Remove-Item -LiteralPath $visibleBinDir -Recurse -Force + } + Move-Item -LiteralPath $oldStandaloneBackup -Destination $visibleBinDir + } + throw + } + if ($null -ne $oldStandaloneBackup) { + Remove-Item -LiteralPath $oldStandaloneBackup -Recurse -Force + } } } finally { Remove-Item -Recurse -Force $tempDir -ErrorAction SilentlyContinue } +Maybe-HandleConflictingInstall -Conflict $conflictingInstall + $userPath = [Environment]::GetEnvironmentVariable("Path", "User") -$pathNeedsNewShell = $false -if (-not (Path-Contains -PathValue $userPath -Entry $installDir)) { +if (-not (Path-Contains -PathValue $userPath -Entry $visibleBinDir)) { if ([string]::IsNullOrWhiteSpace($userPath)) { - $newUserPath = $installDir + $newUserPath = $visibleBinDir } else { - $newUserPath = "$installDir;$userPath" + $newUserPath = "$visibleBinDir;$userPath" } [Environment]::SetEnvironmentVariable("Path", $newUserPath, "User") - if (-not (Path-Contains -PathValue $env:Path -Entry $installDir)) { - if ([string]::IsNullOrWhiteSpace($env:Path)) { - $env:Path = $installDir - } else { - $env:Path = "$installDir;$env:Path" - } - } Write-Step "PATH updated for future PowerShell sessions." - $pathNeedsNewShell = $true -} elseif (Path-Contains -PathValue $env:Path -Entry $installDir) { - Write-Step "$installDir is already on PATH." +} elseif (Path-Contains -PathValue $env:Path -Entry $visibleBinDir) { + Write-Step "$visibleBinDir is already on PATH." } else { Write-Step "PATH is already configured for future PowerShell sessions." - $pathNeedsNewShell = $true } -if ($pathNeedsNewShell) { - Write-Step ('Run now: $env:Path = "{0};$env:Path"; codex' -f $installDir) - Write-Step "Or open a new PowerShell window and run: codex" -} else { - Write-Step "Run: codex" +if (-not (Path-Contains -PathValue $env:Path -Entry $visibleBinDir)) { + if ([string]::IsNullOrWhiteSpace($env:Path)) { + $env:Path = $visibleBinDir + } else { + $env:Path = "$visibleBinDir;$env:Path" + } } +Write-Step "Current PowerShell session: codex" +Write-Step "Future PowerShell windows: open a new PowerShell window and run: codex" Write-Host "Codex CLI $resolvedVersion installed successfully." + +$codexCommand = Join-Path $visibleBinDir "codex.exe" +if (Prompt-YesNo "Start Codex now?") { + Write-Step "Launching Codex" + & $codexCommand +} diff --git a/scripts/install/install.sh b/scripts/install/install.sh index 86e2940d55..8c225e4d3b 100755 --- a/scripts/install/install.sh +++ b/scripts/install/install.sh @@ -2,15 +2,33 @@ set -eu -VERSION="${1:-latest}" -INSTALL_DIR="${CODEX_INSTALL_DIR:-$HOME/.local/bin}" +RELEASE="latest" + +BIN_DIR="${CODEX_INSTALL_DIR:-$HOME/.local/bin}" +BIN_PATH="$BIN_DIR/codex" +CODEX_HOME_DIR="${CODEX_HOME:-$HOME/.codex}" +STANDALONE_ROOT="$CODEX_HOME_DIR/packages/standalone" +RELEASES_DIR="$STANDALONE_ROOT/releases" +CURRENT_LINK="$STANDALONE_ROOT/current" +LOCK_FILE="$STANDALONE_ROOT/install.lock" +LOCK_DIR="$STANDALONE_ROOT/install.lock.d" +LOCK_STALE_AFTER_SECS=600 + path_action="already" path_profile="" +conflict_manager="" +conflict_path="" +lock_kind="" +tmp_dir="" step() { printf '==> %s\n' "$1" } +warn() { + printf 'WARNING: %s\n' "$1" >&2 +} + normalize_version() { case "$1" in "" | latest) @@ -28,6 +46,32 @@ normalize_version() { esac } +parse_args() { + while [ "$#" -gt 0 ]; do + case "$1" in + --release) + if [ "$#" -lt 2 ]; then + echo "--release requires a value." >&2 + exit 1 + fi + RELEASE="$2" + shift + ;; + --help | -h) + cat <&2 + exit 1 + ;; + esac + shift + done +} + download_file() { url="$1" output="$2" @@ -63,40 +107,6 @@ download_text() { exit 1 } -add_to_path() { - path_action="already" - path_profile="" - - case ":$PATH:" in - *":$INSTALL_DIR:"*) - return - ;; - esac - - profile="$HOME/.profile" - case "${SHELL:-}" in - */zsh) - profile="$HOME/.zshrc" - ;; - */bash) - profile="$HOME/.bashrc" - ;; - esac - - path_profile="$profile" - path_line="export PATH=\"$INSTALL_DIR:\$PATH\"" - if [ -f "$profile" ] && grep -F "$path_line" "$profile" >/dev/null 2>&1; then - path_action="configured" - return - fi - - { - printf '\n# Added by Codex installer\n' - printf '%s\n' "$path_line" - } >>"$profile" - path_action="added" -} - release_url_for_asset() { asset="$1" resolved_version="$2" @@ -104,6 +114,92 @@ release_url_for_asset() { printf 'https://github.com/openai/codex/releases/download/rust-v%s/%s\n' "$resolved_version" "$asset" } +release_metadata_url() { + resolved_version="$1" + + printf 'https://api.github.com/repos/openai/codex/releases/tags/rust-v%s\n' "$resolved_version" +} + +release_asset_digest() { + asset="$1" + resolved_version="$2" + release_json="$(download_text "$(release_metadata_url "$resolved_version")")" + + digest="$(printf '%s\n' "$release_json" | awk -v asset="$asset" ' + { + if ($0 ~ "\"name\":[[:space:]]*\"" asset "\"") { + in_asset = 1 + asset_depth = depth + } + + if (in_asset && /"digest":[[:space:]]*"[^"]+"/) { + sub(/^.*"digest":[[:space:]]*"/, "") + sub(/".*$/, "") + digest = $0 + } + + line = $0 + opens = gsub(/\{/, "{", line) + closes = gsub(/\}/, "}", line) + depth += opens - closes + + if (in_asset && depth < asset_depth) { + in_asset = 0 + } + } + END { + if (digest != "") { + print digest + } + } + ')" + + case "$digest" in + sha256:????????????????????????????????????????????????????????????????) + printf '%s\n' "${digest#sha256:}" + ;; + *) + echo "Could not find SHA-256 digest for release asset $asset." >&2 + exit 1 + ;; + esac +} + +file_sha256() { + path="$1" + + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$path" | awk '{print $1}' + return + fi + + if command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$path" | awk '{print $1}' + return + fi + + if command -v openssl >/dev/null 2>&1; then + openssl dgst -sha256 "$path" | sed 's/^.*= //' + return + fi + + echo "sha256sum, shasum, or openssl is required to verify the Codex download." >&2 + exit 1 +} + +verify_archive_digest() { + archive_path="$1" + expected_digest="$2" + actual_digest="$(file_sha256 "$archive_path")" + + if [ "$actual_digest" != "$expected_digest" ]; then + echo "Downloaded Codex archive checksum did not match release metadata." >&2 + echo "expected: $expected_digest" >&2 + echo "actual: $actual_digest" >&2 + exit 1 + fi +} + require_command() { if ! command -v "$1" >/dev/null 2>&1; then echo "$1 is required to install Codex." >&2 @@ -111,11 +207,8 @@ require_command() { fi } -require_command mktemp -require_command tar - resolve_version() { - normalized_version="$(normalize_version "$VERSION")" + normalized_version="$(normalize_version "$RELEASE")" if [ "$normalized_version" != "latest" ]; then printf '%s\n' "$normalized_version" @@ -133,6 +226,417 @@ resolve_version() { printf '%s\n' "$resolved" } +pick_profile() { + # Use the same shell-specific split Homebrew documents because there is no + # universal startup file across macOS/Linux login and interactive shells. + case "$os:${SHELL:-}" in + darwin:*/zsh) + printf '%s\n' "$HOME/.zprofile" + ;; + darwin:*/bash) + printf '%s\n' "$HOME/.bash_profile" + ;; + linux:*/zsh) + printf '%s\n' "$HOME/.zshrc" + ;; + linux:*/bash) + printf '%s\n' "$HOME/.bashrc" + ;; + *) + printf '%s\n' "$HOME/.profile" + ;; + esac +} + +add_to_path() { + path_action="already" + path_profile="" + + case ":$PATH:" in + *":$BIN_DIR:"*) + return + ;; + esac + + profile="$(pick_profile)" + path_profile="$profile" + begin_marker="# >>> Codex installer >>>" + end_marker="# <<< Codex installer <<<" + path_line="export PATH=\"$BIN_DIR:\$PATH\"" + + if [ -f "$profile" ] && grep -F "$begin_marker" "$profile" >/dev/null 2>&1; then + if grep -F "$path_line" "$profile" >/dev/null 2>&1; then + path_action="configured" + return + fi + + if grep -F "$end_marker" "$profile" >/dev/null 2>&1; then + rewrite_path_block "$profile" "$begin_marker" "$end_marker" "$path_line" + path_action="updated" + return + fi + fi + + append_path_block "$profile" "$begin_marker" "$end_marker" "$path_line" + path_action="added" +} + +append_path_block() { + profile="$1" + begin_marker="$2" + end_marker="$3" + path_line="$4" + + { + printf '\n%s\n' "$begin_marker" + printf '%s\n' "$path_line" + printf '%s\n' "$end_marker" + } >>"$profile" +} + +rewrite_path_block() { + profile="$1" + begin_marker="$2" + end_marker="$3" + path_line="$4" + tmp_profile="$tmp_dir/profile.$$.tmp" + + awk -v begin="$begin_marker" -v end="$end_marker" -v line="$path_line" ' + BEGIN { + in_block = 0 + replaced = 0 + } + $0 == begin { + if (!replaced) { + print begin + print line + print end + replaced = 1 + } + in_block = 1 + next + } + in_block { + if ($0 == end) { + in_block = 0 + } + next + } + { + print + } + END { + if (in_block != 0) { + exit 1 + } + } + ' "$profile" >"$tmp_profile" + mv "$tmp_profile" "$profile" +} + +mkdir_lock_is_stale() { + [ -d "$LOCK_DIR" ] || return 1 + + pid="$(cat "$LOCK_DIR/pid" 2>/dev/null || true)" + started_at="$(cat "$LOCK_DIR/started_at" 2>/dev/null || true)" + now="$(date +%s 2>/dev/null || printf '0')" + + case "$started_at" in + ''|*[!0-9]*) + started_at=0 + ;; + esac + + if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then + return 1 + fi + + if [ "$started_at" -eq 0 ] || [ "$now" -eq 0 ]; then + return 0 + fi + + [ $((now - started_at)) -ge "$LOCK_STALE_AFTER_SECS" ] +} + +acquire_install_lock() { + mkdir -p "$STANDALONE_ROOT" + + if [ "$os" = "darwin" ] && command -v lockf >/dev/null 2>&1; then + : >>"$LOCK_FILE" + exec 9<>"$LOCK_FILE" + lockf 9 + lock_kind="lockf" + return + fi + + if command -v flock >/dev/null 2>&1; then + exec 9>"$LOCK_FILE" + flock 9 + lock_kind="flock" + return + fi + + while ! mkdir "$LOCK_DIR" 2>/dev/null; do + if mkdir_lock_is_stale; then + warn "Removing stale installer lock at $LOCK_DIR" + rm -rf "$LOCK_DIR" + continue + fi + sleep 1 + done + + printf '%s\n' "$$" >"$LOCK_DIR/pid" + date +%s >"$LOCK_DIR/started_at" 2>/dev/null || true + lock_kind="mkdir" +} + +release_install_lock() { + if [ "$lock_kind" = "mkdir" ]; then + rm -rf "$LOCK_DIR" 2>/dev/null || true + elif [ "$lock_kind" = "flock" ] || [ "$lock_kind" = "lockf" ]; then + exec 9>&- 2>/dev/null || true + fi + lock_kind="" +} + +cleanup_stale_install_artifacts() { + mkdir -p "$RELEASES_DIR" "$STANDALONE_ROOT" + + find "$RELEASES_DIR" -mindepth 1 -maxdepth 1 -name '.staging.*' -exec rm -rf {} + + find "$STANDALONE_ROOT" -mindepth 1 -maxdepth 1 -name '.current.*' -exec rm -f {} + + + if [ -d "$BIN_DIR" ]; then + find "$BIN_DIR" -mindepth 1 -maxdepth 1 -name '.codex.*' -exec rm -f {} + + fi +} + +replace_path_with_symlink() { + link_path="$1" + link_target="$2" + tmp_link="$3" + + rm -f "$tmp_link" + ln -s "$link_target" "$tmp_link" + + if mv -Tf "$tmp_link" "$link_path" 2>/dev/null; then + return + fi + + if mv -hf "$tmp_link" "$link_path" 2>/dev/null; then + return + fi + + rm -f "$link_path" + mv -f "$tmp_link" "$link_path" +} + +version_from_binary() { + codex_path="$1" + + if [ ! -x "$codex_path" ]; then + return 1 + fi + + "$codex_path" --version 2>/dev/null | sed -n 's/.* \([0-9][0-9A-Za-z.+-]*\)$/\1/p' | head -n 1 +} + +current_installed_version() { + version="$(version_from_binary "$CURRENT_LINK/codex" || true)" + if [ -n "$version" ]; then + printf '%s\n' "$version" + return 0 + fi + + return 0 +} + +resolve_existing_codex() { + command -v codex 2>/dev/null || true +} + +classify_existing_codex() { + existing_path="$1" + + if [ -z "$existing_path" ] || [ "$existing_path" = "$BIN_PATH" ]; then + return 1 + fi + + case "$existing_path" in + /opt/homebrew/* | /usr/local/*) + if [ "$os" = "darwin" ]; then + printf 'brew\n' + return 0 + fi + ;; + esac + + if [ -f "$existing_path" ] && grep -F "#!/usr/bin/env node" "$existing_path" >/dev/null 2>&1; then + case "$existing_path" in + *".bun"*) + printf 'bun\n' + ;; + *) + printf 'npm\n' + ;; + esac + return 0 + fi + + return 1 +} + +prompt_yes_no() { + prompt="$1" + + if ( : /dev/null; then + printf '%s [y/N] ' "$prompt" >/dev/tty + if ! IFS= read -r answer /dev/null +} + +parse_args "$@" + +require_command mktemp +require_command tar + case "$(uname -s)" in Darwin) os="darwin" @@ -187,58 +691,78 @@ else fi fi -if [ -x "$INSTALL_DIR/codex" ]; then - install_mode="Updating" -else - install_mode="Installing" -fi - -step "$install_mode Codex CLI" -step "Detected platform: $platform_label" - resolved_version="$(resolve_version)" asset="codex-npm-$npm_tag-$resolved_version.tgz" download_url="$(release_url_for_asset "$asset" "$resolved_version")" +release_name="$resolved_version-$vendor_target" +release_dir="$RELEASES_DIR/$release_name" +current_version="$(current_installed_version)" +if [ -n "$current_version" ] && [ "$current_version" != "$resolved_version" ]; then + step "Updating Codex CLI from $current_version to $resolved_version" +elif [ -n "$current_version" ]; then + step "Updating Codex CLI" +else + step "Installing Codex CLI" +fi +step "Detected platform: $platform_label" step "Resolved version: $resolved_version" +detect_conflicting_install + tmp_dir="$(mktemp -d)" cleanup() { - rm -rf "$tmp_dir" + release_install_lock + if [ -n "$tmp_dir" ]; then + rm -rf "$tmp_dir" + fi } trap cleanup EXIT INT TERM -archive_path="$tmp_dir/$asset" +acquire_install_lock +cleanup_stale_install_artifacts -step "Downloading Codex CLI" -download_file "$download_url" "$archive_path" +if ! release_dir_is_complete "$release_dir" "$resolved_version" "$vendor_target"; then + if [ -e "$release_dir" ] || [ -L "$release_dir" ]; then + warn "Found incomplete existing release at $release_dir; reinstalling." + fi -tar -xzf "$archive_path" -C "$tmp_dir" + archive_path="$tmp_dir/$asset" + extract_dir="$tmp_dir/extract" -step "Installing to $INSTALL_DIR" -mkdir -p "$INSTALL_DIR" -cp "$tmp_dir/package/vendor/$vendor_target/codex/codex" "$INSTALL_DIR/codex" -cp "$tmp_dir/package/vendor/$vendor_target/path/rg" "$INSTALL_DIR/rg" -chmod 0755 "$INSTALL_DIR/codex" -chmod 0755 "$INSTALL_DIR/rg" + step "Downloading Codex CLI" + expected_digest="$(release_asset_digest "$asset" "$resolved_version")" + download_file "$download_url" "$archive_path" + verify_archive_digest "$archive_path" "$expected_digest" + mkdir -p "$extract_dir" + tar -xzf "$archive_path" -C "$extract_dir" + + step "Installing standalone package to $release_dir" + install_release "$release_dir" "$extract_dir/package/vendor/$vendor_target" +fi +update_current_link "$release_dir" +update_visible_command add_to_path +verify_visible_command +release_install_lock +handle_conflicting_install case "$path_action" in added) - step "PATH updated for future shells in $path_profile" - step "Run now: export PATH=\"$INSTALL_DIR:\$PATH\" && codex" - step "Or open a new terminal and run: codex" + print_launch_instructions + ;; + updated) + print_launch_instructions ;; configured) - step "PATH is already configured for future shells in $path_profile" - step "Run now: export PATH=\"$INSTALL_DIR:\$PATH\" && codex" - step "Or open a new terminal and run: codex" + print_launch_instructions ;; *) - step "$INSTALL_DIR is already on PATH" - step "Run: codex" + step "$BIN_DIR is already on PATH" + print_launch_instructions ;; esac printf 'Codex CLI %s installed successfully.\n' "$resolved_version" +maybe_launch_codex_now diff --git a/scripts/start-codex-exec.sh b/scripts/start-codex-exec.sh index 5699c13efd..8835d4c095 100755 --- a/scripts/start-codex-exec.sh +++ b/scripts/start-codex-exec.sh @@ -67,11 +67,7 @@ repo_root="$(git rev-parse --show-toplevel 2>/dev/null)" || { } ssh "${remote_host}" "mkdir -p ${remote_path}" - -if ! ssh "${remote_host}" 'command -v rsync >/dev/null 2>&1'; then - echo "remote rsync is required on ${remote_host}" >&2 - exit 1 -fi +ssh "${remote_host}" -C "sudo apt-get install rsync libcap-dev" sync_instance_id="$(date +%s)-$$" diff --git a/scripts/test-remote-env.sh b/scripts/test-remote-env.sh index 329fa8a7f3..ba4d35f235 100755 --- a/scripts/test-remote-env.sh +++ b/scripts/test-remote-env.sh @@ -18,6 +18,11 @@ is_sourced() { setup_remote_env() { local container_name local codex_binary_path + local container_ip + local remote_codex_path + local remote_exec_server_pid + local remote_exec_server_port + local remote_exec_server_stdout_path container_name="${CODEX_TEST_REMOTE_ENV_CONTAINER_NAME:-codex-remote-test-env-local-$(date +%s)-${RANDOM}}" codex_binary_path="${REPO_ROOT}/codex-rs/target/debug/codex" @@ -59,14 +64,58 @@ setup_remote_env() { return 1 fi + if [[ -z "${CODEX_TEST_REMOTE_EXEC_SERVER_URL:-}" ]]; then + remote_codex_path="/tmp/codex-remote-env/codex" + remote_exec_server_port="31987" + remote_exec_server_stdout_path="/tmp/codex-remote-env/exec-server.stdout" + docker exec "${container_name}" sh -lc "mkdir -p /tmp/codex-remote-env" + docker cp "${codex_binary_path}" "${container_name}:${remote_codex_path}" + docker exec "${container_name}" chmod +x "${remote_codex_path}" + remote_exec_server_pid="$( + docker exec "${container_name}" sh -lc \ + "rm -f ${remote_exec_server_stdout_path}; nohup ${remote_codex_path} exec-server --listen ws://0.0.0.0:${remote_exec_server_port} > ${remote_exec_server_stdout_path} 2>&1 & echo \$!" + )" + wait_for_remote_exec_server_port "${container_name}" "${remote_exec_server_port}" "${remote_exec_server_stdout_path}" + container_ip="$( + docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "${container_name}" + )" + if [[ -z "${container_ip}" ]]; then + echo "container ${container_name} has no IP address" >&2 + docker rm -f "${container_name}" >/dev/null 2>&1 || true + return 1 + fi + export CODEX_TEST_REMOTE_EXEC_SERVER_PID="${remote_exec_server_pid}" + export CODEX_TEST_REMOTE_EXEC_SERVER_URL="ws://${container_ip}:${remote_exec_server_port}" + fi + export CODEX_TEST_REMOTE_ENV="${container_name}" } +wait_for_remote_exec_server_port() { + local container_name="$1" + local port="$2" + local stdout_path="$3" + local deadline=$((SECONDS + 5)) + + while (( SECONDS < deadline )); do + if docker exec "${container_name}" python3 -c "import socket; socket.create_connection(('127.0.0.1', ${port}), timeout=0.2).close()" >/dev/null 2>&1; then + return 0 + fi + sleep 0.025 + done + + echo "timed out waiting for remote exec-server on ${container_name}:${port}" >&2 + docker exec "${container_name}" sh -lc "cat ${stdout_path} 2>/dev/null || true" >&2 || true + return 1 +} + codex_remote_env_cleanup() { if [[ -n "${CODEX_TEST_REMOTE_ENV:-}" ]]; then docker rm -f "${CODEX_TEST_REMOTE_ENV}" >/dev/null 2>&1 || true unset CODEX_TEST_REMOTE_ENV fi + unset CODEX_TEST_REMOTE_EXEC_SERVER_PID + unset CODEX_TEST_REMOTE_EXEC_SERVER_URL } if ! is_sourced; then @@ -79,6 +128,7 @@ set -euo pipefail if setup_remote_env; then status=0 echo "CODEX_TEST_REMOTE_ENV=${CODEX_TEST_REMOTE_ENV}" + echo "CODEX_TEST_REMOTE_EXEC_SERVER_URL=${CODEX_TEST_REMOTE_EXEC_SERVER_URL}" echo "Remote env ready. Run your command, then call: codex_remote_env_cleanup" else status=$?