diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 5d30916f9f..c5a4c0c1c7 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -94,7 +94,7 @@ jobs: "${bazel_wrapper_args[@]}" \ -- \ test \ - --test_tag_filters=-argument-comment-lint \ + --test_tag_filters=-argument-comment-lint,-flaky \ --test_verbose_timeout_warnings \ --build_metadata=COMMIT_SHA=${GITHUB_SHA} \ -- \ diff --git a/.github/workflows/rust-ci-full.yml b/.github/workflows/rust-ci-full.yml index a39c38857b..7cda682952 100644 --- a/.github/workflows/rust-ci-full.yml +++ b/.github/workflows/rust-ci-full.yml @@ -664,6 +664,7 @@ jobs: export CODEX_TEST_REMOTE_ENV_CONTAINER_NAME=codex-remote-test-env source "${GITHUB_WORKSPACE}/scripts/test-remote-env.sh" echo "CODEX_TEST_REMOTE_ENV=${CODEX_TEST_REMOTE_ENV}" >> "$GITHUB_ENV" + echo "CODEX_TEST_REMOTE_EXEC_SERVER_URL=${CODEX_TEST_REMOTE_EXEC_SERVER_URL}" >> "$GITHUB_ENV" - name: tests id: test diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index c270e6a9fa..5d7572f461 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -819,6 +819,8 @@ "dylint_linting_5.0.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"assert_cmd\",\"req\":\"^2.0\"},{\"name\":\"cargo_metadata\",\"req\":\"^0.23\"},{\"features\":[\"config\"],\"name\":\"dylint_internal\",\"req\":\"=5.0.0\"},{\"name\":\"paste\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rustc_version\",\"req\":\"^0.4\"},{\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.23\"},{\"name\":\"thiserror\",\"req\":\"^2.0\"},{\"name\":\"toml\",\"req\":\"^0.9\"},{\"kind\":\"build\",\"name\":\"toml\",\"req\":\"^0.9\"}],\"features\":{\"constituent\":[]}}", "dylint_testing_5.0.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0\"},{\"name\":\"cargo_metadata\",\"req\":\"^0.23\"},{\"name\":\"compiletest_rs\",\"req\":\"^0.11\"},{\"name\":\"dylint\",\"req\":\"=5.0.0\"},{\"name\":\"dylint_internal\",\"req\":\"=5.0.0\"},{\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"once_cell\",\"req\":\"^1.21\"},{\"name\":\"regex\",\"req\":\"^1.11\"},{\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"tempfile\",\"req\":\"^3.23\"}],\"features\":{\"default\":[],\"deny_warnings\":[]}}", "dyn-clone_1.0.20": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.66\"}],\"features\":{}}", + "ed25519-dalek_2.2.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"blake2\",\"req\":\"^0.10\"},{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"default_features\":false,\"features\":[\"digest\"],\"name\":\"curve25519-dalek\",\"req\":\"^4\"},{\"default_features\":false,\"features\":[\"digest\",\"rand_core\"],\"kind\":\"dev\",\"name\":\"curve25519-dalek\",\"req\":\"^4\"},{\"default_features\":false,\"name\":\"ed25519\",\"req\":\">=2.2, <2.3\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"hex-literal\",\"req\":\"^0.4\"},{\"default_features\":false,\"name\":\"merlin\",\"optional\":true,\"req\":\"^3\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"default_features\":false,\"name\":\"rand_core\",\"optional\":true,\"req\":\"^0.6.4\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"rand_core\",\"req\":\"^0.6.4\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"sha2\",\"req\":\"^0.10\"},{\"kind\":\"dev\",\"name\":\"sha3\",\"req\":\"^0.10\"},{\"default_features\":false,\"name\":\"signature\",\"optional\":true,\"req\":\">=2.0, <2.3\"},{\"default_features\":false,\"name\":\"subtle\",\"req\":\"^2.3.0\"},{\"kind\":\"dev\",\"name\":\"toml\",\"req\":\"^0.7\"},{\"default_features\":false,\"features\":[\"static_secrets\"],\"kind\":\"dev\",\"name\":\"x25519-dalek\",\"req\":\"^2\"},{\"default_features\":false,\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1.5\"}],\"features\":{\"alloc\":[\"curve25519-dalek/alloc\",\"ed25519/alloc\",\"serde?/alloc\",\"zeroize/alloc\"],\"asm\":[\"sha2/asm\"],\"batch\":[\"alloc\",\"merlin\",\"rand_core\"],\"default\":[\"fast\",\"std\",\"zeroize\"],\"digest\":[\"signature/digest\"],\"fast\":[\"curve25519-dalek/precomputed-tables\"],\"hazmat\":[],\"legacy_compatibility\":[\"curve25519-dalek/legacy_compatibility\"],\"pem\":[\"alloc\",\"ed25519/pem\",\"pkcs8\"],\"pkcs8\":[\"ed25519/pkcs8\"],\"rand_core\":[\"dep:rand_core\"],\"serde\":[\"dep:serde\",\"ed25519/serde\"],\"std\":[\"alloc\",\"ed25519/std\",\"serde?/std\",\"sha2/std\"],\"zeroize\":[\"dep:zeroize\",\"curve25519-dalek/zeroize\"]}}", + "ed25519_2.2.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1\"},{\"features\":[\"rand_core\"],\"kind\":\"dev\",\"name\":\"ed25519-dalek\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"hex-literal\",\"req\":\"^0.4\"},{\"name\":\"pkcs8\",\"optional\":true,\"req\":\"^0.10\"},{\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"rand_core\",\"req\":\"^0.6\"},{\"default_features\":false,\"features\":[\"signature\"],\"kind\":\"dev\",\"name\":\"ring-compat\",\"req\":\"^0.8\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"serde_bytes\",\"optional\":true,\"req\":\"^0.11\"},{\"default_features\":false,\"name\":\"signature\",\"req\":\"^2\"},{\"default_features\":false,\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"alloc\":[\"pkcs8?/alloc\"],\"default\":[\"std\"],\"pem\":[\"alloc\",\"pkcs8/pem\"],\"serde_bytes\":[\"serde\",\"dep:serde_bytes\"],\"std\":[\"pkcs8?/std\",\"signature/std\"]}}", "either_1.15.0": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"alloc\",\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.95\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[],\"use_std\":[\"std\"]}}", "ena_0.14.3": "{\"dependencies\":[{\"name\":\"dogged\",\"optional\":true,\"req\":\"^0.2.0\"},{\"name\":\"log\",\"req\":\"^0.4\"}],\"features\":{\"bench\":[],\"persistent\":[\"dogged\"]}}", "encode_unicode_1.0.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"ascii\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.0\",\"target\":\"cfg(unix)\"},{\"features\":[\"https-native\"],\"kind\":\"dev\",\"name\":\"minreq\",\"req\":\"^2.6\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", @@ -1239,7 +1241,7 @@ "rama-unix_0.3.0-alpha.4": "{\"dependencies\":[{\"name\":\"pin-project-lite\",\"req\":\"^0.2\"},{\"name\":\"rama-core\",\"req\":\"^0.3.0-alpha.4\"},{\"name\":\"rama-net\",\"req\":\"^0.3.0-alpha.4\"},{\"features\":[\"macros\",\"net\"],\"name\":\"tokio\",\"req\":\"^1.48\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.48\"}],\"features\":{\"default\":[]}}", "rama-utils_0.3.0-alpha.4": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"ahash\",\"req\":\"^0.8\"},{\"name\":\"const_format\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"loom\",\"req\":\"^0.7\",\"target\":\"cfg(loom)\"},{\"name\":\"parking_lot\",\"req\":\"^0.12\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"name\":\"rama-macros\",\"req\":\"^0.3.0-alpha.4\"},{\"name\":\"regex\",\"req\":\"^1.12\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1\"},{\"features\":[\"write\",\"serde\",\"const_generics\",\"const_new\"],\"name\":\"smallvec\",\"req\":\"^1.15\"},{\"name\":\"smol_str\",\"req\":\"^0.3\"},{\"features\":[\"time\",\"macros\"],\"name\":\"tokio\",\"req\":\"^1.48\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0\"},{\"name\":\"wildcard\",\"req\":\"^0.3\"}],\"features\":{}}", "rand_0.8.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.2.1\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.22\",\"target\":\"cfg(unix)\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.4\"},{\"features\":[\"into_bits\"],\"name\":\"packed_simd\",\"optional\":true,\"package\":\"packed_simd_2\",\"req\":\"^0.3.7\"},{\"default_features\":false,\"name\":\"rand_chacha\",\"optional\":true,\"req\":\"^0.3.0\"},{\"name\":\"rand_core\",\"req\":\"^0.6.0\"},{\"kind\":\"dev\",\"name\":\"rand_pcg\",\"req\":\"^0.3.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"}],\"features\":{\"alloc\":[\"rand_core/alloc\"],\"default\":[\"std\",\"std_rng\"],\"getrandom\":[\"rand_core/getrandom\"],\"min_const_gen\":[],\"nightly\":[],\"serde1\":[\"serde\",\"rand_core/serde1\"],\"simd_support\":[\"packed_simd\"],\"small_rng\":[],\"std\":[\"rand_core/std\",\"rand_chacha/std\",\"alloc\",\"getrandom\",\"libc\"],\"std_rng\":[\"rand_chacha\"]}}", - "rand_0.9.2": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.2.1\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.4\"},{\"default_features\":false,\"name\":\"rand_chacha\",\"optional\":true,\"req\":\"^0.9.0\"},{\"default_features\":false,\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rand_pcg\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.7\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.140\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"std_rng\",\"os_rng\",\"small_rng\",\"thread_rng\"],\"log\":[\"dep:log\"],\"nightly\":[],\"os_rng\":[\"rand_core/os_rng\"],\"serde\":[\"dep:serde\",\"rand_core/serde\"],\"simd_support\":[],\"small_rng\":[],\"std\":[\"rand_core/std\",\"rand_chacha?/std\",\"alloc\"],\"std_rng\":[\"dep:rand_chacha\"],\"thread_rng\":[\"std\",\"std_rng\",\"os_rng\"],\"unbiased\":[]}}", + "rand_0.9.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.2.1\"},{\"default_features\":false,\"name\":\"rand_chacha\",\"optional\":true,\"req\":\"^0.9.0\"},{\"default_features\":false,\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rand_pcg\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.7\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.140\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"std_rng\",\"os_rng\",\"small_rng\",\"thread_rng\"],\"log\":[],\"nightly\":[],\"os_rng\":[\"rand_core/os_rng\"],\"serde\":[\"dep:serde\",\"rand_core/serde\"],\"simd_support\":[],\"small_rng\":[],\"std\":[\"rand_core/std\",\"rand_chacha?/std\",\"alloc\"],\"std_rng\":[\"dep:rand_chacha\"],\"thread_rng\":[\"std\",\"std_rng\",\"os_rng\"],\"unbiased\":[]}}", "rand_chacha_0.3.1": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"ppv-lite86\",\"req\":\"^0.2.8\"},{\"name\":\"rand_core\",\"req\":\"^0.6.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"serde1\":[\"serde\"],\"simd\":[],\"std\":[\"ppv-lite86/std\"]}}", "rand_chacha_0.9.0": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"ppv-lite86\",\"req\":\"^0.2.14\"},{\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"features\":[\"os_rng\"],\"kind\":\"dev\",\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"os_rng\":[\"rand_core/os_rng\"],\"serde\":[\"dep:serde\"],\"std\":[\"ppv-lite86/std\",\"rand_core/std\"]}}", "rand_core_0.6.4": "{\"dependencies\":[{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.2\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"serde1\":[\"serde\"],\"std\":[\"alloc\",\"getrandom\",\"getrandom/std\"]}}", @@ -1287,7 +1289,7 @@ "rustix_1.1.4": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bitflags\",\"req\":\"^2.4.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4\",\"target\":\"cfg(all(criterion, not(any(target_os = \\\"emscripten\\\", target_os = \\\"wasi\\\"))))\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.182\",\"target\":\"cfg(all(not(windows), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.182\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.171\"},{\"default_features\":false,\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(all(not(windows), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(windows)\"},{\"default_features\":false,\"name\":\"libc_errno\",\"optional\":true,\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\"},{\"default_features\":false,\"features\":[\"general\",\"ioctl\",\"no_std\"],\"name\":\"linux-raw-sys\",\"req\":\"^0.12\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"features\":[\"auxvec\",\"general\",\"errno\",\"ioctl\",\"no_std\",\"elf\"],\"name\":\"linux-raw-sys\",\"req\":\"^0.12\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"kind\":\"dev\",\"name\":\"memoffset\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.20.3\",\"target\":\"cfg(windows)\"},{\"name\":\"rustc-std-workspace-alloc\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"serial_test\",\"req\":\"^2.0.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.5.0\"},{\"features\":[\"Win32_Foundation\",\"Win32_Networking_WinSock\"],\"name\":\"windows-sys\",\"req\":\">=0.52, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"all-apis\":[\"event\",\"fs\",\"io_uring\",\"mm\",\"mount\",\"net\",\"param\",\"pipe\",\"process\",\"pty\",\"rand\",\"runtime\",\"shm\",\"stdio\",\"system\",\"termios\",\"thread\",\"time\"],\"alloc\":[],\"default\":[\"std\"],\"event\":[],\"fs\":[],\"io_uring\":[\"event\",\"fs\",\"net\",\"thread\",\"linux-raw-sys/io_uring\"],\"linux_4_11\":[],\"linux_5_1\":[\"linux_4_11\"],\"linux_5_11\":[\"linux_5_1\"],\"linux_latest\":[\"linux_5_11\"],\"mm\":[],\"mount\":[],\"net\":[\"linux-raw-sys/net\",\"linux-raw-sys/netlink\",\"linux-raw-sys/if_ether\",\"linux-raw-sys/xdp\"],\"param\":[],\"pipe\":[],\"process\":[\"linux-raw-sys/prctl\"],\"pty\":[\"fs\"],\"rand\":[],\"runtime\":[\"linux-raw-sys/prctl\"],\"rustc-dep-of-std\":[\"core\",\"rustc-std-workspace-alloc\",\"linux-raw-sys/rustc-dep-of-std\",\"bitflags/rustc-dep-of-std\"],\"shm\":[\"fs\"],\"std\":[\"bitflags/std\",\"alloc\",\"libc?/std\",\"libc_errno?/std\"],\"stdio\":[],\"system\":[\"linux-raw-sys/system\"],\"termios\":[],\"thread\":[\"linux-raw-sys/prctl\"],\"time\":[],\"try_close\":[],\"use-explicitly-provided-auxv\":[],\"use-libc\":[\"libc_errno\",\"libc\"],\"use-libc-auxv\":[]}}", "rustls-native-certs_0.8.3": "{\"dependencies\":[{\"name\":\"openssl-probe\",\"req\":\"^0.2\",\"target\":\"cfg(all(unix, not(target_os = \\\"macos\\\")))\"},{\"features\":[\"std\"],\"name\":\"pki-types\",\"package\":\"rustls-pki-types\",\"req\":\"^1.10\"},{\"kind\":\"dev\",\"name\":\"ring\",\"req\":\"^0.17\"},{\"kind\":\"dev\",\"name\":\"rustls\",\"req\":\"^0.23\"},{\"kind\":\"dev\",\"name\":\"rustls-webpki\",\"req\":\"^0.103\"},{\"name\":\"schannel\",\"req\":\"^0.1\",\"target\":\"cfg(windows)\"},{\"name\":\"security-framework\",\"req\":\"^3\",\"target\":\"cfg(target_os = \\\"macos\\\")\"},{\"kind\":\"dev\",\"name\":\"serial_test\",\"req\":\"^3\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.5\"},{\"kind\":\"dev\",\"name\":\"untrusted\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"webpki-roots\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"x509-parser\",\"req\":\"^0.18\"}],\"features\":{}}", "rustls-pki-types_1.14.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"crabgrind\",\"req\":\"=0.1.9\",\"target\":\"cfg(all(target_os = \\\"linux\\\", target_arch = \\\"x86_64\\\"))\"},{\"name\":\"web-time\",\"optional\":true,\"req\":\"^1\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", target_os = \\\"unknown\\\"))\"},{\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"alloc\":[\"dep:zeroize\"],\"default\":[\"alloc\"],\"std\":[\"alloc\"],\"web\":[\"web-time\"]}}", - "rustls-webpki_0.103.10": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aws-lc-rs\",\"optional\":true,\"req\":\"^1.14\"},{\"kind\":\"dev\",\"name\":\"base64\",\"req\":\"^0.22\"},{\"kind\":\"dev\",\"name\":\"bencher\",\"req\":\"^0.1.5\"},{\"kind\":\"dev\",\"name\":\"bzip2\",\"req\":\"^0.6\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.17.2\"},{\"default_features\":false,\"name\":\"pki-types\",\"package\":\"rustls-pki-types\",\"req\":\"^1.12\"},{\"default_features\":false,\"features\":[\"aws_lc_rs\"],\"kind\":\"dev\",\"name\":\"rcgen\",\"req\":\"^0.14.2\"},{\"default_features\":false,\"name\":\"ring\",\"optional\":true,\"req\":\"^0.17\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"untrusted\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"x509-parser\",\"req\":\"^0.18.1\"}],\"features\":{\"alloc\":[\"ring?/alloc\",\"pki-types/alloc\"],\"aws-lc-rs\":[\"dep:aws-lc-rs\",\"aws-lc-rs/aws-lc-sys\",\"aws-lc-rs/prebuilt-nasm\"],\"aws-lc-rs-fips\":[\"dep:aws-lc-rs\",\"aws-lc-rs/fips\"],\"aws-lc-rs-unstable\":[\"aws-lc-rs\",\"aws-lc-rs/unstable\"],\"default\":[\"std\"],\"ring\":[\"dep:ring\"],\"std\":[\"alloc\",\"pki-types/std\"]}}", + "rustls-webpki_0.103.12": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aws-lc-rs\",\"optional\":true,\"req\":\"^1.14\"},{\"kind\":\"dev\",\"name\":\"base64\",\"req\":\"^0.22\"},{\"kind\":\"dev\",\"name\":\"bencher\",\"req\":\"^0.1.5\"},{\"kind\":\"dev\",\"name\":\"bzip2\",\"req\":\"^0.6\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.17.2\"},{\"default_features\":false,\"name\":\"pki-types\",\"package\":\"rustls-pki-types\",\"req\":\"^1.12\"},{\"default_features\":false,\"features\":[\"aws_lc_rs\"],\"kind\":\"dev\",\"name\":\"rcgen\",\"req\":\"^0.14.2\"},{\"default_features\":false,\"name\":\"ring\",\"optional\":true,\"req\":\"^0.17\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"untrusted\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"x509-parser\",\"req\":\"^0.18.1\"}],\"features\":{\"alloc\":[\"ring?/alloc\",\"pki-types/alloc\"],\"aws-lc-rs\":[\"dep:aws-lc-rs\",\"aws-lc-rs/aws-lc-sys\",\"aws-lc-rs/prebuilt-nasm\"],\"aws-lc-rs-fips\":[\"dep:aws-lc-rs\",\"aws-lc-rs/fips\"],\"aws-lc-rs-unstable\":[\"aws-lc-rs\",\"aws-lc-rs/unstable\"],\"default\":[\"std\"],\"ring\":[\"dep:ring\"],\"std\":[\"alloc\",\"pki-types/std\"]}}", "rustls_0.23.36": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aws-lc-rs\",\"optional\":true,\"req\":\"^1.14\"},{\"kind\":\"dev\",\"name\":\"base64\",\"req\":\"^0.22\"},{\"kind\":\"dev\",\"name\":\"bencher\",\"req\":\"^0.1.5\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"brotli\",\"optional\":true,\"req\":\"^8\"},{\"name\":\"brotli-decompressor\",\"optional\":true,\"req\":\"^5.0.0\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"default-hasher\",\"inline-more\"],\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.15\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.8\"},{\"kind\":\"dev\",\"name\":\"log\",\"req\":\"^0.4.8\"},{\"kind\":\"dev\",\"name\":\"macro_rules_attribute\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"num-bigint\",\"req\":\"^0.4.4\"},{\"default_features\":false,\"features\":[\"alloc\",\"race\"],\"name\":\"once_cell\",\"req\":\"^1.16\"},{\"features\":[\"alloc\"],\"name\":\"pki-types\",\"package\":\"rustls-pki-types\",\"req\":\"^1.12\"},{\"default_features\":false,\"features\":[\"pem\",\"aws_lc_rs\"],\"kind\":\"dev\",\"name\":\"rcgen\",\"req\":\"^0.14\"},{\"name\":\"ring\",\"optional\":true,\"req\":\"^0.17\"},{\"kind\":\"build\",\"name\":\"rustversion\",\"optional\":true,\"req\":\"^1.0.6\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"subtle\",\"req\":\"^2.5.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"time\",\"req\":\"^0.3.6\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"webpki\",\"package\":\"rustls-webpki\",\"req\":\"^0.103.5\"},{\"kind\":\"dev\",\"name\":\"webpki-roots\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"x509-parser\",\"req\":\"^0.17\"},{\"name\":\"zeroize\",\"req\":\"^1.8\"},{\"name\":\"zlib-rs\",\"optional\":true,\"req\":\"^0.5\"}],\"features\":{\"aws-lc-rs\":[\"aws_lc_rs\"],\"aws_lc_rs\":[\"dep:aws-lc-rs\",\"webpki/aws-lc-rs\",\"aws-lc-rs/aws-lc-sys\",\"aws-lc-rs/prebuilt-nasm\"],\"brotli\":[\"dep:brotli\",\"dep:brotli-decompressor\",\"std\"],\"custom-provider\":[],\"default\":[\"aws_lc_rs\",\"logging\",\"prefer-post-quantum\",\"std\",\"tls12\"],\"fips\":[\"aws_lc_rs\",\"aws-lc-rs?/fips\",\"webpki/aws-lc-rs-fips\"],\"logging\":[\"log\"],\"prefer-post-quantum\":[\"aws_lc_rs\"],\"read_buf\":[\"rustversion\",\"std\"],\"ring\":[\"dep:ring\",\"webpki/ring\"],\"std\":[\"webpki/std\",\"pki-types/std\",\"once_cell/std\"],\"tls12\":[],\"zlib\":[\"dep:zlib-rs\"]}}", "rustversion_1.0.22": "{\"dependencies\":[{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", "rustyline_14.0.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"assert_matches\",\"req\":\"^1.2\"},{\"name\":\"bitflags\",\"req\":\"^2.0\"},{\"default_features\":false,\"name\":\"buffer-redux\",\"optional\":true,\"req\":\"^1.0\",\"target\":\"cfg(unix)\"},{\"name\":\"cfg-if\",\"req\":\"^1.0\"},{\"name\":\"clipboard-win\",\"req\":\"^5.0\",\"target\":\"cfg(windows)\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"fd-lock\",\"optional\":true,\"req\":\"^4.0.0\"},{\"name\":\"home\",\"optional\":true,\"req\":\"^0.5.4\"},{\"name\":\"libc\",\"req\":\"^0.2\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"memchr\",\"req\":\"^2.0\"},{\"default_features\":false,\"features\":[\"fs\",\"ioctl\",\"poll\",\"signal\",\"term\"],\"name\":\"nix\",\"req\":\"^0.28\",\"target\":\"cfg(unix)\"},{\"name\":\"radix_trie\",\"optional\":true,\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"name\":\"regex\",\"optional\":true,\"req\":\"^1.5.5\"},{\"default_features\":false,\"features\":[\"bundled\",\"backup\"],\"name\":\"rusqlite\",\"optional\":true,\"req\":\"^0.31.0\"},{\"name\":\"rustyline-derive\",\"optional\":true,\"req\":\"^0.10.0\"},{\"default_features\":false,\"name\":\"signal-hook\",\"optional\":true,\"req\":\"^0.3\",\"target\":\"cfg(unix)\"},{\"default_features\":false,\"name\":\"skim\",\"optional\":true,\"req\":\"^0.10\",\"target\":\"cfg(unix)\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.1.0\"},{\"name\":\"termios\",\"optional\":true,\"req\":\"^0.3.3\",\"target\":\"cfg(unix)\"},{\"name\":\"unicode-segmentation\",\"req\":\"^1.0\"},{\"name\":\"unicode-width\",\"req\":\"^0.1\"},{\"name\":\"utf8parse\",\"req\":\"^0.2\",\"target\":\"cfg(unix)\"},{\"features\":[\"Win32_Foundation\",\"Win32_System_Console\",\"Win32_Security\",\"Win32_System_Threading\",\"Win32_UI_Input_KeyboardAndMouse\"],\"name\":\"windows-sys\",\"req\":\"^0.52.0\",\"target\":\"cfg(windows)\"}],\"features\":{\"case_insensitive_history_search\":[\"regex\"],\"custom-bindings\":[\"radix_trie\"],\"default\":[\"custom-bindings\",\"with-dirs\",\"with-file-history\"],\"derive\":[\"rustyline-derive\"],\"with-dirs\":[\"home\"],\"with-file-history\":[\"fd-lock\"],\"with-fuzzy\":[\"skim\"],\"with-sqlite-history\":[\"rusqlite\"]}}", diff --git a/SECURITY.md b/SECURITY.md index d6dd568910..29a9b903df 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -11,3 +11,7 @@ Our security program is managed through Bugcrowd, and we ask that any validated ## Vulnerability Disclosure Program Our Vulnerability Program Guidelines are defined on our [Bugcrowd program page](https://bugcrowd.com/engagements/openai). + +## How to operate CODEX safely + +For details on Codex security boundaries, including sandboxing, approvals, and network controls, see [Agent approvals & security](https://developers.openai.com/codex/agent-approvals-security). diff --git a/announcement_tip.toml b/announcement_tip.toml index 2473258368..79c0daea76 100644 --- a/announcement_tip.toml +++ b/announcement_tip.toml @@ -4,20 +4,14 @@ # version_regex matches against the CLI version (env!("CARGO_PKG_VERSION")); omit to apply to all versions. # target_app specify which app should display the announcement (cli, vsce, ...). -[[announcements]] -content = "Welcome to Codex! Check out the new onboarding flow." -from_date = "2024-10-01" -to_date = "2024-10-15" -target_app = "cli" - -# Test announcement only for local build version until 2026-01-10 excluded (past) +# Test announcement only for local build version until 2027-05-10 excluded [[announcements]] content = "This is a test announcement" version_regex = "^0\\.0\\.0$" -to_date = "2026-05-10" +to_date = "2027-05-10" [[announcements]] -content = "**BREAKING NEWS**: `gpt-5.3-codex` is out! Upgrade to `0.98.0` for a faster, smarter, more steerable agent." -from_date = "2026-02-01" -to_date = "2026-02-16" -version_regex = "^0\\.(?:[0-9]|[1-8][0-9]|9[0-7])\\." +content = "Update Required - This version will no longer be supported starting May 8th. Please upgrade to the latest version (https://github.com/openai/codex/releases/latest) using your preferred package manager." +# Matches 0.x.y versions from 0.0.y through 0.119.y; excludes 0.120.0 and newer. +version_regex = "^0\\.(?:[0-9]|[1-9][0-9]|1[01][0-9])\\." +to_date = "2026-05-08" diff --git a/codex-rs/.cargo/audit.toml b/codex-rs/.cargo/audit.toml index 425997f6ae..3760d28648 100644 --- a/codex-rs/.cargo/audit.toml +++ b/codex-rs/.cargo/audit.toml @@ -1,9 +1,10 @@ [advisories] -# Reviewed 2026-04-11. Keep this list in sync with ../deny.toml. +# Reviewed 2026-04-15. Keep this list in sync with ../deny.toml. ignore = [ "RUSTSEC-2024-0388", # derivative 2.2.0 via starlark; upstream crate is unmaintained "RUSTSEC-2025-0057", # fxhash 0.2.1 via starlark_map; upstream crate is unmaintained "RUSTSEC-2024-0436", # paste 1.0.15 via starlark/ratatui; upstream crate is unmaintained "RUSTSEC-2024-0320", # yaml-rust via syntect; remove when syntect drops or updates it "RUSTSEC-2025-0141", # bincode via syntect; remove when syntect drops or updates it + "RUSTSEC-2026-0097", # rand 0.8.5 via age/codex-secrets and zbus/keyring; remove when transitive deps move to rand >=0.9.3 ] diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index ea79776bdc..68e0ec092f 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1461,6 +1461,7 @@ dependencies = [ "codex-sandboxing", "codex-shell-command", "codex-state", + "codex-thread-store", "codex-tools", "codex-utils-absolute-path", "codex-utils-cargo-bin", @@ -1694,6 +1695,7 @@ dependencies = [ "codex-stdio-to-uds", "codex-terminal-detection", "codex-tui", + "codex-utils-absolute-path", "codex-utils-cargo-bin", "codex-utils-cli", "codex-utils-path", @@ -1728,7 +1730,7 @@ dependencies = [ "opentelemetry", "opentelemetry_sdk", "pretty_assertions", - "rand 0.9.2", + "rand 0.9.3", "reqwest", "rustls", "rustls-native-certs", @@ -1830,6 +1832,7 @@ name = "codex-code-mode" version = "0.0.0" dependencies = [ "async-trait", + "codex-protocol", "deno_core_icudata", "pretty_assertions", "serde", @@ -1905,6 +1908,7 @@ dependencies = [ "codex-api", "codex-app-server-protocol", "codex-apply-patch", + "codex-arg0", "codex-async-utils", "codex-code-mode", "codex-config", @@ -1935,6 +1939,7 @@ dependencies = [ "codex-state", "codex-terminal-detection", "codex-test-binary-support", + "codex-thread-store", "codex-tools", "codex-utils-absolute-path", "codex-utils-cache", @@ -1956,6 +1961,7 @@ dependencies = [ "ctor 0.6.3", "dirs", "dunce", + "ed25519-dalek", "env-flags", "eventsource-stream", "futures", @@ -1973,7 +1979,7 @@ dependencies = [ "opentelemetry_sdk", "predicates", "pretty_assertions", - "rand 0.9.2", + "rand 0.9.3", "regex-lite", "reqwest", "rmcp", @@ -2326,7 +2332,7 @@ dependencies = [ "once_cell", "os_info", "pretty_assertions", - "rand 0.9.2", + "rand 0.9.3", "regex-lite", "reqwest", "serde", @@ -2694,6 +2700,8 @@ dependencies = [ name = "codex-sandboxing" version = "0.0.0" dependencies = [ + "anyhow", + "async-trait", "codex-network-proxy", "codex-protocol", "codex-utils-absolute-path", @@ -2702,6 +2710,7 @@ dependencies = [ "pretty_assertions", "serde_json", "tempfile", + "tokio", "tracing", "url", "which 8.0.0", @@ -2718,7 +2727,7 @@ dependencies = [ "codex-keyring-store", "keyring", "pretty_assertions", - "rand 0.9.2", + "rand 0.9.3", "regex", "schemars 0.8.22", "serde", @@ -2834,9 +2843,17 @@ version = "0.0.0" dependencies = [ "async-trait", "chrono", + "codex-git-utils", "codex-protocol", + "codex-rollout", + "codex-state", + "pretty_assertions", "serde", + "serde_json", + "tempfile", "thiserror 2.0.18", + "tokio", + "uuid", ] [[package]] @@ -2917,7 +2934,7 @@ dependencies = [ "pathdiff", "pretty_assertions", "pulldown-cmark", - "rand 0.9.2", + "rand 0.9.3", "ratatui", "ratatui-macros", "regex-lite", @@ -3650,6 +3667,7 @@ dependencies = [ "cfg-if", "cpufeatures", "curve25519-dalek-derive", + "digest", "fiat-crypto", "rustc_version", "subtle", @@ -4126,7 +4144,7 @@ dependencies = [ "libc", "option-ext", "redox_users 0.5.2", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4230,6 +4248,30 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "serde", + "sha2", + "subtle", + "zeroize", +] + [[package]] name = "either" version = "1.15.0" @@ -5159,7 +5201,7 @@ dependencies = [ "idna", "ipnet", "once_cell", - "rand 0.9.2", + "rand 0.9.3", "ring", "thiserror 2.0.18", "tinyvec", @@ -5181,7 +5223,7 @@ dependencies = [ "moka", "once_cell", "parking_lot", - "rand 0.9.2", + "rand 0.9.3", "resolv-conf", "smallvec", "thiserror 2.0.18", @@ -6728,7 +6770,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7327,7 +7369,7 @@ dependencies = [ "futures-util", "opentelemetry", "percent-encoding", - "rand 0.9.2", + "rand 0.9.3", "thiserror 2.0.18", "tokio", "tokio-stream", @@ -7876,7 +7918,7 @@ checksum = "bee689443a2bd0a16ab0348b52ee43e3b2d1b1f931c8aa5c9f8de4c86fbe8c40" dependencies = [ "bitflags 2.10.0", "num-traits", - "rand 0.9.2", + "rand 0.9.3", "rand_chacha 0.9.0", "rand_xorshift", "regex-syntax 0.8.8", @@ -8047,7 +8089,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.2", + "rand 0.9.3", "ring", "rustc-hash 2.1.1", "rustls", @@ -8179,7 +8221,7 @@ dependencies = [ "rama-http-types", "rama-net", "rama-utils", - "rand 0.9.2", + "rand 0.9.3", "serde", "serde_html_form", "serde_json", @@ -8249,7 +8291,7 @@ dependencies = [ "rama-macros", "rama-net", "rama-utils", - "rand 0.9.2", + "rand 0.9.3", "serde", "sha1", ] @@ -8277,7 +8319,7 @@ dependencies = [ "rama-error", "rama-macros", "rama-utils", - "rand 0.9.2", + "rand 0.9.3", "serde", "serde_json", "sync_wrapper", @@ -8351,7 +8393,7 @@ dependencies = [ "rama-http-types", "rama-net", "rama-utils", - "rand 0.9.2", + "rand 0.9.3", "tokio", ] @@ -8431,9 +8473,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "7ec095654a25171c2124e9e3393a930bddbffdc939556c914957a4c3e0a87166" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -8748,7 +8790,7 @@ dependencies = [ "pastey", "pin-project-lite", "process-wrap", - "rand 0.9.2", + "rand 0.9.3", "reqwest", "rmcp-macros", "schemars 1.2.1", @@ -8955,9 +8997,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.10" +version = "0.103.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06" dependencies = [ "aws-lc-rs", "ring", @@ -9332,7 +9374,7 @@ version = "0.46.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26ab054c34b87f96c3e4701bea1888317cde30cc7e4a6136d2c48454ab96661c" dependencies = [ - "rand 0.9.2", + "rand 0.9.3", "sentry-types", "serde", "serde_json", @@ -9380,7 +9422,7 @@ checksum = "eecbd63e9d15a26a40675ed180d376fcb434635d2e33de1c24003f61e3e2230d" dependencies = [ "debugid", "hex", - "rand 0.9.2", + "rand 0.9.3", "serde", "serde_json", "thiserror 2.0.18", @@ -11165,7 +11207,7 @@ dependencies = [ "http 1.4.0", "httparse", "log", - "rand 0.9.2", + "rand 0.9.3", "rustls", "rustls-pki-types", "sha1", @@ -11852,7 +11894,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index e4985e4582..4bb8837211 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -166,6 +166,7 @@ codex-state = { path = "state" } codex-stdio-to-uds = { path = "stdio-to-uds" } codex-terminal-detection = { path = "terminal-detection" } codex-test-binary-support = { path = "test-binary-support" } +codex-thread-store = { path = "thread-store" } codex-tools = { path = "tools" } codex-tui = { path = "tui" } codex-utils-absolute-path = { path = "utils/absolute-path" } @@ -227,6 +228,7 @@ diffy = "0.4.2" dirs = "6" dotenvy = "0.15.7" dunce = "1.0.4" +ed25519-dalek = { version = "2.2.0", features = ["pkcs8"] } encoding_rs = "0.8.35" env-flags = "0.1.1" env_logger = "0.11.9" diff --git a/codex-rs/app-server-client/src/lib.rs b/codex-rs/app-server-client/src/lib.rs index eb3d984965..35f9068712 100644 --- a/codex-rs/app-server-client/src/lib.rs +++ b/codex-rs/app-server-client/src/lib.rs @@ -28,6 +28,7 @@ use std::time::Duration; pub use codex_app_server::in_process::DEFAULT_IN_PROCESS_CHANNEL_CAPACITY; pub use codex_app_server::in_process::InProcessServerEvent; use codex_app_server::in_process::InProcessStartArgs; +use codex_app_server::in_process::LogDbLayer; use codex_app_server_protocol::ClientInfo; use codex_app_server_protocol::ClientNotification; use codex_app_server_protocol::ClientRequest; @@ -354,6 +355,8 @@ pub struct InProcessClientStartArgs { pub cloud_requirements: CloudRequirementsLoader, /// Feedback sink used by app-server/core telemetry and logs. pub feedback: CodexFeedback, + /// SQLite tracing layer used to flush recently emitted logs before feedback upload. + pub log_db: Option, /// Environment manager used by core execution and filesystem operations. pub environment_manager: Arc, /// Startup warnings emitted after initialize succeeds. @@ -405,6 +408,7 @@ impl InProcessClientStartArgs { loader_overrides: self.loader_overrides, cloud_requirements: self.cloud_requirements, feedback: self.feedback, + log_db: self.log_db, environment_manager: self.environment_manager, config_warnings: self.config_warnings, session_source: self.session_source, @@ -982,6 +986,7 @@ mod tests { loader_overrides: LoaderOverrides::default(), cloud_requirements: CloudRequirementsLoader::default(), feedback: CodexFeedback::new(), + log_db: None, environment_manager: Arc::new(EnvironmentManager::new(/*exec_server_url*/ None)), config_warnings: Vec::new(), session_source, @@ -1994,6 +1999,7 @@ mod tests { loader_overrides: LoaderOverrides::default(), cloud_requirements: CloudRequirementsLoader::default(), feedback: CodexFeedback::new(), + log_db: None, environment_manager: environment_manager.clone(), config_warnings: Vec::new(), session_source: SessionSource::Exec, diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index 9efc6571f2..3d6bf9d872 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -289,6 +289,11 @@ client_request_definitions! { params: v2::ThreadMemoryModeSetParams, response: v2::ThreadMemoryModeSetResponse, }, + #[experimental("memory/reset")] + MemoryReset => "memory/reset" { + params: #[ts(type = "undefined")] #[serde(skip_serializing_if = "Option::is_none")] Option<()>, + response: v2::MemoryResetResponse, + }, ThreadUnarchive => "thread/unarchive" { params: v2::ThreadUnarchiveParams, response: v2::ThreadUnarchiveResponse, diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 0843241f32..5897625de1 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -3114,6 +3114,11 @@ pub struct ThreadMemoryModeSetParams { #[ts(export_to = "v2/")] pub struct ThreadMemoryModeSetResponse {} +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct MemoryResetResponse {} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] diff --git a/codex-rs/app-server/BUILD.bazel b/codex-rs/app-server/BUILD.bazel index 872f533be1..f32f811627 100644 --- a/codex-rs/app-server/BUILD.bazel +++ b/codex-rs/app-server/BUILD.bazel @@ -3,6 +3,9 @@ load("//:defs.bzl", "codex_rust_crate") codex_rust_crate( name = "app-server", crate_name = "codex_app_server", + integration_test_tags_extra_by_stem = { + "all": ["flaky"], + }, integration_test_timeout = "long", test_tags = ["no-sandbox"], ) diff --git a/codex-rs/app-server/Cargo.toml b/codex-rs/app-server/Cargo.toml index 5dc3a31485..408f39fafe 100644 --- a/codex-rs/app-server/Cargo.toml +++ b/codex-rs/app-server/Cargo.toml @@ -54,6 +54,7 @@ codex-rmcp-client = { workspace = true } codex-rollout = { workspace = true } codex-sandboxing = { workspace = true } codex-state = { workspace = true } +codex-thread-store = { workspace = true } codex-tools = { workspace = true } codex-utils-absolute-path = { workspace = true } codex-utils-json-to-toml = { workspace = true } diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index e00bbefc91..5cfe052b50 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -41,8 +41,11 @@ Security note: - Non-loopback websocket listeners currently allow unauthenticated connections by default during rollout. If you expose one remotely, configure websocket auth explicitly now. - Supported auth modes are app-server flags: - `--ws-auth capability-token --ws-token-file /absolute/path` + - `--ws-auth capability-token --ws-token-sha256 HEX` - `--ws-auth signed-bearer-token --ws-shared-secret-file /absolute/path` for HMAC-signed JWT/JWS bearer tokens, with optional `--ws-issuer`, `--ws-audience`, `--ws-max-clock-skew-seconds` - Clients present the credential as `Authorization: Bearer ` during the websocket handshake. Auth is enforced before JSON-RPC `initialize`. +- When starting `codex app-server` manually, prefer `--ws-token-file` over passing raw bearer tokens on the command line. Store a high-entropy token in a file readable only by your user, then have your client present that token in the websocket `Authorization` header. +- `--ws-token-sha256` is intended for clients that keep the raw token in a separate local secret store and only need the server to know the SHA-256 verifier. The hash may appear in process listings, but it is not sufficient to authenticate; clients still need the original raw token. Only use this mode with randomly generated high-entropy tokens, not passwords or other guessable values. Tracing/log output: @@ -141,6 +144,7 @@ Example with notification opt-out: - `thread/read` — read a stored thread by id without resuming it; optionally include turns via `includeTurns`. The returned `thread` includes `status` (`ThreadStatus`), defaulting to `notLoaded` when the thread is not currently loaded. - `thread/metadata/update` — patch stored thread metadata in sqlite; currently supports updating persisted `gitInfo` fields and returns the refreshed `thread`. - `thread/memoryMode/set` — experimental; set a thread’s persisted memory eligibility to `"enabled"` or `"disabled"` for either a loaded thread or a stored rollout; returns `{}` on success. +- `memory/reset` — experimental; clear the current `CODEX_HOME/memories` directory and reset persisted memory stage data in sqlite while preserving existing thread memory modes; returns `{}` on success. - `thread/status/changed` — notification emitted when a loaded thread’s status changes (`threadId` + new `status`). - `thread/archive` — move a thread’s rollout file into the archived directory; returns `{}` on success and emits `thread/archived`. - `thread/unsubscribe` — unsubscribe this connection from thread turn/item events. If this was the last subscriber, the server keeps the thread loaded and unloads it only after it has had no subscribers and no thread activity for 30 minutes, then emits `thread/closed`. @@ -413,6 +417,13 @@ Experimental: use `thread/memoryMode/set` to change whether a thread remains eli { "id": 26, "result": {} } ``` +Experimental: use `memory/reset` to clear local memory artifacts and sqlite-backed memory stage data for the current Codex home. This preserves existing thread memory modes; use `thread/memoryMode/set` separately when a thread's future memory eligibility should change. + +```json +{ "method": "memory/reset", "id": 27 } +{ "id": 27, "result": {} } +``` + ### Example: Archive a thread Use `thread/archive` to move the persisted rollout (stored as a JSONL file on disk) into the archived sessions directory. diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index f31aafeac5..42cb8b2d1a 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -92,6 +92,7 @@ use codex_app_server_protocol::McpServerStatus; use codex_app_server_protocol::McpServerStatusDetail; use codex_app_server_protocol::McpServerToolCallParams; use codex_app_server_protocol::McpServerToolCallResponse; +use codex_app_server_protocol::MemoryResetResponse; use codex_app_server_protocol::MockExperimentalMethodParams; use codex_app_server_protocol::MockExperimentalMethodResponse; use codex_app_server_protocol::ModelListParams; @@ -202,7 +203,6 @@ use codex_chatgpt::connectors; use codex_cloud_requirements::cloud_requirements_loader; use codex_config::types::McpServerTransportConfig; use codex_core::CodexThread; -use codex_core::Cursor as RolloutCursor; use codex_core::ForkSnapshot; use codex_core::NewThread; use codex_core::RolloutRecorder; @@ -210,7 +210,6 @@ use codex_core::SessionMeta; use codex_core::SteerInputError; use codex_core::ThreadConfigSnapshot; use codex_core::ThreadManager; -use codex_core::ThreadSortKey as CoreThreadSortKey; use codex_core::append_thread_name; use codex_core::config::Config; use codex_core::config::ConfigOverrides; @@ -231,7 +230,6 @@ use codex_core::find_archived_thread_path_by_id_str; use codex_core::find_thread_name_by_id; use codex_core::find_thread_names_by_ids; use codex_core::find_thread_path_by_id_str; -use codex_core::parse_cursor; use codex_core::path_utils; use codex_core::plugins::MarketplaceAddError; use codex_core::plugins::MarketplaceError; @@ -246,7 +244,6 @@ use codex_core::plugins::load_plugin_apps; use codex_core::plugins::load_plugin_mcp_servers; use codex_core::read_head_for_summary; use codex_core::read_session_meta_line; -use codex_core::rollout_date_parts; use codex_core::sandboxing::SandboxPermissions; use codex_core::windows_sandbox::WindowsSandboxLevelExt; use codex_core::windows_sandbox::WindowsSandboxSetupMode as CoreWindowsSandboxSetupMode; @@ -321,15 +318,20 @@ use codex_state::StateRuntime; use codex_state::ThreadMetadata; use codex_state::ThreadMetadataBuilder; use codex_state::log_db::LogDbLayer; +use codex_thread_store::ArchiveThreadParams as StoreArchiveThreadParams; +use codex_thread_store::ListThreadsParams as StoreListThreadsParams; +use codex_thread_store::LocalThreadStore; +use codex_thread_store::ReadThreadParams as StoreReadThreadParams; +use codex_thread_store::StoredThread; +use codex_thread_store::ThreadSortKey as StoreThreadSortKey; +use codex_thread_store::ThreadStore; +use codex_thread_store::ThreadStoreError; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_json_to_toml::json_to_toml; use codex_utils_pty::DEFAULT_OUTPUT_BYTES_CAP; use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::HashSet; -use std::ffi::OsStr; -use std::fs::FileTimes; -use std::fs::OpenOptions; use std::io::Error as IoError; use std::path::Path; use std::path::PathBuf; @@ -339,7 +341,6 @@ use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; use std::time::Duration; use std::time::Instant; -use std::time::SystemTime; use tokio::sync::Mutex; use tokio::sync::broadcast; use tokio::sync::oneshot; @@ -443,6 +444,7 @@ pub(crate) struct CodexMessageProcessor { analytics_events_client: AnalyticsEventsClient, arg0_paths: Arg0DispatchPaths, config: Arc, + thread_store: LocalThreadStore, cli_overrides: Arc>>, runtime_feature_enablement: Arc>>, cloud_requirements: Arc>, @@ -701,6 +703,7 @@ impl CodexMessageProcessor { outgoing: outgoing.clone(), analytics_events_client, arg0_paths, + thread_store: LocalThreadStore::new(codex_rollout::RolloutConfig::from_view(&config)), config, cli_overrides, runtime_feature_enablement, @@ -908,6 +911,10 @@ impl CodexMessageProcessor { self.thread_memory_mode_set(to_connection_request_id(request_id), params) .await; } + ClientRequest::MemoryReset { request_id, params } => { + self.memory_reset(to_connection_request_id(request_id), params) + .await; + } ClientRequest::ThreadUnarchive { request_id, params } => { self.thread_unarchive(to_connection_request_id(request_id), params) .await; @@ -2709,7 +2716,6 @@ impl CodexMessageProcessor { } async fn thread_archive(&self, request_id: ConnectionRequestId, params: ThreadArchiveParams) { - // TODO(jif) mostly rewrite this using sqlite after phase 1 let thread_id = match ThreadId::from_string(¶ms.thread_id) { Ok(id) => id, Err(err) => { @@ -2723,32 +2729,28 @@ impl CodexMessageProcessor { } }; - let rollout_path = - match find_thread_path_by_id_str(&self.config.codex_home, &thread_id.to_string()).await - { - Ok(Some(p)) => p, - Ok(None) => { - let error = JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!("no rollout found for thread id {thread_id}"), - data: None, - }; - self.outgoing.send_error(request_id, error).await; - return; - } - Err(err) => { - let error = JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!("failed to locate thread id {thread_id}: {err}"), - data: None, - }; - self.outgoing.send_error(request_id, error).await; - return; - } - }; - let thread_id_str = thread_id.to_string(); - match self.archive_thread_common(thread_id, &rollout_path).await { + if let Err(err) = self + .thread_store + .read_thread(StoreReadThreadParams { + thread_id, + include_archived: false, + include_history: false, + }) + .await + { + self.outgoing + .send_error(request_id, thread_store_archive_error("archive", err)) + .await; + return; + } + self.prepare_thread_for_archive(thread_id).await; + + match self + .thread_store + .archive_thread(StoreArchiveThreadParams { thread_id }) + .await + { Ok(()) => { let response = ThreadArchiveResponse {}; self.outgoing.send_response(request_id, response).await; @@ -2760,7 +2762,9 @@ impl CodexMessageProcessor { .await; } Err(err) => { - self.outgoing.send_error(request_id, err).await; + self.outgoing + .send_error(request_id, thread_store_archive_error("archive", err)) + .await; } } } @@ -3046,6 +3050,86 @@ impl CodexMessageProcessor { .await; } + async fn memory_reset(&self, request_id: ConnectionRequestId, _params: Option<()>) { + let state_db = match StateRuntime::init( + self.config.sqlite_home.clone(), + self.config.model_provider_id.clone(), + ) + .await + { + Ok(state_db) => state_db, + Err(err) => { + self.send_internal_error( + request_id, + format!("failed to open state db for memory reset: {err}"), + ) + .await; + return; + } + }; + + if let Err(err) = state_db.clear_memory_data().await { + self.send_internal_error( + request_id, + format!("failed to clear memory rows in state db: {err}"), + ) + .await; + return; + } + + let memory_root = self.config.codex_home.join("memories"); + let memory_extensions_root = self.config.codex_home.join("memories_extensions"); + let clear_memory_root_result: std::io::Result<()> = async { + for directory in [memory_root.as_path(), memory_extensions_root.as_path()] { + match tokio::fs::symlink_metadata(directory).await { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!( + "refusing to clear symlinked memory root {}", + directory.display() + ), + )); + } + Ok(_) => {} + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => return Err(err), + } + + tokio::fs::create_dir_all(directory).await?; + let mut entries = tokio::fs::read_dir(directory).await?; + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); + let file_type = entry.file_type().await?; + if file_type.is_dir() { + tokio::fs::remove_dir_all(path).await?; + } else { + tokio::fs::remove_file(path).await?; + } + } + } + + Ok(()) + } + .await; + + if let Err(err) = clear_memory_root_result { + self.send_internal_error( + request_id, + format!( + "failed to clear memory directory {}: {err}", + memory_root.display() + ), + ) + .await; + return; + } + + self.outgoing + .send_response(request_id, MemoryResetResponse {}) + .await; + } + async fn thread_metadata_update( &self, request_id: ConnectionRequestId, @@ -3349,7 +3433,6 @@ impl CodexMessageProcessor { request_id: ConnectionRequestId, params: ThreadUnarchiveParams, ) { - // TODO(jif) mostly rewrite this using sqlite after phase 1 let thread_id = match ThreadId::from_string(¶ms.thread_id) { Ok(id) => id, Err(err) => { @@ -3363,152 +3446,21 @@ impl CodexMessageProcessor { } }; - let archived_path = match find_archived_thread_path_by_id_str( - &self.config.codex_home, - &thread_id.to_string(), - ) - .await - { - Ok(Some(path)) => path, - Ok(None) => { - let error = JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!("no archived rollout found for thread id {thread_id}"), - data: None, - }; - self.outgoing.send_error(request_id, error).await; - return; - } - Err(err) => { - let error = JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!("failed to locate archived thread id {thread_id}: {err}"), - data: None, - }; - self.outgoing.send_error(request_id, error).await; - return; - } - }; - - let rollout_path_display = archived_path.display().to_string(); let fallback_provider = self.config.model_provider_id.clone(); - let state_db_ctx = get_state_db(&self.config).await; - let archived_folder = self - .config - .codex_home - .join(codex_core::ARCHIVED_SESSIONS_SUBDIR); - - let result: Result = async { - let canonical_archived_dir = tokio::fs::canonicalize(&archived_folder).await.map_err( - |err| JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!( - "failed to unarchive thread: unable to resolve archived directory: {err}" - ), - data: None, - }, - )?; - let canonical_rollout_path = tokio::fs::canonicalize(&archived_path).await; - let canonical_rollout_path = if let Ok(path) = canonical_rollout_path - && path.starts_with(&canonical_archived_dir) - { - path - } else { - return Err(JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!( - "rollout path `{rollout_path_display}` must be in archived directory" - ), - data: None, - }); - }; - - let required_suffix = format!("{thread_id}.jsonl"); - let Some(file_name) = canonical_rollout_path.file_name().map(OsStr::to_owned) else { - return Err(JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!("rollout path `{rollout_path_display}` missing file name"), - data: None, - }); - }; - if !file_name - .to_string_lossy() - .ends_with(required_suffix.as_str()) - { - return Err(JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!( - "rollout path `{rollout_path_display}` does not match thread id {thread_id}" - ), - data: None, - }); - } - - let Some((year, month, day)) = rollout_date_parts(&file_name) else { - return Err(JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!( - "rollout path `{rollout_path_display}` missing filename timestamp" - ), - data: None, - }); - }; - - let sessions_folder = self.config.codex_home.join(codex_core::SESSIONS_SUBDIR); - let dest_dir = sessions_folder.join(year).join(month).join(day); - let restored_path = dest_dir.join(&file_name); - tokio::fs::create_dir_all(&dest_dir) - .await - .map_err(|err| JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!("failed to unarchive thread: {err}"), - data: None, - })?; - tokio::fs::rename(&canonical_rollout_path, &restored_path) - .await - .map_err(|err| JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!("failed to unarchive thread: {err}"), - data: None, - })?; - tokio::task::spawn_blocking({ - let restored_path = restored_path.clone(); - move || -> std::io::Result<()> { - let times = FileTimes::new().set_modified(SystemTime::now()); - OpenOptions::new() - .append(true) - .open(&restored_path)? - .set_times(times)?; - Ok(()) - } - }) + let result = self + .thread_store + .unarchive_thread(StoreArchiveThreadParams { thread_id }) .await - .map_err(|err| JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!("failed to update unarchived thread timestamp: {err}"), - data: None, - })? - .map_err(|err| JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!("failed to update unarchived thread timestamp: {err}"), - data: None, - })?; - if let Some(ctx) = state_db_ctx { - let _ = ctx - .mark_unarchived(thread_id, restored_path.as_path()) - .await; - } - let summary = - read_summary_from_rollout(restored_path.as_path(), fallback_provider.as_str()) - .await - .map_err(|err| JSONRPCErrorError { + .map_err(|err| thread_store_archive_error("unarchive", err)) + .and_then(|stored_thread| { + summary_from_stored_thread(stored_thread, fallback_provider.as_str()) + .map(|summary| summary_to_thread(summary, &self.config.cwd)) + .ok_or_else(|| JSONRPCErrorError { code: INTERNAL_ERROR_CODE, - message: format!("failed to read unarchived thread: {err}"), + message: format!("failed to read unarchived thread {thread_id}"), data: None, - })?; - Ok(summary_to_thread(summary, &self.config.cwd)) - } - .await; + }) + }); match result { Ok(mut thread) => { @@ -3734,15 +3686,15 @@ impl CodexMessageProcessor { .map(|value| value as usize) .unwrap_or(THREAD_LIST_DEFAULT_LIMIT) .clamp(1, THREAD_LIST_MAX_LIMIT); - let core_sort_key = match sort_key.unwrap_or(ThreadSortKey::CreatedAt) { - ThreadSortKey::CreatedAt => CoreThreadSortKey::CreatedAt, - ThreadSortKey::UpdatedAt => CoreThreadSortKey::UpdatedAt, + let store_sort_key = match sort_key.unwrap_or(ThreadSortKey::CreatedAt) { + ThreadSortKey::CreatedAt => StoreThreadSortKey::CreatedAt, + ThreadSortKey::UpdatedAt => StoreThreadSortKey::UpdatedAt, }; let (summaries, next_cursor) = match self .list_threads_common( requested_page_size, cursor, - core_sort_key, + store_sort_key, ThreadListFilters { model_providers, source_kinds, @@ -4986,7 +4938,7 @@ impl CodexMessageProcessor { &self, requested_page_size: usize, cursor: Option, - sort_key: CoreThreadSortKey, + sort_key: StoreThreadSortKey, filters: ThreadListFilters, ) -> Result<(Vec, Option), JSONRPCErrorError> { let ThreadListFilters { @@ -4996,16 +4948,7 @@ impl CodexMessageProcessor { cwd, search_term, } = filters; - let mut cursor_obj: Option = match cursor.as_ref() { - Some(cursor_str) => { - Some(parse_cursor(cursor_str).ok_or_else(|| JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!("invalid cursor: {cursor_str}"), - data: None, - })?) - } - None => None, - }; + let mut cursor_obj = cursor; let mut last_cursor = cursor_obj.clone(); let mut remaining = requested_page_size; let mut items = Vec::with_capacity(requested_page_size); @@ -5024,54 +4967,26 @@ impl CodexMessageProcessor { let fallback_provider = self.config.model_provider_id.clone(); let (allowed_sources_vec, source_kind_filter) = compute_source_filters(source_kinds); let allowed_sources = allowed_sources_vec.as_slice(); - let state_db_ctx = get_state_db(&self.config).await; while remaining > 0 { let page_size = remaining.min(THREAD_LIST_MAX_LIMIT); - let page = if archived { - RolloutRecorder::list_archived_threads( - &self.config, + let page = self + .thread_store + .list_threads(StoreListThreadsParams { page_size, - cursor_obj.as_ref(), + cursor: cursor_obj.clone(), sort_key, - allowed_sources, - model_provider_filter.as_deref(), - fallback_provider.as_str(), - search_term.as_deref(), - ) + allowed_sources: allowed_sources.to_vec(), + model_providers: model_provider_filter.clone(), + archived, + search_term: search_term.clone(), + }) .await - .map_err(|err| JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!("failed to list threads: {err}"), - data: None, - })? - } else { - RolloutRecorder::list_threads( - &self.config, - page_size, - cursor_obj.as_ref(), - sort_key, - allowed_sources, - model_provider_filter.as_deref(), - fallback_provider.as_str(), - search_term.as_deref(), - ) - .await - .map_err(|err| JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!("failed to list threads: {err}"), - data: None, - })? - }; + .map_err(thread_store_list_error)?; let mut filtered = Vec::with_capacity(page.items.len()); for it in page.items { - let Some(summary) = summary_from_thread_list_item( - it, - fallback_provider.as_str(), - state_db_ctx.as_ref(), - ) - .await + let Some(summary) = summary_from_stored_thread(it, fallback_provider.as_str()) else { continue; }; @@ -5091,12 +5006,8 @@ impl CodexMessageProcessor { items.extend(filtered); remaining = requested_page_size.saturating_sub(items.len()); - // Encode RolloutCursor into the JSON-RPC string form returned to clients. let next_cursor_value = page.next_cursor.clone(); - next_cursor = next_cursor_value - .as_ref() - .and_then(|cursor| serde_json::to_value(cursor).ok()) - .and_then(|value| value.as_str().map(str::to_owned)); + next_cursor = next_cursor_value.clone(); if remaining == 0 { break; } @@ -5897,76 +5808,10 @@ impl CodexMessageProcessor { .await; } - async fn archive_thread_common( - &self, - thread_id: ThreadId, - rollout_path: &Path, - ) -> Result<(), JSONRPCErrorError> { - // Verify rollout_path is under sessions dir. - let rollout_folder = self.config.codex_home.join(codex_core::SESSIONS_SUBDIR); - - let canonical_sessions_dir = match tokio::fs::canonicalize(&rollout_folder).await { - Ok(path) => path, - Err(err) => { - return Err(JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!( - "failed to archive thread: unable to resolve sessions directory: {err}" - ), - data: None, - }); - } - }; - let canonical_rollout_path = tokio::fs::canonicalize(rollout_path).await; - let canonical_rollout_path = if let Ok(path) = canonical_rollout_path - && path.starts_with(&canonical_sessions_dir) - { - path - } else { - return Err(JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!( - "rollout path `{}` must be in sessions directory", - rollout_path.display() - ), - data: None, - }); - }; - - // Verify file name matches thread id. - let required_suffix = format!("{thread_id}.jsonl"); - let Some(file_name) = canonical_rollout_path.file_name().map(OsStr::to_owned) else { - return Err(JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!( - "rollout path `{}` missing file name", - rollout_path.display() - ), - data: None, - }); - }; - if !file_name - .to_string_lossy() - .ends_with(required_suffix.as_str()) - { - return Err(JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!( - "rollout path `{}` does not match thread id {thread_id}", - rollout_path.display() - ), - data: None, - }); - } - - let mut state_db_ctx = None; - + async fn prepare_thread_for_archive(&self, thread_id: ThreadId) { // If the thread is active, request shutdown and wait briefly. let removed_conversation = self.thread_manager.remove_thread(&thread_id).await; if let Some(conversation) = removed_conversation { - if let Some(ctx) = conversation.state_db() { - state_db_ctx = Some(ctx); - } info!("thread {thread_id} was active; shutting down"); match Self::wait_for_thread_shutdown(&conversation).await { ThreadShutdownResult::Complete => {} @@ -5981,34 +5826,6 @@ impl CodexMessageProcessor { } } self.finalize_thread_teardown(thread_id).await; - - if state_db_ctx.is_none() { - state_db_ctx = get_state_db(&self.config).await; - } - - // Move the rollout file to archived. - let result: std::io::Result<()> = async move { - let archive_folder = self - .config - .codex_home - .join(codex_core::ARCHIVED_SESSIONS_SUBDIR); - tokio::fs::create_dir_all(&archive_folder).await?; - let archived_path = archive_folder.join(&file_name); - tokio::fs::rename(&canonical_rollout_path, &archived_path).await?; - if let Some(ctx) = state_db_ctx { - let _ = ctx - .mark_archived(thread_id, archived_path.as_path(), Utc::now()) - .await; - } - Ok(()) - } - .await; - - result.map_err(|err| JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!("failed to archive thread: {err}"), - data: None, - }) } async fn apps_list(&self, request_id: ConnectionRequestId, params: AppsListParams) { @@ -9336,67 +9153,67 @@ fn set_thread_name_from_title(thread: &mut Thread, title: String) { thread.name = Some(title); } -async fn summary_from_thread_list_item( - it: codex_core::ThreadItem, - fallback_provider: &str, - state_db_ctx: Option<&StateDbHandle>, -) -> Option { - if let Some(thread_id) = it.thread_id { - let timestamp = it.created_at.clone(); - let updated_at = it.updated_at.clone().or_else(|| timestamp.clone()); - let model_provider = it - .model_provider - .clone() - .unwrap_or_else(|| fallback_provider.to_string()); - let cwd = it.cwd?; - let cli_version = it.cli_version.unwrap_or_default(); - let source = with_thread_spawn_agent_metadata( - it.source - .unwrap_or(codex_protocol::protocol::SessionSource::Unknown), - it.agent_nickname.clone(), - it.agent_role.clone(), - ); - return Some(ConversationSummary { - conversation_id: thread_id, - path: it.path, - preview: it.first_user_message.unwrap_or_default(), - timestamp, - updated_at, - model_provider, - cwd, - cli_version, - source, - git_info: if it.git_sha.is_none() - && it.git_branch.is_none() - && it.git_origin_url.is_none() - { - None - } else { - Some(ConversationGitInfo { - sha: it.git_sha, - branch: it.git_branch, - origin_url: it.git_origin_url, - }) - }, - }); +fn thread_store_list_error(err: ThreadStoreError) -> JSONRPCErrorError { + match err { + ThreadStoreError::InvalidRequest { message } => JSONRPCErrorError { + code: INVALID_REQUEST_ERROR_CODE, + message, + data: None, + }, + err => JSONRPCErrorError { + code: INTERNAL_ERROR_CODE, + message: format!("failed to list threads: {err}"), + data: None, + }, } - if let Some(thread_id) = thread_id_from_rollout_path(it.path.as_path()) { - return read_summary_from_state_db_context_by_thread_id(state_db_ctx, thread_id).await; - } - None } -fn thread_id_from_rollout_path(path: &Path) -> Option { - let file_name = path.file_name()?.to_str()?; - let stem = file_name.strip_suffix(".jsonl")?; - if stem.len() < 37 { - return None; +fn thread_store_archive_error(operation: &str, err: ThreadStoreError) -> JSONRPCErrorError { + match err { + ThreadStoreError::InvalidRequest { message } => JSONRPCErrorError { + code: INVALID_REQUEST_ERROR_CODE, + message, + data: None, + }, + err => JSONRPCErrorError { + code: INTERNAL_ERROR_CODE, + message: format!("failed to {operation} thread: {err}"), + data: None, + }, } - let uuid_start = stem.len().saturating_sub(36); - if !stem[..uuid_start].ends_with('-') { - return None; - } - ThreadId::from_string(&stem[uuid_start..]).ok() +} + +fn summary_from_stored_thread( + thread: StoredThread, + fallback_provider: &str, +) -> Option { + let path = thread.rollout_path?; + let source = with_thread_spawn_agent_metadata( + thread.source, + thread.agent_nickname.clone(), + thread.agent_role.clone(), + ); + let git_info = thread.git_info.map(|git| ConversationGitInfo { + sha: git.commit_hash.map(|sha| sha.0), + branch: git.branch, + origin_url: git.repository_url, + }); + Some(ConversationSummary { + conversation_id: thread.thread_id, + path, + preview: thread.first_user_message.unwrap_or(thread.preview), + timestamp: Some(thread.created_at.to_rfc3339_opts(SecondsFormat::Secs, true)), + updated_at: Some(thread.updated_at.to_rfc3339_opts(SecondsFormat::Secs, true)), + model_provider: if thread.model_provider.is_empty() { + fallback_provider.to_string() + } else { + thread.model_provider + }, + cwd: thread.cwd, + cli_version: thread.cli_version, + source, + git_info, + }) } #[allow(clippy::too_many_arguments)] diff --git a/codex-rs/app-server/src/in_process.rs b/codex-rs/app-server/src/in_process.rs index 9ac0e6cf42..84dfd282cd 100644 --- a/codex-rs/app-server/src/in_process.rs +++ b/codex-rs/app-server/src/in_process.rs @@ -82,6 +82,7 @@ use codex_exec_server::EnvironmentManager; use codex_feedback::CodexFeedback; use codex_login::AuthManager; use codex_protocol::protocol::SessionSource; +pub use codex_state::log_db::LogDbLayer; use tokio::sync::mpsc; use tokio::sync::oneshot; use tokio::time::timeout; @@ -117,6 +118,8 @@ pub struct InProcessStartArgs { pub cloud_requirements: CloudRequirementsLoader, /// Feedback sink used by app-server/core telemetry and logs. pub feedback: CodexFeedback, + /// SQLite tracing layer used to flush recently emitted logs before feedback upload. + pub log_db: Option, /// Environment manager used by core execution and filesystem operations. pub environment_manager: Arc, /// Startup warnings emitted after initialize succeeds. @@ -395,7 +398,7 @@ fn start_uninitialized(args: InProcessStartArgs) -> InProcessClientHandle { loader_overrides: args.loader_overrides, cloud_requirements: args.cloud_requirements, feedback: args.feedback, - log_db: None, + log_db: args.log_db, config_warnings: args.config_warnings, session_source: args.session_source, auth_manager, @@ -728,6 +731,7 @@ mod tests { loader_overrides: LoaderOverrides::default(), cloud_requirements: CloudRequirementsLoader::default(), feedback: CodexFeedback::new(), + log_db: None, environment_manager: Arc::new(EnvironmentManager::new(/*exec_server_url*/ None)), config_warnings: Vec::new(), session_source, diff --git a/codex-rs/app-server/src/transport/auth.rs b/codex-rs/app-server/src/transport/auth.rs index a67c692b7d..45f44a36c9 100644 --- a/codex-rs/app-server/src/transport/auth.rs +++ b/codex-rs/app-server/src/transport/auth.rs @@ -34,6 +34,10 @@ pub struct AppServerWebsocketAuthArgs { #[arg(long = "ws-token-file", value_name = "PATH")] pub ws_token_file: Option, + /// Hex-encoded SHA-256 digest of the capability token. + #[arg(long = "ws-token-sha256", value_name = "HEX")] + pub ws_token_sha256: Option, + /// Absolute path to the shared secret file for signed JWT bearer tokens. #[arg(long = "ws-shared-secret-file", value_name = "PATH")] pub ws_shared_secret_file: Option, @@ -65,7 +69,7 @@ pub struct AppServerWebsocketAuthSettings { #[derive(Debug, Clone, PartialEq, Eq)] pub enum AppServerWebsocketAuthConfig { CapabilityToken { - token_file: AbsolutePathBuf, + source: AppServerWebsocketCapabilityTokenSource, }, SignedBearerToken { shared_secret_file: AbsolutePathBuf, @@ -75,6 +79,12 @@ pub enum AppServerWebsocketAuthConfig { }, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AppServerWebsocketCapabilityTokenSource { + TokenFile { token_file: AbsolutePathBuf }, + TokenSha256 { token_sha256: [u8; 32] }, +} + #[derive(Clone, Debug, Default)] pub(crate) struct WebsocketAuthPolicy { pub(crate) mode: Option, @@ -144,17 +154,34 @@ impl AppServerWebsocketAuthArgs { "`--ws-shared-secret-file`, `--ws-issuer`, `--ws-audience`, and `--ws-max-clock-skew-seconds` require `--ws-auth signed-bearer-token`" ); } - let token_file = self.ws_token_file.context( - "`--ws-token-file` is required when `--ws-auth capability-token` is set", - )?; - Some(AppServerWebsocketAuthConfig::CapabilityToken { - token_file: absolute_path_arg("--ws-token-file", token_file)?, - }) + let source = match (self.ws_token_file, self.ws_token_sha256) { + (Some(_), Some(_)) => { + anyhow::bail!( + "`--ws-token-file` and `--ws-token-sha256` are mutually exclusive" + ); + } + (Some(token_file), None) => { + AppServerWebsocketCapabilityTokenSource::TokenFile { + token_file: absolute_path_arg("--ws-token-file", token_file)?, + } + } + (None, Some(token_sha256)) => { + AppServerWebsocketCapabilityTokenSource::TokenSha256 { + token_sha256: sha256_digest_arg("--ws-token-sha256", &token_sha256)?, + } + } + (None, None) => { + anyhow::bail!( + "`--ws-token-file` or `--ws-token-sha256` is required when `--ws-auth capability-token` is set" + ); + } + }; + Some(AppServerWebsocketAuthConfig::CapabilityToken { source }) } Some(WebsocketAuthCliMode::SignedBearerToken) => { - if self.ws_token_file.is_some() { + if self.ws_token_file.is_some() || self.ws_token_sha256.is_some() { anyhow::bail!( - "`--ws-token-file` requires `--ws-auth capability-token`, not `signed-bearer-token`" + "`--ws-token-file` and `--ws-token-sha256` require `--ws-auth capability-token`, not `signed-bearer-token`" ); } let shared_secret_file = self.ws_shared_secret_file.context( @@ -174,6 +201,7 @@ impl AppServerWebsocketAuthArgs { } None => { if self.ws_token_file.is_some() + || self.ws_token_sha256.is_some() || self.ws_shared_secret_file.is_some() || self.ws_issuer.is_some() || self.ws_audience.is_some() @@ -195,12 +223,19 @@ pub(crate) fn policy_from_settings( settings: &AppServerWebsocketAuthSettings, ) -> io::Result { let mode = match settings.config.as_ref() { - Some(AppServerWebsocketAuthConfig::CapabilityToken { token_file }) => { - let token = read_trimmed_secret(token_file.as_ref())?; - Some(WebsocketAuthMode::CapabilityToken { - token_sha256: sha256_digest(token.as_bytes()), - }) - } + Some(AppServerWebsocketAuthConfig::CapabilityToken { source }) => match source { + AppServerWebsocketCapabilityTokenSource::TokenFile { token_file } => { + let token = read_trimmed_secret(token_file.as_ref())?; + Some(WebsocketAuthMode::CapabilityToken { + token_sha256: sha256_digest(token.as_bytes()), + }) + } + AppServerWebsocketCapabilityTokenSource::TokenSha256 { token_sha256 } => { + Some(WebsocketAuthMode::CapabilityToken { + token_sha256: *token_sha256, + }) + } + }, Some(AppServerWebsocketAuthConfig::SignedBearerToken { shared_secret_file, issuer, @@ -387,6 +422,30 @@ fn absolute_path_arg(flag_name: &str, path: PathBuf) -> anyhow::Result anyhow::Result<[u8; 32]> { + let trimmed = value.trim(); + if trimmed.len() != 64 { + anyhow::bail!("{flag_name} must be a 64-character hex SHA-256 digest"); + } + + let mut digest = [0u8; 32]; + for (index, pair) in trimmed.as_bytes().chunks_exact(2).enumerate() { + let high = hex_nibble(flag_name, pair[0])?; + let low = hex_nibble(flag_name, pair[1])?; + digest[index] = (high << 4) | low; + } + Ok(digest) +} + +fn hex_nibble(flag_name: &str, byte: u8) -> anyhow::Result { + match byte { + b'0'..=b'9' => Ok(byte - b'0'), + b'a'..=b'f' => Ok(byte - b'a' + 10), + b'A'..=b'F' => Ok(byte - b'A' + 10), + _ => anyhow::bail!("{flag_name} must be a 64-character hex SHA-256 digest"), + } +} + fn sha256_digest(input: &[u8]) -> [u8; 32] { let mut digest = [0u8; 32]; digest.copy_from_slice(&Sha256::digest(input)); @@ -403,6 +462,7 @@ fn unauthorized(message: &'static str) -> WebsocketAuthError { #[cfg(test)] mod tests { use super::*; + use axum::http::HeaderValue; use base64::Engine; use base64::engine::general_purpose::URL_SAFE_NO_PAD; use hmac::Hmac; @@ -443,19 +503,98 @@ mod tests { } #[test] - fn capability_token_args_require_token_file() { + fn capability_token_args_require_token_file_or_hash() { let err = AppServerWebsocketAuthArgs { ws_auth: Some(WebsocketAuthCliMode::CapabilityToken), ..Default::default() } .try_into_settings() - .expect_err("capability-token mode should require a token file"); + .expect_err("capability-token mode should require a token source"); assert!( - err.to_string().contains("--ws-token-file"), + err.to_string().contains("--ws-token-file") + && err.to_string().contains("--ws-token-sha256"), "unexpected error: {err}" ); } + #[test] + fn capability_token_args_accept_token_hash() { + let settings = AppServerWebsocketAuthArgs { + ws_auth: Some(WebsocketAuthCliMode::CapabilityToken), + ws_token_sha256: Some("ab".repeat(32)), + ..Default::default() + } + .try_into_settings() + .expect("capability-token hash args should parse"); + + assert_eq!( + settings, + AppServerWebsocketAuthSettings { + config: Some(AppServerWebsocketAuthConfig::CapabilityToken { + source: AppServerWebsocketCapabilityTokenSource::TokenSha256 { + token_sha256: [0xab; 32], + }, + }), + } + ); + } + + #[test] + fn capability_token_args_reject_multiple_token_sources() { + let err = AppServerWebsocketAuthArgs { + ws_auth: Some(WebsocketAuthCliMode::CapabilityToken), + ws_token_file: Some(PathBuf::from("/tmp/token")), + ws_token_sha256: Some("ab".repeat(32)), + ..Default::default() + } + .try_into_settings() + .expect_err("capability-token mode should reject multiple token sources"); + assert!( + err.to_string().contains("mutually exclusive"), + "unexpected error: {err}" + ); + } + + #[test] + fn capability_token_args_reject_malformed_token_hash() { + let err = AppServerWebsocketAuthArgs { + ws_auth: Some(WebsocketAuthCliMode::CapabilityToken), + ws_token_sha256: Some("not-a-sha256".to_string()), + ..Default::default() + } + .try_into_settings() + .expect_err("capability-token mode should reject malformed token hashes"); + assert!( + err.to_string().contains("64-character hex"), + "unexpected error: {err}" + ); + } + + #[test] + fn capability_token_hash_policy_authorizes_matching_bearer_token() { + let settings = AppServerWebsocketAuthSettings { + config: Some(AppServerWebsocketAuthConfig::CapabilityToken { + source: AppServerWebsocketCapabilityTokenSource::TokenSha256 { + token_sha256: sha256_digest(b"super-secret-token"), + }, + }), + }; + let policy = policy_from_settings(&settings).expect("hash policy should build"); + let mut headers = HeaderMap::new(); + headers.insert( + AUTHORIZATION, + HeaderValue::from_static("Bearer super-secret-token"), + ); + authorize_upgrade(&headers, &policy).expect("matching token should authorize"); + + headers.insert( + AUTHORIZATION, + HeaderValue::from_static("Bearer wrong-token"), + ); + let err = authorize_upgrade(&headers, &policy).expect_err("wrong token should fail"); + assert_eq!(err.status_code(), StatusCode::UNAUTHORIZED); + } + #[test] fn signed_bearer_args_require_mode_when_mode_specific_flags_are_set() { let err = AppServerWebsocketAuthArgs { diff --git a/codex-rs/app-server/src/transport/remote_control/protocol.rs b/codex-rs/app-server/src/transport/remote_control/protocol.rs index 857855f2a0..dcb79e5c35 100644 --- a/codex-rs/app-server/src/transport/remote_control/protocol.rs +++ b/codex-rs/app-server/src/transport/remote_control/protocol.rs @@ -48,9 +48,8 @@ pub enum ClientEvent { message: JSONRPCMessage, }, /// Backend-generated acknowledgement for all server envelopes addressed to - /// `client_id` whose envelope `seq_id` is less than or equal to this ack's - /// `seq_id`. This cursor is client-scoped, not stream-scoped, so receivers - /// must not use `stream_id` to partition acks. + /// `client_id` and `stream_id` whose envelope `seq_id` is less than or equal + /// to this ack's `seq_id`. This cursor is stream-scoped. Ack, Ping, ClientClosed, @@ -65,7 +64,7 @@ pub(crate) struct ClientEnvelope { pub(crate) client_id: ClientId, #[serde(rename = "stream_id", skip_serializing_if = "Option::is_none")] pub(crate) stream_id: Option, - /// For `Ack`, this is the backend-generated per-client cursor over + /// For `Ack`, this is the backend-generated per-stream cursor over /// `ServerEnvelope.seq_id`. #[serde(rename = "seq_id", skip_serializing_if = "Option::is_none")] pub(crate) seq_id: Option, diff --git a/codex-rs/app-server/src/transport/remote_control/tests.rs b/codex-rs/app-server/src/transport/remote_control/tests.rs index 21808a3a18..2dd687a3d1 100644 --- a/codex-rs/app-server/src/transport/remote_control/tests.rs +++ b/codex-rs/app-server/src/transport/remote_control/tests.rs @@ -5,6 +5,7 @@ use super::enroll::update_persisted_remote_control_enrollment; use super::protocol::ClientEnvelope; use super::protocol::ClientEvent; use super::protocol::ClientId; +use super::protocol::StreamId; use super::protocol::normalize_remote_control_url; use super::websocket::REMOTE_CONTROL_PROTOCOL_VERSION; use super::*; @@ -96,6 +97,7 @@ fn remote_control_auth_dot_json(account_id: Option<&str>) -> AuthDotJson { account_id: account_id.map(str::to_string), }), last_refresh: Some(chrono::Utc::now()), + agent_identity: None, } } @@ -162,7 +164,7 @@ async fn remote_control_transport_manages_virtual_clients_and_routes_messages() json!({ "type": "pong", "client_id": "client-1", - "seq_id": 0, + "seq_id": 1, "status": "unknown", }) ); @@ -368,7 +370,7 @@ async fn remote_control_transport_manages_virtual_clients_and_routes_messages() json!({ "type": "pong", "client_id": "client-1", - "seq_id": 3, + "seq_id": 1, "status": "unknown", }) ); @@ -617,12 +619,13 @@ async fn remote_control_transport_clears_outgoing_buffer_when_backend_acks() { )) .await .expect("remote writer should accept outgoing message"); + let (server_event, stream_id) = read_server_event_with_stream_id(&mut first_websocket).await; assert_eq!( - read_server_event(&mut first_websocket).await, + server_event, json!({ "type": "server_message", "client_id": "client-1", - "seq_id": 0, + "seq_id": 1, "message": { "method": "configWarning", "params": { @@ -638,8 +641,8 @@ async fn remote_control_transport_clears_outgoing_buffer_when_backend_acks() { ClientEnvelope { event: ClientEvent::Ack, client_id: client_id.clone(), - stream_id: None, - seq_id: Some(0), + stream_id: Some(stream_id), + seq_id: Some(1), cursor: None, }, ) @@ -853,7 +856,7 @@ async fn remote_control_http_mode_enrolls_before_connecting() { json!({ "type": "server_message", "client_id": backend_client_id.0.clone(), - "seq_id": 0, + "seq_id": 1, "message": { "id": 11, "result": { @@ -881,7 +884,7 @@ async fn remote_control_http_mode_enrolls_before_connecting() { json!({ "type": "server_message", "client_id": backend_client_id.0.clone(), - "seq_id": 1, + "seq_id": 2, "message": { "method": "configWarning", "params": { @@ -1355,6 +1358,12 @@ async fn send_client_event( } async fn read_server_event(websocket: &mut WebSocketStream) -> serde_json::Value { + read_server_event_with_stream_id(websocket).await.0 +} + +async fn read_server_event_with_stream_id( + websocket: &mut WebSocketStream, +) -> (serde_json::Value, StreamId) { loop { let frame = timeout(Duration::from_secs(5), websocket.next()) .await @@ -1365,13 +1374,15 @@ async fn read_server_event(websocket: &mut WebSocketStream) -> serde_ tungstenite::Message::Text(text) => { let mut event: serde_json::Value = serde_json::from_str(text.as_ref()).expect("server event should deserialize"); - if let Some(stream_id) = event + let stream_id = event .as_object_mut() .and_then(|event| event.remove("stream_id")) - { - assert!(stream_id.is_string(), "stream_id should be a string"); - } - return event; + .expect("stream_id should be present"); + let stream_id = stream_id + .as_str() + .expect("stream_id should be a string") + .to_string(); + return (event, StreamId(stream_id)); } tungstenite::Message::Ping(payload) => { websocket diff --git a/codex-rs/app-server/src/transport/remote_control/websocket.rs b/codex-rs/app-server/src/transport/remote_control/websocket.rs index 42924c2d34..82e5f35b33 100644 --- a/codex-rs/app-server/src/transport/remote_control/websocket.rs +++ b/codex-rs/app-server/src/transport/remote_control/websocket.rs @@ -14,6 +14,7 @@ use super::protocol::ClientEvent; use super::protocol::ClientId; use super::protocol::RemoteControlTarget; use super::protocol::ServerEnvelope; +use super::protocol::StreamId; use axum::http::HeaderValue; use base64::Engine; use codex_core::util::backoff; @@ -25,8 +26,8 @@ use futures::SinkExt; use futures::StreamExt; use futures::stream::SplitSink; use futures::stream::SplitStream; -use std::collections::BTreeMap; use std::collections::HashMap; +use std::collections::VecDeque; use std::io; use std::io::ErrorKind; use std::sync::Arc; @@ -57,10 +58,7 @@ const REMOTE_CONTROL_ACCOUNT_ID_RETRY_INTERVAL: std::time::Duration = std::time::Duration::from_secs(1); struct BoundedOutboundBuffer { - // Remote-control acks are generated by the backend at client scope, so - // retransmit retention is keyed by client_id only. stream_id stays on each - // envelope for routing, but it is not part of the ack cursor. - buffer_by_client: HashMap>, + buffer_by_stream: HashMap<(ClientId, StreamId), VecDeque>, used_tx: watch::Sender, } @@ -68,46 +66,50 @@ impl BoundedOutboundBuffer { fn new() -> (Self, watch::Receiver) { let (used_tx, used_rx) = watch::channel(0); let buffer = Self { - buffer_by_client: HashMap::new(), + buffer_by_stream: HashMap::new(), used_tx, }; (buffer, used_rx) } fn insert(&mut self, server_envelope: &ServerEnvelope) { - self.buffer_by_client - .entry(server_envelope.client_id.clone()) + self.buffer_by_stream + .entry(( + server_envelope.client_id.clone(), + server_envelope.stream_id.clone(), + )) .or_default() - .insert(server_envelope.seq_id, server_envelope.clone()); + .push_back(server_envelope.clone()); self.used_tx.send_modify(|used| *used += 1); } - fn ack(&mut self, client_id: &ClientId, acked_seq_id: u64) { - let Some(buffer) = self.buffer_by_client.get_mut(client_id) else { + fn ack(&mut self, client_id: &ClientId, stream_id: &StreamId, acked_seq_id: u64) { + let key = (client_id.clone(), stream_id.clone()); + let Some(buffer) = self.buffer_by_stream.get_mut(&key) else { return; }; - while let Some(seq_id) = buffer.first_key_value().map(|(seq_id, _)| seq_id) - && *seq_id <= acked_seq_id + while let Some(server_envelope) = buffer.front() + && server_envelope.seq_id <= acked_seq_id { - buffer.pop_first(); + buffer.pop_front(); self.used_tx.send_modify(|used| *used -= 1); } if buffer.is_empty() { - self.buffer_by_client.remove(client_id); + self.buffer_by_stream.remove(&key); } } fn server_envelopes(&self) -> impl Iterator { - self.buffer_by_client + self.buffer_by_stream .values() - .flat_map(|buffer| buffer.values()) + .flat_map(|buffer| buffer.iter()) } } struct WebsocketState { outbound_buffer: BoundedOutboundBuffer, subscribe_cursor: Option, - next_seq_id: u64, + next_seq_id_by_stream: HashMap<(ClientId, StreamId), u64>, } pub(crate) struct RemoteControlWebsocket { @@ -162,7 +164,7 @@ impl RemoteControlWebsocket { state: Arc::new(Mutex::new(WebsocketState { outbound_buffer, subscribe_cursor: None, - next_seq_id: 0, + next_seq_id_by_stream: HashMap::new(), })), server_event_rx: Arc::new(Mutex::new(server_event_rx)), used_rx, @@ -445,10 +447,16 @@ impl RemoteControlWebsocket { } } }; - let (server_envelope, write_complete_tx) = { + let (payload, write_complete_tx) = { let mut state = state.lock().await; - let seq_id = state.next_seq_id; - state.next_seq_id = state.next_seq_id.saturating_add(1); + let seq_key = ( + queued_server_envelope.client_id.clone(), + queued_server_envelope.stream_id.clone(), + ); + let seq_id = *state + .next_seq_id_by_stream + .entry(seq_key.clone()) + .or_insert(1); let server_envelope = ServerEnvelope { event: queued_server_envelope.event, @@ -456,17 +464,19 @@ impl RemoteControlWebsocket { seq_id, stream_id: queued_server_envelope.stream_id, }; + let payload = match serde_json::to_string(&server_envelope) { + Ok(payload) => payload, + Err(err) => { + error!("failed to serialize remote-control server event: {err}"); + continue; + } + }; + state + .next_seq_id_by_stream + .insert(seq_key, seq_id.saturating_add(1)); state.outbound_buffer.insert(&server_envelope); - (server_envelope, queued_server_envelope.write_complete_tx) - }; - - let payload = match serde_json::to_string(&server_envelope) { - Ok(payload) => payload, - Err(err) => { - error!("failed to serialize remote-control server event: {err}"); - continue; - } + (payload, queued_server_envelope.write_complete_tx) }; tokio::select! { @@ -590,10 +600,13 @@ impl RemoteControlWebsocket { } if let ClientEvent::Ack = &client_envelope.event && let Some(acked_seq_id) = client_envelope.seq_id + && let Some(stream_id) = client_envelope.stream_id.as_ref() { - websocket_state - .outbound_buffer - .ack(&client_envelope.client_id, acked_seq_id); + websocket_state.outbound_buffer.ack( + &client_envelope.client_id, + stream_id, + acked_seq_id, + ); } drop(websocket_state); @@ -971,6 +984,7 @@ mod tests { account_id: Some("account_id".to_string()), }), last_refresh: Some(Utc::now()), + agent_identity: None, } } @@ -1219,7 +1233,7 @@ mod tests { let state = Arc::new(Mutex::new(WebsocketState { outbound_buffer, subscribe_cursor: None, - next_seq_id: 0, + next_seq_id_by_stream: HashMap::new(), })); let (_server_event_tx, server_event_rx) = mpsc::channel(super::super::CHANNEL_CAPACITY); let server_event_rx = Arc::new(Mutex::new(server_event_rx)); @@ -1247,6 +1261,83 @@ mod tests { .expect("writer should stop cleanly"); } + #[tokio::test] + async fn run_server_writer_inner_assigns_contiguous_seq_ids_per_stream() { + let (client_stream, mut server_stream) = connected_websocket_pair().await; + let (websocket_writer, _websocket_reader) = client_stream.split(); + let (outbound_buffer, used_rx) = BoundedOutboundBuffer::new(); + let state = Arc::new(Mutex::new(WebsocketState { + outbound_buffer, + subscribe_cursor: None, + next_seq_id_by_stream: HashMap::new(), + })); + let (server_event_tx, server_event_rx) = mpsc::channel(super::super::CHANNEL_CAPACITY); + let server_event_rx = Arc::new(Mutex::new(server_event_rx)); + let shutdown_token = CancellationToken::new(); + let writer_task = tokio::spawn(RemoteControlWebsocket::run_server_writer_inner( + state, + server_event_rx, + used_rx, + websocket_writer, + Duration::from_secs(60), + shutdown_token.clone(), + )); + + let client_id = ClientId("client-1".to_string()); + let first_stream = StreamId("stream-1".to_string()); + let second_stream = StreamId("stream-2".to_string()); + for stream_id in [&first_stream, &second_stream, &first_stream] { + server_event_tx + .send(super::super::QueuedServerEnvelope { + event: ServerEvent::Pong { + status: crate::transport::remote_control::protocol::PongStatus::Active, + }, + client_id: client_id.clone(), + stream_id: stream_id.clone(), + write_complete_tx: None, + }) + .await + .expect("server event should queue"); + } + + assert_eq!( + read_server_text_event(&mut server_stream).await, + serde_json::json!({ + "type": "pong", + "client_id": "client-1", + "stream_id": "stream-1", + "seq_id": 1, + "status": "active", + }) + ); + assert_eq!( + read_server_text_event(&mut server_stream).await, + serde_json::json!({ + "type": "pong", + "client_id": "client-1", + "stream_id": "stream-2", + "seq_id": 1, + "status": "active", + }) + ); + assert_eq!( + read_server_text_event(&mut server_stream).await, + serde_json::json!({ + "type": "pong", + "client_id": "client-1", + "stream_id": "stream-1", + "seq_id": 2, + "status": "active", + }) + ); + + shutdown_token.cancel(); + writer_task + .await + .expect("writer task should join") + .expect("writer should stop cleanly"); + } + #[tokio::test] async fn run_websocket_reader_inner_times_out_without_pong_frames() { let (client_stream, _server_stream) = connected_websocket_pair().await; @@ -1255,7 +1346,7 @@ mod tests { let state = Arc::new(Mutex::new(WebsocketState { outbound_buffer, subscribe_cursor: None, - next_seq_id: 0, + next_seq_id_by_stream: HashMap::new(), })); let (server_event_tx, _server_event_rx) = mpsc::channel(super::super::CHANNEL_CAPACITY); let (transport_event_tx, _transport_event_rx) = @@ -1286,70 +1377,32 @@ mod tests { } #[test] - fn outbound_buffer_acks_by_client_id_across_stream_ids() { + fn outbound_buffer_acks_by_stream_id() { let (mut outbound_buffer, used_rx) = BoundedOutboundBuffer::new(); let client_1 = ClientId("client-1".to_string()); let client_2 = ClientId("client-2".to_string()); + let stream_1 = StreamId("stream-1".to_string()); outbound_buffer.insert(&server_envelope( &client_1, "stream-1", - /*seq_id*/ 0, + /*seq_id*/ 1, "first-client-old-stream", )); outbound_buffer.insert(&server_envelope( &client_2, "stream-1", - /*seq_id*/ 1, + /*seq_id*/ 2, "second-client", )); outbound_buffer.insert(&server_envelope( &client_1, "stream-2", - /*seq_id*/ 2, + /*seq_id*/ 3, "first-client-new-stream", )); - outbound_buffer.ack(&client_1, /*acked_seq_id*/ 2); - - let mut retained = outbound_buffer - .server_envelopes() - .map(|server_envelope| { - ( - server_envelope.client_id.0.as_str(), - server_envelope.stream_id.0.as_str(), - server_envelope.seq_id, - ) - }) - .collect::>(); - retained.sort_unstable(); - assert_eq!(retained, vec![("client-2", "stream-1", 1)]); - assert_eq!(*used_rx.borrow(), 1); - } - - #[test] - fn outbound_buffer_retains_unacked_messages_until_ack_advances() { - let (mut outbound_buffer, used_rx) = BoundedOutboundBuffer::new(); - let client_1 = ClientId("client-1".to_string()); - let client_2 = ClientId("client-2".to_string()); - - outbound_buffer.insert(&server_envelope( - &client_1, - "stream-1", - /*seq_id*/ 0, - "first-old", - )); - outbound_buffer.insert(&server_envelope( - &client_1, - "stream-2", - /*seq_id*/ 1, - "first-new", - )); - outbound_buffer.insert(&server_envelope( - &client_2, "stream-1", /*seq_id*/ 2, "second", - )); - - outbound_buffer.ack(&client_1, /*acked_seq_id*/ 0); + outbound_buffer.ack(&client_1, &stream_1, /*acked_seq_id*/ 3); let mut retained = outbound_buffer .server_envelopes() @@ -1364,7 +1417,50 @@ mod tests { retained.sort_unstable(); assert_eq!( retained, - vec![("client-1", "stream-2", 1), ("client-2", "stream-1", 2)] + vec![("client-1", "stream-2", 3), ("client-2", "stream-1", 2)] + ); + assert_eq!(*used_rx.borrow(), 2); + } + + #[test] + fn outbound_buffer_retains_unacked_messages_until_ack_advances() { + let (mut outbound_buffer, used_rx) = BoundedOutboundBuffer::new(); + let client_1 = ClientId("client-1".to_string()); + let client_2 = ClientId("client-2".to_string()); + let stream_1 = StreamId("stream-1".to_string()); + + outbound_buffer.insert(&server_envelope( + &client_1, + "stream-1", + /*seq_id*/ 1, + "first-old", + )); + outbound_buffer.insert(&server_envelope( + &client_1, + "stream-2", + /*seq_id*/ 2, + "first-new", + )); + outbound_buffer.insert(&server_envelope( + &client_2, "stream-1", /*seq_id*/ 3, "second", + )); + + outbound_buffer.ack(&client_1, &stream_1, /*acked_seq_id*/ 1); + + let mut retained = outbound_buffer + .server_envelopes() + .map(|server_envelope| { + ( + server_envelope.client_id.0.as_str(), + server_envelope.stream_id.0.as_str(), + server_envelope.seq_id, + ) + }) + .collect::>(); + retained.sort_unstable(); + assert_eq!( + retained, + vec![("client-1", "stream-2", 2), ("client-2", "stream-1", 3)] ); assert_eq!(*used_rx.borrow(), 2); } @@ -1449,6 +1545,20 @@ mod tests { (client_stream, server_stream) } + async fn read_server_text_event( + server_stream: &mut WebSocketStream, + ) -> serde_json::Value { + let message = timeout(Duration::from_secs(5), server_stream.next()) + .await + .expect("server event should arrive in time") + .expect("server websocket should stay open") + .expect("server event should read"); + let tungstenite::Message::Text(text) = message else { + panic!("expected text event, got {message:?}"); + }; + serde_json::from_str(text.as_ref()).expect("server event should deserialize") + } + async fn respond_with_status_and_headers( mut stream: TcpStream, status: &str, diff --git a/codex-rs/app-server/tests/common/auth_fixtures.rs b/codex-rs/app-server/tests/common/auth_fixtures.rs index 99334f0770..86f0fb456d 100644 --- a/codex-rs/app-server/tests/common/auth_fixtures.rs +++ b/codex-rs/app-server/tests/common/auth_fixtures.rs @@ -163,6 +163,7 @@ pub fn write_chatgpt_auth( openai_api_key: None, tokens: Some(tokens), last_refresh, + agent_identity: None, }; save_auth(codex_home, &auth, cli_auth_credentials_store_mode).context("write auth.json") diff --git a/codex-rs/app-server/tests/suite/v2/app_list.rs b/codex-rs/app-server/tests/suite/v2/app_list.rs index 57a27961af..dbe61524f5 100644 --- a/codex-rs/app-server/tests/suite/v2/app_list.rs +++ b/codex-rs/app-server/tests/suite/v2/app_list.rs @@ -117,6 +117,7 @@ async fn list_apps_returns_empty_with_api_key_auth() -> Result<()> { openai_api_key: Some("test-api-key".to_string()), tokens: None, last_refresh: None, + agent_identity: None, }, AuthCredentialsStoreMode::File, )?; diff --git a/codex-rs/app-server/tests/suite/v2/mcp_server_elicitation.rs b/codex-rs/app-server/tests/suite/v2/mcp_server_elicitation.rs index 862bdf8487..13ebe0b99c 100644 --- a/codex-rs/app-server/tests/suite/v2/mcp_server_elicitation.rs +++ b/codex-rs/app-server/tests/suite/v2/mcp_server_elicitation.rs @@ -66,8 +66,9 @@ use tokio::time::timeout; const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); const CONNECTOR_ID: &str = "calendar"; const CONNECTOR_NAME: &str = "Calendar"; +const TOOL_NAMESPACE: &str = "mcp__codex_apps__calendar"; +const CALLABLE_TOOL_NAME: &str = "_confirm_action"; const TOOL_NAME: &str = "calendar_confirm_action"; -const QUALIFIED_TOOL_NAME: &str = "mcp__codex_apps__calendar_confirm_action"; const TOOL_CALL_ID: &str = "call-calendar-confirm"; const ELICITATION_MESSAGE: &str = "Allow this request?"; @@ -85,9 +86,10 @@ async fn mcp_server_elicitation_round_trip() -> Result<()> { ]), responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call( + responses::ev_function_call_with_namespace( TOOL_CALL_ID, - QUALIFIED_TOOL_NAME, + TOOL_NAMESPACE, + CALLABLE_TOOL_NAME, &tool_call_arguments, ), responses::ev_completed("resp-1"), diff --git a/codex-rs/app-server/tests/suite/v2/memory_reset.rs b/codex-rs/app-server/tests/suite/v2/memory_reset.rs new file mode 100644 index 0000000000..3c7ae38671 --- /dev/null +++ b/codex-rs/app-server/tests/suite/v2/memory_reset.rs @@ -0,0 +1,145 @@ +use anyhow::Result; +use app_test_support::McpProcess; +use app_test_support::to_response; +use chrono::Utc; +use codex_app_server_protocol::JSONRPCResponse; +use codex_app_server_protocol::MemoryResetResponse; +use codex_app_server_protocol::RequestId; +use codex_protocol::ThreadId; +use codex_protocol::protocol::SessionSource; +use codex_state::Stage1JobClaimOutcome; +use codex_state::StateRuntime; +use codex_state::ThreadMetadataBuilder; +use pretty_assertions::assert_eq; +use std::path::Path; +use std::sync::Arc; +use tempfile::TempDir; +use tokio::time::timeout; +use uuid::Uuid; + +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + +#[tokio::test] +async fn memory_reset_clears_memory_files_and_rows_preserves_threads() -> Result<()> { + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path())?; + let state_db = init_state_db(codex_home.path()).await?; + + let memory_root = codex_home.path().join("memories"); + tokio::fs::create_dir_all(memory_root.join("rollout_summaries")).await?; + tokio::fs::write(memory_root.join("MEMORY.md"), "stale memory\n").await?; + tokio::fs::write( + memory_root.join("rollout_summaries").join("stale.md"), + "stale rollout summary\n", + ) + .await?; + + let thread_id = seed_stage1_output(&state_db, codex_home.path()).await?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let request_id = mcp + .send_raw_request("memory/reset", /*params*/ None) + .await?; + let response: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let _: MemoryResetResponse = to_response::(response)?; + + let stage1_outputs = state_db.list_stage1_outputs_for_global(/*n*/ 10).await?; + assert_eq!(stage1_outputs, Vec::new()); + assert_eq!( + state_db.get_thread_memory_mode(thread_id).await?.as_deref(), + Some("enabled") + ); + + let mut remaining_entries = tokio::fs::read_dir(&memory_root).await?; + assert!( + remaining_entries.next_entry().await?.is_none(), + "memory root should be empty after reset" + ); + + Ok(()) +} + +async fn seed_stage1_output(state_db: &Arc, codex_home: &Path) -> Result { + let now = Utc::now(); + let thread_id = ThreadId::from_string(&Uuid::new_v4().to_string())?; + let worker_id = ThreadId::from_string(&Uuid::new_v4().to_string())?; + let mut builder = ThreadMetadataBuilder::new( + thread_id, + codex_home.join("sessions").join("test.jsonl"), + now, + SessionSource::Cli, + ); + builder.updated_at = Some(now); + builder.cwd = codex_home.to_path_buf(); + let metadata = builder.build("mock_provider"); + state_db.upsert_thread(&metadata).await?; + + let claim = state_db + .try_claim_stage1_job( + thread_id, + worker_id, + now.timestamp(), + /*lease_seconds*/ 3600, + /*max_running_jobs*/ 64, + ) + .await?; + let Stage1JobClaimOutcome::Claimed { ownership_token } = claim else { + anyhow::bail!("unexpected stage1 claim outcome: {claim:?}"); + }; + assert!( + state_db + .mark_stage1_job_succeeded( + thread_id, + ownership_token.as_str(), + now.timestamp(), + "raw memory", + "rollout summary", + /*rollout_slug*/ None, + ) + .await?, + "stage1 success should be recorded" + ); + state_db + .enqueue_global_consolidation(now.timestamp()) + .await?; + + Ok(thread_id) +} + +async fn init_state_db(codex_home: &Path) -> Result> { + let state_db = StateRuntime::init(codex_home.to_path_buf(), "mock_provider".into()).await?; + state_db + .mark_backfill_complete(/*last_watermark*/ None) + .await?; + Ok(state_db) +} + +fn create_config_toml(codex_home: &Path) -> std::io::Result<()> { + let config_toml = codex_home.join("config.toml"); + std::fs::write( + config_toml, + r#" +model = "mock-model" +approval_policy = "never" +sandbox_mode = "read-only" +model_provider = "mock_provider" +suppress_unstable_features_warning = true + +[features] +sqlite = true + +[model_providers.mock_provider] +name = "Mock provider for test" +base_url = "http://127.0.0.1:9/v1" +wire_api = "responses" +request_max_retries = 0 +stream_max_retries = 0 +"#, + ) +} diff --git a/codex-rs/app-server/tests/suite/v2/mod.rs b/codex-rs/app-server/tests/suite/v2/mod.rs index 95e07c031e..bbf04006e6 100644 --- a/codex-rs/app-server/tests/suite/v2/mod.rs +++ b/codex-rs/app-server/tests/suite/v2/mod.rs @@ -20,6 +20,7 @@ mod mcp_resource; mod mcp_server_elicitation; mod mcp_server_status; mod mcp_tool; +mod memory_reset; mod model_list; mod output_schema; mod plan_item; diff --git a/codex-rs/app-server/tests/suite/v2/thread_list.rs b/codex-rs/app-server/tests/suite/v2/thread_list.rs index cf1ac4ff11..8fd4e47303 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_list.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_list.rs @@ -563,15 +563,34 @@ sqlite = true /*git_info*/ None, )?; - // `thread/list` only applies `search_term` on the sqlite path. In this test we - // create rollouts manually, so we must also create the sqlite DB and mark backfill - // complete; otherwise app-server will permanently use filesystem fallback. + // `thread/list` applies `search_term` on the sqlite fast path. This test creates + // rollouts manually, so mark the DB backfill complete and then run an unsearched + // list large enough to repair every rollout the searched list should find. let state_db = codex_state::StateRuntime::init(codex_home.path().to_path_buf(), "mock_provider".into()) .await?; state_db .mark_backfill_complete(/*last_watermark*/ None) .await?; + let rollout_config = codex_rollout::RolloutConfig { + codex_home: codex_home.path().to_path_buf(), + sqlite_home: codex_home.path().to_path_buf(), + cwd: codex_home.path().to_path_buf(), + model_provider_id: "mock_provider".to_string(), + generate_memories: false, + }; + let repaired_page = codex_core::RolloutRecorder::list_threads( + &rollout_config, + /*page_size*/ 10, + /*cursor*/ None, + codex_core::ThreadSortKey::CreatedAt, + &[], + /*model_providers*/ None, + "mock_provider", + /*search_term*/ None, + ) + .await?; + assert_eq!(repaired_page.items.len(), 3); let mut mcp = init_mcp(codex_home.path()).await?; let request_id = mcp diff --git a/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs b/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs index b553137752..f8eaf799da 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs @@ -127,10 +127,17 @@ async fn turn_interrupt_aborts_running_turn() -> Result<()> { #[tokio::test] async fn turn_interrupt_resolves_pending_command_approval_request() -> Result<()> { + #[cfg(target_os = "windows")] + let shell_command = vec![ + "powershell".to_string(), + "-Command".to_string(), + "Start-Sleep -Seconds 10".to_string(), + ]; + #[cfg(not(target_os = "windows"))] let shell_command = vec![ "python3".to_string(), "-c".to_string(), - "print(42)".to_string(), + "import time; time.sleep(10)".to_string(), ]; let tmp = TempDir::new()?; @@ -143,7 +150,7 @@ async fn turn_interrupt_resolves_pending_command_approval_request() -> Result<() shell_command.clone(), Some(&working_directory), Some(10_000), - "call_python_approval", + "call_sleep_approval", )?]) .await; create_config_toml(&codex_home, &server.uri(), "untrusted", "read-only")?; @@ -172,6 +179,7 @@ async fn turn_interrupt_resolves_pending_command_approval_request() -> Result<() text_elements: Vec::new(), }], cwd: Some(working_directory), + approval_policy: Some(codex_app_server_protocol::AskForApproval::UnlessTrusted), ..Default::default() }) .await?; @@ -190,7 +198,7 @@ async fn turn_interrupt_resolves_pending_command_approval_request() -> Result<() let ServerRequest::CommandExecutionRequestApproval { request_id, params } = request else { panic!("expected CommandExecutionRequestApproval request"); }; - assert_eq!(params.item_id, "call_python_approval"); + assert_eq!(params.item_id, "call_sleep_approval"); assert_eq!(params.thread_id, thread.id); assert_eq!(params.turn_id, turn.id); @@ -251,6 +259,7 @@ fn create_config_toml( r#" model = "mock-model" approval_policy = "{approval_policy}" +approvals_reviewer = "user" sandbox_mode = "{sandbox_mode}" model_provider = "mock_provider" diff --git a/codex-rs/cli/Cargo.toml b/codex-rs/cli/Cargo.toml index 60caf888f8..11d3eb3bbd 100644 --- a/codex-rs/cli/Cargo.toml +++ b/codex-rs/cli/Cargo.toml @@ -45,6 +45,7 @@ codex-state = { workspace = true } codex-stdio-to-uds = { workspace = true } codex-terminal-detection = { workspace = true } codex-tui = { workspace = true } +codex-utils-absolute-path = { workspace = true } codex-utils-path = { workspace = true } libc = { workspace = true } owo-colors = { workspace = true } diff --git a/codex-rs/cli/src/debug_sandbox.rs b/codex-rs/cli/src/debug_sandbox.rs index 0ad2f36382..a2cbf61874 100644 --- a/codex-rs/cli/src/debug_sandbox.rs +++ b/codex-rs/cli/src/debug_sandbox.rs @@ -18,7 +18,10 @@ use codex_protocol::config_types::SandboxMode; use codex_protocol::permissions::NetworkSandboxPolicy; use codex_sandboxing::landlock::create_linux_sandbox_command_args_for_policies; #[cfg(target_os = "macos")] -use codex_sandboxing::seatbelt::create_seatbelt_command_args_for_policies; +use codex_sandboxing::seatbelt::CreateSeatbeltCommandArgsParams; +#[cfg(target_os = "macos")] +use codex_sandboxing::seatbelt::create_seatbelt_command_args; +use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_cli::CliConfigOverrides; use tokio::process::Child; use tokio::process::Command as TokioCommand; @@ -39,6 +42,7 @@ pub async fn run_command_under_seatbelt( ) -> anyhow::Result<()> { let SeatbeltCommand { full_auto, + allow_unix_sockets, log_denials, config_overrides, command, @@ -50,6 +54,7 @@ pub async fn run_command_under_seatbelt( codex_linux_sandbox_exe, SandboxType::Seatbelt, log_denials, + &allow_unix_sockets, ) .await } @@ -78,6 +83,7 @@ pub async fn run_command_under_landlock( codex_linux_sandbox_exe, SandboxType::Landlock, /*log_denials*/ false, + &[], ) .await } @@ -98,6 +104,7 @@ pub async fn run_command_under_windows( codex_linux_sandbox_exe, SandboxType::Windows, /*log_denials*/ false, + &[], ) .await } @@ -116,6 +123,8 @@ async fn run_command_under_sandbox( codex_linux_sandbox_exe: Option, sandbox_type: SandboxType, log_denials: bool, + #[cfg_attr(not(target_os = "macos"), allow(unused_variables))] + allow_unix_sockets: &[AbsolutePathBuf], ) -> anyhow::Result<()> { let config = load_debug_sandbox_config( config_overrides @@ -252,14 +261,15 @@ async fn run_command_under_sandbox( let mut child = match sandbox_type { #[cfg(target_os = "macos")] SandboxType::Seatbelt => { - let args = create_seatbelt_command_args_for_policies( + let args = create_seatbelt_command_args(CreateSeatbeltCommandArgsParams { command, - &config.permissions.file_system_sandbox_policy, - config.permissions.network_sandbox_policy, - sandbox_policy_cwd.as_path(), - /*enforce_managed_network*/ false, - network.as_ref(), - ); + file_system_sandbox_policy: &config.permissions.file_system_sandbox_policy, + network_sandbox_policy: config.permissions.network_sandbox_policy, + sandbox_policy_cwd: sandbox_policy_cwd.as_path(), + enforce_managed_network: false, + network: network.as_ref(), + extra_allow_unix_sockets: allow_unix_sockets, + }); let network_policy = config.permissions.network_sandbox_policy; spawn_debug_sandbox_child( PathBuf::from("/usr/bin/sandbox-exec"), diff --git a/codex-rs/cli/src/lib.rs b/codex-rs/cli/src/lib.rs index 848391293c..cac34b3b61 100644 --- a/codex-rs/cli/src/lib.rs +++ b/codex-rs/cli/src/lib.rs @@ -3,6 +3,7 @@ mod exit_status; pub(crate) mod login; use clap::Parser; +use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_cli::CliConfigOverrides; pub use debug_sandbox::run_command_under_landlock; @@ -22,6 +23,10 @@ pub struct SeatbeltCommand { #[arg(long = "full-auto", default_value_t = false)] pub full_auto: bool, + /// Allow the sandboxed command to bind/connect AF_UNIX sockets rooted at this path. Relative paths are resolved against the current directory. Repeat to allow multiple paths. + #[arg(long = "allow-unix-socket", value_parser = parse_allow_unix_socket_path)] + pub allow_unix_sockets: Vec, + /// While the command runs, capture macOS sandbox denials via `log stream` and print them after exit #[arg(long = "log-denials", default_value_t = false)] pub log_denials: bool, @@ -34,6 +39,11 @@ pub struct SeatbeltCommand { pub command: Vec, } +fn parse_allow_unix_socket_path(raw: &str) -> Result { + AbsolutePathBuf::relative_to_current_dir(raw) + .map_err(|err| format!("invalid path {raw}: {err}")) +} + #[derive(Debug, Parser)] pub struct LandlockCommand { /// Convenience alias for low-friction sandboxed automatic execution (network-disabled sandbox that can write to cwd and TMPDIR) diff --git a/codex-rs/cli/src/main.rs b/codex-rs/cli/src/main.rs index 5ad86a80fd..4e962cf091 100644 --- a/codex-rs/cli/src/main.rs +++ b/codex-rs/cli/src/main.rs @@ -1269,7 +1269,7 @@ async fn run_debug_clear_memories_command( let state_db = StateRuntime::init(config.sqlite_home.clone(), config.model_provider_id.clone()) .await?; - state_db.reset_memory_data_for_fresh_start().await?; + state_db.clear_memory_data().await?; cleared_state_db = true; } diff --git a/codex-rs/cli/tests/debug_clear_memories.rs b/codex-rs/cli/tests/debug_clear_memories.rs index c68172ba35..164e4bd3e2 100644 --- a/codex-rs/cli/tests/debug_clear_memories.rs +++ b/codex-rs/cli/tests/debug_clear_memories.rs @@ -125,12 +125,6 @@ INSERT INTO jobs ( .fetch_one(&pool) .await?; assert_eq!(memory_jobs_count, 0); - - let memory_mode: String = sqlx::query_scalar("SELECT memory_mode FROM threads WHERE id = ?") - .bind(thread_id) - .fetch_one(&pool) - .await?; - assert_eq!(memory_mode, "disabled"); assert!(!memory_root.exists()); Ok(()) diff --git a/codex-rs/code-mode/Cargo.toml b/codex-rs/code-mode/Cargo.toml index fa7ce63a42..d2f42359d4 100644 --- a/codex-rs/code-mode/Cargo.toml +++ b/codex-rs/code-mode/Cargo.toml @@ -14,6 +14,7 @@ workspace = true [dependencies] async-trait = { workspace = true } +codex-protocol = { workspace = true } deno_core_icudata = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } diff --git a/codex-rs/code-mode/src/description.rs b/codex-rs/code-mode/src/description.rs index ac18fafd89..1aee1f7a3b 100644 --- a/codex-rs/code-mode/src/description.rs +++ b/codex-rs/code-mode/src/description.rs @@ -1,3 +1,4 @@ +use codex_protocol::ToolName; use serde::Deserialize; use serde::Serialize; use serde_json::Value as JsonValue; @@ -129,6 +130,7 @@ pub enum CodeModeToolKind { #[derive(Clone, Debug, PartialEq)] pub struct ToolDefinition { pub name: String, + pub tool_name: ToolName, pub description: String, pub kind: CodeModeToolKind, pub input_schema: Option, @@ -269,11 +271,15 @@ pub fn build_exec_tool_description( for tool in enabled_tools { let name = tool.name.as_str(); let nested_description = render_code_mode_sample_for_definition(tool); - let next_namespace = namespace_descriptions - .get(name) + let namespace_description = tool + .tool_name + .namespace + .as_ref() + .and_then(|namespace| namespace_descriptions.get(namespace)); + let next_namespace = namespace_description .map(|namespace_description| namespace_description.name.as_str()); if next_namespace != current_namespace { - if let Some(namespace_description) = namespace_descriptions.get(name) { + if let Some(namespace_description) = namespace_description { let namespace_description_text = namespace_description.description.trim(); if !namespace_description_text.is_empty() { nested_tool_sections.push(format!( @@ -346,7 +352,7 @@ pub fn augment_tool_definition(mut definition: ToolDefinition) -> ToolDefinition pub fn enabled_tool_metadata(definition: &ToolDefinition) -> EnabledToolMetadata { EnabledToolMetadata { - tool_name: definition.name.clone(), + tool_name: definition.tool_name.clone(), global_name: normalize_code_mode_identifier(&definition.name), description: definition.description.clone(), kind: definition.kind, @@ -355,7 +361,7 @@ pub fn enabled_tool_metadata(definition: &ToolDefinition) -> EnabledToolMetadata #[derive(Clone, Debug, Eq, PartialEq, Serialize)] pub struct EnabledToolMetadata { - pub tool_name: String, + pub tool_name: ToolName, pub global_name: String, pub description: String, pub kind: CodeModeToolKind, @@ -706,6 +712,7 @@ mod tests { use super::build_exec_tool_description; use super::normalize_code_mode_identifier; use super::parse_exec_source; + use codex_protocol::ToolName; use pretty_assertions::assert_eq; use serde_json::Value as JsonValue; use serde_json::json; @@ -770,6 +777,7 @@ mod tests { fn augment_tool_definition_appends_typed_declaration() { let definition = ToolDefinition { name: "hidden_dynamic_tool".to_string(), + tool_name: ToolName::plain("hidden_dynamic_tool"), description: "Test tool".to_string(), kind: CodeModeToolKind::Function, input_schema: Some(json!({ @@ -798,6 +806,7 @@ mod tests { fn augment_tool_definition_includes_property_descriptions_as_comments() { let definition = ToolDefinition { name: "weather_tool".to_string(), + tool_name: ToolName::plain("weather_tool"), description: "Weather tool".to_string(), kind: CodeModeToolKind::Function, input_schema: Some(json!({ @@ -846,6 +855,7 @@ mod tests { let description = build_exec_tool_description( &[ToolDefinition { name: "foo".to_string(), + tool_name: ToolName::plain("foo"), description: "bar".to_string(), kind: CodeModeToolKind::Function, input_schema: None, @@ -870,26 +880,18 @@ bar" #[test] fn code_mode_only_description_groups_namespace_instructions_once() { - let namespace_descriptions = BTreeMap::from([ - ( - "mcp__sample__alpha".to_string(), - ToolNamespaceDescription { - name: "mcp__sample".to_string(), - description: "Shared namespace guidance.".to_string(), - }, - ), - ( - "mcp__sample__beta".to_string(), - ToolNamespaceDescription { - name: "mcp__sample".to_string(), - description: "Shared namespace guidance.".to_string(), - }, - ), - ]); + let namespace_descriptions = BTreeMap::from([( + "mcp__sample__".to_string(), + ToolNamespaceDescription { + name: "mcp__sample".to_string(), + description: "Shared namespace guidance.".to_string(), + }, + )]); let description = build_exec_tool_description( &[ ToolDefinition { name: "mcp__sample__alpha".to_string(), + tool_name: ToolName::namespaced("mcp__sample__", "alpha"), description: "First tool".to_string(), kind: CodeModeToolKind::Function, input_schema: Some(json!({ @@ -905,6 +907,7 @@ bar" }, ToolDefinition { name: "mcp__sample__beta".to_string(), + tool_name: ToolName::namespaced("mcp__sample__", "beta"), description: "Second tool".to_string(), kind: CodeModeToolKind::Function, input_schema: Some(json!({ @@ -935,7 +938,7 @@ bar" #[test] fn code_mode_only_description_omits_empty_namespace_sections() { let namespace_descriptions = BTreeMap::from([( - "mcp__sample__alpha".to_string(), + "mcp__sample__".to_string(), ToolNamespaceDescription { name: "mcp__sample".to_string(), description: String::new(), @@ -944,6 +947,7 @@ bar" let description = build_exec_tool_description( &[ToolDefinition { name: "mcp__sample__alpha".to_string(), + tool_name: ToolName::namespaced("mcp__sample__", "alpha"), description: "First tool".to_string(), kind: CodeModeToolKind::Function, input_schema: Some(json!({ @@ -969,6 +973,7 @@ bar" fn code_mode_only_description_renders_shared_mcp_types_once() { let first_tool = augment_tool_definition(ToolDefinition { name: "mcp__sample__alpha".to_string(), + tool_name: ToolName::namespaced("mcp__sample__", "alpha"), description: "First tool".to_string(), kind: CodeModeToolKind::Function, input_schema: Some(json!({ @@ -1002,6 +1007,7 @@ bar" }); let second_tool = augment_tool_definition(ToolDefinition { name: "mcp__sample__beta".to_string(), + tool_name: ToolName::namespaced("mcp__sample__", "beta"), description: "Second tool".to_string(), kind: CodeModeToolKind::Function, input_schema: Some(json!({ @@ -1038,6 +1044,7 @@ bar" &[ ToolDefinition { name: first_tool.name, + tool_name: first_tool.tool_name, description: "First tool".to_string(), kind: first_tool.kind, input_schema: first_tool.input_schema, @@ -1045,6 +1052,7 @@ bar" }, ToolDefinition { name: second_tool.name, + tool_name: second_tool.tool_name, description: "Second tool".to_string(), kind: second_tool.kind, input_schema: second_tool.input_schema, diff --git a/codex-rs/code-mode/src/runtime/callbacks.rs b/codex-rs/code-mode/src/runtime/callbacks.rs index 5511baca23..ace6f61144 100644 --- a/codex-rs/code-mode/src/runtime/callbacks.rs +++ b/codex-rs/code-mode/src/runtime/callbacks.rs @@ -15,7 +15,13 @@ pub(super) fn tool_callback( args: v8::FunctionCallbackArguments, mut retval: v8::ReturnValue, ) { - let tool_name = args.data().to_rust_string_lossy(scope); + let tool_index = match args.data().to_rust_string_lossy(scope).parse::() { + Ok(tool_index) => tool_index, + Err(_) => { + throw_type_error(scope, "invalid tool callback data"); + return; + } + }; let input = if args.length() == 0 { Ok(None) } else { @@ -36,6 +42,22 @@ pub(super) fn tool_callback( let promise = resolver.get_promise(scope); let resolver = v8::Global::new(scope, resolver); + let tool_name = { + let Some(state) = scope.get_slot::() else { + throw_type_error(scope, "runtime state unavailable"); + return; + }; + let Some(tool_name) = state + .enabled_tools + .get(tool_index) + .map(|tool| tool.tool_name.clone()) + else { + throw_type_error(scope, "tool callback data is out of range"); + return; + }; + tool_name + }; + let Some(state) = scope.get_slot_mut::() else { throw_type_error(scope, "runtime state unavailable"); return; diff --git a/codex-rs/code-mode/src/runtime/globals.rs b/codex-rs/code-mode/src/runtime/globals.rs index 2d419db908..b40136c44c 100644 --- a/codex-rs/code-mode/src/runtime/globals.rs +++ b/codex-rs/code-mode/src/runtime/globals.rs @@ -53,10 +53,10 @@ fn build_tools_object<'s>( .map(|state| state.enabled_tools.clone()) .unwrap_or_default(); - for tool in enabled_tools { + for (tool_index, tool) in enabled_tools.iter().enumerate() { let name = v8::String::new(scope, &tool.global_name) .ok_or_else(|| "failed to allocate tool name".to_string())?; - let function = tool_function(scope, &tool.tool_name)?; + let function = tool_function(scope, tool_index)?; tools.set(scope, name.into(), function.into()); } Ok(tools) @@ -116,9 +116,9 @@ where fn tool_function<'s>( scope: &mut v8::PinScope<'s, '_>, - tool_name: &str, + tool_index: usize, ) -> Result, String> { - let data = v8::String::new(scope, tool_name) + let data = v8::String::new(scope, &tool_index.to_string()) .ok_or_else(|| "failed to allocate tool callback data".to_string())?; let template = v8::FunctionTemplate::builder(tool_callback) .data(data.into()) diff --git a/codex-rs/code-mode/src/runtime/mod.rs b/codex-rs/code-mode/src/runtime/mod.rs index febe54216a..0f50edd329 100644 --- a/codex-rs/code-mode/src/runtime/mod.rs +++ b/codex-rs/code-mode/src/runtime/mod.rs @@ -9,6 +9,7 @@ use std::sync::OnceLock; use std::sync::mpsc as std_mpsc; use std::thread; +use codex_protocol::ToolName; use serde_json::Value as JsonValue; use tokio::sync::mpsc; @@ -62,7 +63,7 @@ pub(crate) enum TurnMessage { ToolCall { cell_id: String, id: String, - name: String, + name: ToolName, input: Option, }, Notify { @@ -87,7 +88,7 @@ pub(crate) enum RuntimeEvent { YieldRequested, ToolCall { id: String, - name: String, + name: ToolName, input: Option, }, Notify { diff --git a/codex-rs/code-mode/src/service.rs b/codex-rs/code-mode/src/service.rs index c14fee1957..8a37880131 100644 --- a/codex-rs/code-mode/src/service.rs +++ b/codex-rs/code-mode/src/service.rs @@ -5,6 +5,7 @@ use std::sync::atomic::Ordering; use std::time::Duration; use async_trait::async_trait; +use codex_protocol::ToolName; use serde_json::Value as JsonValue; use tokio::sync::Mutex; use tokio::sync::mpsc; @@ -26,7 +27,7 @@ use crate::runtime::spawn_runtime; pub trait CodeModeTurnHost: Send + Sync { async fn invoke_tool( &self, - tool_name: String, + tool_name: ToolName, input: Option, cancellation_token: CancellationToken, ) -> Result; diff --git a/codex-rs/codex-api/src/api_bridge.rs b/codex-rs/codex-api/src/api_bridge.rs index 0ad2b13979..7c36c67fdd 100644 --- a/codex-rs/codex-api/src/api_bridge.rs +++ b/codex-rs/codex-api/src/api_bridge.rs @@ -12,6 +12,7 @@ use codex_protocol::error::RetryLimitReachedError; use codex_protocol::error::UnexpectedResponseError; use codex_protocol::error::UsageLimitReachedError; use http::HeaderMap; +use http::HeaderValue; use serde::Deserialize; use serde_json::Value; @@ -200,11 +201,16 @@ impl CoreAuthProvider { } impl ApiAuthProvider for CoreAuthProvider { - fn bearer_token(&self) -> Option { - self.token.clone() - } - - fn account_id(&self) -> Option { - self.account_id.clone() + fn add_auth_headers(&self, headers: &mut HeaderMap) { + if let Some(token) = self.token.as_ref() + && let Ok(header) = HeaderValue::from_str(&format!("Bearer {token}")) + { + let _ = headers.insert(http::header::AUTHORIZATION, header); + } + if let Some(account_id) = self.account_id.as_ref() + && let Ok(header) = HeaderValue::from_str(account_id) + { + let _ = headers.insert("ChatGPT-Account-ID", header); + } } } diff --git a/codex-rs/codex-api/src/api_bridge_tests.rs b/codex-rs/codex-api/src/api_bridge_tests.rs index 71d3889915..50247c1314 100644 --- a/codex-rs/codex-api/src/api_bridge_tests.rs +++ b/codex-rs/codex-api/src/api_bridge_tests.rs @@ -141,3 +141,24 @@ fn core_auth_provider_reports_when_auth_header_will_attach() { assert!(auth.auth_header_attached()); assert_eq!(auth.auth_header_name(), Some("authorization")); } + +#[test] +fn core_auth_provider_adds_auth_headers() { + let auth = CoreAuthProvider::for_test(Some("access-token"), Some("workspace-123")); + let mut headers = HeaderMap::new(); + + crate::AuthProvider::add_auth_headers(&auth, &mut headers); + + assert_eq!( + headers + .get(http::header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()), + Some("Bearer access-token") + ); + assert_eq!( + headers + .get("ChatGPT-Account-ID") + .and_then(|value| value.to_str().ok()), + Some("workspace-123") + ); +} diff --git a/codex-rs/codex-api/src/auth.rs b/codex-rs/codex-api/src/auth.rs index f649062db1..a7b1e69d14 100644 --- a/codex-rs/codex-api/src/auth.rs +++ b/codex-rs/codex-api/src/auth.rs @@ -1,33 +1,10 @@ -use codex_client::Request; use http::HeaderMap; -use http::HeaderValue; -/// Provides bearer and account identity information for API requests. +/// Adds authentication headers to API requests. /// /// Implementations should be cheap and non-blocking; any asynchronous /// refresh or I/O should be handled by higher layers before requests /// reach this interface. pub trait AuthProvider: Send + Sync { - fn bearer_token(&self) -> Option; - fn account_id(&self) -> Option { - None - } -} - -pub(crate) fn add_auth_headers_to_header_map(auth: &A, headers: &mut HeaderMap) { - if let Some(token) = auth.bearer_token() - && let Ok(header) = HeaderValue::from_str(&format!("Bearer {token}")) - { - let _ = headers.insert(http::header::AUTHORIZATION, header); - } - if let Some(account_id) = auth.account_id() - && let Ok(header) = HeaderValue::from_str(&account_id) - { - let _ = headers.insert("ChatGPT-Account-ID", header); - } -} - -pub(crate) fn add_auth_headers(auth: &A, mut req: Request) -> Request { - add_auth_headers_to_header_map(auth, &mut req.headers); - req + fn add_auth_headers(&self, headers: &mut HeaderMap); } diff --git a/codex-rs/codex-api/src/endpoint/compact.rs b/codex-rs/codex-api/src/endpoint/compact.rs index 44a56a11a7..748ac35558 100644 --- a/codex-rs/codex-api/src/endpoint/compact.rs +++ b/codex-rs/codex-api/src/endpoint/compact.rs @@ -90,9 +90,7 @@ mod tests { struct DummyAuth; impl AuthProvider for DummyAuth { - fn bearer_token(&self) -> Option { - None - } + fn add_auth_headers(&self, _headers: &mut HeaderMap) {} } #[test] diff --git a/codex-rs/codex-api/src/endpoint/memories.rs b/codex-rs/codex-api/src/endpoint/memories.rs index 5cb2a65b1f..3047c859db 100644 --- a/codex-rs/codex-api/src/endpoint/memories.rs +++ b/codex-rs/codex-api/src/endpoint/memories.rs @@ -103,9 +103,7 @@ mod tests { struct DummyAuth; impl AuthProvider for DummyAuth { - fn bearer_token(&self) -> Option { - None - } + fn add_auth_headers(&self, _headers: &mut HeaderMap) {} } #[derive(Clone)] diff --git a/codex-rs/codex-api/src/endpoint/models.rs b/codex-rs/codex-api/src/endpoint/models.rs index 97781ac419..17342d6f99 100644 --- a/codex-rs/codex-api/src/endpoint/models.rs +++ b/codex-rs/codex-api/src/endpoint/models.rs @@ -132,9 +132,7 @@ mod tests { struct DummyAuth; impl AuthProvider for DummyAuth { - fn bearer_token(&self) -> Option { - None - } + fn add_auth_headers(&self, _headers: &mut HeaderMap) {} } fn provider(base_url: &str) -> Provider { diff --git a/codex-rs/codex-api/src/endpoint/realtime_call.rs b/codex-rs/codex-api/src/endpoint/realtime_call.rs index 4d0bcfa7bf..a9a8b963cf 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_call.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_call.rs @@ -284,8 +284,11 @@ mod tests { struct DummyAuth; impl AuthProvider for DummyAuth { - fn bearer_token(&self) -> Option { - Some("test-token".to_string()) + fn add_auth_headers(&self, headers: &mut HeaderMap) { + headers.insert( + http::header::AUTHORIZATION, + HeaderValue::from_static("Bearer test-token"), + ); } } diff --git a/codex-rs/codex-api/src/endpoint/responses_websocket.rs b/codex-rs/codex-api/src/endpoint/responses_websocket.rs index a60d188bda..d2b775cdd5 100644 --- a/codex-rs/codex-api/src/endpoint/responses_websocket.rs +++ b/codex-rs/codex-api/src/endpoint/responses_websocket.rs @@ -1,5 +1,4 @@ use crate::auth::AuthProvider; -use crate::auth::add_auth_headers_to_header_map; use crate::common::ResponseEvent; use crate::common::ResponseStream; use crate::common::ResponsesWsRequest; @@ -310,7 +309,7 @@ impl ResponsesWebsocketClient { let mut headers = merge_request_headers(&self.provider.headers, extra_headers, default_headers); - add_auth_headers_to_header_map(&self.auth, &mut headers); + self.auth.add_auth_headers(&mut headers); let (stream, server_reasoning_included, models_etag, server_model) = connect_websocket(ws_url, headers, turn_state.clone()).await?; diff --git a/codex-rs/codex-api/src/endpoint/session.rs b/codex-rs/codex-api/src/endpoint/session.rs index 00919a0c54..e4a470ceee 100644 --- a/codex-rs/codex-api/src/endpoint/session.rs +++ b/codex-rs/codex-api/src/endpoint/session.rs @@ -1,5 +1,4 @@ use crate::auth::AuthProvider; -use crate::auth::add_auth_headers; use crate::error::ApiError; use crate::provider::Provider; use crate::telemetry::run_with_request_telemetry; @@ -56,7 +55,8 @@ impl EndpointSession { if let Some(body) = body { req.body = Some(RequestBody::Json(body.clone())); } - add_auth_headers(&self.auth, req) + self.auth.add_auth_headers(&mut req.headers); + req } pub(crate) async fn execute( diff --git a/codex-rs/codex-api/src/files.rs b/codex-rs/codex-api/src/files.rs index 6fad5b62f5..ebe35af5ba 100644 --- a/codex-rs/codex-api/src/files.rs +++ b/codex-rs/codex-api/src/files.rs @@ -256,17 +256,14 @@ fn authorized_request( method: reqwest::Method, url: &str, ) -> reqwest::RequestBuilder { + let mut headers = http::HeaderMap::new(); + auth.add_auth_headers(&mut headers); + let client = build_reqwest_client(); - let mut request = client + client .request(method, url) - .timeout(OPENAI_FILE_REQUEST_TIMEOUT); - if let Some(token) = auth.bearer_token() { - request = request.bearer_auth(token); - } - if let Some(account_id) = auth.account_id() { - request = request.header("chatgpt-account-id", account_id); - } - request + .timeout(OPENAI_FILE_REQUEST_TIMEOUT) + .headers(headers) } fn build_reqwest_client() -> reqwest::Client { diff --git a/codex-rs/codex-api/src/lib.rs b/codex-rs/codex-api/src/lib.rs index f4f90b289c..82a1004849 100644 --- a/codex-rs/codex-api/src/lib.rs +++ b/codex-rs/codex-api/src/lib.rs @@ -57,7 +57,7 @@ pub use crate::error::ApiError; pub use crate::files::upload_local_file; pub use crate::provider::Provider; pub use crate::provider::RetryConfig; -pub use crate::provider::is_azure_responses_wire_base_url; +pub use crate::provider::is_azure_responses_provider; pub use crate::requests::Compression; pub use crate::sse::stream_from_fixture; pub use crate::telemetry::SseTelemetry; diff --git a/codex-rs/codex-api/src/provider.rs b/codex-rs/codex-api/src/provider.rs index 81a168ffd7..45f2512dc3 100644 --- a/codex-rs/codex-api/src/provider.rs +++ b/codex-rs/codex-api/src/provider.rs @@ -86,7 +86,7 @@ impl Provider { } pub fn is_azure_responses_endpoint(&self) -> bool { - is_azure_responses_wire_base_url(&self.name, Some(&self.base_url)) + is_azure_responses_provider(&self.name, Some(&self.base_url)) } pub fn websocket_url_for_path(&self, path: &str) -> Result { @@ -103,21 +103,20 @@ impl Provider { } } -pub fn is_azure_responses_wire_base_url(name: &str, base_url: Option<&str>) -> bool { +pub fn is_azure_responses_provider(name: &str, base_url: Option<&str>) -> bool { if name.eq_ignore_ascii_case("azure") { - return true; + true + } else if let Some(base_url) = base_url { + matches_azure_responses_base_url(base_url) + } else { + false } - - let Some(base_url) = base_url else { - return false; - }; - - let base = base_url.to_ascii_lowercase(); - base.contains("openai.azure.") || matches_azure_responses_base_url(&base) } fn matches_azure_responses_base_url(base_url: &str) -> bool { - const AZURE_MARKERS: [&str; 5] = [ + let base_url = base_url.to_ascii_lowercase(); + const AZURE_MARKERS: [&str; 6] = [ + "openai.azure.", "cognitiveservices.azure.", "aoai.azure.", "azure-api.", @@ -144,12 +143,12 @@ mod tests { for base_url in positive_cases { assert!( - is_azure_responses_wire_base_url("test", Some(base_url)), + is_azure_responses_provider("test", Some(base_url)), "expected {base_url} to be detected as Azure" ); } - assert!(is_azure_responses_wire_base_url( + assert!(is_azure_responses_provider( "Azure", Some("https://example.com") )); @@ -162,7 +161,7 @@ mod tests { for base_url in negative_cases { assert!( - !is_azure_responses_wire_base_url("test", Some(base_url)), + !is_azure_responses_provider("test", Some(base_url)), "expected {base_url} not to be detected as Azure" ); } diff --git a/codex-rs/codex-api/tests/clients.rs b/codex-rs/codex-api/tests/clients.rs index b11c6f9d6b..d82fcc14ce 100644 --- a/codex-rs/codex-api/tests/clients.rs +++ b/codex-rs/codex-api/tests/clients.rs @@ -91,9 +91,7 @@ impl HttpTransport for RecordingTransport { struct NoAuth; impl AuthProvider for NoAuth { - fn bearer_token(&self) -> Option { - None - } + fn add_auth_headers(&self, _headers: &mut HeaderMap) {} } #[derive(Clone)] @@ -112,12 +110,14 @@ impl StaticAuth { } impl AuthProvider for StaticAuth { - fn bearer_token(&self) -> Option { - Some(self.token.clone()) - } - - fn account_id(&self) -> Option { - Some(self.account_id.clone()) + fn add_auth_headers(&self, headers: &mut HeaderMap) { + let token = &self.token; + if let Ok(header) = HeaderValue::from_str(&format!("Bearer {token}")) { + headers.insert(http::header::AUTHORIZATION, header); + } + if let Ok(header) = HeaderValue::from_str(&self.account_id) { + headers.insert("ChatGPT-Account-ID", header); + } } } diff --git a/codex-rs/codex-api/tests/models_integration.rs b/codex-rs/codex-api/tests/models_integration.rs index 91a8477a92..fab135e0ab 100644 --- a/codex-rs/codex-api/tests/models_integration.rs +++ b/codex-rs/codex-api/tests/models_integration.rs @@ -24,9 +24,7 @@ use wiremock::matchers::path; struct DummyAuth; impl AuthProvider for DummyAuth { - fn bearer_token(&self) -> Option { - None - } + fn add_auth_headers(&self, _headers: &mut HeaderMap) {} } fn provider(base_url: &str) -> Provider { diff --git a/codex-rs/codex-api/tests/sse_end_to_end.rs b/codex-rs/codex-api/tests/sse_end_to_end.rs index b15de296a8..4d32e82242 100644 --- a/codex-rs/codex-api/tests/sse_end_to_end.rs +++ b/codex-rs/codex-api/tests/sse_end_to_end.rs @@ -53,9 +53,7 @@ impl HttpTransport for FixtureSseTransport { struct NoAuth; impl AuthProvider for NoAuth { - fn bearer_token(&self) -> Option { - None - } + fn add_auth_headers(&self, _headers: &mut HeaderMap) {} } fn provider(name: &str) -> Provider { diff --git a/codex-rs/codex-mcp/src/lib.rs b/codex-rs/codex-mcp/src/lib.rs index bfbe1c60f8..ed0d9b4122 100644 --- a/codex-rs/codex-mcp/src/lib.rs +++ b/codex-rs/codex-mcp/src/lib.rs @@ -36,8 +36,7 @@ pub use mcp::tool_plugin_provenance; pub use mcp::with_codex_apps_mcp; pub use mcp_connection_manager::CodexAppsToolsCacheKey; pub use mcp_connection_manager::DEFAULT_STARTUP_TIMEOUT; -pub use mcp_connection_manager::MCP_SANDBOX_STATE_CAPABILITY; -pub use mcp_connection_manager::MCP_SANDBOX_STATE_METHOD; +pub use mcp_connection_manager::MCP_SANDBOX_STATE_META_CAPABILITY; pub use mcp_connection_manager::McpConnectionManager; pub use mcp_connection_manager::SandboxState; pub use mcp_connection_manager::ToolInfo; diff --git a/codex-rs/codex-mcp/src/mcp/mod.rs b/codex-rs/codex-mcp/src/mcp/mod.rs index d53532b951..421ed14c36 100644 --- a/codex-rs/codex-mcp/src/mcp/mod.rs +++ b/codex-rs/codex-mcp/src/mcp/mod.rs @@ -35,7 +35,6 @@ use codex_protocol::protocol::SandboxPolicy; use serde_json::Value; use crate::mcp_connection_manager::McpConnectionManager; -use crate::mcp_connection_manager::SandboxState; use crate::mcp_connection_manager::codex_apps_tools_cache_key; pub type McpManager = McpConnectionManager; @@ -347,14 +346,6 @@ pub async fn collect_mcp_snapshot_with_detail( let (tx_event, rx_event) = unbounded(); drop(rx_event); - // Use ReadOnly sandbox policy for MCP snapshot collection (safest default) - let sandbox_state = SandboxState { - sandbox_policy: SandboxPolicy::new_read_only_policy(), - codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(), - sandbox_cwd: env::current_dir().unwrap_or_else(|_| PathBuf::from("/")), - use_legacy_landlock: config.use_legacy_landlock, - }; - let (mcp_connection_manager, cancel_token) = McpConnectionManager::new( &mcp_servers, config.mcp_oauth_credentials_store_mode, @@ -362,7 +353,7 @@ pub async fn collect_mcp_snapshot_with_detail( &config.approval_policy, submit_id, tx_event, - sandbox_state, + SandboxPolicy::new_read_only_policy(), config.codex_home.clone(), codex_apps_tools_cache_key(auth), tool_plugin_provenance, @@ -421,13 +412,6 @@ pub async fn collect_mcp_server_status_snapshot_with_detail( let (tx_event, rx_event) = unbounded(); drop(rx_event); - let sandbox_state = SandboxState { - sandbox_policy: SandboxPolicy::new_read_only_policy(), - codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(), - sandbox_cwd: env::current_dir().unwrap_or_else(|_| PathBuf::from("/")), - use_legacy_landlock: config.use_legacy_landlock, - }; - let (mcp_connection_manager, cancel_token) = McpConnectionManager::new( &mcp_servers, config.mcp_oauth_credentials_store_mode, @@ -435,7 +419,7 @@ pub async fn collect_mcp_server_status_snapshot_with_detail( &config.approval_policy, submit_id, tx_event, - sandbox_state, + SandboxPolicy::new_read_only_policy(), config.codex_home.clone(), codex_apps_tools_cache_key(auth), tool_plugin_provenance, diff --git a/codex-rs/codex-mcp/src/mcp_connection_manager.rs b/codex-rs/codex-mcp/src/mcp_connection_manager.rs index 9021f7d7a9..1542c7b30b 100644 --- a/codex-rs/codex-mcp/src/mcp_connection_manager.rs +++ b/codex-rs/codex-mcp/src/mcp_connection_manager.rs @@ -36,6 +36,7 @@ use codex_async_utils::CancelErr; use codex_async_utils::OrCancelExt; use codex_config::Constrained; use codex_config::types::OAuthCredentialsStoreMode; +use codex_protocol::ToolName; use codex_protocol::approvals::ElicitationRequest; use codex_protocol::approvals::ElicitationRequestEvent; use codex_protocol::mcp::CallToolResult; @@ -155,6 +156,12 @@ pub struct ToolInfo { pub connector_description: Option, } +impl ToolInfo { + pub fn canonical_tool_name(&self) -> ToolName { + ToolName::namespaced(self.callable_namespace.clone(), self.callable_name.clone()) + } +} + const META_OPENAI_FILE_PARAMS: &str = "openai/fileParams"; pub fn declared_openai_file_input_param_names( @@ -432,7 +439,7 @@ struct ManagedClient { tool_filter: ToolFilter, tool_timeout: Option, server_instructions: Option, - server_supports_sandbox_state_capability: bool, + server_supports_sandbox_state_meta_capability: bool, codex_apps_tools_cache_context: Option, } @@ -461,22 +468,6 @@ impl ManagedClient { self.tools.clone() } - - /// Returns once the server has ack'd the sandbox state update. - async fn notify_sandbox_state_change(&self, sandbox_state: &SandboxState) -> Result<()> { - if !self.server_supports_sandbox_state_capability { - return Ok(()); - } - - let _response = self - .client - .send_custom_request( - MCP_SANDBOX_STATE_METHOD, - Some(serde_json::to_value(sandbox_state)?), - ) - .await?; - Ok(()) - } } #[derive(Clone)] @@ -634,18 +625,11 @@ impl AsyncManagedClient { }; tools.map(annotate_tools) } - - async fn notify_sandbox_state_change(&self, sandbox_state: &SandboxState) -> Result<()> { - let managed = self.client().await?; - managed.notify_sandbox_state_change(sandbox_state).await - } } -pub const MCP_SANDBOX_STATE_CAPABILITY: &str = "codex/sandbox-state"; - -/// Custom MCP request to push sandbox state updates. -/// When used, the `params` field of the notification is [`SandboxState`]. -pub const MCP_SANDBOX_STATE_METHOD: &str = "codex/sandbox-state/update"; +/// MCP server capability indicating that Codex should include [`SandboxState`] +/// in tool-call request `_meta` under this key. +pub const MCP_SANDBOX_STATE_META_CAPABILITY: &str = "codex/sandbox-state-meta"; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -723,7 +707,7 @@ impl McpConnectionManager { approval_policy: &Constrained, submit_id: String, tx_event: Sender, - initial_sandbox_state: SandboxState, + initial_sandbox_policy: SandboxPolicy, codex_home: PathBuf, codex_apps_tools_cache_key: CodexAppsToolsCacheKey, tool_plugin_provenance: ToolPluginProvenance, @@ -732,10 +716,8 @@ impl McpConnectionManager { let mut clients = HashMap::new(); let mut server_origins = HashMap::new(); let mut join_set = JoinSet::new(); - let elicitation_requests = ElicitationRequestManager::new( - approval_policy.value(), - initial_sandbox_state.sandbox_policy.clone(), - ); + let elicitation_requests = + ElicitationRequestManager::new(approval_policy.value(), initial_sandbox_policy); let tool_plugin_provenance = Arc::new(tool_plugin_provenance); let startup_submit_id = submit_id.clone(); let mcp_servers = mcp_servers.clone(); @@ -775,25 +757,13 @@ impl McpConnectionManager { let tx_event = tx_event.clone(); let submit_id = startup_submit_id.clone(); let auth_entry = auth_entries.get(&server_name).cloned(); - let sandbox_state = initial_sandbox_state.clone(); join_set.spawn(async move { let outcome = async_managed_client.client().await; if cancel_token.is_cancelled() { return (server_name, Err(StartupOutcomeError::Cancelled)); } let status = match &outcome { - Ok(_) => { - // Send sandbox state notification immediately after Ready - if let Err(e) = async_managed_client - .notify_sandbox_state_change(&sandbox_state) - .await - { - warn!( - "Failed to notify sandbox state to MCP server {server_name}: {e:#}", - ); - } - McpStartupStatus::Ready - } + Ok(_) => McpStartupStatus::Ready, Err(error) => { let error_str = mcp_init_error_display( server_name.as_str(), @@ -1142,6 +1112,16 @@ impl McpConnectionManager { }) } + pub async fn server_supports_sandbox_state_meta_capability( + &self, + server: &str, + ) -> Result { + Ok(self + .client_by_name(server) + .await? + .server_supports_sandbox_state_meta_capability) + } + /// List resources from the specified server. pub async fn list_resources( &self, @@ -1191,42 +1171,11 @@ impl McpConnectionManager { .with_context(|| format!("resources/read failed for `{server}` ({uri})")) } - pub async fn resolve_tool_info(&self, name: &str, namespace: Option<&str>) -> Option { - let qualified_name = match namespace { - Some(namespace) if name.starts_with(namespace) => name.to_string(), - Some(namespace) => format!("{namespace}{name}"), - None => name.to_string(), - }; - - self.list_all_tools().await.get(&qualified_name).cloned() - } - - pub async fn notify_sandbox_state_change(&self, sandbox_state: &SandboxState) -> Result<()> { - let mut join_set = JoinSet::new(); - - for async_managed_client in self.clients.values() { - let sandbox_state = sandbox_state.clone(); - let async_managed_client = async_managed_client.clone(); - join_set.spawn(async move { - async_managed_client - .notify_sandbox_state_change(&sandbox_state) - .await - }); - } - - while let Some(join_res) = join_set.join_next().await { - match join_res { - Ok(Ok(())) => {} - Ok(Err(err)) => { - warn!("Failed to notify sandbox state change to MCP server: {err:#}"); - } - Err(err) => { - warn!("Task panic when notifying sandbox state change to MCP server: {err:#}"); - } - } - } - - Ok(()) + pub async fn resolve_tool_info(&self, tool_name: &ToolName) -> Option { + let all_tools = self.list_all_tools().await; + all_tools + .into_values() + .find(|tool| tool.canonical_tool_name() == *tool_name) } } @@ -1473,11 +1422,11 @@ async fn start_server_task( .await .map_err(StartupOutcomeError::from)?; - let server_supports_sandbox_state_capability = initialize_result + let server_supports_sandbox_state_meta_capability = initialize_result .capabilities .experimental .as_ref() - .and_then(|exp| exp.get(MCP_SANDBOX_STATE_CAPABILITY)) + .and_then(|exp| exp.get(MCP_SANDBOX_STATE_META_CAPABILITY)) .is_some(); let list_start = Instant::now(); let fetch_start = Instant::now(); @@ -1514,7 +1463,7 @@ async fn start_server_task( tool_timeout: Some(tool_timeout), tool_filter, server_instructions: initialize_result.instructions, - server_supports_sandbox_state_capability, + server_supports_sandbox_state_meta_capability, codex_apps_tools_cache_context, }; diff --git a/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs b/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs index bfa50c4182..663f76d5d0 100644 --- a/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs +++ b/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs @@ -1,4 +1,5 @@ use super::*; +use codex_protocol::ToolName; use codex_protocol::protocol::GranularApprovalConfig; use codex_protocol::protocol::McpAuthStatus; use pretty_assertions::assert_eq; @@ -646,6 +647,42 @@ async fn list_all_tools_uses_startup_snapshot_while_client_is_pending() { assert_eq!(tool.callable_name, "calendar_create_event"); } +#[tokio::test] +async fn resolve_tool_info_accepts_canonical_namespaced_tool_names() { + let startup_tools = vec![create_test_tool("rmcp", "echo")]; + let pending_client = futures::future::pending::>() + .boxed() + .shared(); + let approval_policy = Constrained::allow_any(AskForApproval::OnFailure); + let sandbox_policy = Constrained::allow_any(SandboxPolicy::new_read_only_policy()); + let mut manager = McpConnectionManager::new_uninitialized(&approval_policy, &sandbox_policy); + manager.clients.insert( + "rmcp".to_string(), + AsyncManagedClient { + client: pending_client, + startup_snapshot: Some(startup_tools), + startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(false)), + tool_plugin_provenance: Arc::new(ToolPluginProvenance::default()), + }, + ); + + let tool = manager + .resolve_tool_info(&ToolName::namespaced("mcp__rmcp__", "echo")) + .await + .expect("split MCP tool namespace and name should resolve"); + + let expected = ("rmcp", "mcp__rmcp__", "echo", "echo"); + assert_eq!( + ( + tool.server_name.as_str(), + tool.callable_namespace.as_str(), + tool.callable_name.as_str(), + tool.tool.name.as_ref(), + ), + expected + ); +} + #[tokio::test] async fn list_all_tools_blocks_while_client_is_pending_without_startup_snapshot() { let pending_client = futures::future::pending::>() diff --git a/codex-rs/core/BUILD.bazel b/codex-rs/core/BUILD.bazel index ed01996273..be07ef7999 100644 --- a/codex-rs/core/BUILD.bazel +++ b/codex-rs/core/BUILD.bazel @@ -1,4 +1,4 @@ -load("//:defs.bzl", "codex_rust_crate") +load("//:defs.bzl", "codex_rust_crate", "workspace_root_test") filegroup( name = "model_availability_nux_fixtures", @@ -31,6 +31,15 @@ codex_rust_crate( "//codex-rs/apply-patch:apply_patch_tool_instructions.md", "templates/realtime/backend_prompt.md", ], + integration_test_flaky_by_stem = { + "all": [ + "suite::abort_tasks::interrupt_tool_records_history_entries", + "suite::apply_patch_cli::apply_patch_cli_rejects_invalid_hunk_header::applypatchmodeloutput_function_expects", + ], + "responses_headers": [ + "responses_stream_includes_turn_metadata_header_for_git_workspace_e2e", + ], + }, integration_test_timeout = "long", test_data_extra = [ "config.schema.json", @@ -48,11 +57,56 @@ codex_rust_crate( "//:AGENTS.md", ], test_tags = ["no-sandbox"], + unit_test_args = [ + "--skip", + "guardian::tests::guardian_parallel_reviews_fork_from_last_committed_trunk_history", + "--skip", + "agent::control::tests::completion_watcher_notifies_parent_when_child_is_missing", + ], unit_test_timeout = "long", extra_binaries = [ "//codex-rs/linux-sandbox:codex-linux-sandbox", "//codex-rs/rmcp-client:test_stdio_server", "//codex-rs/rmcp-client:test_streamable_http_server", + "//codex-rs/responses-api-proxy:codex-responses-api-proxy", "//codex-rs/cli:codex", ], ) + +workspace_root_test( + name = "core-guardian-parallel-reviews-flaky-test", + env = { + "INSTA_WORKSPACE_ROOT": ".", + "INSTA_SNAPSHOT_PATH": "src", + }, + test_bin = ":core-unit-tests-bin", + workspace_root_marker = "//codex-rs/utils/cargo-bin:repo_root.marker", + args = [ + "--exact", + "guardian::tests::guardian_parallel_reviews_fork_from_last_committed_trunk_history", + ], + tags = [ + "flaky", + "no-sandbox", + ], + timeout = "long", +) + +workspace_root_test( + name = "core-completion-watcher-missing-child-flaky-test", + env = { + "INSTA_WORKSPACE_ROOT": ".", + "INSTA_SNAPSHOT_PATH": "src", + }, + test_bin = ":core-unit-tests-bin", + workspace_root_marker = "//codex-rs/utils/cargo-bin:repo_root.marker", + args = [ + "--exact", + "agent::control::tests::completion_watcher_notifies_parent_when_child_is_missing", + ], + tags = [ + "flaky", + "no-sandbox", + ], + timeout = "long", +) diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index c91f07f705..78dfd3c070 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -41,6 +41,7 @@ codex-login = { workspace = true } codex-mcp = { workspace = true } codex-model-provider-info = { workspace = true } codex-models-manager = { workspace = true } +ed25519-dalek = { workspace = true } codex-shell-command = { workspace = true } codex-execpolicy = { workspace = true } codex-git-utils = { workspace = true } @@ -56,6 +57,7 @@ codex-rmcp-client = { workspace = true } codex-sandboxing = { workspace = true } codex-state = { workspace = true } codex-terminal-detection = { workspace = true } +codex-thread-store = { workspace = true } codex-tools = { workspace = true } codex-utils-absolute-path = { workspace = true } codex-utils-cache = { workspace = true } @@ -143,6 +145,7 @@ codex-shell-escalation = { workspace = true } [dev-dependencies] assert_cmd = { workspace = true } assert_matches = { workspace = true } +codex-arg0 = { workspace = true } codex-otel = { workspace = true } codex-test-binary-support = { workspace = true } codex-utils-cargo-bin = { workspace = true } diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index d502e06dd5..a02525fc6e 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -526,6 +526,9 @@ }, "web_search_request": { "type": "boolean" + }, + "workspace_dependencies": { + "type": "boolean" } }, "type": "object" @@ -2417,6 +2420,9 @@ }, "web_search_request": { "type": "boolean" + }, + "workspace_dependencies": { + "type": "boolean" } }, "type": "object" diff --git a/codex-rs/core/src/agent/control_tests.rs b/codex-rs/core/src/agent/control_tests.rs index 6fc74b30e8..79a96d11e5 100644 --- a/codex-rs/core/src/agent/control_tests.rs +++ b/codex-rs/core/src/agent/control_tests.rs @@ -7,7 +7,6 @@ use crate::config::Config; use crate::config::ConfigBuilder; use crate::contextual_user_message::SUBAGENT_NOTIFICATION_OPEN_TAG; use assert_matches::assert_matches; -use chrono::Utc; use codex_features::Feature; use codex_login::CodexAuth; use codex_protocol::AgentPath; @@ -24,6 +23,9 @@ use codex_protocol::protocol::TurnAbortReason; use codex_protocol::protocol::TurnAbortedEvent; use codex_protocol::protocol::TurnCompleteEvent; use codex_protocol::protocol::TurnStartedEvent; +use codex_thread_store::ArchiveThreadParams; +use codex_thread_store::LocalThreadStore; +use codex_thread_store::ThreadStore; use pretty_assertions::assert_eq; use tempfile::TempDir; use tokio::time::Duration; @@ -1658,38 +1660,18 @@ async fn resume_agent_from_rollout_reads_archived_rollout_path() { .await .expect("child thread should exist"); persist_thread_for_tree_resume(&child_thread, "persist before archiving").await; - let rollout_path = child_thread - .rollout_path() - .expect("thread should have rollout path"); - let state_db = child_thread - .state_db() - .expect("thread should have state db handle"); - let _ = harness .control .shutdown_live_agent(child_thread_id) .await .expect("child shutdown should succeed"); - - let archived_root = harness - .config - .codex_home - .join(crate::ARCHIVED_SESSIONS_SUBDIR); - tokio::fs::create_dir_all(&archived_root) + let store = LocalThreadStore::new(codex_rollout::RolloutConfig::from_view(&harness.config)); + store + .archive_thread(ArchiveThreadParams { + thread_id: child_thread_id, + }) .await - .expect("archived root should exist"); - let archived_rollout_path = archived_root.join( - rollout_path - .file_name() - .expect("rollout file name should be present"), - ); - tokio::fs::rename(&rollout_path, &archived_rollout_path) - .await - .expect("rollout should move to archived path"); - state_db - .mark_archived(child_thread_id, archived_rollout_path.as_path(), Utc::now()) - .await - .expect("state db archive update should succeed"); + .expect("child thread should archive"); let resumed_thread_id = harness .control diff --git a/codex-rs/core/src/agent_identity.rs b/codex-rs/core/src/agent_identity.rs new file mode 100644 index 0000000000..13a07cd961 --- /dev/null +++ b/codex-rs/core/src/agent_identity.rs @@ -0,0 +1,766 @@ +use std::sync::Arc; +use std::time::Duration; + +use anyhow::Context; +use anyhow::Result; +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use chrono::SecondsFormat; +use chrono::Utc; +use codex_features::Feature; +use codex_login::AgentIdentityAuthRecord; +use codex_login::AuthManager; +use codex_login::CodexAuth; +use codex_login::default_client::create_client; +use codex_protocol::protocol::SessionSource; +use ed25519_dalek::SigningKey; +use ed25519_dalek::VerifyingKey; +use ed25519_dalek::pkcs8::DecodePrivateKey; +use ed25519_dalek::pkcs8::EncodePrivateKey; +use rand::TryRngCore; +use rand::rngs::OsRng; +use serde::Deserialize; +use serde::Serialize; +use tokio::sync::Mutex; +use tracing::debug; +use tracing::info; +use tracing::warn; + +use crate::config::Config; + +const AGENT_REGISTRATION_TIMEOUT: Duration = Duration::from_secs(15); +const AGENT_IDENTITY_BISCUIT_TIMEOUT: Duration = Duration::from_secs(15); + +#[derive(Clone)] +pub(crate) struct AgentIdentityManager { + auth_manager: Arc, + chatgpt_base_url: String, + feature_enabled: bool, + abom: AgentBillOfMaterials, + ensure_lock: Arc>, +} + +impl std::fmt::Debug for AgentIdentityManager { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AgentIdentityManager") + .field("chatgpt_base_url", &self.chatgpt_base_url) + .field("feature_enabled", &self.feature_enabled) + .field("abom", &self.abom) + .finish_non_exhaustive() + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub(crate) struct StoredAgentIdentity { + pub(crate) binding_id: String, + pub(crate) chatgpt_account_id: String, + pub(crate) chatgpt_user_id: Option, + pub(crate) agent_runtime_id: String, + pub(crate) private_key_pkcs8_base64: String, + pub(crate) public_key_ssh: String, + pub(crate) registered_at: String, + pub(crate) abom: AgentBillOfMaterials, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub(crate) struct AgentBillOfMaterials { + pub(crate) agent_version: String, + pub(crate) agent_harness_id: String, + pub(crate) running_location: String, +} + +#[derive(Debug, Serialize)] +struct RegisterAgentRequest { + abom: AgentBillOfMaterials, + agent_public_key: String, + capabilities: Vec, +} + +#[derive(Debug, Deserialize)] +struct RegisterAgentResponse { + agent_runtime_id: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct AgentIdentityBinding { + binding_id: String, + chatgpt_account_id: String, + chatgpt_user_id: Option, + access_token: String, +} + +struct GeneratedAgentKeyMaterial { + private_key_pkcs8_base64: String, + public_key_ssh: String, +} + +impl AgentIdentityManager { + pub(crate) fn new( + config: &Config, + auth_manager: Arc, + session_source: SessionSource, + ) -> Self { + Self { + auth_manager, + chatgpt_base_url: config.chatgpt_base_url.clone(), + feature_enabled: config.features.enabled(Feature::UseAgentIdentity), + abom: build_abom(session_source), + ensure_lock: Arc::new(Mutex::new(())), + } + } + + pub(crate) fn is_enabled(&self) -> bool { + self.feature_enabled + } + + pub(crate) async fn ensure_registered_identity(&self) -> Result> { + if !self.feature_enabled { + return Ok(None); + } + + let Some(auth) = self.auth_manager.auth().await else { + debug!("skipping agent identity registration because no auth is available"); + return Ok(None); + }; + + let Some(binding) = + AgentIdentityBinding::from_auth(&auth, self.auth_manager.forced_chatgpt_workspace_id()) + else { + debug!("skipping agent identity registration because ChatGPT auth is unavailable"); + return Ok(None); + }; + + let _guard = self.ensure_lock.lock().await; + + if let Some(stored_identity) = self.load_stored_identity(&auth, &binding)? { + info!( + agent_runtime_id = %stored_identity.agent_runtime_id, + binding_id = %binding.binding_id, + "reusing stored agent identity" + ); + return Ok(Some(stored_identity)); + } + + let stored_identity = self.register_agent_identity(&binding).await?; + self.store_identity(&auth, &stored_identity)?; + Ok(Some(stored_identity)) + } + + async fn register_agent_identity( + &self, + binding: &AgentIdentityBinding, + ) -> Result { + let key_material = generate_agent_key_material()?; + let request_body = RegisterAgentRequest { + abom: self.abom.clone(), + agent_public_key: key_material.public_key_ssh.clone(), + capabilities: Vec::new(), + }; + + let url = agent_registration_url(&self.chatgpt_base_url); + let human_biscuit = self.mint_human_biscuit(binding, "POST", &url).await?; + let client = create_client(); + let response = client + .post(&url) + .header("X-OpenAI-Authorization", human_biscuit) + .json(&request_body) + .timeout(AGENT_REGISTRATION_TIMEOUT) + .send() + .await + .with_context(|| { + format!("failed to send agent identity registration request to {url}") + })?; + + if response.status().is_success() { + let response_body = response + .json::() + .await + .with_context(|| format!("failed to parse agent identity response from {url}"))?; + let stored_identity = StoredAgentIdentity { + binding_id: binding.binding_id.clone(), + chatgpt_account_id: binding.chatgpt_account_id.clone(), + chatgpt_user_id: binding.chatgpt_user_id.clone(), + agent_runtime_id: response_body.agent_runtime_id, + private_key_pkcs8_base64: key_material.private_key_pkcs8_base64, + public_key_ssh: key_material.public_key_ssh, + registered_at: Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), + abom: self.abom.clone(), + }; + info!( + agent_runtime_id = %stored_identity.agent_runtime_id, + binding_id = %binding.binding_id, + "registered agent identity" + ); + return Ok(stored_identity); + } + + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("agent identity registration failed with status {status} from {url}: {body}") + } + + async fn mint_human_biscuit( + &self, + binding: &AgentIdentityBinding, + target_method: &str, + target_url: &str, + ) -> Result { + let url = agent_identity_biscuit_url(&self.chatgpt_base_url); + let request_id = agent_identity_request_id()?; + let client = create_client(); + let response = client + .get(&url) + .bearer_auth(&binding.access_token) + .header("X-Request-Id", request_id.clone()) + .header("X-Original-Method", target_method) + .header("X-Original-Url", target_url) + .timeout(AGENT_IDENTITY_BISCUIT_TIMEOUT) + .send() + .await + .with_context(|| format!("failed to send agent identity biscuit request to {url}"))?; + + if response.status().is_success() { + let human_biscuit = response + .headers() + .get("x-openai-authorization") + .context("agent identity biscuit response did not include x-openai-authorization")? + .to_str() + .context("agent identity biscuit response header was not valid UTF-8")? + .to_string(); + info!( + request_id = %request_id, + "minted human biscuit for agent identity registration" + ); + return Ok(human_biscuit); + } + + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + anyhow::bail!( + "agent identity biscuit minting failed with status {status} from {url}: {body}" + ) + } + + fn load_stored_identity( + &self, + auth: &CodexAuth, + binding: &AgentIdentityBinding, + ) -> Result> { + let Some(record) = auth.get_agent_identity(&binding.chatgpt_account_id) else { + return Ok(None); + }; + + let stored_identity = + match StoredAgentIdentity::from_auth_record(binding, record, self.abom.clone()) { + Ok(stored_identity) => stored_identity, + Err(error) => { + warn!( + binding_id = %binding.binding_id, + error = %error, + "stored agent identity is invalid; deleting cached value" + ); + auth.remove_agent_identity()?; + return Ok(None); + } + }; + + if !stored_identity.matches_binding(binding) { + warn!( + binding_id = %binding.binding_id, + "stored agent identity binding no longer matches current auth; deleting cached value" + ); + auth.remove_agent_identity()?; + return Ok(None); + } + + if let Err(error) = stored_identity.validate_key_material() { + warn!( + agent_runtime_id = %stored_identity.agent_runtime_id, + binding_id = %binding.binding_id, + error = %error, + "stored agent identity key material is invalid; deleting cached value" + ); + auth.remove_agent_identity()?; + return Ok(None); + } + + Ok(Some(stored_identity)) + } + + fn store_identity( + &self, + auth: &CodexAuth, + stored_identity: &StoredAgentIdentity, + ) -> Result<()> { + auth.set_agent_identity(stored_identity.to_auth_record())?; + Ok(()) + } + + #[cfg(test)] + fn new_for_tests( + auth_manager: Arc, + feature_enabled: bool, + chatgpt_base_url: String, + session_source: SessionSource, + ) -> Self { + Self { + auth_manager, + chatgpt_base_url, + feature_enabled, + abom: build_abom(session_source), + ensure_lock: Arc::new(Mutex::new(())), + } + } +} + +impl StoredAgentIdentity { + fn from_auth_record( + binding: &AgentIdentityBinding, + record: AgentIdentityAuthRecord, + abom: AgentBillOfMaterials, + ) -> Result { + if record.workspace_id != binding.chatgpt_account_id { + anyhow::bail!( + "stored agent identity workspace {:?} does not match current workspace {:?}", + record.workspace_id, + binding.chatgpt_account_id + ); + } + let signing_key = signing_key_from_private_key_pkcs8_base64(&record.agent_private_key)?; + Ok(Self { + binding_id: binding.binding_id.clone(), + chatgpt_account_id: binding.chatgpt_account_id.clone(), + chatgpt_user_id: record.chatgpt_user_id, + agent_runtime_id: record.agent_runtime_id, + private_key_pkcs8_base64: record.agent_private_key, + public_key_ssh: encode_ssh_ed25519_public_key(&signing_key.verifying_key()), + registered_at: record.registered_at, + abom, + }) + } + + fn to_auth_record(&self) -> AgentIdentityAuthRecord { + AgentIdentityAuthRecord { + workspace_id: self.chatgpt_account_id.clone(), + chatgpt_user_id: self.chatgpt_user_id.clone(), + agent_runtime_id: self.agent_runtime_id.clone(), + agent_private_key: self.private_key_pkcs8_base64.clone(), + registered_at: self.registered_at.clone(), + } + } + + fn matches_binding(&self, binding: &AgentIdentityBinding) -> bool { + self.binding_id == binding.binding_id + && self.chatgpt_account_id == binding.chatgpt_account_id + && match binding.chatgpt_user_id.as_deref() { + Some(chatgpt_user_id) => self.chatgpt_user_id.as_deref() == Some(chatgpt_user_id), + None => true, + } + } + + fn validate_key_material(&self) -> Result<()> { + let signing_key = self.signing_key()?; + let derived_public_key = encode_ssh_ed25519_public_key(&signing_key.verifying_key()); + anyhow::ensure!( + self.public_key_ssh == derived_public_key, + "stored public key does not match the private key" + ); + Ok(()) + } + + pub(crate) fn signing_key(&self) -> Result { + signing_key_from_private_key_pkcs8_base64(&self.private_key_pkcs8_base64) + } +} + +impl AgentIdentityBinding { + fn from_auth(auth: &CodexAuth, forced_workspace_id: Option) -> Option { + if !auth.is_chatgpt_auth() { + return None; + } + + let token_data = auth.get_token_data().ok()?; + let resolved_account_id = + forced_workspace_id + .filter(|value| !value.is_empty()) + .or(token_data + .account_id + .clone() + .filter(|value| !value.is_empty()))?; + + Some(Self { + binding_id: format!("chatgpt-account-{resolved_account_id}"), + chatgpt_account_id: resolved_account_id, + chatgpt_user_id: token_data + .id_token + .chatgpt_user_id + .filter(|value| !value.is_empty()), + access_token: token_data.access_token, + }) + } +} + +fn build_abom(session_source: SessionSource) -> AgentBillOfMaterials { + AgentBillOfMaterials { + agent_version: env!("CARGO_PKG_VERSION").to_string(), + agent_harness_id: match &session_source { + SessionSource::VSCode => "codex-app".to_string(), + SessionSource::Cli + | SessionSource::Exec + | SessionSource::Mcp + | SessionSource::Custom(_) + | SessionSource::SubAgent(_) + | SessionSource::Unknown => "codex-cli".to_string(), + }, + running_location: format!("{}-{}", session_source, std::env::consts::OS), + } +} + +fn generate_agent_key_material() -> Result { + let mut secret_key_bytes = [0u8; 32]; + OsRng + .try_fill_bytes(&mut secret_key_bytes) + .context("failed to generate agent identity private key bytes")?; + let signing_key = SigningKey::from_bytes(&secret_key_bytes); + let private_key_pkcs8 = signing_key + .to_pkcs8_der() + .context("failed to encode agent identity private key as PKCS#8")?; + + Ok(GeneratedAgentKeyMaterial { + private_key_pkcs8_base64: BASE64_STANDARD.encode(private_key_pkcs8.as_bytes()), + public_key_ssh: encode_ssh_ed25519_public_key(&signing_key.verifying_key()), + }) +} + +fn encode_ssh_ed25519_public_key(verifying_key: &VerifyingKey) -> String { + let mut blob = Vec::with_capacity(4 + 11 + 4 + 32); + append_ssh_string(&mut blob, b"ssh-ed25519"); + append_ssh_string(&mut blob, verifying_key.as_bytes()); + format!("ssh-ed25519 {}", BASE64_STANDARD.encode(blob)) +} + +fn append_ssh_string(buf: &mut Vec, value: &[u8]) { + buf.extend_from_slice(&(value.len() as u32).to_be_bytes()); + buf.extend_from_slice(value); +} + +fn agent_registration_url(chatgpt_base_url: &str) -> String { + let trimmed = chatgpt_base_url.trim_end_matches('/'); + format!("{trimmed}/v1/agent/register") +} + +fn signing_key_from_private_key_pkcs8_base64(private_key_pkcs8_base64: &str) -> Result { + let private_key = BASE64_STANDARD + .decode(private_key_pkcs8_base64) + .context("stored agent identity private key is not valid base64")?; + SigningKey::from_pkcs8_der(&private_key) + .context("stored agent identity private key is not valid PKCS#8") +} + +fn agent_identity_biscuit_url(chatgpt_base_url: &str) -> String { + let trimmed = chatgpt_base_url.trim_end_matches('/'); + format!("{trimmed}/authenticate_app_v2") +} + +fn agent_identity_request_id() -> Result { + let mut request_id_bytes = [0u8; 16]; + OsRng + .try_fill_bytes(&mut request_id_bytes) + .context("failed to generate agent identity request id")?; + Ok(format!( + "codex-agent-identity-{}", + URL_SAFE_NO_PAD.encode(request_id_bytes) + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + use base64::engine::general_purpose::URL_SAFE_NO_PAD; + use codex_app_server_protocol::AuthMode as ApiAuthMode; + use codex_login::AuthCredentialsStoreMode; + use codex_login::AuthDotJson; + use codex_login::save_auth; + use codex_login::token_data::IdTokenInfo; + use codex_login::token_data::TokenData; + use pretty_assertions::assert_eq; + use wiremock::Mock; + use wiremock::MockServer; + use wiremock::ResponseTemplate; + use wiremock::matchers::header; + use wiremock::matchers::method; + use wiremock::matchers::path; + + #[tokio::test] + async fn ensure_registered_identity_skips_when_feature_is_disabled() { + let auth_manager = + AuthManager::from_auth_for_testing(make_chatgpt_auth("account-123", Some("user-123"))); + let manager = AgentIdentityManager::new_for_tests( + auth_manager, + /*feature_enabled*/ false, + "https://chatgpt.com/backend-api/".to_string(), + SessionSource::Cli, + ); + + assert_eq!(manager.ensure_registered_identity().await.unwrap(), None); + } + + #[tokio::test] + async fn ensure_registered_identity_skips_for_api_key_auth() { + let auth_manager = AuthManager::from_auth_for_testing(CodexAuth::from_api_key("test-key")); + let manager = AgentIdentityManager::new_for_tests( + auth_manager, + /*feature_enabled*/ true, + "https://chatgpt.com/backend-api/".to_string(), + SessionSource::Cli, + ); + + assert_eq!(manager.ensure_registered_identity().await.unwrap(), None); + } + + #[tokio::test] + async fn ensure_registered_identity_registers_and_reuses_cached_identity() { + let server = MockServer::start().await; + let chatgpt_base_url = server.uri(); + mount_human_biscuit(&server, &chatgpt_base_url).await; + Mock::given(method("POST")) + .and(path("/v1/agent/register")) + .and(header("x-openai-authorization", "human-biscuit")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "agent_runtime_id": "agent_123", + }))) + .expect(1) + .mount(&server) + .await; + + let auth_manager = + AuthManager::from_auth_for_testing(make_chatgpt_auth("account-123", Some("user-123"))); + let manager = AgentIdentityManager::new_for_tests( + auth_manager, + /*feature_enabled*/ true, + chatgpt_base_url, + SessionSource::Cli, + ); + + let first = manager + .ensure_registered_identity() + .await + .unwrap() + .expect("identity should be registered"); + let second = manager + .ensure_registered_identity() + .await + .unwrap() + .expect("identity should be reused"); + + assert_eq!(first.agent_runtime_id, "agent_123"); + assert_eq!(first, second); + assert_eq!(first.abom.agent_harness_id, "codex-cli"); + assert_eq!(first.chatgpt_account_id, "account-123"); + assert_eq!(first.chatgpt_user_id.as_deref(), Some("user-123")); + } + + #[tokio::test] + async fn ensure_registered_identity_deletes_invalid_cached_identity_and_reregisters() { + let server = MockServer::start().await; + let chatgpt_base_url = server.uri(); + mount_human_biscuit(&server, &chatgpt_base_url).await; + Mock::given(method("POST")) + .and(path("/v1/agent/register")) + .and(header("x-openai-authorization", "human-biscuit")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "agent_runtime_id": "agent_456", + }))) + .expect(1) + .mount(&server) + .await; + + let auth = make_chatgpt_auth("account-123", Some("user-123")); + let auth_manager = AuthManager::from_auth_for_testing(auth.clone()); + let manager = AgentIdentityManager::new_for_tests( + auth_manager, + /*feature_enabled*/ true, + chatgpt_base_url, + SessionSource::Cli, + ); + + let binding = + AgentIdentityBinding::from_auth(&auth, /*forced_workspace_id*/ None).expect("binding"); + auth.set_agent_identity(AgentIdentityAuthRecord { + workspace_id: "account-123".to_string(), + chatgpt_user_id: Some("user-123".to_string()), + agent_runtime_id: "agent_invalid".to_string(), + agent_private_key: "not-valid-base64".to_string(), + registered_at: "2026-01-01T00:00:00Z".to_string(), + }) + .expect("seed invalid identity"); + + let stored = manager + .ensure_registered_identity() + .await + .unwrap() + .expect("identity should be registered"); + + assert_eq!(stored.agent_runtime_id, "agent_456"); + let persisted = auth + .get_agent_identity(&binding.chatgpt_account_id) + .expect("stored identity"); + assert_eq!(persisted.agent_runtime_id, "agent_456"); + } + + #[tokio::test] + async fn ensure_registered_identity_deletes_different_user_identity_and_reregisters() { + let server = MockServer::start().await; + let chatgpt_base_url = server.uri(); + mount_human_biscuit(&server, &chatgpt_base_url).await; + Mock::given(method("POST")) + .and(path("/v1/agent/register")) + .and(header("x-openai-authorization", "human-biscuit")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "agent_runtime_id": "agent_new", + }))) + .expect(1) + .mount(&server) + .await; + + let auth = make_chatgpt_auth("account-123", Some("user-new")); + let stale_key = generate_agent_key_material().expect("key material"); + auth.set_agent_identity(AgentIdentityAuthRecord { + workspace_id: "account-123".to_string(), + chatgpt_user_id: Some("user-old".to_string()), + agent_runtime_id: "agent_old".to_string(), + agent_private_key: stale_key.private_key_pkcs8_base64, + registered_at: "2026-01-01T00:00:00Z".to_string(), + }) + .expect("seed stale identity"); + + let auth_manager = AuthManager::from_auth_for_testing(auth.clone()); + let manager = AgentIdentityManager::new_for_tests( + auth_manager, + /*feature_enabled*/ true, + chatgpt_base_url, + SessionSource::Cli, + ); + + let stored = manager + .ensure_registered_identity() + .await + .unwrap() + .expect("identity should be registered"); + + assert_eq!(stored.agent_runtime_id, "agent_new"); + assert_eq!(stored.chatgpt_user_id.as_deref(), Some("user-new")); + let persisted = auth + .get_agent_identity("account-123") + .expect("stored identity"); + assert_eq!(persisted.agent_runtime_id, "agent_new"); + assert_eq!(persisted.chatgpt_user_id.as_deref(), Some("user-new")); + } + + #[tokio::test] + async fn ensure_registered_identity_uses_chatgpt_base_url() { + let server = MockServer::start().await; + let chatgpt_base_url = format!("{}/backend-api", server.uri()); + mount_human_biscuit(&server, &chatgpt_base_url).await; + Mock::given(method("POST")) + .and(path("/backend-api/v1/agent/register")) + .and(header("x-openai-authorization", "human-biscuit")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "agent_runtime_id": "agent_canonical", + }))) + .expect(1) + .mount(&server) + .await; + + let auth_manager = + AuthManager::from_auth_for_testing(make_chatgpt_auth("account-123", Some("user-123"))); + let manager = AgentIdentityManager::new_for_tests( + auth_manager, + /*feature_enabled*/ true, + chatgpt_base_url, + SessionSource::Cli, + ); + + let stored = manager + .ensure_registered_identity() + .await + .unwrap() + .expect("identity should be registered"); + assert_eq!(stored.agent_runtime_id, "agent_canonical"); + } + + async fn mount_human_biscuit(server: &MockServer, chatgpt_base_url: &str) { + let biscuit_url = agent_identity_biscuit_url(chatgpt_base_url); + let biscuit_path = reqwest::Url::parse(&biscuit_url) + .expect("biscuit URL parses") + .path() + .to_string(); + let target_url = agent_registration_url(chatgpt_base_url); + Mock::given(method("GET")) + .and(path(biscuit_path)) + .and(header("authorization", "Bearer access-token-account-123")) + .and(header("x-original-method", "POST")) + .and(header("x-original-url", target_url)) + .respond_with( + ResponseTemplate::new(200).insert_header("x-openai-authorization", "human-biscuit"), + ) + .expect(1) + .mount(server) + .await; + } + + #[test] + fn encode_ssh_ed25519_public_key_matches_expected_wire_shape() { + let key_material = generate_agent_key_material().expect("key material"); + let (_, encoded_blob) = key_material + .public_key_ssh + .split_once(' ') + .expect("public key contains scheme"); + let decoded = BASE64_STANDARD.decode(encoded_blob).expect("base64"); + + assert_eq!(&decoded[..4], 11u32.to_be_bytes().as_slice()); + assert_eq!(&decoded[4..15], b"ssh-ed25519"); + assert_eq!(&decoded[15..19], 32u32.to_be_bytes().as_slice()); + assert_eq!(decoded.len(), 51); + } + + fn make_chatgpt_auth(account_id: &str, user_id: Option<&str>) -> CodexAuth { + let tempdir = tempfile::tempdir().expect("tempdir"); + let auth_json = AuthDotJson { + auth_mode: Some(ApiAuthMode::Chatgpt), + openai_api_key: None, + tokens: Some(TokenData { + id_token: IdTokenInfo { + email: None, + chatgpt_plan_type: None, + chatgpt_user_id: user_id.map(ToOwned::to_owned), + chatgpt_account_id: Some(account_id.to_string()), + raw_jwt: fake_id_token(account_id, user_id), + }, + access_token: format!("access-token-{account_id}"), + refresh_token: "refresh-token".to_string(), + account_id: Some(account_id.to_string()), + }), + last_refresh: Some(Utc::now()), + agent_identity: None, + }; + save_auth(tempdir.path(), &auth_json, AuthCredentialsStoreMode::File).expect("save auth"); + CodexAuth::from_auth_storage(tempdir.path(), AuthCredentialsStoreMode::File) + .expect("load auth") + .expect("auth") + } + + fn fake_id_token(account_id: &str, user_id: Option<&str>) -> String { + let header = URL_SAFE_NO_PAD.encode(r#"{"alg":"none","typ":"JWT"}"#); + let payload = serde_json::json!({ + "https://api.openai.com/auth": { + "chatgpt_user_id": user_id, + "chatgpt_account_id": account_id, + } + }); + let payload = URL_SAFE_NO_PAD.encode(payload.to_string()); + format!("{header}.{payload}.signature") + } +} diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 16f743943a..bd83c81f0d 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -32,6 +32,7 @@ use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; use codex_api::ApiError; +use codex_api::AuthProvider; use codex_api::CompactClient as ApiCompactClient; use codex_api::CompactionInput as ApiCompactionInput; use codex_api::Compression; @@ -83,7 +84,6 @@ use futures::StreamExt; use http::HeaderMap as ApiHeaderMap; use http::HeaderValue; use http::StatusCode as HttpStatusCode; -use http::header::AUTHORIZATION; use reqwest::StatusCode; use std::time::Duration; use std::time::Instant; @@ -277,16 +277,7 @@ pub(crate) struct RealtimeWebrtcCallStart { /// `api.openai.com` sideband path. fn sideband_websocket_auth_headers(api_auth: &CoreAuthProvider) -> ApiHeaderMap { let mut headers = ApiHeaderMap::new(); - if let Some(token) = api_auth.token.as_ref() - && let Ok(value) = HeaderValue::from_str(&format!("Bearer {token}")) - { - headers.insert(AUTHORIZATION, value); - } - if let Some(account_id) = api_auth.account_id.as_ref() - && let Ok(value) = HeaderValue::from_str(account_id) - { - headers.insert("ChatGPT-Account-ID", value); - } + api_auth.add_auth_headers(&mut headers); headers } diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 95e932b1dc..cc314a66e9 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -13,6 +13,7 @@ use crate::agent::Mailbox; use crate::agent::MailboxReceiver; use crate::agent::agent_status_from_event; use crate::agent::status::is_final; +use crate::agent_identity::AgentIdentityManager; use crate::apps::render_apps_section; use crate::commit_attribution::commit_message_trailer_instruction; use crate::compact; @@ -77,7 +78,6 @@ use codex_login::CodexAuth; use codex_login::auth_env_telemetry::collect_auth_env_telemetry; use codex_login::default_client::originator; use codex_mcp::McpConnectionManager; -use codex_mcp::SandboxState; use codex_mcp::ToolInfo; use codex_mcp::codex_apps_tools_cache_key; #[cfg(test)] @@ -91,6 +91,7 @@ use codex_otel::current_span_trace_id; use codex_otel::current_span_w3c_trace_context; use codex_otel::set_parent_from_w3c_trace_context; use codex_protocol::ThreadId; +use codex_protocol::ToolName; use codex_protocol::approvals::ElicitationRequestEvent; use codex_protocol::approvals::ExecPolicyAmendment; use codex_protocol::approvals::NetworkPolicyAmendment; @@ -134,9 +135,11 @@ use codex_protocol::request_permissions::RequestPermissionsResponse; use codex_protocol::request_user_input::RequestUserInputArgs; use codex_protocol::request_user_input::RequestUserInputResponse; use codex_rmcp_client::ElicitationResponse; +use codex_rollout::RolloutConfig; use codex_rollout::state_db; use codex_shell_command::parse_command::parse_command; use codex_terminal_detection::user_agent; +use codex_thread_store::LocalThreadStore; use codex_tools::filter_tool_suggest_discoverable_tools_for_client; use codex_utils_output_truncation::TruncationPolicy; use codex_utils_stream_parser::AssistantTextChunk; @@ -1509,6 +1512,56 @@ impl Session { }); } + fn start_agent_identity_registration(self: &Arc) { + if !self.services.agent_identity_manager.is_enabled() { + return; + } + + let weak_sess = Arc::downgrade(self); + let mut auth_state_rx = self.services.auth_manager.subscribe_auth_state(); + tokio::spawn(async move { + loop { + let Some(sess) = weak_sess.upgrade() else { + return; + }; + match sess + .services + .agent_identity_manager + .ensure_registered_identity() + .await + { + Ok(Some(_)) => return, + Ok(None) => { + drop(sess); + if auth_state_rx.changed().await.is_err() { + return; + } + } + Err(error) => { + sess.fail_agent_identity_registration(error).await; + return; + } + } + } + }); + } + + async fn fail_agent_identity_registration(self: &Arc, error: anyhow::Error) { + warn!(error = %error, "agent identity registration failed"); + let message = format!( + "Agent identity registration failed. Codex cannot continue while `features.use_agent_identity` is enabled: {error}" + ); + self.send_event_raw(Event { + id: self.next_internal_sub_id(), + msg: EventMsg::Error(ErrorEvent { + message, + codex_error_info: Some(CodexErrorInfo::Other), + }), + }) + .await; + handlers::shutdown(self, self.next_internal_sub_id()).await; + } + #[allow(clippy::too_many_arguments)] fn make_turn_context( conversation_id: ThreadId, @@ -2054,6 +2107,11 @@ impl Session { hooks, rollout: Mutex::new(rollout_recorder), user_shell: Arc::new(default_shell), + agent_identity_manager: Arc::new(AgentIdentityManager::new( + config.as_ref(), + Arc::clone(&auth_manager), + session_configuration.session_source.clone(), + )), shell_snapshot_tx, show_raw_agent_reasoning: config.show_raw_agent_reasoning, exec_policy, @@ -2070,6 +2128,7 @@ impl Session { network_proxy, network_approval: Arc::clone(&network_approval), state_db: state_db_ctx.clone(), + thread_store: LocalThreadStore::new(RolloutConfig::from_view(config.as_ref())), model_client: ModelClient::new( Some(Arc::clone(&auth_manager)), conversation_id, @@ -2151,14 +2210,7 @@ impl Session { // Start the watcher after SessionConfigured so it cannot emit earlier events. sess.start_skills_watcher_listener(); - // Construct sandbox_state before MCP startup so it can be sent to each - // MCP server immediately after it becomes ready (avoiding blocking). - let sandbox_state = SandboxState { - sandbox_policy: session_configuration.sandbox_policy.get().clone(), - codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(), - sandbox_cwd: session_configuration.cwd.to_path_buf(), - use_legacy_landlock: config.features.use_legacy_landlock(), - }; + sess.start_agent_identity_registration(); let mut required_mcp_servers: Vec = mcp_servers .iter() .filter(|(_, server)| server.enabled && server.required) @@ -2180,7 +2232,7 @@ impl Session { &session_configuration.approval_policy, INITIAL_SUBMIT_ID.to_owned(), tx_event.clone(), - sandbox_state, + session_configuration.sandbox_policy.get().clone(), config.codex_home.to_path_buf(), codex_apps_tools_cache_key(auth), tool_plugin_provenance, @@ -2612,12 +2664,16 @@ impl Session { &session_source, ); + if sandbox_policy_changed { + self.refresh_managed_network_proxy_for_current_sandbox_policy() + .await; + } + Ok(self .new_turn_from_configuration( sub_id, session_configuration, updates.final_output_json_schema, - sandbox_policy_changed, ) .await) } @@ -2627,7 +2683,6 @@ impl Session { sub_id: String, session_configuration: SessionConfiguration, final_output_json_schema: Option>, - sandbox_policy_changed: bool, ) -> Arc { let per_turn_config = Self::build_per_turn_config(&session_configuration); { @@ -2637,27 +2692,6 @@ impl Session { .set_sandbox_policy(per_turn_config.permissions.sandbox_policy.get()); } - if sandbox_policy_changed { - self.refresh_managed_network_proxy_for_current_sandbox_policy() - .await; - let sandbox_state = SandboxState { - sandbox_policy: per_turn_config.permissions.sandbox_policy.get().clone(), - codex_linux_sandbox_exe: per_turn_config.codex_linux_sandbox_exe.clone(), - sandbox_cwd: per_turn_config.cwd.to_path_buf(), - use_legacy_landlock: per_turn_config.features.use_legacy_landlock(), - }; - if let Err(e) = self - .services - .mcp_connection_manager - .read() - .await - .notify_sandbox_state_change(&sandbox_state) - .await - { - warn!("Failed to notify sandbox state change to MCP servers: {e:#}"); - } - } - let model_info = self .services .models_manager @@ -2806,7 +2840,6 @@ impl Session { sub_id, session_configuration, /*final_output_json_schema*/ None, - /*sandbox_policy_changed*/ false, ) .await } @@ -4453,16 +4486,12 @@ impl Session { .await } - pub(crate) async fn resolve_mcp_tool_info( - &self, - name: &str, - namespace: Option<&str>, - ) -> Option { + pub(crate) async fn resolve_mcp_tool_info(&self, tool_name: &ToolName) -> Option { self.services .mcp_connection_manager .read() .await - .resolve_tool_info(name, namespace) + .resolve_tool_info(tool_name) .await } @@ -4522,12 +4551,6 @@ impl Session { .await; let mcp_servers = with_codex_apps_mcp(mcp_servers, auth.as_ref(), &mcp_config); let auth_statuses = compute_auth_statuses(mcp_servers.iter(), store_mode).await; - let sandbox_state = SandboxState { - sandbox_policy: turn_context.sandbox_policy.get().clone(), - codex_linux_sandbox_exe: turn_context.codex_linux_sandbox_exe.clone(), - sandbox_cwd: turn_context.cwd.to_path_buf(), - use_legacy_landlock: turn_context.features.use_legacy_landlock(), - }; { let mut guard = self.services.mcp_startup_cancellation_token.lock().await; guard.cancel(); @@ -4540,7 +4563,7 @@ impl Session { &turn_context.config.permissions.approval_policy, turn_context.sub_id.clone(), self.get_tx_event(), - sandbox_state, + turn_context.sandbox_policy.get().clone(), config.codex_home.to_path_buf(), codex_apps_tools_cache_key(auth.as_ref()), tool_plugin_provenance, diff --git a/codex-rs/core/src/codex_tests.rs b/codex-rs/core/src/codex_tests.rs index 6de8909e6d..eaa2099098 100644 --- a/codex-rs/core/src/codex_tests.rs +++ b/codex-rs/core/src/codex_tests.rs @@ -110,6 +110,7 @@ use opentelemetry::trace::TraceId; use std::path::Path; use std::time::Duration; use tokio::time::sleep; +use tokio::time::timeout; use tracing_opentelemetry::OpenTelemetrySpanExt; use codex_protocol::mcp::CallToolResult as McpCallToolResult; @@ -655,6 +656,89 @@ async fn managed_network_proxy_decider_survives_full_access_start() -> anyhow::R Ok(()) } +#[tokio::test] +async fn new_turn_refreshes_managed_network_proxy_for_sandbox_change() -> anyhow::Result<()> { + let (mut session, _turn_context) = make_session_and_context().await; + let initial_policy = SandboxPolicy::new_workspace_write_policy(); + + let mut network_config = NetworkProxyConfig::default(); + network_config + .network + .set_allowed_domains(vec!["evil.com".to_string()]); + let requirements = NetworkConstraints { + domains: Some(NetworkDomainPermissionsToml { + entries: std::collections::BTreeMap::from([( + "*.example.com".to_string(), + NetworkDomainPermissionToml::Allow, + )]), + }), + ..Default::default() + }; + let spec = crate::config::NetworkProxySpec::from_config_and_constraints( + network_config, + Some(requirements), + &initial_policy, + )?; + let (started_proxy, _) = Session::start_managed_network_proxy( + &spec, + &Policy::empty(), + &initial_policy, + /*network_policy_decider*/ None, + /*blocked_request_observer*/ None, + /*managed_network_requirements_enabled*/ false, + crate::config::NetworkProxyAuditMetadata::default(), + ) + .await?; + assert_eq!( + started_proxy + .proxy() + .current_cfg() + .await? + .network + .allowed_domains(), + Some(vec!["*.example.com".to_string(), "evil.com".to_string()]) + ); + + { + let mut state = session.state.lock().await; + let mut config = (*state.session_configuration.original_config_do_not_use).clone(); + config.permissions.network = Some(spec); + config.permissions.sandbox_policy = + codex_config::Constrained::allow_any(initial_policy.clone()); + state.session_configuration.original_config_do_not_use = Arc::new(config); + state.session_configuration.sandbox_policy = + codex_config::Constrained::allow_any(initial_policy); + } + session.services.network_proxy = Some(started_proxy); + + session + .new_turn_with_sub_id( + "sandbox-policy-change".to_string(), + SessionSettingsUpdate { + sandbox_policy: Some(SandboxPolicy::DangerFullAccess), + ..Default::default() + }, + ) + .await?; + + let started_proxy = session + .services + .network_proxy + .as_ref() + .expect("managed network proxy should be present"); + assert_eq!( + started_proxy + .proxy() + .current_cfg() + .await? + .network + .allowed_domains(), + Some(vec!["*.example.com".to_string()]) + ); + + Ok(()) +} + #[tokio::test] async fn get_base_instructions_no_user_content() { let prompt_with_apply_patch_instructions = @@ -904,7 +988,7 @@ fn mcp_tool_exposure_searches_large_effective_tool_sets() { } #[test] -fn mcp_tool_exposure_directly_exposes_explicit_apps_in_large_search_sets() { +fn mcp_tool_exposure_directly_exposes_explicit_apps_without_deferred_overlap() { let config = test_config(); let tools_config = tools_config_for_mcp_tool_exposure(/*search_tool*/ true); let mut mcp_tools = numbered_mcp_tools(DIRECT_MCP_TOOL_EXPOSURE_THRESHOLD - 1); @@ -935,13 +1019,19 @@ fn mcp_tool_exposure_directly_exposes_explicit_apps_in_large_search_sets() { ); assert_eq!( exposure.deferred_tools.as_ref().map(HashMap::len), - Some(DIRECT_MCP_TOOL_EXPOSURE_THRESHOLD) + Some(DIRECT_MCP_TOOL_EXPOSURE_THRESHOLD - 1) ); let deferred_tools = exposure .deferred_tools .as_ref() .expect("large tool sets should be discoverable through tool_search"); - assert!(deferred_tools.contains_key("mcp__codex_apps__calendar_create_event")); + assert!( + tool_names + .iter() + .all(|direct_tool_name| !deferred_tools.contains_key(direct_tool_name)), + "direct tools should not also be deferred: {tool_names:?}" + ); + assert!(!deferred_tools.contains_key("mcp__codex_apps__calendar_create_event")); assert!(deferred_tools.contains_key("mcp__rmcp__tool_0")); } @@ -2808,6 +2898,11 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) { }), rollout: Mutex::new(None), user_shell: Arc::new(default_user_shell()), + agent_identity_manager: Arc::new(crate::agent_identity::AgentIdentityManager::new( + config.as_ref(), + Arc::clone(&auth_manager), + session_configuration.session_source.clone(), + )), shell_snapshot_tx: watch::channel(None).0, show_raw_agent_reasoning: config.show_raw_agent_reasoning, exec_policy, @@ -2824,6 +2919,9 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) { network_proxy: None, network_approval: Arc::clone(&network_approval), state_db: None, + thread_store: codex_thread_store::LocalThreadStore::new( + codex_rollout::RolloutConfig::from_view(config.as_ref()), + ), model_client: ModelClient::new( Some(auth_manager.clone()), conversation_id, @@ -3660,6 +3758,11 @@ pub(crate) async fn make_session_and_context_with_dynamic_tools_and_rx( }), rollout: Mutex::new(None), user_shell: Arc::new(default_user_shell()), + agent_identity_manager: Arc::new(crate::agent_identity::AgentIdentityManager::new( + config.as_ref(), + Arc::clone(&auth_manager), + session_configuration.session_source.clone(), + )), shell_snapshot_tx: watch::channel(None).0, show_raw_agent_reasoning: config.show_raw_agent_reasoning, exec_policy, @@ -3676,6 +3779,9 @@ pub(crate) async fn make_session_and_context_with_dynamic_tools_and_rx( network_proxy: None, network_approval: Arc::clone(&network_approval), state_db: None, + thread_store: codex_thread_store::LocalThreadStore::new( + codex_rollout::RolloutConfig::from_view(config.as_ref()), + ), model_client: ModelClient::new( Some(Arc::clone(&auth_manager)), conversation_id, @@ -3764,6 +3870,42 @@ pub(crate) async fn make_session_and_context_with_rx() -> ( make_session_and_context_with_dynamic_tools_and_rx(Vec::new()).await } +#[tokio::test] +async fn fail_agent_identity_registration_emits_error_and_shutdown() { + let (session, _turn_context, rx_event) = make_session_and_context_with_rx().await; + + session + .fail_agent_identity_registration(anyhow::anyhow!("registration exploded")) + .await; + + let error_event = timeout(Duration::from_secs(1), rx_event.recv()) + .await + .expect("error event should arrive") + .expect("error event should be readable"); + match error_event.msg { + EventMsg::Error(ErrorEvent { + message, + codex_error_info, + }) => { + assert_eq!( + message, + "Agent identity registration failed. Codex cannot continue while `features.use_agent_identity` is enabled: registration exploded".to_string() + ); + assert_eq!(codex_error_info, Some(CodexErrorInfo::Other)); + } + other => panic!("expected error event, got {other:?}"), + } + + let shutdown_event = timeout(Duration::from_secs(1), rx_event.recv()) + .await + .expect("shutdown event should arrive") + .expect("shutdown event should be readable"); + match shutdown_event.msg { + EventMsg::ShutdownComplete => {} + other => panic!("expected shutdown event, got {other:?}"), + } +} + #[tokio::test] async fn refresh_mcp_servers_is_deferred_until_next_turn() { let (session, turn_context) = make_session_and_context().await; diff --git a/codex-rs/core/src/compact.rs b/codex-rs/core/src/compact.rs index 59fb45454d..eaf1d92109 100644 --- a/codex-rs/core/src/compact.rs +++ b/codex-rs/core/src/compact.rs @@ -58,7 +58,7 @@ pub(crate) enum InitialContextInjection { } pub(crate) fn should_use_remote_compact_task(provider: &ModelProviderInfo) -> bool { - provider.is_openai() + provider.supports_remote_compaction() } pub(crate) async fn run_inline_auto_compact_task( diff --git a/codex-rs/core/src/compact_tests.rs b/codex-rs/core/src/compact_tests.rs index cecf9ce9d7..f7a8d1dfff 100644 --- a/codex-rs/core/src/compact_tests.rs +++ b/codex-rs/core/src/compact_tests.rs @@ -1,4 +1,6 @@ use super::*; +use codex_model_provider_info::ModelProviderInfo; +use codex_model_provider_info::WireApi; use pretty_assertions::assert_eq; async fn process_compacted_history_with_test_session( @@ -185,6 +187,30 @@ fn build_token_limited_compacted_history_appends_summary_message() { assert_eq!(summary, summary_text); } +#[test] +fn should_use_remote_compact_task_for_azure_provider() { + let provider = ModelProviderInfo { + name: "Azure".into(), + base_url: Some("https://example.com/openai".into()), + env_key: Some("AZURE_OPENAI_API_KEY".into()), + env_key_instructions: None, + experimental_bearer_token: None, + auth: None, + wire_api: WireApi::Responses, + query_params: None, + http_headers: None, + env_http_headers: None, + request_max_retries: None, + stream_max_retries: None, + stream_idle_timeout_ms: None, + websocket_connect_timeout_ms: None, + requires_openai_auth: false, + supports_websockets: false, + }; + + assert!(should_use_remote_compact_task(&provider)); +} + #[tokio::test] async fn process_compacted_history_replaces_developer_messages() { let compacted_history = vec![ diff --git a/codex-rs/core/src/connectors.rs b/codex-rs/core/src/connectors.rs index eef6d5747b..77603adcf1 100644 --- a/codex-rs/core/src/connectors.rs +++ b/codex-rs/core/src/connectors.rs @@ -1,8 +1,6 @@ use std::collections::BTreeSet; use std::collections::HashMap; use std::collections::HashSet; -use std::env; -use std::path::PathBuf; use std::sync::Arc; use std::sync::LazyLock; use std::sync::Mutex as StdMutex; @@ -42,7 +40,6 @@ use codex_login::default_client::is_first_party_chat_originator; use codex_login::default_client::originator; use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; use codex_mcp::McpConnectionManager; -use codex_mcp::SandboxState; use codex_mcp::ToolInfo; use codex_mcp::ToolPluginProvenance; use codex_mcp::codex_apps_tools_cache_key; @@ -228,13 +225,6 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_options_and_status( let (tx_event, rx_event) = unbounded(); drop(rx_event); - let sandbox_state = SandboxState { - sandbox_policy: SandboxPolicy::new_read_only_policy(), - codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(), - sandbox_cwd: env::current_dir().unwrap_or_else(|_| PathBuf::from("/")), - use_legacy_landlock: config.features.use_legacy_landlock(), - }; - let (mcp_connection_manager, cancel_token) = McpConnectionManager::new( &mcp_servers, config.mcp_oauth_credentials_store_mode, @@ -242,7 +232,7 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_options_and_status( &config.permissions.approval_policy, INITIAL_SUBMIT_ID.to_owned(), tx_event, - sandbox_state, + SandboxPolicy::new_read_only_policy(), config.codex_home.to_path_buf(), codex_apps_tools_cache_key(auth.as_ref()), ToolPluginProvenance::default(), diff --git a/codex-rs/core/src/exec.rs b/codex-rs/core/src/exec.rs index d5b96902dc..10a1dc40ff 100644 --- a/codex-rs/core/src/exec.rs +++ b/codex-rs/core/src/exec.rs @@ -251,8 +251,24 @@ pub fn build_exec_request( codex_linux_sandbox_exe: &Option, use_legacy_landlock: bool, ) -> Result { - let windows_sandbox_level = params.windows_sandbox_level; - let enforce_managed_network = params.network.is_some(); + let ExecParams { + command, + cwd, + mut env, + expiration, + capture_policy, + network, + windows_sandbox_level, + windows_sandbox_private_desktop, + + // TODO: Should arg0 be set on the ExecRequest that is returned? + arg0: _, + // These fields are related to approvals, so can be ignored here. + justification: _, + sandbox_permissions: _, + } = params; + + let enforce_managed_network = network.is_some(); let sandbox_type = select_process_exec_tool_sandbox_type( file_system_sandbox_policy, network_sandbox_policy, @@ -261,19 +277,6 @@ pub fn build_exec_request( ); tracing::debug!("Sandbox type: {sandbox_type:?}"); - let ExecParams { - command, - cwd, - mut env, - expiration, - capture_policy, - network, - sandbox_permissions: _, - windows_sandbox_level, - windows_sandbox_private_desktop, - justification: _, - arg0: _, - } = params; if let Some(network) = network.as_ref() { network.apply_to_env(&mut env); } @@ -357,7 +360,8 @@ pub(crate) async fn execute_exec_request( windows_sandbox_level, windows_sandbox_private_desktop, sandbox_policy, - file_system_sandbox_policy, + // TODO(mbolin): Use file_system_sandbox_policy instead of sandbox_policy. + file_system_sandbox_policy: _, network_sandbox_policy, windows_sandbox_filesystem_overrides, arg0, @@ -378,21 +382,40 @@ pub(crate) async fn execute_exec_request( }; let start = Instant::now(); - let raw_output_result = exec( + let raw_output_result = get_raw_output_result( params, - sandbox, - &sandbox_policy, - &file_system_sandbox_policy, - windows_sandbox_filesystem_overrides.as_ref(), network_sandbox_policy, stdout_stream, after_spawn, + sandbox, + &sandbox_policy, + windows_sandbox_filesystem_overrides.as_ref(), ) .await; let duration = start.elapsed(); finalize_exec_result(raw_output_result, sandbox, duration) } +async fn get_raw_output_result( + params: ExecParams, + network_sandbox_policy: NetworkSandboxPolicy, + stdout_stream: Option, + after_spawn: Option>, + #[cfg_attr(not(windows), allow(unused_variables))] sandbox: SandboxType, + #[cfg_attr(not(windows), allow(unused_variables))] sandbox_policy: &SandboxPolicy, + #[cfg_attr(not(windows), allow(unused_variables))] windows_sandbox_filesystem_overrides: Option< + &WindowsSandboxFilesystemOverrides, + >, +) -> Result { + #[cfg(target_os = "windows")] + if sandbox == SandboxType::WindowsRestrictedToken { + return exec_windows_sandbox(params, sandbox_policy, windows_sandbox_filesystem_overrides) + .await; + } + + exec(params, network_sandbox_policy, stdout_stream, after_spawn).await +} + #[cfg(target_os = "windows")] fn extract_create_process_as_user_error_code(err: &str) -> Option { let marker = "CreateProcessAsUserW failed: "; @@ -799,26 +822,24 @@ fn aggregate_output( } } -#[allow(clippy::too_many_arguments)] +/// This is a general-purpose function for executing a command specified by +/// [ExecParams]. Events are reported via `stdout_stream`, if specified, and +/// `after_spawn` is invoked once the child process has been spawned, before +/// output consumption begins. +/// +/// `network_sandbox_policy` is used to determine whether +/// CODEX_SANDBOX_NETWORK_DISABLED=1 is added to the environment of the spawned +/// process. +/// +/// Note this command does not apply any sandboxing logic. The caller is +/// responsible for constructing [ExecParams::command] to include any sandboxing +/// wrapper args, as appropriate. async fn exec( params: ExecParams, - _sandbox: SandboxType, - _sandbox_policy: &SandboxPolicy, - _file_system_sandbox_policy: &FileSystemSandboxPolicy, - _windows_sandbox_filesystem_overrides: Option<&WindowsSandboxFilesystemOverrides>, network_sandbox_policy: NetworkSandboxPolicy, stdout_stream: Option, after_spawn: Option>, ) -> Result { - #[cfg(target_os = "windows")] - if _sandbox == SandboxType::WindowsRestrictedToken { - return exec_windows_sandbox( - params, - _sandbox_policy, - _windows_sandbox_filesystem_overrides, - ) - .await; - } let ExecParams { command, cwd, @@ -827,8 +848,14 @@ async fn exec( arg0, expiration, capture_policy, + + // If applicable, these fields should have been honored upstream of + // this exec call. windows_sandbox_level: _, - .. + windows_sandbox_private_desktop: _, + // These fields are related to approvals, so can be ignored here. + sandbox_permissions: _, + justification: _, } = params; if let Some(network) = network.as_ref() { network.apply_to_env(&mut env); diff --git a/codex-rs/core/src/exec_tests.rs b/codex-rs/core/src/exec_tests.rs index 796062b3fc..acc92f5585 100644 --- a/codex-rs/core/src/exec_tests.rs +++ b/codex-rs/core/src/exec_tests.rs @@ -277,10 +277,6 @@ async fn exec_full_buffer_capture_ignores_expiration() -> Result<()> { justification: None, arg0: None, }, - SandboxType::None, - &SandboxPolicy::DangerFullAccess, - &FileSystemSandboxPolicy::unrestricted(), - /*windows_sandbox_filesystem_overrides*/ None, NetworkSandboxPolicy::Enabled, /*stdout_stream*/ None, /*after_spawn*/ None, @@ -317,10 +313,6 @@ async fn exec_full_buffer_capture_keeps_io_drain_timeout_when_descendant_holds_p justification: None, arg0: None, }, - SandboxType::None, - &SandboxPolicy::DangerFullAccess, - &FileSystemSandboxPolicy::unrestricted(), - /*windows_sandbox_filesystem_overrides*/ None, NetworkSandboxPolicy::Enabled, /*stdout_stream*/ None, /*after_spawn*/ None, @@ -931,10 +923,6 @@ async fn kill_child_process_group_kills_grandchildren_on_timeout() -> Result<()> let output = exec( params, - SandboxType::None, - &SandboxPolicy::new_read_only_policy(), - &FileSystemSandboxPolicy::from(&SandboxPolicy::new_read_only_policy()), - /*windows_sandbox_filesystem_overrides*/ None, NetworkSandboxPolicy::Restricted, /*stdout_stream*/ None, /*after_spawn*/ None, diff --git a/codex-rs/core/src/external_agent_config.rs b/codex-rs/core/src/external_agent_config.rs index a688fce12a..7cc7888012 100644 --- a/codex-rs/core/src/external_agent_config.rs +++ b/codex-rs/core/src/external_agent_config.rs @@ -3,8 +3,11 @@ use crate::plugins::PluginId; use crate::plugins::PluginInstallRequest; use crate::plugins::PluginsManager; use crate::plugins::add_marketplace; +<<<<<<< HEAD use crate::plugins::validate_marketplace_add_source; use codex_utils_absolute_path::AbsolutePathBuf; +======= +>>>>>>> main use serde_json::Value as JsonValue; use std::collections::BTreeMap; use std::collections::HashSet; @@ -177,7 +180,7 @@ impl ExternalAgentConfigService { |repo_root| repo_root.join(".codex").join("config.toml"), ); if let Some(settings) = settings.as_ref() { - let migrated = build_config_from_external(&settings)?; + let migrated = build_config_from_external(settings)?; if !is_empty_toml_table(&migrated) { let mut should_include = true; if target_config.exists() { @@ -348,23 +351,12 @@ impl ExternalAgentConfigService { let add_marketplace_outcome = add_marketplace(self.codex_home.clone(), request).await; let marketplace_path = match add_marketplace_outcome { Ok(add_marketplace_outcome) => { - match AbsolutePathBuf::try_from( - add_marketplace_outcome - .installed_root - .join(INSTALLED_MARKETPLACE_MANIFEST_RELATIVE_PATH), - ) { - Ok(path) => { - outcome - .succeeded_marketplaces - .push(marketplace_name.clone()); - path - } - Err(_) => { - outcome.failed_marketplaces.push(marketplace_name); - outcome.failed_plugin_ids.extend(plugin_ids); - continue; - } - } + outcome + .succeeded_marketplaces + .push(marketplace_name.clone()); + add_marketplace_outcome + .installed_root + .join(INSTALLED_MARKETPLACE_MANIFEST_RELATIVE_PATH) } Err(_) => { outcome.failed_marketplaces.push(marketplace_name); diff --git a/codex-rs/core/src/guardian/review_session.rs b/codex-rs/core/src/guardian/review_session.rs index 239754c508..6cb9aefa14 100644 --- a/codex-rs/core/src/guardian/review_session.rs +++ b/codex-rs/core/src/guardian/review_session.rs @@ -748,6 +748,7 @@ pub(crate) fn build_guardian_review_session_config( for feature in [ Feature::SpawnCsv, Feature::Collab, + Feature::CodexHooks, Feature::WebSearchRequest, Feature::WebSearchCached, ] { @@ -854,6 +855,25 @@ mod tests { ); } + #[test] + fn guardian_review_session_config_disables_hooks() { + let mut parent_config = crate::config::test_config(); + parent_config + .features + .enable(Feature::CodexHooks) + .expect("enable hooks on parent config"); + + let guardian_config = build_guardian_review_session_config( + &parent_config, + /*live_network_config*/ None, + "active-model", + /*reasoning_effort*/ None, + ) + .expect("guardian config"); + + assert!(!guardian_config.features.enabled(Feature::CodexHooks)); + } + #[tokio::test(flavor = "current_thread")] async fn run_before_review_deadline_times_out_before_future_completes() { let outcome = run_before_review_deadline( diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 5e62199d2d..8d463f3b51 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -5,6 +5,7 @@ // the TUI or the tracing stack). #![deny(clippy::print_stdout, clippy::print_stderr)] +mod agent_identity; mod apply_patch; mod apps; mod arc_monitor; @@ -54,8 +55,6 @@ pub use network_proxy_loader::MtimeConfigReloader; pub use network_proxy_loader::build_network_proxy_state; pub use network_proxy_loader::build_network_proxy_state_and_reloader; mod original_image_detail; -pub use codex_mcp::MCP_SANDBOX_STATE_CAPABILITY; -pub use codex_mcp::MCP_SANDBOX_STATE_METHOD; pub use codex_mcp::SandboxState; mod mcp_openai_file; mod mcp_tool_call; @@ -139,7 +138,6 @@ pub use project_doc::discover_project_doc_paths; pub use project_doc::read_project_docs; mod rollout; pub(crate) mod safety; -pub mod seatbelt; mod session_rollout_init_error; pub mod shell; pub(crate) mod shell_snapshot; diff --git a/codex-rs/core/src/mcp_tool_call.rs b/codex-rs/core/src/mcp_tool_call.rs index 9aea9b87c8..0dd5deae73 100644 --- a/codex-rs/core/src/mcp_tool_call.rs +++ b/codex-rs/core/src/mcp_tool_call.rs @@ -36,6 +36,7 @@ use codex_analytics::build_track_events_context; use codex_config::types::AppToolApproval; use codex_features::Feature; use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; +use codex_mcp::SandboxState; use codex_mcp::declared_openai_file_input_param_names; use codex_mcp::mcp_permission_prompt_is_auto_approved; use codex_otel::sanitize_metric_tag_value; @@ -466,6 +467,10 @@ async fn execute_mcp_tool_call( metadata.and_then(|metadata| metadata.openai_file_input_params.as_deref()), ) .await?; + let request_meta = + augment_mcp_tool_request_meta_with_sandbox_state(sess, turn_context, server, request_meta) + .await + .map_err(|e| format!("failed to build MCP tool request metadata: {e:#}"))?; let result = sess .call_tool(server, tool_name, rewritten_arguments, request_meta) .await @@ -479,6 +484,52 @@ async fn execute_mcp_tool_call( ) } +async fn augment_mcp_tool_request_meta_with_sandbox_state( + sess: &Session, + turn_context: &TurnContext, + server: &str, + mut meta: Option, +) -> anyhow::Result> { + let supports_sandbox_state_meta = sess + .services + .mcp_connection_manager + .read() + .await + .server_supports_sandbox_state_meta_capability(server) + .await + .unwrap_or(false); + if !supports_sandbox_state_meta { + return Ok(meta); + } + + let sandbox_state = serde_json::to_value(SandboxState { + sandbox_policy: turn_context.sandbox_policy.get().clone(), + codex_linux_sandbox_exe: turn_context.codex_linux_sandbox_exe.clone(), + sandbox_cwd: turn_context.cwd.to_path_buf(), + use_legacy_landlock: turn_context.features.use_legacy_landlock(), + })?; + + match meta.as_mut() { + Some(serde_json::Value::Object(map)) => { + map.insert( + codex_mcp::MCP_SANDBOX_STATE_META_CAPABILITY.to_string(), + sandbox_state, + ); + } + Some(_) => {} + None => { + let mut map = serde_json::Map::new(); + map.insert( + codex_mcp::MCP_SANDBOX_STATE_META_CAPABILITY.to_string(), + sandbox_state, + ); + meta = Some(serde_json::Value::Object(map)); + } + } + + Ok(meta) +} + async fn maybe_mark_thread_memory_mode_polluted(sess: &Session, turn_context: &TurnContext) { if !turn_context .config diff --git a/codex-rs/core/src/mcp_tool_exposure.rs b/codex-rs/core/src/mcp_tool_exposure.rs index d7858c17f0..16aa50cfd4 100644 --- a/codex-rs/core/src/mcp_tool_exposure.rs +++ b/codex-rs/core/src/mcp_tool_exposure.rs @@ -41,9 +41,13 @@ pub(crate) fn build_mcp_tool_exposure( let direct_tools = filter_codex_apps_mcp_tools(all_mcp_tools, explicitly_enabled_connectors, config); + for direct_tool_name in direct_tools.keys() { + deferred_tools.remove(direct_tool_name); + } + McpToolExposure { direct_tools, - deferred_tools: Some(deferred_tools), + deferred_tools: (!deferred_tools.is_empty()).then_some(deferred_tools), } } diff --git a/codex-rs/core/src/memories/README.md b/codex-rs/core/src/memories/README.md index eecaeaea28..a1d365435b 100644 --- a/codex-rs/core/src/memories/README.md +++ b/codex-rs/core/src/memories/README.md @@ -84,7 +84,11 @@ What it does: - `raw_memories.md` (merged raw memories, latest first) - `rollout_summaries/` (one summary file per retained rollout) - prunes stale rollout summaries that are no longer retained -- if there are no inputs, marks the job successful and exits +- finds old resource files from memory extensions under + `memories_extensions//resources/` for extension directories that + have an `instructions.md`, using the memory module retention window +- if there are no Phase 1 inputs or old extension resources, marks the job + successful and exits If there is input, it then: @@ -92,10 +96,13 @@ If there is input, it then: - builds the Phase 2 prompt with a diff of the current Phase 1 input selection versus the last successful Phase 2 selection (`added`, `retained`, `removed`) +- includes old extension resource paths in the prompt diff - runs it with no approvals, no network, and local write access only - disables collab for that agent (to prevent recursive delegation) - watches the agent status and heartbeats the global job lease while it runs - marks the phase-2 job success/failure in the state DB when the agent finishes +- prunes old extension resource files after the consolidation agent completes + and the successful Phase 2 job is recorded Selection diff behavior: diff --git a/codex-rs/core/src/memories/extensions.rs b/codex-rs/core/src/memories/extensions.rs new file mode 100644 index 0000000000..91c96ebff0 --- /dev/null +++ b/codex-rs/core/src/memories/extensions.rs @@ -0,0 +1,251 @@ +use crate::memories::memory_extensions_root; +use chrono::DateTime; +use chrono::Duration; +use chrono::NaiveDateTime; +use chrono::Utc; +use std::path::Path; +use std::path::PathBuf; +use tracing::warn; + +const FILENAME_TS_FORMAT: &str = "%Y-%m-%dT%H-%M-%S"; +pub(super) const EXTENSION_RESOURCE_RETENTION_DAYS: i64 = 7; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct RemovedExtensionResource { + pub(super) extension: String, + pub(super) resource_path: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct PendingExtensionResourceRemoval { + pub(super) removed: RemovedExtensionResource, + path: PathBuf, +} + +pub(super) async fn find_old_extension_resources( + memory_root: &Path, +) -> Vec { + find_old_extension_resources_with_now(memory_root, Utc::now()).await +} + +async fn find_old_extension_resources_with_now( + memory_root: &Path, + now: DateTime, +) -> Vec { + let mut pending = Vec::new(); + let cutoff = now - Duration::days(EXTENSION_RESOURCE_RETENTION_DAYS); + let extensions_root = memory_extensions_root(memory_root); + let mut extensions = match tokio::fs::read_dir(&extensions_root).await { + Ok(extensions) => extensions, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return pending, + Err(err) => { + warn!( + "failed reading memory extensions root {}: {err}", + extensions_root.display() + ); + return pending; + } + }; + + while let Ok(Some(extension_entry)) = extensions.next_entry().await { + let extension_path = extension_entry.path(); + let Ok(file_type) = extension_entry.file_type().await else { + continue; + }; + if !file_type.is_dir() { + continue; + } + let Some(extension) = extension_path + .file_name() + .and_then(|name| name.to_str()) + .map(ToOwned::to_owned) + else { + continue; + }; + if !tokio::fs::try_exists(extension_path.join("instructions.md")) + .await + .unwrap_or(false) + { + continue; + } + + let resources_path = extension_path.join("resources"); + let mut resources = match tokio::fs::read_dir(&resources_path).await { + Ok(resources) => resources, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue, + Err(err) => { + warn!( + "failed reading memory extension resources {}: {err}", + resources_path.display() + ); + continue; + } + }; + + while let Ok(Some(resource_entry)) = resources.next_entry().await { + let resource_file_path = resource_entry.path(); + let Ok(file_type) = resource_entry.file_type().await else { + continue; + }; + if !file_type.is_file() { + continue; + } + let Some(file_name) = resource_file_path + .file_name() + .and_then(|name| name.to_str()) + else { + continue; + }; + if !file_name.ends_with(".md") { + continue; + } + let Some(resource_timestamp) = resource_timestamp(file_name) else { + continue; + }; + if resource_timestamp > cutoff { + continue; + } + + pending.push(PendingExtensionResourceRemoval { + removed: RemovedExtensionResource { + extension: extension.clone(), + resource_path: format!("resources/{file_name}"), + }, + path: resource_file_path, + }); + } + } + + pending.sort_by(|left, right| { + left.removed + .extension + .cmp(&right.removed.extension) + .then_with(|| left.removed.resource_path.cmp(&right.removed.resource_path)) + }); + pending +} + +pub(super) async fn remove_extension_resources(resources: &[PendingExtensionResourceRemoval]) { + for resource in resources { + if let Err(err) = tokio::fs::remove_file(&resource.path).await + && err.kind() != std::io::ErrorKind::NotFound + { + warn!( + "failed pruning old memory extension resource {}: {err}", + resource.path.display() + ); + } + } +} + +fn resource_timestamp(file_name: &str) -> Option> { + let timestamp = file_name.get(..19)?; + let naive = NaiveDateTime::parse_from_str(timestamp, FILENAME_TS_FORMAT).ok()?; + Some(DateTime::from_naive_utc_and_offset(naive, Utc)) +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use tempfile::TempDir; + + #[tokio::test] + async fn finds_only_old_resources_from_extensions_with_instructions() { + let codex_home = TempDir::new().expect("create temp codex home"); + let memory_root = codex_home.path().join("memories"); + let extensions_root = memory_extensions_root(&memory_root); + let telepathy_resources = extensions_root.join("telepathy/resources"); + tokio::fs::create_dir_all(&telepathy_resources) + .await + .expect("create telepathy resources"); + tokio::fs::write( + extensions_root.join("telepathy/instructions.md"), + "instructions", + ) + .await + .expect("write telepathy instructions"); + + let now = DateTime::from_naive_utc_and_offset( + NaiveDateTime::parse_from_str("2026-04-14T12-00-00", FILENAME_TS_FORMAT) + .expect("parse now"), + Utc, + ); + let old_file = telepathy_resources.join("2026-04-06T11-59-59-abcd-10min-old.md"); + let exact_cutoff_file = + telepathy_resources.join("2026-04-07T12-00-00-abcd-10min-cutoff.md"); + let recent_file = telepathy_resources.join("2026-04-08T12-00-00-abcd-10min-recent.md"); + let invalid_file = telepathy_resources.join("not-a-timestamp.md"); + for file in [&old_file, &exact_cutoff_file, &recent_file, &invalid_file] { + tokio::fs::write(file, "resource") + .await + .expect("write telepathy resource"); + } + + let ignored_resources = extensions_root.join("ignored/resources"); + tokio::fs::create_dir_all(&ignored_resources) + .await + .expect("create ignored resources"); + let ignored_old_file = ignored_resources.join("2026-04-06T11-59-59-abcd-10min-old.md"); + tokio::fs::write(&ignored_old_file, "ignored") + .await + .expect("write ignored resource"); + + let pending = find_old_extension_resources_with_now(&memory_root, now).await; + + assert_eq!( + pending + .iter() + .map(|resource| resource.removed.clone()) + .collect::>(), + vec![ + RemovedExtensionResource { + extension: "telepathy".to_string(), + resource_path: "resources/2026-04-06T11-59-59-abcd-10min-old.md".to_string(), + }, + RemovedExtensionResource { + extension: "telepathy".to_string(), + resource_path: "resources/2026-04-07T12-00-00-abcd-10min-cutoff.md".to_string(), + }, + ] + ); + assert!( + tokio::fs::try_exists(&old_file) + .await + .expect("check old file before remove") + ); + assert!( + tokio::fs::try_exists(&exact_cutoff_file) + .await + .expect("check cutoff file before remove") + ); + + remove_extension_resources(&pending).await; + + assert!( + !tokio::fs::try_exists(&old_file) + .await + .expect("check old file") + ); + assert!( + !tokio::fs::try_exists(&exact_cutoff_file) + .await + .expect("check cutoff file") + ); + assert!( + tokio::fs::try_exists(&recent_file) + .await + .expect("check recent file") + ); + assert!( + tokio::fs::try_exists(&invalid_file) + .await + .expect("check invalid file") + ); + assert!( + tokio::fs::try_exists(&ignored_old_file) + .await + .expect("check ignored old file") + ); + } +} diff --git a/codex-rs/core/src/memories/mod.rs b/codex-rs/core/src/memories/mod.rs index fb033a32c1..29c9d375f2 100644 --- a/codex-rs/core/src/memories/mod.rs +++ b/codex-rs/core/src/memories/mod.rs @@ -30,6 +30,8 @@ mod artifacts { pub(super) const RAW_MEMORIES_FILENAME: &str = "raw_memories.md"; } +mod extensions; + /// Phase 1 (startup extraction). mod phase_one { /// Default model used for phase 1. diff --git a/codex-rs/core/src/memories/phase1.rs b/codex-rs/core/src/memories/phase1.rs index 37cb0e9f48..c005f4da4e 100644 --- a/codex-rs/core/src/memories/phase1.rs +++ b/codex-rs/core/src/memories/phase1.rs @@ -477,9 +477,10 @@ mod job { } }) .collect::>(); - serde_json::to_string(&filtered).map_err(|err| { + let serialized = serde_json::to_string(&filtered).map_err(|err| { CodexErr::InvalidRequest(format!("failed to serialize rollout memory: {err}")) - }) + })?; + Ok(redact_secrets(serialized)) } fn sanitize_response_item_for_memories(item: &ResponseItem) -> Option { diff --git a/codex-rs/core/src/memories/phase1_tests.rs b/codex-rs/core/src/memories/phase1_tests.rs index 9d824ab2b1..89bde1a877 100644 --- a/codex-rs/core/src/memories/phase1_tests.rs +++ b/codex-rs/core/src/memories/phase1_tests.rs @@ -3,6 +3,8 @@ use super::JobResult; use super::aggregate_stats; use super::job::serialize_filtered_rollout_response_items; use codex_protocol::models::ContentItem; +use codex_protocol::models::FunctionCallOutputBody; +use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::TokenUsage; @@ -72,6 +74,25 @@ fn serializes_memory_rollout_with_agents_removed_but_environment_kept() { ); } +#[test] +fn serializes_memory_rollout_redacts_secrets_before_prompt_upload() { + let serialized = serialize_filtered_rollout_response_items(&[RolloutItem::ResponseItem( + ResponseItem::FunctionCallOutput { + call_id: "call_123".to_string(), + output: FunctionCallOutputPayload { + body: FunctionCallOutputBody::Text( + r#"{"token":"sk-abcdefghijklmnopqrstuvwxyz123456"}"#.to_string(), + ), + success: Some(true), + }, + }, + )]) + .expect("serialize"); + + assert!(!serialized.contains("sk-abcdefghijklmnopqrstuvwxyz123456")); + assert!(serialized.contains("[REDACTED_SECRET]")); +} + #[test] fn count_outcomes_sums_token_usage_across_all_jobs() { let counts = aggregate_stats(vec![ diff --git a/codex-rs/core/src/memories/phase2.rs b/codex-rs/core/src/memories/phase2.rs index 2abc435ca7..dd194edfbf 100644 --- a/codex-rs/core/src/memories/phase2.rs +++ b/codex-rs/core/src/memories/phase2.rs @@ -3,6 +3,9 @@ use crate::agent::status::is_final as is_final_agent_status; use crate::codex::Session; use crate::codex::emit_subagent_session_started; use crate::config::Config; +use crate::memories::extensions::PendingExtensionResourceRemoval; +use crate::memories::extensions::find_old_extension_resources; +use crate::memories::extensions::remove_extension_resources; use crate::memories::memory_root; use crate::memories::metrics; use crate::memories::phase_two; @@ -112,7 +115,12 @@ pub(super) async fn run(session: &Arc, config: Arc) { job::failed(session, db, &claim, "failed_rebuild_raw_memories").await; return; } - if raw_memories.is_empty() { + let pending_extension_resource_removals = find_old_extension_resources(&root).await; + let removed_extension_resources = pending_extension_resource_removals + .iter() + .map(|resource| resource.removed.clone()) + .collect::>(); + if raw_memories.is_empty() && pending_extension_resource_removals.is_empty() { // We check only after sync of the file system. job::succeed( session, @@ -127,7 +135,7 @@ pub(super) async fn run(session: &Arc, config: Arc) { } // 5. Spawn the agent - let prompt = agent::get_prompt(config, &selection); + let prompt = agent::get_prompt(config, &selection, &removed_extension_resources); let source = SessionSource::SubAgent(SubAgentSource::MemoryConsolidation); let thread_id = match session .services @@ -170,6 +178,7 @@ pub(super) async fn run(session: &Arc, config: Arc) { claim, new_watermark, raw_memories.clone(), + pending_extension_resource_removals, thread_id, phase_two_e2e_timer, ); @@ -268,15 +277,15 @@ mod job { completion_watermark: i64, selected_outputs: &[codex_state::Stage1Output], reason: &'static str, - ) { + ) -> bool { session.services.session_telemetry.counter( metrics::MEMORY_PHASE_TWO_JOBS, /*inc*/ 1, &[("status", reason)], ); - let _ = db - .mark_global_phase2_job_succeeded(&claim.token, completion_watermark, selected_outputs) - .await; + db.mark_global_phase2_job_succeeded(&claim.token, completion_watermark, selected_outputs) + .await + .unwrap_or(false) } } @@ -328,9 +337,10 @@ mod agent { pub(super) fn get_prompt( config: Arc, selection: &codex_state::Phase2InputSelection, + removed_extension_resources: &[crate::memories::extensions::RemovedExtensionResource], ) -> Vec { let root = memory_root(&config.codex_home); - let prompt = build_consolidation_prompt(&root, selection); + let prompt = build_consolidation_prompt(&root, selection, removed_extension_resources); vec![UserInput::Text { text: prompt, text_elements: vec![], @@ -343,6 +353,7 @@ mod agent { claim: Claim, new_watermark: i64, selected_outputs: Vec, + pending_extension_resource_removals: Vec, thread_id: ThreadId, phase_two_e2e_timer: Option, ) { @@ -379,7 +390,7 @@ mod agent { if let Some(token_usage) = agent_control.get_total_token_usage(thread_id).await { emit_token_usage_metrics(&session, &token_usage); } - job::succeed( + if job::succeed( &session, &db, &claim, @@ -387,7 +398,10 @@ mod agent { &selected_outputs, "succeeded", ) - .await; + .await + { + remove_extension_resources(&pending_extension_resource_removals).await; + } } else { job::failed(&session, &db, &claim, "failed_agent").await; } diff --git a/codex-rs/core/src/memories/prompts.rs b/codex-rs/core/src/memories/prompts.rs index b6e18b4057..fc85917c9b 100644 --- a/codex-rs/core/src/memories/prompts.rs +++ b/codex-rs/core/src/memories/prompts.rs @@ -1,3 +1,5 @@ +use crate::memories::extensions::EXTENSION_RESOURCE_RETENTION_DAYS; +use crate::memories::extensions::RemovedExtensionResource; use crate::memories::memory_extensions_root; use crate::memories::memory_root; use crate::memories::phase_one; @@ -10,6 +12,7 @@ use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_output_truncation::TruncationPolicy; use codex_utils_output_truncation::truncate_text; use codex_utils_template::Template; +use std::fmt::Write as _; use std::path::Path; use std::sync::LazyLock; use tokio::fs; @@ -62,8 +65,9 @@ Memory extensions (under {{ memory_extensions_root }}/): source. If the user has any memory extensions, you MUST read the instructions for each extension to -determine how to use the memory source. If it has no extension folders, continue with the standard -memory inputs only. +determine how to use the memory source. If the Phase 2 diff lists removed memory extension +resources, use that extension-specific deletion diff to remove stale memories derived only from +those resources. If it has no extension folders, continue with the standard memory inputs only. "#; const MEMORY_EXTENSIONS_PRIMARY_INPUTS: &str = r#" @@ -73,12 +77,16 @@ Under `{{ memory_extensions_root }}/`: - `/instructions.md` - If extension folders exist, read each instructions.md first and follow it when interpreting that extension's memory source. + +If the Phase 2 diff lists removed memory extension resources, use that extension-specific deletion +diff to remove stale memories derived only from those resources. "#; /// Builds the consolidation subagent prompt for a specific memory root. pub(super) fn build_consolidation_prompt( memory_root: &Path, selection: &Phase2InputSelection, + removed_extension_resources: &[RemovedExtensionResource], ) -> String { let memory_extensions_root = memory_extensions_root(memory_root); let memory_extensions_exist = memory_extensions_root.is_dir(); @@ -100,7 +108,8 @@ pub(super) fn build_consolidation_prompt( } else { String::new() }; - let phase2_input_selection = render_phase2_input_selection(selection); + let phase2_input_selection = + render_phase2_input_selection(selection, removed_extension_resources); CONSOLIDATION_PROMPT_TEMPLATE .render([ ("memory_root", memory_root.as_str()), @@ -131,7 +140,10 @@ fn render_memory_extensions_block(template: &Template, memory_extensions_root: & }) } -fn render_phase2_input_selection(selection: &Phase2InputSelection) -> String { +fn render_phase2_input_selection( + selection: &Phase2InputSelection, + removed_extension_resources: &[RemovedExtensionResource], +) -> String { let retained = selection.retained_thread_ids.len(); let added = selection.selected.len().saturating_sub(retained); let selected = if selection.selected.is_empty() { @@ -160,11 +172,29 @@ fn render_phase2_input_selection(selection: &Phase2InputSelection) -> String { .join("\n") }; - format!( + let mut rendered = format!( "- selected inputs this run: {}\n- newly added since the last successful Phase 2 run: {added}\n- retained from the last successful Phase 2 run: {retained}\n- removed from the last successful Phase 2 run: {}\n\nCurrent selected Phase 1 inputs:\n{selected}\n\nRemoved from the last successful Phase 2 selection:\n{removed}\n", selection.selected.len(), selection.removed.len(), - ) + ); + + if !removed_extension_resources.is_empty() { + rendered.push_str("\nMemory extension resources removed by retention pruning:\n"); + let _ = writeln!( + rendered, + "- retention window: {EXTENSION_RESOURCE_RETENTION_DAYS} days" + ); + let mut current_extension = ""; + for removed_resource in removed_extension_resources { + if removed_resource.extension != current_extension { + current_extension = &removed_resource.extension; + let _ = writeln!(rendered, "- extension: {current_extension}"); + } + let _ = writeln!(rendered, " - {}", removed_resource.resource_path); + } + } + + rendered } fn render_selected_input_line(item: &Stage1Output, retained: bool) -> String { diff --git a/codex-rs/core/src/memories/prompts_tests.rs b/codex-rs/core/src/memories/prompts_tests.rs index 325d5e9234..959bb6911d 100644 --- a/codex-rs/core/src/memories/prompts_tests.rs +++ b/codex-rs/core/src/memories/prompts_tests.rs @@ -1,5 +1,7 @@ use super::*; +use crate::memories::extensions::RemovedExtensionResource; use codex_models_manager::model_info::model_info_from_slug; +use codex_state::Phase2InputSelection; use core_test_support::PathExt; use pretty_assertions::assert_eq; use tempfile::tempdir; @@ -54,6 +56,36 @@ fn build_stage_one_input_message_uses_default_limit_when_model_context_window_mi assert!(message.contains(&expected_truncated)); } +#[test] +fn build_consolidation_prompt_includes_removed_extension_resources() { + let temp = tempdir().unwrap(); + let memory_root = temp.path().join("memories"); + std::fs::create_dir_all(temp.path().join("memories_extensions")).unwrap(); + let removed_extension_resources = vec![ + RemovedExtensionResource { + extension: "telepathy".to_string(), + resource_path: "resources/2026-04-06T11-59-59-abcd-10min-old.md".to_string(), + }, + RemovedExtensionResource { + extension: "telepathy".to_string(), + resource_path: "resources/2026-04-07T12-00-00-abcd-10min-cutoff.md".to_string(), + }, + ]; + + let prompt = build_consolidation_prompt( + &memory_root, + &Phase2InputSelection::default(), + &removed_extension_resources, + ); + + assert!(prompt.contains("Memory extension resources removed by retention pruning:")); + assert!(prompt.contains("- retention window: 7 days")); + assert!(prompt.contains("- extension: telepathy")); + assert!(prompt.contains(" - resources/2026-04-06T11-59-59-abcd-10min-old.md")); + assert!(prompt.contains(" - resources/2026-04-07T12-00-00-abcd-10min-cutoff.md")); + assert!(prompt.contains("extension-specific deletion diff")); +} + #[tokio::test] async fn build_memory_tool_developer_instructions_renders_embedded_template() { let temp = tempdir().unwrap(); diff --git a/codex-rs/core/src/memories/tests.rs b/codex-rs/core/src/memories/tests.rs index a0ed29dbec..71809d382d 100644 --- a/codex-rs/core/src/memories/tests.rs +++ b/codex-rs/core/src/memories/tests.rs @@ -424,6 +424,7 @@ mod phase2 { use crate::memories::phase2; use crate::memories::raw_memories_file; use crate::memories::rollout_summaries_dir; + use chrono::Duration as ChronoDuration; use chrono::Utc; use codex_config::Constrained; use codex_login::CodexAuth; @@ -960,6 +961,28 @@ mod phase2 { "stage-1 success should enqueue global consolidation" ); + let telepathy_resources = config + .codex_home + .join("memories_extensions/telepathy/resources"); + tokio::fs::create_dir_all(&telepathy_resources) + .await + .expect("create telepathy resources"); + tokio::fs::write( + config + .codex_home + .join("memories_extensions/telepathy/instructions.md"), + "instructions", + ) + .await + .expect("write telepathy instructions"); + let old_file = telepathy_resources.join(format!( + "{}-abcd-10min-old.md", + (Utc::now() - ChronoDuration::days(8)).format("%Y-%m-%dT%H-%M-%S") + )); + tokio::fs::write(&old_file, "old resource") + .await + .expect("write old extension resource"); + phase2::run(&session, Arc::clone(&config)).await; let retry_claim = state_db @@ -971,5 +994,11 @@ mod phase2 { Phase2JobClaimOutcome::SkippedNotDirty, "spawn failures should leave the job in retry backoff instead of running" ); + assert!( + tokio::fs::try_exists(&old_file) + .await + .expect("check old extension resource"), + "spawn failures should not prune extension resources before retry" + ); } } diff --git a/codex-rs/core/src/personality_migration.rs b/codex-rs/core/src/personality_migration.rs index 52cabf55de..4b53a2629e 100644 --- a/codex-rs/core/src/personality_migration.rs +++ b/codex-rs/core/src/personality_migration.rs @@ -1,14 +1,10 @@ use crate::config::edit::ConfigEditsBuilder; -use crate::rollout::ARCHIVED_SESSIONS_SUBDIR; -use crate::rollout::SESSIONS_SUBDIR; -use crate::rollout::list::ThreadListConfig; -use crate::rollout::list::ThreadListLayout; -use crate::rollout::list::ThreadSortKey; -use crate::rollout::list::get_threads_in_root; use codex_config::config_toml::ConfigToml; use codex_protocol::config_types::Personality; -use codex_protocol::protocol::SessionSource; -use codex_rollout::state_db; +use codex_thread_store::ListThreadsParams; +use codex_thread_store::LocalThreadStore; +use codex_thread_store::ThreadSortKey; +use codex_thread_store::ThreadStore; use std::io; use std::path::Path; use tokio::fs::OpenOptions; @@ -64,57 +60,33 @@ pub async fn maybe_migrate_personality( } async fn has_recorded_sessions(codex_home: &Path, default_provider: &str) -> io::Result { - let allowed_sources: &[SessionSource] = &[]; + let store = LocalThreadStore::new(codex_rollout::RolloutConfig { + codex_home: codex_home.to_path_buf(), + sqlite_home: codex_home.to_path_buf(), + cwd: codex_home.to_path_buf(), + model_provider_id: default_provider.to_string(), + generate_memories: false, + }); + if has_threads(&store, /*archived*/ false).await? { + return Ok(true); + } + has_threads(&store, /*archived*/ true).await +} - if let Some(state_db_ctx) = state_db::open_if_present(codex_home, default_provider).await - && let Some(ids) = state_db::list_thread_ids_db( - Some(state_db_ctx.as_ref()), - codex_home, - /*page_size*/ 1, - /*cursor*/ None, - ThreadSortKey::CreatedAt, - allowed_sources, - /*model_providers*/ None, - /*archived_only*/ false, - "personality_migration", - ) +async fn has_threads(store: &LocalThreadStore, archived: bool) -> io::Result { + store + .list_threads(ListThreadsParams { + page_size: 1, + cursor: None, + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived, + search_term: None, + }) .await - && !ids.is_empty() - { - return Ok(true); - } - - let sessions = get_threads_in_root( - codex_home.join(SESSIONS_SUBDIR), - /*page_size*/ 1, - /*cursor*/ None, - ThreadSortKey::CreatedAt, - ThreadListConfig { - allowed_sources, - model_providers: None, - default_provider, - layout: ThreadListLayout::NestedByDate, - }, - ) - .await?; - if !sessions.items.is_empty() { - return Ok(true); - } - - let archived_sessions = get_threads_in_root( - codex_home.join(ARCHIVED_SESSIONS_SUBDIR), - /*page_size*/ 1, - /*cursor*/ None, - ThreadSortKey::CreatedAt, - ThreadListConfig { - allowed_sources, - model_providers: None, - default_provider, - layout: ThreadListLayout::Flat, - }, - ) - .await?; - Ok(!archived_sessions.items.is_empty()) + .map(|page| !page.items.is_empty()) + .map_err(io::Error::other) } async fn create_marker(marker_path: &Path) -> io::Result<()> { diff --git a/codex-rs/core/src/personality_migration_tests.rs b/codex-rs/core/src/personality_migration_tests.rs index de1070ad34..4aef53a5c4 100644 --- a/codex-rs/core/src/personality_migration_tests.rs +++ b/codex-rs/core/src/personality_migration_tests.rs @@ -7,6 +7,8 @@ use codex_protocol::protocol::SessionMeta; use codex_protocol::protocol::SessionMetaLine; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::UserMessageEvent; +use codex_rollout::ARCHIVED_SESSIONS_SUBDIR; +use codex_rollout::SESSIONS_SUBDIR; use pretty_assertions::assert_eq; use tempfile::TempDir; use tokio::io::AsyncWriteExt; @@ -25,6 +27,16 @@ async fn write_session_with_user_event(codex_home: &Path) -> io::Result<()> { .join("2025") .join("01") .join("01"); + write_rollout_with_user_event(&dir, thread_id).await +} + +async fn write_archived_session_with_user_event(codex_home: &Path) -> io::Result<()> { + let thread_id = ThreadId::new(); + let dir = codex_home.join(ARCHIVED_SESSIONS_SUBDIR); + write_rollout_with_user_event(&dir, thread_id).await +} + +async fn write_rollout_with_user_event(dir: &Path, thread_id: ThreadId) -> io::Result<()> { tokio::fs::create_dir_all(&dir).await?; let file_path = dir.join(format!("rollout-{TEST_TIMESTAMP}-{thread_id}.jsonl")); let mut file = tokio::fs::File::create(&file_path).await?; @@ -85,6 +97,22 @@ async fn applies_when_sessions_exist_and_no_personality() -> io::Result<()> { Ok(()) } +#[tokio::test] +async fn applies_when_only_archived_sessions_exist_and_no_personality() -> io::Result<()> { + let temp = TempDir::new()?; + write_archived_session_with_user_event(temp.path()).await?; + + let config_toml = ConfigToml::default(); + let status = maybe_migrate_personality(temp.path(), &config_toml).await?; + + assert_eq!(status, PersonalityMigrationStatus::Applied); + assert!(temp.path().join(PERSONALITY_MIGRATION_FILENAME).exists()); + + let persisted = read_config_toml(temp.path()).await?; + assert_eq!(persisted.personality, Some(Personality::Pragmatic)); + Ok(()) +} + #[tokio::test] async fn skips_when_marker_exists() -> io::Result<()> { let temp = TempDir::new()?; diff --git a/codex-rs/core/src/plugins/installed_marketplaces.rs b/codex-rs/core/src/plugins/installed_marketplaces.rs index df69111a27..ad1918d51d 100644 --- a/codex-rs/core/src/plugins/installed_marketplaces.rs +++ b/codex-rs/core/src/plugins/installed_marketplaces.rs @@ -4,6 +4,7 @@ use std::path::Path; use std::path::PathBuf; use tracing::warn; +use super::marketplace::find_marketplace_manifest_path; use super::validate_plugin_segment; pub const INSTALLED_MARKETPLACES_DIR: &str = ".tmp/marketplaces"; @@ -50,9 +51,7 @@ pub(crate) fn installed_marketplace_roots_from_config( marketplace, &default_install_root, )?; - path.join(".agents/plugins/marketplace.json") - .is_file() - .then_some(path) + find_marketplace_manifest_path(&path).map(|_| path) }) .filter_map(|path| AbsolutePathBuf::try_from(path).ok()) .collect::>(); diff --git a/codex-rs/core/src/plugins/marketplace.rs b/codex-rs/core/src/plugins/marketplace.rs index 3cb1756f80..96c73e8e19 100644 --- a/codex-rs/core/src/plugins/marketplace.rs +++ b/codex-rs/core/src/plugins/marketplace.rs @@ -9,6 +9,8 @@ use codex_protocol::protocol::Product; use codex_utils_absolute_path::AbsolutePathBuf; use dirs::home_dir; use serde::Deserialize; +use serde::Deserializer; +use serde_json::Value as JsonValue; use std::fs; use std::io; use std::path::Component; @@ -16,7 +18,10 @@ use std::path::Path; use std::path::PathBuf; use tracing::warn; -const MARKETPLACE_RELATIVE_PATH: &str = ".agents/plugins/marketplace.json"; +const MARKETPLACE_MANIFEST_RELATIVE_PATHS: &[&str] = &[ + ".agents/plugins/marketplace.json", + ".claude-plugin/marketplace.json", +]; #[derive(Debug, Clone, PartialEq, Eq)] pub struct ResolvedMarketplacePlugin { @@ -162,46 +167,50 @@ pub fn resolve_marketplace_plugin( ) -> Result { let marketplace = load_raw_marketplace_manifest(marketplace_path)?; let marketplace_name = marketplace.name; - let plugin = marketplace - .plugins - .into_iter() - .find(|plugin| plugin.name == plugin_name); - - let Some(plugin) = plugin else { - return Err(MarketplaceError::PluginNotFound { - plugin_name: plugin_name.to_string(), - marketplace_name, - }); - }; - - let RawMarketplaceManifestPlugin { - name, - source, - policy, - .. - } = plugin; - let install_policy = policy.installation; - let product_allowed = match policy.products.as_deref() { - None => true, - Some([]) => false, - Some(products) => { - restriction_product.is_some_and(|product| product.matches_product_restriction(products)) + let marketplace_name_for_not_found = marketplace_name.clone(); + for plugin in marketplace.plugins { + if plugin.name != plugin_name { + continue; } - }; - if install_policy == MarketplacePluginInstallPolicy::NotAvailable || !product_allowed { - return Err(MarketplaceError::PluginNotAvailable { - plugin_name: name, - marketplace_name, + + let RawMarketplaceManifestPlugin { + name, + source, + policy, + .. + } = plugin; + let install_policy = policy.installation; + let product_allowed = match policy.products.as_deref() { + None => true, + Some([]) => false, + Some(products) => restriction_product + .is_some_and(|product| product.matches_product_restriction(products)), + }; + if install_policy == MarketplacePluginInstallPolicy::NotAvailable || !product_allowed { + return Err(MarketplaceError::PluginNotAvailable { + plugin_name: name, + marketplace_name, + }); + } + + let Some(source_path) = + resolve_supported_plugin_source_path(marketplace_path, &name, source) + else { + continue; + }; + + return Ok(ResolvedMarketplacePlugin { + plugin_id: PluginId::new(name, marketplace_name).map_err(|err| match err { + PluginIdError::Invalid(message) => MarketplaceError::InvalidPlugin(message), + })?, + source_path, + auth_policy: policy.authentication, }); } - let plugin_id = PluginId::new(name, marketplace_name).map_err(|err| match err { - PluginIdError::Invalid(message) => MarketplaceError::InvalidPlugin(message), - })?; - Ok(ResolvedMarketplacePlugin { - plugin_id, - source_path: resolve_plugin_source_path(marketplace_path, source)?, - auth_policy: policy.authentication, + Err(MarketplaceError::PluginNotFound { + plugin_name: plugin_name.to_string(), + marketplace_name: marketplace_name_for_not_found, }) } @@ -212,16 +221,50 @@ pub fn list_marketplaces( } pub fn validate_marketplace_root(root: &Path) -> Result { - let path = AbsolutePathBuf::try_from(root.join(MARKETPLACE_RELATIVE_PATH)).map_err(|err| { - MarketplaceError::InvalidMarketplaceFile { - path: root.join(MARKETPLACE_RELATIVE_PATH), - message: format!("marketplace path must resolve to an absolute path: {err}"), - } - })?; + let Some(path) = find_marketplace_manifest_path(root) else { + return Err(MarketplaceError::InvalidMarketplaceFile { + path: root.to_path_buf(), + message: "marketplace root does not contain a supported manifest".to_string(), + }); + }; let marketplace = load_marketplace(&path)?; Ok(marketplace.name) } +pub(crate) fn find_marketplace_manifest_path(root: &Path) -> Option { + MARKETPLACE_MANIFEST_RELATIVE_PATHS + .iter() + .find_map(|relative_path| { + let path = root.join(relative_path); + if !path.is_file() { + return None; + } + AbsolutePathBuf::try_from(path).ok() + }) +} + +fn invalid_marketplace_layout_error(path: &AbsolutePathBuf) -> MarketplaceError { + MarketplaceError::InvalidMarketplaceFile { + path: path.to_path_buf(), + message: "marketplace file is not in a supported location".to_string(), + } +} + +fn marketplace_root_from_layout(marketplace_path: &Path, relative_path: &str) -> Option { + let mut current = marketplace_path; + for component in Path::new(relative_path).components().rev() { + let expected = match component { + Component::Normal(expected) => expected, + _ => return None, + }; + if current.file_name() != Some(expected) { + return None; + } + current = current.parent()?; + } + Some(current.to_path_buf()) +} + pub(crate) fn load_marketplace(path: &AbsolutePathBuf) -> Result { let marketplace = load_raw_marketplace_manifest(path)?; let mut plugins = Vec::new(); @@ -233,7 +276,9 @@ pub(crate) fn load_marketplace(path: &AbsolutePathBuf) -> Result Vec { let mut paths = Vec::new(); - if let Some(home) = home_dir { - let path = home.join(MARKETPLACE_RELATIVE_PATH); - if path.is_file() - && let Ok(path) = AbsolutePathBuf::try_from(path) - { - paths.push(path); - } + if let Some(home) = home_dir + && let Some(path) = find_marketplace_manifest_path(home) + { + paths.push(path); } for root in additional_roots { // Curated marketplaces can now come from an HTTP-downloaded directory that is not a git // checkout, so check the root directly before falling back to repo-root discovery. - let path = root.join(MARKETPLACE_RELATIVE_PATH); - if path.as_path().is_file() && !paths.contains(&path) { + if let Some(path) = find_marketplace_manifest_path(root.as_path()) + && !paths.contains(&path) + { paths.push(path); continue; } if let Some(repo_root) = get_git_repo_root(root.as_path()) && let Ok(repo_root) = AbsolutePathBuf::try_from(repo_root) + && let Some(path) = find_marketplace_manifest_path(repo_root.as_path()) + && !paths.contains(&path) { - let path = repo_root.join(MARKETPLACE_RELATIVE_PATH); - if path.as_path().is_file() && !paths.contains(&path) { - paths.push(path); - } + paths.push(path); } } @@ -346,80 +388,83 @@ fn load_raw_marketplace_manifest( }) } -fn resolve_plugin_source_path( +fn resolve_supported_plugin_source_path( marketplace_path: &AbsolutePathBuf, + plugin_name: &str, source: RawMarketplaceManifestPluginSource, -) -> Result { +) -> Option { match source { RawMarketplaceManifestPluginSource::Local { path } => { - let Some(path) = path.strip_prefix("./") else { - return Err(MarketplaceError::InvalidMarketplaceFile { - path: marketplace_path.to_path_buf(), - message: "local plugin source path must start with `./`".to_string(), - }); - }; - if path.is_empty() { - return Err(MarketplaceError::InvalidMarketplaceFile { - path: marketplace_path.to_path_buf(), - message: "local plugin source path must not be empty".to_string(), - }); + match resolve_local_plugin_source_path(marketplace_path, &path) { + Ok(path) => Some(path), + Err(err) => { + warn!( + path = %marketplace_path.display(), + plugin = plugin_name, + error = %err, + "skipping marketplace plugin that failed to resolve" + ); + None + } } - - let relative_source_path = Path::new(path); - if relative_source_path - .components() - .any(|component| !matches!(component, Component::Normal(_))) - { - return Err(MarketplaceError::InvalidMarketplaceFile { - path: marketplace_path.to_path_buf(), - message: "local plugin source path must stay within the marketplace root" - .to_string(), - }); - } - - // `marketplace.json` lives under `/.agents/plugins/`, but local plugin paths - // are resolved relative to ``, not relative to the `plugins/` directory. - Ok(marketplace_root_dir(marketplace_path)?.join(relative_source_path)) + } + RawMarketplaceManifestPluginSource::Unsupported => { + warn!( + path = %marketplace_path.display(), + plugin = plugin_name, + "skipping marketplace plugin with unsupported source" + ); + None } } } +fn resolve_local_plugin_source_path( + marketplace_path: &AbsolutePathBuf, + source_path: &str, +) -> Result { + let Some(source_path) = source_path.strip_prefix("./") else { + return Err(MarketplaceError::InvalidMarketplaceFile { + path: marketplace_path.to_path_buf(), + message: "local plugin source path must start with `./`".to_string(), + }); + }; + if source_path.is_empty() { + return Err(MarketplaceError::InvalidMarketplaceFile { + path: marketplace_path.to_path_buf(), + message: "local plugin source path must not be empty".to_string(), + }); + } + + let relative_source_path = Path::new(source_path); + if relative_source_path + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(MarketplaceError::InvalidMarketplaceFile { + path: marketplace_path.to_path_buf(), + message: "local plugin source path must stay within the marketplace root".to_string(), + }); + } + + // `marketplace.json` lives under a supported marketplace layout beneath ``, + // but local plugin paths are resolved relative to ``. + Ok(marketplace_root_dir(marketplace_path)?.join(relative_source_path)) +} + fn marketplace_root_dir( marketplace_path: &AbsolutePathBuf, ) -> Result { - let Some(plugins_dir) = marketplace_path.parent() else { - return Err(MarketplaceError::InvalidMarketplaceFile { - path: marketplace_path.to_path_buf(), - message: "marketplace file must live under `/.agents/plugins/`".to_string(), - }); - }; - let Some(dot_agents_dir) = plugins_dir.parent() else { - return Err(MarketplaceError::InvalidMarketplaceFile { - path: marketplace_path.to_path_buf(), - message: "marketplace file must live under `/.agents/plugins/`".to_string(), - }); - }; - let Some(marketplace_root) = dot_agents_dir.parent() else { - return Err(MarketplaceError::InvalidMarketplaceFile { - path: marketplace_path.to_path_buf(), - message: "marketplace file must live under `/.agents/plugins/`".to_string(), - }); - }; - - if plugins_dir.as_path().file_name().and_then(|s| s.to_str()) != Some("plugins") - || dot_agents_dir - .as_path() - .file_name() - .and_then(|s| s.to_str()) - != Some(".agents") - { - return Err(MarketplaceError::InvalidMarketplaceFile { - path: marketplace_path.to_path_buf(), - message: "marketplace file must live under `/.agents/plugins/`".to_string(), - }); + for relative_path in MARKETPLACE_MANIFEST_RELATIVE_PATHS { + if let Some(marketplace_root) = + marketplace_root_from_layout(marketplace_path.as_path(), relative_path) + { + return AbsolutePathBuf::try_from(marketplace_root) + .map_err(|_| invalid_marketplace_layout_error(marketplace_path)); + } } - Ok(marketplace_root) + Err(invalid_marketplace_layout_error(marketplace_path)) } #[derive(Debug, Deserialize)] @@ -459,10 +504,33 @@ struct RawMarketplaceManifestPluginPolicy { products: Option>, } -#[derive(Debug, Deserialize)] -#[serde(tag = "source", rename_all = "lowercase")] +#[derive(Debug)] enum RawMarketplaceManifestPluginSource { Local { path: String }, + // Mixed-source marketplaces should still contribute the local plugins we can load. + Unsupported, +} + +impl<'de> Deserialize<'de> for RawMarketplaceManifestPluginSource { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let source = JsonValue::deserialize(deserializer)?; + Ok(match source { + JsonValue::String(path) => Self::Local { path }, + JsonValue::Object(object) => match object.get("source").and_then(JsonValue::as_str) { + Some("local") => match object.get("path").and_then(JsonValue::as_str) { + Some(path) => Self::Local { + path: path.to_string(), + }, + None => Self::Unsupported, + }, + _ => Self::Unsupported, + }, + _ => Self::Unsupported, + }) + } } fn resolve_marketplace_interface( diff --git a/codex-rs/core/src/plugins/marketplace_add.rs b/codex-rs/core/src/plugins/marketplace_add.rs index 2ef11106f8..245ab9d4dd 100644 --- a/codex-rs/core/src/plugins/marketplace_add.rs +++ b/codex-rs/core/src/plugins/marketplace_add.rs @@ -282,9 +282,15 @@ mod tests { ); let config = fs::read_to_string(codex_home.path().join(codex_config::CONFIG_TOML_FILE))?; - assert!(config.contains("[marketplaces.debug]")); - assert!(config.contains("source_type = \"local\"")); - assert!(config.contains(&format!("source = \"{expected_source}\""))); + let config: toml::Value = toml::from_str(&config)?; + assert_eq!( + config["marketplaces"]["debug"]["source_type"].as_str(), + Some("local") + ); + assert_eq!( + config["marketplaces"]["debug"]["source"].as_str(), + Some(expected_source.as_str()) + ); Ok(()) } diff --git a/codex-rs/core/src/plugins/marketplace_add/metadata.rs b/codex-rs/core/src/plugins/marketplace_add/metadata.rs index ce271b116d..b9412e7f0b 100644 --- a/codex-rs/core/src/plugins/marketplace_add/metadata.rs +++ b/codex-rs/core/src/plugins/marketplace_add/metadata.rs @@ -296,19 +296,11 @@ mod tests { r#"{"name":"debug","plugins":[]}"#, ) .unwrap(); - fs::write( - codex_home.path().join(CONFIG_TOML_FILE), - format!( - "[marketplaces.debug]\nsource_type = \"local\"\nsource = \"{}\"\n", - source_root.display() - ), - ) - .unwrap(); - let source = MarketplaceSource::Local { path: source_root.clone(), }; let install_metadata = MarketplaceInstallMetadata::from_source(&source, &[]); + record_added_marketplace_entry(codex_home.path(), "debug", &install_metadata).unwrap(); let root = installed_marketplace_root_for_source( codex_home.path(), diff --git a/codex-rs/core/src/plugins/marketplace_add/source.rs b/codex-rs/core/src/plugins/marketplace_add/source.rs index 19fec57ade..53d52058aa 100644 --- a/codex-rs/core/src/plugins/marketplace_add/source.rs +++ b/codex-rs/core/src/plugins/marketplace_add/source.rs @@ -124,9 +124,9 @@ fn normalize_git_url(url: &str) -> String { } fn looks_like_local_path(source: &str) -> bool { - source.starts_with("./") + Path::new(source).is_absolute() + || source.starts_with("./") || source.starts_with("../") - || source.starts_with('/') || source.starts_with("~/") || source == "." || source == ".." diff --git a/codex-rs/core/src/plugins/marketplace_tests.rs b/codex-rs/core/src/plugins/marketplace_tests.rs index 226ae3ff6e..3412eff72c 100644 --- a/codex-rs/core/src/plugins/marketplace_tests.rs +++ b/codex-rs/core/src/plugins/marketplace_tests.rs @@ -1,8 +1,18 @@ use super::*; use codex_protocol::protocol::Product; use pretty_assertions::assert_eq; +use std::path::Path; use tempfile::tempdir; +const ALTERNATE_MARKETPLACE_RELATIVE_PATH: &str = ".claude-plugin/marketplace.json"; + +fn write_alternate_marketplace(repo_root: &Path, contents: &str) -> AbsolutePathBuf { + let marketplace_path = repo_root.join(ALTERNATE_MARKETPLACE_RELATIVE_PATH); + fs::create_dir_all(marketplace_path.parent().unwrap()).unwrap(); + fs::write(&marketplace_path, contents).unwrap(); + AbsolutePathBuf::try_from(marketplace_path).unwrap() +} + #[test] fn resolve_marketplace_plugin_finds_repo_marketplace_plugin() { let tmp = tempdir().unwrap(); @@ -45,6 +55,46 @@ fn resolve_marketplace_plugin_finds_repo_marketplace_plugin() { ); } +#[test] +fn resolve_marketplace_plugin_supports_alternate_layout_and_string_local_source() { + let tmp = tempdir().unwrap(); + let repo_root = tmp.path().join("repo"); + fs::create_dir_all(repo_root.join(".git")).unwrap(); + let marketplace_path = write_alternate_marketplace( + &repo_root, + r#"{ + "name": "alternate-marketplace", + "plugins": [ + { + "name": "string-source-plugin", + "source": "./plugins/string-source-plugin" + } + ] +}"#, + ); + + let resolved = resolve_marketplace_plugin( + &marketplace_path, + "string-source-plugin", + Some(Product::Codex), + ) + .unwrap(); + + assert_eq!( + resolved, + ResolvedMarketplacePlugin { + plugin_id: PluginId::new( + "string-source-plugin".to_string(), + "alternate-marketplace".to_string() + ) + .unwrap(), + source_path: AbsolutePathBuf::try_from(repo_root.join("plugins/string-source-plugin")) + .unwrap(), + auth_policy: MarketplacePluginAuthPolicy::OnInstall, + } + ); +} + #[test] fn resolve_marketplace_plugin_reports_missing_plugin() { let tmp = tempdir().unwrap(); @@ -70,6 +120,106 @@ fn resolve_marketplace_plugin_reports_missing_plugin() { ); } +#[test] +fn list_marketplaces_supports_alternate_manifest_layout() { + let tmp = tempdir().unwrap(); + let repo_root = tmp.path().join("repo"); + + fs::create_dir_all(repo_root.join(".git")).unwrap(); + let marketplace_path = write_alternate_marketplace( + &repo_root, + r#"{ + "name": "alternate-marketplace", + "plugins": [ + { + "name": "string-source-plugin", + "source": "./plugins/string-source-plugin" + } + ] +}"#, + ); + + let marketplaces = list_marketplaces_with_home( + &[AbsolutePathBuf::try_from(repo_root.clone()).unwrap()], + /*home_dir*/ None, + ) + .unwrap() + .marketplaces; + + assert_eq!( + marketplaces, + vec![Marketplace { + name: "alternate-marketplace".to_string(), + path: marketplace_path, + interface: None, + plugins: vec![MarketplacePlugin { + name: "string-source-plugin".to_string(), + source: MarketplacePluginSource::Local { + path: AbsolutePathBuf::try_from(repo_root.join("plugins/string-source-plugin")) + .unwrap(), + }, + policy: MarketplacePluginPolicy { + installation: MarketplacePluginInstallPolicy::Available, + authentication: MarketplacePluginAuthPolicy::OnInstall, + products: None, + }, + interface: None, + }], + }] + ); +} + +#[test] +fn list_marketplaces_prefers_first_supported_manifest_layout() { + let tmp = tempdir().unwrap(); + let repo_root = tmp.path().join("repo"); + + fs::create_dir_all(repo_root.join(".git")).unwrap(); + fs::create_dir_all(repo_root.join(".agents/plugins")).unwrap(); + fs::write( + repo_root.join(".agents/plugins/marketplace.json"), + r#"{ + "name": "agents-marketplace", + "plugins": [ + { + "name": "agents-plugin", + "source": { + "source": "local", + "path": "./plugins/agents-plugin" + } + } + ] +}"#, + ) + .unwrap(); + write_alternate_marketplace( + &repo_root, + r#"{ + "name": "alternate-marketplace", + "plugins": [ + { + "name": "string-source-plugin", + "source": "./plugins/string-source-plugin" + } + ] +}"#, + ); + + let marketplaces = list_marketplaces_with_home( + &[AbsolutePathBuf::try_from(repo_root.clone()).unwrap()], + /*home_dir*/ None, + ) + .unwrap() + .marketplaces; + + assert_eq!(marketplaces.len(), 1); + assert_eq!(marketplaces[0].name, "agents-marketplace"); + assert_eq!( + marketplaces[0].path, + AbsolutePathBuf::try_from(repo_root.join(".agents/plugins/marketplace.json")).unwrap() + ); +} + #[test] fn list_marketplaces_returns_home_and_repo_marketplaces() { let tmp = tempdir().unwrap(); @@ -413,7 +563,7 @@ fn list_marketplaces_reads_marketplace_display_name() { } #[test] -fn list_marketplaces_skips_marketplaces_that_fail_to_load() { +fn list_marketplaces_skips_invalid_plugins_but_keeps_marketplace() { let tmp = tempdir().unwrap(); let valid_repo_root = tmp.path().join("valid-repo"); let invalid_repo_root = tmp.path().join("invalid-repo"); @@ -465,8 +615,10 @@ fn list_marketplaces_skips_marketplaces_that_fail_to_load() { .unwrap() .marketplaces; - assert_eq!(marketplaces.len(), 1); + assert_eq!(marketplaces.len(), 2); assert_eq!(marketplaces[0].name, "valid-marketplace"); + assert_eq!(marketplaces[1].name, "invalid-marketplace"); + assert!(marketplaces[1].plugins.is_empty()); } #[test] @@ -522,6 +674,60 @@ fn list_marketplaces_reports_marketplace_load_errors() { ); } +#[test] +fn list_marketplaces_skips_unsupported_plugin_sources_but_keeps_local_plugins() { + let tmp = tempdir().unwrap(); + let repo_root = tmp.path().join("repo"); + + fs::create_dir_all(repo_root.join(".git")).unwrap(); + write_alternate_marketplace( + &repo_root, + r#"{ + "name": "mixed-source-marketplace", + "plugins": [ + { + "name": "local-plugin", + "source": "./plugins/local-plugin" + }, + { + "name": "url-plugin", + "source": { + "source": "url", + "url": "https://github.com/example/plugin.git" + } + }, + { + "name": "git-subdir-plugin", + "source": { + "source": "git-subdir", + "url": "owner/repo", + "path": "plugins/example", + "ref": "main" + } + } + ] +}"#, + ); + + let marketplaces = list_marketplaces_with_home( + &[AbsolutePathBuf::try_from(repo_root.clone()).unwrap()], + /*home_dir*/ None, + ) + .unwrap() + .marketplaces; + + assert_eq!(marketplaces.len(), 1); + assert_eq!(marketplaces[0].name, "mixed-source-marketplace"); + assert_eq!(marketplaces[0].plugins.len(), 1); + assert_eq!(marketplaces[0].plugins[0].name, "local-plugin"); + assert_eq!( + marketplaces[0].plugins[0].source, + MarketplacePluginSource::Local { + path: AbsolutePathBuf::try_from(repo_root.join("plugins/local-plugin")).unwrap(), + } + ); +} + #[test] fn list_marketplaces_resolves_plugin_interface_paths_to_absolute() { let tmp = tempdir().unwrap(); @@ -734,7 +940,7 @@ fn list_marketplaces_ignores_plugin_interface_assets_without_dot_slash() { } #[test] -fn resolve_marketplace_plugin_rejects_non_relative_local_paths() { +fn resolve_marketplace_plugin_skips_invalid_local_paths() { let tmp = tempdir().unwrap(); let repo_root = tmp.path().join("repo"); fs::create_dir_all(repo_root.join(".git")).unwrap(); @@ -756,17 +962,46 @@ fn resolve_marketplace_plugin_rejects_non_relative_local_paths() { ) .unwrap(); - let marketplace_path = - AbsolutePathBuf::try_from(repo_root.join(".agents/plugins/marketplace.json")).unwrap(); - let err = resolve_marketplace_plugin(&marketplace_path, "local-plugin", Some(Product::Codex)) + let err = resolve_marketplace_plugin( + &AbsolutePathBuf::try_from(repo_root.join(".agents/plugins/marketplace.json")).unwrap(), + "local-plugin", + Some(Product::Codex), + ) + .unwrap_err(); + + assert_eq!( + err.to_string(), + "plugin `local-plugin` was not found in marketplace `codex-curated`" + ); +} + +#[test] +fn resolve_marketplace_plugin_skips_unsupported_sources() { + let tmp = tempdir().unwrap(); + let repo_root = tmp.path().join("repo"); + fs::create_dir_all(repo_root.join(".git")).unwrap(); + let marketplace_path = write_alternate_marketplace( + &repo_root, + r#"{ + "name": "alternate-marketplace", + "plugins": [ + { + "name": "remote-plugin", + "source": { + "source": "url", + "url": "https://github.com/example/plugin.git" + } + } + ] +}"#, + ); + + let err = resolve_marketplace_plugin(&marketplace_path, "remote-plugin", Some(Product::Codex)) .unwrap_err(); assert_eq!( err.to_string(), - format!( - "invalid marketplace file `{}`: local plugin source path must start with `./`", - marketplace_path.display() - ) + "plugin `remote-plugin` was not found in marketplace `alternate-marketplace`" ); } diff --git a/codex-rs/core/src/realtime_context.rs b/codex-rs/core/src/realtime_context.rs index 6e9815f562..30cbd6f636 100644 --- a/codex-rs/core/src/realtime_context.rs +++ b/codex-rs/core/src/realtime_context.rs @@ -4,8 +4,10 @@ use crate::event_mapping::is_contextual_user_message_content; use chrono::Utc; use codex_git_utils::resolve_root_git_project_for_trust; use codex_protocol::models::ResponseItem; -use codex_state::SortKey; -use codex_state::ThreadMetadata; +use codex_thread_store::ListThreadsParams; +use codex_thread_store::StoredThread; +use codex_thread_store::ThreadSortKey; +use codex_thread_store::ThreadStore; use codex_utils_output_truncation::TruncationPolicy; use codex_utils_output_truncation::truncate_text; use dirs::home_dir; @@ -98,7 +100,7 @@ pub(crate) async fn build_realtime_startup_context( } if let Some(section) = format_section( "Notes", - Some("Built at realtime startup from the current thread history, persisted thread metadata in the state DB, and a bounded local workspace scan. This excludes repo memory instructions, AGENTS files, project-doc prompt blends, and memory summaries.".to_string()), + Some("Built at realtime startup from the current thread history, local thread metadata, and a bounded local workspace scan. This excludes repo memory instructions, AGENTS files, project-doc prompt blends, and memory summaries.".to_string()), NOTES_SECTION_TOKEN_BUDGET, ) { parts.push(section); @@ -117,33 +119,31 @@ pub(crate) async fn build_realtime_startup_context( Some(context) } -async fn load_recent_threads(sess: &Session) -> Vec { - let Some(state_db) = sess.services.state_db.as_ref() else { - return Vec::new(); - }; - - match state_db - .list_threads( - MAX_RECENT_THREADS, - /*anchor*/ None, - SortKey::UpdatedAt, - &[], - /*model_providers*/ None, - /*archived_only*/ false, - /*search_term*/ None, - ) +async fn load_recent_threads(sess: &Session) -> Vec { + match sess + .services + .thread_store + .list_threads(ListThreadsParams { + page_size: MAX_RECENT_THREADS, + cursor: None, + sort_key: ThreadSortKey::UpdatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived: false, + search_term: None, + }) .await { Ok(page) => page.items, Err(err) => { - warn!("failed to load realtime startup threads from state db: {err}"); + warn!("failed to load realtime startup threads from thread store: {err}"); Vec::new() } } } -fn build_recent_work_section(cwd: &Path, recent_threads: &[ThreadMetadata]) -> Option { - let mut groups: HashMap> = HashMap::new(); +fn build_recent_work_section(cwd: &Path, recent_threads: &[StoredThread]) -> Option { + let mut groups: HashMap> = HashMap::new(); for entry in recent_threads { let group = resolve_root_git_project_for_trust(&entry.cwd).unwrap_or_else(|| entry.cwd.clone()); @@ -446,7 +446,7 @@ fn format_section(title: &str, body: Option, budget_tokens: usize) -> Op fn format_thread_group( current_group: &Path, group: &Path, - entries: Vec<&ThreadMetadata>, + entries: Vec<&StoredThread>, ) -> Option { let latest = entries.first()?; let group_label = if resolve_root_git_project_for_trust(latest.cwd.as_path()).is_some() { @@ -461,8 +461,9 @@ fn format_thread_group( ]; if let Some(git_branch) = latest - .git_branch - .as_deref() + .git_info + .as_ref() + .and_then(|git| git.branch.as_deref()) .filter(|git_branch| !git_branch.is_empty()) { lines.push(format!("Latest branch: {git_branch}")); diff --git a/codex-rs/core/src/realtime_context_tests.rs b/codex-rs/core/src/realtime_context_tests.rs index 7495161d46..7e1e82476f 100644 --- a/codex-rs/core/src/realtime_context_tests.rs +++ b/codex-rs/core/src/realtime_context_tests.rs @@ -3,20 +3,31 @@ use super::build_recent_work_section; use super::build_workspace_section_with_user_root; use chrono::TimeZone; use chrono::Utc; +use codex_git_utils::GitSha; use codex_protocol::ThreadId; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; -use codex_state::ThreadMetadata; +use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::GitInfo; +use codex_protocol::protocol::SandboxPolicy; +use codex_protocol::protocol::SessionSource; +use codex_thread_store::StoredThread; use pretty_assertions::assert_eq; use std::fs; use std::path::PathBuf; use std::process::Command; use tempfile::TempDir; -fn thread_metadata(cwd: &str, title: &str, first_user_message: &str) -> ThreadMetadata { - ThreadMetadata { - id: ThreadId::new(), - rollout_path: PathBuf::from("/tmp/rollout.jsonl"), +fn stored_thread(cwd: &str, title: &str, first_user_message: &str) -> StoredThread { + StoredThread { + thread_id: ThreadId::new(), + rollout_path: Some(PathBuf::from("/tmp/rollout.jsonl")), + forked_from_id: None, + preview: first_user_message.to_string(), + name: (!title.is_empty()).then(|| title.to_string()), + model_provider: "test-provider".to_string(), + model: Some("gpt-5".to_string()), + reasoning_effort: None, created_at: Utc .timestamp_opt(1_709_251_100, 0) .single() @@ -25,24 +36,23 @@ fn thread_metadata(cwd: &str, title: &str, first_user_message: &str) -> ThreadMe .timestamp_opt(1_709_251_200, 0) .single() .expect("valid timestamp"), - source: "cli".to_string(), - agent_path: None, - agent_nickname: None, - agent_role: None, - model_provider: "test-provider".to_string(), - model: Some("gpt-5".to_string()), - reasoning_effort: None, + archived_at: None, cwd: PathBuf::from(cwd), cli_version: "test".to_string(), - title: title.to_string(), - sandbox_policy: "workspace-write".to_string(), - approval_mode: "never".to_string(), - tokens_used: 0, + source: SessionSource::Cli, + agent_nickname: None, + agent_role: None, + agent_path: None, + git_info: Some(GitInfo { + commit_hash: Some(GitSha::new("abcdef")), + branch: Some("main".to_string()), + repository_url: None, + }), + approval_mode: AskForApproval::Never, + sandbox_policy: SandboxPolicy::new_read_only_policy(), + token_usage: None, first_user_message: Some(first_user_message.to_string()), - archived_at: None, - git_sha: None, - git_branch: Some("main".to_string()), - git_origin_url: None, + history: None, } } @@ -224,17 +234,17 @@ fn recent_work_section_groups_threads_by_cwd() { fs::create_dir_all(&outside).expect("create outside dir"); let recent_threads = vec![ - thread_metadata( + stored_thread( workspace_a.to_string_lossy().as_ref(), "Investigate realtime startup context", "Log the startup context before sending it", ), - thread_metadata( + stored_thread( workspace_b.to_string_lossy().as_ref(), "Trim websocket startup payload", "Remove memories from the realtime startup context", ), - thread_metadata(outside.to_string_lossy().as_ref(), "", "Inspect flaky test"), + stored_thread(outside.to_string_lossy().as_ref(), "", "Inspect flaky test"), ]; let current_cwd = workspace_a; let repo = fs::canonicalize(repo).expect("canonicalize repo"); diff --git a/codex-rs/core/src/realtime_conversation.rs b/codex-rs/core/src/realtime_conversation.rs index e749736710..f81d6c0948 100644 --- a/codex-rs/core/src/realtime_conversation.rs +++ b/codex-rs/core/src/realtime_conversation.rs @@ -805,7 +805,9 @@ async fn handle_start_inner( if let Some(text) = maybe_routed_text { debug!(text = %text, "[realtime-text] realtime conversation text output"); let sess_for_routed_text = Arc::clone(&sess_clone); - sess_for_routed_text.route_realtime_text_input(text).await; + sess_for_routed_text + .route_realtime_text_input(wrap_realtime_delegation_input(&text)) + .await; } if !fanout_realtime_active.load(Ordering::Relaxed) { break; @@ -867,6 +869,20 @@ fn realtime_text_from_handoff_request(handoff: &RealtimeHandoffRequested) -> Opt .or((!handoff.input_transcript.is_empty()).then_some(handoff.input_transcript.clone())) } +fn wrap_realtime_delegation_input(input: &str) -> String { + format!( + "\n {}\n", + escape_xml_text(input) + ) +} + +fn escape_xml_text(input: &str) -> String { + input + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") +} + fn realtime_api_key(auth: Option<&CodexAuth>, provider: &ModelProviderInfo) -> CodexResult { if let Some(api_key) = provider.api_key()? { return Ok(api_key); diff --git a/codex-rs/core/src/realtime_conversation_tests.rs b/codex-rs/core/src/realtime_conversation_tests.rs index 0a32d063c0..cda130e182 100644 --- a/codex-rs/core/src/realtime_conversation_tests.rs +++ b/codex-rs/core/src/realtime_conversation_tests.rs @@ -1,6 +1,7 @@ use super::RealtimeHandoffState; use super::RealtimeSessionKind; use super::realtime_text_from_handoff_request; +use super::wrap_realtime_delegation_input; use async_channel::bounded; use codex_protocol::protocol::RealtimeHandoffRequested; use codex_protocol::protocol::RealtimeTranscriptEntry; @@ -54,6 +55,22 @@ fn ignores_empty_handoff_request_input_transcript() { assert_eq!(realtime_text_from_handoff_request(&handoff), None); } +#[test] +fn wraps_realtime_delegation_input() { + assert_eq!( + wrap_realtime_delegation_input("hello"), + "\n hello\n" + ); +} + +#[test] +fn wraps_realtime_delegation_input_with_xml_escaping() { + assert_eq!( + wrap_realtime_delegation_input("use a < b && c > d"), + "\n use a < b && c > d\n" + ); +} + #[tokio::test] async fn clears_active_handoff_explicitly() { let (tx, _rx) = bounded(1); diff --git a/codex-rs/core/src/rollout.rs b/codex-rs/core/src/rollout.rs index 26e3a84d5c..4913f957ea 100644 --- a/codex-rs/core/src/rollout.rs +++ b/codex-rs/core/src/rollout.rs @@ -46,11 +46,7 @@ impl codex_rollout::RolloutConfigView for Config { } pub(crate) mod list { - pub use codex_rollout::ThreadListConfig; - pub use codex_rollout::ThreadListLayout; - pub use codex_rollout::ThreadSortKey; pub use codex_rollout::find_thread_path_by_id_str; - pub use codex_rollout::get_threads_in_root; } pub(crate) mod metadata { diff --git a/codex-rs/core/src/seatbelt.rs b/codex-rs/core/src/seatbelt.rs deleted file mode 100644 index c54ff3fd5b..0000000000 --- a/codex-rs/core/src/seatbelt.rs +++ /dev/null @@ -1,48 +0,0 @@ -#![cfg(target_os = "macos")] - -use crate::spawn::CODEX_SANDBOX_ENV_VAR; -use crate::spawn::SpawnChildRequest; -use crate::spawn::StdioPolicy; -use crate::spawn::spawn_child_async; -use codex_network_proxy::NetworkProxy; -use codex_protocol::permissions::FileSystemSandboxPolicy; -use codex_protocol::permissions::NetworkSandboxPolicy; -use codex_protocol::protocol::SandboxPolicy; -use codex_sandboxing::seatbelt::MACOS_PATH_TO_SEATBELT_EXECUTABLE; -use codex_sandboxing::seatbelt::create_seatbelt_command_args_for_policies; -use codex_utils_absolute_path::AbsolutePathBuf; -use std::collections::HashMap; -use std::path::PathBuf; -use tokio::process::Child; - -pub async fn spawn_command_under_seatbelt( - command: Vec, - command_cwd: AbsolutePathBuf, - sandbox_policy: &SandboxPolicy, - sandbox_policy_cwd: &AbsolutePathBuf, - stdio_policy: StdioPolicy, - network: Option<&NetworkProxy>, - mut env: HashMap, -) -> std::io::Result { - let args = create_seatbelt_command_args_for_policies( - command, - &FileSystemSandboxPolicy::from_legacy_sandbox_policy(sandbox_policy, sandbox_policy_cwd), - NetworkSandboxPolicy::from(sandbox_policy), - sandbox_policy_cwd, - /*enforce_managed_network*/ false, - network, - ); - let arg0 = None; - env.insert(CODEX_SANDBOX_ENV_VAR.to_string(), "seatbelt".to_string()); - spawn_child_async(SpawnChildRequest { - program: PathBuf::from(MACOS_PATH_TO_SEATBELT_EXECUTABLE), - args, - arg0, - cwd: command_cwd, - network_sandbox_policy: NetworkSandboxPolicy::from(sandbox_policy), - network, - stdio_policy, - env, - }) - .await -} diff --git a/codex-rs/core/src/state/service.rs b/codex-rs/core/src/state/service.rs index 2e89585f99..5db38f7b72 100644 --- a/codex-rs/core/src/state/service.rs +++ b/codex-rs/core/src/state/service.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use crate::RolloutRecorder; use crate::SkillsManager; use crate::agent::AgentControl; +use crate::agent_identity::AgentIdentityManager; use crate::client::ModelClient; use crate::config::StartedNetworkProxy; use crate::exec_policy::ExecPolicyManager; @@ -23,6 +24,7 @@ use codex_mcp::McpConnectionManager; use codex_models_manager::manager::ModelsManager; use codex_otel::SessionTelemetry; use codex_rollout::state_db::StateDbHandle; +use codex_thread_store::LocalThreadStore; use std::path::PathBuf; use tokio::sync::Mutex; use tokio::sync::RwLock; @@ -41,6 +43,7 @@ pub(crate) struct SessionServices { pub(crate) hooks: Hooks, pub(crate) rollout: Mutex>, pub(crate) user_shell: Arc, + pub(crate) agent_identity_manager: Arc, pub(crate) shell_snapshot_tx: watch::Sender>>, pub(crate) show_raw_agent_reasoning: bool, pub(crate) exec_policy: Arc, @@ -57,6 +60,7 @@ pub(crate) struct SessionServices { pub(crate) network_proxy: Option, pub(crate) network_approval: Arc, pub(crate) state_db: Option, + pub(crate) thread_store: LocalThreadStore, /// Session-scoped model client shared across turns. pub(crate) model_client: ModelClient, pub(crate) code_mode_service: CodeModeService, diff --git a/codex-rs/core/src/tools/code_mode/mod.rs b/codex-rs/core/src/tools/code_mode/mod.rs index de3a1fa330..0004ec6f51 100644 --- a/codex-rs/core/src/tools/code_mode/mod.rs +++ b/codex-rs/core/src/tools/code_mode/mod.rs @@ -116,7 +116,7 @@ struct CoreTurnHost { impl CodeModeTurnHost for CoreTurnHost { async fn invoke_tool( &self, - tool_name: String, + tool_name: ToolName, input: Option, cancellation_token: CancellationToken, ) -> Result { @@ -288,38 +288,39 @@ async fn build_nested_router(exec: &ExecContext) -> ToolRouter { async fn call_nested_tool( exec: ExecContext, tool_runtime: ToolCallRuntime, - tool_name: String, + tool_name: ToolName, input: Option, cancellation_token: CancellationToken, ) -> Result { - if tool_name == PUBLIC_TOOL_NAME { + if tool_name.namespace.is_none() && tool_name.name == PUBLIC_TOOL_NAME { return Err(FunctionCallError::RespondToModel(format!( "{PUBLIC_TOOL_NAME} cannot invoke itself" ))); } - let payload = if let Some(tool_info) = exec - .session - .resolve_mcp_tool_info(&tool_name, /*namespace*/ None) - .await - { - match serialize_function_tool_arguments(&tool_name, input) { - Ok(raw_arguments) => ToolPayload::Mcp { - server: tool_info.server_name, - tool: tool_info.tool.name.to_string(), - raw_arguments, - }, - Err(error) => return Err(FunctionCallError::RespondToModel(error)), - } - } else { - match build_nested_tool_payload(tool_runtime.find_spec(&tool_name), &tool_name, input) { - Ok(payload) => payload, - Err(error) => return Err(FunctionCallError::RespondToModel(error)), - } - }; + let (tool_call_name, payload) = + if let Some(tool_info) = exec.session.resolve_mcp_tool_info(&tool_name).await { + let raw_arguments = match serialize_function_tool_arguments(&tool_name, input) { + Ok(raw_arguments) => raw_arguments, + Err(error) => return Err(FunctionCallError::RespondToModel(error)), + }; + ( + tool_info.canonical_tool_name(), + ToolPayload::Mcp { + server: tool_info.server_name, + tool: tool_info.tool.name.to_string(), + raw_arguments, + }, + ) + } else { + match build_nested_tool_payload(tool_runtime.find_spec(&tool_name), &tool_name, input) { + Ok(payload) => (tool_name, payload), + Err(error) => return Err(FunctionCallError::RespondToModel(error)), + } + }; let call = ToolCall { - tool_name: ToolName::plain(tool_name.clone()), + tool_name: tool_call_name, call_id: format!("{PUBLIC_TOOL_NAME}-{}", uuid::Uuid::new_v4()), payload, }; @@ -339,7 +340,7 @@ fn tool_kind_for_spec(spec: &ToolSpec) -> codex_code_mode::CodeModeToolKind { fn tool_kind_for_name( spec: Option, - tool_name: &str, + tool_name: &ToolName, ) -> Result { spec.as_ref() .map(tool_kind_for_spec) @@ -348,7 +349,7 @@ fn tool_kind_for_name( fn build_nested_tool_payload( spec: Option, - tool_name: &str, + tool_name: &ToolName, input: Option, ) -> Result { let actual_kind = tool_kind_for_name(spec, tool_name)?; @@ -363,7 +364,7 @@ fn build_nested_tool_payload( } fn build_function_tool_payload( - tool_name: &str, + tool_name: &ToolName, input: Option, ) -> Result { let arguments = serialize_function_tool_arguments(tool_name, input)?; @@ -371,7 +372,7 @@ fn build_function_tool_payload( } fn serialize_function_tool_arguments( - tool_name: &str, + tool_name: &ToolName, input: Option, ) -> Result { match input { @@ -385,7 +386,7 @@ fn serialize_function_tool_arguments( } fn build_freeform_tool_payload( - tool_name: &str, + tool_name: &ToolName, input: Option, ) -> Result { match input { diff --git a/codex-rs/core/src/tools/handlers/multi_agents_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_tests.rs index e35d6a06fb..10f41d97eb 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_tests.rs @@ -1492,24 +1492,25 @@ async fn multi_agent_v2_followup_task_interrupts_busy_child_without_losing_messa let session = Arc::new(session); let turn = Arc::new(turn); - SpawnAgentHandlerV2 - .handle(invocation( - session.clone(), - turn.clone(), - "spawn_agent", - function_payload(json!({ - "message": "boot worker", - "task_name": "worker" - })), - )) - .await - .expect("spawn worker"); + let worker_path = AgentPath::try_from("/root/worker").expect("worker path"); let agent_id = session .services .agent_control - .resolve_agent_reference(session.conversation_id, &turn.session_source, "worker") + .spawn_agent_with_metadata( + (*turn.config).clone(), + Op::CleanBackgroundTerminals, + Some(SessionSource::SubAgent(SubAgentSource::ThreadSpawn { + parent_thread_id: root.thread_id, + depth: 1, + agent_path: Some(worker_path.clone()), + agent_nickname: None, + agent_role: None, + })), + crate::agent::control::SpawnAgentOptions::default(), + ) .await - .expect("worker should resolve"); + .expect("worker spawn should succeed") + .thread_id; let thread = manager .get_thread(agent_id) .await @@ -1566,7 +1567,7 @@ async fn multi_agent_v2_followup_task_interrupts_busy_child_without_losing_messa &thread, &InterAgentCommunication::new( AgentPath::root(), - AgentPath::try_from("/root/worker").expect("agent path"), + worker_path, Vec::new(), "continue".to_string(), /*trigger_turn*/ true, diff --git a/codex-rs/core/src/tools/js_repl/mod.rs b/codex-rs/core/src/tools/js_repl/mod.rs index b728009499..92bd4c610f 100644 --- a/codex-rs/core/src/tools/js_repl/mod.rs +++ b/codex-rs/core/src/tools/js_repl/mod.rs @@ -45,6 +45,8 @@ use codex_sandboxing::SandboxCommand; use codex_sandboxing::SandboxManager; use codex_sandboxing::SandboxTransformRequest; use codex_sandboxing::SandboxablePreference; +use codex_tools::ResponsesApiNamespaceTool; +use codex_tools::ToolName; use codex_tools::ToolSpec; use codex_utils_output_truncation::TruncationPolicy; use codex_utils_output_truncation::truncate_text; @@ -1574,29 +1576,67 @@ impl JsReplManager { }, ); - let payload = if let Some(tool_info) = exec + let specs = router.specs(); + let requested_tool_name = specs + .iter() + .find_map(|spec| match spec { + ToolSpec::Function(tool) if tool.name == req.tool_name => { + Some(ToolName::plain(req.tool_name.clone())) + } + ToolSpec::Freeform(tool) if tool.name == req.tool_name => { + Some(ToolName::plain(req.tool_name.clone())) + } + ToolSpec::Namespace(namespace) => { + namespace.tools.iter().find_map(|tool| match tool { + ResponsesApiNamespaceTool::Function(tool) => { + let tool_name = + ToolName::namespaced(namespace.name.clone(), tool.name.clone()); + (tool_name.display() == req.tool_name).then_some(tool_name) + } + }) + } + ToolSpec::LocalShell {} + | ToolSpec::ImageGeneration { .. } + | ToolSpec::ToolSearch { .. } + | ToolSpec::WebSearch { .. } + | ToolSpec::Function(_) + | ToolSpec::Freeform(_) => None, + }) + .unwrap_or_else(|| ToolName::plain(req.tool_name.clone())); + let (tool_call_name, payload) = if let Some(tool_info) = exec .session - .resolve_mcp_tool_info(&req.tool_name, /*namespace*/ None) + .resolve_mcp_tool_info(&requested_tool_name) .await { - crate::tools::context::ToolPayload::Mcp { - server: tool_info.server_name, - tool: tool_info.tool.name.to_string(), - raw_arguments: req.arguments.clone(), - } - } else if is_freeform_tool(&router.specs(), &req.tool_name) { - crate::tools::context::ToolPayload::Custom { - input: req.arguments.clone(), - } + ( + tool_info.canonical_tool_name(), + crate::tools::context::ToolPayload::Mcp { + server: tool_info.server_name, + tool: tool_info.tool.name.to_string(), + raw_arguments: req.arguments.clone(), + }, + ) + } else if matches!( + router.find_spec(&requested_tool_name), + Some(ToolSpec::Freeform(_)) + ) { + ( + requested_tool_name, + crate::tools::context::ToolPayload::Custom { + input: req.arguments.clone(), + }, + ) } else { - crate::tools::context::ToolPayload::Function { - arguments: req.arguments.clone(), - } + ( + requested_tool_name, + crate::tools::context::ToolPayload::Function { + arguments: req.arguments.clone(), + }, + ) }; - let tool_name = req.tool_name.clone(); let call = crate::tools::router::ToolCall { - tool_name: codex_tools::ToolName::plain(tool_name.clone()), + tool_name: tool_call_name, call_id: req.id.clone(), payload, }; @@ -1755,12 +1795,6 @@ fn split_exec_result_content_items( } } -fn is_freeform_tool(specs: &[ToolSpec], name: &str) -> bool { - specs - .iter() - .any(|spec| spec.name() == name && matches!(spec, ToolSpec::Freeform(_))) -} - fn is_js_repl_internal_tool(name: &str) -> bool { matches!(name, "js_repl" | "js_repl_reset") } diff --git a/codex-rs/core/src/tools/parallel.rs b/codex-rs/core/src/tools/parallel.rs index a1965db5cc..b4dbfaa6cf 100644 --- a/codex-rs/core/src/tools/parallel.rs +++ b/codex-rs/core/src/tools/parallel.rs @@ -48,7 +48,7 @@ impl ToolCallRuntime { } } - pub(crate) fn find_spec(&self, tool_name: &str) -> Option { + pub(crate) fn find_spec(&self, tool_name: &codex_tools::ToolName) -> Option { self.router.find_spec(tool_name) } diff --git a/codex-rs/core/src/tools/router.rs b/codex-rs/core/src/tools/router.rs index 645650525c..183b9fb416 100644 --- a/codex-rs/core/src/tools/router.rs +++ b/codex-rs/core/src/tools/router.rs @@ -16,6 +16,7 @@ use codex_protocol::models::SearchToolCallParams; use codex_protocol::models::ShellToolCallParams; use codex_tools::ConfiguredToolSpec; use codex_tools::DiscoverableTool; +use codex_tools::ResponsesApiNamespaceTool; use codex_tools::ToolName; use codex_tools::ToolSpec; use codex_tools::ToolsConfig; @@ -102,20 +103,48 @@ impl ToolRouter { self.model_visible_specs.clone() } - pub fn find_spec(&self, tool_name: &str) -> Option { - self.specs - .iter() - .find(|config| config.name() == tool_name) - .map(|config| config.spec.clone()) + pub fn find_spec(&self, tool_name: &ToolName) -> Option { + self.specs.iter().find_map(|config| match &config.spec { + ToolSpec::Function(tool) + if tool_name.namespace.is_none() && tool.name == tool_name.name => + { + Some(config.spec.clone()) + } + ToolSpec::Freeform(tool) + if tool_name.namespace.is_none() && tool.name == tool_name.name => + { + Some(config.spec.clone()) + } + ToolSpec::Namespace(namespace) => namespace.tools.iter().find_map(|tool| match tool { + ResponsesApiNamespaceTool::Function(tool) + if tool_name.namespace.as_deref() == Some(namespace.name.as_str()) + && tool.name == tool_name.name => + { + Some(ToolSpec::Function(tool.clone())) + } + _ => None, + }), + _ => None, + }) } fn configured_tool_supports_parallel(&self, tool_name: &ToolName) -> bool { - tool_name.namespace.is_none() - && self - .specs - .iter() - .filter(|config| config.supports_parallel_tool_calls) - .any(|config| config.name() == tool_name.name.as_str()) + if tool_name.namespace.is_some() { + return false; + } + + self.specs + .iter() + .filter(|config| config.supports_parallel_tool_calls) + .any(|config| match &config.spec { + ToolSpec::Function(tool) => tool.name == tool_name.name.as_str(), + ToolSpec::Freeform(tool) => tool.name == tool_name.name.as_str(), + ToolSpec::Namespace(_) + | ToolSpec::ToolSearch { .. } + | ToolSpec::LocalShell {} + | ToolSpec::ImageGeneration { .. } + | ToolSpec::WebSearch { .. } => false, + }) } pub fn tool_supports_parallel(&self, call: &ToolCall) -> bool { @@ -141,16 +170,10 @@ impl ToolRouter { call_id, .. } => { - let mcp_tool = session - .resolve_mcp_tool_info(&name, namespace.as_deref()) - .await; - let tool_name = match namespace { - Some(namespace) => ToolName::namespaced(namespace, name), - None => ToolName::plain(name), - }; - if let Some(tool_info) = mcp_tool { + let tool_name = ToolName::new(namespace, name); + if let Some(tool_info) = session.resolve_mcp_tool_info(&tool_name).await { Ok(Some(ToolCall { - tool_name, + tool_name: tool_info.canonical_tool_name(), call_id, payload: ToolPayload::Mcp { server: tool_info.server_name, diff --git a/codex-rs/core/src/tools/spec.rs b/codex-rs/core/src/tools/spec.rs index 188be42ba1..9c05e6cc35 100644 --- a/codex-rs/core/src/tools/spec.rs +++ b/codex-rs/core/src/tools/spec.rs @@ -11,6 +11,7 @@ use codex_tools::DiscoverableTool; use codex_tools::ToolHandlerKind; use codex_tools::ToolNamespace; use codex_tools::ToolRegistryPlanDeferredTool; +use codex_tools::ToolRegistryPlanMcpTool; use codex_tools::ToolRegistryPlanParams; use codex_tools::ToolUserShellType; use codex_tools::ToolsConfig; @@ -29,25 +30,31 @@ pub(crate) fn tool_user_shell_type(user_shell: &Shell) -> ToolUserShellType { } } -struct McpToolPlanInputs { - mcp_tools: HashMap, +struct McpToolPlanInputs<'a> { + mcp_tools: Vec>, tool_namespaces: HashMap, } -fn map_mcp_tools_for_plan(mcp_tools: &HashMap) -> McpToolPlanInputs { +fn map_mcp_tools_for_plan(mcp_tools: &HashMap) -> McpToolPlanInputs<'_> { McpToolPlanInputs { mcp_tools: mcp_tools - .iter() - .map(|(name, tool)| (name.clone(), tool.tool.clone())) + .values() + .map(|tool| ToolRegistryPlanMcpTool { + name: tool.canonical_tool_name(), + tool: &tool.tool, + }) .collect(), tool_namespaces: mcp_tools - .iter() - .map(|(name, tool)| { + .values() + .map(|tool| { ( - name.clone(), + tool.callable_namespace.clone(), ToolNamespace { name: tool.callable_namespace.clone(), - description: tool.server_instructions.clone(), + description: tool + .connector_description + .clone() + .or_else(|| tool.server_instructions.clone()), }, ) }) @@ -99,8 +106,7 @@ pub(crate) fn build_specs_with_discoverable_tools( tools .values() .map(|tool| ToolRegistryPlanDeferredTool { - tool_name: tool.callable_name.as_str(), - tool_namespace: tool.callable_namespace.as_str(), + name: tool.canonical_tool_name(), server_name: tool.server_name.as_str(), connector_name: tool.connector_name.as_deref(), connector_description: tool.connector_description.as_deref(), @@ -114,7 +120,7 @@ pub(crate) fn build_specs_with_discoverable_tools( ToolRegistryPlanParams { mcp_tools: mcp_tool_plan_inputs .as_ref() - .map(|inputs| &inputs.mcp_tools), + .map(|inputs| inputs.mcp_tools.as_slice()), deferred_mcp_tools: deferred_mcp_tool_sources.as_deref(), tool_namespaces: mcp_tool_plan_inputs .as_ref() diff --git a/codex-rs/core/src/tools/spec_tests.rs b/codex-rs/core/src/tools/spec_tests.rs index b0fe3f8436..ee993b0d53 100644 --- a/codex-rs/core/src/tools/spec_tests.rs +++ b/codex-rs/core/src/tools/spec_tests.rs @@ -19,6 +19,7 @@ use codex_protocol::protocol::SessionSource; use codex_tools::ConfiguredToolSpec; use codex_tools::DiscoverableTool; use codex_tools::JsonSchema; +use codex_tools::ResponsesApiNamespaceTool; use codex_tools::ResponsesApiTool; use codex_tools::ShellCommandBackendConfig; use codex_tools::TOOL_SEARCH_TOOL_NAME; @@ -67,6 +68,25 @@ fn mcp_tool_info(tool: rmcp::model::Tool) -> ToolInfo { } } +fn mcp_tool_info_with_display_name(display_name: &str, tool: rmcp::model::Tool) -> ToolInfo { + let (callable_namespace, callable_name) = display_name + .rsplit_once('/') + .map(|(namespace, callable_name)| (format!("{namespace}/"), callable_name.to_string())) + .unwrap_or_else(|| ("".to_string(), display_name.to_string())); + + ToolInfo { + server_name: "test_server".to_string(), + callable_name, + callable_namespace, + server_instructions: None, + tool, + connector_id: None, + connector_name: None, + plugin_display_names: Vec::new(), + connector_description: None, + } +} + fn discoverable_connector(id: &str, name: &str, description: &str) -> DiscoverableTool { let slug = name.replace(' ', "-").to_lowercase(); DiscoverableTool::Connector(Box::new(AppInfo { @@ -109,8 +129,11 @@ fn deferred_responses_api_tool_serializes_with_defer_loading() { ); let serialized = serde_json::to_value(ToolSpec::Function( - mcp_tool_to_deferred_responses_api_tool("mcp__codex_apps__lookup_order".to_string(), &tool) - .expect("convert deferred tool"), + mcp_tool_to_deferred_responses_api_tool( + &ToolName::namespaced("mcp__codex_apps__", "lookup_order"), + &tool, + ) + .expect("convert deferred tool"), )) .expect("serialize deferred tool"); @@ -118,7 +141,7 @@ fn deferred_responses_api_tool_serializes_with_defer_loading() { serialized, serde_json::json!({ "type": "function", - "name": "mcp__codex_apps__lookup_order", + "name": "lookup_order", "description": "Look up an order", "strict": false, "defer_loading": true, @@ -173,6 +196,25 @@ fn find_tool<'a>(tools: &'a [ConfiguredToolSpec], expected_name: &str) -> &'a Co .unwrap_or_else(|| panic!("expected tool {expected_name}")) } +fn find_namespace_function_tool<'a>( + tools: &'a [ConfiguredToolSpec], + expected_namespace: &str, + expected_name: &str, +) -> &'a ResponsesApiTool { + let namespace_tool = find_tool(tools, expected_namespace); + let ToolSpec::Namespace(namespace) = &namespace_tool.spec else { + panic!("expected namespace tool {expected_namespace}"); + }; + namespace + .tools + .iter() + .find_map(|tool| match tool { + ResponsesApiNamespaceTool::Function(tool) if tool.name == expected_name => Some(tool), + _ => None, + }) + .unwrap_or_else(|| panic!("expected tool {expected_namespace}{expected_name} in namespace")) +} + fn multi_agent_v2_tools_config() -> ToolsConfig { let config = test_config(); let model_info = construct_model_info_offline("gpt-5-codex", &config); @@ -910,8 +952,43 @@ fn search_tool_registers_namespaced_mcp_tool_aliases() { assert!(registry.has_handler(&ToolName::plain(TOOL_SEARCH_TOOL_NAME))); assert!(registry.has_handler(&app_alias)); assert!(registry.has_handler(&mcp_alias)); - assert!(registry.has_handler(&ToolName::plain("mcp__codex_apps__calendar_create_event"))); - assert!(registry.has_handler(&ToolName::plain("mcp__rmcp__echo"))); +} + +#[test] +fn direct_mcp_tools_register_namespaced_handlers() { + let config = test_config(); + let model_info = construct_model_info_offline("gpt-5-codex", &config); + let mut features = Features::with_defaults(); + features.enable(Feature::UnifiedExec); + let available_models = Vec::new(); + let tools_config = ToolsConfig::new(&ToolsConfigParams { + model_info: &model_info, + available_models: &available_models, + features: &features, + image_generation_tool_auth_allowed: true, + web_search_mode: Some(WebSearchMode::Cached), + session_source: SessionSource::Cli, + sandbox_policy: &SandboxPolicy::DangerFullAccess, + windows_sandbox_level: WindowsSandboxLevel::Disabled, + }); + + let (_, registry) = build_specs( + &tools_config, + Some(HashMap::from([( + "mcp__test_server__echo".to_string(), + mcp_tool_info(mcp_tool( + "echo", + "Echo", + serde_json::json!({"type": "object"}), + )), + )])), + /*deferred_mcp_tools*/ None, + &[], + ) + .build(); + + assert!(registry.has_handler(&ToolName::namespaced("mcp__test_server__", "echo"))); + assert!(!registry.has_handler(&ToolName::plain("mcp__test_server__echo"))); } #[test] @@ -936,27 +1013,30 @@ fn test_mcp_tool_property_missing_type_defaults_to_string() { &tools_config, Some(HashMap::from([( "dash/search".to_string(), - mcp_tool_info(mcp_tool( - "search", - "Search docs", - serde_json::json!({ - "type": "object", - "properties": { - "query": {"description": "search query"} - } - }), - )), + mcp_tool_info_with_display_name( + "dash/search", + mcp_tool( + "search", + "Search docs", + serde_json::json!({ + "type": "object", + "properties": { + "query": {"description": "search query"} + } + }), + ), + ), )])), /*deferred_mcp_tools*/ None, &[], ) .build(); - let tool = find_tool(&tools, "dash/search"); + let tool = find_namespace_function_tool(&tools, "dash/", "search"); assert_eq!( - tool.spec, - ToolSpec::Function(ResponsesApiTool { - name: "dash/search".to_string(), + *tool, + ResponsesApiTool { + name: "search".to_string(), parameters: JsonSchema::object( /*properties*/ BTreeMap::from([( @@ -970,7 +1050,7 @@ fn test_mcp_tool_property_missing_type_defaults_to_string() { strict: false, output_schema: Some(mcp_call_tool_result_output_schema(serde_json::json!({}))), defer_loading: None, - }) + } ); } @@ -996,25 +1076,28 @@ fn test_mcp_tool_preserves_integer_schema() { &tools_config, Some(HashMap::from([( "dash/paginate".to_string(), - mcp_tool_info(mcp_tool( - "paginate", - "Pagination", - serde_json::json!({ - "type": "object", - "properties": {"page": {"type": "integer"}} - }), - )), + mcp_tool_info_with_display_name( + "dash/paginate", + mcp_tool( + "paginate", + "Pagination", + serde_json::json!({ + "type": "object", + "properties": {"page": {"type": "integer"}} + }), + ), + ), )])), /*deferred_mcp_tools*/ None, &[], ) .build(); - let tool = find_tool(&tools, "dash/paginate"); + let tool = find_namespace_function_tool(&tools, "dash/", "paginate"); assert_eq!( - tool.spec, - ToolSpec::Function(ResponsesApiTool { - name: "dash/paginate".to_string(), + *tool, + ResponsesApiTool { + name: "paginate".to_string(), parameters: JsonSchema::object( /*properties*/ BTreeMap::from([( @@ -1028,7 +1111,7 @@ fn test_mcp_tool_preserves_integer_schema() { strict: false, output_schema: Some(mcp_call_tool_result_output_schema(serde_json::json!({}))), defer_loading: None, - }) + } ); } @@ -1055,25 +1138,28 @@ fn test_mcp_tool_array_without_items_gets_default_string_items() { &tools_config, Some(HashMap::from([( "dash/tags".to_string(), - mcp_tool_info(mcp_tool( - "tags", - "Tags", - serde_json::json!({ - "type": "object", - "properties": {"tags": {"type": "array"}} - }), - )), + mcp_tool_info_with_display_name( + "dash/tags", + mcp_tool( + "tags", + "Tags", + serde_json::json!({ + "type": "object", + "properties": {"tags": {"type": "array"}} + }), + ), + ), )])), /*deferred_mcp_tools*/ None, &[], ) .build(); - let tool = find_tool(&tools, "dash/tags"); + let tool = find_namespace_function_tool(&tools, "dash/", "tags"); assert_eq!( - tool.spec, - ToolSpec::Function(ResponsesApiTool { - name: "dash/tags".to_string(), + *tool, + ResponsesApiTool { + name: "tags".to_string(), parameters: JsonSchema::object( /*properties*/ BTreeMap::from([( @@ -1090,7 +1176,7 @@ fn test_mcp_tool_array_without_items_gets_default_string_items() { strict: false, output_schema: Some(mcp_call_tool_result_output_schema(serde_json::json!({}))), defer_loading: None, - }) + } ); } @@ -1116,27 +1202,30 @@ fn test_mcp_tool_anyof_defaults_to_string() { &tools_config, Some(HashMap::from([( "dash/value".to_string(), - mcp_tool_info(mcp_tool( - "value", - "AnyOf Value", - serde_json::json!({ - "type": "object", - "properties": { - "value": {"anyOf": [{"type": "string"}, {"type": "number"}]} - } - }), - )), + mcp_tool_info_with_display_name( + "dash/value", + mcp_tool( + "value", + "AnyOf Value", + serde_json::json!({ + "type": "object", + "properties": { + "value": {"anyOf": [{"type": "string"}, {"type": "number"}]} + } + }), + ), + ), )])), /*deferred_mcp_tools*/ None, &[], ) .build(); - let tool = find_tool(&tools, "dash/value"); + let tool = find_namespace_function_tool(&tools, "dash/", "value"); assert_eq!( - tool.spec, - ToolSpec::Function(ResponsesApiTool { - name: "dash/value".to_string(), + *tool, + ResponsesApiTool { + name: "value".to_string(), parameters: JsonSchema::object( /*properties*/ BTreeMap::from([( @@ -1156,7 +1245,7 @@ fn test_mcp_tool_anyof_defaults_to_string() { strict: false, output_schema: Some(mcp_call_tool_result_output_schema(serde_json::json!({}))), defer_loading: None, - }) + } ); } @@ -1181,12 +1270,14 @@ fn test_get_openai_tools_mcp_tools_with_additional_properties_schema() { &tools_config, Some(HashMap::from([( "test_server/do_something_cool".to_string(), - mcp_tool_info(mcp_tool( - "do_something_cool", - "Do something cool", - serde_json::json!({ - "type": "object", - "properties": { + mcp_tool_info_with_display_name( + "test_server/do_something_cool", + mcp_tool( + "do_something_cool", + "Do something cool", + serde_json::json!({ + "type": "object", + "properties": { "string_argument": {"type": "string"}, "number_argument": {"type": "number"}, "object_argument": { @@ -1203,22 +1294,23 @@ fn test_get_openai_tools_mcp_tools_with_additional_properties_schema() { }, "required": ["addtl_prop"], "additionalProperties": false + } } } - } - }), - )), + }), + ), + ), )])), /*deferred_mcp_tools*/ None, &[], ) .build(); - let tool = find_tool(&tools, "test_server/do_something_cool"); + let tool = find_namespace_function_tool(&tools, "test_server/", "do_something_cool"); assert_eq!( - tool.spec, - ToolSpec::Function(ResponsesApiTool { - name: "test_server/do_something_cool".to_string(), + *tool, + ResponsesApiTool { + name: "do_something_cool".to_string(), parameters: JsonSchema::object( /*properties*/ BTreeMap::from([ @@ -1268,7 +1360,7 @@ fn test_get_openai_tools_mcp_tools_with_additional_properties_schema() { strict: false, output_schema: Some(mcp_call_tool_result_output_schema(serde_json::json!({}))), defer_loading: None, - }) + } ); } diff --git a/codex-rs/core/tests/common/responses.rs b/codex-rs/core/tests/common/responses.rs index 2e2155ebdd..8def3ac6ea 100644 --- a/codex-rs/core/tests/common/responses.rs +++ b/codex-rs/core/tests/common/responses.rs @@ -125,6 +125,10 @@ impl ResponsesRequest { self.body_json().to_string().contains(&json_fragment) } + pub fn tool_by_name(&self, namespace: &str, tool_name: &str) -> Option { + namespace_child_tool(&self.body_json(), namespace, tool_name).cloned() + } + pub fn instructions_text(&self) -> String { self.body_json()["instructions"] .as_str() @@ -315,6 +319,31 @@ pub(crate) fn output_value_to_text(value: &Value) -> Option { } } +pub fn namespace_child_tool<'a>( + body: &'a Value, + namespace: &str, + tool_name: &str, +) -> Option<&'a Value> { + let tools = body.get("tools")?.as_array()?; + for tool in tools { + if tool.get("name").and_then(Value::as_str) != Some(namespace) + || tool.get("type").and_then(Value::as_str) != Some("namespace") + { + continue; + } + + let child_tools = tool.get("tools")?.as_array()?; + if let Some(child_tool) = child_tools + .iter() + .find(|tool| tool.get("name").and_then(Value::as_str) == Some(tool_name)) + { + return Some(child_tool); + } + } + + None +} + #[cfg(test)] mod tests { use super::*; @@ -780,6 +809,24 @@ pub fn ev_function_call(call_id: &str, name: &str, arguments: &str) -> Value { }) } +pub fn ev_function_call_with_namespace( + call_id: &str, + namespace: &str, + name: &str, + arguments: &str, +) -> Value { + serde_json::json!({ + "type": "response.output_item.done", + "item": { + "type": "function_call", + "call_id": call_id, + "namespace": namespace, + "name": name, + "arguments": arguments + } + }) +} + pub fn ev_tool_search_call(call_id: &str, arguments: &serde_json::Value) -> Value { serde_json::json!({ "type": "response.output_item.done", diff --git a/codex-rs/core/tests/common/test_codex.rs b/codex-rs/core/tests/common/test_codex.rs index b1274d5a94..92176c338f 100644 --- a/codex-rs/core/tests/common/test_codex.rs +++ b/codex-rs/core/tests/common/test_codex.rs @@ -7,8 +7,6 @@ use std::process::Command; use std::sync::Arc; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; -use std::time::Duration; -use std::time::Instant; use anyhow::Context; use anyhow::Result; @@ -44,7 +42,6 @@ use tempfile::TempDir; use wiremock::MockServer; use crate::PathBufExt; -use crate::RemoteEnvConfig; use crate::TempDirExt; use crate::get_remote_test_env; use crate::load_default_config_for_test; @@ -62,50 +59,15 @@ type PreBuildHook = dyn FnOnce(&Path) + Send + 'static; type WorkspaceSetup = dyn FnOnce(AbsolutePathBuf, Arc) -> BoxFuture<'static, Result<()>> + Send; const TEST_MODEL_WITH_EXPERIMENTAL_TOOLS: &str = "test-gpt-5.1-codex"; -const REMOTE_EXEC_SERVER_START_TIMEOUT: Duration = Duration::from_secs(5); -const REMOTE_EXEC_SERVER_POLL_INTERVAL: Duration = Duration::from_millis(25); -static REMOTE_EXEC_SERVER_INSTANCE_COUNTER: AtomicU64 = AtomicU64::new(0); - -#[derive(Debug)] -struct RemoteExecServerProcess { - container_name: String, - pid: u32, - remote_exec_server_path: String, - stdout_path: String, - cleanup_paths: Vec, -} - -impl Drop for RemoteExecServerProcess { - fn drop(&mut self) { - let cleanup_paths = self.cleanup_paths.join(" "); - let cleanup_paths_script = if cleanup_paths.is_empty() { - String::new() - } else { - format!("rm -rf {cleanup_paths}; ") - }; - let script = format!( - "if kill -0 {pid} 2>/dev/null; then kill {pid}; fi; {cleanup_paths_script}rm -f {remote_exec_server_path} {stdout_path}", - pid = self.pid, - cleanup_paths_script = cleanup_paths_script, - remote_exec_server_path = self.remote_exec_server_path, - stdout_path = self.stdout_path - ); - let _ = docker_command_capture_stdout(["exec", &self.container_name, "sh", "-lc", &script]); - } -} - -impl RemoteExecServerProcess { - fn register_cleanup_path(&mut self, path: &Path) { - self.cleanup_paths.push(path.display().to_string()); - } -} +const REMOTE_EXEC_SERVER_URL_ENV_VAR: &str = "CODEX_TEST_REMOTE_EXEC_SERVER_URL"; +static REMOTE_TEST_INSTANCE_COUNTER: AtomicU64 = AtomicU64::new(0); #[derive(Debug)] pub struct TestEnv { environment: codex_exec_server::Environment, cwd: AbsolutePathBuf, local_cwd_temp_dir: Option>, - _remote_exec_server_process: Option, + remote_container_name: Option, } impl TestEnv { @@ -117,7 +79,7 @@ impl TestEnv { environment, cwd, local_cwd_temp_dir: Some(local_cwd_temp_dir), - _remote_exec_server_process: None, + remote_container_name: None, }) } @@ -138,12 +100,19 @@ impl TestEnv { } } +impl Drop for TestEnv { + fn drop(&mut self) { + if let Some(container_name) = &self.remote_container_name { + let script = format!("rm -rf {}", self.cwd.as_path().display()); + let _ = docker_command_capture_stdout(["exec", container_name, "sh", "-lc", &script]); + } + } +} + pub async fn test_env() -> Result { match get_remote_test_env() { Some(remote_env) => { - let mut remote_process = start_remote_exec_server(&remote_env)?; - let remote_ip = remote_container_ip(&remote_env.container_name)?; - let websocket_url = rewrite_websocket_host(&remote_process.listen_url, &remote_ip)?; + let websocket_url = remote_exec_server_url()?; let environment = codex_exec_server::Environment::create(Some(websocket_url)).await?; let cwd = remote_aware_cwd_path(); environment @@ -154,184 +123,43 @@ pub async fn test_env() -> Result { /*sandbox*/ None, ) .await?; - remote_process.process.register_cleanup_path(cwd.as_path()); Ok(TestEnv { environment, cwd, local_cwd_temp_dir: None, - _remote_exec_server_process: Some(remote_process.process), + remote_container_name: Some(remote_env.container_name), }) } None => TestEnv::local().await, } } -struct RemoteExecServerStart { - process: RemoteExecServerProcess, - listen_url: String, -} - -fn start_remote_exec_server(remote_env: &RemoteEnvConfig) -> Result { - let container_name = remote_env.container_name.as_str(); - let instance_id = remote_exec_server_instance_id(); - let remote_exec_server_path = format!("/tmp/codex-{instance_id}"); - let remote_linux_sandbox_path = format!("/tmp/codex-linux-sandbox-{instance_id}"); - let stdout_path = format!("/tmp/codex-exec-server-{instance_id}.stdout"); - let local_binary = codex_utils_cargo_bin::cargo_bin("codex").context("resolve codex binary")?; - let local_linux_sandbox = codex_utils_cargo_bin::cargo_bin("codex-linux-sandbox") - .context("resolve codex-linux-sandbox binary")?; - let local_binary = local_binary.to_string_lossy().to_string(); - let local_linux_sandbox = local_linux_sandbox.to_string_lossy().to_string(); - let remote_binary = format!("{container_name}:{remote_exec_server_path}"); - let remote_linux_sandbox = format!("{container_name}:{remote_linux_sandbox_path}"); - - docker_command_success(["cp", &local_binary, &remote_binary])?; - docker_command_success(["cp", &local_linux_sandbox, &remote_linux_sandbox])?; - docker_command_success([ - "exec", - container_name, - "chmod", - "+x", - &remote_exec_server_path, - ])?; - docker_command_success([ - "exec", - container_name, - "chmod", - "+x", - &remote_linux_sandbox_path, - ])?; - probe_remote_linux_sandbox(container_name, &remote_linux_sandbox_path)?; - - let start_script = format!( - "rm -f {stdout_path}; \ -nohup {remote_exec_server_path} exec-server --listen ws://0.0.0.0:0 > {stdout_path} 2>&1 & \ -echo $!" - ); - let pid_output = - docker_command_capture_stdout(["exec", container_name, "sh", "-lc", &start_script])?; - let pid = pid_output - .trim() - .parse::() - .with_context(|| format!("parse remote exec-server PID from {pid_output:?}"))?; - - let listen_url = wait_for_remote_listen_url(container_name, &stdout_path)?; - - Ok(RemoteExecServerStart { - process: RemoteExecServerProcess { - container_name: container_name.to_string(), - pid, - remote_exec_server_path, - stdout_path, - cleanup_paths: vec![remote_linux_sandbox_path], - }, - listen_url, - }) -} - -fn probe_remote_linux_sandbox(container_name: &str, remote_linux_sandbox_path: &str) -> Result<()> { - let policy = serde_json::to_string(&SandboxPolicy::new_read_only_policy()) - .context("serialize remote sandbox probe policy")?; - let probe_script = format!( - "{remote_linux_sandbox_path} --sandbox-policy-cwd /tmp --sandbox-policy '{policy}' -- /bin/true" - ); - let output = Command::new("docker") - .args(["exec", container_name, "sh", "-lc", &probe_script]) - .output() - .with_context(|| format!("probe remote linux sandbox in container `{container_name}`"))?; - if !output.status.success() { - return Err(anyhow!( - "remote linux sandbox probe failed in container `{container_name}`: stdout={} stderr={}", - String::from_utf8_lossy(&output.stdout).trim(), - String::from_utf8_lossy(&output.stderr).trim() - )); - } - Ok(()) -} - fn remote_aware_cwd_path() -> AbsolutePathBuf { PathBuf::from(format!( "/tmp/codex-core-test-cwd-{}", - remote_exec_server_instance_id() + remote_test_instance_id() )) .abs() } -fn wait_for_remote_listen_url(container_name: &str, stdout_path: &str) -> Result { - let deadline = Instant::now() + REMOTE_EXEC_SERVER_START_TIMEOUT; - loop { - let line = docker_command_capture_stdout([ - "exec", - container_name, - "sh", - "-lc", - &format!("head -n 1 {stdout_path} 2>/dev/null || true"), - ])?; - let listen_url = line.trim(); - if listen_url.starts_with("ws://") { - return Ok(listen_url.to_string()); - } - - if Instant::now() >= deadline { - return Err(anyhow!( - "timed out waiting for remote exec-server listen URL in container `{container_name}` after {REMOTE_EXEC_SERVER_START_TIMEOUT:?}" - )); - } - std::thread::sleep(REMOTE_EXEC_SERVER_POLL_INTERVAL); +fn remote_exec_server_url() -> Result { + let listen_url = std::env::var(REMOTE_EXEC_SERVER_URL_ENV_VAR).with_context(|| { + format!("{REMOTE_EXEC_SERVER_URL_ENV_VAR} must be set for remote tests") + })?; + let listen_url = listen_url.trim(); + if listen_url.is_empty() { + return Err(anyhow!( + "{REMOTE_EXEC_SERVER_URL_ENV_VAR} must not be empty" + )); } + Ok(listen_url.to_string()) } -fn remote_exec_server_instance_id() -> String { - let instance = REMOTE_EXEC_SERVER_INSTANCE_COUNTER.fetch_add(1, Ordering::Relaxed); +fn remote_test_instance_id() -> String { + let instance = REMOTE_TEST_INSTANCE_COUNTER.fetch_add(1, Ordering::Relaxed); format!("{}-{instance}", std::process::id()) } -fn remote_container_ip(container_name: &str) -> Result { - let ip = docker_command_capture_stdout([ - "inspect", - "-f", - "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", - container_name, - ])?; - let ip = ip.trim(); - if ip.is_empty() { - return Err(anyhow!( - "container `{container_name}` has no IP address; cannot connect to remote exec-server" - )); - } - Ok(ip.to_string()) -} - -fn rewrite_websocket_host(listen_url: &str, host: &str) -> Result { - let Some(address) = listen_url.strip_prefix("ws://") else { - return Err(anyhow!( - "unexpected websocket listen URL `{listen_url}`; expected ws://IP:PORT" - )); - }; - let Some((_, port)) = address.rsplit_once(':') else { - return Err(anyhow!( - "unexpected websocket listen URL `{listen_url}`; expected ws://IP:PORT" - )); - }; - Ok(format!("ws://{host}:{port}")) -} - -fn docker_command_success(args: [&str; N]) -> Result<()> { - let output = Command::new("docker") - .args(args) - .output() - .with_context(|| format!("run docker {args:?}"))?; - if !output.status.success() { - return Err(anyhow!( - "docker {:?} failed: stdout={} stderr={}", - args, - String::from_utf8_lossy(&output.stdout).trim(), - String::from_utf8_lossy(&output.stderr).trim() - )); - } - Ok(()) -} - fn docker_command_capture_stdout(args: [&str; N]) -> Result { let output = Command::new("docker") .args(args) diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index 76a70d5076..f010c9e587 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -177,12 +177,25 @@ async fn run_code_mode_turn_with_rmcp( server: &MockServer, prompt: &str, code: &str, +) -> Result<(TestCodex, ResponseMock)> { + run_code_mode_turn_with_rmcp_mode(server, prompt, code, /*code_mode_only*/ false).await +} + +async fn run_code_mode_turn_with_rmcp_mode( + server: &MockServer, + prompt: &str, + code: &str, + code_mode_only: bool, ) -> Result<(TestCodex, ResponseMock)> { let rmcp_test_server_bin = stdio_server_bin()?; let mut builder = test_codex() .with_model("test-gpt-5.1-codex") .with_config(move |config| { - let _ = config.features.enable(Feature::CodeMode); + let _ = if code_mode_only { + config.features.enable(Feature::CodeModeOnly) + } else { + config.features.enable(Feature::CodeMode) + }; let mut servers = config.mcp_servers.get().clone(); servers.insert( @@ -1989,6 +2002,36 @@ contentLength=0" Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn code_mode_only_can_call_mcp_tool() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let code = r#" +const result = await tools.mcp__rmcp__echo({ message: "ping" }); +text(`echo=${result.structuredContent?.echo ?? "missing"}`); +"#; + + let (_test, second_mock) = run_code_mode_turn_with_rmcp_mode( + &server, + "use exec to run the rmcp echo tool in code mode only", + code, + /*code_mode_only*/ true, + ) + .await?; + + let req = second_mock.single_request(); + let (output, success) = custom_tool_output_body_and_success(&req, "call-1"); + assert_ne!( + success, + Some(false), + "code_mode_only rmcp tool call failed unexpectedly: {output}" + ); + assert_eq!(output, "echo=ECHOING: ping"); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn code_mode_exposes_mcp_tools_on_global_tools_object() -> Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/core/tests/suite/exec.rs b/codex-rs/core/tests/suite/exec.rs index 19869ebb2b..c80c8a3340 100644 --- a/codex-rs/core/tests/suite/exec.rs +++ b/codex-rs/core/tests/suite/exec.rs @@ -1,8 +1,5 @@ #![cfg(target_os = "macos")] -use std::collections::HashMap; -use std::string::ToString; - use codex_core::exec::ExecCapturePolicy; use codex_core::exec::ExecParams; use codex_core::exec::process_exec_tool_call; @@ -17,6 +14,7 @@ use codex_protocol::protocol::SandboxPolicy; use codex_sandboxing::SandboxType; use codex_sandboxing::get_platform_sandbox; use core_test_support::PathExt; +use std::collections::HashMap; use tempfile::TempDir; fn skip_test() -> bool { @@ -29,14 +27,18 @@ fn skip_test() -> bool { } #[expect(clippy::expect_used)] -async fn run_test_cmd(tmp: TempDir, cmd: Vec<&str>) -> Result { +async fn run_test_cmd(tmp: TempDir, command: I) -> Result +where + I: IntoIterator, + S: Into, +{ let sandbox_type = get_platform_sandbox(/*windows_sandbox_enabled*/ false) .expect("should be able to get sandbox type"); assert_eq!(sandbox_type, SandboxType::MacosSeatbelt); let cwd = tmp.path().abs(); let params = ExecParams { - command: cmd.iter().map(ToString::to_string).collect(), + command: command.into_iter().map(Into::into).collect(), cwd: cwd.clone(), expiration: 1000.into(), capture_policy: ExecCapturePolicy::ShellTool, @@ -129,6 +131,37 @@ async fn exit_command_not_found_is_ok() { run_test_cmd(tmp, cmd).await.unwrap(); } +#[tokio::test] +async fn openpty_works_under_real_exec_seatbelt_path() { + if skip_test() { + return; + } + + let python = match which::which("python3") { + Ok(path) => path, + Err(_) => { + eprintln!("python3 not found in PATH, skipping test."); + return; + } + }; + + let tmp = TempDir::new().expect("should be able to create temp dir"); + let cmd = vec![ + python.to_string_lossy().into_owned(), + "-c".to_string(), + r#"import os + +master, slave = os.openpty() +os.write(slave, b"ping") +assert os.read(master, 4) == b"ping""# + .to_string(), + ]; + + let output = run_test_cmd(tmp, cmd).await.unwrap(); + assert_eq!(output.stdout.text, ""); + assert_eq!(output.stderr.text, ""); +} + /// Writing a file fails and should be considered a sandbox error #[tokio::test] async fn write_file_fails_as_sandbox_error() { @@ -139,7 +172,7 @@ async fn write_file_fails_as_sandbox_error() { let tmp = TempDir::new().expect("should be able to create temp dir"); let path = tmp.path().join("test.txt"); let cmd = vec![ - "/user/bin/touch", + "/usr/bin/touch", path.to_str().expect("should be able to get path"), ]; diff --git a/codex-rs/core/tests/suite/js_repl.rs b/codex-rs/core/tests/suite/js_repl.rs index 5f52c24e19..2016897b3b 100644 --- a/codex-rs/core/tests/suite/js_repl.rs +++ b/codex-rs/core/tests/suite/js_repl.rs @@ -1,6 +1,8 @@ #![allow(clippy::expect_used, clippy::unwrap_used)] use anyhow::Result; +use codex_config::types::McpServerConfig; +use codex_config::types::McpServerTransportConfig; use codex_features::Feature; use codex_protocol::protocol::EventMsg; use core_test_support::responses; @@ -12,12 +14,15 @@ use core_test_support::responses::ev_custom_tool_call; use core_test_support::responses::ev_response_created; use core_test_support::responses::sse; use core_test_support::skip_if_no_network; +use core_test_support::stdio_server_bin; use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event_match; +use std::collections::HashMap; use std::fs; #[cfg(unix)] use std::os::unix::fs::PermissionsExt; use std::path::Path; +use std::time::Duration; use tempfile::tempdir; use wiremock::MockServer; @@ -594,6 +599,82 @@ async fn js_repl_can_invoke_builtin_tools() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn js_repl_can_invoke_mcp_tools_by_display_name() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let rmcp_test_server_bin = stdio_server_bin()?; + let mut builder = test_codex().with_config(move |config| { + config + .features + .enable(Feature::JsRepl) + .expect("test config should allow feature update"); + + let mut servers = config.mcp_servers.get().clone(); + servers.insert( + "rmcp".to_string(), + McpServerConfig { + transport: McpServerTransportConfig::Stdio { + command: rmcp_test_server_bin, + args: Vec::new(), + env: None, + env_vars: Vec::new(), + cwd: None, + }, + enabled: true, + required: false, + supports_parallel_tool_calls: false, + disabled_reason: None, + startup_timeout_sec: Some(Duration::from_secs(10)), + tool_timeout_sec: None, + enabled_tools: None, + disabled_tools: None, + scopes: None, + oauth_resource: None, + tools: HashMap::new(), + }, + ); + config + .mcp_servers + .set(servers) + .expect("test mcp servers should accept any configuration"); + }); + let test = builder.build(&server).await?; + + responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-1"), + ev_custom_tool_call( + "call-1", + "js_repl", + r#" +const result = await codex.tool("mcp__rmcp__echo", { message: "ping" }); +console.log(result.output); +"#, + ), + ev_completed("resp-1"), + ]), + ) + .await; + let final_mock = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]), + ) + .await; + + test.submit_turn("use js_repl to call an MCP tool").await?; + + let req = final_mock.single_request(); + assert_js_repl_ok(&req, "call-1", "ECHOING: ping"); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn js_repl_tool_call_rejects_recursive_js_repl_invocation() -> Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/core/tests/suite/memories.rs b/codex-rs/core/tests/suite/memories.rs index 98e6186437..60685592f6 100644 --- a/codex-rs/core/tests/suite/memories.rs +++ b/codex-rs/core/tests/suite/memories.rs @@ -159,6 +159,145 @@ async fn memories_startup_phase2_tracks_added_and_removed_inputs_across_runs() - Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn memories_startup_phase2_prunes_old_extension_resources_and_reports_them() -> Result<()> { + let server = start_mock_server().await; + let home = Arc::new(TempDir::new()?); + let db = init_state_db(&home).await?; + let now = Utc::now(); + let thread_id = seed_stage1_output( + db.as_ref(), + home.path(), + now - ChronoDuration::hours(1), + "raw memory", + "rollout summary", + "rollout", + ) + .await?; + + let telepathy_resources = home.path().join("memories_extensions/telepathy/resources"); + tokio::fs::create_dir_all(&telepathy_resources).await?; + tokio::fs::write( + home.path() + .join("memories_extensions/telepathy/instructions.md"), + "instructions", + ) + .await?; + let old_file_name = format!( + "{}-abcd-10min-old.md", + (now - ChronoDuration::days(8)).format("%Y-%m-%dT%H-%M-%S") + ); + let old_file = telepathy_resources.join(&old_file_name); + tokio::fs::write(&old_file, "old resource").await?; + let recent_file = telepathy_resources.join(format!( + "{}-abcd-10min-recent.md", + (now - ChronoDuration::days(6)).format("%Y-%m-%dT%H-%M-%S") + )); + tokio::fs::write(&recent_file, "recent resource").await?; + + let phase2 = mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-phase2"), + ev_assistant_message("msg-phase2", "phase2 complete"), + ev_completed("resp-phase2"), + ]), + ) + .await; + + let codex = build_test_codex(&server, home.clone()).await?; + let request = wait_for_single_request(&phase2).await; + let prompt = phase2_prompt_text(&request); + + assert!( + prompt.contains("Memory extension resources removed by retention pruning:"), + "expected extension resource prune report in prompt: {prompt}" + ); + assert!( + prompt.contains("- retention window: 7 days"), + "expected retention window in prompt: {prompt}" + ); + assert!( + prompt.contains("- extension: telepathy"), + "expected extension name in prompt: {prompt}" + ); + assert!( + prompt.contains(&format!(" - resources/{old_file_name}")), + "expected old resource in prompt: {prompt}" + ); + + wait_for_phase2_success(db.as_ref(), thread_id).await?; + wait_for_file_removed(&old_file).await?; + assert!( + !tokio::fs::try_exists(&old_file).await?, + "old extension resource should be pruned" + ); + assert!( + tokio::fs::try_exists(&recent_file).await?, + "recent extension resource should be retained" + ); + + shutdown_test_codex(&codex).await?; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn memories_startup_phase2_processes_old_extension_resources_without_stage1_input() +-> Result<()> { + let server = start_mock_server().await; + let home = Arc::new(TempDir::new()?); + let db = init_state_db(&home).await?; + db.enqueue_global_consolidation(/*input_watermark*/ 1) + .await?; + + let now = Utc::now(); + let telepathy_resources = home.path().join("memories_extensions/telepathy/resources"); + tokio::fs::create_dir_all(&telepathy_resources).await?; + tokio::fs::write( + home.path() + .join("memories_extensions/telepathy/instructions.md"), + "instructions", + ) + .await?; + let old_file_name = format!( + "{}-abcd-10min-old.md", + (now - ChronoDuration::days(8)).format("%Y-%m-%dT%H-%M-%S") + ); + let old_file = telepathy_resources.join(&old_file_name); + tokio::fs::write(&old_file, "old resource").await?; + + let phase2 = mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-phase2-empty"), + ev_assistant_message("msg-phase2-empty", "phase2 complete"), + ev_completed("resp-phase2-empty"), + ]), + ) + .await; + + let codex = build_test_codex(&server, home.clone()).await?; + let request = wait_for_single_request(&phase2).await; + let prompt = phase2_prompt_text(&request); + + assert!( + prompt.contains("- selected inputs this run: 0"), + "expected no selected raw inputs in prompt: {prompt}" + ); + assert!( + prompt.contains("Memory extension resources removed by retention pruning:"), + "expected extension resource prune report in prompt: {prompt}" + ); + assert!( + prompt.contains(&format!(" - resources/{old_file_name}")), + "expected old resource in prompt: {prompt}" + ); + wait_for_file_removed(&old_file).await?; + + shutdown_test_codex(&codex).await?; + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn web_search_pollution_moves_selected_thread_into_removed_phase2_inputs() -> Result<()> { let server = start_mock_server().await; @@ -386,6 +525,21 @@ async fn wait_for_single_request(mock: &ResponseMock) -> ResponsesRequest { wait_for_request(mock, /*expected_count*/ 1).await.remove(0) } +async fn wait_for_file_removed(path: &Path) -> Result<()> { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + if !tokio::fs::try_exists(path).await? { + return Ok(()); + } + assert!( + Instant::now() < deadline, + "timed out waiting for {} to be removed", + path.display() + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + async fn wait_for_request(mock: &ResponseMock, expected_count: usize) -> Vec { let deadline = Instant::now() + Duration::from_secs(10); loop { diff --git a/codex-rs/core/tests/suite/mod.rs b/codex-rs/core/tests/suite/mod.rs index 8f3d9e15d9..9f8389c45e 100644 --- a/codex-rs/core/tests/suite/mod.rs +++ b/codex-rs/core/tests/suite/mod.rs @@ -85,7 +85,6 @@ mod rmcp_client; mod rollout_list_find; mod safety_check_downgrade; mod search_tool; -mod seatbelt; mod shell_command; mod shell_serialization; mod shell_snapshot; diff --git a/codex-rs/core/tests/suite/openai_file_mcp.rs b/codex-rs/core/tests/suite/openai_file_mcp.rs index 134ad6dfde..211aa67381 100644 --- a/codex-rs/core/tests/suite/openai_file_mcp.rs +++ b/codex-rs/core/tests/suite/openai_file_mcp.rs @@ -10,7 +10,7 @@ use core_test_support::apps_test_server::AppsTestServer; use core_test_support::apps_test_server::DOCUMENT_EXTRACT_TEXT_RESOURCE_URI; use core_test_support::responses::ev_assistant_message; use core_test_support::responses::ev_completed; -use core_test_support::responses::ev_function_call; +use core_test_support::responses::ev_function_call_with_namespace; use core_test_support::responses::ev_response_created; use core_test_support::responses::mount_sse_sequence; use core_test_support::responses::sse; @@ -26,7 +26,8 @@ use wiremock::matchers::header; use wiremock::matchers::method; use wiremock::matchers::path; -const DOCUMENT_EXTRACT_TOOL: &str = "mcp__codex_apps__calendar_extract_text"; +const DOCUMENT_EXTRACT_NAMESPACE: &str = "mcp__codex_apps__calendar"; +const DOCUMENT_EXTRACT_TOOL: &str = "_extract_text"; fn configure_apps(config: &mut Config, chatgpt_base_url: &str) { if let Err(err) = config.features.enable(Feature::Apps) { @@ -35,18 +36,6 @@ fn configure_apps(config: &mut Config, chatgpt_base_url: &str) { config.chatgpt_base_url = chatgpt_base_url.to_string(); } -fn tool_by_name<'a>(body: &'a Value, name: &str) -> &'a Value { - body.get("tools") - .and_then(Value::as_array) - .and_then(|tools| { - tools.iter().find(|tool| { - tool.get("name").and_then(Value::as_str) == Some(name) - || tool.get("type").and_then(Value::as_str) == Some(name) - }) - }) - .unwrap_or_else(|| panic!("missing tool {name} in /v1/responses request: {body:?}")) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn codex_apps_file_params_upload_local_paths_before_mcp_tool_call() -> Result<()> { let server = start_mock_server().await; @@ -93,8 +82,9 @@ async fn codex_apps_file_params_upload_local_paths_before_mcp_tool_call() -> Res vec![ sse(vec![ ev_response_created("resp-1"), - ev_function_call( + ev_function_call_with_namespace( call_id, + DOCUMENT_EXTRACT_NAMESPACE, DOCUMENT_EXTRACT_TOOL, &json!({"file": "report.txt"}).to_string(), ), @@ -123,8 +113,14 @@ async fn codex_apps_file_params_upload_local_paths_before_mcp_tool_call() -> Res .await?; let requests = mock.requests(); - let body = requests[0].body_json(); - let extract_tool = tool_by_name(&body, DOCUMENT_EXTRACT_TOOL); + let Some(extract_tool) = + requests[0].tool_by_name(DOCUMENT_EXTRACT_NAMESPACE, DOCUMENT_EXTRACT_TOOL) + else { + let body = requests[0].body_json(); + panic!( + "missing tool {DOCUMENT_EXTRACT_NAMESPACE}{DOCUMENT_EXTRACT_TOOL} in /v1/responses request: {body:?}" + ) + }; assert_eq!( extract_tool.pointer("/parameters/properties/file"), Some(&json!({ diff --git a/codex-rs/core/tests/suite/plugins.rs b/codex-rs/core/tests/suite/plugins.rs index ab2c85ac0c..b19385cd5d 100644 --- a/codex-rs/core/tests/suite/plugins.rs +++ b/codex-rs/core/tests/suite/plugins.rs @@ -168,22 +168,6 @@ fn tool_names(body: &serde_json::Value) -> Vec { .unwrap_or_default() } -fn tool_description(body: &serde_json::Value, tool_name: &str) -> Option { - body.get("tools") - .and_then(serde_json::Value::as_array) - .and_then(|tools| { - tools.iter().find_map(|tool| { - if tool.get("name").and_then(serde_json::Value::as_str) == Some(tool_name) { - tool.get("description") - .and_then(serde_json::Value::as_str) - .map(str::to_string) - } else { - None - } - }) - }) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn capability_sections_render_in_developer_message_in_order() -> Result<()> { skip_if_no_network!(Ok(())); @@ -319,20 +303,27 @@ async fn explicit_plugin_mentions_inject_plugin_guidance() -> Result<()> { assert!( request_tools .iter() - .any(|name| name == "mcp__codex_apps__google_calendar_create_event"), + .any(|name| name == "mcp__codex_apps__google_calendar"), "expected plugin app tools to become visible for this turn: {request_tools:?}" ); - let echo_description = tool_description(&request_body, "mcp__sample__echo") + let echo_tool = request + .tool_by_name("mcp__sample__", "echo") + .expect("plugin MCP tool should be present"); + let echo_description = echo_tool + .get("description") + .and_then(serde_json::Value::as_str) .expect("plugin MCP tool description should be present"); assert!( echo_description.contains("This tool is part of plugin `sample`."), "expected plugin MCP provenance in tool description: {echo_description:?}" ); - let calendar_description = tool_description( - &request_body, - "mcp__codex_apps__google_calendar_create_event", - ) - .expect("plugin app tool description should be present"); + let calendar_tool = request + .tool_by_name("mcp__codex_apps__google_calendar", "_create_event") + .expect("plugin app tool should be present"); + let calendar_description = calendar_tool + .get("description") + .and_then(serde_json::Value::as_str) + .expect("plugin app tool description should be present"); assert!( calendar_description.contains("This tool is part of plugin `sample`."), "expected plugin app provenance in tool description: {calendar_description:?}" diff --git a/codex-rs/core/tests/suite/realtime_conversation.rs b/codex-rs/core/tests/suite/realtime_conversation.rs index 4759f5daf5..ecaaeb8cba 100644 --- a/codex-rs/core/tests/suite/realtime_conversation.rs +++ b/codex-rs/core/tests/suite/realtime_conversation.rs @@ -194,10 +194,16 @@ async fn seed_recent_thread( let db = test.codex.state_db().context("state db enabled")?; let thread_id = ThreadId::new(); let updated_at = Utc::now(); + let rollout_path = test + .codex_home_path() + .join(format!("rollout-{thread_id}.jsonl")); + // This helper seeds SQLite metadata directly. Local listing drops stale metadata rows whose + // rollout path no longer exists, so create the placeholder path that the test metadata points + // at without exercising rollout writing in this realtime-context test. + std::fs::write(&rollout_path, "")?; let mut metadata_builder = codex_state::ThreadMetadataBuilder::new( thread_id, - test.codex_home_path() - .join(format!("rollout-{thread_id}.jsonl")), + rollout_path, updated_at, SessionSource::Cli, ); @@ -2417,11 +2423,8 @@ async fn inbound_handoff_request_starts_turn() -> Result<()> { let request = response_mock.single_request(); let user_texts = request.message_input_texts("user"); - assert!( - user_texts - .iter() - .any(|text| text == "user: text from realtime") - ); + assert!(user_texts.iter().any(|text| text + == "\n user: text from realtime\n")); realtime_server.shutdown().await; Ok(()) @@ -2503,7 +2506,7 @@ async fn inbound_handoff_request_uses_active_transcript() -> Result<()> { let request = response_mock.single_request(); let user_texts = request.message_input_texts("user"); assert!(user_texts.iter().any(|text| text - == "assistant: assistant context\nuser: delegated query\nassistant: assist confirm")); + == "\n assistant: assistant context\nuser: delegated query\nassistant: assist confirm\n")); realtime_server.shutdown().await; Ok(()) @@ -2617,23 +2620,14 @@ async fn inbound_handoff_request_clears_active_transcript_after_each_handoff() - assert_eq!(requests.len(), 2); let first_user_texts = requests[0].message_input_texts("user"); - assert!( - first_user_texts - .iter() - .any(|text| text == "user: first question") - ); + assert!(first_user_texts.iter().any(|text| text + == "\n user: first question\n")); let second_user_texts = requests[1].message_input_texts("user"); - assert!( - second_user_texts - .iter() - .any(|text| text == "user: second question") - ); - assert!( - !second_user_texts - .iter() - .any(|text| text == "user: first question\nuser: second question") - ); + assert!(second_user_texts.iter().any(|text| text + == "\n user: second question\n")); + assert!(!second_user_texts.iter().any(|text| text + == "\n user: first question\nuser: second question\n")); realtime_server.shutdown().await; Ok(()) @@ -3156,14 +3150,12 @@ async fn inbound_handoff_request_steers_active_turn() -> Result<()> { assert!( !first_texts .iter() - .any(|text| text == "user: steer via realtime") + .any(|text| text + == "\n user: steer via realtime\n") ); assert!(second_texts.iter().any(|text| text == "first prompt")); - assert!( - second_texts - .iter() - .any(|text| text == "user: steer via realtime") - ); + assert!(second_texts.iter().any(|text| text + == "\n user: steer via realtime\n")); realtime_server.shutdown().await; api_server.shutdown().await; @@ -3278,7 +3270,9 @@ async fn inbound_handoff_request_starts_turn_and_does_not_block_realtime_audio() assert_eq!(requests.len(), 1); let first_body: Value = serde_json::from_slice(&requests[0]).expect("parse first request"); let first_texts = message_input_texts(&first_body, "user"); - let expected_text = format!("user: {delegated_text}"); + let expected_text = format!( + "\n user: {delegated_text}\n" + ); assert!(first_texts.iter().any(|text| text == &expected_text)); realtime_server.shutdown().await; diff --git a/codex-rs/core/tests/suite/responses_api_proxy_headers.rs b/codex-rs/core/tests/suite/responses_api_proxy_headers.rs index 7dc888ffce..8199956ee0 100644 --- a/codex-rs/core/tests/suite/responses_api_proxy_headers.rs +++ b/codex-rs/core/tests/suite/responses_api_proxy_headers.rs @@ -5,6 +5,11 @@ use anyhow::Result; use anyhow::anyhow; use codex_features::Feature; +use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::Op; +use codex_protocol::protocol::SandboxPolicy; +use codex_protocol::user_input::UserInput; use core_test_support::responses::ev_assistant_message; use core_test_support::responses::ev_completed; use core_test_support::responses::ev_function_call; @@ -13,10 +18,12 @@ use core_test_support::responses::mount_sse_once_match; use core_test_support::responses::sse; use core_test_support::responses::start_mock_server; use core_test_support::skip_if_no_network; +use core_test_support::test_codex::TestCodex; use core_test_support::test_codex::test_codex; use pretty_assertions::assert_eq; use serde_json::Value; use serde_json::json; +use std::io::ErrorKind; use std::io::Write; use std::path::Path; use std::process::Child; @@ -29,8 +36,9 @@ use tempfile::TempDir; const PARENT_PROMPT: &str = "spawn a subagent and report when it is started"; const CHILD_PROMPT: &str = "child: say done"; const SPAWN_CALL_ID: &str = "spawn-call-1"; -const PROXY_START_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 5); +const PROXY_START_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 30); const PROXY_POLL_INTERVAL: Duration = Duration::from_millis(/*millis*/ 20); +const TURN_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 60); struct ResponsesApiProxy { child: Child, @@ -40,8 +48,17 @@ struct ResponsesApiProxy { impl ResponsesApiProxy { fn start(upstream_url: &str, dump_dir: &Path) -> Result { let server_info = dump_dir.join("server-info.json"); - let mut child = StdCommand::new(codex_utils_cargo_bin::cargo_bin("codex")?) - .args(["responses-api-proxy", "--server-info"]) + let (proxy_program, use_codex_multitool) = + match codex_utils_cargo_bin::cargo_bin("codex-responses-api-proxy") { + Ok(path) => (path, false), + Err(_) => (codex_utils_cargo_bin::cargo_bin("codex")?, true), + }; + let mut command = StdCommand::new(proxy_program); + if use_codex_multitool { + command.arg("responses-api-proxy"); + } + let mut child = command + .args(["--server-info"]) .arg(&server_info) .args(["--upstream-url", upstream_url, "--dump-dir"]) .arg(dump_dir) @@ -58,15 +75,27 @@ impl ResponsesApiProxy { let deadline = Instant::now() + PROXY_START_TIMEOUT; loop { - if let Ok(info) = std::fs::read_to_string(&server_info) { - let port = serde_json::from_str::(&info)? - .get("port") - .and_then(Value::as_u64) - .ok_or_else(|| anyhow!("proxy server info missing port"))?; - return Ok(Self { - child, - port: u16::try_from(port)?, - }); + match std::fs::read_to_string(&server_info) { + Ok(info) => { + if !info.trim().is_empty() { + match serde_json::from_str::(&info) { + Ok(info) => { + let port = info + .get("port") + .and_then(Value::as_u64) + .ok_or_else(|| anyhow!("proxy server info missing port"))?; + return Ok(Self { + child, + port: u16::try_from(port)?, + }); + } + Err(err) if err.is_eof() => {} + Err(err) => return Err(err.into()), + } + } + } + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(err.into()), } if let Some(status) = child.try_wait()? { return Err(anyhow!( @@ -144,7 +173,7 @@ async fn responses_api_proxy_dumps_parent_and_subagent_identity_headers() -> Res .expect("test config should allow feature update"); }); let test = builder.build(&server).await?; - test.submit_turn(PARENT_PROMPT).await?; + submit_turn_with_timeout(&test, PARENT_PROMPT, dump_dir.path()).await?; let dumps = wait_for_proxy_request_dumps(dump_dir.path())?; let parent = dumps @@ -178,6 +207,85 @@ async fn responses_api_proxy_dumps_parent_and_subagent_identity_headers() -> Res Ok(()) } +async fn submit_turn_with_timeout(test: &TestCodex, prompt: &str, dump_dir: &Path) -> Result<()> { + let session_model = test.session_configured.model.clone(); + test.codex + .submit(Op::UserTurn { + items: vec![UserInput::Text { + text: prompt.into(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + cwd: test.config.cwd.to_path_buf(), + approval_policy: AskForApproval::OnRequest, + approvals_reviewer: None, + sandbox_policy: SandboxPolicy::WorkspaceWrite { + writable_roots: Vec::new(), + read_only_access: Default::default(), + network_access: false, + exclude_tmpdir_env_var: false, + exclude_slash_tmp: false, + }, + model: session_model, + effort: None, + summary: None, + service_tier: None, + collaboration_mode: None, + personality: None, + }) + .await?; + + let turn_started = wait_for_event_result(test, "turn started", dump_dir, |event| { + matches!(event, EventMsg::TurnStarted(_)) + }) + .await?; + let EventMsg::TurnStarted(turn_started) = turn_started else { + unreachable!("event predicate only matches turn started events"); + }; + wait_for_event_result(test, "turn complete", dump_dir, |event| match event { + EventMsg::TurnComplete(event) => event.turn_id == turn_started.turn_id, + _ => false, + }) + .await?; + + Ok(()) +} + +async fn wait_for_event_result( + test: &TestCodex, + stage: &str, + dump_dir: &Path, + mut predicate: F, +) -> Result +where + F: FnMut(&EventMsg) -> bool, +{ + let mut seen_events = Vec::new(); + tokio::time::timeout(TURN_TIMEOUT, async { + loop { + let event = test.codex.next_event().await?; + seen_events.push(event_summary(&event.msg)); + if predicate(&event.msg) { + return Ok::(event.msg); + } + } + }) + .await + .map_err(|_| { + anyhow!( + "timed out waiting for {stage}; saw events: {}; {}", + seen_events.join(" | "), + proxy_dump_summary(dump_dir) + ) + })? +} + +fn event_summary(event: &EventMsg) -> String { + let mut summary = format!("{event:?}"); + summary.truncate(240); + summary +} + fn request_body_contains(req: &wiremock::Request, text: &str) -> bool { std::str::from_utf8(&req.body).is_ok_and(|body| body.contains(text)) } @@ -212,12 +320,54 @@ fn read_proxy_request_dumps(dump_dir: &Path) -> Result> { .and_then(|name| name.to_str()) .is_some_and(|name| name.ends_with("-request.json")) { - dumps.push(serde_json::from_str(&std::fs::read_to_string(&path)?)?); + let contents = std::fs::read_to_string(&path)?; + if contents.trim().is_empty() { + continue; + } + + match serde_json::from_str(&contents) { + Ok(dump) => dumps.push(dump), + Err(err) if err.is_eof() => continue, + Err(err) => return Err(err.into()), + } } } Ok(dumps) } +fn proxy_dump_summary(dump_dir: &Path) -> String { + match read_proxy_request_dumps(dump_dir) { + Ok(dumps) => { + let bodies = dumps + .iter() + .filter_map(|dump| dump.get("body")) + .map(Value::to_string) + .collect::>() + .join("; "); + format!("proxy wrote {} request dumps: {bodies}", dumps.len()) + } + Err(err) => format!("failed to read proxy request dumps: {err}"), + } +} + +#[test] +fn read_proxy_request_dumps_ignores_in_progress_files() -> Result<()> { + let dump_dir = TempDir::new()?; + std::fs::write(dump_dir.path().join("empty-request.json"), "")?; + std::fs::write(dump_dir.path().join("partial-request.json"), "{\"body\"")?; + std::fs::write( + dump_dir.path().join("complete-request.json"), + serde_json::to_string(&json!({ "body": "ready" }))?, + )?; + + assert_eq!( + read_proxy_request_dumps(dump_dir.path())?, + vec![json!({ "body": "ready" })] + ); + + Ok(()) +} + fn dump_body_contains(dump: &Value, text: &str) -> bool { dump.get("body") .is_some_and(|body| body.to_string().contains(text)) diff --git a/codex-rs/core/tests/suite/rmcp_client.rs b/codex-rs/core/tests/suite/rmcp_client.rs index b0ca1d7561..b86094f910 100644 --- a/codex-rs/core/tests/suite/rmcp_client.rs +++ b/codex-rs/core/tests/suite/rmcp_client.rs @@ -12,6 +12,7 @@ use std::time::UNIX_EPOCH; use codex_config::types::McpServerConfig; use codex_config::types::McpServerTransportConfig; use codex_login::CodexAuth; +use codex_mcp::MCP_SANDBOX_STATE_META_CAPABILITY; use codex_models_manager::manager::RefreshStrategy; use codex_protocol::config_types::ReasoningSummary; @@ -90,13 +91,18 @@ async fn stdio_server_round_trip() -> anyhow::Result<()> { let call_id = "call-123"; let server_name = "rmcp"; - let tool_name = format!("mcp__{server_name}__echo"); + let namespace = format!("mcp__{server_name}__"); - mount_sse_once( + let call_mock = mount_sse_once( &server, responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, &tool_name, "{\"message\":\"ping\"}"), + responses::ev_function_call_with_namespace( + call_id, + &namespace, + "echo", + "{\"message\":\"ping\"}", + ), responses::ev_completed("resp-1"), ]), ) @@ -222,6 +228,13 @@ async fn stdio_server_round_trip() -> anyhow::Result<()> { wait_for_event(&fixture.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; let output_item = final_mock.single_request().function_call_output(call_id); + let request = call_mock.single_request(); + assert!( + request.tool_by_name(&namespace, "echo").is_some(), + "direct MCP tool should be sent as a namespace child tool: {:?}", + request.body_json() + ); + let output_text = output_item .get("output") .and_then(Value::as_str) @@ -237,6 +250,138 @@ async fn stdio_server_round_trip() -> anyhow::Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn stdio_mcp_tool_call_includes_sandbox_state_meta() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + + let call_id = "sandbox-meta-call"; + let server_name = "rmcp"; + let namespace = format!("mcp__{server_name}__"); + let tool_name = format!("{namespace}sandbox_meta"); + + let call_mock = mount_sse_once( + &server, + responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_function_call_with_namespace(call_id, &namespace, "sandbox_meta", "{}"), + responses::ev_completed("resp-1"), + ]), + ) + .await; + let final_mock = mount_sse_once( + &server, + responses::sse(vec![ + responses::ev_assistant_message("msg-1", "rmcp sandbox meta completed successfully."), + responses::ev_completed("resp-2"), + ]), + ) + .await; + + let rmcp_test_server_bin = stdio_server_bin()?; + let fixture = test_codex() + .with_config(move |config| { + let mut servers = config.mcp_servers.get().clone(); + servers.insert( + server_name.to_string(), + McpServerConfig { + transport: McpServerTransportConfig::Stdio { + command: rmcp_test_server_bin, + args: Vec::new(), + env: None, + env_vars: Vec::new(), + cwd: None, + }, + enabled: true, + required: false, + supports_parallel_tool_calls: false, + disabled_reason: None, + startup_timeout_sec: Some(Duration::from_secs(10)), + tool_timeout_sec: None, + enabled_tools: None, + disabled_tools: None, + scopes: None, + oauth_resource: None, + tools: HashMap::new(), + }, + ); + config + .mcp_servers + .set(servers) + .expect("test mcp servers should accept any configuration"); + }) + .build(&server) + .await?; + + let tools_ready_deadline = Instant::now() + Duration::from_secs(30); + loop { + fixture.codex.submit(Op::ListMcpTools).await?; + let list_event = wait_for_event_with_timeout( + &fixture.codex, + |ev| matches!(ev, EventMsg::McpListToolsResponse(_)), + Duration::from_secs(10), + ) + .await; + let EventMsg::McpListToolsResponse(tool_list) = list_event else { + unreachable!("event guard guarantees McpListToolsResponse"); + }; + if tool_list.tools.contains_key(&tool_name) { + break; + } + + let available_tools: Vec<&str> = tool_list.tools.keys().map(String::as_str).collect(); + if Instant::now() >= tools_ready_deadline { + panic!( + "timed out waiting for MCP tool {tool_name} to become available; discovered tools: {available_tools:?}" + ); + } + sleep(Duration::from_millis(200)).await; + } + + let sandbox_policy = SandboxPolicy::new_read_only_policy(); + fixture + .submit_turn_with_policy("call the rmcp sandbox_meta tool", sandbox_policy.clone()) + .await?; + + let request = call_mock.single_request(); + assert!( + request.tool_by_name(&namespace, "sandbox_meta").is_some(), + "direct MCP tool should be sent as a namespace child tool: {:?}", + request.body_json() + ); + + let output_item = final_mock.single_request().function_call_output(call_id); + let output_text = output_item + .get("output") + .and_then(Value::as_str) + .expect("function_call_output output should be a string"); + let wrapped_payload = split_wall_time_wrapped_output(output_text); + let output_json: Value = serde_json::from_str(wrapped_payload) + .expect("wrapped MCP output should preserve sandbox metadata JSON"); + let Value::Object(meta) = output_json else { + panic!("sandbox_meta should return metadata object: {output_json:?}"); + }; + + let sandbox_meta = meta + .get(MCP_SANDBOX_STATE_META_CAPABILITY) + .expect("sandbox state metadata should be present"); + let expected_sandbox_policy = serde_json::to_value(&sandbox_policy)?; + assert_eq!( + sandbox_meta.get("sandboxPolicy"), + Some(&expected_sandbox_policy) + ); + assert_eq!( + sandbox_meta.get("sandboxCwd").and_then(Value::as_str), + fixture.cwd.path().to_str() + ); + assert_eq!(sandbox_meta.get("useLegacyLandlock"), Some(&json!(false))); + + server.verify().await; + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn stdio_mcp_parallel_tool_calls_default_false_runs_serially() -> anyhow::Result<()> { skip_if_no_network!(Ok(())); @@ -246,15 +391,15 @@ async fn stdio_mcp_parallel_tool_calls_default_false_runs_serially() -> anyhow:: let first_call_id = "sync-serial-1"; let second_call_id = "sync-serial-2"; let server_name = "rmcp"; - let tool_name = format!("mcp__{server_name}__sync"); + let namespace = format!("mcp__{server_name}__"); let args = json!({ "sleep_after_ms": 100 }).to_string(); mount_sse_once( &server, responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(first_call_id, &tool_name, &args), - responses::ev_function_call(second_call_id, &tool_name, &args), + responses::ev_function_call_with_namespace(first_call_id, &namespace, "sync", &args), + responses::ev_function_call_with_namespace(second_call_id, &namespace, "sync", &args), responses::ev_completed("resp-1"), ]), ) @@ -388,7 +533,7 @@ async fn stdio_mcp_parallel_tool_calls_opt_in_runs_concurrently() -> anyhow::Res let first_call_id = "sync-1"; let second_call_id = "sync-2"; let server_name = "rmcp"; - let tool_name = format!("mcp__{server_name}__sync"); + let namespace = format!("mcp__{server_name}__"); let args = json!({ "sleep_after_ms": 100, "barrier": { @@ -403,8 +548,8 @@ async fn stdio_mcp_parallel_tool_calls_opt_in_runs_concurrently() -> anyhow::Res &server, responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(first_call_id, &tool_name, &args), - responses::ev_function_call(second_call_id, &tool_name, &args), + responses::ev_function_call_with_namespace(first_call_id, &namespace, "sync", &args), + responses::ev_function_call_with_namespace(second_call_id, &namespace, "sync", &args), responses::ev_completed("resp-1"), ]), ) @@ -504,13 +649,14 @@ async fn stdio_image_responses_round_trip() -> anyhow::Result<()> { let call_id = "img-1"; let server_name = "rmcp"; let tool_name = format!("mcp__{server_name}__image"); + let namespace = format!("mcp__{server_name}__"); // First stream: model decides to call the image tool. mount_sse_once( &server, responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, &tool_name, "{}"), + responses::ev_function_call_with_namespace(call_id, &namespace, "image", "{}"), responses::ev_completed("resp-1"), ]), ) @@ -693,7 +839,7 @@ async fn stdio_image_responses_are_sanitized_for_text_only_model() -> anyhow::Re let call_id = "img-text-only-1"; let server_name = "rmcp"; - let tool_name = format!("mcp__{server_name}__image"); + let namespace = format!("mcp__{server_name}__"); let text_only_model_slug = "rmcp-text-only-model"; let models_mock = mount_models_once( @@ -743,7 +889,7 @@ async fn stdio_image_responses_are_sanitized_for_text_only_model() -> anyhow::Re &server, responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, &tool_name, "{}"), + responses::ev_function_call_with_namespace(call_id, &namespace, "image", "{}"), responses::ev_completed("resp-1"), ]), ) @@ -864,13 +1010,18 @@ async fn stdio_server_propagates_whitelisted_env_vars() -> anyhow::Result<()> { let call_id = "call-1234"; let server_name = "rmcp_whitelist"; - let tool_name = format!("mcp__{server_name}__echo"); + let namespace = format!("mcp__{server_name}__"); mount_sse_once( &server, responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, &tool_name, "{\"message\":\"ping\"}"), + responses::ev_function_call_with_namespace( + call_id, + &namespace, + "echo", + "{\"message\":\"ping\"}", + ), responses::ev_completed("resp-1"), ]), ) @@ -1006,13 +1157,18 @@ async fn streamable_http_tool_call_round_trip() -> anyhow::Result<()> { let call_id = "call-456"; let server_name = "rmcp_http"; - let tool_name = format!("mcp__{server_name}__echo"); + let namespace = format!("mcp__{server_name}__"); mount_sse_once( &server, responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, &tool_name, "{\"message\":\"ping\"}"), + responses::ev_function_call_with_namespace( + call_id, + &namespace, + "echo", + "{\"message\":\"ping\"}", + ), responses::ev_completed("resp-1"), ]), ) @@ -1211,12 +1367,18 @@ async fn streamable_http_with_oauth_round_trip_impl() -> anyhow::Result<()> { let call_id = "call-789"; let server_name = "rmcp_http_oauth"; let tool_name = format!("mcp__{server_name}__echo"); + let namespace = format!("mcp__{server_name}__"); mount_sse_once( &server, responses::sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, &tool_name, "{\"message\":\"ping\"}"), + responses::ev_function_call_with_namespace( + call_id, + &namespace, + "echo", + "{\"message\":\"ping\"}", + ), responses::ev_completed("resp-1"), ]), ) diff --git a/codex-rs/core/tests/suite/search_tool.rs b/codex-rs/core/tests/suite/search_tool.rs index c524316650..74de499d70 100644 --- a/codex-rs/core/tests/suite/search_tool.rs +++ b/codex-rs/core/tests/suite/search_tool.rs @@ -23,6 +23,7 @@ use core_test_support::responses::ev_response_created; use core_test_support::responses::ev_tool_search_call; use core_test_support::responses::mount_sse_once; use core_test_support::responses::mount_sse_sequence; +use core_test_support::responses::namespace_child_tool; use core_test_support::responses::sse; use core_test_support::responses::start_mock_server; use core_test_support::skip_if_no_network; @@ -45,6 +46,7 @@ const CALENDAR_CREATE_TOOL: &str = "mcp__codex_apps__calendar_create_event"; const CALENDAR_LIST_TOOL: &str = "mcp__codex_apps__calendar_list_events"; const SEARCH_CALENDAR_NAMESPACE: &str = "mcp__codex_apps__calendar"; const SEARCH_CALENDAR_CREATE_TOOL: &str = "_create_event"; +const SEARCH_CALENDAR_LIST_TOOL: &str = "_list_events"; fn tool_names(body: &Value) -> Vec { body.get("tools") @@ -215,8 +217,17 @@ async fn tool_search_disabled_by_default_exposes_apps_tools_directly() -> Result let body = mock.single_request().body_json(); let tools = tool_names(&body); assert!(!tools.iter().any(|name| name == TOOL_SEARCH_TOOL_NAME)); - assert!(tools.iter().any(|name| name == CALENDAR_CREATE_TOOL)); - assert!(tools.iter().any(|name| name == CALENDAR_LIST_TOOL)); + assert!( + namespace_child_tool( + &body, + SEARCH_CALENDAR_NAMESPACE, + SEARCH_CALENDAR_CREATE_TOOL + ) + .is_some() + ); + assert!( + namespace_child_tool(&body, SEARCH_CALENDAR_NAMESPACE, SEARCH_CALENDAR_LIST_TOOL).is_some() + ); Ok(()) } @@ -332,6 +343,7 @@ async fn search_tool_hides_apps_tools_without_search() -> Result<()> { assert!(tools.iter().any(|name| name == TOOL_SEARCH_TOOL_NAME)); assert!(!tools.iter().any(|name| name == CALENDAR_CREATE_TOOL)); assert!(!tools.iter().any(|name| name == CALENDAR_LIST_TOOL)); + assert!(!tools.iter().any(|name| name == SEARCH_CALENDAR_NAMESPACE)); Ok(()) } @@ -365,11 +377,16 @@ async fn explicit_app_mentions_expose_apps_tools_without_search() -> Result<()> let body = mock.single_request().body_json(); let tools = tool_names(&body); assert!( - tools.iter().any(|name| name == CALENDAR_CREATE_TOOL), + namespace_child_tool( + &body, + SEARCH_CALENDAR_NAMESPACE, + SEARCH_CALENDAR_CREATE_TOOL + ) + .is_some(), "expected explicit app mention to expose create tool, got tools: {tools:?}" ); assert!( - tools.iter().any(|name| name == CALENDAR_LIST_TOOL), + namespace_child_tool(&body, SEARCH_CALENDAR_NAMESPACE, SEARCH_CALENDAR_LIST_TOOL).is_some(), "expected explicit app mention to expose list tool, got tools: {tools:?}" ); @@ -523,6 +540,12 @@ async fn tool_search_returns_deferred_tools_without_follow_up_tool_injection() - .any(|name| name == CALENDAR_CREATE_TOOL), "app tools should still be hidden before search: {first_request_tools:?}" ); + assert!( + !first_request_tools + .iter() + .any(|name| name == SEARCH_CALENDAR_NAMESPACE), + "app namespace should still be hidden before search: {first_request_tools:?}" + ); let output_item = tool_search_output_item(&requests[1], call_id); assert_eq!( @@ -570,6 +593,12 @@ async fn tool_search_returns_deferred_tools_without_follow_up_tool_injection() - .any(|name| name == CALENDAR_CREATE_TOOL), "follow-up request should rely on tool_search_output history, not tool injection: {second_request_tools:?}" ); + assert!( + !second_request_tools + .iter() + .any(|name| name == SEARCH_CALENDAR_NAMESPACE), + "follow-up request should rely on tool_search_output history, not namespace injection: {second_request_tools:?}" + ); let output_item = requests[2].function_call_output("calendar-call-1"); assert_eq!( @@ -584,6 +613,12 @@ async fn tool_search_returns_deferred_tools_without_follow_up_tool_injection() - .any(|name| name == CALENDAR_CREATE_TOOL), "post-tool follow-up should still rely on tool_search_output history, not tool injection: {third_request_tools:?}" ); + assert!( + !third_request_tools + .iter() + .any(|name| name == SEARCH_CALENDAR_NAMESPACE), + "post-tool follow-up should still rely on tool_search_output history, not namespace injection: {third_request_tools:?}" + ); Ok(()) } @@ -683,16 +718,19 @@ async fn tool_search_indexes_only_enabled_non_app_mcp_tools() -> Result<()> { .any(|name| name == "mcp__rmcp__echo"), "non-app MCP tools should be hidden before search in large-search mode: {first_request_tools:?}" ); + assert!( + !first_request_tools.iter().any(|name| name == "mcp__rmcp__"), + "non-app MCP namespace should be hidden before search in large-search mode: {first_request_tools:?}" + ); let echo_tools = tool_search_output_tools(&requests[1], echo_call_id); - let rmcp_echo_tools = echo_tools - .iter() - .filter(|tool| tool.get("name").and_then(Value::as_str) == Some("mcp__rmcp__")) - .flat_map(|namespace| namespace.get("tools").and_then(Value::as_array)) - .flatten() - .filter_map(|tool| tool.get("name").and_then(Value::as_str).map(str::to_string)) - .collect::>(); - assert_eq!(rmcp_echo_tools, vec!["echo".to_string()]); + let echo_output = json!({ "tools": echo_tools }); + let rmcp_echo_tool = namespace_child_tool(&echo_output, "mcp__rmcp__", "echo") + .expect("tool_search should return rmcp echo as a namespace child tool"); + assert_eq!( + rmcp_echo_tool.get("type").and_then(Value::as_str), + Some("function") + ); let image_tools = tool_search_output_tools(&requests[1], image_call_id); let found_rmcp_image_tool = image_tools diff --git a/codex-rs/core/tests/suite/seatbelt.rs b/codex-rs/core/tests/suite/seatbelt.rs deleted file mode 100644 index 808a8a022d..0000000000 --- a/codex-rs/core/tests/suite/seatbelt.rs +++ /dev/null @@ -1,316 +0,0 @@ -#![cfg(target_os = "macos")] - -//! Tests for the macOS sandboxing that are specific to Seatbelt. -//! Tests that apply to both Mac and Linux sandboxing should go in sandbox.rs. - -use std::collections::HashMap; -use std::path::Path; -use std::path::PathBuf; - -use codex_core::seatbelt::spawn_command_under_seatbelt; -use codex_core::spawn::CODEX_SANDBOX_ENV_VAR; -use codex_core::spawn::StdioPolicy; -use codex_protocol::protocol::SandboxPolicy; -use codex_utils_absolute_path::AbsolutePathBuf; -use tempfile::TempDir; - -struct TestScenario { - repo_parent: PathBuf, - file_outside_repo: PathBuf, - repo_root: PathBuf, - file_in_repo_root: PathBuf, - file_in_dot_git_dir: PathBuf, -} - -struct TestExpectations { - file_outside_repo_is_writable: bool, - file_in_repo_root_is_writable: bool, - file_in_dot_git_dir_is_writable: bool, -} - -impl TestScenario { - async fn run_test(&self, policy: &SandboxPolicy, expectations: TestExpectations) { - if std::env::var(CODEX_SANDBOX_ENV_VAR) == Ok("seatbelt".to_string()) { - eprintln!("{CODEX_SANDBOX_ENV_VAR} is set to 'seatbelt', skipping test."); - return; - } - - assert_eq!( - touch(&self.file_outside_repo, policy).await, - expectations.file_outside_repo_is_writable - ); - assert_eq!( - self.file_outside_repo.exists(), - expectations.file_outside_repo_is_writable - ); - - assert_eq!( - touch(&self.file_in_repo_root, policy).await, - expectations.file_in_repo_root_is_writable - ); - assert_eq!( - self.file_in_repo_root.exists(), - expectations.file_in_repo_root_is_writable - ); - - assert_eq!( - touch(&self.file_in_dot_git_dir, policy).await, - expectations.file_in_dot_git_dir_is_writable - ); - assert_eq!( - self.file_in_dot_git_dir.exists(), - expectations.file_in_dot_git_dir_is_writable - ); - } -} - -/// If the user has added a workspace root that is not a Git repo root, then -/// the user has to specify `--skip-git-repo-check` or go through some -/// interstitial that indicates they are taking on some risk because Git -/// cannot be used to backup their work before the agent begins. -/// -/// Because the user has agreed to this risk, we do not try find all .git -/// folders in the workspace and block them (though we could change our -/// position on this in the future). -#[tokio::test] -async fn if_parent_of_repo_is_writable_then_dot_git_folder_is_writable() { - let tmp = TempDir::new().expect("should be able to create temp dir"); - let test_scenario = create_test_scenario(&tmp); - let policy = SandboxPolicy::WorkspaceWrite { - writable_roots: vec![test_scenario.repo_parent.as_path().try_into().unwrap()], - read_only_access: Default::default(), - network_access: false, - exclude_tmpdir_env_var: true, - exclude_slash_tmp: true, - }; - - test_scenario - .run_test( - &policy, - TestExpectations { - file_outside_repo_is_writable: true, - file_in_repo_root_is_writable: true, - file_in_dot_git_dir_is_writable: true, - }, - ) - .await; -} - -/// When the writable root is the root of a Git repository (as evidenced by the -/// presence of a .git folder), then the .git folder should be read-only if -/// the policy is `WorkspaceWrite`. -#[tokio::test] -async fn if_git_repo_is_writable_root_then_dot_git_folder_is_read_only() { - let tmp = TempDir::new().expect("should be able to create temp dir"); - let test_scenario = create_test_scenario(&tmp); - let policy = SandboxPolicy::WorkspaceWrite { - writable_roots: vec![test_scenario.repo_root.as_path().try_into().unwrap()], - read_only_access: Default::default(), - network_access: false, - exclude_tmpdir_env_var: true, - exclude_slash_tmp: true, - }; - - test_scenario - .run_test( - &policy, - TestExpectations { - file_outside_repo_is_writable: false, - file_in_repo_root_is_writable: true, - file_in_dot_git_dir_is_writable: false, - }, - ) - .await; -} - -/// Under DangerFullAccess, all writes should be permitted anywhere on disk, -/// including inside the .git folder. -#[tokio::test] -async fn danger_full_access_allows_all_writes() { - let tmp = TempDir::new().expect("should be able to create temp dir"); - let test_scenario = create_test_scenario(&tmp); - let policy = SandboxPolicy::DangerFullAccess; - - test_scenario - .run_test( - &policy, - TestExpectations { - file_outside_repo_is_writable: true, - file_in_repo_root_is_writable: true, - file_in_dot_git_dir_is_writable: true, - }, - ) - .await; -} - -/// Under ReadOnly, writes should not be permitted anywhere on disk. -#[tokio::test] -async fn read_only_forbids_all_writes() { - let tmp = TempDir::new().expect("should be able to create temp dir"); - let test_scenario = create_test_scenario(&tmp); - let policy = SandboxPolicy::new_read_only_policy(); - - test_scenario - .run_test( - &policy, - TestExpectations { - file_outside_repo_is_writable: false, - file_in_repo_root_is_writable: false, - file_in_dot_git_dir_is_writable: false, - }, - ) - .await; -} - -#[tokio::test] -async fn openpty_works_under_seatbelt() { - if std::env::var(CODEX_SANDBOX_ENV_VAR) == Ok("seatbelt".to_string()) { - eprintln!("{CODEX_SANDBOX_ENV_VAR} is set to 'seatbelt', skipping test."); - return; - } - - if which::which("python3").is_err() { - eprintln!("python3 not found in PATH, skipping test."); - return; - } - - let policy = SandboxPolicy::new_read_only_policy(); - let command_cwd = AbsolutePathBuf::current_dir().expect("getcwd"); - let sandbox_cwd = command_cwd.clone(); - - let mut child = spawn_command_under_seatbelt( - vec![ - "python3".to_string(), - "-c".to_string(), - r#"import os - -master, slave = os.openpty() -os.write(slave, b"ping") -assert os.read(master, 4) == b"ping""# - .to_string(), - ], - command_cwd, - &policy, - &sandbox_cwd, - StdioPolicy::RedirectForShellTool, - /*network*/ None, - HashMap::new(), - ) - .await - .expect("should be able to spawn python under seatbelt"); - - let status = child - .wait() - .await - .expect("should be able to wait for child process"); - assert!(status.success(), "python exited with {status:?}"); -} - -#[tokio::test] -async fn java_home_finds_runtime_under_seatbelt() { - if std::env::var(CODEX_SANDBOX_ENV_VAR) == Ok("seatbelt".to_string()) { - eprintln!("{CODEX_SANDBOX_ENV_VAR} is set to 'seatbelt', skipping test."); - return; - } - - let java_home_path = Path::new("/usr/libexec/java_home"); - if !java_home_path.exists() { - eprintln!("/usr/libexec/java_home is not present, skipping test."); - return; - } - - let baseline_output = tokio::process::Command::new(java_home_path) - .env_remove("JAVA_HOME") - .output() - .await - .expect("should be able to invoke java_home outside seatbelt"); - if !baseline_output.status.success() { - eprintln!( - "java_home exited with {:?} outside seatbelt, skipping test", - baseline_output.status - ); - return; - } - - let policy = SandboxPolicy::new_read_only_policy(); - let command_cwd = AbsolutePathBuf::current_dir().expect("getcwd"); - let sandbox_cwd = command_cwd.clone(); - - let mut env: HashMap = std::env::vars().collect(); - env.remove("JAVA_HOME"); - env.remove(CODEX_SANDBOX_ENV_VAR); - - let child = spawn_command_under_seatbelt( - vec![java_home_path.to_string_lossy().to_string()], - command_cwd, - &policy, - &sandbox_cwd, - StdioPolicy::RedirectForShellTool, - /*network*/ None, - env, - ) - .await - .expect("should be able to spawn java_home under seatbelt"); - - let output = child - .wait_with_output() - .await - .expect("should be able to wait for java_home child"); - assert!( - output.status.success(), - "java_home under seatbelt exited with {:?}, stderr: {}", - output.status, - String::from_utf8_lossy(&output.stderr) - ); - - let stdout = String::from_utf8_lossy(&output.stdout); - assert!( - !stdout.trim().is_empty(), - "java_home stdout unexpectedly empty under seatbelt" - ); -} - -#[expect(clippy::expect_used)] -fn create_test_scenario(tmp: &TempDir) -> TestScenario { - let repo_parent = tmp.path().to_path_buf(); - let repo_root = repo_parent.join("repo"); - let dot_git_dir = repo_root.join(".git"); - - std::fs::create_dir(&repo_root).expect("should be able to create repo root"); - std::fs::create_dir(&dot_git_dir).expect("should be able to create .git dir"); - - TestScenario { - file_outside_repo: repo_parent.join("outside.txt"), - repo_parent, - file_in_repo_root: repo_root.join("repo_file.txt"), - repo_root, - file_in_dot_git_dir: dot_git_dir.join("dot_git_file.txt"), - } -} - -#[expect(clippy::expect_used)] -/// Note that `path` must be absolute. -async fn touch(path: &Path, policy: &SandboxPolicy) -> bool { - assert!(path.is_absolute(), "Path must be absolute: {path:?}"); - let command_cwd = AbsolutePathBuf::current_dir().expect("getcwd"); - let sandbox_cwd = command_cwd.clone(); - let mut child = spawn_command_under_seatbelt( - vec![ - "/usr/bin/touch".to_string(), - path.to_string_lossy().to_string(), - ], - command_cwd, - policy, - &sandbox_cwd, - StdioPolicy::RedirectForShellTool, - /*network*/ None, - HashMap::new(), - ) - .await - .expect("should be able to spawn command under seatbelt"); - child - .wait() - .await - .expect("should be able to wait for child process") - .success() -} diff --git a/codex-rs/core/tests/suite/sqlite_state.rs b/codex-rs/core/tests/suite/sqlite_state.rs index 1c9d6a87e1..ef517d74fd 100644 --- a/codex-rs/core/tests/suite/sqlite_state.rs +++ b/codex-rs/core/tests/suite/sqlite_state.rs @@ -325,12 +325,17 @@ async fn mcp_call_marks_thread_memory_mode_polluted_when_configured() -> Result< let server = start_mock_server().await; let call_id = "call-123"; let server_name = "rmcp"; - let tool_name = format!("mcp__{server_name}__echo"); + let namespace = format!("mcp__{server_name}__"); mount_sse_once( &server, responses::sse(vec![ ev_response_created("resp-1"), - ev_function_call(call_id, &tool_name, "{\"message\":\"ping\"}"), + responses::ev_function_call_with_namespace( + call_id, + &namespace, + "echo", + "{\"message\":\"ping\"}", + ), ev_completed("resp-1"), ]), ) diff --git a/codex-rs/core/tests/suite/truncation.rs b/codex-rs/core/tests/suite/truncation.rs index 288e8f4bdb..5bca13de50 100644 --- a/codex-rs/core/tests/suite/truncation.rs +++ b/codex-rs/core/tests/suite/truncation.rs @@ -331,7 +331,7 @@ async fn mcp_tool_call_output_exceeds_limit_truncated_for_model() -> Result<()> let call_id = "rmcp-truncated"; let server_name = "rmcp"; - let tool_name = format!("mcp__{server_name}__echo"); + let namespace = format!("mcp__{server_name}__"); // Build a very large message to exceed 10KiB once serialized. let large_msg = "long-message-with-newlines-".repeat(6000); @@ -341,7 +341,12 @@ async fn mcp_tool_call_output_exceeds_limit_truncated_for_model() -> Result<()> &server, sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, &tool_name, &args_json.to_string()), + responses::ev_function_call_with_namespace( + call_id, + &namespace, + "echo", + &args_json.to_string(), + ), responses::ev_completed("resp-1"), ]), ) @@ -426,13 +431,13 @@ async fn mcp_image_output_preserves_image_and_no_text_summary() -> Result<()> { let call_id = "rmcp-image-no-trunc"; let server_name = "rmcp"; - let tool_name = format!("mcp__{server_name}__image"); + let namespace = format!("mcp__{server_name}__"); mount_sse_once( &server, sse(vec![ ev_response_created("resp-1"), - ev_function_call(call_id, &tool_name, "{}"), + responses::ev_function_call_with_namespace(call_id, &namespace, "image", "{}"), ev_completed("resp-1"), ]), ) @@ -705,7 +710,7 @@ async fn mcp_tool_call_output_not_truncated_with_custom_limit() -> Result<()> { let call_id = "rmcp-untruncated"; let server_name = "rmcp"; - let tool_name = format!("mcp__{server_name}__echo"); + let namespace = format!("mcp__{server_name}__"); let large_msg = "a".repeat(80_000); let args_json = serde_json::json!({ "message": large_msg }); @@ -713,7 +718,12 @@ async fn mcp_tool_call_output_not_truncated_with_custom_limit() -> Result<()> { &server, sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, &tool_name, &args_json.to_string()), + responses::ev_function_call_with_namespace( + call_id, + &namespace, + "echo", + &args_json.to_string(), + ), responses::ev_completed("resp-1"), ]), ) diff --git a/codex-rs/deny.toml b/codex-rs/deny.toml index 088cda0ba1..bed4a4f2fa 100644 --- a/codex-rs/deny.toml +++ b/codex-rs/deny.toml @@ -70,7 +70,7 @@ feature-depth = 1 # A list of advisory IDs to ignore. Note that ignored advisories will still # output a note when they are encountered. ignore = [ - # Reviewed 2026-04-11. Keep this list in sync with .cargo/audit.toml. + # Reviewed 2026-04-15. Keep this list in sync with .cargo/audit.toml. # Each exception must identify the dependency path and removal condition. { id = "RUSTSEC-2024-0388", reason = "derivative is unmaintained; pulled in via starlark v0.13.0 used by execpolicy/cli/core; no fixed release yet" }, { id = "RUSTSEC-2025-0057", reason = "fxhash is unmaintained; pulled in via starlark_map/starlark v0.13.0 used by execpolicy/cli/core; no fixed release yet" }, @@ -78,6 +78,7 @@ ignore = [ # TODO(fcoury): remove this exception when syntect drops yaml-rust and bincode, or updates to versions that have fixed the vulnerabilities. { id = "RUSTSEC-2024-0320", reason = "yaml-rust is unmaintained; pulled in via syntect v5.3.0 used by codex-tui for syntax highlighting; no fixed release yet" }, { id = "RUSTSEC-2025-0141", reason = "bincode is unmaintained; pulled in via syntect v5.3.0 used by codex-tui for syntax highlighting; no fixed release yet" }, + { id = "RUSTSEC-2026-0097", reason = "rand 0.8.5 is pulled in via age v0.11.2/codex-secrets and zbus v4.4.0/keyring; no compatible rand 0.8 fixed release, remove when transitive dependencies move to rand >=0.9.3" }, ] # If this is true, then cargo deny will use the git executable to fetch advisory database. # If this is false, then it uses a built-in git library. diff --git a/codex-rs/exec-server/src/fs_sandbox.rs b/codex-rs/exec-server/src/fs_sandbox.rs index d0765afa35..7b0ac256d4 100644 --- a/codex-rs/exec-server/src/fs_sandbox.rs +++ b/codex-rs/exec-server/src/fs_sandbox.rs @@ -1,10 +1,8 @@ use std::collections::HashMap; -use std::path::PathBuf; use codex_app_server_protocol::JSONRPCErrorError; use codex_protocol::models::FileSystemPermissions; use codex_protocol::models::PermissionProfile; -use codex_protocol::permissions::FileSystemAccessMode; use codex_protocol::permissions::FileSystemSandboxPolicy; use codex_protocol::permissions::NetworkSandboxPolicy; use codex_protocol::protocol::ReadOnlyAccess; @@ -26,14 +24,6 @@ use crate::fs_helper::FsHelperPayload; use crate::fs_helper::FsHelperRequest; use crate::fs_helper::FsHelperResponse; use crate::local_file_system::current_sandbox_cwd; -use crate::local_file_system::resolve_existing_path; -use crate::protocol::FsCopyParams; -use crate::protocol::FsCreateDirectoryParams; -use crate::protocol::FsGetMetadataParams; -use crate::protocol::FsReadDirectoryParams; -use crate::protocol::FsReadFileParams; -use crate::protocol::FsRemoveParams; -use crate::protocol::FsWriteFileParams; use crate::rpc::internal_error; use crate::rpc::invalid_request; @@ -52,23 +42,16 @@ impl FileSystemSandboxRunner { sandbox: &FileSystemSandboxContext, request: FsHelperRequest, ) -> Result { - let request_sandbox_policy = - normalize_sandbox_policy_root_aliases(sandbox.sandbox_policy.clone()); let helper_sandbox_policy = normalize_sandbox_policy_root_aliases( sandbox_policy_with_helper_runtime_defaults(&sandbox.sandbox_policy), ); let cwd = current_sandbox_cwd().map_err(io_error)?; let cwd = AbsolutePathBuf::from_absolute_path(cwd.as_path()) .map_err(|err| invalid_request(format!("current directory is not absolute: {err}")))?; - let request_file_system_policy = FileSystemSandboxPolicy::from_legacy_sandbox_policy( - &request_sandbox_policy, - cwd.as_path(), - ); let file_system_policy = FileSystemSandboxPolicy::from_legacy_sandbox_policy( &helper_sandbox_policy, cwd.as_path(), ); - let request = resolve_request_paths(request, &request_file_system_policy, &cwd)?; let network_policy = NetworkSandboxPolicy::Restricted; let command = self.sandbox_exec_request( &helper_sandbox_policy, @@ -158,132 +141,6 @@ impl FileSystemSandboxRunner { } } -fn resolve_request_paths( - request: FsHelperRequest, - file_system_policy: &FileSystemSandboxPolicy, - cwd: &AbsolutePathBuf, -) -> Result { - match request { - FsHelperRequest::ReadFile(FsReadFileParams { path, sandbox }) => { - let path = resolve_sandbox_path(&path, PreserveTerminalSymlink::No)?; - ensure_path_access(file_system_policy, cwd, &path, FileSystemAccessMode::Read)?; - Ok(FsHelperRequest::ReadFile(FsReadFileParams { - path, - sandbox, - })) - } - FsHelperRequest::WriteFile(FsWriteFileParams { - path, - data_base64, - sandbox, - }) => Ok(FsHelperRequest::WriteFile(FsWriteFileParams { - path: { - let path = resolve_sandbox_path(&path, PreserveTerminalSymlink::No)?; - ensure_path_access(file_system_policy, cwd, &path, FileSystemAccessMode::Write)?; - path - }, - data_base64, - sandbox, - })), - FsHelperRequest::CreateDirectory(FsCreateDirectoryParams { - path, - recursive, - sandbox, - }) => Ok(FsHelperRequest::CreateDirectory(FsCreateDirectoryParams { - path: { - let path = resolve_sandbox_path(&path, PreserveTerminalSymlink::No)?; - ensure_path_access(file_system_policy, cwd, &path, FileSystemAccessMode::Write)?; - path - }, - recursive, - sandbox, - })), - FsHelperRequest::GetMetadata(FsGetMetadataParams { path, sandbox }) => { - let path = resolve_sandbox_path(&path, PreserveTerminalSymlink::No)?; - ensure_path_access(file_system_policy, cwd, &path, FileSystemAccessMode::Read)?; - Ok(FsHelperRequest::GetMetadata(FsGetMetadataParams { - path, - sandbox, - })) - } - FsHelperRequest::ReadDirectory(FsReadDirectoryParams { path, sandbox }) => { - let path = resolve_sandbox_path(&path, PreserveTerminalSymlink::No)?; - ensure_path_access(file_system_policy, cwd, &path, FileSystemAccessMode::Read)?; - Ok(FsHelperRequest::ReadDirectory(FsReadDirectoryParams { - path, - sandbox, - })) - } - FsHelperRequest::Remove(FsRemoveParams { - path, - recursive, - force, - sandbox, - }) => Ok(FsHelperRequest::Remove(FsRemoveParams { - path: { - let path = resolve_sandbox_path(&path, PreserveTerminalSymlink::Yes)?; - ensure_path_access(file_system_policy, cwd, &path, FileSystemAccessMode::Write)?; - path - }, - recursive, - force, - sandbox, - })), - FsHelperRequest::Copy(FsCopyParams { - source_path, - destination_path, - recursive, - sandbox, - }) => Ok(FsHelperRequest::Copy(FsCopyParams { - source_path: { - let source_path = resolve_sandbox_path(&source_path, PreserveTerminalSymlink::Yes)?; - ensure_path_access( - file_system_policy, - cwd, - &source_path, - FileSystemAccessMode::Read, - )?; - source_path - }, - destination_path: { - let destination_path = - resolve_sandbox_path(&destination_path, PreserveTerminalSymlink::No)?; - ensure_path_access( - file_system_policy, - cwd, - &destination_path, - FileSystemAccessMode::Write, - )?; - destination_path - }, - recursive, - sandbox, - })), - } -} - -#[derive(Clone, Copy)] -enum PreserveTerminalSymlink { - Yes, - No, -} - -fn resolve_sandbox_path( - path: &AbsolutePathBuf, - preserve_terminal_symlink: PreserveTerminalSymlink, -) -> Result { - if matches!(preserve_terminal_symlink, PreserveTerminalSymlink::Yes) - && std::fs::symlink_metadata(path.as_path()) - .map(|metadata| metadata.file_type().is_symlink()) - .unwrap_or(false) - { - return Ok(normalize_top_level_alias(path.clone())); - } - - let resolved = resolve_existing_path(path.as_path()).map_err(io_error)?; - absolute_path(resolved) -} - fn normalize_sandbox_policy_root_aliases(sandbox_policy: SandboxPolicy) -> SandboxPolicy { let mut sandbox_policy = sandbox_policy; match &mut sandbox_policy { @@ -338,33 +195,6 @@ fn normalize_top_level_alias(path: AbsolutePathBuf) -> AbsolutePathBuf { path } -fn absolute_path(path: PathBuf) -> Result { - AbsolutePathBuf::from_absolute_path(path.as_path()) - .map_err(|err| invalid_request(format!("resolved sandbox path is not absolute: {err}"))) -} - -fn ensure_path_access( - file_system_policy: &FileSystemSandboxPolicy, - cwd: &AbsolutePathBuf, - path: &AbsolutePathBuf, - required_access: FileSystemAccessMode, -) -> Result<(), JSONRPCErrorError> { - let actual_access = file_system_policy.resolve_access_with_cwd(path.as_path(), cwd.as_path()); - let permitted = match required_access { - FileSystemAccessMode::Read => actual_access.can_read(), - FileSystemAccessMode::Write => actual_access.can_write(), - FileSystemAccessMode::None => true, - }; - if permitted { - return Ok(()); - } - - Err(invalid_request(format!( - "{} is not permitted by filesystem sandbox", - path.display() - ))) -} - async fn run_command( command: SandboxExecRequest, request_json: Vec, diff --git a/codex-rs/exec-server/src/local_file_system.rs b/codex-rs/exec-server/src/local_file_system.rs index fe9a4a84f4..c7d628ce8f 100644 --- a/codex-rs/exec-server/src/local_file_system.rs +++ b/codex-rs/exec-server/src/local_file_system.rs @@ -305,7 +305,9 @@ impl ExecutorFileSystem for DirectFileSystem { let mut entries = Vec::new(); let mut read_dir = tokio::fs::read_dir(path.as_path()).await?; while let Some(entry) = read_dir.next_entry().await? { - let metadata = tokio::fs::metadata(entry.path()).await?; + let Ok(metadata) = tokio::fs::metadata(entry.path()).await else { + continue; + }; entries.push(ReadDirectoryEntry { file_name: entry.file_name().to_string_lossy().into_owned(), is_directory: metadata.is_dir(), diff --git a/codex-rs/exec-server/tests/file_system.rs b/codex-rs/exec-server/tests/file_system.rs index e24a2605f0..8b9fb90ecc 100644 --- a/codex-rs/exec-server/tests/file_system.rs +++ b/codex-rs/exec-server/tests/file_system.rs @@ -3,6 +3,8 @@ mod common; use std::os::unix::fs::symlink; +use std::path::Path; +use std::path::PathBuf; use std::process::Command; use std::sync::Arc; @@ -17,6 +19,8 @@ use codex_exec_server::FileSystemSandboxContext; use codex_exec_server::LocalFileSystem; use codex_exec_server::ReadDirectoryEntry; use codex_exec_server::RemoveOptions; +use codex_protocol::models::FileSystemPermissions; +use codex_protocol::models::PermissionProfile; use codex_protocol::protocol::ReadOnlyAccess; use codex_protocol::protocol::SandboxPolicy; use codex_utils_absolute_path::AbsolutePathBuf; @@ -94,20 +98,26 @@ fn workspace_write_sandbox(writable_root: std::path::PathBuf) -> FileSystemSandb } fn assert_sandbox_denied(error: &std::io::Error) { - assert!( - matches!( - error.kind(), - std::io::ErrorKind::InvalidInput | std::io::ErrorKind::PermissionDenied + match error.kind() { + std::io::ErrorKind::InvalidInput | std::io::ErrorKind::PermissionDenied => { + let message = error.to_string(); + assert!( + message.contains("is not permitted") + || message.contains("Operation not permitted") + || message.contains("Permission denied"), + "unexpected sandbox error message: {message}", + ); + } + std::io::ErrorKind::NotFound => assert!( + error.to_string().contains("No such file or directory"), + "unexpected sandbox not-found message: {error}", ), - "unexpected sandbox error kind: {error:?}", - ); - let message = error.to_string(); - assert!( - message.contains("is not permitted") - || message.contains("Operation not permitted") - || message.contains("Permission denied"), - "unexpected sandbox error message: {message}", - ); + std::io::ErrorKind::Other => assert!( + error.to_string().contains("Read-only file system"), + "unexpected sandbox other error message: {error}", + ), + other => panic!("unexpected sandbox error kind: {other:?}: {error:?}"), + } } fn assert_normalized_path_rejected(error: &std::io::Error) { @@ -129,6 +139,15 @@ fn assert_normalized_path_rejected(error: &std::io::Error) { } } +fn alias_root_candidate() -> Result> { + for root in [Path::new("/tmp").to_path_buf(), std::env::temp_dir()] { + if root.is_dir() && root.canonicalize().is_ok_and(|canonical| canonical != root) { + return Ok(Some(root)); + } + } + Ok(None) +} + #[test_case(false ; "local")] #[test_case(true ; "remote")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -253,6 +272,11 @@ async fn file_system_methods_cover_surface_area(use_remote: bool) -> Result<()> "hello from trait" ); + symlink( + source_dir.join("missing-target"), + source_dir.join("broken-link"), + )?; + let mut entries = file_system .read_directory(&absolute_path(source_dir), /*sandbox*/ None) .await @@ -353,9 +377,11 @@ async fn file_system_copy_rejects_directory_without_recursive(use_remote: bool) Ok(()) } +#[test_case(false ; "local")] +#[test_case(true ; "remote")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_sandboxed_read_allows_readable_root() -> Result<()> { - let context = create_file_system_context(/*use_remote*/ false).await?; +async fn file_system_sandboxed_read_allows_readable_root(use_remote: bool) -> Result<()> { + let context = create_file_system_context(use_remote).await?; let file_system = context.file_system; let tmp = TempDir::new()?; @@ -367,7 +393,8 @@ async fn file_system_sandboxed_read_allows_readable_root() -> Result<()> { let contents = file_system .read_file(&absolute_path(file_path), Some(&sandbox)) - .await?; + .await + .with_context(|| format!("mode={use_remote}"))?; assert_eq!(contents, b"sandboxed hello"); Ok(()) @@ -381,11 +408,9 @@ async fn file_system_sandboxed_write_rejects_unwritable_path(use_remote: bool) - let file_system = context.file_system; let tmp = TempDir::new()?; - let allowed_dir = tmp.path().join("allowed"); let blocked_path = tmp.path().join("blocked.txt"); - std::fs::create_dir_all(&allowed_dir)?; - let sandbox = read_only_sandbox(allowed_dir); + let sandbox = read_only_sandbox(tmp.path().to_path_buf()); let error = match file_system .write_file( &absolute_path(blocked_path.clone()), @@ -403,6 +428,72 @@ async fn file_system_sandboxed_write_rejects_unwritable_path(use_remote: bool) - Ok(()) } +#[test_case(false ; "local")] +#[test_case(true ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_sandboxed_write_allows_explicit_alias_roots(use_remote: bool) -> Result<()> { + let Some(alias_root) = alias_root_candidate()? else { + return Ok(()); + }; + + let context = create_file_system_context(use_remote).await?; + let file_system = context.file_system; + + let tmp = tempfile::Builder::new() + .prefix("codex-fs-sandbox-alias-") + .tempdir_in(&alias_root)?; + let file_path = tmp.path().join("note.txt"); + let sandbox = workspace_write_sandbox(alias_root.clone()); + + file_system + .write_file( + &absolute_path(file_path.clone()), + b"created".to_vec(), + Some(&sandbox), + ) + .await + .with_context(|| format!("write file through alias root mode={use_remote}"))?; + assert_eq!(std::fs::read(&file_path)?, b"created"); + + Ok(()) +} + +#[test_case(false ; "local")] +#[test_case(true ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_sandboxed_write_allows_additional_write_root(use_remote: bool) -> Result<()> { + let context = create_file_system_context(use_remote).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let readable_dir = tmp.path().join("readable"); + let writable_dir = tmp.path().join("writable"); + let file_path = writable_dir.join("note.txt"); + std::fs::create_dir_all(&readable_dir)?; + std::fs::create_dir_all(&writable_dir)?; + + let mut sandbox = read_only_sandbox(readable_dir); + sandbox.additional_permissions = Some(PermissionProfile { + network: None, + file_system: Some(FileSystemPermissions { + read: None, + write: Some(vec![absolute_path(writable_dir)]), + }), + }); + + file_system + .write_file( + &absolute_path(file_path.clone()), + b"created".to_vec(), + Some(&sandbox), + ) + .await + .with_context(|| format!("write file through additional root mode={use_remote}"))?; + assert_eq!(std::fs::read(&file_path)?, b"created"); + + Ok(()) +} + #[test_case(false ; "local")] #[test_case(true ; "remote")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -432,9 +523,13 @@ async fn file_system_sandboxed_read_rejects_symlink_escape(use_remote: bool) -> Ok(()) } +#[test_case(false ; "local")] +#[test_case(true ; "remote")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_sandboxed_read_rejects_symlink_parent_dotdot_escape() -> Result<()> { - let context = create_file_system_context(/*use_remote*/ false).await?; +async fn file_system_sandboxed_read_rejects_symlink_parent_dotdot_escape( + use_remote: bool, +) -> Result<()> { + let context = create_file_system_context(use_remote).await?; let file_system = context.file_system; let tmp = TempDir::new()?; @@ -528,35 +623,6 @@ async fn file_system_create_directory_rejects_symlink_escape(use_remote: bool) - Ok(()) } -#[test_case(false ; "local")] -#[test_case(true ; "remote")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_get_metadata_rejects_symlink_escape(use_remote: bool) -> Result<()> { - let context = create_file_system_context(use_remote).await?; - let file_system = context.file_system; - - let tmp = TempDir::new()?; - let allowed_dir = tmp.path().join("allowed"); - let outside_dir = tmp.path().join("outside"); - std::fs::create_dir_all(&allowed_dir)?; - std::fs::create_dir_all(&outside_dir)?; - std::fs::write(outside_dir.join("secret.txt"), "nope")?; - symlink(&outside_dir, allowed_dir.join("link"))?; - - let requested_path = allowed_dir.join("link").join("secret.txt"); - let sandbox = read_only_sandbox(allowed_dir); - let error = match file_system - .get_metadata(&absolute_path(requested_path.clone()), Some(&sandbox)) - .await - { - Ok(_) => anyhow::bail!("get_metadata should be blocked"), - Err(error) => error, - }; - assert_sandbox_denied(&error); - - Ok(()) -} - #[test_case(false ; "local")] #[test_case(true ; "remote")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -621,9 +687,11 @@ async fn file_system_copy_rejects_symlink_escape_destination(use_remote: bool) - Ok(()) } +#[test_case(false ; "local")] +#[test_case(true ; "remote")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_remove_removes_symlink_not_target() -> Result<()> { - let context = create_file_system_context(/*use_remote*/ false).await?; +async fn file_system_remove_removes_symlink_not_target(use_remote: bool) -> Result<()> { + let context = create_file_system_context(use_remote).await?; let file_system = context.file_system; let tmp = TempDir::new()?; @@ -646,7 +714,8 @@ async fn file_system_remove_removes_symlink_not_target() -> Result<()> { }, Some(&sandbox), ) - .await?; + .await + .with_context(|| format!("mode={use_remote}"))?; assert!(!symlink_path.exists()); assert!(outside_file.exists()); @@ -655,9 +724,11 @@ async fn file_system_remove_removes_symlink_not_target() -> Result<()> { Ok(()) } +#[test_case(false ; "local")] +#[test_case(true ; "remote")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_copy_preserves_symlink_source() -> Result<()> { - let context = create_file_system_context(/*use_remote*/ false).await?; +async fn file_system_copy_preserves_symlink_source(use_remote: bool) -> Result<()> { + let context = create_file_system_context(use_remote).await?; let file_system = context.file_system; let tmp = TempDir::new()?; @@ -679,7 +750,8 @@ async fn file_system_copy_preserves_symlink_source() -> Result<()> { CopyOptions { recursive: false }, Some(&sandbox), ) - .await?; + .await + .with_context(|| format!("mode={use_remote}"))?; let copied_metadata = std::fs::symlink_metadata(&copied_symlink)?; assert!(copied_metadata.file_type().is_symlink()); diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index 758da9f43d..123038ffaa 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -481,6 +481,7 @@ pub async fn run_main(cli: Cli, arg0_paths: Arg0DispatchPaths) -> anyhow::Result loader_overrides: run_loader_overrides, cloud_requirements: run_cloud_requirements, feedback: CodexFeedback::new(), + log_db: None, environment_manager: std::sync::Arc::new(EnvironmentManager::from_env_with_runtime_paths( Some(local_runtime_paths), )), diff --git a/codex-rs/exec/tests/suite/sandbox.rs b/codex-rs/exec/tests/suite/sandbox.rs index 7a8c3238e8..691b590f4e 100644 --- a/codex-rs/exec/tests/suite/sandbox.rs +++ b/codex-rs/exec/tests/suite/sandbox.rs @@ -19,17 +19,67 @@ async fn spawn_command_under_sandbox( stdio_policy: StdioPolicy, env: HashMap, ) -> std::io::Result { - use codex_core::seatbelt::spawn_command_under_seatbelt; - spawn_command_under_seatbelt( - command, - command_cwd, + use codex_core::exec::ExecCapturePolicy; + use codex_core::exec::ExecParams; + use codex_core::exec::build_exec_request; + use codex_core::sandboxing::SandboxPermissions; + use codex_protocol::config_types::WindowsSandboxLevel; + use codex_protocol::permissions::FileSystemSandboxPolicy; + use codex_protocol::permissions::NetworkSandboxPolicy; + use std::process::Stdio; + + let codex_linux_sandbox_exe = None; + let exec_request = build_exec_request( + ExecParams { + command, + cwd: command_cwd, + expiration: 1000.into(), + capture_policy: ExecCapturePolicy::ShellTool, + env, + network: None, + sandbox_permissions: SandboxPermissions::UseDefault, + windows_sandbox_level: WindowsSandboxLevel::Disabled, + windows_sandbox_private_desktop: false, + justification: None, + arg0: None, + }, sandbox_policy, + &FileSystemSandboxPolicy::from_legacy_sandbox_policy(sandbox_policy, sandbox_cwd), + NetworkSandboxPolicy::from(sandbox_policy), sandbox_cwd, - stdio_policy, - /*network*/ None, - env, + &codex_linux_sandbox_exe, + /*use_legacy_landlock*/ false, ) - .await + .map_err(|err| io::Error::other(err.to_string()))?; + + let (program, args) = exec_request + .command + .split_first() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "command args are empty"))?; + + let mut child = tokio::process::Command::new(program); + if let Some(arg0) = exec_request.arg0.as_deref() { + child.arg0(arg0); + } + child.args(args); + child.current_dir(exec_request.cwd); + child.env_clear(); + child.envs(exec_request.env); + + match stdio_policy { + StdioPolicy::RedirectForShellTool => { + child.stdin(Stdio::null()); + child.stdout(Stdio::piped()).stderr(Stdio::piped()); + } + StdioPolicy::Inherit => { + child + .stdin(Stdio::inherit()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()); + } + } + + child.kill_on_drop(true).spawn() } #[cfg(target_os = "linux")] diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rs index df4a1f5f80..a463741c9b 100644 --- a/codex-rs/features/src/lib.rs +++ b/codex-rs/features/src/lib.rs @@ -193,6 +193,8 @@ pub enum Feature { ResponsesWebsocketsV2, /// Use the agent identity registration flow for ChatGPT-authenticated sessions. UseAgentIdentity, + /// Enable workspace dependency support. + WorkspaceDependencies, } impl Feature { @@ -674,7 +676,11 @@ pub const FEATURES: &[FeatureSpec] = &[ FeatureSpec { id: Feature::MemoryTool, key: "memories", - stage: Stage::UnderDevelopment, + stage: Stage::Experimental { + name: "Memories", + menu_description: "Allow Codex to create new memories from conversations and bring relevant memories into new conversations.", + announcement: "NEW: Codex can now generate and uses memories. Try is now with `/memories`", + }, default_enabled: false, }, FeatureSpec { @@ -937,6 +943,12 @@ pub const FEATURES: &[FeatureSpec] = &[ stage: Stage::UnderDevelopment, default_enabled: false, }, + FeatureSpec { + id: Feature::WorkspaceDependencies, + key: "workspace_dependencies", + stage: Stage::Stable, + default_enabled: true, + }, ]; pub fn unstable_features_warning_event( diff --git a/codex-rs/features/src/tests.rs b/codex-rs/features/src/tests.rs index eb9afbfe7c..102680a6db 100644 --- a/codex-rs/features/src/tests.rs +++ b/codex-rs/features/src/tests.rs @@ -200,6 +200,16 @@ fn use_agent_identity_is_under_development() { assert_eq!(Feature::UseAgentIdentity.default_enabled(), false); } +#[test] +fn workspace_dependencies_is_stable_and_enabled_by_default() { + assert_eq!(Feature::WorkspaceDependencies.stage(), Stage::Stable); + assert_eq!(Feature::WorkspaceDependencies.default_enabled(), true); + assert_eq!( + feature_for_key("workspace_dependencies"), + Some(Feature::WorkspaceDependencies) + ); +} + #[test] fn collab_is_legacy_alias_for_multi_agent() { assert_eq!(feature_for_key("multi_agent"), Some(Feature::Collab)); diff --git a/codex-rs/login/src/auth/auth_tests.rs b/codex-rs/login/src/auth/auth_tests.rs index 27064b6831..8a16f5939c 100644 --- a/codex-rs/login/src/auth/auth_tests.rs +++ b/codex-rs/login/src/auth/auth_tests.rs @@ -16,6 +16,8 @@ use serde_json::json; use std::sync::Arc; use tempfile::TempDir; use tempfile::tempdir; +use tokio::time::Duration; +use tokio::time::timeout; #[tokio::test] async fn refresh_without_id_token() { @@ -135,6 +137,7 @@ async fn pro_account_with_no_api_key_uses_chatgpt_auth() { account_id: None, }), last_refresh: Some(last_refresh), + agent_identity: None, }, auth_dot_json ); @@ -172,6 +175,7 @@ fn logout_removes_auth_file() -> Result<(), std::io::Error> { openai_api_key: Some("sk-test-key".to_string()), tokens: None, last_refresh: None, + agent_identity: None, }; super::save_auth(dir.path(), &auth_dot_json, AuthCredentialsStoreMode::File)?; let auth_file = get_auth_file(dir.path()); @@ -181,6 +185,49 @@ fn logout_removes_auth_file() -> Result<(), std::io::Error> { Ok(()) } +#[test] +fn chatgpt_auth_persists_agent_identity_for_workspace() { + let codex_home = tempdir().unwrap(); + write_auth_file( + AuthFileParams { + openai_api_key: None, + chatgpt_plan_type: Some("pro".to_string()), + chatgpt_account_id: Some("account-123".to_string()), + }, + codex_home.path(), + ) + .expect("failed to write auth file"); + let auth = super::load_auth( + codex_home.path(), + /*enable_codex_api_key_env*/ false, + AuthCredentialsStoreMode::File, + ) + .expect("load auth") + .expect("auth available"); + let record = AgentIdentityAuthRecord { + workspace_id: "account-123".to_string(), + chatgpt_user_id: Some("user-123".to_string()), + agent_runtime_id: "agent_123".to_string(), + agent_private_key: "pkcs8-base64".to_string(), + registered_at: "2026-04-13T12:00:00Z".to_string(), + }; + + auth.set_agent_identity(record.clone()) + .expect("set agent identity"); + + assert_eq!(auth.get_agent_identity("account-123"), Some(record.clone())); + assert_eq!(auth.get_agent_identity("other-account"), None); + let storage = FileAuthStorage::new(codex_home.path().to_path_buf()); + let persisted = storage + .load() + .expect("load auth") + .expect("auth should exist"); + assert_eq!(persisted.agent_identity, Some(record)); + + assert!(auth.remove_agent_identity().expect("remove agent identity")); + assert_eq!(auth.get_agent_identity("account-123"), None); +} + #[test] fn unauthorized_recovery_reports_mode_and_step_names() { let dir = tempdir().unwrap(); @@ -474,6 +521,67 @@ exit 1 } } +#[tokio::test] +async fn auth_manager_notifies_when_auth_state_changes() { + let dir = tempdir().unwrap(); + let manager = AuthManager::shared( + dir.path().to_path_buf(), + /*enable_codex_api_key_env*/ false, + AuthCredentialsStoreMode::File, + ); + let mut auth_state_rx = manager.subscribe_auth_state(); + + save_auth( + dir.path(), + &AuthDotJson { + auth_mode: Some(ApiAuthMode::ApiKey), + openai_api_key: Some("sk-test-key".to_string()), + tokens: None, + last_refresh: None, + agent_identity: None, + }, + AuthCredentialsStoreMode::File, + ) + .expect("save auth"); + + assert!( + manager.reload(), + "reload should report a changed auth state" + ); + timeout(Duration::from_secs(1), auth_state_rx.changed()) + .await + .expect("auth change notification should arrive") + .expect("auth state watch should remain open"); + + save_auth( + dir.path(), + &AuthDotJson { + auth_mode: Some(ApiAuthMode::ApiKey), + openai_api_key: Some("sk-updated-key".to_string()), + tokens: None, + last_refresh: None, + agent_identity: None, + }, + AuthCredentialsStoreMode::File, + ) + .expect("save updated auth"); + + assert!( + !manager.reload(), + "reload remains mode-stable even when the underlying credentials change" + ); + timeout(Duration::from_secs(1), auth_state_rx.changed()) + .await + .expect("auth reload notification should still arrive") + .expect("auth state watch should remain open"); + + manager.set_forced_chatgpt_workspace_id(Some("workspace-123".to_string())); + timeout(Duration::from_secs(1), auth_state_rx.changed()) + .await + .expect("workspace change notification should arrive") + .expect("auth state watch should remain open"); +} + struct AuthFileParams { openai_api_key: Option, chatgpt_plan_type: Option, diff --git a/codex-rs/login/src/auth/manager.rs b/codex-rs/login/src/auth/manager.rs index 15b129a169..8768171d95 100644 --- a/codex-rs/login/src/auth/manager.rs +++ b/codex-rs/login/src/auth/manager.rs @@ -13,6 +13,7 @@ use std::sync::Arc; use std::sync::Mutex; use std::sync::RwLock; use tokio::sync::Mutex as AsyncMutex; +use tokio::sync::watch; use codex_app_server_protocol::AuthMode; use codex_app_server_protocol::AuthMode as ApiAuthMode; @@ -20,6 +21,7 @@ use codex_protocol::config_types::ForcedLoginMethod; use codex_protocol::config_types::ModelProviderAuthInfo; use super::external_bearer::BearerTokenRefresher; +pub use crate::auth::storage::AgentIdentityAuthRecord; pub use crate::auth::storage::AuthDotJson; use crate::auth::storage::AuthStorageBackend; use crate::auth::storage::create_auth_storage; @@ -60,6 +62,7 @@ pub struct ChatgptAuth { #[derive(Debug, Clone)] pub struct ChatgptAuthTokens { state: ChatgptAuthState, + storage: Arc, } #[derive(Debug, Clone)] @@ -204,14 +207,13 @@ impl CodexAuth { client, }; + let storage = create_auth_storage(codex_home.to_path_buf(), storage_mode); match auth_mode { - ApiAuthMode::Chatgpt => { - let storage = create_auth_storage(codex_home.to_path_buf(), storage_mode); - Ok(Self::Chatgpt(ChatgptAuth { state, storage })) - } - ApiAuthMode::ChatgptAuthTokens => { - Ok(Self::ChatgptAuthTokens(ChatgptAuthTokens { state })) - } + ApiAuthMode::Chatgpt => Ok(Self::Chatgpt(ChatgptAuth { state, storage })), + ApiAuthMode::ChatgptAuthTokens => Ok(Self::ChatgptAuthTokens(ChatgptAuthTokens { + state, + storage, + })), ApiAuthMode::ApiKey => unreachable!("api key mode is handled above"), } } @@ -352,6 +354,52 @@ impl CodexAuth { self.get_current_auth_json().and_then(|t| t.tokens) } + pub fn get_agent_identity(&self, workspace_id: &str) -> Option { + self.get_current_auth_json() + .and_then(|auth| auth.agent_identity) + .filter(|identity| identity.workspace_id == workspace_id) + } + + pub fn set_agent_identity(&self, record: AgentIdentityAuthRecord) -> std::io::Result<()> { + let (state, storage) = match self { + Self::Chatgpt(auth) => (&auth.state, &auth.storage), + Self::ChatgptAuthTokens(auth) => (&auth.state, &auth.storage), + Self::ApiKey(_) => return Ok(()), + }; + let mut guard = state + .auth_dot_json + .lock() + .map_err(|_| std::io::Error::other("failed to lock auth state"))?; + let mut auth = guard + .clone() + .ok_or_else(|| std::io::Error::other("auth data is not available"))?; + auth.agent_identity = Some(record); + storage.save(&auth)?; + *guard = Some(auth); + Ok(()) + } + + pub fn remove_agent_identity(&self) -> std::io::Result { + let (state, storage) = match self { + Self::Chatgpt(auth) => (&auth.state, &auth.storage), + Self::ChatgptAuthTokens(auth) => (&auth.state, &auth.storage), + Self::ApiKey(_) => return Ok(false), + }; + let mut guard = state + .auth_dot_json + .lock() + .map_err(|_| std::io::Error::other("failed to lock auth state"))?; + let Some(mut auth) = guard.clone() else { + return Ok(false); + }; + let removed = auth.agent_identity.take().is_some(); + if removed { + storage.save(&auth)?; + *guard = Some(auth); + } + Ok(removed) + } + /// Consider this private to integration tests. pub fn create_dummy_chatgpt_auth_for_testing() -> Self { let auth_dot_json = AuthDotJson { @@ -364,6 +412,7 @@ impl CodexAuth { account_id: Some("account_id".to_string()), }), last_refresh: Some(Utc::now()), + agent_identity: None, }; let client = create_client(); @@ -439,6 +488,7 @@ pub fn login_with_api_key( openai_api_key: Some(api_key.to_string()), tokens: None, last_refresh: None, + agent_identity: None, }; save_auth(codex_home, &auth_dot_json, auth_credentials_store_mode) } @@ -812,6 +862,7 @@ impl AuthDotJson { openai_api_key: None, tokens: Some(tokens), last_refresh: Some(Utc::now()), + agent_identity: None, }) } @@ -1107,6 +1158,7 @@ pub struct AuthManager { forced_chatgpt_workspace_id: RwLock>, refresh_lock: AsyncMutex<()>, external_auth: RwLock>>, + auth_state_tx: watch::Sender<()>, } /// Configuration view required to construct a shared [`AuthManager`]. @@ -1155,6 +1207,7 @@ impl AuthManager { enable_codex_api_key_env: bool, auth_credentials_store_mode: AuthCredentialsStoreMode, ) -> Self { + let (auth_state_tx, _) = watch::channel(()); let managed_auth = load_auth( &codex_home, enable_codex_api_key_env, @@ -1173,11 +1226,13 @@ impl AuthManager { forced_chatgpt_workspace_id: RwLock::new(None), refresh_lock: AsyncMutex::new(()), external_auth: RwLock::new(None), + auth_state_tx, } } /// Create an AuthManager with a specific CodexAuth, for testing only. pub fn from_auth_for_testing(auth: CodexAuth) -> Arc { + let (auth_state_tx, _) = watch::channel(()); let cached = CachedAuth { auth: Some(auth), permanent_refresh_failure: None, @@ -1191,11 +1246,13 @@ impl AuthManager { forced_chatgpt_workspace_id: RwLock::new(None), refresh_lock: AsyncMutex::new(()), external_auth: RwLock::new(None), + auth_state_tx, }) } /// Create an AuthManager with a specific CodexAuth and codex home, for testing only. pub fn from_auth_for_testing_with_home(auth: CodexAuth, codex_home: PathBuf) -> Arc { + let (auth_state_tx, _) = watch::channel(()); let cached = CachedAuth { auth: Some(auth), permanent_refresh_failure: None, @@ -1208,10 +1265,12 @@ impl AuthManager { forced_chatgpt_workspace_id: RwLock::new(None), refresh_lock: AsyncMutex::new(()), external_auth: RwLock::new(None), + auth_state_tx, }) } pub fn external_bearer_only(config: ModelProviderAuthInfo) -> Arc { + let (auth_state_tx, _) = watch::channel(()); Arc::new(Self { codex_home: PathBuf::from("non-existent"), inner: RwLock::new(CachedAuth { @@ -1225,6 +1284,7 @@ impl AuthManager { external_auth: RwLock::new(Some( Arc::new(BearerTokenRefresher::new(config)) as Arc )), + auth_state_tx, }) } @@ -1364,6 +1424,7 @@ impl AuthManager { } tracing::info!("Reloaded auth, changed: {changed}"); guard.auth = new_auth; + self.auth_state_tx.send_replace(()); changed } else { false @@ -1373,18 +1434,23 @@ impl AuthManager { pub fn set_external_auth(&self, external_auth: Arc) { if let Ok(mut guard) = self.external_auth.write() { *guard = Some(external_auth); + self.auth_state_tx.send_replace(()); } } pub fn clear_external_auth(&self) { if let Ok(mut guard) = self.external_auth.write() { *guard = None; + self.auth_state_tx.send_replace(()); } } pub fn set_forced_chatgpt_workspace_id(&self, workspace_id: Option) { - if let Ok(mut guard) = self.forced_chatgpt_workspace_id.write() { + if let Ok(mut guard) = self.forced_chatgpt_workspace_id.write() + && *guard != workspace_id + { *guard = workspace_id; + self.auth_state_tx.send_replace(()); } } @@ -1395,6 +1461,10 @@ impl AuthManager { .and_then(|guard| guard.clone()) } + pub fn subscribe_auth_state(&self) -> watch::Receiver<()> { + self.auth_state_tx.subscribe() + } + pub fn has_external_auth(&self) -> bool { self.external_auth().is_some() } @@ -1638,8 +1708,14 @@ impl AuthManager { ), ))); } - let auth_dot_json = + let mut auth_dot_json = AuthDotJson::from_external_tokens(&refreshed).map_err(RefreshTokenError::Transient)?; + if let Some(previous_auth) = self + .auth_cached() + .and_then(|auth| auth.get_current_auth_json()) + { + auth_dot_json.agent_identity = previous_auth.agent_identity; + } save_auth( &self.codex_home, &auth_dot_json, diff --git a/codex-rs/login/src/auth/storage.rs b/codex-rs/login/src/auth/storage.rs index 97e801415c..f0d7a3169c 100644 --- a/codex-rs/login/src/auth/storage.rs +++ b/codex-rs/login/src/auth/storage.rs @@ -39,6 +39,19 @@ pub struct AuthDotJson { #[serde(default, skip_serializing_if = "Option::is_none")] pub last_refresh: Option>, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub agent_identity: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, PartialEq, Eq)] +pub struct AgentIdentityAuthRecord { + pub workspace_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub chatgpt_user_id: Option, + pub agent_runtime_id: String, + pub agent_private_key: String, + pub registered_at: String, } pub(super) fn get_auth_file(codex_home: &Path) -> PathBuf { diff --git a/codex-rs/login/src/auth/storage_tests.rs b/codex-rs/login/src/auth/storage_tests.rs index 4bf72c11b9..2e1cc85029 100644 --- a/codex-rs/login/src/auth/storage_tests.rs +++ b/codex-rs/login/src/auth/storage_tests.rs @@ -18,6 +18,7 @@ async fn file_storage_load_returns_auth_dot_json() -> anyhow::Result<()> { openai_api_key: Some("test-key".to_string()), tokens: None, last_refresh: Some(Utc::now()), + agent_identity: None, }; storage @@ -38,6 +39,7 @@ async fn file_storage_save_persists_auth_dot_json() -> anyhow::Result<()> { openai_api_key: Some("test-key".to_string()), tokens: None, last_refresh: Some(Utc::now()), + agent_identity: None, }; let file = get_auth_file(codex_home.path()); @@ -52,6 +54,30 @@ async fn file_storage_save_persists_auth_dot_json() -> anyhow::Result<()> { Ok(()) } +#[tokio::test] +async fn file_storage_persists_agent_identity() -> anyhow::Result<()> { + let codex_home = tempdir()?; + let storage = FileAuthStorage::new(codex_home.path().to_path_buf()); + let auth_dot_json = AuthDotJson { + auth_mode: Some(AuthMode::Chatgpt), + openai_api_key: None, + tokens: None, + last_refresh: Some(Utc::now()), + agent_identity: Some(AgentIdentityAuthRecord { + workspace_id: "account-123".to_string(), + chatgpt_user_id: Some("user-123".to_string()), + agent_runtime_id: "agent_123".to_string(), + agent_private_key: "pkcs8-base64".to_string(), + registered_at: "2026-04-13T12:00:00Z".to_string(), + }), + }; + + storage.save(&auth_dot_json)?; + + assert_eq!(storage.load()?, Some(auth_dot_json)); + Ok(()) +} + #[test] fn file_storage_delete_removes_auth_file() -> anyhow::Result<()> { let dir = tempdir()?; @@ -60,6 +86,7 @@ fn file_storage_delete_removes_auth_file() -> anyhow::Result<()> { openai_api_key: Some("sk-test-key".to_string()), tokens: None, last_refresh: None, + agent_identity: None, }; let storage = create_auth_storage(dir.path().to_path_buf(), AuthCredentialsStoreMode::File); storage.save(&auth_dot_json)?; @@ -83,6 +110,7 @@ fn ephemeral_storage_save_load_delete_is_in_memory_only() -> anyhow::Result<()> openai_api_key: Some("sk-ephemeral".to_string()), tokens: None, last_refresh: Some(Utc::now()), + agent_identity: None, }; storage.save(&auth_dot_json)?; @@ -181,6 +209,7 @@ fn auth_with_prefix(prefix: &str) -> AuthDotJson { account_id: Some(format!("{prefix}-account-id")), }), last_refresh: None, + agent_identity: None, } } @@ -197,6 +226,7 @@ fn keyring_auth_storage_load_returns_deserialized_auth() -> anyhow::Result<()> { openai_api_key: Some("sk-test".to_string()), tokens: None, last_refresh: None, + agent_identity: None, }; seed_keyring_with_auth( &mock_keyring, @@ -239,6 +269,7 @@ fn keyring_auth_storage_save_persists_and_removes_fallback_file() -> anyhow::Res account_id: Some("account".to_string()), }), last_refresh: Some(Utc::now()), + agent_identity: None, }; storage.save(&auth)?; diff --git a/codex-rs/login/src/lib.rs b/codex-rs/login/src/lib.rs index 1b52223bc3..4880e43106 100644 --- a/codex-rs/login/src/lib.rs +++ b/codex-rs/login/src/lib.rs @@ -20,6 +20,7 @@ pub use server::ShutdownHandle; pub use server::run_login_server; pub use api_bridge::auth_provider_from_auth; +pub use auth::AgentIdentityAuthRecord; pub use auth::AuthConfig; pub use auth::AuthDotJson; pub use auth::AuthManager; diff --git a/codex-rs/login/src/server.rs b/codex-rs/login/src/server.rs index 169a8a3091..0c7b810184 100644 --- a/codex-rs/login/src/server.rs +++ b/codex-rs/login/src/server.rs @@ -781,6 +781,7 @@ pub(crate) async fn persist_tokens_async( openai_api_key: api_key, tokens: Some(tokens), last_refresh: Some(Utc::now()), + agent_identity: None, }; save_auth(&codex_home, &auth, auth_credentials_store_mode) }) diff --git a/codex-rs/login/tests/suite/auth_refresh.rs b/codex-rs/login/tests/suite/auth_refresh.rs index bf9e03bc26..d754c9589d 100644 --- a/codex-rs/login/tests/suite/auth_refresh.rs +++ b/codex-rs/login/tests/suite/auth_refresh.rs @@ -54,6 +54,7 @@ async fn refresh_token_succeeds_updates_storage() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -117,6 +118,7 @@ async fn refresh_token_refreshes_when_auth_is_unchanged() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -171,6 +173,7 @@ async fn refresh_token_skips_refresh_when_auth_changed() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -180,6 +183,7 @@ async fn refresh_token_skips_refresh_when_auth_changed() -> Result<()> { openai_api_key: None, tokens: Some(disk_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; save_auth( ctx.codex_home.path(), @@ -234,6 +238,7 @@ async fn refresh_token_errors_on_account_mismatch() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -244,6 +249,7 @@ async fn refresh_token_errors_on_account_mismatch() -> Result<()> { openai_api_key: None, tokens: Some(disk_tokens), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; save_auth( ctx.codex_home.path(), @@ -302,6 +308,7 @@ async fn returns_fresh_tokens_as_is() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(stale_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -349,6 +356,7 @@ async fn refreshes_token_when_access_token_is_expired() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(fresh_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -398,6 +406,7 @@ async fn auth_reloads_disk_auth_when_cached_auth_is_stale() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens), last_refresh: Some(stale_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -408,6 +417,7 @@ async fn auth_reloads_disk_auth_when_cached_auth_is_stale() -> Result<()> { openai_api_key: None, tokens: Some(disk_tokens.clone()), last_refresh: Some(fresh_refresh), + agent_identity: None, }; save_auth( ctx.codex_home.path(), @@ -459,6 +469,7 @@ async fn auth_reloads_disk_auth_without_calling_expired_refresh_token() -> Resul openai_api_key: None, tokens: Some(initial_tokens), last_refresh: Some(stale_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -469,6 +480,7 @@ async fn auth_reloads_disk_auth_without_calling_expired_refresh_token() -> Resul openai_api_key: None, tokens: Some(disk_tokens.clone()), last_refresh: Some(fresh_refresh), + agent_identity: None, }; save_auth( ctx.codex_home.path(), @@ -518,6 +530,7 @@ async fn refresh_token_returns_permanent_error_for_expired_refresh_token() -> Re openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -570,6 +583,7 @@ async fn refresh_token_does_not_retry_after_permanent_failure() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -636,6 +650,7 @@ async fn refresh_token_reloads_changed_auth_after_permanent_failure() -> Result< openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -657,6 +672,7 @@ async fn refresh_token_reloads_changed_auth_after_permanent_failure() -> Result< openai_api_key: None, tokens: Some(disk_tokens.clone()), last_refresh: Some(fresh_refresh), + agent_identity: None, }; save_auth( ctx.codex_home.path(), @@ -715,6 +731,7 @@ async fn refresh_token_returns_transient_error_on_server_failure() -> Result<()> openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -767,6 +784,7 @@ async fn unauthorized_recovery_reloads_then_refreshes_tokens() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -776,6 +794,7 @@ async fn unauthorized_recovery_reloads_then_refreshes_tokens() -> Result<()> { openai_api_key: None, tokens: Some(disk_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; save_auth( ctx.codex_home.path(), @@ -859,6 +878,7 @@ async fn unauthorized_recovery_errors_on_account_mismatch() -> Result<()> { openai_api_key: None, tokens: Some(initial_tokens.clone()), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; ctx.write_auth(&initial_auth)?; @@ -869,6 +889,7 @@ async fn unauthorized_recovery_errors_on_account_mismatch() -> Result<()> { openai_api_key: None, tokens: Some(disk_tokens), last_refresh: Some(initial_last_refresh), + agent_identity: None, }; save_auth( ctx.codex_home.path(), @@ -926,6 +947,7 @@ async fn unauthorized_recovery_requires_chatgpt_auth() -> Result<()> { openai_api_key: Some("sk-test".to_string()), tokens: None, last_refresh: None, + agent_identity: None, }; ctx.write_auth(&auth)?; diff --git a/codex-rs/model-provider-info/src/lib.rs b/codex-rs/model-provider-info/src/lib.rs index f39ff0c7fe..80fb705605 100644 --- a/codex-rs/model-provider-info/src/lib.rs +++ b/codex-rs/model-provider-info/src/lib.rs @@ -7,6 +7,7 @@ use codex_api::Provider as ApiProvider; use codex_api::RetryConfig as ApiRetryConfig; +use codex_api::is_azure_responses_provider; use codex_app_server_protocol::AuthMode; use codex_protocol::config_types::ModelProviderAuthInfo; use codex_protocol::error::CodexErr; @@ -300,6 +301,10 @@ impl ModelProviderInfo { self.name == OPENAI_PROVIDER_NAME } + pub fn supports_remote_compaction(&self) -> bool { + self.is_openai() || is_azure_responses_provider(&self.name, self.base_url.as_deref()) + } + pub fn has_command_auth(&self) -> bool { self.auth.is_some() } diff --git a/codex-rs/model-provider-info/src/model_provider_info_tests.rs b/codex-rs/model-provider-info/src/model_provider_info_tests.rs index bdc2e73b1a..6cd1ef2366 100644 --- a/codex-rs/model-provider-info/src/model_provider_info_tests.rs +++ b/codex-rs/model-provider-info/src/model_provider_info_tests.rs @@ -129,6 +129,61 @@ supports_websockets = true assert_eq!(provider.websocket_connect_timeout_ms, Some(15_000)); } +#[test] +fn test_supports_remote_compaction_for_openai() { + let provider = ModelProviderInfo::create_openai_provider(/*base_url*/ None); + + assert!(provider.supports_remote_compaction()); +} + +#[test] +fn test_supports_remote_compaction_for_azure_name() { + let provider = ModelProviderInfo { + name: "Azure".into(), + base_url: Some("https://example.com/openai".into()), + env_key: Some("AZURE_OPENAI_API_KEY".into()), + env_key_instructions: None, + experimental_bearer_token: None, + auth: None, + wire_api: WireApi::Responses, + query_params: None, + http_headers: None, + env_http_headers: None, + request_max_retries: None, + stream_max_retries: None, + stream_idle_timeout_ms: None, + websocket_connect_timeout_ms: None, + requires_openai_auth: false, + supports_websockets: false, + }; + + assert!(provider.supports_remote_compaction()); +} + +#[test] +fn test_supports_remote_compaction_for_non_openai_non_azure_provider() { + let provider = ModelProviderInfo { + name: "Example".into(), + base_url: Some("https://example.com/v1".into()), + env_key: Some("API_KEY".into()), + env_key_instructions: None, + experimental_bearer_token: None, + auth: None, + wire_api: WireApi::Responses, + query_params: None, + http_headers: None, + env_http_headers: None, + request_max_retries: None, + stream_max_retries: None, + stream_idle_timeout_ms: None, + websocket_connect_timeout_ms: None, + requires_openai_auth: false, + supports_websockets: false, + }; + + assert!(!provider.supports_remote_compaction()); +} + #[test] fn test_deserialize_provider_auth_config_defaults() { let base_dir = tempdir().unwrap(); diff --git a/codex-rs/protocol/src/lib.rs b/codex-rs/protocol/src/lib.rs index de580de6f3..2506dae747 100644 --- a/codex-rs/protocol/src/lib.rs +++ b/codex-rs/protocol/src/lib.rs @@ -2,8 +2,10 @@ pub mod account; mod agent_path; pub mod auth; mod thread_id; +mod tool_name; pub use agent_path::AgentPath; pub use thread_id::ThreadId; +pub use tool_name::ToolName; pub mod approvals; pub mod config_types; pub mod dynamic_tools; diff --git a/codex-rs/tools/src/tool_name.rs b/codex-rs/protocol/src/tool_name.rs similarity index 62% rename from codex-rs/tools/src/tool_name.rs rename to codex-rs/protocol/src/tool_name.rs index da18233abb..d09bc1ce72 100644 --- a/codex-rs/tools/src/tool_name.rs +++ b/codex-rs/protocol/src/tool_name.rs @@ -1,12 +1,23 @@ +use serde::Deserialize; +use serde::Serialize; +use std::fmt; + /// Identifies a callable tool, preserving the namespace split when the model /// provides one. -#[derive(Clone, Debug, Eq, Hash, PartialEq)] +#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] pub struct ToolName { pub name: String, pub namespace: Option, } impl ToolName { + pub fn new(namespace: Option, name: impl Into) -> Self { + Self { + name: name.into(), + namespace, + } + } + pub fn plain(name: impl Into) -> Self { Self { name: name.into(), @@ -29,6 +40,15 @@ impl ToolName { } } +impl fmt::Display for ToolName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.namespace { + Some(namespace) => write!(f, "{namespace}{}", self.name), + None => f.write_str(&self.name), + } + } +} + impl From for ToolName { fn from(name: String) -> Self { Self::plain(name) diff --git a/codex-rs/rmcp-client/src/bin/test_stdio_server.rs b/codex-rs/rmcp-client/src/bin/test_stdio_server.rs index 7295b63159..326fd94279 100644 --- a/codex-rs/rmcp-client/src/bin/test_stdio_server.rs +++ b/codex-rs/rmcp-client/src/bin/test_stdio_server.rs @@ -1,4 +1,5 @@ use std::borrow::Cow; +use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::hash_map::Entry; use std::sync::Arc; @@ -41,6 +42,7 @@ struct TestToolServer { const MEMO_URI: &str = "memo://codex/example-note"; const MEMO_CONTENT: &str = "This is a sample MCP resource served by the rmcp test server."; +const SANDBOX_STATE_META_CAPABILITY: &str = "codex/sandbox-state-meta"; const SMALL_PNG_BASE64: &str = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg=="; pub fn stdio() -> (tokio::io::Stdin, tokio::io::Stdout) { @@ -49,12 +51,27 @@ pub fn stdio() -> (tokio::io::Stdin, tokio::io::Stdout) { impl TestToolServer { fn new() -> Self { + #[expect(clippy::expect_used)] + let sandbox_meta_schema: JsonObject = serde_json::from_value(serde_json::json!({ + "type": "object", + "properties": {}, + "additionalProperties": false + })) + .expect("sandbox_meta tool schema should deserialize"); + let mut sandbox_meta_tool = Tool::new( + Cow::Borrowed("sandbox_meta"), + Cow::Borrowed("Return the MCP request metadata received by this test server."), + Arc::new(sandbox_meta_schema), + ); + sandbox_meta_tool.annotations = Some(ToolAnnotations::new().read_only(true)); + let tools = vec![ Self::echo_tool(), Self::echo_dash_tool(), Self::sync_tool(), Self::image_tool(), Self::image_scenario_tool(), + sandbox_meta_tool, ]; let resources = vec![Self::memo_resource()]; let resource_templates = vec![Self::memo_template()]; @@ -341,12 +358,18 @@ struct ImageScenarioArgs { impl ServerHandler for TestToolServer { fn get_info(&self) -> ServerInfo { + let mut capabilities = ServerCapabilities::builder() + .enable_tools() + .enable_tool_list_changed() + .enable_resources() + .build(); + capabilities.experimental = Some(BTreeMap::from([( + SANDBOX_STATE_META_CAPABILITY.to_string(), + JsonObject::new(), + )])); + ServerInfo { - capabilities: ServerCapabilities::builder() - .enable_tools() - .enable_tool_list_changed() - .enable_resources() - .build(), + capabilities, ..ServerInfo::default() } } @@ -418,9 +441,15 @@ impl ServerHandler for TestToolServer { async fn call_tool( &self, request: CallToolRequestParams, - _context: rmcp::service::RequestContext, + context: rmcp::service::RequestContext, ) -> Result { match request.name.as_ref() { + "sandbox_meta" => Ok(CallToolResult { + content: Vec::new(), + structured_content: Some(serde_json::Value::Object(context.meta.0)), + is_error: Some(false), + meta: None, + }), "echo" | "echo-tool" => { let args: EchoArgs = match request.arguments { Some(arguments) => serde_json::from_value(serde_json::Value::Object( diff --git a/codex-rs/rollout/src/lib.rs b/codex-rs/rollout/src/lib.rs index 31fcf47240..de169db1f7 100644 --- a/codex-rs/rollout/src/lib.rs +++ b/codex-rs/rollout/src/lib.rs @@ -48,6 +48,7 @@ pub use list::get_threads_in_root; pub use list::parse_cursor; pub use list::read_head_for_summary; pub use list::read_session_meta_line; +pub use list::read_thread_item_from_rollout; pub use list::rollout_date_parts; pub use metadata::builder_from_items; pub use policy::EventPersistenceMode; diff --git a/codex-rs/rollout/src/list.rs b/codex-rs/rollout/src/list.rs index afa4103307..640f0b0278 100644 --- a/codex-rs/rollout/src/list.rs +++ b/codex-rs/rollout/src/list.rs @@ -756,6 +756,21 @@ async fn build_thread_item( None } +/// Read a single rollout file into the same summary item shape used by thread listing. +/// +/// This is for callers that already resolved a rollout path and need the same +/// metadata/preview extraction as list operations without scanning the whole +/// sessions tree. +pub async fn read_thread_item_from_rollout(path: PathBuf) -> Option { + build_thread_item( + path, + &[], + /*provider_matcher*/ None, + /*updated_at*/ None, + ) + .await +} + /// Collects immediate subdirectories of `parent`, parses their (string) names with `parse`, /// and returns them sorted descending by the parsed key. async fn collect_dirs_desc(parent: &Path, parse: F) -> io::Result> diff --git a/codex-rs/rollout/src/recorder.rs b/codex-rs/rollout/src/recorder.rs index 4f88ff1a21..ab1ae0a5bf 100644 --- a/codex-rs/rollout/src/recorder.rs +++ b/codex-rs/rollout/src/recorder.rs @@ -279,6 +279,10 @@ impl RolloutRecorder { let codex_home = config.codex_home(); let state_db_ctx = state_db::get_state_db(config).await; + // Search is the SQLite-optimized path and assumes a DB marked backfill-complete is + // actually populated enough to answer the query. If unmigrated rollout files still exist + // on disk, the repair path below may or may not run and catch them depending on whether + // SQLite already has another matching search hit. if search_term.is_some() && let Some(db_page) = state_db::list_threads_db( state_db_ctx.as_deref(), diff --git a/codex-rs/sandboxing/Cargo.toml b/codex-rs/sandboxing/Cargo.toml index 274e5bce22..18c5ffb45d 100644 --- a/codex-rs/sandboxing/Cargo.toml +++ b/codex-rs/sandboxing/Cargo.toml @@ -23,5 +23,8 @@ url = { workspace = true } which = { workspace = true } [dev-dependencies] +anyhow = { workspace = true } +async-trait = { workspace = true } pretty_assertions = { workspace = true } tempfile = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/codex-rs/sandboxing/src/manager.rs b/codex-rs/sandboxing/src/manager.rs index ebff33f105..0836eb2565 100644 --- a/codex-rs/sandboxing/src/manager.rs +++ b/codex-rs/sandboxing/src/manager.rs @@ -9,10 +9,6 @@ use crate::policy_transforms::EffectiveSandboxPermissions; use crate::policy_transforms::effective_file_system_sandbox_policy; use crate::policy_transforms::effective_network_sandbox_policy; use crate::policy_transforms::should_require_platform_sandbox; -#[cfg(target_os = "macos")] -use crate::seatbelt::MACOS_PATH_TO_SEATBELT_EXECUTABLE; -#[cfg(target_os = "macos")] -use crate::seatbelt::create_seatbelt_command_args_for_policies; use codex_network_proxy::NetworkProxy; use codex_protocol::config_types::WindowsSandboxLevel; use codex_protocol::models::PermissionProfile; @@ -208,14 +204,19 @@ impl SandboxManager { SandboxType::None => (os_argv_to_strings(argv), None), #[cfg(target_os = "macos")] SandboxType::MacosSeatbelt => { - let mut args = create_seatbelt_command_args_for_policies( - os_argv_to_strings(argv), - &effective_file_system_policy, - effective_network_policy, + use crate::seatbelt::CreateSeatbeltCommandArgsParams; + use crate::seatbelt::MACOS_PATH_TO_SEATBELT_EXECUTABLE; + use crate::seatbelt::create_seatbelt_command_args; + + let mut args = create_seatbelt_command_args(CreateSeatbeltCommandArgsParams { + command: os_argv_to_strings(argv), + file_system_sandbox_policy: &effective_file_system_policy, + network_sandbox_policy: effective_network_policy, sandbox_policy_cwd, enforce_managed_network, network, - ); + extra_allow_unix_sockets: &[], + }); let mut full_command = Vec::with_capacity(1 + args.len()); full_command.push(MACOS_PATH_TO_SEATBELT_EXECUTABLE.to_string()); full_command.append(&mut args); diff --git a/codex-rs/sandboxing/src/seatbelt.rs b/codex-rs/sandboxing/src/seatbelt.rs index 517a9163df..b12492210c 100644 --- a/codex-rs/sandboxing/src/seatbelt.rs +++ b/codex-rs/sandboxing/src/seatbelt.rs @@ -100,41 +100,54 @@ struct UnixSocketPathParam { path: AbsolutePathBuf, } -fn proxy_policy_inputs(network: Option<&NetworkProxy>) -> ProxyPolicyInputs { - if let Some(network) = network { - let mut env = HashMap::new(); - network.apply_to_env(&mut env); - let unix_domain_socket_policy = if network.dangerously_allow_all_unix_sockets() { - UnixDomainSocketPolicy::AllowAll - } else { - let allowed = network - .allow_unix_sockets() - .iter() - .filter_map( - |socket_path| match normalize_path_for_sandbox(Path::new(socket_path)) { - Some(path) => Some((path.to_string_lossy().to_string(), path)), - None => { - warn!( - "ignoring network.allow_unix_sockets entry because it could not be normalized: {socket_path}" - ); - None - } - }, - ) - .collect::>() - .into_values() - .collect(); - UnixDomainSocketPolicy::Restricted { allowed } - }; - return ProxyPolicyInputs { - ports: proxy_loopback_ports_from_env(&env), - has_proxy_config: has_proxy_url_env_vars(&env), - allow_local_binding: network.allow_local_binding(), - unix_domain_socket_policy, - }; - } +fn proxy_policy_inputs( + network: Option<&NetworkProxy>, + extra_allow_unix_sockets: &[AbsolutePathBuf], +) -> ProxyPolicyInputs { + let extra_allowed = extra_allow_unix_sockets + .iter() + .filter_map(|socket_path| normalize_path_for_sandbox(socket_path.as_path())) + .collect::>(); - ProxyPolicyInputs::default() + match network { + Some(network) => { + let mut env = HashMap::new(); + network.apply_to_env(&mut env); + let unix_domain_socket_policy = if network.dangerously_allow_all_unix_sockets() { + UnixDomainSocketPolicy::AllowAll + } else { + let mut allowed = network + .allow_unix_sockets() + .iter() + .filter_map(|socket_path| { + match normalize_path_for_sandbox(Path::new(socket_path)) { + Some(path) => Some(path), + None => { + warn!( + "ignoring network.allow_unix_sockets entry because it could not be normalized: {socket_path}" + ); + None + } + } + }) + .collect::>(); + allowed.extend(extra_allowed); + UnixDomainSocketPolicy::Restricted { allowed } + }; + ProxyPolicyInputs { + ports: proxy_loopback_ports_from_env(&env), + has_proxy_config: has_proxy_url_env_vars(&env), + allow_local_binding: network.allow_local_binding(), + unix_domain_socket_policy, + } + } + None => ProxyPolicyInputs { + unix_domain_socket_policy: UnixDomainSocketPolicy::Restricted { + allowed: extra_allowed, + }, + ..Default::default() + }, + } } fn normalize_path_for_sandbox(path: &Path) -> Option { @@ -244,8 +257,14 @@ fn dynamic_network_policy_for_network( enforce_managed_network: bool, proxy: &ProxyPolicyInputs, ) -> String { - let should_use_restricted_network_policy = - !proxy.ports.is_empty() || proxy.has_proxy_config || enforce_managed_network; + let has_some_unix_socket_access = match &proxy.unix_domain_socket_policy { + UnixDomainSocketPolicy::AllowAll => true, + UnixDomainSocketPolicy::Restricted { allowed } => !allowed.is_empty(), + }; + let should_use_restricted_network_policy = !proxy.ports.is_empty() + || proxy.has_proxy_config + || enforce_managed_network + || (!network_policy.is_enabled() && has_some_unix_socket_access); if should_use_restricted_network_policy { let mut policy = String::new(); if proxy.allow_local_binding { @@ -285,9 +304,13 @@ fn dynamic_network_policy_for_network( if network_policy.is_enabled() { // No proxy env is configured: retain the existing full-network behavior. - format!( - "(allow network-outbound)\n(allow network-inbound)\n{MACOS_SEATBELT_NETWORK_POLICY}" - ) + let mut policy = String::from("(allow network-outbound)\n(allow network-inbound)\n"); + let unix_socket_policy = unix_socket_policy(proxy); + if !unix_socket_policy.is_empty() { + policy.push_str("; allow unix domain sockets for local IPC\n"); + policy.push_str(&unix_socket_policy); + } + format!("{policy}{MACOS_SEATBELT_NETWORK_POLICY}") } else { String::new() } @@ -357,31 +380,48 @@ fn build_seatbelt_access_policy( } #[cfg_attr(not(test), allow(dead_code))] -fn create_seatbelt_command_args( +fn create_seatbelt_command_args_for_legacy_policy( command: Vec, sandbox_policy: &SandboxPolicy, sandbox_policy_cwd: &Path, enforce_managed_network: bool, network: Option<&NetworkProxy>, ) -> Vec { - create_seatbelt_command_args_for_policies( + let file_system_sandbox_policy = + FileSystemSandboxPolicy::from_legacy_sandbox_policy(sandbox_policy, sandbox_policy_cwd); + create_seatbelt_command_args(CreateSeatbeltCommandArgsParams { command, - &FileSystemSandboxPolicy::from_legacy_sandbox_policy(sandbox_policy, sandbox_policy_cwd), - NetworkSandboxPolicy::from(sandbox_policy), + file_system_sandbox_policy: &file_system_sandbox_policy, + network_sandbox_policy: NetworkSandboxPolicy::from(sandbox_policy), sandbox_policy_cwd, enforce_managed_network, network, - ) + extra_allow_unix_sockets: &[], + }) } -pub fn create_seatbelt_command_args_for_policies( - command: Vec, - file_system_sandbox_policy: &FileSystemSandboxPolicy, - network_sandbox_policy: NetworkSandboxPolicy, - sandbox_policy_cwd: &Path, - enforce_managed_network: bool, - network: Option<&NetworkProxy>, -) -> Vec { +#[derive(Debug)] +pub struct CreateSeatbeltCommandArgsParams<'a> { + pub command: Vec, + pub file_system_sandbox_policy: &'a FileSystemSandboxPolicy, + pub network_sandbox_policy: NetworkSandboxPolicy, + pub sandbox_policy_cwd: &'a Path, + pub enforce_managed_network: bool, + pub network: Option<&'a NetworkProxy>, + pub extra_allow_unix_sockets: &'a [AbsolutePathBuf], +} + +pub fn create_seatbelt_command_args(args: CreateSeatbeltCommandArgsParams<'_>) -> Vec { + let CreateSeatbeltCommandArgsParams { + command, + file_system_sandbox_policy, + network_sandbox_policy, + sandbox_policy_cwd, + enforce_managed_network, + network, + extra_allow_unix_sockets, + } = args; + let unreadable_roots = file_system_sandbox_policy.get_unreadable_roots_with_cwd(sandbox_policy_cwd); let (file_write_policy, file_write_dir_params) = @@ -465,7 +505,7 @@ pub fn create_seatbelt_command_args_for_policies( } }; - let proxy = proxy_policy_inputs(network); + let proxy = proxy_policy_inputs(network, extra_allow_unix_sockets); let network_policy = dynamic_network_policy_for_network(network_sandbox_policy, enforce_managed_network, &proxy); diff --git a/codex-rs/sandboxing/src/seatbelt_tests.rs b/codex-rs/sandboxing/src/seatbelt_tests.rs index 419f3968d6..f222dd15d1 100644 --- a/codex-rs/sandboxing/src/seatbelt_tests.rs +++ b/codex-rs/sandboxing/src/seatbelt_tests.rs @@ -1,14 +1,23 @@ +use super::CreateSeatbeltCommandArgsParams; use super::MACOS_PATH_TO_SEATBELT_EXECUTABLE; use super::MACOS_SEATBELT_BASE_POLICY; use super::ProxyPolicyInputs; use super::UnixDomainSocketPolicy; use super::create_seatbelt_command_args; -use super::create_seatbelt_command_args_for_policies; +use super::create_seatbelt_command_args_for_legacy_policy; use super::dynamic_network_policy; use super::macos_dir_params; use super::normalize_path_for_sandbox; use super::unix_socket_dir_params; use super::unix_socket_policy; +use codex_network_proxy::ConfigReloader; +use codex_network_proxy::ConfigState; +use codex_network_proxy::NetworkMode; +use codex_network_proxy::NetworkProxy; +use codex_network_proxy::NetworkProxyConfig; +use codex_network_proxy::NetworkProxyConstraints; +use codex_network_proxy::NetworkProxyState; +use codex_network_proxy::build_config_state; use codex_protocol::permissions::FileSystemAccessMode; use codex_protocol::permissions::FileSystemPath; use codex_protocol::permissions::FileSystemSandboxEntry; @@ -23,6 +32,7 @@ use std::fs; use std::path::Path; use std::path::PathBuf; use std::process::Command; +use std::sync::Arc; use tempfile::TempDir; fn assert_seatbelt_denied(stderr: &[u8], path: &Path) { @@ -48,6 +58,23 @@ fn seatbelt_policy_arg(args: &[String]) -> &str { .expect("seatbelt args should include policy text") } +struct TestConfigReloader; + +#[async_trait::async_trait] +impl ConfigReloader for TestConfigReloader { + fn source_label(&self) -> String { + "seatbelt test config".to_string() + } + + async fn maybe_reload(&self) -> anyhow::Result> { + Ok(None) + } + + async fn reload_now(&self) -> anyhow::Result { + Err(anyhow::anyhow!("seatbelt test config cannot reload")) + } +} + #[test] fn base_policy_allows_node_cpu_sysctls() { assert!( @@ -128,14 +155,15 @@ fn explicit_unreadable_paths_are_excluded_from_full_disk_read_and_write_access() }, ]); - let args = create_seatbelt_command_args_for_policies( - vec!["/bin/true".to_string()], - &file_system_policy, - NetworkSandboxPolicy::Restricted, - Path::new("/"), - /*enforce_managed_network*/ false, - /*network*/ None, - ); + let args = create_seatbelt_command_args(CreateSeatbeltCommandArgsParams { + command: vec!["/bin/true".to_string()], + file_system_sandbox_policy: &file_system_policy, + network_sandbox_policy: NetworkSandboxPolicy::Restricted, + sandbox_policy_cwd: Path::new("/"), + enforce_managed_network: false, + network: None, + extra_allow_unix_sockets: &[], + }); let policy = seatbelt_policy_arg(&args); let unreadable_roots = file_system_policy.get_unreadable_roots_with_cwd(Path::new("/")); @@ -193,14 +221,15 @@ fn explicit_unreadable_paths_are_excluded_from_readable_roots() { }, ]); - let args = create_seatbelt_command_args_for_policies( - vec!["/bin/true".to_string()], - &file_system_policy, - NetworkSandboxPolicy::Restricted, - Path::new("/"), - /*enforce_managed_network*/ false, - /*network*/ None, - ); + let args = create_seatbelt_command_args(CreateSeatbeltCommandArgsParams { + command: vec!["/bin/true".to_string()], + file_system_sandbox_policy: &file_system_policy, + network_sandbox_policy: NetworkSandboxPolicy::Restricted, + sandbox_policy_cwd: Path::new("/"), + enforce_managed_network: false, + network: None, + extra_allow_unix_sockets: &[], + }); let policy = seatbelt_policy_arg(&args); let readable_roots = file_system_policy.get_readable_roots_with_cwd(Path::new("/")); @@ -231,7 +260,7 @@ fn explicit_unreadable_paths_are_excluded_from_readable_roots() { #[test] fn seatbelt_args_without_extension_profile_keep_legacy_preferences_read_access() { let cwd = std::env::temp_dir(); - let args = create_seatbelt_command_args( + let args = create_seatbelt_command_args_for_legacy_policy( vec!["echo".to_string(), "ok".to_string()], &SandboxPolicy::new_read_only_policy(), cwd.as_path(), @@ -249,7 +278,7 @@ fn seatbelt_legacy_workspace_write_nested_readable_root_stays_writable() { let cwd = tmp.path().join("workspace"); fs::create_dir_all(cwd.join("docs")).expect("create docs"); let docs = AbsolutePathBuf::from_absolute_path(cwd.join("docs")).expect("absolute docs"); - let args = create_seatbelt_command_args( + let args = create_seatbelt_command_args_for_legacy_policy( vec!["/bin/true".to_string()], &SandboxPolicy::WorkspaceWrite { writable_roots: vec![], @@ -451,6 +480,140 @@ fn create_seatbelt_args_allowlists_unix_socket_paths() { ); } +#[test] +fn create_seatbelt_args_allowlists_explicit_unix_socket_paths_without_proxy() { + let cwd = TempDir::new().expect("temp cwd"); + let file_system_policy = FileSystemSandboxPolicy::from_legacy_sandbox_policy( + &SandboxPolicy::new_read_only_policy(), + cwd.path(), + ); + let extra_allow_unix_sockets = vec![absolute_path("/tmp/codex-browser-use")]; + let args = create_seatbelt_command_args(CreateSeatbeltCommandArgsParams { + command: vec!["/usr/bin/true".to_string()], + file_system_sandbox_policy: &file_system_policy, + network_sandbox_policy: NetworkSandboxPolicy::Restricted, + sandbox_policy_cwd: cwd.path(), + enforce_managed_network: false, + network: None, + extra_allow_unix_sockets: &extra_allow_unix_sockets, + }); + let policy = seatbelt_policy_arg(&args); + + assert!( + policy.contains("(allow system-socket (socket-domain AF_UNIX))"), + "policy should allow AF_UNIX when explicit socket paths are requested:\n{policy}" + ); + assert!( + policy.contains( + "(allow network-outbound (remote unix-socket (subpath (param \"UNIX_SOCKET_PATH_0\"))))" + ), + "policy should allow outbound AF_UNIX traffic for explicit socket paths:\n{policy}" + ); + let expected_socket_root = normalize_path_for_sandbox(Path::new("/tmp/codex-browser-use")) + .expect("socket root should normalize") + .to_string_lossy() + .into_owned(); + assert!( + args.iter() + .any(|arg| arg == &format!("-DUNIX_SOCKET_PATH_0={expected_socket_root}")), + "seatbelt args should pass the configured socket root as a sandbox param: {args:?}" + ); +} + +#[tokio::test] +async fn create_seatbelt_args_merges_proxy_and_explicit_unix_socket_paths() -> anyhow::Result<()> { + let cwd = TempDir::new().expect("temp cwd"); + let file_system_policy = FileSystemSandboxPolicy::from_legacy_sandbox_policy( + &SandboxPolicy::new_read_only_policy(), + cwd.path(), + ); + let network_socket = "/tmp/codex-proxy-use"; + let explicit_socket = "/tmp/codex-browser-use"; + let mut network_config = NetworkProxyConfig::default(); + network_config.network.enabled = true; + network_config.network.mode = NetworkMode::Full; + network_config + .network + .set_allow_unix_sockets(vec![network_socket.to_string()]); + let state = build_config_state(network_config, NetworkProxyConstraints::default())?; + let network_proxy = NetworkProxy::builder() + .state(Arc::new(NetworkProxyState::with_reloader( + state, + Arc::new(TestConfigReloader), + ))) + .managed_by_codex(/*managed_by_codex*/ false) + .build() + .await?; + let extra_allow_unix_sockets = vec![absolute_path(explicit_socket)]; + + let args = create_seatbelt_command_args(CreateSeatbeltCommandArgsParams { + command: vec!["/usr/bin/true".to_string()], + file_system_sandbox_policy: &file_system_policy, + network_sandbox_policy: NetworkSandboxPolicy::Restricted, + sandbox_policy_cwd: cwd.path(), + enforce_managed_network: false, + network: Some(&network_proxy), + extra_allow_unix_sockets: &extra_allow_unix_sockets, + }); + + let expected_explicit_socket = normalize_path_for_sandbox(Path::new(explicit_socket)) + .expect("explicit socket root should normalize"); + let expected_network_socket = normalize_path_for_sandbox(Path::new(network_socket)) + .expect("network socket root should normalize"); + let unix_socket_definitions = args + .iter() + .filter(|arg| arg.starts_with("-DUNIX_SOCKET_PATH_")) + .cloned() + .collect::>(); + assert_eq!( + unix_socket_definitions, + vec![ + format!( + "-DUNIX_SOCKET_PATH_0={}", + expected_explicit_socket.display() + ), + format!("-DUNIX_SOCKET_PATH_1={}", expected_network_socket.display()), + ], + "seatbelt args should include both explicit and network proxy socket roots: {args:?}" + ); + Ok(()) +} + +#[test] +fn create_seatbelt_args_preserves_full_network_with_explicit_unix_socket_paths() { + let cwd = TempDir::new().expect("temp cwd"); + let file_system_policy = FileSystemSandboxPolicy::from_legacy_sandbox_policy( + &SandboxPolicy::new_read_only_policy(), + cwd.path(), + ); + let extra_allow_unix_sockets = vec![absolute_path("/tmp/codex-browser-use")]; + let args = create_seatbelt_command_args(CreateSeatbeltCommandArgsParams { + command: vec!["/usr/bin/true".to_string()], + file_system_sandbox_policy: &file_system_policy, + network_sandbox_policy: NetworkSandboxPolicy::Enabled, + sandbox_policy_cwd: cwd.path(), + enforce_managed_network: false, + network: None, + extra_allow_unix_sockets: &extra_allow_unix_sockets, + }); + let policy = seatbelt_policy_arg(&args); + + assert!( + policy.contains("(allow network-outbound)\n"), + "policy should preserve full outbound network access:\n{policy}" + ); + assert!( + policy.contains("(allow network-inbound)\n"), + "policy should preserve full inbound network access:\n{policy}" + ); + assert!( + policy.contains( + "(allow network-outbound (remote unix-socket (subpath (param \"UNIX_SOCKET_PATH_0\"))))" + ), + "policy should still allow outbound AF_UNIX traffic for explicit socket paths:\n{policy}" + ); +} + #[test] fn unix_socket_policy_non_empty_output_is_newline_terminated() { let allowlist_policy = unix_socket_policy(&ProxyPolicyInputs { @@ -615,7 +778,7 @@ fn create_seatbelt_args_with_read_only_git_and_codex_subpaths() { .iter() .map(std::string::ToString::to_string) .collect(); - let args = create_seatbelt_command_args( + let args = create_seatbelt_command_args_for_legacy_policy( shell_command.clone(), &policy, &cwd, @@ -729,7 +892,7 @@ fn create_seatbelt_args_with_read_only_git_and_codex_subpaths() { .iter() .map(std::string::ToString::to_string) .collect(); - let write_hooks_file_args = create_seatbelt_command_args( + let write_hooks_file_args = create_seatbelt_command_args_for_legacy_policy( shell_command_git, &policy, &cwd, @@ -765,7 +928,7 @@ fn create_seatbelt_args_with_read_only_git_and_codex_subpaths() { .iter() .map(std::string::ToString::to_string) .collect(); - let write_allowed_file_args = create_seatbelt_command_args( + let write_allowed_file_args = create_seatbelt_command_args_for_legacy_policy( shell_command_allowed, &policy, &cwd, @@ -830,7 +993,7 @@ fn create_seatbelt_args_block_first_time_dot_codex_creation_with_exact_and_desce .iter() .map(std::string::ToString::to_string) .collect(); - let args = create_seatbelt_command_args( + let args = create_seatbelt_command_args_for_legacy_policy( shell_command, &policy, repo_root.as_path(), @@ -885,7 +1048,7 @@ fn create_seatbelt_args_with_read_only_git_pointer_file() { .iter() .map(std::string::ToString::to_string) .collect(); - let args = create_seatbelt_command_args( + let args = create_seatbelt_command_args_for_legacy_policy( shell_command, &policy, &cwd, @@ -921,7 +1084,7 @@ fn create_seatbelt_args_with_read_only_git_pointer_file() { .iter() .map(std::string::ToString::to_string) .collect(); - let gitdir_args = create_seatbelt_command_args( + let gitdir_args = create_seatbelt_command_args_for_legacy_policy( shell_command_gitdir, &policy, &cwd, @@ -984,7 +1147,7 @@ fn create_seatbelt_args_for_cwd_as_git_repo() { .iter() .map(std::string::ToString::to_string) .collect(); - let args = create_seatbelt_command_args( + let args = create_seatbelt_command_args_for_legacy_policy( shell_command.clone(), &policy, vulnerable_root.as_path(), diff --git a/codex-rs/state/src/runtime/memories.rs b/codex-rs/state/src/runtime/memories.rs index 50f2f0cd8c..4817d5db67 100644 --- a/codex-rs/state/src/runtime/memories.rs +++ b/codex-rs/state/src/runtime/memories.rs @@ -30,21 +30,6 @@ impl StateRuntime { /// stage-1 (`memory_stage1`) and phase-2 (`memory_consolidate_global`) /// memory pipelines. pub async fn clear_memory_data(&self) -> anyhow::Result<()> { - self.clear_memory_data_inner(/*disable_existing_threads*/ false) - .await - } - - /// Resets persisted memory state for a clean-slate local start. - /// - /// In addition to clearing persisted stage-1 outputs and memory pipeline - /// jobs, this disables memory generation for all existing threads so - /// historical rollouts are not immediately picked up again. - pub async fn reset_memory_data_for_fresh_start(&self) -> anyhow::Result<()> { - self.clear_memory_data_inner(/*disable_existing_threads*/ true) - .await - } - - async fn clear_memory_data_inner(&self, disable_existing_threads: bool) -> anyhow::Result<()> { let mut tx = self.pool.begin().await?; sqlx::query( @@ -66,18 +51,6 @@ WHERE kind = ? OR kind = ? .execute(&mut *tx) .await?; - if disable_existing_threads { - sqlx::query( - r#" -UPDATE threads -SET memory_mode = 'disabled' -WHERE memory_mode = 'enabled' - "#, - ) - .execute(&mut *tx) - .await?; - } - tx.commit().await?; Ok(()) } @@ -1823,7 +1796,7 @@ mod tests { } #[tokio::test] - async fn reset_memory_data_for_fresh_start_clears_rows_and_disables_threads() { + async fn clear_memory_data_clears_rows_and_preserves_thread_memory_modes() { let codex_home = unique_temp_dir(); let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string()) .await @@ -1893,9 +1866,9 @@ mod tests { .expect("disable existing thread"); runtime - .reset_memory_data_for_fresh_start() + .clear_memory_data() .await - .expect("reset memory data"); + .expect("clear memory data"); let stage1_outputs_count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM stage1_outputs") .fetch_one(runtime.pool.as_ref()) @@ -1918,7 +1891,7 @@ mod tests { .fetch_one(runtime.pool.as_ref()) .await .expect("read enabled thread memory mode"); - assert_eq!(enabled_memory_mode, "disabled"); + assert_eq!(enabled_memory_mode, "enabled"); let disabled_memory_mode: String = sqlx::query_scalar("SELECT memory_mode FROM threads WHERE id = ?") diff --git a/codex-rs/thread-store/Cargo.toml b/codex-rs/thread-store/Cargo.toml index a8f22fbfe5..328d9a4a7f 100644 --- a/codex-rs/thread-store/Cargo.toml +++ b/codex-rs/thread-store/Cargo.toml @@ -14,6 +14,16 @@ workspace = true [dependencies] async-trait = { workspace = true } chrono = { workspace = true, features = ["serde"] } +codex-git-utils = { workspace = true } codex-protocol = { workspace = true } +codex-rollout = { workspace = true } serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } thiserror = { workspace = true } + +[dev-dependencies] +codex-state = { workspace = true } +pretty_assertions = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true, features = ["macros"] } +uuid = { workspace = true } diff --git a/codex-rs/thread-store/src/lib.rs b/codex-rs/thread-store/src/lib.rs index 5525db1394..a64bf023be 100644 --- a/codex-rs/thread-store/src/lib.rs +++ b/codex-rs/thread-store/src/lib.rs @@ -5,12 +5,14 @@ //! any other backing store. mod error; +mod local; mod recorder; mod store; mod types; pub use error::ThreadStoreError; pub use error::ThreadStoreResult; +pub use local::LocalThreadStore; pub use recorder::ThreadRecorder; pub use store::ThreadStore; pub use types::AppendThreadItemsParams; diff --git a/codex-rs/thread-store/src/local/archive_thread.rs b/codex-rs/thread-store/src/local/archive_thread.rs new file mode 100644 index 0000000000..4fcea6dbac --- /dev/null +++ b/codex-rs/thread-store/src/local/archive_thread.rs @@ -0,0 +1,170 @@ +use chrono::Utc; +use codex_rollout::find_thread_path_by_id_str; + +use super::LocalThreadStore; +use super::helpers::matching_rollout_file_name; +use super::helpers::scoped_rollout_path; +use crate::ArchiveThreadParams; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; + +pub(super) async fn archive_thread( + store: &LocalThreadStore, + params: ArchiveThreadParams, +) -> ThreadStoreResult<()> { + let thread_id = params.thread_id; + let rollout_path = + find_thread_path_by_id_str(store.config.codex_home.as_path(), &thread_id.to_string()) + .await + .map_err(|err| ThreadStoreError::InvalidRequest { + message: format!("failed to locate thread id {thread_id}: {err}"), + })? + .ok_or_else(|| ThreadStoreError::InvalidRequest { + message: format!("no rollout found for thread id {thread_id}"), + })?; + + let canonical_rollout_path = scoped_rollout_path( + store.config.codex_home.join(codex_rollout::SESSIONS_SUBDIR), + rollout_path.as_path(), + "sessions", + )?; + let file_name = matching_rollout_file_name( + canonical_rollout_path.as_path(), + thread_id, + rollout_path.as_path(), + )?; + + let archive_folder = store + .config + .codex_home + .join(codex_rollout::ARCHIVED_SESSIONS_SUBDIR); + std::fs::create_dir_all(&archive_folder).map_err(|err| ThreadStoreError::Internal { + message: format!("failed to archive thread: {err}"), + })?; + let archived_path = archive_folder.join(&file_name); + std::fs::rename(&canonical_rollout_path, &archived_path).map_err(|err| { + ThreadStoreError::Internal { + message: format!("failed to archive thread: {err}"), + } + })?; + + if let Some(ctx) = codex_rollout::state_db::get_state_db(&store.config).await { + let _ = ctx + .mark_archived(thread_id, archived_path.as_path(), Utc::now()) + .await; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use chrono::Utc; + use codex_protocol::ThreadId; + use codex_protocol::protocol::SessionSource; + use codex_rollout::ARCHIVED_SESSIONS_SUBDIR; + use pretty_assertions::assert_eq; + use tempfile::TempDir; + use uuid::Uuid; + + use super::*; + use crate::ListThreadsParams; + use crate::ThreadSortKey; + use crate::ThreadStore; + use crate::local::LocalThreadStore; + use crate::local::test_support::test_config; + use crate::local::test_support::write_session_file; + + #[tokio::test] + async fn archive_thread_moves_rollout_to_archived_collection() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path())); + let uuid = Uuid::from_u128(201); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + let active_path = + write_session_file(home.path(), "2025-01-03T12-00-00", uuid).expect("session file"); + + store + .archive_thread(ArchiveThreadParams { thread_id }) + .await + .expect("archive thread"); + + assert!(!active_path.exists()); + let archived_path = home + .path() + .join(ARCHIVED_SESSIONS_SUBDIR) + .join(active_path.file_name().expect("file name")); + assert!(archived_path.exists()); + + let archived = store + .list_threads(ListThreadsParams { + page_size: 10, + cursor: None, + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived: true, + search_term: None, + }) + .await + .expect("archived listing"); + assert_eq!(archived.items.len(), 1); + assert_eq!(archived.items[0].thread_id, thread_id); + assert_eq!(archived.items[0].rollout_path, Some(archived_path)); + assert_eq!( + archived.items[0].archived_at, + Some(archived.items[0].updated_at) + ); + } + + #[tokio::test] + async fn archive_thread_updates_sqlite_metadata_when_present() { + let home = TempDir::new().expect("temp dir"); + let config = test_config(home.path()); + let store = LocalThreadStore::new(config.clone()); + let uuid = Uuid::from_u128(202); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + let active_path = + write_session_file(home.path(), "2025-01-03T12-00-00", uuid).expect("session file"); + let runtime = codex_state::StateRuntime::init( + home.path().to_path_buf(), + config.model_provider_id.clone(), + ) + .await + .expect("state db should initialize"); + runtime + .mark_backfill_complete(/*last_watermark*/ None) + .await + .expect("backfill should be complete"); + let mut builder = codex_state::ThreadMetadataBuilder::new( + thread_id, + active_path.clone(), + Utc::now(), + SessionSource::Cli, + ); + builder.model_provider = Some(config.model_provider_id.clone()); + builder.cwd = home.path().to_path_buf(); + builder.cli_version = Some("test_version".to_string()); + let metadata = builder.build(config.model_provider_id.as_str()); + runtime + .upsert_thread(&metadata) + .await + .expect("state db upsert should succeed"); + + store + .archive_thread(ArchiveThreadParams { thread_id }) + .await + .expect("archive thread"); + + let archived_path = home + .path() + .join(ARCHIVED_SESSIONS_SUBDIR) + .join(active_path.file_name().expect("file name")); + let updated = runtime + .get_thread(thread_id) + .await + .expect("state db read should succeed") + .expect("thread metadata should exist"); + assert_eq!(updated.rollout_path, archived_path); + assert!(updated.archived_at.is_some()); + } +} diff --git a/codex-rs/thread-store/src/local/helpers.rs b/codex-rs/thread-store/src/local/helpers.rs new file mode 100644 index 0000000000..e28ecc6612 --- /dev/null +++ b/codex-rs/thread-store/src/local/helpers.rs @@ -0,0 +1,168 @@ +use std::ffi::OsStr; +use std::fs::FileTimes; +use std::fs::OpenOptions; +use std::path::Path; +use std::path::PathBuf; +use std::time::SystemTime; + +use chrono::DateTime; +use chrono::Utc; +use codex_git_utils::GitSha; +use codex_protocol::ThreadId; +use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::GitInfo; +use codex_protocol::protocol::SandboxPolicy; +use codex_protocol::protocol::SessionSource; +use codex_rollout::ThreadItem; + +use crate::StoredThread; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; + +pub(super) fn scoped_rollout_path( + root: PathBuf, + rollout_path: &Path, + root_name: &str, +) -> ThreadStoreResult { + let canonical_root = + std::fs::canonicalize(&root).map_err(|err| ThreadStoreError::Internal { + message: format!( + "failed to resolve {root_name} directory `{}`: {err}", + root.display() + ), + })?; + let canonical_rollout_path = + std::fs::canonicalize(rollout_path).map_err(|_| ThreadStoreError::InvalidRequest { + message: format!( + "rollout path `{}` must be in {root_name} directory", + rollout_path.display() + ), + })?; + if canonical_rollout_path.starts_with(&canonical_root) { + Ok(canonical_rollout_path) + } else { + Err(ThreadStoreError::InvalidRequest { + message: format!( + "rollout path `{}` must be in {root_name} directory", + rollout_path.display() + ), + }) + } +} + +pub(super) fn matching_rollout_file_name( + rollout_path: &Path, + thread_id: ThreadId, + display_path: &Path, +) -> ThreadStoreResult { + let Some(file_name) = rollout_path.file_name().map(OsStr::to_owned) else { + return Err(ThreadStoreError::InvalidRequest { + message: format!( + "rollout path `{}` missing file name", + display_path.display() + ), + }); + }; + let required_suffix = format!("{thread_id}.jsonl"); + if file_name + .to_string_lossy() + .ends_with(required_suffix.as_str()) + { + Ok(file_name) + } else { + Err(ThreadStoreError::InvalidRequest { + message: format!( + "rollout path `{}` does not match thread id {thread_id}", + display_path.display() + ), + }) + } +} + +pub(super) fn touch_modified_time(path: &Path) -> std::io::Result<()> { + let times = FileTimes::new().set_modified(SystemTime::now()); + OpenOptions::new().append(true).open(path)?.set_times(times) +} + +pub(super) fn stored_thread_from_rollout_item( + item: ThreadItem, + archived: bool, + default_provider: &str, +) -> Option { + let thread_id = item + .thread_id + .or_else(|| thread_id_from_rollout_path(item.path.as_path()))?; + let created_at = parse_rfc3339(item.created_at.as_deref()).unwrap_or_else(Utc::now); + let updated_at = parse_rfc3339(item.updated_at.as_deref()).unwrap_or(created_at); + let archived_at = archived.then_some(updated_at); + let git_info = git_info_from_parts( + item.git_sha.clone(), + item.git_branch.clone(), + item.git_origin_url.clone(), + ); + let source = item.source.unwrap_or(SessionSource::Unknown); + let preview = item.first_user_message.clone().unwrap_or_default(); + + Some(StoredThread { + thread_id, + rollout_path: Some(item.path), + forked_from_id: None, + preview, + name: None, + model_provider: item + .model_provider + .filter(|provider| !provider.is_empty()) + .unwrap_or_else(|| default_provider.to_string()), + model: None, + reasoning_effort: None, + created_at, + updated_at, + archived_at, + cwd: item.cwd.unwrap_or_default(), + cli_version: item.cli_version.unwrap_or_default(), + source, + agent_nickname: item.agent_nickname, + agent_role: item.agent_role, + agent_path: None, + git_info, + approval_mode: AskForApproval::OnRequest, + sandbox_policy: SandboxPolicy::new_read_only_policy(), + token_usage: None, + first_user_message: item.first_user_message, + history: None, + }) +} + +fn parse_rfc3339(value: Option<&str>) -> Option> { + DateTime::parse_from_rfc3339(value?) + .ok() + .map(|dt| dt.with_timezone(&Utc)) +} + +fn git_info_from_parts( + sha: Option, + branch: Option, + origin_url: Option, +) -> Option { + if sha.is_none() && branch.is_none() && origin_url.is_none() { + return None; + } + Some(GitInfo { + commit_hash: sha.as_deref().map(GitSha::new), + branch, + repository_url: origin_url, + }) +} + +fn thread_id_from_rollout_path(path: &Path) -> Option { + let file_name = path.file_name()?.to_str()?; + let stem = file_name.strip_suffix(".jsonl")?; + if stem.len() < 37 { + return None; + } + let uuid_start = stem.len().saturating_sub(36); + if !stem[..uuid_start].ends_with('-') { + return None; + } + ThreadId::from_string(&stem[uuid_start..]).ok() +} diff --git a/codex-rs/thread-store/src/local/list_threads.rs b/codex-rs/thread-store/src/local/list_threads.rs new file mode 100644 index 0000000000..35c6f3de43 --- /dev/null +++ b/codex-rs/thread-store/src/local/list_threads.rs @@ -0,0 +1,319 @@ +use codex_rollout::RolloutConfig; +use codex_rollout::RolloutRecorder; +use codex_rollout::parse_cursor; + +use super::LocalThreadStore; +use super::helpers::stored_thread_from_rollout_item; +use crate::ListThreadsParams; +use crate::ThreadPage; +use crate::ThreadSortKey; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; + +pub(super) async fn list_threads( + store: &LocalThreadStore, + params: ListThreadsParams, +) -> ThreadStoreResult { + let cursor = params + .cursor + .as_deref() + .map(|cursor| { + parse_cursor(cursor).ok_or_else(|| ThreadStoreError::InvalidRequest { + message: format!("invalid cursor: {cursor}"), + }) + }) + .transpose()?; + let sort_key = match params.sort_key { + ThreadSortKey::CreatedAt => codex_rollout::ThreadSortKey::CreatedAt, + ThreadSortKey::UpdatedAt => codex_rollout::ThreadSortKey::UpdatedAt, + }; + let page = list_rollout_threads(&store.config, ¶ms, cursor.as_ref(), sort_key).await?; + + let next_cursor = page + .next_cursor + .as_ref() + .and_then(|cursor| serde_json::to_value(cursor).ok()) + .and_then(|value| value.as_str().map(str::to_owned)); + let items = page + .items + .into_iter() + .filter_map(|item| { + stored_thread_from_rollout_item( + item, + params.archived, + store.config.model_provider_id.as_str(), + ) + }) + .collect::>(); + + Ok(ThreadPage { items, next_cursor }) +} + +async fn list_rollout_threads( + config: &RolloutConfig, + params: &ListThreadsParams, + cursor: Option<&codex_rollout::Cursor>, + sort_key: codex_rollout::ThreadSortKey, +) -> ThreadStoreResult { + let page = if params.archived { + RolloutRecorder::list_archived_threads( + config, + params.page_size, + cursor, + sort_key, + params.allowed_sources.as_slice(), + params.model_providers.as_deref(), + config.model_provider_id.as_str(), + params.search_term.as_deref(), + ) + .await + } else { + RolloutRecorder::list_threads( + config, + params.page_size, + cursor, + sort_key, + params.allowed_sources.as_slice(), + params.model_providers.as_deref(), + config.model_provider_id.as_str(), + params.search_term.as_deref(), + ) + .await + }; + page.map_err(|err| ThreadStoreError::Internal { + message: format!("failed to list threads: {err}"), + }) +} + +#[cfg(test)] +mod tests { + use chrono::Utc; + use codex_protocol::ThreadId; + use codex_protocol::protocol::SessionSource; + use pretty_assertions::assert_eq; + use std::fs; + use tempfile::TempDir; + use uuid::Uuid; + + use super::*; + use crate::ThreadStore; + use crate::local::LocalThreadStore; + use crate::local::test_support::test_config; + use crate::local::test_support::write_archived_session_file; + use crate::local::test_support::write_session_file; + use crate::local::test_support::write_session_file_with; + + #[tokio::test] + async fn list_threads_uses_default_provider_when_rollout_omits_provider() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path())); + write_session_file_with( + home.path(), + home.path().join("sessions/2025/01/03"), + "2025-01-03T12-00-00", + Uuid::from_u128(102), + "Hello from user", + /*model_provider*/ None, + ) + .expect("session file"); + + let page = store + .list_threads(ListThreadsParams { + page_size: 10, + cursor: None, + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived: false, + search_term: None, + }) + .await + .expect("thread listing"); + + assert_eq!(page.items.len(), 1); + assert_eq!(page.items[0].model_provider, "test-provider"); + } + + #[tokio::test] + async fn list_threads_preserves_sqlite_title_search_results() { + let home = TempDir::new().expect("temp dir"); + let config = test_config(home.path()); + let store = LocalThreadStore::new(config.clone()); + let uuid = Uuid::from_u128(103); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + let rollout_path = home.path().join("rollout-title-search.jsonl"); + fs::write(&rollout_path, "").expect("placeholder rollout file"); + + let runtime = codex_state::StateRuntime::init( + home.path().to_path_buf(), + config.model_provider_id.clone(), + ) + .await + .expect("state db should initialize"); + runtime + .mark_backfill_complete(/*last_watermark*/ None) + .await + .expect("backfill should be complete"); + let created_at = Utc::now(); + let mut builder = codex_state::ThreadMetadataBuilder::new( + thread_id, + rollout_path, + created_at, + SessionSource::Cli, + ); + builder.model_provider = Some(config.model_provider_id.clone()); + builder.cwd = home.path().to_path_buf(); + builder.cli_version = Some("test_version".to_string()); + let mut metadata = builder.build(config.model_provider_id.as_str()); + metadata.title = "needle title".to_string(); + metadata.first_user_message = Some("plain preview".to_string()); + runtime + .upsert_thread(&metadata) + .await + .expect("state db upsert should succeed"); + + let page = store + .list_threads(ListThreadsParams { + page_size: 10, + cursor: None, + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived: false, + search_term: Some("needle".to_string()), + }) + .await + .expect("thread listing"); + + let ids = page + .items + .iter() + .map(|item| item.thread_id) + .collect::>(); + assert_eq!(ids, vec![thread_id]); + assert_eq!( + page.items[0].first_user_message.as_deref(), + Some("plain preview") + ); + } + + #[tokio::test] + async fn list_threads_selects_active_or_archived_collection() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path())); + let active_uuid = Uuid::from_u128(105); + let archived_uuid = Uuid::from_u128(106); + write_session_file(home.path(), "2025-01-03T12-00-00", active_uuid) + .expect("active session file"); + write_archived_session_file(home.path(), "2025-01-03T13-00-00", archived_uuid) + .expect("archived session file"); + + let active = store + .list_threads(ListThreadsParams { + page_size: 10, + cursor: None, + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived: false, + search_term: None, + }) + .await + .expect("active listing"); + let archived = store + .list_threads(ListThreadsParams { + page_size: 10, + cursor: None, + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived: true, + search_term: None, + }) + .await + .expect("archived listing"); + + let active_id = ThreadId::from_string(&active_uuid.to_string()).expect("valid thread id"); + let archived_id = + ThreadId::from_string(&archived_uuid.to_string()).expect("valid thread id"); + assert_eq!( + active + .items + .iter() + .map(|item| item.thread_id) + .collect::>(), + vec![active_id] + ); + assert_eq!( + archived + .items + .iter() + .map(|item| item.thread_id) + .collect::>(), + vec![archived_id] + ); + assert_eq!(active.items[0].archived_at, None); + assert_eq!( + archived.items[0].archived_at, + Some(archived.items[0].updated_at) + ); + } + + #[tokio::test] + async fn list_threads_returns_local_rollout_summary() { + let home = TempDir::new().expect("temp dir"); + let config = test_config(home.path()); + let store = LocalThreadStore::new(config); + let uuid = Uuid::from_u128(101); + let path = + write_session_file(home.path(), "2025-01-03T12-00-00", uuid).expect("session file"); + + let page = store + .list_threads(ListThreadsParams { + page_size: 10, + cursor: None, + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: vec![SessionSource::Cli], + model_providers: Some(vec!["test-provider".to_string()]), + archived: false, + search_term: None, + }) + .await + .expect("thread listing"); + + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + assert_eq!(page.next_cursor, None); + assert_eq!(page.items.len(), 1); + assert_eq!(page.items[0].thread_id, thread_id); + assert_eq!(page.items[0].rollout_path, Some(path)); + assert_eq!(page.items[0].preview, "Hello from user"); + assert_eq!( + page.items[0].first_user_message.as_deref(), + Some("Hello from user") + ); + assert_eq!(page.items[0].model_provider, "test-provider"); + assert_eq!(page.items[0].cli_version, "test_version"); + assert_eq!(page.items[0].source, SessionSource::Cli); + } + + #[tokio::test] + async fn list_threads_rejects_invalid_cursor() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path())); + + let err = store + .list_threads(ListThreadsParams { + page_size: 10, + cursor: Some("not-a-cursor".to_string()), + sort_key: ThreadSortKey::CreatedAt, + allowed_sources: Vec::new(), + model_providers: None, + archived: false, + search_term: None, + }) + .await + .expect_err("invalid cursor should fail"); + + assert!(matches!(err, ThreadStoreError::InvalidRequest { .. })); + } +} diff --git a/codex-rs/thread-store/src/local/mod.rs b/codex-rs/thread-store/src/local/mod.rs new file mode 100644 index 0000000000..81428cac8c --- /dev/null +++ b/codex-rs/thread-store/src/local/mod.rs @@ -0,0 +1,105 @@ +mod archive_thread; +mod helpers; +mod list_threads; +mod read_thread; +mod unarchive_thread; + +#[cfg(test)] +mod test_support; + +use async_trait::async_trait; +use codex_rollout::RolloutConfig; + +use crate::AppendThreadItemsParams; +use crate::ArchiveThreadParams; +use crate::CreateThreadParams; +use crate::ListThreadsParams; +use crate::LoadThreadHistoryParams; +use crate::ReadThreadParams; +use crate::ResumeThreadRecorderParams; +use crate::SetThreadNameParams; +use crate::StoredThread; +use crate::StoredThreadHistory; +use crate::ThreadPage; +use crate::ThreadRecorder; +use crate::ThreadStore; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; +use crate::UpdateThreadMetadataParams; + +/// Local filesystem/SQLite-backed implementation of [`ThreadStore`]. +#[derive(Clone, Debug)] +pub struct LocalThreadStore { + pub(super) config: RolloutConfig, +} + +impl LocalThreadStore { + /// Create a local store from the rollout configuration used by existing local persistence. + pub fn new(config: RolloutConfig) -> Self { + Self { config } + } +} + +#[async_trait] +impl ThreadStore for LocalThreadStore { + async fn create_thread( + &self, + _params: CreateThreadParams, + ) -> ThreadStoreResult> { + unsupported("create_thread") + } + + async fn resume_thread_recorder( + &self, + _params: ResumeThreadRecorderParams, + ) -> ThreadStoreResult> { + unsupported("resume_thread_recorder") + } + + async fn append_items(&self, _params: AppendThreadItemsParams) -> ThreadStoreResult<()> { + unsupported("append_items") + } + + async fn load_history( + &self, + _params: LoadThreadHistoryParams, + ) -> ThreadStoreResult { + unsupported("load_history") + } + + async fn read_thread(&self, params: ReadThreadParams) -> ThreadStoreResult { + read_thread::read_thread(self, params).await + } + + async fn list_threads(&self, params: ListThreadsParams) -> ThreadStoreResult { + list_threads::list_threads(self, params).await + } + + async fn set_thread_name(&self, _params: SetThreadNameParams) -> ThreadStoreResult<()> { + unsupported("set_thread_name") + } + + async fn update_thread_metadata( + &self, + _params: UpdateThreadMetadataParams, + ) -> ThreadStoreResult { + unsupported("update_thread_metadata") + } + + async fn archive_thread(&self, params: ArchiveThreadParams) -> ThreadStoreResult<()> { + archive_thread::archive_thread(self, params).await + } + + async fn unarchive_thread( + &self, + params: ArchiveThreadParams, + ) -> ThreadStoreResult { + unarchive_thread::unarchive_thread(self, params).await + } +} + +fn unsupported(operation: &str) -> ThreadStoreResult { + Err(ThreadStoreError::Internal { + message: format!("local thread store does not implement {operation} in this slice"), + }) +} diff --git a/codex-rs/thread-store/src/local/read_thread.rs b/codex-rs/thread-store/src/local/read_thread.rs new file mode 100644 index 0000000000..189bc1938d --- /dev/null +++ b/codex-rs/thread-store/src/local/read_thread.rs @@ -0,0 +1,138 @@ +use codex_rollout::RolloutRecorder; +use codex_rollout::find_archived_thread_path_by_id_str; +use codex_rollout::find_thread_path_by_id_str; +use codex_rollout::read_thread_item_from_rollout; + +use super::LocalThreadStore; +use super::helpers::stored_thread_from_rollout_item; +use crate::ReadThreadParams; +use crate::StoredThread; +use crate::StoredThreadHistory; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; + +pub(super) async fn read_thread( + store: &LocalThreadStore, + params: ReadThreadParams, +) -> ThreadStoreResult { + let thread_id = params.thread_id; + let path = if params.include_archived { + match find_thread_path_by_id_str(store.config.codex_home.as_path(), &thread_id.to_string()) + .await + .map_err(|err| ThreadStoreError::InvalidRequest { + message: format!("failed to locate thread id {thread_id}: {err}"), + })? { + Some(path) => Some(path), + None => find_archived_thread_path_by_id_str( + store.config.codex_home.as_path(), + &thread_id.to_string(), + ) + .await + .map_err(|err| ThreadStoreError::InvalidRequest { + message: format!("failed to locate archived thread id {thread_id}: {err}"), + })?, + } + } else { + find_thread_path_by_id_str(store.config.codex_home.as_path(), &thread_id.to_string()) + .await + .map_err(|err| ThreadStoreError::InvalidRequest { + message: format!("failed to locate thread id {thread_id}: {err}"), + })? + } + .ok_or_else(|| ThreadStoreError::InvalidRequest { + message: format!("no rollout found for thread id {thread_id}"), + })?; + + let item = read_thread_item_from_rollout(path.clone()) + .await + .ok_or_else(|| ThreadStoreError::Internal { + message: format!("failed to read thread {}", path.display()), + })?; + let archived = item.path.starts_with( + store + .config + .codex_home + .join(codex_rollout::ARCHIVED_SESSIONS_SUBDIR), + ); + let mut thread = + stored_thread_from_rollout_item(item, archived, store.config.model_provider_id.as_str()) + .ok_or_else(|| ThreadStoreError::Internal { + message: format!("failed to read thread id from {}", path.display()), + })?; + if params.include_history { + let (items, _, _) = RolloutRecorder::load_rollout_items(path.as_path()) + .await + .map_err(|err| ThreadStoreError::Internal { + message: format!("failed to load thread history {}: {err}", path.display()), + })?; + thread.history = Some(StoredThreadHistory { thread_id, items }); + } + Ok(thread) +} + +#[cfg(test)] +mod tests { + use codex_protocol::ThreadId; + use pretty_assertions::assert_eq; + use tempfile::TempDir; + use uuid::Uuid; + + use super::*; + use crate::ThreadStore; + use crate::local::LocalThreadStore; + use crate::local::test_support::test_config; + use crate::local::test_support::write_session_file; + + #[tokio::test] + async fn read_thread_returns_active_rollout_summary() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path())); + let uuid = Uuid::from_u128(205); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + let active_path = + write_session_file(home.path(), "2025-01-03T12-00-00", uuid).expect("session file"); + + let thread = store + .read_thread(ReadThreadParams { + thread_id, + include_archived: false, + include_history: true, + }) + .await + .expect("read thread"); + + assert_eq!(thread.thread_id, thread_id); + assert_eq!(thread.rollout_path, Some(active_path)); + assert_eq!(thread.archived_at, None); + assert_eq!(thread.preview, "Hello from user"); + assert_eq!( + thread.history.expect("history should load").thread_id, + thread_id + ); + } + + #[tokio::test] + async fn read_thread_fails_without_rollout() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path())); + let uuid = Uuid::from_u128(206); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + + let err = store + .read_thread(ReadThreadParams { + thread_id, + include_archived: false, + include_history: false, + }) + .await + .expect_err("read should fail without rollout"); + + let ThreadStoreError::InvalidRequest { message } = err else { + panic!("expected invalid request error"); + }; + assert_eq!( + message, + format!("no rollout found for thread id {thread_id}") + ); + } +} diff --git a/codex-rs/thread-store/src/local/test_support.rs b/codex-rs/thread-store/src/local/test_support.rs new file mode 100644 index 0000000000..d8ddcf8dde --- /dev/null +++ b/codex-rs/thread-store/src/local/test_support.rs @@ -0,0 +1,87 @@ +use std::fs; +use std::io::Write; +use std::path::Path; +use std::path::PathBuf; + +use codex_rollout::ARCHIVED_SESSIONS_SUBDIR; +use codex_rollout::RolloutConfig; +use uuid::Uuid; + +pub(super) fn test_config(codex_home: &Path) -> RolloutConfig { + RolloutConfig { + codex_home: codex_home.to_path_buf(), + sqlite_home: codex_home.to_path_buf(), + cwd: codex_home.to_path_buf(), + model_provider_id: "test-provider".to_string(), + generate_memories: true, + } +} + +pub(super) fn write_session_file(root: &Path, ts: &str, uuid: Uuid) -> std::io::Result { + write_session_file_with( + root, + root.join("sessions/2025/01/03"), + ts, + uuid, + "Hello from user", + Some("test-provider"), + ) +} + +pub(super) fn write_archived_session_file( + root: &Path, + ts: &str, + uuid: Uuid, +) -> std::io::Result { + write_session_file_with( + root, + root.join(ARCHIVED_SESSIONS_SUBDIR), + ts, + uuid, + "Archived user message", + Some("test-provider"), + ) +} + +pub(super) fn write_session_file_with( + root: &Path, + day_dir: PathBuf, + ts: &str, + uuid: Uuid, + first_user_message: &str, + model_provider: Option<&str>, +) -> std::io::Result { + fs::create_dir_all(&day_dir)?; + let path = day_dir.join(format!("rollout-{ts}-{uuid}.jsonl")); + let mut file = fs::File::create(&path)?; + let meta = serde_json::json!({ + "timestamp": ts, + "type": "session_meta", + "payload": { + "id": uuid, + "timestamp": ts, + "cwd": root, + "originator": "test_originator", + "cli_version": "test_version", + "source": "cli", + "model_provider": model_provider, + "git": { + "commit_hash": "abcdef", + "branch": "main", + "repository_url": "https://example.com/repo.git" + } + }, + }); + writeln!(file, "{meta}")?; + let user_event = serde_json::json!({ + "timestamp": ts, + "type": "event_msg", + "payload": { + "type": "user_message", + "message": first_user_message, + "kind": "plain", + }, + }); + writeln!(file, "{user_event}")?; + Ok(path) +} diff --git a/codex-rs/thread-store/src/local/unarchive_thread.rs b/codex-rs/thread-store/src/local/unarchive_thread.rs new file mode 100644 index 0000000000..17e484bace --- /dev/null +++ b/codex-rs/thread-store/src/local/unarchive_thread.rs @@ -0,0 +1,198 @@ +use codex_rollout::find_archived_thread_path_by_id_str; +use codex_rollout::read_thread_item_from_rollout; +use codex_rollout::rollout_date_parts; + +use super::LocalThreadStore; +use super::helpers::matching_rollout_file_name; +use super::helpers::scoped_rollout_path; +use super::helpers::stored_thread_from_rollout_item; +use super::helpers::touch_modified_time; +use crate::ArchiveThreadParams; +use crate::StoredThread; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; + +pub(super) async fn unarchive_thread( + store: &LocalThreadStore, + params: ArchiveThreadParams, +) -> ThreadStoreResult { + let thread_id = params.thread_id; + let archived_path = find_archived_thread_path_by_id_str( + store.config.codex_home.as_path(), + &thread_id.to_string(), + ) + .await + .map_err(|err| ThreadStoreError::InvalidRequest { + message: format!("failed to locate archived thread id {thread_id}: {err}"), + })? + .ok_or_else(|| ThreadStoreError::InvalidRequest { + message: format!("no archived rollout found for thread id {thread_id}"), + })?; + + let canonical_archived_path = scoped_rollout_path( + store + .config + .codex_home + .join(codex_rollout::ARCHIVED_SESSIONS_SUBDIR), + archived_path.as_path(), + "archived", + )?; + let file_name = matching_rollout_file_name( + canonical_archived_path.as_path(), + thread_id, + archived_path.as_path(), + )?; + let Some((year, month, day)) = rollout_date_parts(&file_name) else { + return Err(ThreadStoreError::InvalidRequest { + message: format!( + "rollout path `{}` missing filename timestamp", + archived_path.display() + ), + }); + }; + + let dest_dir = store + .config + .codex_home + .join(codex_rollout::SESSIONS_SUBDIR) + .join(year) + .join(month) + .join(day); + std::fs::create_dir_all(&dest_dir).map_err(|err| ThreadStoreError::Internal { + message: format!("failed to unarchive thread: {err}"), + })?; + let restored_path = dest_dir.join(&file_name); + std::fs::rename(&canonical_archived_path, &restored_path).map_err(|err| { + ThreadStoreError::Internal { + message: format!("failed to unarchive thread: {err}"), + } + })?; + touch_modified_time(restored_path.as_path()).map_err(|err| ThreadStoreError::Internal { + message: format!("failed to update unarchived thread timestamp: {err}"), + })?; + + if let Some(ctx) = codex_rollout::state_db::get_state_db(&store.config).await { + let _ = ctx + .mark_unarchived(thread_id, restored_path.as_path()) + .await; + } + + let item = read_thread_item_from_rollout(restored_path.clone()) + .await + .ok_or_else(|| ThreadStoreError::Internal { + message: format!( + "failed to read unarchived thread {}", + restored_path.display() + ), + })?; + stored_thread_from_rollout_item( + item, + /*archived*/ false, + store.config.model_provider_id.as_str(), + ) + .ok_or_else(|| ThreadStoreError::Internal { + message: format!( + "failed to read unarchived thread id from {}", + restored_path.display() + ), + }) +} + +#[cfg(test)] +mod tests { + use chrono::Utc; + use codex_protocol::ThreadId; + use codex_protocol::protocol::SessionSource; + use pretty_assertions::assert_eq; + use tempfile::TempDir; + use uuid::Uuid; + + use super::*; + use crate::ThreadStore; + use crate::local::LocalThreadStore; + use crate::local::test_support::test_config; + use crate::local::test_support::write_archived_session_file; + + #[tokio::test] + async fn unarchive_thread_restores_rollout_and_returns_updated_thread() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path())); + let uuid = Uuid::from_u128(203); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + let archived_path = write_archived_session_file(home.path(), "2025-01-03T13-00-00", uuid) + .expect("archived session file"); + + let thread = store + .unarchive_thread(ArchiveThreadParams { thread_id }) + .await + .expect("unarchive thread"); + + assert!(!archived_path.exists()); + let restored_path = home + .path() + .join("sessions/2025/01/03") + .join(archived_path.file_name().expect("file name")); + assert!(restored_path.exists()); + assert_eq!(thread.thread_id, thread_id); + assert_eq!(thread.rollout_path, Some(restored_path)); + assert_eq!(thread.archived_at, None); + assert_eq!(thread.preview, "Archived user message"); + assert_eq!( + thread.first_user_message.as_deref(), + Some("Archived user message") + ); + } + + #[tokio::test] + async fn unarchive_thread_updates_sqlite_metadata_when_present() { + let home = TempDir::new().expect("temp dir"); + let config = test_config(home.path()); + let store = LocalThreadStore::new(config.clone()); + let uuid = Uuid::from_u128(204); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + let archived_path = write_archived_session_file(home.path(), "2025-01-03T13-00-00", uuid) + .expect("archived session file"); + let runtime = codex_state::StateRuntime::init( + home.path().to_path_buf(), + config.model_provider_id.clone(), + ) + .await + .expect("state db should initialize"); + runtime + .mark_backfill_complete(/*last_watermark*/ None) + .await + .expect("backfill should be complete"); + let mut builder = codex_state::ThreadMetadataBuilder::new( + thread_id, + archived_path.clone(), + Utc::now(), + SessionSource::Cli, + ); + builder.model_provider = Some(config.model_provider_id.clone()); + builder.cwd = home.path().to_path_buf(); + builder.cli_version = Some("test_version".to_string()); + let mut metadata = builder.build(config.model_provider_id.as_str()); + metadata.archived_at = Some(metadata.updated_at); + runtime + .upsert_thread(&metadata) + .await + .expect("state db upsert should succeed"); + + store + .unarchive_thread(ArchiveThreadParams { thread_id }) + .await + .expect("unarchive thread"); + + let restored_path = home + .path() + .join("sessions/2025/01/03") + .join(archived_path.file_name().expect("file name")); + let updated = runtime + .get_thread(thread_id) + .await + .expect("state db read should succeed") + .expect("thread metadata should exist"); + assert_eq!(updated.rollout_path, restored_path); + assert_eq!(updated.archived_at, None); + } +} diff --git a/codex-rs/thread-store/src/types.rs b/codex-rs/thread-store/src/types.rs index 73f983a466..8d489ea6cb 100644 --- a/codex-rs/thread-store/src/types.rs +++ b/codex-rs/thread-store/src/types.rs @@ -135,6 +135,8 @@ pub struct ThreadPage { pub struct StoredThread { /// Thread id. pub thread_id: ThreadId, + /// Local rollout path when the backing store is filesystem-based. + pub rollout_path: Option, /// Source thread id when this thread was forked from another thread. pub forked_from_id: Option, /// Best available user-facing preview, usually the first user message. @@ -155,6 +157,8 @@ pub struct StoredThread { pub archived_at: Option>, /// Working directory captured for the thread. pub cwd: PathBuf, + /// CLI version captured for the thread. + pub cli_version: String, /// Runtime source for the thread. pub source: SessionSource, /// Optional random nickname for thread-spawn sub-agents. diff --git a/codex-rs/tools/src/code_mode.rs b/codex-rs/tools/src/code_mode.rs index 35d95b271a..e45c7be1d9 100644 --- a/codex-rs/tools/src/code_mode.rs +++ b/codex-rs/tools/src/code_mode.rs @@ -1,7 +1,9 @@ use crate::FreeformTool; use crate::FreeformToolFormat; use crate::JsonSchema; +use crate::ResponsesApiNamespaceTool; use crate::ResponsesApiTool; +use crate::ToolName; use crate::ToolSpec; use codex_code_mode::CodeModeToolKind; use codex_code_mode::ToolDefinition as CodeModeToolDefinition; @@ -9,22 +11,46 @@ use std::collections::BTreeMap; /// Augment tool descriptions with code-mode-specific exec samples. pub fn augment_tool_spec_for_code_mode(spec: ToolSpec) -> ToolSpec { - let Some(description) = code_mode_tool_definition_for_spec(&spec) - .map(codex_code_mode::augment_tool_definition) - .map(|definition| definition.description) - else { - return spec; - }; - match spec { ToolSpec::Function(mut tool) => { + let Some(description) = + augmented_description_for_spec(&ToolSpec::Function(tool.clone())) + else { + return ToolSpec::Function(tool); + }; tool.description = description; ToolSpec::Function(tool) } ToolSpec::Freeform(mut tool) => { + let Some(description) = + augmented_description_for_spec(&ToolSpec::Freeform(tool.clone())) + else { + return ToolSpec::Freeform(tool); + }; tool.description = description; ToolSpec::Freeform(tool) } + ToolSpec::Namespace(mut namespace) => { + for tool in &mut namespace.tools { + match tool { + ResponsesApiNamespaceTool::Function(tool) => { + let tool_name = + ToolName::namespaced(namespace.name.clone(), tool.name.clone()); + let definition = CodeModeToolDefinition { + name: tool_name.display(), + tool_name, + description: tool.description.clone(), + kind: CodeModeToolKind::Function, + input_schema: serde_json::to_value(&tool.parameters).ok(), + output_schema: tool.output_schema.clone(), + }; + tool.description = + codex_code_mode::augment_tool_definition(definition).description; + } + } + } + ToolSpec::Namespace(namespace) + } other => other, } } @@ -42,7 +68,9 @@ pub fn collect_code_mode_tool_definitions<'a>( ) -> Vec { let mut tool_definitions = specs .into_iter() - .filter_map(tool_spec_to_code_mode_tool_definition) + .flat_map(code_mode_tool_definitions_for_spec) + .filter(|definition| codex_code_mode::is_code_mode_nested_tool(&definition.name)) + .map(codex_code_mode::augment_tool_definition) .collect::>(); tool_definitions.sort_by(|left, right| left.name.cmp(&right.name)); tool_definitions.dedup_by(|left, right| left.name == right.name); @@ -54,7 +82,7 @@ pub fn collect_code_mode_exec_prompt_tool_definitions<'a>( ) -> Vec { let mut tool_definitions = specs .into_iter() - .filter_map(code_mode_tool_definition_for_spec) + .flat_map(code_mode_tool_definitions_for_spec) .filter(|definition| codex_code_mode::is_code_mode_nested_tool(&definition.name)) .collect::>(); tool_definitions.sort_by(|left, right| left.name.cmp(&right.name)); @@ -137,26 +165,61 @@ SOURCE: /[\s\S]+/ }) } +fn augmented_description_for_spec(spec: &ToolSpec) -> Option { + code_mode_tool_definition_for_spec(spec) + .map(codex_code_mode::augment_tool_definition) + .map(|definition| definition.description) +} + fn code_mode_tool_definition_for_spec(spec: &ToolSpec) -> Option { + code_mode_tool_definitions_for_spec(spec).into_iter().next() +} + +fn code_mode_tool_definitions_for_spec(spec: &ToolSpec) -> Vec { match spec { - ToolSpec::Function(tool) => Some(CodeModeToolDefinition { - name: tool.name.clone(), - description: tool.description.clone(), - kind: CodeModeToolKind::Function, - input_schema: serde_json::to_value(&tool.parameters).ok(), - output_schema: tool.output_schema.clone(), - }), - ToolSpec::Freeform(tool) => Some(CodeModeToolDefinition { - name: tool.name.clone(), - description: tool.description.clone(), - kind: CodeModeToolKind::Freeform, - input_schema: None, - output_schema: None, - }), + ToolSpec::Function(tool) => { + let name = tool.name.clone(); + vec![CodeModeToolDefinition { + tool_name: ToolName::plain(name.clone()), + name, + description: tool.description.clone(), + kind: CodeModeToolKind::Function, + input_schema: serde_json::to_value(&tool.parameters).ok(), + output_schema: tool.output_schema.clone(), + }] + } + ToolSpec::Freeform(tool) => { + let name = tool.name.clone(); + vec![CodeModeToolDefinition { + tool_name: ToolName::plain(name.clone()), + name, + description: tool.description.clone(), + kind: CodeModeToolKind::Freeform, + input_schema: None, + output_schema: None, + }] + } + ToolSpec::Namespace(namespace) => namespace + .tools + .iter() + .map(|tool| match tool { + ResponsesApiNamespaceTool::Function(tool) => { + let tool_name = ToolName::namespaced(namespace.name.clone(), tool.name.clone()); + CodeModeToolDefinition { + name: tool_name.display(), + tool_name, + description: tool.description.clone(), + kind: CodeModeToolKind::Function, + input_schema: serde_json::to_value(&tool.parameters).ok(), + output_schema: tool.output_schema.clone(), + } + } + }) + .collect(), ToolSpec::LocalShell {} | ToolSpec::ImageGeneration { .. } | ToolSpec::ToolSearch { .. } - | ToolSpec::WebSearch { .. } => None, + | ToolSpec::WebSearch { .. } => Vec::new(), } } diff --git a/codex-rs/tools/src/code_mode_tests.rs b/codex-rs/tools/src/code_mode_tests.rs index ec30c6e8bf..15cca164ec 100644 --- a/codex-rs/tools/src/code_mode_tests.rs +++ b/codex-rs/tools/src/code_mode_tests.rs @@ -7,6 +7,7 @@ use crate::FreeformTool; use crate::FreeformToolFormat; use crate::JsonSchema; use crate::ResponsesApiTool; +use crate::ToolName; use crate::ToolSpec; use pretty_assertions::assert_eq; use serde_json::json; @@ -106,6 +107,7 @@ fn tool_spec_to_code_mode_tool_definition_returns_augmented_nested_tools() { tool_spec_to_code_mode_tool_definition(&spec), Some(codex_code_mode::ToolDefinition { name: "apply_patch".to_string(), + tool_name: ToolName::plain("apply_patch"), description: r#"Apply a patch exec tool declaration: @@ -184,6 +186,7 @@ fn create_wait_tool_matches_expected_spec() { fn create_code_mode_tool_matches_expected_spec() { let enabled_tools = vec![codex_code_mode::ToolDefinition { name: "update_plan".to_string(), + tool_name: ToolName::plain("update_plan"), description: "Update the plan".to_string(), kind: codex_code_mode::CodeModeToolKind::Function, input_schema: None, diff --git a/codex-rs/tools/src/lib.rs b/codex-rs/tools/src/lib.rs index c310222a0f..4a3ccc01ae 100644 --- a/codex-rs/tools/src/lib.rs +++ b/codex-rs/tools/src/lib.rs @@ -18,7 +18,6 @@ mod responses_api; mod tool_config; mod tool_definition; mod tool_discovery; -mod tool_name; mod tool_registry_plan; mod tool_registry_plan_types; mod tool_spec; @@ -50,6 +49,7 @@ pub use code_mode::collect_code_mode_tool_definitions; pub use code_mode::create_code_mode_tool; pub use code_mode::create_wait_tool; pub use code_mode::tool_spec_to_code_mode_tool_definition; +pub use codex_protocol::ToolName; pub use dynamic_tool::parse_dynamic_tool; pub use image_detail::can_request_original_image_detail; pub use image_detail::normalize_output_image_detail; @@ -85,6 +85,7 @@ pub use responses_api::ResponsesApiNamespace; pub use responses_api::ResponsesApiNamespaceTool; pub use responses_api::ResponsesApiTool; pub use responses_api::ToolSearchOutputTool; +pub(crate) use responses_api::default_namespace_description; pub use responses_api::dynamic_tool_to_responses_api_tool; pub use responses_api::mcp_tool_to_deferred_responses_api_tool; pub use responses_api::mcp_tool_to_responses_api_tool; @@ -113,13 +114,13 @@ pub use tool_discovery::collect_tool_suggest_entries; pub use tool_discovery::create_tool_search_tool; pub use tool_discovery::create_tool_suggest_tool; pub use tool_discovery::filter_tool_suggest_discoverable_tools_for_client; -pub use tool_name::ToolName; pub use tool_registry_plan::build_tool_registry_plan; pub use tool_registry_plan_types::ToolHandlerKind; pub use tool_registry_plan_types::ToolHandlerSpec; pub use tool_registry_plan_types::ToolNamespace; pub use tool_registry_plan_types::ToolRegistryPlan; pub use tool_registry_plan_types::ToolRegistryPlanDeferredTool; +pub use tool_registry_plan_types::ToolRegistryPlanMcpTool; pub use tool_registry_plan_types::ToolRegistryPlanParams; pub use tool_spec::ConfiguredToolSpec; pub use tool_spec::ResponsesApiWebSearchFilters; diff --git a/codex-rs/tools/src/responses_api.rs b/codex-rs/tools/src/responses_api.rs index 50bb4f2232..5317a082f1 100644 --- a/codex-rs/tools/src/responses_api.rs +++ b/codex-rs/tools/src/responses_api.rs @@ -1,5 +1,6 @@ use crate::JsonSchema; use crate::ToolDefinition; +use crate::ToolName; use crate::parse_dynamic_tool; use crate::parse_mcp_tool; use codex_protocol::dynamic_tools::DynamicToolSpec; @@ -54,6 +55,10 @@ pub struct ResponsesApiNamespace { pub tools: Vec, } +pub(crate) fn default_namespace_description(namespace_name: &str) -> String { + format!("Tools in the {namespace_name} namespace.") +} + #[derive(Debug, Clone, Serialize, PartialEq)] #[serde(tag = "type")] pub enum ResponsesApiNamespaceTool { @@ -70,20 +75,22 @@ pub fn dynamic_tool_to_responses_api_tool( } pub fn mcp_tool_to_responses_api_tool( - name: String, + tool_name: &ToolName, tool: &rmcp::model::Tool, ) -> Result { Ok(tool_definition_to_responses_api_tool( - parse_mcp_tool(tool)?.renamed(name), + parse_mcp_tool(tool)?.renamed(tool_name.name.clone()), )) } pub fn mcp_tool_to_deferred_responses_api_tool( - name: String, + tool_name: &ToolName, tool: &rmcp::model::Tool, ) -> Result { Ok(tool_definition_to_responses_api_tool( - parse_mcp_tool(tool)?.renamed(name).into_deferred(), + parse_mcp_tool(tool)? + .renamed(tool_name.name.clone()) + .into_deferred(), )) } diff --git a/codex-rs/tools/src/responses_api_tests.rs b/codex-rs/tools/src/responses_api_tests.rs index c3ce4cec2b..799a2385d2 100644 --- a/codex-rs/tools/src/responses_api_tests.rs +++ b/codex-rs/tools/src/responses_api_tests.rs @@ -7,6 +7,7 @@ use super::mcp_tool_to_deferred_responses_api_tool; use super::tool_definition_to_responses_api_tool; use crate::JsonSchema; use crate::ToolDefinition; +use crate::ToolName; use codex_protocol::dynamic_tools::DynamicToolSpec; use pretty_assertions::assert_eq; use serde_json::json; @@ -106,19 +107,23 @@ fn mcp_tool_to_deferred_responses_api_tool_sets_defer_loading() { assert_eq!( mcp_tool_to_deferred_responses_api_tool( - "mcp__codex_apps__lookup_order".to_string(), + &ToolName::namespaced("mcp__codex_apps__", "lookup_order"), &tool, ) .expect("convert deferred tool"), ResponsesApiTool { - name: "mcp__codex_apps__lookup_order".to_string(), + name: "lookup_order".to_string(), description: "Look up an order".to_string(), strict: false, defer_loading: Some(true), - parameters: JsonSchema::object(BTreeMap::from([( + parameters: JsonSchema::object( + BTreeMap::from([( "order_id".to_string(), JsonSchema::string(/*description*/ None), - )]), Some(vec!["order_id".to_string()]), Some(false.into())), + )]), + Some(vec!["order_id".to_string()]), + Some(false.into()) + ), output_schema: None, } ); diff --git a/codex-rs/tools/src/tool_discovery.rs b/codex-rs/tools/src/tool_discovery.rs index f5d84a952e..b773623edf 100644 --- a/codex-rs/tools/src/tool_discovery.rs +++ b/codex-rs/tools/src/tool_discovery.rs @@ -2,8 +2,10 @@ use crate::JsonSchema; use crate::ResponsesApiNamespace; use crate::ResponsesApiNamespaceTool; use crate::ResponsesApiTool; +use crate::ToolName; use crate::ToolSearchOutputTool; use crate::ToolSpec; +use crate::default_namespace_description; use crate::mcp_tool_to_deferred_responses_api_tool; use codex_app_server_protocol::AppInfo; use serde::Deserialize; @@ -224,6 +226,8 @@ pub fn collect_tool_search_output_tools<'a>( let description = first_tool .connector_description + .map(str::trim) + .filter(|description| !description.is_empty()) .map(str::to_string) .or_else(|| { first_tool @@ -231,25 +235,21 @@ pub fn collect_tool_search_output_tools<'a>( .map(str::trim) .filter(|connector_name| !connector_name.is_empty()) .map(|connector_name| format!("Tools for working with {connector_name}.")) - }) - .or_else(|| { - Some(format!( - "Tools from the {} MCP server.", - first_tool.server_name - )) }); let tools = tools .iter() .map(|tool| { - mcp_tool_to_deferred_responses_api_tool(tool.tool_name.to_string(), tool.tool) + let tool_name = ToolName::namespaced(tool.tool_namespace, tool.tool_name); + mcp_tool_to_deferred_responses_api_tool(&tool_name, tool.tool) .map(ResponsesApiNamespaceTool::Function) }) .collect::, _>>()?; results.push(ToolSearchOutputTool::Namespace(ResponsesApiNamespace { name: tool_namespace.to_string(), - description: description.unwrap_or_default(), + description: description + .unwrap_or_else(|| default_namespace_description(tool_namespace)), tools, })); } diff --git a/codex-rs/tools/src/tool_discovery_tests.rs b/codex-rs/tools/src/tool_discovery_tests.rs index 10e108fe04..b745f756cc 100644 --- a/codex-rs/tools/src/tool_discovery_tests.rs +++ b/codex-rs/tools/src/tool_discovery_tests.rs @@ -253,7 +253,7 @@ fn collect_tool_search_output_tools_preserves_search_order_while_grouping_by_nam }), ToolSearchOutputTool::Namespace(ResponsesApiNamespace { name: "mcp__docs__".to_string(), - description: "Tools from the docs MCP server.".to_string(), + description: "Tools in the mcp__docs__ namespace.".to_string(), tools: vec![ResponsesApiNamespaceTool::Function(ResponsesApiTool { name: "search".to_string(), description: "Search docs.".to_string(), @@ -272,7 +272,7 @@ fn collect_tool_search_output_tools_preserves_search_order_while_grouping_by_nam } #[test] -fn collect_tool_search_output_tools_falls_back_to_connector_name_description() { +fn collect_tool_search_output_tools_ignores_blank_connector_description() { let gmail_batch_read_email = mcp_tool("gmail-batch-read-email", "Read multiple emails."); let tools = collect_tool_search_output_tools([ToolSearchResultSource { @@ -281,7 +281,7 @@ fn collect_tool_search_output_tools_falls_back_to_connector_name_description() { tool_name: "_batch_read_email", tool: &gmail_batch_read_email, connector_name: Some("Gmail"), - connector_description: None, + connector_description: Some(" "), }]) .expect("collect tool search output tools"); diff --git a/codex-rs/tools/src/tool_registry_plan.rs b/codex-rs/tools/src/tool_registry_plan.rs index 82b4c3a142..b5da9afb25 100644 --- a/codex-rs/tools/src/tool_registry_plan.rs +++ b/codex-rs/tools/src/tool_registry_plan.rs @@ -1,12 +1,13 @@ use crate::CommandToolOptions; use crate::REQUEST_USER_INPUT_TOOL_NAME; +use crate::ResponsesApiNamespace; +use crate::ResponsesApiNamespaceTool; use crate::ShellToolOptions; use crate::SpawnAgentToolOptions; use crate::TOOL_SEARCH_DEFAULT_LIMIT; use crate::TOOL_SEARCH_TOOL_NAME; use crate::TOOL_SUGGEST_TOOL_NAME; use crate::ToolHandlerKind; -use crate::ToolName; use crate::ToolRegistryPlan; use crate::ToolRegistryPlanParams; use crate::ToolSearchSource; @@ -54,6 +55,7 @@ use crate::create_wait_agent_tool_v2; use crate::create_wait_tool; use crate::create_web_search_tool; use crate::create_write_stdin_tool; +use crate::default_namespace_description; use crate::dynamic_tool_to_responses_api_tool; use crate::mcp_tool_to_responses_api_tool; use crate::request_permissions_tool_description; @@ -61,7 +63,6 @@ use crate::request_user_input_tool_description; use crate::tool_registry_plan_types::agent_type_description; use codex_protocol::openai_models::ApplyPatchToolType; use codex_protocol::openai_models::ConfigShellToolType; -use rmcp::model::Tool as McpTool; use std::collections::BTreeMap; pub fn build_tool_registry_plan( @@ -76,9 +77,9 @@ pub fn build_tool_registry_plan( .tool_namespaces .into_iter() .flatten() - .map(|(name, detail)| { + .map(|(namespace, detail)| { ( - name.clone(), + namespace.clone(), codex_code_mode::ToolNamespaceDescription { name: detail.name.clone(), description: detail.description.clone().unwrap_or_default(), @@ -100,9 +101,8 @@ pub fn build_tool_registry_plan( .iter() .map(|configured_tool| &configured_tool.spec), ); - enabled_tools.sort_by(|left, right| { - compare_code_mode_tool_names(&left.name, &right.name, &namespace_descriptions) - }); + enabled_tools + .sort_by(|left, right| compare_code_mode_tools(left, right, &namespace_descriptions)); plan.push_spec( create_code_mode_tool( &enabled_tools, @@ -266,10 +266,7 @@ pub fn build_tool_registry_plan( plan.register_handler(TOOL_SEARCH_TOOL_NAME, ToolHandlerKind::ToolSearch); for tool in deferred_mcp_tools { - plan.register_handler( - ToolName::namespaced(tool.tool_namespace, tool.tool_name), - ToolHandlerKind::Mcp, - ); + plan.register_handler(tool.name.clone(), ToolHandlerKind::Mcp); } } @@ -471,28 +468,65 @@ pub fn build_tool_registry_plan( } if let Some(mcp_tools) = params.mcp_tools { - let mut entries: Vec<(String, &McpTool)> = mcp_tools - .iter() - .map(|(name, tool)| (name.clone(), tool)) - .collect(); - entries.sort_by(|left, right| left.0.cmp(&right.0)); + let mut entries = mcp_tools.to_vec(); + entries.sort_by_key(|tool| tool.name.display()); + let mut namespace_entries = BTreeMap::new(); - for (name, tool) in entries { - match mcp_tool_to_responses_api_tool(name.clone(), tool) { - Ok(converted_tool) => { - plan.push_spec( - ToolSpec::Function(converted_tool), - /*supports_parallel_tool_calls*/ false, - config.code_mode_enabled, - ); - plan.register_handler(name, ToolHandlerKind::Mcp); - } - Err(error) => { - tracing::error!( - "Failed to convert {name:?} MCP tool to OpenAI tool: {error:?}" - ); + for tool in entries { + let Some(namespace) = tool.name.namespace.as_ref() else { + let tool_name = &tool.name; + tracing::error!("Skipping MCP tool `{tool_name}`: MCP tools must be namespaced"); + continue; + }; + namespace_entries + .entry(namespace.clone()) + .or_insert_with(Vec::new) + .push(tool); + } + + for (namespace, mut entries) in namespace_entries { + entries.sort_by_key(|tool| tool.name.name.clone()); + let tool_namespace = params + .tool_namespaces + .and_then(|namespaces| namespaces.get(&namespace)); + let description = tool_namespace + .and_then(|namespace| namespace.description.as_deref()) + .map(str::trim) + .filter(|description| !description.is_empty()) + .map(str::to_string) + .unwrap_or_else(|| { + let namespace_name = tool_namespace + .map(|namespace| namespace.name.as_str()) + .unwrap_or(namespace.as_str()); + default_namespace_description(namespace_name) + }); + let mut tools = Vec::new(); + for tool in entries { + match mcp_tool_to_responses_api_tool(&tool.name, tool.tool) { + Ok(converted_tool) => { + tools.push(ResponsesApiNamespaceTool::Function(converted_tool)); + plan.register_handler(tool.name, ToolHandlerKind::Mcp); + } + Err(error) => { + let tool_name = &tool.name; + tracing::error!( + "Failed to convert `{tool_name}` MCP tool to OpenAI tool: {error:?}" + ); + } } } + + if !tools.is_empty() { + plan.push_spec( + ToolSpec::Namespace(ResponsesApiNamespace { + name: namespace, + description, + tools, + }), + /*supports_parallel_tool_calls*/ false, + config.code_mode_enabled, + ); + } } } @@ -518,41 +552,31 @@ pub fn build_tool_registry_plan( plan } -fn compare_code_mode_tool_names( - left_name: &str, - right_name: &str, +fn compare_code_mode_tools( + left: &codex_code_mode::ToolDefinition, + right: &codex_code_mode::ToolDefinition, namespace_descriptions: &BTreeMap, ) -> std::cmp::Ordering { - let left_namespace = code_mode_namespace_name(left_name, namespace_descriptions); - let right_namespace = code_mode_namespace_name(right_name, namespace_descriptions); + let left_namespace = code_mode_namespace_name(left, namespace_descriptions); + let right_namespace = code_mode_namespace_name(right, namespace_descriptions); left_namespace .cmp(&right_namespace) - .then_with(|| { - code_mode_function_name(left_name, left_namespace) - .cmp(code_mode_function_name(right_name, right_namespace)) - }) - .then_with(|| left_name.cmp(right_name)) + .then_with(|| left.tool_name.name.cmp(&right.tool_name.name)) + .then_with(|| left.name.cmp(&right.name)) } fn code_mode_namespace_name<'a>( - name: &str, + tool: &codex_code_mode::ToolDefinition, namespace_descriptions: &'a BTreeMap, ) -> Option<&'a str> { - namespace_descriptions - .get(name) + tool.tool_name + .namespace + .as_ref() + .and_then(|namespace| namespace_descriptions.get(namespace)) .map(|namespace_description| namespace_description.name.as_str()) } -fn code_mode_function_name<'a>(name: &'a str, namespace: Option<&str>) -> &'a str { - namespace - .and_then(|namespace| { - name.strip_prefix(namespace) - .and_then(|suffix| suffix.strip_prefix("__")) - }) - .unwrap_or(name) -} - #[cfg(test)] #[path = "tool_registry_plan_tests.rs"] mod tests; diff --git a/codex-rs/tools/src/tool_registry_plan_tests.rs b/codex-rs/tools/src/tool_registry_plan_tests.rs index 69b8b55b54..a643543ee7 100644 --- a/codex-rs/tools/src/tool_registry_plan_tests.rs +++ b/codex-rs/tools/src/tool_registry_plan_tests.rs @@ -7,12 +7,15 @@ use crate::FreeformTool; use crate::JsonSchema; use crate::JsonSchemaPrimitiveType; use crate::JsonSchemaType; +use crate::ResponsesApiNamespaceTool; use crate::ResponsesApiTool; use crate::ResponsesApiWebSearchFilters; use crate::ResponsesApiWebSearchUserLocation; use crate::ToolHandlerSpec; +use crate::ToolName; use crate::ToolNamespace; use crate::ToolRegistryPlanDeferredTool; +use crate::ToolRegistryPlanMcpTool; use crate::ToolsConfigParams; use crate::WaitAgentTimeoutOptions; use crate::mcp_call_tool_result_output_schema; @@ -1075,7 +1078,7 @@ fn test_build_specs_mcp_tools_converted() { let (tools, _) = build_specs( &tools_config, Some(HashMap::from([( - "test_server/do_something_cool".to_string(), + ToolName::namespaced("test_server/", "do_something_cool"), mcp_tool( "do_something_cool", "Do something cool", @@ -1101,11 +1104,11 @@ fn test_build_specs_mcp_tools_converted() { &[], ); - let tool = find_tool(&tools, "test_server/do_something_cool"); + let tool = find_namespace_function_tool(&tools, "test_server/", "do_something_cool"); assert_eq!( - &tool.spec, - &ToolSpec::Function(ResponsesApiTool { - name: "test_server/do_something_cool".to_string(), + tool, + &ResponsesApiTool { + name: "do_something_cool".to_string(), parameters: JsonSchema::object( BTreeMap::from([ ( @@ -1144,7 +1147,47 @@ fn test_build_specs_mcp_tools_converted() { strict: false, output_schema: Some(mcp_call_tool_result_output_schema(serde_json::json!({}))), defer_loading: None, - }) + } + ); +} + +#[test] +fn test_build_specs_mcp_namespace_description_falls_back_when_missing() { + let model_info = model_info(); + let mut features = Features::with_defaults(); + features.enable(Feature::UnifiedExec); + let available_models = Vec::new(); + let tools_config = ToolsConfig::new(&ToolsConfigParams { + model_info: &model_info, + available_models: &available_models, + features: &features, + image_generation_tool_auth_allowed: true, + web_search_mode: Some(WebSearchMode::Cached), + session_source: SessionSource::Cli, + sandbox_policy: &SandboxPolicy::DangerFullAccess, + windows_sandbox_level: WindowsSandboxLevel::Disabled, + }); + let (tools, _) = build_specs( + &tools_config, + Some(HashMap::from([( + ToolName::namespaced("test_server/", "do_something_cool"), + mcp_tool( + "do_something_cool", + "Do something cool", + serde_json::json!({"type": "object"}), + ), + )])), + /*deferred_mcp_tools*/ None, + &[], + ); + + let namespace_tool = find_tool(&tools, "test_server/"); + let ToolSpec::Namespace(namespace) = &namespace_tool.spec else { + panic!("expected namespace tool"); + }; + assert_eq!( + namespace.description, + "Tools in the test_server/ namespace." ); } @@ -1167,16 +1210,16 @@ fn test_build_specs_mcp_tools_sorted_by_name() { let tools_map = HashMap::from([ ( - "test_server/do".to_string(), - mcp_tool("a", "a", serde_json::json!({"type": "object"})), + ToolName::namespaced("test_server/", "do"), + mcp_tool("do", "a", serde_json::json!({"type": "object"})), ), ( - "test_server/something".to_string(), - mcp_tool("b", "b", serde_json::json!({"type": "object"})), + ToolName::namespaced("test_server/", "something"), + mcp_tool("something", "b", serde_json::json!({"type": "object"})), ), ( - "test_server/cool".to_string(), - mcp_tool("c", "c", serde_json::json!({"type": "object"})), + ToolName::namespaced("test_server/", "cool"), + mcp_tool("cool", "c", serde_json::json!({"type": "object"})), ), ]); @@ -1187,17 +1230,14 @@ fn test_build_specs_mcp_tools_sorted_by_name() { &[], ); - let mcp_names: Vec<_> = tools - .iter() - .map(|tool| tool.name().to_string()) - .filter(|name| name.starts_with("test_server/")) - .collect(); - let expected = vec![ - "test_server/cool".to_string(), - "test_server/do".to_string(), - "test_server/something".to_string(), - ]; - assert_eq!(mcp_names, expected); + assert_eq!( + namespace_function_names(&tools, "test_server/"), + vec![ + "cool".to_string(), + "do".to_string(), + "something".to_string(), + ] + ); } #[test] @@ -1222,7 +1262,7 @@ fn search_tool_description_lists_each_mcp_source_once() { &tools_config, Some(HashMap::from([ ( - "mcp__codex_apps__calendar_create_event".to_string(), + ToolName::namespaced("mcp__codex_apps__calendar", "_create_event"), mcp_tool( "calendar_create_event", "Create calendar event", @@ -1230,7 +1270,7 @@ fn search_tool_description_lists_each_mcp_source_once() { ), ), ( - "mcp__rmcp__echo".to_string(), + ToolName::namespaced("mcp__rmcp__", "echo"), mcp_tool("echo", "Echo", serde_json::json!({"type": "object"})), ), ])), @@ -1577,7 +1617,7 @@ fn code_mode_augments_mcp_tool_descriptions_with_namespaced_sample() { let (tools, _) = build_specs( &tools_config, Some(HashMap::from([( - "mcp__sample__echo".to_string(), + ToolName::namespaced("mcp__sample__", "echo"), mcp_tool( "echo", "Echo text", @@ -1595,11 +1635,8 @@ fn code_mode_augments_mcp_tool_descriptions_with_namespaced_sample() { &[], ); - let ToolSpec::Function(ResponsesApiTool { description, .. }) = - &find_tool(&tools, "mcp__sample__echo").spec - else { - panic!("expected function tool"); - }; + let ResponsesApiTool { description, .. } = + find_namespace_function_tool(&tools, "mcp__sample__", "echo"); assert_eq!( description, @@ -1633,7 +1670,7 @@ fn code_mode_preserves_nullable_and_literal_mcp_input_shapes() { let (tools, _) = build_specs( &tools_config, Some(HashMap::from([( - "mcp__sample__fn".to_string(), + ToolName::namespaced("mcp__sample__", "fn"), mcp_tool( "fn", "Sample fn", @@ -1684,11 +1721,8 @@ fn code_mode_preserves_nullable_and_literal_mcp_input_shapes() { &[], ); - let ToolSpec::Function(ResponsesApiTool { description, .. }) = - &find_tool(&tools, "mcp__sample__fn").spec - else { - panic!("expected function tool"); - }; + let ResponsesApiTool { description, .. } = + find_namespace_function_tool(&tools, "mcp__sample__", "fn"); assert!(description.contains( r#"exec tool declaration: @@ -1851,7 +1885,7 @@ fn search_capable_model_info() -> ModelInfo { fn build_specs<'a>( config: &ToolsConfig, - mcp_tools: Option>, + mcp_tools: Option>, deferred_mcp_tools: Option>>, dynamic_tools: &[DynamicToolSpec], ) -> (Vec, Vec) { @@ -1866,7 +1900,7 @@ fn build_specs<'a>( fn build_specs_with_discoverable_tools<'a>( config: &ToolsConfig, - mcp_tools: Option>, + mcp_tools: Option>, deferred_mcp_tools: Option>>, discoverable_tools: Option>, dynamic_tools: &[DynamicToolSpec], @@ -1883,16 +1917,25 @@ fn build_specs_with_discoverable_tools<'a>( fn build_specs_with_optional_tool_namespaces<'a>( config: &ToolsConfig, - mcp_tools: Option>, + mcp_tools: Option>, deferred_mcp_tools: Option>>, tool_namespaces: Option>, discoverable_tools: Option>, dynamic_tools: &[DynamicToolSpec], ) -> (Vec, Vec) { + let mcp_tool_inputs = mcp_tools.as_ref().map(|mcp_tools| { + mcp_tools + .iter() + .map(|(name, tool)| ToolRegistryPlanMcpTool { + name: name.clone(), + tool, + }) + .collect::>() + }); let plan = build_tool_registry_plan( config, ToolRegistryPlanParams { - mcp_tools: mcp_tools.as_ref(), + mcp_tools: mcp_tool_inputs.as_deref(), deferred_mcp_tools: deferred_mcp_tools.as_deref(), tool_namespaces: tool_namespaces.as_ref(), discoverable_tools: discoverable_tools.as_deref(), @@ -1968,16 +2011,16 @@ fn code_mode_augments_mcp_tool_descriptions_with_structured_output_sample() { let (tools, _) = build_specs( &tools_config, - Some(HashMap::from([("mcp__sample__echo".to_string(), tool)])), + Some(HashMap::from([( + ToolName::namespaced("mcp__sample__", "echo"), + tool, + )])), /*deferred_mcp_tools*/ None, &[], ); - let ToolSpec::Function(ResponsesApiTool { description, .. }) = - &find_tool(&tools, "mcp__sample__echo").spec - else { - panic!("expected function tool"); - }; + let ResponsesApiTool { description, .. } = + find_namespace_function_tool(&tools, "mcp__sample__", "echo"); assert_eq!( description, @@ -2017,8 +2060,7 @@ fn deferred_mcp_tool<'a>( connector_description: Option<&'a str>, ) -> ToolRegistryPlanDeferredTool<'a> { ToolRegistryPlanDeferredTool { - tool_name, - tool_namespace, + name: ToolName::namespaced(tool_namespace, tool_name), server_name, connector_name, connector_description, @@ -2089,6 +2131,39 @@ fn find_tool<'a>(tools: &'a [ConfiguredToolSpec], expected_name: &str) -> &'a Co .unwrap_or_else(|| panic!("expected tool {expected_name}")) } +fn find_namespace_function_tool<'a>( + tools: &'a [ConfiguredToolSpec], + expected_namespace: &str, + expected_name: &str, +) -> &'a ResponsesApiTool { + let namespace_tool = find_tool(tools, expected_namespace); + let ToolSpec::Namespace(namespace) = &namespace_tool.spec else { + panic!("expected namespace tool {expected_namespace}"); + }; + namespace + .tools + .iter() + .find_map(|tool| match tool { + ResponsesApiNamespaceTool::Function(tool) if tool.name == expected_name => Some(tool), + _ => None, + }) + .unwrap_or_else(|| panic!("expected tool {expected_namespace}{expected_name} in namespace")) +} + +fn namespace_function_names(tools: &[ConfiguredToolSpec], expected_namespace: &str) -> Vec { + let namespace_tool = find_tool(tools, expected_namespace); + let ToolSpec::Namespace(namespace) = &namespace_tool.spec else { + panic!("expected namespace tool {expected_namespace}"); + }; + namespace + .tools + .iter() + .map(|tool| match tool { + ResponsesApiNamespaceTool::Function(tool) => tool.name.clone(), + }) + .collect() +} + fn expect_object_schema( schema: &JsonSchema, ) -> (&BTreeMap, Option<&Vec>) { @@ -2137,6 +2212,17 @@ fn strip_descriptions_tool(spec: &mut ToolSpec) { ToolSpec::Function(ResponsesApiTool { parameters, .. }) => { strip_descriptions_schema(parameters); } + ToolSpec::Namespace(namespace) => { + for tool in &mut namespace.tools { + match tool { + ResponsesApiNamespaceTool::Function(ResponsesApiTool { + parameters, .. + }) => { + strip_descriptions_schema(parameters); + } + } + } + } ToolSpec::Freeform(FreeformTool { .. }) | ToolSpec::LocalShell {} | ToolSpec::ImageGeneration { .. } diff --git a/codex-rs/tools/src/tool_registry_plan_types.rs b/codex-rs/tools/src/tool_registry_plan_types.rs index 7459954dcd..b9d66a0c2c 100644 --- a/codex-rs/tools/src/tool_registry_plan_types.rs +++ b/codex-rs/tools/src/tool_registry_plan_types.rs @@ -6,7 +6,6 @@ use crate::ToolsConfig; use crate::WaitAgentTimeoutOptions; use crate::augment_tool_spec_for_code_mode; use codex_protocol::dynamic_tools::DynamicToolSpec; -use rmcp::model::Tool as McpTool; use std::collections::HashMap; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -58,7 +57,7 @@ pub struct ToolRegistryPlan { #[derive(Debug, Clone, Copy)] pub struct ToolRegistryPlanParams<'a> { - pub mcp_tools: Option<&'a HashMap>, + pub mcp_tools: Option<&'a [ToolRegistryPlanMcpTool<'a>]>, pub deferred_mcp_tools: Option<&'a [ToolRegistryPlanDeferredTool<'a>]>, pub tool_namespaces: Option<&'a HashMap>, pub discoverable_tools: Option<&'a [DiscoverableTool]>, @@ -73,10 +72,18 @@ pub struct ToolNamespace { pub description: Option, } -#[derive(Debug, Clone, Copy)] +/// Direct MCP tool metadata needed to expose the Responses API namespace tool +/// while registering its runtime handler with the canonical namespace/name +/// identity. +#[derive(Debug, Clone)] +pub struct ToolRegistryPlanMcpTool<'a> { + pub name: ToolName, + pub tool: &'a rmcp::model::Tool, +} + +#[derive(Debug, Clone)] pub struct ToolRegistryPlanDeferredTool<'a> { - pub tool_name: &'a str, - pub tool_namespace: &'a str, + pub name: ToolName, pub server_name: &'a str, pub connector_name: Option<&'a str>, pub connector_description: Option<&'a str>, diff --git a/codex-rs/tools/src/tool_spec.rs b/codex-rs/tools/src/tool_spec.rs index 24644e2260..81db2f5584 100644 --- a/codex-rs/tools/src/tool_spec.rs +++ b/codex-rs/tools/src/tool_spec.rs @@ -1,5 +1,6 @@ use crate::FreeformTool; use crate::JsonSchema; +use crate::ResponsesApiNamespace; use crate::ResponsesApiTool; use codex_protocol::config_types::WebSearchConfig; use codex_protocol::config_types::WebSearchContextSize; @@ -20,6 +21,8 @@ const WEB_SEARCH_TEXT_AND_IMAGE_CONTENT_TYPES: [&str; 2] = ["text", "image"]; pub enum ToolSpec { #[serde(rename = "function")] Function(ResponsesApiTool), + #[serde(rename = "namespace")] + Namespace(ResponsesApiNamespace), #[serde(rename = "tool_search")] ToolSearch { execution: String, @@ -57,6 +60,7 @@ impl ToolSpec { pub fn name(&self) -> &str { match self { ToolSpec::Function(tool) => tool.name.as_str(), + ToolSpec::Namespace(namespace) => namespace.name.as_str(), ToolSpec::ToolSearch { .. } => "tool_search", ToolSpec::LocalShell {} => "local_shell", ToolSpec::ImageGeneration { .. } => "image_generation", diff --git a/codex-rs/tools/src/tool_spec_tests.rs b/codex-rs/tools/src/tool_spec_tests.rs index c94f54e6b5..0b98625626 100644 --- a/codex-rs/tools/src/tool_spec_tests.rs +++ b/codex-rs/tools/src/tool_spec_tests.rs @@ -1,4 +1,5 @@ use super::ConfiguredToolSpec; +use super::ResponsesApiNamespace; use super::ResponsesApiWebSearchFilters; use super::ResponsesApiWebSearchUserLocation; use super::ToolSpec; @@ -6,6 +7,7 @@ use crate::AdditionalProperties; use crate::FreeformTool; use crate::FreeformToolFormat; use crate::JsonSchema; +use crate::ResponsesApiNamespaceTool; use crate::ResponsesApiTool; use crate::create_tools_json_for_responses_api; use codex_protocol::config_types::WebSearchContextSize; @@ -34,6 +36,15 @@ fn tool_spec_name_covers_all_variants() { .name(), "lookup_order" ); + assert_eq!( + ToolSpec::Namespace(ResponsesApiNamespace { + name: "mcp__demo__".to_string(), + description: "Demo tools".to_string(), + tools: Vec::new(), + }) + .name(), + "mcp__demo__" + ); assert_eq!( ToolSpec::ToolSearch { execution: "sync".to_string(), @@ -163,6 +174,51 @@ fn create_tools_json_for_responses_api_includes_top_level_name() { ); } +#[test] +fn namespace_tool_spec_serializes_expected_wire_shape() { + assert_eq!( + serde_json::to_value(ToolSpec::Namespace(ResponsesApiNamespace { + name: "mcp__demo__".to_string(), + description: "Demo tools".to_string(), + tools: vec![ResponsesApiNamespaceTool::Function(ResponsesApiTool { + name: "lookup_order".to_string(), + description: "Look up an order".to_string(), + strict: false, + defer_loading: None, + parameters: JsonSchema::object( + BTreeMap::from([( + "order_id".to_string(), + JsonSchema::string(/*description*/ None), + )]), + /*required*/ None, + /*additional_properties*/ None, + ), + output_schema: None, + })], + })) + .expect("serialize namespace tool"), + json!({ + "type": "namespace", + "name": "mcp__demo__", + "description": "Demo tools", + "tools": [ + { + "type": "function", + "name": "lookup_order", + "description": "Look up an order", + "strict": false, + "parameters": { + "type": "object", + "properties": { + "order_id": { "type": "string" }, + }, + }, + }, + ], + }) + ); +} + #[test] fn web_search_tool_spec_serializes_expected_wire_shape() { assert_eq!( diff --git a/codex-rs/tui/BUILD.bazel b/codex-rs/tui/BUILD.bazel index 33cc133431..a8cf752f96 100644 --- a/codex-rs/tui/BUILD.bazel +++ b/codex-rs/tui/BUILD.bazel @@ -1,4 +1,4 @@ -load("//:defs.bzl", "MACOS_WEBRTC_RUSTC_LINK_FLAGS", "codex_rust_crate") +load("//:defs.bzl", "MACOS_WEBRTC_RUSTC_LINK_FLAGS", "codex_rust_crate", "workspace_root_test") codex_rust_crate( name = "tui", @@ -20,8 +20,27 @@ codex_rust_crate( "src/**/snapshots/**", ]) + ["//codex-rs/core:model_availability_nux_fixtures"], integration_compile_data_extra = ["src/test_backend.rs"], + unit_test_args = [ + "--skip", + "app::tests::update_memory_settings_updates_current_thread_memory_mode", + ], extra_binaries = [ "//codex-rs/cli:codex", ], rustc_flags_extra = MACOS_WEBRTC_RUSTC_LINK_FLAGS, ) + +workspace_root_test( + name = "tui-update-memory-settings-flaky-test", + env = { + "INSTA_WORKSPACE_ROOT": ".", + "INSTA_SNAPSHOT_PATH": "src", + }, + test_bin = ":tui-unit-tests-bin", + workspace_root_marker = "//codex-rs/utils/cargo-bin:repo_root.marker", + args = [ + "--exact", + "app::tests::update_memory_settings_updates_current_thread_memory_mode", + ], + tags = ["flaky"], +) diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index e003866cf3..41e542fd69 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -100,6 +100,7 @@ use codex_app_server_protocol::ServerRequest; use codex_app_server_protocol::SkillsListResponse; use codex_app_server_protocol::ThreadItem; use codex_app_server_protocol::ThreadLoadedListParams; +use codex_app_server_protocol::ThreadMemoryMode; use codex_app_server_protocol::ThreadRollbackResponse; use codex_app_server_protocol::ThreadStartSource; use codex_app_server_protocol::Turn; @@ -168,7 +169,7 @@ use toml::Value as TomlValue; use uuid::Uuid; mod agent_navigation; mod app_server_adapter; -mod app_server_requests; +pub(crate) mod app_server_requests; mod loaded_threads; mod pending_interactive_replay; @@ -1641,10 +1642,16 @@ impl App { } self.config = next_config; + let show_memory_enable_notice = feature_updates_to_apply + .iter() + .any(|(feature, enabled)| *feature == Feature::MemoryTool && *enabled); for (feature, effective_enabled) in feature_updates_to_apply { self.chat_widget .set_feature_enabled(feature, effective_enabled); } + if show_memory_enable_notice { + self.chat_widget.add_memories_enable_notice(); + } if approvals_reviewer_override.is_some() { self.set_approvals_reviewer_in_app_and_widget(self.config.approvals_reviewer); } @@ -1728,6 +1735,101 @@ impl App { } } + async fn update_memory_settings( + &mut self, + use_memories: bool, + generate_memories: bool, + ) -> bool { + let active_profile = self.active_profile.clone(); + let scoped_memory_segments = |key: &str| { + if let Some(profile) = active_profile.as_deref() { + vec![ + "profiles".to_string(), + profile.to_string(), + "memories".to_string(), + key.to_string(), + ] + } else { + vec!["memories".to_string(), key.to_string()] + } + }; + let edits = [ + ConfigEdit::SetPath { + segments: scoped_memory_segments("use_memories"), + value: use_memories.into(), + }, + ConfigEdit::SetPath { + segments: scoped_memory_segments("generate_memories"), + value: generate_memories.into(), + }, + ]; + + if let Err(err) = ConfigEditsBuilder::new(&self.config.codex_home) + .with_edits(edits) + .apply() + .await + { + tracing::error!(error = %err, "failed to persist memory settings"); + self.chat_widget + .add_error_message(format!("Failed to save memory settings: {err}")); + return false; + } + + self.config.memories.use_memories = use_memories; + self.config.memories.generate_memories = generate_memories; + self.chat_widget + .set_memory_settings(use_memories, generate_memories); + true + } + + async fn update_memory_settings_with_app_server( + &mut self, + app_server: &mut AppServerSession, + use_memories: bool, + generate_memories: bool, + ) { + let previous_generate_memories = self.config.memories.generate_memories; + if !self + .update_memory_settings(use_memories, generate_memories) + .await + { + return; + } + + if previous_generate_memories == generate_memories { + return; + } + + let Some(thread_id) = self.current_displayed_thread_id() else { + return; + }; + + let mode = if generate_memories { + ThreadMemoryMode::Enabled + } else { + ThreadMemoryMode::Disabled + }; + + if let Err(err) = app_server.thread_memory_mode_set(thread_id, mode).await { + tracing::error!(error = %err, %thread_id, "failed to update thread memory mode"); + self.chat_widget.add_error_message(format!( + "Saved memory settings, but failed to update the current thread: {err}" + )); + } + } + + async fn reset_memories_with_app_server(&mut self, app_server: &mut AppServerSession) { + if let Err(err) = app_server.memory_reset().await { + tracing::error!(error = %err, "failed to reset memories"); + self.chat_widget + .add_error_message(format!("Failed to reset memories: {err}")); + return; + } + + self.chat_widget + .add_info_message("Reset local memories.".to_string(), /*hint*/ None); + } + fn open_url_in_browser(&mut self, url: String) { if let Err(err) = webbrowser::open(&url) { self.chat_widget @@ -5594,6 +5696,20 @@ impl App { AppEvent::UpdateFeatureFlags { updates } => { self.update_feature_flags(updates).await; } + AppEvent::UpdateMemorySettings { + use_memories, + generate_memories, + } => { + self.update_memory_settings_with_app_server( + app_server, + use_memories, + generate_memories, + ) + .await; + } + AppEvent::ResetMemories => { + self.reset_memories_with_app_server(app_server).await; + } AppEvent::SkipNextWorldWritableScan => { self.windows_sandbox.skip_world_writable_scan_once = true; } @@ -6061,8 +6177,13 @@ impl App { .handle_server_notification(notification, /*replay_kind*/ None); } ThreadBufferedEvent::Request(request) => { - self.chat_widget - .handle_server_request(request, /*replay_kind*/ None); + if self + .pending_app_server_requests + .contains_server_request(&request) + { + self.chat_widget + .handle_server_request(request, /*replay_kind*/ None); + } } ThreadBufferedEvent::HistoryEntryResponse(event) => { self.chat_widget.handle_history_entry_response(event); @@ -6517,7 +6638,7 @@ async fn fetch_plugins_list( ) -> Result { let cwd = AbsolutePathBuf::try_from(cwd).wrap_err("plugin list cwd must be absolute")?; let request_id = RequestId::String(format!("plugin-list-{}", Uuid::new_v4())); - request_handle + let mut response = request_handle .request_typed(ClientRequest::PluginList { request_id, params: PluginListParams { @@ -6526,7 +6647,17 @@ async fn fetch_plugins_list( }, }) .await - .wrap_err("plugin/list failed in TUI") + .wrap_err("plugin/list failed in TUI")?; + hide_cli_only_plugin_marketplaces(&mut response); + Ok(response) +} + +const CLI_HIDDEN_PLUGIN_MARKETPLACES: &[&str] = &["openai-bundled"]; + +fn hide_cli_only_plugin_marketplaces(response: &mut PluginListResponse) { + response + .marketplaces + .retain(|marketplace| !CLI_HIDDEN_PLUGIN_MARKETPLACES.contains(&marketplace.name.as_str())); } async fn fetch_plugin_detail( @@ -6704,6 +6835,7 @@ mod tests { use codex_app_server_protocol::NetworkPolicyRuleAction as AppServerNetworkPolicyRuleAction; use codex_app_server_protocol::NonSteerableTurnKind as AppServerNonSteerableTurnKind; use codex_app_server_protocol::PermissionsRequestApprovalParams; + use codex_app_server_protocol::PluginMarketplaceEntry; use codex_app_server_protocol::RequestId as AppServerRequestId; use codex_app_server_protocol::ServerNotification; use codex_app_server_protocol::ServerRequest; @@ -6762,6 +6894,41 @@ mod tests { AbsolutePathBuf::try_from(PathBuf::from(path)).expect("absolute test path") } + #[test] + fn hide_cli_only_plugin_marketplaces_removes_openai_bundled() { + let mut response = PluginListResponse { + marketplaces: vec![ + PluginMarketplaceEntry { + name: "openai-bundled".to_string(), + path: test_absolute_path("/marketplaces/openai-bundled"), + interface: None, + plugins: Vec::new(), + }, + PluginMarketplaceEntry { + name: "openai-curated".to_string(), + path: test_absolute_path("/marketplaces/openai-curated"), + interface: None, + plugins: Vec::new(), + }, + ], + marketplace_load_errors: Vec::new(), + remote_sync_error: None, + featured_plugin_ids: Vec::new(), + }; + + hide_cli_only_plugin_marketplaces(&mut response); + + assert_eq!( + response.marketplaces, + vec![PluginMarketplaceEntry { + name: "openai-curated".to_string(), + path: test_absolute_path("/marketplaces/openai-curated"), + interface: None, + plugins: Vec::new(), + }] + ); + } + #[test] fn normalize_harness_overrides_resolves_relative_add_dirs() -> Result<()> { let temp_dir = tempdir()?; @@ -7004,6 +7171,11 @@ mod tests { let approval_request = exec_approval_request(thread_id, "turn-1", "call-1", /*approval_id*/ None); + assert_eq!( + app.pending_app_server_requests + .note_server_request(&approval_request), + None + ); app.enqueue_primary_thread_request(approval_request).await?; app.enqueue_primary_thread_session( test_thread_session(thread_id, test_path_buf("/tmp/project")), @@ -7046,6 +7218,66 @@ mod tests { panic!("expected approval action to submit a thread-scoped op"); } + #[tokio::test] + async fn resolved_buffered_approval_does_not_become_actionable_after_drain() -> Result<()> { + let (mut app, mut app_event_rx, _op_rx) = make_test_app_with_channels().await; + let thread_id = ThreadId::new(); + let approval_request = + exec_approval_request(thread_id, "turn-1", "call-1", /*approval_id*/ None); + + app.enqueue_primary_thread_session( + test_thread_session(thread_id, test_path_buf("/tmp/project")), + Vec::new(), + ) + .await?; + while app_event_rx.try_recv().is_ok() {} + + assert_eq!( + app.pending_app_server_requests + .note_server_request(&approval_request), + None + ); + app.enqueue_thread_request(thread_id, approval_request) + .await?; + + let resolved = app + .pending_app_server_requests + .resolve_notification(&AppServerRequestId::Integer(1)) + .expect("matching app-server request should resolve"); + app.chat_widget.dismiss_app_server_request(&resolved); + while app_event_rx.try_recv().is_ok() {} + + let rx = app + .active_thread_rx + .as_mut() + .expect("primary thread receiver should be active"); + let event = time::timeout(Duration::from_millis(50), rx.recv()) + .await + .expect("timed out waiting for buffered approval event") + .expect("channel closed unexpectedly"); + + assert!(matches!( + &event, + ThreadBufferedEvent::Request(ServerRequest::CommandExecutionRequestApproval { + params, + .. + }) if params.turn_id == "turn-1" + )); + + app.handle_thread_event_now(event); + app.chat_widget + .handle_key_event(KeyEvent::new(KeyCode::Char('y'), KeyModifiers::NONE)); + + while let Ok(app_event) = app_event_rx.try_recv() { + assert!( + !matches!(app_event, AppEvent::SubmitThreadOp { .. }), + "resolved buffered approval should not become actionable" + ); + } + + Ok(()) + } + #[tokio::test] async fn enqueue_primary_thread_session_replays_turns_before_initial_prompt_submit() -> Result<()> { @@ -8226,6 +8458,111 @@ mod tests { Ok(()) } + #[tokio::test] + async fn update_memory_settings_persists_and_updates_widget_config() -> Result<()> { + let (mut app, _app_event_rx, _op_rx) = make_test_app_with_channels().await; + let codex_home = tempdir()?; + app.config.codex_home = codex_home.path().to_path_buf().abs(); + let mut app_server = crate::start_embedded_app_server_for_picker(&app.config).await?; + + app.update_memory_settings_with_app_server( + &mut app_server, + /*use_memories*/ false, + /*generate_memories*/ false, + ) + .await; + + assert!(!app.config.memories.use_memories); + assert!(!app.config.memories.generate_memories); + assert!(!app.chat_widget.config_ref().memories.use_memories); + assert!(!app.chat_widget.config_ref().memories.generate_memories); + + let config = std::fs::read_to_string(codex_home.path().join("config.toml"))?; + let config_value = toml::from_str::(&config)?; + let memories = config_value + .as_table() + .and_then(|table| table.get("memories")) + .and_then(TomlValue::as_table) + .expect("memories table should exist"); + assert_eq!( + memories.get("use_memories"), + Some(&TomlValue::Boolean(false)) + ); + assert_eq!( + memories.get("generate_memories"), + Some(&TomlValue::Boolean(false)) + ); + assert!( + !memories.contains_key("no_memories_if_mcp_or_web_search"), + "the TUI menu should not write the MCP pollution setting" + ); + app_server.shutdown().await?; + Ok(()) + } + + #[tokio::test] + async fn update_memory_settings_updates_current_thread_memory_mode() -> Result<()> { + let (mut app, _app_event_rx, _op_rx) = make_test_app_with_channels().await; + let codex_home = tempdir()?; + app.config.codex_home = codex_home.path().to_path_buf().abs(); + + let mut app_server = crate::start_embedded_app_server_for_picker(&app.config).await?; + let started = app_server.start_thread(&app.config).await?; + let thread_id = started.session.thread_id; + app.active_thread_id = Some(thread_id); + + app.update_memory_settings_with_app_server( + &mut app_server, + /*use_memories*/ true, + /*generate_memories*/ false, + ) + .await; + + let state_db = codex_state::StateRuntime::init( + codex_home.path().to_path_buf(), + app.config.model_provider_id.clone(), + ) + .await + .expect("state db should initialize"); + let memory_mode = state_db + .get_thread_memory_mode(thread_id) + .await + .expect("thread memory mode should be readable"); + assert_eq!(memory_mode.as_deref(), Some("disabled")); + + app_server.shutdown().await?; + Ok(()) + } + + #[tokio::test] + async fn reset_memories_clears_local_memory_directories() -> Result<()> { + let (mut app, _app_event_rx, _op_rx) = make_test_app_with_channels().await; + let codex_home = tempdir()?; + app.config.codex_home = codex_home.path().to_path_buf().abs(); + app.config.sqlite_home = codex_home.path().to_path_buf(); + + let memory_root = codex_home.path().join("memories"); + let extensions_root = codex_home.path().join("memories_extensions"); + std::fs::create_dir_all(memory_root.join("rollout_summaries"))?; + std::fs::create_dir_all(&extensions_root)?; + std::fs::write(memory_root.join("MEMORY.md"), "stale memory\n")?; + std::fs::write( + memory_root.join("rollout_summaries").join("stale.md"), + "stale summary\n", + )?; + std::fs::write(extensions_root.join("stale.txt"), "stale extension\n")?; + + let mut app_server = crate::start_embedded_app_server_for_picker(&app.config).await?; + + app.reset_memories_with_app_server(&mut app_server).await; + + assert_eq!(std::fs::read_dir(&memory_root)?.count(), 0); + assert_eq!(std::fs::read_dir(&extensions_root)?.count(), 0); + + app_server.shutdown().await?; + Ok(()) + } + #[tokio::test] async fn update_feature_flags_enabling_guardian_selects_guardian_approvals() -> Result<()> { let (mut app, mut app_event_rx, mut op_rx) = make_test_app_with_channels().await; diff --git a/codex-rs/tui/src/app/app_server_adapter.rs b/codex-rs/tui/src/app/app_server_adapter.rs index 09b8ce06cd..f2490ca6a2 100644 --- a/codex-rs/tui/src/app/app_server_adapter.rs +++ b/codex-rs/tui/src/app/app_server_adapter.rs @@ -158,8 +158,12 @@ impl App { ) { match ¬ification { ServerNotification::ServerRequestResolved(notification) => { - self.pending_app_server_requests - .resolve_notification(¬ification.request_id); + if let Some(request) = self + .pending_app_server_requests + .resolve_notification(¬ification.request_id) + { + self.chat_widget.dismiss_app_server_request(&request); + } } ServerNotification::McpServerStatusUpdated(_) => { self.refresh_mcp_startup_expected_servers_from_config(); diff --git a/codex-rs/tui/src/app/app_server_requests.rs b/codex-rs/tui/src/app/app_server_requests.rs index b1393e95e0..e2483eb25d 100644 --- a/codex-rs/tui/src/app/app_server_requests.rs +++ b/codex-rs/tui/src/app/app_server_requests.rs @@ -1,4 +1,5 @@ use std::collections::HashMap; +use std::collections::VecDeque; use crate::app_command::AppCommand; use crate::app_command::AppCommandView; @@ -27,12 +28,32 @@ pub(super) struct UnsupportedAppServerRequest { pub(super) message: String, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ResolvedAppServerRequest { + ExecApproval { + id: String, + }, + FileChangeApproval { + id: String, + }, + PermissionsApproval { + id: String, + }, + UserInput { + call_id: String, + }, + McpElicitation { + server_name: String, + request_id: McpRequestId, + }, +} + #[derive(Debug, Default)] pub(super) struct PendingAppServerRequests { exec_approvals: HashMap, file_change_approvals: HashMap, permissions_approvals: HashMap, - user_inputs: HashMap, + user_inputs: HashMap>, mcp_requests: HashMap, } @@ -70,7 +91,12 @@ impl PendingAppServerRequests { } ServerRequest::ToolRequestUserInput { request_id, params } => { self.user_inputs - .insert(params.turn_id.clone(), request_id.clone()); + .entry(params.turn_id.clone()) + .or_default() + .push_back(PendingUserInputRequest { + item_id: params.item_id.clone(), + request_id: request_id.clone(), + }); None } ServerRequest::McpServerElicitationRequest { request_id, params } => { @@ -165,11 +191,10 @@ impl PendingAppServerRequests { }) .transpose()?, AppCommandView::UserInputAnswer { id, response } => self - .user_inputs - .remove(id) - .map(|request_id| { + .pop_user_input_request_for_turn(id) + .map(|pending| { Ok::(AppServerRequestResolution { - request_id, + request_id: pending.request_id, result: serde_json::to_value( serde_json::from_value::( serde_json::to_value(response).map_err(|err| { @@ -229,15 +254,131 @@ impl PendingAppServerRequests { Ok(resolution) } - pub(super) fn resolve_notification(&mut self, request_id: &AppServerRequestId) { - self.exec_approvals.retain(|_, value| value != request_id); - self.file_change_approvals - .retain(|_, value| value != request_id); - self.permissions_approvals - .retain(|_, value| value != request_id); - self.user_inputs.retain(|_, value| value != request_id); - self.mcp_requests.retain(|_, value| value != request_id); + pub(super) fn resolve_notification( + &mut self, + request_id: &AppServerRequestId, + ) -> Option { + if let Some(id) = self + .exec_approvals + .iter() + .find_map(|(id, value)| (value == request_id).then(|| id.clone())) + { + self.exec_approvals.remove(&id); + return Some(ResolvedAppServerRequest::ExecApproval { id }); + } + + if let Some(id) = self + .file_change_approvals + .iter() + .find_map(|(id, value)| (value == request_id).then(|| id.clone())) + { + self.file_change_approvals.remove(&id); + return Some(ResolvedAppServerRequest::FileChangeApproval { id }); + } + + if let Some(id) = self + .permissions_approvals + .iter() + .find_map(|(id, value)| (value == request_id).then(|| id.clone())) + { + self.permissions_approvals.remove(&id); + return Some(ResolvedAppServerRequest::PermissionsApproval { id }); + } + + if let Some(pending) = self.remove_user_input_request(request_id) { + return Some(ResolvedAppServerRequest::UserInput { + call_id: pending.item_id, + }); + } + + if let Some(key) = self + .mcp_requests + .iter() + .find_map(|(key, value)| (value == request_id).then(|| key.clone())) + { + self.mcp_requests.remove(&key); + return Some(ResolvedAppServerRequest::McpElicitation { + server_name: key.server_name, + request_id: key.request_id, + }); + } + + None } + + pub(super) fn contains_server_request(&self, request: &ServerRequest) -> bool { + match request { + ServerRequest::CommandExecutionRequestApproval { request_id, .. } => self + .exec_approvals + .values() + .any(|pending_request_id| pending_request_id == request_id), + ServerRequest::FileChangeRequestApproval { request_id, .. } => self + .file_change_approvals + .values() + .any(|pending_request_id| pending_request_id == request_id), + ServerRequest::PermissionsRequestApproval { request_id, .. } => self + .permissions_approvals + .values() + .any(|pending_request_id| pending_request_id == request_id), + ServerRequest::ToolRequestUserInput { request_id, .. } => { + self.user_inputs.values().any(|queue| { + queue + .iter() + .any(|pending| &pending.request_id == request_id) + }) + } + ServerRequest::McpServerElicitationRequest { request_id, .. } => self + .mcp_requests + .values() + .any(|pending_request_id| pending_request_id == request_id), + ServerRequest::DynamicToolCall { .. } + | ServerRequest::ChatgptAuthTokensRefresh { .. } + | ServerRequest::ApplyPatchApproval { .. } + | ServerRequest::ExecCommandApproval { .. } => true, + } + } + + fn pop_user_input_request_for_turn( + &mut self, + turn_id: &str, + ) -> Option { + let pending = self + .user_inputs + .get_mut(turn_id) + .and_then(VecDeque::pop_front); + if self + .user_inputs + .get(turn_id) + .is_some_and(VecDeque::is_empty) + { + self.user_inputs.remove(turn_id); + } + pending + } + + fn remove_user_input_request( + &mut self, + request_id: &AppServerRequestId, + ) -> Option { + let (turn_id, index) = self.user_inputs.iter().find_map(|(turn_id, queue)| { + queue + .iter() + .position(|pending| &pending.request_id == request_id) + .map(|index| (turn_id.clone(), index)) + })?; + let queue = self.user_inputs.get_mut(&turn_id)?; + let removed = queue.remove(index); + if queue.is_empty() { + self.user_inputs.remove(&turn_id); + } + removed + } +} + +#[derive(Debug)] +struct PendingUserInputRequest { + item_id: String, + request_id: AppServerRequestId, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -272,6 +413,7 @@ fn file_change_decision(decision: &ReviewDecision) -> Result, }, + /// Update memory settings and persist them to config.toml. + UpdateMemorySettings { + use_memories: bool, + generate_memories: bool, + }, + + /// Clear all persisted local memory artifacts via the app-server. + ResetMemories, + /// Update whether the full access warning prompt has been acknowledged. UpdateFullAccessWarningAcknowledged(bool), diff --git a/codex-rs/tui/src/app_server_session.rs b/codex-rs/tui/src/app_server_session.rs index 292a935dd9..36ed977a3f 100644 --- a/codex-rs/tui/src/app_server_session.rs +++ b/codex-rs/tui/src/app_server_session.rs @@ -23,6 +23,7 @@ use codex_app_server_protocol::GetAccountParams; use codex_app_server_protocol::GetAccountRateLimitsResponse; use codex_app_server_protocol::GetAccountResponse; use codex_app_server_protocol::JSONRPCErrorError; +use codex_app_server_protocol::MemoryResetResponse; use codex_app_server_protocol::Model as ApiModel; use codex_app_server_protocol::ModelListParams; use codex_app_server_protocol::ModelListResponse; @@ -43,6 +44,9 @@ use codex_app_server_protocol::ThreadListParams; use codex_app_server_protocol::ThreadListResponse; use codex_app_server_protocol::ThreadLoadedListParams; use codex_app_server_protocol::ThreadLoadedListResponse; +use codex_app_server_protocol::ThreadMemoryMode; +use codex_app_server_protocol::ThreadMemoryModeSetParams; +use codex_app_server_protocol::ThreadMemoryModeSetResponse; use codex_app_server_protocol::ThreadReadParams; use codex_app_server_protocol::ThreadReadResponse; use codex_app_server_protocol::ThreadRealtimeAppendAudioParams; @@ -542,6 +546,39 @@ impl AppServerSession { Ok(()) } + pub(crate) async fn thread_memory_mode_set( + &mut self, + thread_id: ThreadId, + mode: ThreadMemoryMode, + ) -> Result<()> { + let request_id = self.next_request_id(); + let _: ThreadMemoryModeSetResponse = self + .client + .request_typed(ClientRequest::ThreadMemoryModeSet { + request_id, + params: ThreadMemoryModeSetParams { + thread_id: thread_id.to_string(), + mode, + }, + }) + .await + .wrap_err("thread/memoryMode/set failed in TUI")?; + Ok(()) + } + + pub(crate) async fn memory_reset(&mut self) -> Result<()> { + let request_id = self.next_request_id(); + let _: MemoryResetResponse = self + .client + .request_typed(ClientRequest::MemoryReset { + request_id, + params: None, + }) + .await + .wrap_err("memory/reset failed in TUI")?; + Ok(()) + } + pub(crate) async fn thread_unsubscribe(&mut self, thread_id: ThreadId) -> Result<()> { let request_id = self.next_request_id(); let _: ThreadUnsubscribeResponse = self diff --git a/codex-rs/tui/src/bottom_pane/app_link_view.rs b/codex-rs/tui/src/bottom_pane/app_link_view.rs index e39d547625..4ffdb304d7 100644 --- a/codex-rs/tui/src/bottom_pane/app_link_view.rs +++ b/codex-rs/tui/src/bottom_pane/app_link_view.rs @@ -24,6 +24,7 @@ use super::scroll_state::ScrollState; use super::selection_popup_common::GenericDisplayRow; use super::selection_popup_common::measure_rows_height; use super::selection_popup_common::render_rows; +use crate::app::app_server_requests::ResolvedAppServerRequest; use crate::app_event::AppEvent; use crate::app_event_sender::AppEventSender; use crate::key_hint; @@ -474,6 +475,25 @@ impl BottomPaneView for AppLinkView { fn is_complete(&self) -> bool { self.complete } + + fn dismiss_app_server_request(&mut self, request: &ResolvedAppServerRequest) -> bool { + let ResolvedAppServerRequest::McpElicitation { + server_name, + request_id, + } = request + else { + return false; + }; + let Some(target) = self.elicitation_target.as_ref() else { + return false; + }; + if target.server_name != *server_name || target.request_id != *request_id { + return false; + } + + self.complete = true; + true + } } impl crate::render::renderable::Renderable for AppLinkView { @@ -546,9 +566,11 @@ impl crate::render::renderable::Renderable for AppLinkView { #[cfg(test)] mod tests { use super::*; + use crate::app::app_server_requests::ResolvedAppServerRequest; use crate::app_event::AppEvent; use crate::render::renderable::Renderable; use insta::assert_snapshot; + use pretty_assertions::assert_eq; use tokio::sync::mpsc::unbounded_channel; fn suggestion_target() -> AppLinkElicitationTarget { @@ -889,6 +911,64 @@ mod tests { assert!(view.is_complete()); } + #[test] + fn resolved_tool_suggestion_dismisses_matching_view() { + let (tx_raw, _rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let mut view = AppLinkView::new( + AppLinkViewParams { + app_id: "connector_google_calendar".to_string(), + title: "Google Calendar".to_string(), + description: Some("Plan events and schedules.".to_string()), + instructions: "Enable this app to use it for the current request.".to_string(), + url: "https://example.test/google-calendar".to_string(), + is_installed: true, + is_enabled: false, + suggest_reason: Some("Plan and reference events from your calendar".to_string()), + suggestion_type: Some(AppLinkSuggestionType::Enable), + elicitation_target: Some(suggestion_target()), + }, + tx, + ); + + assert!( + view.dismiss_app_server_request(&ResolvedAppServerRequest::McpElicitation { + server_name: "codex_apps".to_string(), + request_id: McpRequestId::String("request-1".to_string()), + }) + ); + assert!(view.is_complete()); + } + + #[test] + fn resolved_tool_suggestion_ignores_non_matching_request() { + let (tx_raw, _rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let mut view = AppLinkView::new( + AppLinkViewParams { + app_id: "connector_google_calendar".to_string(), + title: "Google Calendar".to_string(), + description: Some("Plan events and schedules.".to_string()), + instructions: "Enable this app to use it for the current request.".to_string(), + url: "https://example.test/google-calendar".to_string(), + is_installed: true, + is_enabled: false, + suggest_reason: Some("Plan and reference events from your calendar".to_string()), + suggestion_type: Some(AppLinkSuggestionType::Enable), + elicitation_target: Some(suggestion_target()), + }, + tx, + ); + + assert!( + !view.dismiss_app_server_request(&ResolvedAppServerRequest::McpElicitation { + server_name: "other_server".to_string(), + request_id: McpRequestId::String("request-1".to_string()), + }) + ); + assert!(!view.is_complete()); + } + #[test] fn install_suggestion_with_reason_snapshot() { let (tx_raw, _rx) = unbounded_channel::(); diff --git a/codex-rs/tui/src/bottom_pane/approval_overlay.rs b/codex-rs/tui/src/bottom_pane/approval_overlay.rs index 656e812d66..0e2bb2b2e3 100644 --- a/codex-rs/tui/src/bottom_pane/approval_overlay.rs +++ b/codex-rs/tui/src/bottom_pane/approval_overlay.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use std::path::PathBuf; +use crate::app::app_server_requests::ResolvedAppServerRequest; use crate::app_event::AppEvent; use crate::app_event_sender::AppEventSender; use crate::bottom_pane::BottomPaneView; @@ -97,6 +98,35 @@ impl ApprovalRequest { | ApprovalRequest::McpElicitation { thread_label, .. } => thread_label.as_deref(), } } + + fn matches_resolved_request(&self, request: &ResolvedAppServerRequest) -> bool { + match (self, request) { + ( + ApprovalRequest::Exec { id, .. }, + ResolvedAppServerRequest::ExecApproval { id: resolved_id }, + ) => id == resolved_id, + ( + ApprovalRequest::Permissions { call_id, .. }, + ResolvedAppServerRequest::PermissionsApproval { id }, + ) => call_id == id, + ( + ApprovalRequest::ApplyPatch { id, .. }, + ResolvedAppServerRequest::FileChangeApproval { id: resolved_id }, + ) => id == resolved_id, + ( + ApprovalRequest::McpElicitation { + server_name, + request_id, + .. + }, + ResolvedAppServerRequest::McpElicitation { + server_name: resolved_server_name, + request_id: resolved_request_id, + }, + ) => server_name == resolved_server_name && request_id == resolved_request_id, + _ => false, + } + } } /// Modal overlay asking the user to approve or deny one or more requests. @@ -131,6 +161,23 @@ impl ApprovalOverlay { self.queue.push(req); } + fn dismiss_resolved_request(&mut self, request: &ResolvedAppServerRequest) -> bool { + let queue_len = self.queue.len(); + self.queue + .retain(|queued_request| !queued_request.matches_resolved_request(request)); + if self + .current_request + .as_ref() + .is_some_and(|current_request| current_request.matches_resolved_request(request)) + { + self.current_complete = true; + self.advance_queue(); + return true; + } + + self.queue.len() != queue_len + } + fn set_current(&mut self, request: ApprovalRequest) { self.current_complete = false; let header = build_header(&request); @@ -465,6 +512,10 @@ impl BottomPaneView for ApprovalOverlay { self.enqueue_request(request); None } + + fn dismiss_app_server_request(&mut self, request: &ResolvedAppServerRequest) -> bool { + self.dismiss_resolved_request(request) + } } impl Renderable for ApprovalOverlay { @@ -951,6 +1002,27 @@ mod tests { assert!(saw_op, "expected approval decision to emit an op"); } + #[test] + fn resolved_request_dismisses_overlay_without_emitting_abort() { + let (tx, mut rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx); + let mut view = ApprovalOverlay::new(make_exec_request(), tx, Features::with_defaults()); + + assert!( + view.dismiss_app_server_request(&ResolvedAppServerRequest::ExecApproval { + id: "test".to_string(), + }) + ); + assert!( + view.is_complete(), + "resolved request should close the overlay" + ); + assert!( + rx.try_recv().is_err(), + "dismissing a stale request should not emit an approval op" + ); + } + #[test] fn o_opens_source_thread_for_cross_thread_approval() { let (tx, mut rx) = unbounded_channel::(); diff --git a/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs b/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs index 35165db491..6fa946a5af 100644 --- a/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs +++ b/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs @@ -1,3 +1,4 @@ +use crate::app::app_server_requests::ResolvedAppServerRequest; use crate::bottom_pane::ApprovalRequest; use crate::bottom_pane::McpServerElicitationFormRequest; use crate::render::renderable::Renderable; @@ -87,4 +88,11 @@ pub(crate) trait BottomPaneView: Renderable { ) -> Option { Some(request) } + + /// Dismiss a request that was resolved by another client. + /// + /// Returns `true` when the view changed state. + fn dismiss_app_server_request(&mut self, _request: &ResolvedAppServerRequest) -> bool { + false + } } diff --git a/codex-rs/tui/src/bottom_pane/command_popup.rs b/codex-rs/tui/src/bottom_pane/command_popup.rs index 8bef8ddbc8..29da003dc2 100644 --- a/codex-rs/tui/src/bottom_pane/command_popup.rs +++ b/codex-rs/tui/src/bottom_pane/command_popup.rs @@ -293,7 +293,7 @@ mod tests { CommandItem::Builtin(cmd) => cmd.command(), }) .collect(); - assert_eq!(cmds, vec!["model", "mention", "mcp"]); + assert_eq!(cmds, vec!["model", "memories", "mention", "mcp"]); } #[test] diff --git a/codex-rs/tui/src/bottom_pane/mcp_server_elicitation.rs b/codex-rs/tui/src/bottom_pane/mcp_server_elicitation.rs index 1e3ad65912..c9844ff915 100644 --- a/codex-rs/tui/src/bottom_pane/mcp_server_elicitation.rs +++ b/codex-rs/tui/src/bottom_pane/mcp_server_elicitation.rs @@ -28,6 +28,7 @@ use ratatui::widgets::Widget; use serde_json::Value; use unicode_width::UnicodeWidthStr; +use crate::app::app_server_requests::ResolvedAppServerRequest; use crate::app_event_sender::AppEventSender; use crate::bottom_pane::CancellationEvent; use crate::bottom_pane::ChatComposer; @@ -1145,6 +1146,16 @@ impl McpServerElicitationOverlay { ); } + fn advance_queue_or_complete(&mut self) { + if let Some(next) = self.queue.pop_front() { + self.request = next; + self.reset_for_request(); + self.restore_current_draft(); + } else { + self.done = true; + } + } + fn submit_answers(&mut self) { self.save_current_draft(); if let Some(idx) = self.first_required_unanswered_index() { @@ -1181,13 +1192,7 @@ impl McpServerElicitationOverlay { /*content*/ None, meta, ); - if let Some(next) = self.queue.pop_front() { - self.request = next; - self.reset_for_request(); - self.restore_current_draft(); - } else { - self.done = true; - } + self.advance_queue_or_complete(); return; } let content = self @@ -1205,13 +1210,28 @@ impl McpServerElicitationOverlay { Some(Value::Object(content)), /*meta*/ None, ); - if let Some(next) = self.queue.pop_front() { - self.request = next; - self.reset_for_request(); - self.restore_current_draft(); - } else { - self.done = true; + self.advance_queue_or_complete(); + } + + fn dismiss_resolved_request(&mut self, request: &ResolvedAppServerRequest) -> bool { + let ResolvedAppServerRequest::McpElicitation { + server_name, + request_id, + } = request + else { + return false; + }; + + let queue_len = self.queue.len(); + self.queue.retain(|queued_request| { + queued_request.server_name != *server_name || queued_request.request_id != *request_id + }); + if self.request.server_name == *server_name && self.request.request_id == *request_id { + self.advance_queue_or_complete(); + return true; } + + self.queue.len() != queue_len } fn go_next_or_submit(&mut self) { @@ -1642,6 +1662,10 @@ impl BottomPaneView for McpServerElicitationOverlay { self.queue.push_back(request); None } + + fn dismiss_app_server_request(&mut self, request: &ResolvedAppServerRequest) -> bool { + self.dismiss_resolved_request(request) + } } fn wrap_footer_tips(width: u16, tips: Vec) -> Vec> { @@ -2362,6 +2386,72 @@ mod tests { assert_eq!(overlay.request.message, "Third"); } + #[test] + fn resolved_request_dismisses_overlay_without_emitting_events() { + let (tx, mut rx) = test_sender(); + let thread_id = ThreadId::default(); + let supported_form_schema = serde_json::json!({ + "type": "object", + "properties": { + "confirmed": { + "type": "boolean", + "title": "Confirm", + } + }, + }); + let mut overlay = McpServerElicitationOverlay::new( + McpServerElicitationFormRequest::from_event( + thread_id, + form_request("First", supported_form_schema.clone(), /*meta*/ None), + ) + .expect("expected supported form"), + tx, + /*has_input_focus*/ true, + /*enhanced_keys_supported*/ false, + /*disable_paste_burst*/ false, + ); + overlay.try_consume_mcp_server_elicitation_request( + McpServerElicitationFormRequest::from_event( + thread_id, + ElicitationRequestEvent { + turn_id: Some("turn-2".to_string()), + server_name: "server-1".to_string(), + id: McpRequestId::String("request-2".to_string()), + request: ElicitationRequest::Form { + meta: None, + message: "Second".to_string(), + requested_schema: supported_form_schema, + }, + }, + ) + .expect("expected supported form"), + ); + + assert!( + overlay.dismiss_app_server_request(&ResolvedAppServerRequest::McpElicitation { + server_name: "server-1".to_string(), + request_id: McpRequestId::String("request-1".to_string()), + }) + ); + assert_eq!(overlay.request.message, "Second"); + assert!(matches!( + rx.try_recv(), + Err(tokio::sync::mpsc::error::TryRecvError::Empty) + )); + + assert!( + overlay.dismiss_app_server_request(&ResolvedAppServerRequest::McpElicitation { + server_name: "server-1".to_string(), + request_id: McpRequestId::String("request-2".to_string()), + }) + ); + assert!(overlay.is_complete()); + assert!(matches!( + rx.try_recv(), + Err(tokio::sync::mpsc::error::TryRecvError::Empty) + )); + } + #[test] fn boolean_form_snapshot() { let (tx, _rx) = test_sender(); diff --git a/codex-rs/tui/src/bottom_pane/memories_settings_view.rs b/codex-rs/tui/src/bottom_pane/memories_settings_view.rs new file mode 100644 index 0000000000..d2d3c00ce3 --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/memories_settings_view.rs @@ -0,0 +1,477 @@ +use crossterm::event::KeyCode; +use crossterm::event::KeyEvent; +use crossterm::event::KeyModifiers; +use ratatui::buffer::Buffer; +use ratatui::layout::Constraint; +use ratatui::layout::Layout; +use ratatui::layout::Rect; +use ratatui::style::Stylize; +use ratatui::text::Line; +use ratatui::widgets::Block; +use ratatui::widgets::Widget; + +use crate::app_event::AppEvent; +use crate::app_event_sender::AppEventSender; +use crate::bottom_pane::popup_consts::standard_popup_hint_line; +use crate::key_hint; +use crate::render::Insets; +use crate::render::RectExt as _; +use crate::render::renderable::ColumnRenderable; +use crate::render::renderable::Renderable; +use crate::style::user_message_style; + +use super::CancellationEvent; +use super::bottom_pane_view::BottomPaneView; +use super::popup_consts::MAX_POPUP_ROWS; +use super::scroll_state::ScrollState; +use super::selection_popup_common::GenericDisplayRow; +use super::selection_popup_common::measure_rows_height; +use super::selection_popup_common::render_rows; + +const MEMORIES_DOC_URL: &str = "https://developers.openai.com/codex/memories"; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum MemoriesSetting { + Use, + Generate, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum MemoriesAction { + Reset, +} + +enum MemoriesMenuItem { + Setting { + setting: MemoriesSetting, + name: &'static str, + description: &'static str, + enabled: bool, + }, + Action { + action: MemoriesAction, + name: &'static str, + description: &'static str, + }, +} + +pub(crate) struct MemoriesSettingsView { + items: Vec, + state: ScrollState, + reset_confirmation: Option, + complete: bool, + app_event_tx: AppEventSender, + docs_link: Line<'static>, +} + +impl MemoriesSettingsView { + pub(crate) fn new( + use_memories: bool, + generate_memories: bool, + app_event_tx: AppEventSender, + ) -> Self { + let mut view = Self { + items: vec![ + MemoriesMenuItem::Setting { + setting: MemoriesSetting::Use, + name: "Use memories", + description: "Use memories in the following threads. Applied at next thread.", + enabled: use_memories, + }, + MemoriesMenuItem::Setting { + setting: MemoriesSetting::Generate, + name: "Generate memories", + description: "Generate memories from the following threads. Current thread included.", + enabled: generate_memories, + }, + MemoriesMenuItem::Action { + action: MemoriesAction::Reset, + name: "Reset all memories", + description: "Clear local memory files and summaries. Existing threads stay intact.", + }, + ], + state: ScrollState::new(), + reset_confirmation: None, + complete: false, + app_event_tx, + docs_link: Line::from(vec![ + "Learn more: ".dim(), + MEMORIES_DOC_URL.cyan().underlined(), + ]), + }; + view.initialize_selection(); + view + } + + fn initialize_selection(&mut self) { + self.state.selected_idx = (!self.items.is_empty()).then_some(0); + } + + fn settings_header(&self) -> ColumnRenderable<'_> { + let mut header = ColumnRenderable::new(); + header.push(Line::from("Memories".bold())); + header.push(Line::from( + "Choose how Codex uses and creates memories. Changes are saved to config.toml".dim(), + )); + header + } + + fn reset_confirmation_header(&self) -> ColumnRenderable<'_> { + let mut header = ColumnRenderable::new(); + header.push(Line::from("Reset all memories?".bold())); + header.push(Line::from( + "This clears local memory files and rollout summaries for the current Codex home." + .dim(), + )); + header + } + + fn active_state(&self) -> &ScrollState { + self.reset_confirmation.as_ref().unwrap_or(&self.state) + } + + fn active_state_mut(&mut self) -> &mut ScrollState { + self.reset_confirmation.as_mut().unwrap_or(&mut self.state) + } + + fn visible_len(&self) -> usize { + if self.reset_confirmation.is_some() { + 2 + } else { + self.items.len() + } + } + + fn build_rows(&self) -> Vec { + if let Some(state) = self.reset_confirmation.as_ref() { + return ["Reset all memories", "Go back"] + .into_iter() + .enumerate() + .map(|(idx, name)| GenericDisplayRow { + name: if state.selected_idx == Some(idx) { + format!("› {name}") + } else { + format!(" {name}") + }, + description: Some(match idx { + 0 => "Delete local memory files and rollout summaries.".to_string(), + 1 => "Return to memory settings.".to_string(), + _ => unreachable!("reset confirmation only renders two rows"), + }), + ..Default::default() + }) + .collect(); + } + + let selected_idx = self.state.selected_idx; + self.items + .iter() + .enumerate() + .map(|(idx, item)| { + let prefix = if selected_idx == Some(idx) { + '›' + } else { + ' ' + }; + let (name, description) = match item { + MemoriesMenuItem::Setting { + name, + description, + enabled, + .. + } => ( + format!("{prefix} [{}] {name}", if *enabled { 'x' } else { ' ' }), + description, + ), + MemoriesMenuItem::Action { + name, description, .. + } => (format!("{prefix} {name}"), description), + }; + GenericDisplayRow { + name, + description: Some((*description).to_string()), + ..Default::default() + } + }) + .collect() + } + + fn move_up(&mut self) { + let len = self.visible_len(); + if len == 0 { + return; + } + let state = self.active_state_mut(); + state.move_up_wrap(len); + state.ensure_visible(len, MAX_POPUP_ROWS.min(len)); + } + + fn move_down(&mut self) { + let len = self.visible_len(); + if len == 0 { + return; + } + let state = self.active_state_mut(); + state.move_down_wrap(len); + state.ensure_visible(len, MAX_POPUP_ROWS.min(len)); + } + + fn toggle_selected(&mut self) { + if self.reset_confirmation.is_some() { + return; + } + + let Some(selected_idx) = self.state.selected_idx else { + return; + }; + + if let Some(MemoriesMenuItem::Setting { enabled, .. }) = self.items.get_mut(selected_idx) { + *enabled = !*enabled; + } + } + + fn rows_width(total_width: u16) -> u16 { + total_width.saturating_sub(2) + } + + fn current_setting(&self, setting: MemoriesSetting) -> bool { + self.items + .iter() + .find_map(|item| match item { + MemoriesMenuItem::Setting { + setting: item_setting, + enabled, + .. + } if *item_setting == setting => Some(*enabled), + _ => None, + }) + .unwrap_or(false) + } + + fn open_reset_confirmation(&mut self) { + let mut state = ScrollState::new(); + state.selected_idx = Some(0); + self.reset_confirmation = Some(state); + } + + fn close_reset_confirmation(&mut self) { + self.reset_confirmation = None; + self.state.selected_idx = self.items.len().checked_sub(1); + } + + fn footer_hint(&self) -> Line<'static> { + if self.reset_confirmation.is_some() { + standard_popup_hint_line() + } else { + memories_settings_hint_line() + } + } +} + +impl BottomPaneView for MemoriesSettingsView { + fn handle_key_event(&mut self, key_event: KeyEvent) { + match key_event { + KeyEvent { + code: KeyCode::Up, .. + } + | KeyEvent { + code: KeyCode::Char('p'), + modifiers: KeyModifiers::CONTROL, + .. + } + | KeyEvent { + code: KeyCode::Char('\u{0010}'), + modifiers: KeyModifiers::NONE, + .. + } => self.move_up(), + KeyEvent { + code: KeyCode::Char('k'), + modifiers: KeyModifiers::NONE, + .. + } => self.move_up(), + KeyEvent { + code: KeyCode::Down, + .. + } + | KeyEvent { + code: KeyCode::Char('n'), + modifiers: KeyModifiers::CONTROL, + .. + } + | KeyEvent { + code: KeyCode::Char('\u{000e}'), + modifiers: KeyModifiers::NONE, + .. + } => self.move_down(), + KeyEvent { + code: KeyCode::Char('j'), + modifiers: KeyModifiers::NONE, + .. + } => self.move_down(), + KeyEvent { + code: KeyCode::Char(' '), + modifiers: KeyModifiers::NONE, + .. + } => self.toggle_selected(), + KeyEvent { + code: KeyCode::Enter, + modifiers: KeyModifiers::NONE, + .. + } => self.save(), + KeyEvent { + code: KeyCode::Esc, .. + } => self.cancel(), + _ => {} + } + } + + fn is_complete(&self) -> bool { + self.complete + } + + fn on_ctrl_c(&mut self) -> CancellationEvent { + self.cancel(); + CancellationEvent::Handled + } +} + +impl MemoriesSettingsView { + fn save(&mut self) { + if let Some(state) = self.reset_confirmation.as_ref() { + match state.selected_idx { + Some(0) => { + self.app_event_tx.send(AppEvent::ResetMemories); + self.complete = true; + } + Some(1) | None => self.close_reset_confirmation(), + Some(other) => unreachable!("unexpected reset confirmation row: {other}"), + } + return; + } + + match self.state.selected_idx.and_then(|idx| self.items.get(idx)) { + Some(MemoriesMenuItem::Action { + action: MemoriesAction::Reset, + .. + }) => self.open_reset_confirmation(), + _ => { + self.app_event_tx.send(AppEvent::UpdateMemorySettings { + use_memories: self.current_setting(MemoriesSetting::Use), + generate_memories: self.current_setting(MemoriesSetting::Generate), + }); + self.complete = true; + } + } + } + + fn cancel(&mut self) { + if self.reset_confirmation.is_some() { + self.close_reset_confirmation(); + } else { + self.complete = true; + } + } +} + +impl Renderable for MemoriesSettingsView { + fn render(&self, area: Rect, buf: &mut Buffer) { + if area.height == 0 || area.width == 0 { + return; + } + + let [content_area, footer_area] = + Layout::vertical([Constraint::Fill(1), Constraint::Length(1)]).areas(area); + + Block::default() + .style(user_message_style()) + .render(content_area, buf); + + let header = if self.reset_confirmation.is_some() { + self.reset_confirmation_header() + } else { + self.settings_header() + }; + let header_height = header.desired_height(content_area.width.saturating_sub(4)); + let rows = self.build_rows(); + let rows_width = Self::rows_width(content_area.width); + let rows_height = measure_rows_height( + &rows, + self.active_state(), + MAX_POPUP_ROWS, + rows_width.saturating_add(1), + ); + let [header_area, _, list_area, _, docs_area] = Layout::vertical([ + Constraint::Max(header_height), + Constraint::Max(1), + Constraint::Length(rows_height), + Constraint::Max(1), + Constraint::Length(1), + ]) + .areas(content_area.inset(Insets::vh(/*v*/ 1, /*h*/ 2))); + + header.render(header_area, buf); + + if list_area.height > 0 { + let render_area = Rect { + x: list_area.x.saturating_sub(2), + y: list_area.y, + width: rows_width.max(1), + height: list_area.height, + }; + render_rows( + render_area, + buf, + &rows, + self.active_state(), + MAX_POPUP_ROWS, + " No memory settings available", + ); + } + if self.reset_confirmation.is_none() { + self.docs_link.clone().render(docs_area, buf); + } + + let hint_area = Rect { + x: footer_area.x + 2, + y: footer_area.y, + width: footer_area.width.saturating_sub(2), + height: footer_area.height, + }; + self.footer_hint().render(hint_area, buf); + } + + fn desired_height(&self, width: u16) -> u16 { + let header = if self.reset_confirmation.is_some() { + self.reset_confirmation_header() + } else { + self.settings_header() + }; + let rows = self.build_rows(); + let rows_width = Self::rows_width(width); + let rows_height = measure_rows_height( + &rows, + self.active_state(), + MAX_POPUP_ROWS, + rows_width.saturating_add(1), + ); + + let docs_height = if self.reset_confirmation.is_some() { + 0 + } else { + 1 + }; + let mut height = header.desired_height(width.saturating_sub(4)); + height = height.saturating_add(rows_height + 4 + docs_height); + height.saturating_add(1) + } +} + +fn memories_settings_hint_line() -> Line<'static> { + Line::from(vec![ + "Press ".into(), + key_hint::plain(KeyCode::Char(' ')).into(), + " to toggle; ".into(), + key_hint::plain(KeyCode::Enter).into(), + " to save or select".into(), + ]) +} diff --git a/codex-rs/tui/src/bottom_pane/mod.rs b/codex-rs/tui/src/bottom_pane/mod.rs index b2d080b048..91c8bd48eb 100644 --- a/codex-rs/tui/src/bottom_pane/mod.rs +++ b/codex-rs/tui/src/bottom_pane/mod.rs @@ -15,6 +15,7 @@ //! hint. The pane schedules redraws so those hints can expire even when the UI is otherwise idle. use std::path::PathBuf; +use crate::app::app_server_requests::ResolvedAppServerRequest; use crate::app_event::ConnectorsSnapshot; use crate::app_event_sender::AppEventSender; use crate::bottom_pane::pending_input_preview::PendingInputPreview; @@ -81,6 +82,7 @@ mod experimental_features_view; mod file_search_popup; mod footer; mod list_selection_view; +mod memories_settings_view; mod prompt_args; mod skill_popup; mod skills_toggle_view; @@ -91,6 +93,7 @@ pub(crate) use list_selection_view::SelectionViewParams; pub(crate) use list_selection_view::SideContentWidth; pub(crate) use list_selection_view::popup_content_width; pub(crate) use list_selection_view::side_by_side_layout_widths; +pub(crate) use memories_settings_view::MemoriesSettingsView; mod feedback_view; pub(crate) use feedback_view::FeedbackAudience; pub(crate) use feedback_view::feedback_classification; @@ -377,6 +380,18 @@ impl BottomPane { self.request_redraw(); } + fn pop_active_view(&mut self) { + if self.view_stack.pop().is_some() { + self.on_view_stack_depth_decreased(); + } + } + + fn on_view_stack_depth_decreased(&mut self) { + if self.view_stack.is_empty() { + self.on_active_view_complete(); + } + } + /// Forward a key event to the active view or the composer. pub fn handle_key_event(&mut self, key_event: KeyEvent) -> InputResult { // If a modal/view is active, handle it here; otherwise forward to composer. @@ -406,16 +421,14 @@ impl BottomPane { }; if ctrl_c_completed { - self.view_stack.pop(); - self.on_active_view_complete(); + self.pop_active_view(); if let Some(next_view) = self.view_stack.last() && next_view.is_in_paste_burst() { self.request_redraw_in(ChatComposer::recommended_paste_flush_delay()); } } else if view_complete { - self.view_stack.clear(); - self.on_active_view_complete(); + self.pop_active_view(); } else if view_in_paste_burst { self.request_redraw_in(ChatComposer::recommended_paste_flush_delay()); } @@ -467,10 +480,10 @@ impl BottomPane { pub(crate) fn on_ctrl_c(&mut self) -> CancellationEvent { if let Some(view) = self.view_stack.last_mut() { let event = view.on_ctrl_c(); + let view_complete = view.is_complete(); if matches!(event, CancellationEvent::Handled) { - if view.is_complete() { - self.view_stack.pop(); - self.on_active_view_complete(); + if view_complete { + self.pop_active_view(); } self.show_quit_shortcut_hint(key_hint::ctrl(KeyCode::Char('c'))); self.request_redraw(); @@ -491,17 +504,14 @@ impl BottomPane { } pub fn handle_paste(&mut self, pasted: String) { - if !self.view_stack.is_empty() { - let (needs_redraw, view_complete) = { - let last_index = self.view_stack.len() - 1; - let view = &mut self.view_stack[last_index]; - (view.handle_paste(pasted), view.is_complete()) - }; + if let Some(view) = self.view_stack.last_mut() { + let needs_redraw = view.handle_paste(pasted); + let view_complete = view.is_complete(); if view_complete { self.view_stack.clear(); self.on_active_view_complete(); } - if needs_redraw { + if needs_redraw || view_complete { self.request_redraw(); } } else { @@ -1030,6 +1040,37 @@ impl BottomPane { self.push_view(Box::new(modal)); } + pub(crate) fn dismiss_app_server_request( + &mut self, + request: &ResolvedAppServerRequest, + ) -> bool { + if self.view_stack.is_empty() { + return false; + } + + let mut changed = false; + let mut completed_indices = Vec::new(); + for index in (0..self.view_stack.len()).rev() { + let view = &mut self.view_stack[index]; + if !view.dismiss_app_server_request(request) { + continue; + } + changed = true; + if view.is_complete() { + completed_indices.push(index); + } + } + if !changed { + return false; + } + for index in completed_indices { + self.view_stack.remove(index); + } + self.on_view_stack_depth_decreased(); + self.request_redraw(); + true + } + fn on_active_view_complete(&mut self) { self.resume_status_timer_after_modal(); self.set_composer_input_enabled(/*enabled*/ true, /*placeholder*/ None); @@ -1239,6 +1280,7 @@ impl Renderable for BottomPane { #[cfg(test)] mod tests { use super::*; + use crate::app::app_server_requests::ResolvedAppServerRequest; use crate::app_event::AppEvent; use crate::status_indicator_widget::STATUS_DETAILS_DEFAULT_MAX_LINES; use crate::status_indicator_widget::StatusDetailsCapitalization; @@ -1246,9 +1288,12 @@ mod tests { use crate::test_support::test_path_buf; use codex_protocol::protocol::Op; use codex_protocol::protocol::SkillScope; + use crossterm::event::KeyCode; + use crossterm::event::KeyEvent; use crossterm::event::KeyEventKind; use crossterm::event::KeyModifiers; use insta::assert_snapshot; + use pretty_assertions::assert_eq; use ratatui::buffer::Buffer; use ratatui::layout::Rect; use std::cell::Cell; @@ -1273,6 +1318,19 @@ mod tests { snapshot_buffer(&buf) } + fn test_pane(app_event_tx: AppEventSender) -> BottomPane { + BottomPane::new(BottomPaneParams { + app_event_tx, + frame_requester: FrameRequester::test_dummy(), + has_input_focus: true, + enhanced_keys_supported: false, + placeholder_text: "Ask Codex to do anything".to_string(), + disable_paste_burst: false, + animations_enabled: true, + skills: Some(Vec::new()), + }) + } + fn exec_request() -> ApprovalRequest { ApprovalRequest::Exec { thread_id: codex_protocol::ThreadId::new(), @@ -1289,6 +1347,73 @@ mod tests { } } + #[derive(Default)] + struct DismissibleView { + id: Option<&'static str>, + dismiss_exec_id: Option<&'static str>, + complete: bool, + } + + impl Renderable for DismissibleView { + fn render(&self, _area: Rect, _buf: &mut Buffer) {} + + fn desired_height(&self, _width: u16) -> u16 { + 0 + } + } + + impl BottomPaneView for DismissibleView { + fn is_complete(&self) -> bool { + self.complete + } + + fn view_id(&self) -> Option<&'static str> { + self.id + } + + fn dismiss_app_server_request(&mut self, request: &ResolvedAppServerRequest) -> bool { + let ResolvedAppServerRequest::ExecApproval { id } = request else { + return false; + }; + if self.dismiss_exec_id != Some(id.as_str()) { + return false; + } + + self.complete = true; + true + } + } + + #[derive(Default)] + struct CompletingView { + id: Option<&'static str>, + complete: bool, + } + + impl Renderable for CompletingView { + fn render(&self, _area: Rect, _buf: &mut Buffer) {} + + fn desired_height(&self, _width: u16) -> u16 { + 0 + } + } + + impl BottomPaneView for CompletingView { + fn handle_key_event(&mut self, key_event: KeyEvent) { + if key_event.code == KeyCode::Enter { + self.complete = true; + } + } + + fn is_complete(&self) -> bool { + self.complete + } + + fn view_id(&self) -> Option<&'static str> { + self.id + } + } + #[test] fn ctrl_c_on_modal_consumes_without_showing_quit_hint() { let (tx_raw, _rx) = unbounded_channel::(); @@ -1371,6 +1496,89 @@ mod tests { ); } + #[test] + fn dismiss_app_server_request_removes_matching_buried_view() { + let (tx_raw, _rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let mut pane = test_pane(tx); + + pane.push_view(Box::new(DismissibleView { + id: Some("buried"), + dismiss_exec_id: Some("request-1"), + complete: false, + })); + pane.push_view(Box::new(DismissibleView { + id: Some("top"), + dismiss_exec_id: None, + complete: false, + })); + + assert!( + pane.dismiss_app_server_request(&ResolvedAppServerRequest::ExecApproval { + id: "request-1".to_string(), + }) + ); + assert_eq!(pane.view_stack.len(), 1); + assert_eq!( + pane.view_stack.last().and_then(|view| view.view_id()), + Some("top") + ); + } + + #[test] + fn dismiss_app_server_request_returns_false_when_no_view_matches() { + let (tx_raw, _rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let mut pane = test_pane(tx); + + pane.push_view(Box::new(DismissibleView { + id: Some("first"), + dismiss_exec_id: Some("other-request"), + complete: false, + })); + pane.push_view(Box::new(DismissibleView { + id: Some("second"), + dismiss_exec_id: None, + complete: false, + })); + + assert!( + !pane.dismiss_app_server_request(&ResolvedAppServerRequest::ExecApproval { + id: "request-1".to_string(), + }) + ); + assert_eq!(pane.view_stack.len(), 2); + assert_eq!( + pane.view_stack.last().and_then(|view| view.view_id()), + Some("second") + ); + } + + #[test] + fn completing_top_view_preserves_underlying_view() { + let (tx_raw, _rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let mut pane = test_pane(tx); + + pane.push_view(Box::new(DismissibleView { + id: Some("underlying"), + dismiss_exec_id: None, + complete: false, + })); + pane.push_view(Box::new(CompletingView { + id: Some("top"), + complete: false, + })); + + pane.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + + assert_eq!(pane.view_stack.len(), 1); + assert_eq!( + pane.view_stack.last().and_then(|view| view.view_id()), + Some("underlying") + ); + } + #[test] fn composer_shown_after_denied_while_task_running() { let (tx_raw, _rx) = unbounded_channel::(); diff --git a/codex-rs/tui/src/bottom_pane/request_user_input/mod.rs b/codex-rs/tui/src/bottom_pane/request_user_input/mod.rs index f7db7c65dd..bc07c06e0b 100644 --- a/codex-rs/tui/src/bottom_pane/request_user_input/mod.rs +++ b/codex-rs/tui/src/bottom_pane/request_user_input/mod.rs @@ -10,6 +10,7 @@ use std::collections::HashMap; use std::collections::VecDeque; use std::path::PathBuf; +use crate::app::app_server_requests::ResolvedAppServerRequest; use crossterm::event::KeyCode; use crossterm::event::KeyEvent; use crossterm::event::KeyEventKind; @@ -198,6 +199,17 @@ impl RequestUserInputOverlay { self.request.questions.len() } + fn advance_queue_or_complete(&mut self) { + if let Some(next) = self.queue.pop_front() { + self.request = next; + self.reset_for_request(); + self.ensure_focus_available(); + self.restore_current_draft(); + } else { + self.done = true; + } + } + fn has_options(&self) -> bool { self.current_question() .and_then(|question| question.options.as_ref()) @@ -759,14 +771,23 @@ impl RequestUserInputOverlay { interrupted: false, }, ))); - if let Some(next) = self.queue.pop_front() { - self.request = next; - self.reset_for_request(); - self.ensure_focus_available(); - self.restore_current_draft(); - } else { - self.done = true; + self.advance_queue_or_complete(); + } + + fn dismiss_resolved_request(&mut self, request: &ResolvedAppServerRequest) -> bool { + let ResolvedAppServerRequest::UserInput { call_id } = request else { + return false; + }; + + let queue_len = self.queue.len(); + self.queue + .retain(|queued_request| queued_request.call_id != *call_id); + if self.request.call_id == *call_id { + self.advance_queue_or_complete(); + return true; } + + self.queue.len() != queue_len } fn open_unanswered_confirmation(&mut self) { @@ -1273,6 +1294,10 @@ impl BottomPaneView for RequestUserInputOverlay { self.queue.push_back(request); None } + + fn dismiss_app_server_request(&mut self, request: &ResolvedAppServerRequest) -> bool { + self.dismiss_resolved_request(request) + } } #[cfg(test)] @@ -1532,6 +1557,118 @@ mod tests { expect_interrupt_only(&mut rx); } + #[test] + fn resolved_request_dismisses_overlay_without_emitting_events() { + let (tx, mut rx) = test_sender(); + let mut overlay = RequestUserInputOverlay::new( + RequestUserInputEvent { + call_id: "call-1".to_string(), + turn_id: "turn-1".to_string(), + questions: vec![question_with_options("q1", "First")], + }, + tx, + /*has_input_focus*/ true, + /*enhanced_keys_supported*/ false, + /*disable_paste_burst*/ false, + ); + + assert!( + overlay.dismiss_app_server_request(&ResolvedAppServerRequest::UserInput { + call_id: "call-1".to_string(), + }) + ); + assert!(overlay.done, "resolved request should close the overlay"); + assert!( + rx.try_recv().is_err(), + "dismissing a stale request should not emit an interrupt or answer" + ); + } + + #[test] + fn resolved_current_request_advances_to_next_same_turn_prompt() { + let (tx, mut rx) = test_sender(); + let mut overlay = RequestUserInputOverlay::new( + RequestUserInputEvent { + call_id: "call-1".to_string(), + turn_id: "turn-1".to_string(), + questions: vec![question_with_options("q1", "First")], + }, + tx, + /*has_input_focus*/ true, + /*enhanced_keys_supported*/ false, + /*disable_paste_burst*/ false, + ); + overlay.try_consume_user_input_request(RequestUserInputEvent { + call_id: "call-2".to_string(), + turn_id: "turn-1".to_string(), + questions: vec![question_with_options("q2", "Second")], + }); + + assert!( + overlay.dismiss_app_server_request(&ResolvedAppServerRequest::UserInput { + call_id: "call-1".to_string(), + }) + ); + + assert!(!overlay.done, "newer same-turn prompt should stay pending"); + assert_eq!(overlay.request.call_id, "call-2"); + assert_eq!(overlay.request.turn_id, "turn-1"); + assert_eq!(overlay.request.questions[0].id, "q2"); + assert!( + rx.try_recv().is_err(), + "dismissing a stale request should not emit an interrupt or answer" + ); + } + + #[test] + fn resolved_queued_request_removes_only_that_prompt() { + let (tx, mut rx) = test_sender(); + let mut overlay = RequestUserInputOverlay::new( + RequestUserInputEvent { + call_id: "call-1".to_string(), + turn_id: "turn-1".to_string(), + questions: vec![question_with_options("q1", "First")], + }, + tx, + /*has_input_focus*/ true, + /*enhanced_keys_supported*/ false, + /*disable_paste_burst*/ false, + ); + overlay.try_consume_user_input_request(RequestUserInputEvent { + call_id: "call-2".to_string(), + turn_id: "turn-1".to_string(), + questions: vec![question_with_options("q2", "Second")], + }); + overlay.try_consume_user_input_request(RequestUserInputEvent { + call_id: "call-3".to_string(), + turn_id: "turn-1".to_string(), + questions: vec![question_with_options("q3", "Third")], + }); + + assert!( + overlay.dismiss_app_server_request(&ResolvedAppServerRequest::UserInput { + call_id: "call-2".to_string(), + }) + ); + + assert_eq!(overlay.request.call_id, "call-1"); + assert!( + rx.try_recv().is_err(), + "dismissing a stale queued request should not emit an event" + ); + overlay.submit_answers(); + assert_eq!(overlay.request.call_id, "call-3"); + assert_eq!(overlay.request.questions[0].id, "q3"); + assert!( + rx.try_recv().is_ok(), + "submitting the still-current prompt should emit an answer" + ); + assert!( + rx.try_recv().is_ok(), + "submitting the still-current prompt should emit a history cell" + ); + } + #[test] fn options_can_submit_empty_when_unanswered() { let (tx, mut rx) = test_sender(); diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 1334b6e2f2..599655b364 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -43,6 +43,7 @@ use std::time::Duration; use std::time::Instant; use self::realtime::PendingSteerCompareKey; +use crate::app::app_server_requests::ResolvedAppServerRequest; use crate::app_command::AppCommand; use crate::app_event::RealtimeAudioDeviceKind; use crate::app_server_approval_conversions::network_approval_context_to_core; @@ -251,6 +252,11 @@ const MULTI_AGENT_ENABLE_TITLE: &str = "Enable subagents?"; const MULTI_AGENT_ENABLE_YES: &str = "Yes, enable"; const MULTI_AGENT_ENABLE_NO: &str = "Not now"; const MULTI_AGENT_ENABLE_NOTICE: &str = "Subagents will be enabled in the next session."; +const MEMORIES_DOC_URL: &str = "https://developers.openai.com/codex/memories"; +const MEMORIES_ENABLE_TITLE: &str = "Enable memories?"; +const MEMORIES_ENABLE_YES: &str = "Yes, enable"; +const MEMORIES_ENABLE_NO: &str = "Not now"; +const MEMORIES_ENABLE_NOTICE: &str = "Memories will be enabled in the next session."; const PLAN_MODE_REASONING_SCOPE_TITLE: &str = "Apply reasoning change"; const PLAN_MODE_REASONING_SCOPE_PLAN_ONLY: &str = "Apply to Plan mode override"; const PLAN_MODE_REASONING_SCOPE_ALL_MODES: &str = "Apply to global default and Plan mode override"; @@ -311,6 +317,7 @@ use crate::bottom_pane::ExperimentalFeaturesView; use crate::bottom_pane::InputResult; use crate::bottom_pane::LocalImageAttachment; use crate::bottom_pane::McpServerElicitationFormRequest; +use crate::bottom_pane::MemoriesSettingsView; use crate::bottom_pane::MentionBinding; use crate::bottom_pane::QUIT_SHORTCUT_TIMEOUT; use crate::bottom_pane::SelectionAction; @@ -2157,6 +2164,16 @@ impl ChatWidget { self.request_redraw(); } + pub(crate) fn dismiss_app_server_request(&mut self, request: &ResolvedAppServerRequest) { + // A remotely resolved request must not remain user-actionable. It may be + // materialized in the bottom pane or still deferred behind active streaming. + let removed_deferred = self.interrupts.remove_resolved_prompt(request); + let removed_visible = self.bottom_pane.dismiss_app_server_request(request); + if removed_deferred || removed_visible { + self.request_redraw(); + } + } + pub(crate) fn open_feedback_consent(&mut self, category: crate::app_event::FeedbackCategory) { let snapshot = self.feedback.snapshot(self.thread_id); let params = crate::bottom_pane::feedback_upload_consent_params( @@ -2588,6 +2605,61 @@ impl ChatWidget { }); } + pub(crate) fn open_memories_popup(&mut self) { + if !self.config.features.enabled(Feature::MemoryTool) { + self.open_memories_enable_prompt(); + return; + } + + let view = MemoriesSettingsView::new( + self.config.memories.use_memories, + self.config.memories.generate_memories, + self.app_event_tx.clone(), + ); + self.bottom_pane.show_view(Box::new(view)); + } + + pub(crate) fn open_memories_enable_prompt(&mut self) { + let items = vec![ + SelectionItem { + name: MEMORIES_ENABLE_YES.to_string(), + description: Some( + "Save the setting now. You will need a new session to use it.".to_string(), + ), + actions: vec![Box::new(|tx| { + tx.send(AppEvent::UpdateFeatureFlags { + updates: vec![(Feature::MemoryTool, true)], + }); + })], + dismiss_on_select: true, + ..Default::default() + }, + SelectionItem { + name: MEMORIES_ENABLE_NO.to_string(), + description: Some("Keep memories disabled.".to_string()), + dismiss_on_select: true, + ..Default::default() + }, + ]; + + self.bottom_pane.show_selection_view(SelectionViewParams { + title: Some(MEMORIES_ENABLE_TITLE.to_string()), + subtitle: Some("Memories are currently disabled in your config.".to_string()), + footer_note: Some(Line::from(vec![ + "Learn more: ".dim(), + MEMORIES_DOC_URL.cyan().underlined(), + ])), + footer_hint: Some(standard_popup_hint_line()), + items, + ..Default::default() + }); + } + + pub(crate) fn set_memory_settings(&mut self, use_memories: bool, generate_memories: bool) { + self.config.memories.use_memories = use_memories; + self.config.memories.generate_memories = generate_memories; + } + pub(crate) fn set_token_info(&mut self, info: Option) { match info { Some(info) => self.apply_token_info(info), @@ -9693,6 +9765,13 @@ impl ChatWidget { self.request_redraw(); } + pub(crate) fn add_memories_enable_notice(&mut self) { + self.add_to_history(history_cell::new_warning_event( + MEMORIES_ENABLE_NOTICE.to_string(), + )); + self.request_redraw(); + } + pub(crate) fn add_plain_history_lines(&mut self, lines: Vec>) { self.add_boxed_history(Box::new(PlainHistoryCell::new(lines))); self.request_redraw(); @@ -10372,6 +10451,7 @@ impl ChatWidget { self.config.features = config.features.clone(); self.config.config_layer_stack = config.config_layer_stack.clone(); self.config.realtime = config.realtime.clone(); + self.config.memories = config.memories.clone(); } pub(crate) fn open_review_popup(&mut self) { diff --git a/codex-rs/tui/src/chatwidget/interrupts.rs b/codex-rs/tui/src/chatwidget/interrupts.rs index 0a3fc80016..f41e3b3a47 100644 --- a/codex-rs/tui/src/chatwidget/interrupts.rs +++ b/codex-rs/tui/src/chatwidget/interrupts.rs @@ -1,5 +1,6 @@ use std::collections::VecDeque; +use crate::app::app_server_requests::ResolvedAppServerRequest; use codex_protocol::approvals::ElicitationRequestEvent; use codex_protocol::protocol::ApplyPatchApprovalRequestEvent; use codex_protocol::protocol::ExecApprovalRequestEvent; @@ -86,6 +87,13 @@ impl InterruptManager { self.queue.push_back(QueuedInterrupt::PatchEnd(ev)); } + pub(crate) fn remove_resolved_prompt(&mut self, request: &ResolvedAppServerRequest) -> bool { + let original_len = self.queue.len(); + self.queue + .retain(|queued| !queued.matches_resolved_prompt(request)); + self.queue.len() != original_len + } + pub(crate) fn flush_all(&mut self, chat: &mut ChatWidget) { while let Some(q) = self.queue.pop_front() { match q { @@ -103,3 +111,144 @@ impl InterruptManager { } } } + +impl QueuedInterrupt { + fn matches_resolved_prompt(&self, request: &ResolvedAppServerRequest) -> bool { + match self { + QueuedInterrupt::ExecApproval(ev) => { + matches!(request, ResolvedAppServerRequest::ExecApproval { id } + if ev.effective_approval_id() == id.as_str()) + } + QueuedInterrupt::ApplyPatchApproval(ev) => { + matches!(request, ResolvedAppServerRequest::FileChangeApproval { id } + if ev.call_id == id.as_str()) + } + QueuedInterrupt::Elicitation(ev) => { + matches!(request, ResolvedAppServerRequest::McpElicitation { + server_name, + request_id, + } if ev.server_name == server_name.as_str() && &ev.id == request_id) + } + QueuedInterrupt::RequestPermissions(ev) => { + matches!(request, ResolvedAppServerRequest::PermissionsApproval { id } + if ev.call_id == id.as_str()) + } + QueuedInterrupt::RequestUserInput(ev) => { + matches!(request, ResolvedAppServerRequest::UserInput { call_id } + if ev.call_id == call_id.as_str()) + } + QueuedInterrupt::ExecBegin(_) + | QueuedInterrupt::ExecEnd(_) + | QueuedInterrupt::McpBegin(_) + | QueuedInterrupt::McpEnd(_) + | QueuedInterrupt::PatchEnd(_) => false, + } + } +} + +#[cfg(test)] +mod tests { + use codex_protocol::approvals::ExecApprovalRequestEvent; + use codex_protocol::protocol::ExecCommandBeginEvent; + use codex_protocol::protocol::ExecCommandSource; + use codex_protocol::request_user_input::RequestUserInputEvent; + use codex_utils_absolute_path::AbsolutePathBuf; + use pretty_assertions::assert_eq; + + use super::*; + + fn user_input(call_id: &str, turn_id: &str) -> RequestUserInputEvent { + RequestUserInputEvent { + call_id: call_id.to_string(), + turn_id: turn_id.to_string(), + questions: Vec::new(), + } + } + + fn exec_approval(call_id: &str, approval_id: Option<&str>) -> ExecApprovalRequestEvent { + ExecApprovalRequestEvent { + call_id: call_id.to_string(), + approval_id: approval_id.map(str::to_string), + turn_id: "turn".to_string(), + command: vec!["true".to_string()], + cwd: AbsolutePathBuf::current_dir().expect("current dir"), + reason: None, + network_approval_context: None, + proposed_execpolicy_amendment: None, + proposed_network_policy_amendments: None, + additional_permissions: None, + available_decisions: None, + parsed_cmd: Vec::new(), + } + } + + fn exec_begin(call_id: &str) -> ExecCommandBeginEvent { + ExecCommandBeginEvent { + call_id: call_id.to_string(), + process_id: None, + turn_id: "turn".to_string(), + command: vec!["true".to_string()], + cwd: AbsolutePathBuf::current_dir().expect("current dir"), + parsed_cmd: Vec::new(), + source: ExecCommandSource::Agent, + interaction_input: None, + } + } + + #[test] + fn remove_resolved_prompt_removes_matching_user_input_only() { + let mut manager = InterruptManager::new(); + manager.push_user_input(user_input("call-a", "turn")); + manager.push_user_input(user_input("call-b", "turn")); + + assert!( + manager.remove_resolved_prompt(&ResolvedAppServerRequest::UserInput { + call_id: "call-b".to_string(), + }) + ); + + assert_eq!(manager.queue.len(), 1); + let Some(QueuedInterrupt::RequestUserInput(remaining)) = manager.queue.front() else { + panic!("expected remaining queued user input"); + }; + assert_eq!(remaining.call_id, "call-a"); + } + + #[test] + fn remove_resolved_prompt_matches_exec_approval_id() { + let mut manager = InterruptManager::new(); + manager.push_exec_approval(exec_approval("call", Some("approval"))); + + assert!( + !manager.remove_resolved_prompt(&ResolvedAppServerRequest::ExecApproval { + id: "call".to_string(), + }) + ); + assert_eq!(manager.queue.len(), 1); + + assert!( + manager.remove_resolved_prompt(&ResolvedAppServerRequest::ExecApproval { + id: "approval".to_string(), + }) + ); + assert!(manager.queue.is_empty()); + } + + #[test] + fn remove_resolved_prompt_keeps_lifecycle_events() { + let mut manager = InterruptManager::new(); + manager.push_exec_begin(exec_begin("call")); + + assert!( + !manager.remove_resolved_prompt(&ResolvedAppServerRequest::ExecApproval { + id: "call".to_string(), + }) + ); + + assert_eq!(manager.queue.len(), 1); + assert!(matches!( + manager.queue.front(), + Some(QueuedInterrupt::ExecBegin(_)) + )); + } +} diff --git a/codex-rs/tui/src/chatwidget/slash_dispatch.rs b/codex-rs/tui/src/chatwidget/slash_dispatch.rs index e6e93d17fb..ee4d22e95f 100644 --- a/codex-rs/tui/src/chatwidget/slash_dispatch.rs +++ b/codex-rs/tui/src/chatwidget/slash_dispatch.rs @@ -228,6 +228,9 @@ impl ChatWidget { SlashCommand::Experimental => { self.open_experimental_popup(); } + SlashCommand::Memories => { + self.open_memories_popup(); + } SlashCommand::Quit | SlashCommand::Exit => { self.request_quit_without_confirmation(); } diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_enable_prompt.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_enable_prompt.snap new file mode 100644 index 0000000000..f9b0daefcc --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_enable_prompt.snap @@ -0,0 +1,12 @@ +--- +source: tui/src/chatwidget/tests.rs +expression: popup +--- + Enable memories? + Memories are currently disabled in your config. + +› 1. Yes, enable Save the setting now. You will need a new session to use it. + 2. Not now Keep memories disabled. + + Learn more: https://developers.openai.com/codex/memories + Press enter to confirm or esc to go back diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_reset_confirmation.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_reset_confirmation.snap new file mode 100644 index 0000000000..2538e2b461 --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_reset_confirmation.snap @@ -0,0 +1,12 @@ +--- +source: tui/src/chatwidget/tests/popups_and_settings.rs +expression: popup +--- + Reset all memories? + +› Reset all memories Delete local memory files and rollout summaries. + Go back Return to memory settings. + + + + Press enter to confirm or esc to go back diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_settings_popup.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_settings_popup.snap new file mode 100644 index 0000000000..2fbe641e29 --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__memories_settings_popup.snap @@ -0,0 +1,17 @@ +--- +source: tui/src/chatwidget/tests/popups_and_settings.rs +expression: popup +--- + Memories + Choose how Codex uses and creates memories. Changes are saved to config.toml + +› [x] Use memories Use memories in the following threads. Applied at + next thread. + [ ] Generate memories Generate memories from the following threads. Current + thread included. + Reset all memories Clear local memory files and summaries. Existing + threads stay intact. + + Learn more: https://developers.openai.com/codex/memories + + Press space to toggle; enter to save or select diff --git a/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs b/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs index e2635d322d..375c37314b 100644 --- a/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs +++ b/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs @@ -1501,6 +1501,101 @@ async fn multi_agent_enable_prompt_updates_feature_and_emits_notice() { assert!(rendered.contains("Subagents will be enabled in the next session.")); } +#[tokio::test] +async fn memories_enable_prompt_snapshot() { + let (mut chat, _rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::MemoryTool, /*enabled*/ false); + + chat.open_memories_popup(); + + let popup = render_bottom_popup(&chat, /*width*/ 80); + assert_chatwidget_snapshot!("memories_enable_prompt", popup); +} + +#[tokio::test] +async fn memories_enable_prompt_updates_feature_without_notice() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::MemoryTool, /*enabled*/ false); + + chat.open_memories_popup(); + chat.handle_key_event(KeyEvent::from(KeyCode::Enter)); + + assert_matches!( + rx.try_recv(), + Ok(AppEvent::UpdateFeatureFlags { updates }) if updates == vec![(Feature::MemoryTool, true)] + ); + assert!( + rx.try_recv().is_err(), + "memory enable prompt should not emit the success notice before persistence succeeds" + ); +} + +#[tokio::test] +async fn memories_settings_popup_snapshot() { + let (mut chat, _rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::MemoryTool, /*enabled*/ true); + chat.config.memories.use_memories = true; + chat.config.memories.generate_memories = false; + + chat.open_memories_popup(); + + let popup = render_bottom_popup(&chat, /*width*/ 80); + assert_chatwidget_snapshot!("memories_settings_popup", popup); +} + +#[tokio::test] +async fn memories_reset_confirmation_snapshot() { + let (mut chat, _rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::MemoryTool, /*enabled*/ true); + chat.config.memories.use_memories = true; + chat.config.memories.generate_memories = false; + + chat.open_memories_popup(); + chat.handle_key_event(KeyEvent::from(KeyCode::Down)); + chat.handle_key_event(KeyEvent::from(KeyCode::Down)); + chat.handle_key_event(KeyEvent::from(KeyCode::Enter)); + + let popup = render_bottom_popup(&chat, /*width*/ 80); + assert_chatwidget_snapshot!("memories_reset_confirmation", popup); +} + +#[tokio::test] +async fn memories_settings_toggle_saves_on_enter() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::MemoryTool, /*enabled*/ true); + chat.config.memories.use_memories = true; + chat.config.memories.generate_memories = false; + + chat.open_memories_popup(); + chat.handle_key_event(KeyEvent::from(KeyCode::Down)); + chat.handle_key_event(KeyEvent::from(KeyCode::Char(' '))); + chat.handle_key_event(KeyEvent::from(KeyCode::Enter)); + + assert_matches!( + rx.try_recv(), + Ok(AppEvent::UpdateMemorySettings { + use_memories: true, + generate_memories: true, + }) + ); +} + +#[tokio::test] +async fn memories_reset_confirmation_sends_event_on_confirm() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::MemoryTool, /*enabled*/ true); + chat.config.memories.use_memories = true; + chat.config.memories.generate_memories = false; + + chat.open_memories_popup(); + chat.handle_key_event(KeyEvent::from(KeyCode::Down)); + chat.handle_key_event(KeyEvent::from(KeyCode::Down)); + chat.handle_key_event(KeyEvent::from(KeyCode::Enter)); + chat.handle_key_event(KeyEvent::from(KeyCode::Enter)); + + assert_matches!(rx.try_recv(), Ok(AppEvent::ResetMemories)); +} + #[tokio::test] async fn model_selection_popup_snapshot() { let (mut chat, _rx, _op_rx) = make_chatwidget_manual(Some("gpt-5-codex")).await; diff --git a/codex-rs/tui/src/chatwidget/tests/slash_commands.rs b/codex-rs/tui/src/chatwidget/tests/slash_commands.rs index 2afb959349..7174b28b80 100644 --- a/codex-rs/tui/src/chatwidget/tests/slash_commands.rs +++ b/codex-rs/tui/src/chatwidget/tests/slash_commands.rs @@ -533,6 +533,18 @@ async fn slash_mcp_requests_inventory_via_app_server() { assert!(op_rx.try_recv().is_err(), "expected no core op to be sent"); } +#[tokio::test] +async fn slash_memories_opens_memory_menu() { + let (mut chat, mut rx, mut op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::MemoryTool, /*enabled*/ true); + + chat.dispatch_command(SlashCommand::Memories); + + assert!(render_bottom_popup(&chat, /*width*/ 80).contains("Use memories")); + assert_matches!(rx.try_recv(), Err(TryRecvError::Empty)); + assert!(op_rx.try_recv().is_err(), "expected no core op to be sent"); +} + #[tokio::test] async fn slash_memory_update_reports_stubbed_feature() { let (mut chat, mut rx, mut op_rx) = make_chatwidget_manual(/*model_override*/ None).await; diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 2f78b80ce4..9fcaab5c16 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -67,10 +67,12 @@ use std::future::Future; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; +use tracing::Level; use tracing::error; use tracing::warn; use tracing_appender::non_blocking; use tracing_subscriber::EnvFilter; +use tracing_subscriber::filter::Targets; use tracing_subscriber::prelude::*; use url::Url; use uuid::Uuid; @@ -238,6 +240,7 @@ pub use public_widgets::composer_input::ComposerAction; pub use public_widgets::composer_input::ComposerInput; // (tests access modules directly within the crate) +#[allow(clippy::too_many_arguments)] async fn start_embedded_app_server( arg0_paths: Arg0DispatchPaths, config: Config, @@ -245,6 +248,7 @@ async fn start_embedded_app_server( loader_overrides: LoaderOverrides, cloud_requirements: CloudRequirementsLoader, feedback: codex_feedback::CodexFeedback, + log_db: Option, environment_manager: Arc, ) -> color_eyre::Result { start_embedded_app_server_with( @@ -254,6 +258,7 @@ async fn start_embedded_app_server( loader_overrides, cloud_requirements, feedback, + log_db, environment_manager, InProcessAppServerClient::start, ) @@ -370,6 +375,7 @@ async fn start_app_server( loader_overrides: LoaderOverrides, cloud_requirements: CloudRequirementsLoader, feedback: codex_feedback::CodexFeedback, + log_db: Option, environment_manager: Arc, ) -> color_eyre::Result { match target { @@ -380,6 +386,7 @@ async fn start_app_server( loader_overrides, cloud_requirements, feedback, + log_db, environment_manager, ) .await @@ -404,6 +411,7 @@ pub(crate) async fn start_app_server_for_picker( LoaderOverrides::default(), CloudRequirementsLoader::default(), codex_feedback::CodexFeedback::new(), + /*log_db*/ None, environment_manager, ) .await?; @@ -430,6 +438,7 @@ async fn start_embedded_app_server_with( loader_overrides: LoaderOverrides, cloud_requirements: CloudRequirementsLoader, feedback: codex_feedback::CodexFeedback, + log_db: Option, environment_manager: Arc, start_client: F, ) -> color_eyre::Result @@ -454,6 +463,7 @@ where loader_overrides, cloud_requirements, feedback, + log_db, environment_manager, config_warnings, session_source: codex_protocol::protocol::SessionSource::Cli, @@ -970,9 +980,10 @@ pub async fn run_main( let otel_tracing_layer = otel.as_ref().and_then(|o| o.tracing_layer()); - let log_db_layer = get_state_db(&config) - .await - .map(|db| log_db::start(db).with_filter(env_filter())); + let log_db = get_state_db(&config).await.map(log_db::start); + let log_db_layer = log_db + .clone() + .map(|layer| layer.with_filter(Targets::new().with_default(Level::TRACE))); let _ = tracing_subscriber::registry() .with(file_layer) @@ -994,6 +1005,7 @@ pub async fn run_main( cli_kv_overrides, cloud_requirements, feedback, + log_db, remote_url, remote_auth_token, environment_manager, @@ -1014,6 +1026,7 @@ async fn run_ratatui_app( cli_kv_overrides: Vec<(String, toml::Value)>, mut cloud_requirements: CloudRequirementsLoader, feedback: codex_feedback::CodexFeedback, + log_db: Option, remote_url: Option, remote_auth_token: Option, environment_manager: Arc, @@ -1072,6 +1085,7 @@ async fn run_ratatui_app( loader_overrides.clone(), cloud_requirements.clone(), feedback.clone(), + log_db.clone(), environment_manager.clone(), ) .await @@ -1410,6 +1424,7 @@ async fn run_ratatui_app( loader_overrides, cloud_requirements.clone(), feedback.clone(), + log_db.clone(), environment_manager.clone(), ) .await @@ -1773,6 +1788,7 @@ mod tests { LoaderOverrides::default(), CloudRequirementsLoader::default(), codex_feedback::CodexFeedback::new(), + /*log_db*/ None, Arc::new(EnvironmentManager::new(/*exec_server_url*/ None)), ) .await @@ -2164,6 +2180,7 @@ mod tests { LoaderOverrides::default(), CloudRequirementsLoader::default(), codex_feedback::CodexFeedback::new(), + /*log_db*/ None, Arc::new(EnvironmentManager::new(/*exec_server_url*/ None)), |_args| async { Err(std::io::Error::other("boom")) }, ) diff --git a/codex-rs/tui/src/local_chatgpt_auth.rs b/codex-rs/tui/src/local_chatgpt_auth.rs index e888c0387c..1f84b289a7 100644 --- a/codex-rs/tui/src/local_chatgpt_auth.rs +++ b/codex-rs/tui/src/local_chatgpt_auth.rs @@ -108,6 +108,7 @@ mod tests { account_id: Some("workspace-1".to_string()), }), last_refresh: Some(Utc::now()), + agent_identity: None, }; save_auth(codex_home, &auth, AuthCredentialsStoreMode::File) .expect("chatgpt auth should save"); @@ -154,6 +155,7 @@ mod tests { openai_api_key: Some("sk-test".to_string()), tokens: None, last_refresh: None, + agent_identity: None, }, AuthCredentialsStoreMode::File, ) diff --git a/codex-rs/tui/src/onboarding/auth.rs b/codex-rs/tui/src/onboarding/auth.rs index 57bd58bf5a..aee909cea1 100644 --- a/codex-rs/tui/src/onboarding/auth.rs +++ b/codex-rs/tui/src/onboarding/auth.rs @@ -988,6 +988,7 @@ mod tests { "https://chatgpt.com/backend-api/".to_string(), ), feedback: codex_feedback::CodexFeedback::new(), + log_db: None, environment_manager: Arc::new(codex_app_server_client::EnvironmentManager::new( /*exec_server_url*/ None, )), diff --git a/codex-rs/tui/src/slash_command.rs b/codex-rs/tui/src/slash_command.rs index 635dc317f8..759bedbad7 100644 --- a/codex-rs/tui/src/slash_command.rs +++ b/codex-rs/tui/src/slash_command.rs @@ -21,6 +21,7 @@ pub enum SlashCommand { #[strum(serialize = "sandbox-add-read-dir")] SandboxReadRoot, Experimental, + Memories, Skills, Review, Rename, @@ -109,6 +110,7 @@ impl SlashCommand { "let sandbox read a directory: /sandbox-add-read-dir " } SlashCommand::Experimental => "toggle experimental features", + SlashCommand::Memories => "configure memory use and generation", SlashCommand::Mcp => "list configured MCP tools", SlashCommand::Apps => "manage apps", SlashCommand::Plugins => "browse plugins", @@ -154,6 +156,7 @@ impl SlashCommand { | SlashCommand::ElevateSandbox | SlashCommand::SandboxReadRoot | SlashCommand::Experimental + | SlashCommand::Memories | SlashCommand::Review | SlashCommand::Plan | SlashCommand::Clear diff --git a/defs.bzl b/defs.bzl index 6972b65a65..7cd89ce63e 100644 --- a/defs.bzl +++ b/defs.bzl @@ -137,10 +137,13 @@ def codex_rust_crate( rustc_env = {}, deps_extra = [], integration_compile_data_extra = [], + integration_test_tags_extra_by_stem = {}, + integration_test_flaky_by_stem = {}, integration_test_args = [], integration_test_timeout = None, test_data_extra = [], test_tags = [], + unit_test_args = [], unit_test_timeout = None, extra_binaries = []): """Defines a Rust crate with library, binaries, and tests wired for Bazel + Cargo parity. @@ -170,6 +173,9 @@ def codex_rust_crate( deps_extra: Extra normal deps beyond @crates resolution. Typically only needed when features add additional deps. integration_compile_data_extra: Extra compile_data for integration tests. + integration_test_flaky_by_stem: Mapping from `tests/*.rs` stem to exact + Rust test names that should be split into dedicated flaky targets + and skipped from the main integration test target. integration_test_args: Optional args for integration test binaries. integration_test_timeout: Optional Bazel timeout for integration test targets generated from `tests/*.rs`. @@ -270,6 +276,8 @@ def codex_rust_crate( unit_test_kwargs = {} if unit_test_timeout: unit_test_kwargs["timeout"] = unit_test_timeout + if unit_test_args: + unit_test_kwargs["args"] = unit_test_args workspace_root_test( name = name + "-unit-tests", @@ -305,11 +313,9 @@ def codex_rust_crate( binary = Label(binary_label).name cargo_env["CARGO_BIN_EXE_" + binary] = "$(rlocationpath %s)" % binary_label - integration_test_kwargs = {} - if integration_test_args: - integration_test_kwargs["args"] = integration_test_args + integration_test_base_kwargs = {} if integration_test_timeout: - integration_test_kwargs["timeout"] = integration_test_timeout + integration_test_base_kwargs["timeout"] = integration_test_timeout for test in native.glob(["tests/*.rs"], allow_empty = True): test_file_stem = test.removeprefix("tests/").removesuffix(".rs") @@ -317,6 +323,18 @@ def codex_rust_crate( test_name = name + "-" + test_file_stem.replace("/", "-") if not test_name.endswith("-test"): test_name += "-test" + extra_test_tags = integration_test_tags_extra_by_stem.get(test_file_stem, []) + flaky_tests = integration_test_flaky_by_stem.get(test_file_stem, []) + stem_integration_test_args = [] + integration_test_args + for flaky_test in flaky_tests: + stem_integration_test_args += [ + "--skip", + flaky_test, + ] + + integration_test_kwargs = dict(integration_test_base_kwargs) + if stem_integration_test_args: + integration_test_kwargs["args"] = stem_integration_test_args rust_test( name = test_name, @@ -338,6 +356,32 @@ def codex_rust_crate( # `INSTA_WORKSPACE_ROOT="codex-rs"` is tuned for unit tests that # execute from the repo root and can misplace integration snapshots. env = cargo_env, - tags = test_tags, + tags = test_tags + extra_test_tags, **integration_test_kwargs ) + + for flaky_test in flaky_tests: + flaky_test_suffix = flaky_test.replace("::", "-").replace("/", "-").replace("_", "-") + flaky_test_kwargs = dict(integration_test_base_kwargs) + flaky_test_kwargs["args"] = [ + "--exact", + flaky_test, + ] + + rust_test( + name = test_name.removesuffix("-test") + "-flaky-" + flaky_test_suffix + "-test", + crate_name = test_crate_name + "_flaky_" + flaky_test.replace("::", "_").replace("/", "_").replace("-", "_"), + crate_root = test, + srcs = [test], + data = native.glob(["tests/**"], allow_empty = True) + sanitized_binaries + test_data_extra, + compile_data = native.glob(["tests/**"], allow_empty = True) + integration_compile_data_extra, + deps = all_crate_deps(normal = True, normal_dev = True) + maybe_deps + deps_extra, + rustc_flags = rustc_flags_extra + WINDOWS_RUSTC_LINK_FLAGS + [ + "--remap-path-prefix=../codex-rs=", + "--remap-path-prefix=codex-rs=", + ], + rustc_env = rustc_env, + env = cargo_env, + tags = test_tags + extra_test_tags + ["flaky"], + **flaky_test_kwargs + ) diff --git a/scripts/start-codex-exec.sh b/scripts/start-codex-exec.sh index 5699c13efd..8835d4c095 100755 --- a/scripts/start-codex-exec.sh +++ b/scripts/start-codex-exec.sh @@ -67,11 +67,7 @@ repo_root="$(git rev-parse --show-toplevel 2>/dev/null)" || { } ssh "${remote_host}" "mkdir -p ${remote_path}" - -if ! ssh "${remote_host}" 'command -v rsync >/dev/null 2>&1'; then - echo "remote rsync is required on ${remote_host}" >&2 - exit 1 -fi +ssh "${remote_host}" -C "sudo apt-get install rsync libcap-dev" sync_instance_id="$(date +%s)-$$" diff --git a/scripts/test-remote-env.sh b/scripts/test-remote-env.sh index 329fa8a7f3..ba4d35f235 100755 --- a/scripts/test-remote-env.sh +++ b/scripts/test-remote-env.sh @@ -18,6 +18,11 @@ is_sourced() { setup_remote_env() { local container_name local codex_binary_path + local container_ip + local remote_codex_path + local remote_exec_server_pid + local remote_exec_server_port + local remote_exec_server_stdout_path container_name="${CODEX_TEST_REMOTE_ENV_CONTAINER_NAME:-codex-remote-test-env-local-$(date +%s)-${RANDOM}}" codex_binary_path="${REPO_ROOT}/codex-rs/target/debug/codex" @@ -59,14 +64,58 @@ setup_remote_env() { return 1 fi + if [[ -z "${CODEX_TEST_REMOTE_EXEC_SERVER_URL:-}" ]]; then + remote_codex_path="/tmp/codex-remote-env/codex" + remote_exec_server_port="31987" + remote_exec_server_stdout_path="/tmp/codex-remote-env/exec-server.stdout" + docker exec "${container_name}" sh -lc "mkdir -p /tmp/codex-remote-env" + docker cp "${codex_binary_path}" "${container_name}:${remote_codex_path}" + docker exec "${container_name}" chmod +x "${remote_codex_path}" + remote_exec_server_pid="$( + docker exec "${container_name}" sh -lc \ + "rm -f ${remote_exec_server_stdout_path}; nohup ${remote_codex_path} exec-server --listen ws://0.0.0.0:${remote_exec_server_port} > ${remote_exec_server_stdout_path} 2>&1 & echo \$!" + )" + wait_for_remote_exec_server_port "${container_name}" "${remote_exec_server_port}" "${remote_exec_server_stdout_path}" + container_ip="$( + docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "${container_name}" + )" + if [[ -z "${container_ip}" ]]; then + echo "container ${container_name} has no IP address" >&2 + docker rm -f "${container_name}" >/dev/null 2>&1 || true + return 1 + fi + export CODEX_TEST_REMOTE_EXEC_SERVER_PID="${remote_exec_server_pid}" + export CODEX_TEST_REMOTE_EXEC_SERVER_URL="ws://${container_ip}:${remote_exec_server_port}" + fi + export CODEX_TEST_REMOTE_ENV="${container_name}" } +wait_for_remote_exec_server_port() { + local container_name="$1" + local port="$2" + local stdout_path="$3" + local deadline=$((SECONDS + 5)) + + while (( SECONDS < deadline )); do + if docker exec "${container_name}" python3 -c "import socket; socket.create_connection(('127.0.0.1', ${port}), timeout=0.2).close()" >/dev/null 2>&1; then + return 0 + fi + sleep 0.025 + done + + echo "timed out waiting for remote exec-server on ${container_name}:${port}" >&2 + docker exec "${container_name}" sh -lc "cat ${stdout_path} 2>/dev/null || true" >&2 || true + return 1 +} + codex_remote_env_cleanup() { if [[ -n "${CODEX_TEST_REMOTE_ENV:-}" ]]; then docker rm -f "${CODEX_TEST_REMOTE_ENV}" >/dev/null 2>&1 || true unset CODEX_TEST_REMOTE_ENV fi + unset CODEX_TEST_REMOTE_EXEC_SERVER_PID + unset CODEX_TEST_REMOTE_EXEC_SERVER_URL } if ! is_sourced; then @@ -79,6 +128,7 @@ set -euo pipefail if setup_remote_env; then status=0 echo "CODEX_TEST_REMOTE_ENV=${CODEX_TEST_REMOTE_ENV}" + echo "CODEX_TEST_REMOTE_EXEC_SERVER_URL=${CODEX_TEST_REMOTE_EXEC_SERVER_URL}" echo "Remote env ready. Run your command, then call: codex_remote_env_cleanup" else status=$?